User:MonAx/commonsduplicates.py

See User talk:MGA73/NowCommons.py#User-Agent policy

#!/usr/bin/env python3
"""Find local files that have duplicate files on Commons.

Usage examples:

    python pwb.py commonsduplicates
    python pwb.py -family:wikipedia -lang:uk commonsduplicates
    python pwb.py commonsduplicates -category:Невільні_файли
    python pwb.py commonsduplicates -category:Невільні_файли -output:commons_duplicates.txt
    python pwb.py commonsduplicates -limit:500

To run this script, you should put this in `scripts/userscripts` subdirectory of pywikibot and execute the commands above. 
This script only reports files. It does not edit or delete anything.

Created by:
- **MGA73** (Wikiverse Contributor, made the original script)
- **ChatGPT** (AI Assistant by OpenAI)
- **MonAx** (made this a pywikibot script)
"""

from __future__ import annotations

import os
from typing import Any

import pywikibot
from pywikibot.data.api import Request


DEFAULT_OUTPUT = "commons_duplicates.txt"


def normalize_category_title(raw: str) -> str:
    """Return a category title suitable for gcmtitle."""
    title = raw.strip().replace(" ", "_")

    if ":" in title:
        return title

    return f"Category:{title}"


def get_duplicatefiles_batches(
    site: pywikibot.site.APISite,
    *,
    category: str | None = None,
    start: str | None = None,
):
    """Yield API pages with duplicatefiles data.

    If category is given, scan files in that category.
    Otherwise scan all local files through generator=allimages.
    """
    params: dict[str, Any] = {
        "action": "query",
        "formatversion": "2",
        "prop": "duplicatefiles",
        "dflimit": "max",
    }

    if category:
        params.update({
            "generator": "categorymembers",
            "gcmtitle": normalize_category_title(category),
            "gcmnamespace": "6",
            "gcmlimit": "max",
        })
    else:
        params.update({
            "generator": "allimages",
            "gailimit": "max",
        })

        if start:
            params["gaifrom"] = start

    while True:
        data = Request(site, **params).submit()

        if "error" in data:
            raise RuntimeError(f"API error: {data['error']}")

        pages = data.get("query", {}).get("pages", [])

        if isinstance(pages, dict):
            pages = list(pages.values())

        yield pages

        cont = data.get("continue")
        if not cont:
            break

        params.update(cont)


def commons_duplicates_for_page(page_data: dict[str, Any]) -> list[str]:
    """Return Commons duplicate filenames for one local file page."""
    result = []

    for duplicate in page_data.get("duplicatefiles", []):
        # MediaWiki API flag-style output:
        # "shared": "" means the duplicate is in the shared repo, i.e. Commons.
        if "shared" in duplicate:
            name = duplicate.get("name")
            if name:
                result.append(name)

    return result


def format_result(local_title: str, commons_names: list[str]) -> str:
    """Return one output line in wikitext format."""
    commons_links = ", ".join(
        f"[[:c:File:{name}]]"
        for name in commons_names
    )

    return f"# [[:{local_title}]] → {commons_links}"


def main(*args: str) -> None:
    local_args = pywikibot.handle_args(args)

    output_file = DEFAULT_OUTPUT
    category = None
    start = None
    limit = None

    for arg in local_args:
        if arg == "-help":
            pywikibot.output(__doc__)
            return

        if arg.startswith("-output:"):
            output_file = arg.removeprefix("-output:")
        elif arg.startswith("-category:"):
            category = arg.removeprefix("-category:")
        elif arg.startswith("-start:"):
            start = arg.removeprefix("-start:")
        elif arg.startswith("-limit:"):
            limit = int(arg.removeprefix("-limit:"))
        else:
            raise ValueError(f"Unknown argument: {arg}")

    site = pywikibot.Site()
    site.login()

    pywikibot.output(f"Site: {site}")
    pywikibot.output(f"User: {site.username()}")

    if category:
        pywikibot.output(f"Scanning files in category: {normalize_category_title(category)}")
    else:
        pywikibot.output("Scanning all local files")

    checked = 0
    found = 0
    seen_pairs: set[tuple[str, str]] = set()

    with open(output_file, "w", encoding="utf-8") as out:
        for pages in get_duplicatefiles_batches(site, category=category, start=start):
            for page_data in pages:
                local_title = page_data.get("title")

                if not local_title:
                    continue

                checked += 1

                commons_names = commons_duplicates_for_page(page_data)
                unique_commons_names = []

                for commons_name in commons_names:
                    pair = (local_title, commons_name)

                    if pair in seen_pairs:
                        continue

                    seen_pairs.add(pair)
                    unique_commons_names.append(commons_name)

                if unique_commons_names:
                    line = format_result(local_title, unique_commons_names)
                    out.write(line + "\n")
                    pywikibot.output(line)
                    found += 1

                if limit is not None and checked >= limit:
                    break

            pywikibot.output(f"Checked {checked} files; found {found} files with Commons duplicates")

            if limit is not None and checked >= limit:
                break

        out.write(f"\nTotal checked: {checked}\n")
        out.write(f"Total files with Commons duplicates: {found}\n")

    pywikibot.output(f"Done. Output written to: {os.path.abspath(output_file)}")


if __name__ == "__main__":
    main()