跳至內容

使用者:Srapoj/note/AF180

維基百科,自由的百科全書
code point unicode name category logs
U+200B ZERO WIDTH SPACE Cf 2256
U+200C ZERO WIDTH NON-JOINER Cf 1328
U+FEFF ZERO WIDTH NO-BREAK SPACE Cf 306
U+200E LEFT-TO-RIGHT MARK Cf 196
U+180E MONGOLIAN VOWEL SEPARATOR Cf 161
U+200D ZERO WIDTH JOINER Cf 146
U+2060 WORD JOINER Cf 133
U+202C POP DIRECTIONAL FORMATTING Cf 115
U+00AD SOFT HYPHEN Cf 74
U+202A LEFT-TO-RIGHT EMBEDDING Cf 44
U+2063 INVISIBLE SEPARATOR Cf 24
U+200F RIGHT-TO-LEFT MARK Cf 21
U+202D LEFT-TO-RIGHT OVERRIDE Cf 10
U+2069 POP DIRECTIONAL ISOLATE Cf 9
U+007F DELETE Cc 8
U+0092 PRIVATE USE TWO Cc 6
U+2068 FIRST STRONG ISOLATE Cf 6
U+06DD ARABIC END OF AYAH Cf 4
U+202B RIGHT-TO-LEFT EMBEDDING Cf 3
U+2061 FUNCTION APPLICATION Cf 3
U+2066 LEFT-TO-RIGHT ISOLATE Cf 3
U+0082 BREAK PERMITTED HERE Cc 2
U+0086 START OF SELECTED AREA Cc 1
U+008B PARTIAL LINE FORWARD Cc 1
U+0093 SET TRANSMIT STATE Cc 1
U+0094 CANCEL CHARACTER Cc 1
U+0096 START OF GUARDED AREA Cc 1
U+0097 END OF GUARDED AREA Cc 1
U+009A SINGLE CHARACTER INTRODUCER Cc 1
U+206F NOMINAL DIGIT SHAPES Cf 1
U+13430 EGYPTIAN HIEROGLYPH VERTICAL JOINER Cf 1
U+13431 EGYPTIAN HIEROGLYPH HORIZONTAL JOINER Cf 1
U+13437 EGYPTIAN HIEROGLYPH BEGIN SEGMENT Cf 1
U+13438 EGYPTIAN HIEROGLYPH END SEGMENT Cf 1
U+E0062 TAG LATIN SMALL LETTER B Cf 1
U+E0067 TAG LATIN SMALL LETTER G Cf 1
U+E006C TAG LATIN SMALL LETTER L Cf 1
U+E0073 TAG LATIN SMALL LETTER S Cf 1
U+E0077 TAG LATIN SMALL LETTER W Cf 1
U+E007F CANCEL TAG Cf 1

腳本由GPT-6 Luna生成。建議在PAWS等toolforge環境運行,公網訪問api.php目前似乎會被限速或有些擁堵,下載大響應的速度較慢。

#!/usr/bin/env python3
"""Count Unicode code points from recent zhwiki abuse filter 180 edits.

This script fetches only abuse-log ``details`` and checks ``added_lines``
locally.
"""

from __future__ import annotations

import argparse
import csv
import logging
import sys
import unicodedata
from collections import Counter
from collections.abc import Iterator, Mapping
from typing import Any

import pywikibot
import regex # re in stdlib does not support Unicode properties

import urllib3
urllib3.disable_warnings(category=urllib3.exceptions.InsecureRequestWarning)


#DEFAULT_WIKI_FAMILY = "wikipedia"
DEFAULT_WIKI_FAMILY = "zhc"
DEFAULT_WIKI_CODE = "zh"
FILTER_ID = 180
DEFAULT_MAX_ENTRIES = 5000
# This result was truncated because it would otherwise be larger than the limit of 12,582,912 bytes.
API_PAGE_SIZE = 10
PROGRESS_INTERVAL = 100
USER_AGENT = "afchar/1.0 (wikipedia:zh; User:Srapoj) {pwb} ({revision}) {python} {http_backend}"

# Apply the project user agent to requests made through Pywikibot itself.
pywikibot.config.user_agent_format = USER_AGENT
CHARACTER_PATTERN = regex.compile(r"(?!\t)[\p{Cc}\p{Cf}]")


def result_was_truncated(response: Mapping[str, Any]) -> bool:
    """Check whether the API reported truncating the result payload."""
    warnings = response.get("warnings", {})
    if not isinstance(warnings, Mapping):
        return False
    result_warning = warnings.get("result", {})
    if not isinstance(result_warning, Mapping):
        return False
    message = result_warning.get("warnings", "")
    return isinstance(message, str) and "truncated" in message.lower()


def iter_abuse_log_entries(
    site: pywikibot.site.BaseSite, max_entries: int
) -> Iterator[dict[str, Any]]:
    """Fetch up to ``max_entries`` newest entries with details only."""
    params: dict[str, Any] = {
        "action": "query",
        "formatversion": 2,
        "list": "abuselog",
        "aflfilter": str(FILTER_ID),
        "afldir": "older",
        "afllimit": min(API_PAGE_SIZE, max_entries),
        "aflprop": "details",
    }
    fetched = 0
    next_progress = PROGRESS_INTERVAL

    while fetched < max_entries:
        configured_page_size = min(API_PAGE_SIZE, max_entries - fetched)
        page_size = configured_page_size
        while True:
            params["afllimit"] = page_size
            response = site.simple_request(**params).submit()
            if not result_was_truncated(response):
                break
            if page_size == 1:
                raise RuntimeError(
                    "The AbuseLog result remained truncated with afllimit=1; "
                    "a single entry may exceed the API response size limit."
                )

            smaller_page_size = max(1, page_size // 2)
            logging.warning(
                "AbuseLog response was truncated at afllimit=%d; retrying "
                "the same page with afllimit=%d",
                page_size,
                smaller_page_size,
            )
            page_size = smaller_page_size

        entries = response.get("query", {}).get("abuselog", [])
        if not entries:
            break

        for entry in entries:
            if fetched >= max_entries:
                break
            fetched += 1
            yield entry

        if fetched >= next_progress:
            logging.info(
                "Fetched %d abuse-log entries (last page: %d)",
                fetched,
                len(entries),
            )
            next_progress = fetched + PROGRESS_INTERVAL

        continuation = response.get("continue")
        if not continuation:
            break
        params.update(continuation)


def build_argument_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--wiki-code", default=DEFAULT_WIKI_CODE)
    parser.add_argument("--wiki-family", default=DEFAULT_WIKI_FAMILY)
    parser.add_argument(
        "--max-entries",
        type=int,
        default=DEFAULT_MAX_ENTRIES,
        help="Maximum number of newest entries to fetch (default: 5000).",
    )
    parser.add_argument("--log-level", default="INFO")
    return parser


def main() -> int:
    parser = build_argument_parser()
    args = parser.parse_args()
    if args.max_entries < 1:
        parser.error("--max-entries must be at least 1")

    logging.basicConfig(
        level=getattr(logging, args.log_level.upper(), logging.INFO),
        format="%(levelname)s: %(message)s",
        stream=sys.stderr,
    )

    site = pywikibot.Site(args.wiki_code, args.wiki_family)
    site.login()
    codepoint_log_counts: Counter[int] = Counter()
    fetched = 0
    interrupted = False

    try:
        for entry in iter_abuse_log_entries(site, args.max_entries):
            fetched += 1
            details = entry.get("details", {})
            if not isinstance(details, Mapping):
                continue

            added_lines = details.get("added_lines")
            if not isinstance(added_lines, list):
                continue

            entry_codepoints: set[int] = set()
            for line in added_lines:
                if not isinstance(line, str):
                    continue
                for character in CHARACTER_PATTERN.findall(line):
                    codepoint = ord(character)
                    entry_codepoints.add(codepoint)

            codepoint_log_counts.update(entry_codepoints)
    except KeyboardInterrupt:
        interrupted = True
        print("\nInterrupted; reporting entries fetched so far.", file=sys.stderr)

    writer = csv.writer(sys.stdout, delimiter="\t", lineterminator="\n")
    writer.writerow(
        [
            "code_point",
            "character",
            "unicode_name",
            "category",
            "logs",
        ]
    )
    for codepoint, log_count in sorted(
        codepoint_log_counts.items(), key=lambda item: (-item[1], item[0])
    ):
        character = chr(codepoint)
        writer.writerow(
            [
                f"U+{codepoint:04X}",
                repr(character),
                unicodedata.name(character, "<unnamed>"),
                unicodedata.category(character),
                log_count,
            ]
        )

    logging.info(
        "Processed %d entries; found %d distinct matching code points%s",
        fetched,
        len(codepoint_log_counts),
        " (partial run)" if interrupted else "",
    )
    return 130 if interrupted else 0


if __name__ == "__main__":
    raise SystemExit(main())