使用者:Srapoj/note/AF180
外觀
| code point | unicode name | category | logs |
|---|---|---|---|
| U+200B | ZERO WIDTH SPACE | Cf | 2256 |
| U+200C | ZERO WIDTH NON-JOINER | Cf | 1328 |
| U+FEFF | ZERO WIDTH NO-BREAK SPACE | Cf | 306 |
| U+200E | LEFT-TO-RIGHT MARK | Cf | 196 |
| U+180E | MONGOLIAN VOWEL SEPARATOR | Cf | 161 |
| U+200D | ZERO WIDTH JOINER | Cf | 146 |
| U+2060 | WORD JOINER | Cf | 133 |
| U+202C | POP DIRECTIONAL FORMATTING | Cf | 115 |
| U+00AD | SOFT HYPHEN | Cf | 74 |
| U+202A | LEFT-TO-RIGHT EMBEDDING | Cf | 44 |
| U+2063 | INVISIBLE SEPARATOR | Cf | 24 |
| U+200F | RIGHT-TO-LEFT MARK | Cf | 21 |
| U+202D | LEFT-TO-RIGHT OVERRIDE | Cf | 10 |
| U+2069 | POP DIRECTIONAL ISOLATE | Cf | 9 |
| U+007F | DELETE | Cc | 8 |
| U+0092 | PRIVATE USE TWO | Cc | 6 |
| U+2068 | FIRST STRONG ISOLATE | Cf | 6 |
| U+06DD | ARABIC END OF AYAH | Cf | 4 |
| U+202B | RIGHT-TO-LEFT EMBEDDING | Cf | 3 |
| U+2061 | FUNCTION APPLICATION | Cf | 3 |
| U+2066 | LEFT-TO-RIGHT ISOLATE | Cf | 3 |
| U+0082 | BREAK PERMITTED HERE | Cc | 2 |
| U+0086 | START OF SELECTED AREA | Cc | 1 |
| U+008B | PARTIAL LINE FORWARD | Cc | 1 |
| U+0093 | SET TRANSMIT STATE | Cc | 1 |
| U+0094 | CANCEL CHARACTER | Cc | 1 |
| U+0096 | START OF GUARDED AREA | Cc | 1 |
| U+0097 | END OF GUARDED AREA | Cc | 1 |
| U+009A | SINGLE CHARACTER INTRODUCER | Cc | 1 |
| U+206F | NOMINAL DIGIT SHAPES | Cf | 1 |
| U+13430 | EGYPTIAN HIEROGLYPH VERTICAL JOINER | Cf | 1 |
| U+13431 | EGYPTIAN HIEROGLYPH HORIZONTAL JOINER | Cf | 1 |
| U+13437 | EGYPTIAN HIEROGLYPH BEGIN SEGMENT | Cf | 1 |
| U+13438 | EGYPTIAN HIEROGLYPH END SEGMENT | Cf | 1 |
| U+E0062 | TAG LATIN SMALL LETTER B | Cf | 1 |
| U+E0067 | TAG LATIN SMALL LETTER G | Cf | 1 |
| U+E006C | TAG LATIN SMALL LETTER L | Cf | 1 |
| U+E0073 | TAG LATIN SMALL LETTER S | Cf | 1 |
| U+E0077 | TAG LATIN SMALL LETTER W | Cf | 1 |
| U+E007F | CANCEL TAG | Cf | 1 |
腳本由GPT-6 Luna生成。建議在PAWS等toolforge環境運行,公網訪問api.php目前似乎會被限速或有些擁堵,下載大響應的速度較慢。
#!/usr/bin/env python3
"""Count Unicode code points from recent zhwiki abuse filter 180 edits.
This script fetches only abuse-log ``details`` and checks ``added_lines``
locally.
"""
from __future__ import annotations
import argparse
import csv
import logging
import sys
import unicodedata
from collections import Counter
from collections.abc import Iterator, Mapping
from typing import Any
import pywikibot
import regex # re in stdlib does not support Unicode properties
import urllib3
urllib3.disable_warnings(category=urllib3.exceptions.InsecureRequestWarning)
#DEFAULT_WIKI_FAMILY = "wikipedia"
DEFAULT_WIKI_FAMILY = "zhc"
DEFAULT_WIKI_CODE = "zh"
FILTER_ID = 180
DEFAULT_MAX_ENTRIES = 5000
# This result was truncated because it would otherwise be larger than the limit of 12,582,912 bytes.
API_PAGE_SIZE = 10
PROGRESS_INTERVAL = 100
USER_AGENT = "afchar/1.0 (wikipedia:zh; User:Srapoj) {pwb} ({revision}) {python} {http_backend}"
# Apply the project user agent to requests made through Pywikibot itself.
pywikibot.config.user_agent_format = USER_AGENT
CHARACTER_PATTERN = regex.compile(r"(?!\t)[\p{Cc}\p{Cf}]")
def result_was_truncated(response: Mapping[str, Any]) -> bool:
"""Check whether the API reported truncating the result payload."""
warnings = response.get("warnings", {})
if not isinstance(warnings, Mapping):
return False
result_warning = warnings.get("result", {})
if not isinstance(result_warning, Mapping):
return False
message = result_warning.get("warnings", "")
return isinstance(message, str) and "truncated" in message.lower()
def iter_abuse_log_entries(
site: pywikibot.site.BaseSite, max_entries: int
) -> Iterator[dict[str, Any]]:
"""Fetch up to ``max_entries`` newest entries with details only."""
params: dict[str, Any] = {
"action": "query",
"formatversion": 2,
"list": "abuselog",
"aflfilter": str(FILTER_ID),
"afldir": "older",
"afllimit": min(API_PAGE_SIZE, max_entries),
"aflprop": "details",
}
fetched = 0
next_progress = PROGRESS_INTERVAL
while fetched < max_entries:
configured_page_size = min(API_PAGE_SIZE, max_entries - fetched)
page_size = configured_page_size
while True:
params["afllimit"] = page_size
response = site.simple_request(**params).submit()
if not result_was_truncated(response):
break
if page_size == 1:
raise RuntimeError(
"The AbuseLog result remained truncated with afllimit=1; "
"a single entry may exceed the API response size limit."
)
smaller_page_size = max(1, page_size // 2)
logging.warning(
"AbuseLog response was truncated at afllimit=%d; retrying "
"the same page with afllimit=%d",
page_size,
smaller_page_size,
)
page_size = smaller_page_size
entries = response.get("query", {}).get("abuselog", [])
if not entries:
break
for entry in entries:
if fetched >= max_entries:
break
fetched += 1
yield entry
if fetched >= next_progress:
logging.info(
"Fetched %d abuse-log entries (last page: %d)",
fetched,
len(entries),
)
next_progress = fetched + PROGRESS_INTERVAL
continuation = response.get("continue")
if not continuation:
break
params.update(continuation)
def build_argument_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--wiki-code", default=DEFAULT_WIKI_CODE)
parser.add_argument("--wiki-family", default=DEFAULT_WIKI_FAMILY)
parser.add_argument(
"--max-entries",
type=int,
default=DEFAULT_MAX_ENTRIES,
help="Maximum number of newest entries to fetch (default: 5000).",
)
parser.add_argument("--log-level", default="INFO")
return parser
def main() -> int:
parser = build_argument_parser()
args = parser.parse_args()
if args.max_entries < 1:
parser.error("--max-entries must be at least 1")
logging.basicConfig(
level=getattr(logging, args.log_level.upper(), logging.INFO),
format="%(levelname)s: %(message)s",
stream=sys.stderr,
)
site = pywikibot.Site(args.wiki_code, args.wiki_family)
site.login()
codepoint_log_counts: Counter[int] = Counter()
fetched = 0
interrupted = False
try:
for entry in iter_abuse_log_entries(site, args.max_entries):
fetched += 1
details = entry.get("details", {})
if not isinstance(details, Mapping):
continue
added_lines = details.get("added_lines")
if not isinstance(added_lines, list):
continue
entry_codepoints: set[int] = set()
for line in added_lines:
if not isinstance(line, str):
continue
for character in CHARACTER_PATTERN.findall(line):
codepoint = ord(character)
entry_codepoints.add(codepoint)
codepoint_log_counts.update(entry_codepoints)
except KeyboardInterrupt:
interrupted = True
print("\nInterrupted; reporting entries fetched so far.", file=sys.stderr)
writer = csv.writer(sys.stdout, delimiter="\t", lineterminator="\n")
writer.writerow(
[
"code_point",
"character",
"unicode_name",
"category",
"logs",
]
)
for codepoint, log_count in sorted(
codepoint_log_counts.items(), key=lambda item: (-item[1], item[0])
):
character = chr(codepoint)
writer.writerow(
[
f"U+{codepoint:04X}",
repr(character),
unicodedata.name(character, "<unnamed>"),
unicodedata.category(character),
log_count,
]
)
logging.info(
"Processed %d entries; found %d distinct matching code points%s",
fetched,
len(codepoint_log_counts),
" (partial run)" if interrupted else "",
)
return 130 if interrupted else 0
if __name__ == "__main__":
raise SystemExit(main())