Repository navigation
Expand file tree
/
Copy pathcheck_cjk.py
More file actions
101 lines (82 loc) · 3.83 KB
/
Copy pathcheck_cjk.py
File metadata and controls
101 lines (82 loc) · 3.83 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
#!/usr/bin/env python3
"""Scan tracked text files for CJK characters outside the language-policy allowlist.
Replaces the grep-based reference command that used to live in
``.claude/rules/language-policy.md``. That command silently produced false
negatives on this repo: the ``grep -zZv`` + ``xargs -0`` NUL pipeline
mis-parsed the path list and reported "clean" even when violations existed.
Exit code 0 = clean, 1 = violations found (paths + line numbers printed).
Usage:
python scripts/check_cjk.py # scan all tracked files
python scripts/check_cjk.py a.py b.md # scan specific files (pre-commit)
python scripts/check_cjk.py --quiet # per-file counts only
"""
from __future__ import annotations
import argparse
import os
import re
import subprocess
# CJK / fullwidth code points: CJK symbols & ideographs, Hangul syllables,
# and halfwidth/fullwidth forms. Kept as escapes so this file stays ASCII.
_CJK = re.compile("[\\u3000-\\u9fff\\uac00-\\ud7af\\uff00-\\uffef]")
# A line carrying this marker (in a comment) is exempt: a language switcher
# names the language in itself, a community badge names its group.
_ALLOW_MARKER = "cjk-allow"
def _is_allowlisted(path: str) -> bool:
"""Return True if CJK is permitted in this path (see language-policy.md)."""
name = os.path.basename(path)
# 1. Tests: fixtures, sample inputs, and CJK-behavior assertions.
if path.startswith("tests/"):
return True
# 2. Tokenizer NLP resources (stopword lists, segmentation examples).
if path.startswith("src/everos/component/tokenizer/"):
return True
# 3. Locale-suffixed sample data, e.g. data/solo_chat_zh.json.
if re.match(r"data/.*_(zh|ja|ko)\.", path):
return True
# 4. Translated doc mirrors, e.g. README.zh.md.
if re.search(r"\.(zh|ja|ko)\.md$", path):
return True
# 5. Filenames explicitly marked with a CJK/locale token.
if re.search(r"(^|[._-])(cjk|zh|ja|ko)([._-]|$)", name):
return True
# 6. Third-party integration samples: each use case ships its own demo
# content and branding for its audience, like test fixtures do.
return path.startswith("use-cases/")
def _tracked_files() -> list[str]:
out = subprocess.check_output(["git", "ls-files"], text=True)
return out.splitlines()
def main() -> int:
parser = argparse.ArgumentParser(description="CJK language-policy scanner.")
parser.add_argument("files", nargs="*", help="files to scan (default: all tracked)")
parser.add_argument("--quiet", action="store_true", help="per-file counts only")
args = parser.parse_args()
paths = args.files or _tracked_files()
violations: list[tuple[str, int, str]] = []
for path in paths:
if _is_allowlisted(path):
continue
try:
with open(path, encoding="utf-8") as fh:
lines = fh.readlines()
except (UnicodeDecodeError, FileNotFoundError, IsADirectoryError):
continue # binary / missing / directory: nothing to scan
for i, line in enumerate(lines, start=1):
if _CJK.search(line) and _ALLOW_MARKER not in line:
violations.append((path, i, line.strip()))
if not violations:
print("CJK language-policy: clean")
return 0
by_file: dict[str, int] = {}
for path, _lineno, _text in violations:
by_file[path] = by_file.get(path, 0) + 1
print(f"CJK language-policy: {len(violations)} hit(s) in {len(by_file)} file(s)\n")
if args.quiet:
for path, count in sorted(by_file.items(), key=lambda kv: -kv[1]):
print(f" {count:4d} {path}")
else:
for path, lineno, text in violations:
print(f" {path}:{lineno}: {text[:100]}")
print("\nAllowed CJK locations are defined in .claude/rules/language-policy.md")
return 1
if __name__ == "__main__":
raise SystemExit(main())