Repository navigation
Expand file tree
/
Copy pathcheck_markup.py
More file actions
295 lines (251 loc) · 10.9 KB
/
Copy pathcheck_markup.py
File metadata and controls
295 lines (251 loc) · 10.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
#!/usr/bin/env python3
"""
scripts/check_markup.py
Verify Sphinx markup consistency between msgid and msgstr:
- Roles like :term:`text <target>` — the *target* (or the whole role, if
it has no explicit target) must match; the display text is expected to
be translated, matching Sphinx's own translation convention.
- Literal/code spans (``...``), substitution refs (|...|), and %s/{name}
placeholders — these must match verbatim, since they're not prose.
- Invalid C-string escape sequences in msgstr (a lone backslash followed
by a character that isn't a valid escape, see VALID_ESCAPE_CHARS) — these
make `msgfmt` fail with "invalid control sequence" and must be found
before they break a build.
- Files that cannot be parsed at all are reported (and fail the run)
instead of aborting it, so one broken file doesn't hide findings in the
rest.
Fuzzy entries are checked too: the docs are built with
gettext_allow_fuzzy_translations=1, so they reach the rendered output.
Output is grouped by file, with a per-file mismatch count and a grand
total at the end. Exits 1 if anything was found.
Requires: pip install polib
Usage:
python3 scripts/check_markup.py library/functions.po
python3 scripts/check_markup.py tutorial/*.po
python3 scripts/check_markup.py . # recurse a whole directory
"""
import re
import sys
from collections import Counter
from pathlib import Path
import polib
ROLE_PATTERN = re.compile(r":((?:\w+:)?[\w.-]+):`([^`]+)`")
# Sphinx itself accepts `label<target>` (no space) as well as the more
# common `label <target>` -- both are valid RST role syntax, and the
# upstream English source uses the no-space form in a few places (e.g.
# ":pypi:`file built<blurb>`"). The space before "<" is therefore made
# optional here so those roles are recognized as having an explicit
# target instead of being treated as if the *entire* display text were
# the target (which produced impossible-to-match false positives, since
# no translation could ever reproduce the literal English phrase).
TARGET_PATTERN = re.compile(r"^(.*?)\s?<([^<>]+)>$")
LITERAL_PATTERNS = [
("literal/code span", re.compile(r"``.*?``")),
("substitution ref", re.compile(r"\|[\w.-]+\|")),
("percent placeholder", re.compile(r"%\(\w+\)[a-zA-Z]|%[a-zA-Z]")),
("brace placeholder", re.compile(r"\{[^{}\s]*\}")),
]
# Valid C-string escapes that gettext/msgfmt accept inside a quoted
# string. A backslash followed by anything else is what msgfmt rejects
# with "invalid control sequence". Besides the single-character escapes
# (\\ \" \n \t \r \f \b \a \v), msgfmt also accepts octal (\101) and
# hex (\x41) escapes; only the first character after the backslash matters
# here, since the remaining digits are ordinary characters to this scanner.
VALID_ESCAPE_CHARS = set('\\"ntrfbav') | set("01234567") | {"x"}
# One raw quoted-string line as it appears in the .po file. Matches
# both the first line of an entry, which carries a keyword prefix
# (msgid "..." / msgstr "..." / msgstr[0] "..."), and bare continuation
# lines ("...") that follow it. Captured group is the RAW content
# between the quotes - escapes are NOT decoded, which is required for
# this check (see note below).
RAW_STRING_LINE = re.compile(
r'^(?:msgid|msgstr(?:\[\d+\])?|msgctxt)?\s*"((?:[^"\\]|\\.)*)"\s*$'
)
DISPLAY_ONLY_ROLES = {"dfn", "kbd", "guilabel", "menuselection", "samp", "file"}
DFN_PATTERN = re.compile(r":dfn:`")
DFN_ANGLE = re.compile(r":dfn:`[^`]*<[^`>]+>`")
def find_invalid_escapes_in_raw(raw: str):
"""Scan raw (undecoded) quoted-string content left-to-right the way
a C-string tokenizer would, consuming two characters whenever a
backslash is seen, and return the invalid `\\X` sequences found.
MUST run on raw file text, not on polib's .msgid/.msgstr - polib
decodes \\\\ into a single \\ before this script ever sees it, so
scanning decoded text turns safe, doubled backslashes (\\\\d in the
file, meaning a literal backslash-d) into false positives that look
like a lone backslash followed by an invalid character."""
invalid = []
i = 0
n = len(raw)
while i < n:
if raw[i] == "\\":
if i + 1 < n:
nxt = raw[i + 1]
if nxt not in VALID_ESCAPE_CHARS:
invalid.append("\\" + nxt)
i += 2
continue
else:
invalid.append("\\<end-of-line>")
i += 1
continue
i += 1
return invalid
def check_raw_escapes(path: Path):
"""Line-by-line scan of the raw file for invalid escape sequences,
independent of polib. Returns a list of (line_no, line_text,
invalid_list) for every quoted-string line with a problem."""
findings = []
for lineno, line in enumerate(
path.read_text(encoding="utf-8").splitlines(), start=1
):
m = RAW_STRING_LINE.match(line.strip())
if not m:
continue
invalid = find_invalid_escapes_in_raw(m.group(1))
if invalid:
findings.append((lineno, line.strip(), invalid))
return findings
def extract_role_targets(text: str):
"""For each Sphinx role, return its target: the <target> anchor if
present, otherwise the role's full display text. Display-only roles
(like :dfn:) are skipped, since their text is translatable."""
targets = []
for role, body in ROLE_PATTERN.findall(text):
if role in DISPLAY_ONLY_ROLES:
continue
m = TARGET_PATTERN.match(body)
targets.append(m.group(2).strip() if m else body.strip())
return targets
def check_file(path: Path):
"""Return a list of finding-dicts for this file (empty if none)."""
results = []
po = polib.pofile(str(path))
for entry in po:
if entry.obsolete or not entry.msgid or not entry.msgstr:
continue # obsolete entry, header, or still untranslated
findings = []
expected_targets = Counter(extract_role_targets(entry.msgid))
found_targets = Counter(extract_role_targets(entry.msgstr))
if expected_targets != found_targets:
missing = list((expected_targets - found_targets).elements())
extra = list((found_targets - expected_targets).elements())
if missing:
findings.append(f"missing role target(s): {missing}")
if extra:
findings.append(f"role target(s) not in source: {extra}")
# --- :dfn: checks (display-only role, so compare count, not text) ---
msgid_dfn_count = len(DFN_PATTERN.findall(entry.msgid))
msgstr_dfn_count = len(DFN_PATTERN.findall(entry.msgstr))
if msgid_dfn_count != msgstr_dfn_count:
findings.append(
f":dfn: count differs ({msgid_dfn_count} vs {msgstr_dfn_count})"
)
if DFN_ANGLE.search(entry.msgstr):
findings.append(":dfn: contains <...>, which Sphinx renders literally")
for label, pattern in LITERAL_PATTERNS:
expected = Counter(pattern.findall(entry.msgid))
found = Counter(pattern.findall(entry.msgstr))
if expected != found:
missing = list((expected - found).elements())
extra = list((found - expected).elements())
if missing:
findings.append(f"missing {label}: {missing}")
if extra:
findings.append(f"extra {label} not in source: {extra}")
if findings:
loc = (
f"{entry.occurrences[0][0]}:{entry.occurrences[0][1]}"
if entry.occurrences
else ""
)
results.append(
{
"loc": loc,
"fuzzy": entry.fuzzy,
"findings": findings,
"msgid": entry.msgid,
"msgstr": entry.msgstr,
}
)
return results
def print_file_group(path: Path, results):
print(f"\n{'=' * 70}")
print(f"{path} ({len(results)} mismatch(es))")
print("=" * 70)
for r in results:
tag = " [fuzzy]" if r["fuzzy"] else ""
loc = f" ({r['loc']})" if r["loc"] else ""
print(f"{loc}{tag}: {'; '.join(r['findings'])}")
print(f" msgid : {r['msgid'][:100]}")
print(f" msgstr: {r['msgstr'][:100]}")
def main():
if len(sys.argv) < 2:
print(__doc__)
sys.exit(1)
files = []
for arg in sys.argv[1:]:
p = Path(arg)
if p.is_dir():
files.extend(sorted(f for f in p.rglob("*.po") if ".git" not in f.parts))
else:
files.append(p)
total = 0
per_file_counts = []
escape_total = 0
escape_files = 0
parse_errors = []
for f in files:
# The raw escape scan runs first and independently of polib: a file
# with a bad escape is exactly the kind polib may choke on, and the
# escape findings explain why.
try:
escape_findings = check_raw_escapes(f)
except (OSError, ValueError) as exc: # ValueError: UnicodeDecodeError
escape_findings = []
parse_errors.append((f, exc))
try:
results = check_file(f)
except (OSError, ValueError) as exc: # polib raises OSError on bad syntax
results = []
if not escape_findings: # otherwise the escapes already explain it
parse_errors.append((f, exc))
if results:
print_file_group(f, results)
total += len(results)
per_file_counts.append((f, len(results)))
if escape_findings:
print(f"\n{'=' * 70}")
print(f"{f} ({len(escape_findings)} invalid escape sequence line(s))")
print("=" * 70)
for lineno, line, invalid in escape_findings:
print(f" line {lineno}: invalid {invalid}")
print(f" {line[:120]}")
escape_total += len(escape_findings)
escape_files += 1
if parse_errors:
print(f"\n{'=' * 70}")
print(f"{len(parse_errors)} file(s) could not be parsed:")
print("=" * 70)
for f, exc in parse_errors:
print(f" {f}: {exc}")
if total:
print(f"\n{'=' * 70}")
print("Summary by file (sorted by mismatch count, descending):")
print("=" * 70)
for f, count in sorted(per_file_counts, key=lambda x: -x[1]):
print(f" {count:4d} {f}")
print(
f"\n{total} markup mismatch(es) found across {len(per_file_counts)} file(s)."
)
if escape_total:
print(
f"\n{escape_total} invalid escape sequence line(s) found across "
f"{escape_files} file(s). These will make msgfmt fail with "
f"'invalid control sequence' - fix the offending backslash "
f"in each line above."
)
if total or escape_total or parse_errors:
sys.exit(1)
print("No markup mismatches found.")
if __name__ == "__main__":
main()