Repository navigation
Expand file tree
/
Copy pathteam_stats.py
More file actions
438 lines (354 loc) · 14.3 KB
/
Copy pathteam_stats.py
File metadata and controls
438 lines (354 loc) · 14.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
#!/usr/bin/env python3
"""
scripts/team_stats.py
Compute a per-contributor "Translated Count" for TEAM.md by walking every
.po file and using ``git blame`` to attribute each translated msgstr to the
author who last changed it.
Contributors are identified by their git ``user.name``, which is public
information visible on every commit. Names are mapped to their TEAM.md
handles through the ``NAME_ALIASES`` table (a git username is not always
the same as the handle used on GitHub/TEAM.md).
If two contributors share the same git username, a warning is printed with
a partially redacted email so the conflict can be resolved by asking one of
them to update their ``git config user.name``.
Mechanical commits are excluded:
* bulk ``Sync translations with CPython`` syncs
* header-only maintenance commits (``Update .po files``)
* Transifex/bot import commits and bot accounts
Requires: pip install polib
Usage:
python3 scripts/team_stats.py # report over the whole repo
python3 scripts/team_stats.py tutorial/ # restrict to a directory
python3 scripts/team_stats.py --update-teammd # rewrite TEAM.md in-place
"""
import argparse
import re
import subprocess
from collections import defaultdict
from pathlib import Path
import polib
REPO_ROOT = Path(__file__).resolve().parent.parent
BOT_NAME_RE = re.compile(
r"github[^\w]*actions|\[bot\]|not committed yet", re.IGNORECASE
)
BOT_EMAIL_RE = re.compile(
r"github-actions|transifex|\[bot\]|@users\.noreply\.github\.com",
re.IGNORECASE,
)
MECHANICAL_SUBJECT_RE = re.compile(
r"^(?:sync\s+translations\s+with\s+cpython\b"
r"|update\s+\.po\s+files(?:\s*\(\d+\))?\s*$"
r"|update\s+farsi\s+translations\s+from\s+transifex\b)",
re.IGNORECASE,
)
# git user.name -> TEAM.md User handle (lowercase keys).
NAME_ALIASES = {
"sepehr rasouli": "sepehr-rs",
"revisto": "Revisto",
"alireza shabani": "Revisto",
"alireza shabani (revisto)": "Revisto",
"invincible627": "invincible627",
"khosro": "khosro_o",
"ramiz-22": "Ramiz_222",
"aimer": "aimer.hs872",
}
# Default role used when a contributor is first added to TEAM.md.
# Handles that already have a row keep the role stored in TEAM.md; this
# table is only consulted for brand-new rows.
NEW_ROW_ROLES = {}
SKIP_DIRS = {".git", ".cpython-src", ".venv", "__pycache__", "venv"}
TEAMMD_ROW_RE = re.compile(
r"^\|\s*(?P<user>[^|]+?)\s*\|\s*(?P<role>[^|]+?)\s*\|\s*"
r"(?P<t>\d+(?:\s*\([^)]*\))?)\s*\|$"
)
# ---------------------------------------------------------------------------
# Privacy helper
# ---------------------------------------------------------------------------
def redact_email(email: str) -> str:
"""Return a partially redacted email for warning messages.
``someone@example.com`` → ``s*****e@e******.com``
Only the local part and domain name are obscured; the TLD is kept so
the domain type is still recognisable.
"""
if "@" not in email:
return "***"
local, domain = email.rsplit("@", 1)
def _blur(s: str) -> str:
if len(s) <= 2:
return s[0] + "*"
return s[0] + "*" * (len(s) - 2) + s[-1]
if "." in domain:
domain_name, tld = domain.rsplit(".", 1)
redacted_domain = f"{_blur(domain_name)}.{tld}"
else:
redacted_domain = _blur(domain)
return f"{_blur(local)}@{redacted_domain}"
# ---------------------------------------------------------------------------
# Git helpers
# ---------------------------------------------------------------------------
def _is_mechanical(subject: str) -> bool:
return bool(MECHANICAL_SUBJECT_RE.search(subject))
def _is_bot(name: str, email: str) -> bool:
return bool(BOT_NAME_RE.search(name) or BOT_EMAIL_RE.search(email))
def git_blame_porcelain(path: Path) -> dict[str, dict]:
"""Run ``git blame --porcelain`` and return a commit-hash → info map.
Each value is a dict with keys: ``name``, ``email``, ``subject``,
``author_time`` (unix timestamp, int), and ``lines`` (a set of 1-based
line numbers blamed to that commit).
"""
result = subprocess.run(
[
"git",
"-C",
str(REPO_ROOT),
"blame",
"--porcelain",
"--",
str(path.resolve().relative_to(REPO_ROOT)),
],
capture_output=True,
text=True,
check=False,
)
if result.returncode != 0:
return {}
commits: dict[str, dict] = {}
current_hash = None
for raw_line in result.stdout.splitlines():
# Commit header: "<40-char-hash> <orig-line> <result-line> [<num-lines>]"
parts = raw_line.split()
if (
len(parts) >= 3
and len(parts[0]) == 40
and parts[0].isalnum()
and parts[1].isdigit()
and parts[2].isdigit()
):
h = parts[0]
result_line = int(parts[2])
current_hash = h
if h not in commits:
commits[h] = {
"name": "",
"email": "",
"subject": "",
"author_time": -1,
"lines": set(),
}
commits[h]["lines"].add(result_line)
elif raw_line.startswith("author ") and current_hash:
commits[current_hash]["name"] = raw_line[len("author ") :].strip()
elif raw_line.startswith("author-mail ") and current_hash:
email = raw_line[len("author-mail ") :].strip().strip("<>")
commits[current_hash]["email"] = email.lower()
elif raw_line.startswith("author-time ") and current_hash:
try:
commits[current_hash]["author_time"] = int(
raw_line[len("author-time ") :].strip()
)
except ValueError:
pass
elif raw_line.startswith("summary ") and current_hash:
commits[current_hash]["subject"] = raw_line[len("summary ") :]
return commits
def check_name_collisions(blame: dict[str, dict]) -> None:
"""Warn if the same git username appears with more than one email address.
This indicates two distinct people sharing a username, which would cause
their counts to be merged incorrectly. Ask the affected contributor to
run ``git config user.name`` to pick a unique name.
"""
name_to_emails: dict[str, set[str]] = defaultdict(set)
for info in blame.values():
name = info["name"]
email = info["email"]
if name and email and not _is_bot(name, email):
name_to_emails[name].add(email)
for name, emails in name_to_emails.items():
if len(emails) > 1:
redacted = " vs ".join(redact_email(e) for e in sorted(emails))
print(
f" warning: username '{name}' is used by multiple authors "
f"({redacted}) — counts may be merged incorrectly. "
f"Ask one of them to update their git config user.name."
)
def real_author_for_lines(
blame: dict[str, dict],
line_numbers: set[int],
) -> str | None:
"""Return the git username of the most recent real (non-bot, non-mechanical)
author who touched any of ``line_numbers``, or None.
Recency is determined by each candidate commit's ``author-time`` (a unix
timestamp reported directly by ``git blame --porcelain``), not by line
position in the file. Line number is not a valid proxy for edit time: a
numerically later line is not necessarily edited more recently, so using
it as a stand-in silently misattributes entries whose msgstr lines were
last touched by different commits.
"""
best_name: str | None = None
best_time: int = -1
for info in blame.values():
overlap = info["lines"] & line_numbers
if not overlap:
continue
if _is_bot(info["name"], info["email"]) or _is_mechanical(info["subject"]):
continue
candidate_time = info["author_time"]
if candidate_time > best_time:
best_time = candidate_time
best_name = info["name"]
return best_name
# ---------------------------------------------------------------------------
# .po file walking
# ---------------------------------------------------------------------------
def collect_files(paths: list[str]) -> list[Path]:
files = []
for arg in paths:
p = Path(arg) if Path(arg).is_absolute() else REPO_ROOT / arg
if p.is_dir():
for f in sorted(p.rglob("*.po")):
rel_parts = set(f.relative_to(REPO_ROOT).parts)
if rel_parts & SKIP_DIRS:
continue
if any(part.startswith(".") for part in f.parts):
continue
files.append(f)
elif p.suffix == ".po":
files.append(p)
return files
def msgstr_line_numbers(po_path: Path, entry: polib.POEntry) -> set[int]:
"""Return the 1-based line numbers that belong to an entry's msgstr.
polib exposes ``entry.linenum`` (the msgid line). We scan forward from
there to find the msgstr block, collecting every continuation line too.
"""
lines = po_path.read_text(encoding="utf-8", errors="replace").splitlines()
total = len(lines)
i = entry.linenum # 1-based → use as 0-based index (points just past msgid)
# Walk forward to the msgstr line.
while i < total and not lines[i].startswith("msgstr"):
i += 1
# Collect the msgstr line and any quoted continuation lines.
result: set[int] = set()
while i < total:
stripped = lines[i].strip()
if stripped.startswith("msgstr") or stripped.startswith('"'):
result.add(i + 1) # convert back to 1-based
i += 1
else:
break
return result
def compute_counts(paths: list[str]) -> dict[str, int]:
"""Walk .po files and accumulate per-contributor translated-entry counts."""
counts: dict[str, int] = defaultdict(int)
for po_path in collect_files(paths):
try:
po = polib.pofile(str(po_path))
except Exception as exc:
print(f" warning: could not parse {po_path}: {exc}")
continue
translated = [e for e in po.translated_entries() if "fuzzy" not in e.flags]
if not translated:
continue
blame = git_blame_porcelain(po_path)
if not blame:
continue
check_name_collisions(blame)
for entry in translated:
line_nums = msgstr_line_numbers(po_path, entry)
if not line_nums:
continue
name = real_author_for_lines(blame, line_nums)
if name:
counts[NAME_ALIASES.get(name.lower(), name)] += 1
else:
counts["(unassigned)"] += 1
return dict(counts)
def teammd_totals() -> dict[str, int]:
"""Return {username: translated count} from TEAM.md.
TEAM.md is the canonical contributor record: the nightly run updates it
from git blame for git-visible contributors and preserves the Transifex-era
counts (including restored numbers) for everyone else, so this captures
both eras.
"""
path = REPO_ROOT / "TEAM.md"
totals: dict[str, int] = {}
for line in path.read_text(encoding="utf-8").splitlines():
m = TEAMMD_ROW_RE.match(line)
if not m:
continue
t = int(m.group("t").split()[0])
if t > 0:
totals[m.group("user")] = t
return totals
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def print_report(counts: dict[str, int]) -> None:
total = sum(counts.values())
print(f"Total non-fuzzy translated entries attributed: {total}\n")
print("| Contributor | Translated |")
print("|:------------|-----------:|")
for user, count in sorted(counts.items(), key=lambda kv: -kv[1]):
print(f"| {user} | {count} |")
def update_teammd(counts: dict[str, int]) -> None:
"""Update the Translated column in TEAM.md; append rows for new handles.
An existing count is never decreased: git-blame attribution only grows over
time and can't observe Transifex-era work, so taking the max preserves
restored numbers (e.g. Revisto's) across nightly runs.
"""
path = REPO_ROOT / "TEAM.md"
lines = path.read_text(encoding="utf-8").splitlines()
# Build an index of existing rows by username.
rows: dict[str, tuple[int, str, int]] = {}
for i, line in enumerate(lines):
m = TEAMMD_ROW_RE.match(line)
if m:
rows[m.group("user")] = (i, m.group("role"), int(m.group("t").split()[0]))
touched: list[str] = []
new_rows: list[str] = []
for user, count in sorted(counts.items(), key=lambda kv: -kv[1]):
if user == "(unassigned)" or count <= 0:
continue
if user in rows:
i, role, prev = rows[user]
keep = max(prev, count)
if keep == prev:
continue
lines[i] = f"| {user} | {role} | {keep} |"
touched.append(user)
else:
role = NEW_ROW_ROLES.get(user, "translator")
new_rows.append(f"| {user} | {role} | {count} |")
touched.append(user)
if new_rows:
lines.extend(new_rows)
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
print(f"TEAM.md: updated {len(touched)} contributor row(s)")
if new_rows:
print(f" added {len(new_rows)} new row(s) — review roles manually")
# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"paths",
nargs="*",
default=["."],
help="Files or directories to scan (default: whole repo)",
)
parser.add_argument(
"--update-teammd",
action="store_true",
help="rewrite TEAM.md from the computed counts",
)
args = parser.parse_args()
counts = compute_counts(args.paths)
if args.update_teammd:
update_teammd(counts)
else:
print_report(counts)
if __name__ == "__main__":
main()