Repository navigation
Expand file tree
/
Copy pathcheck_file_sizes.py
More file actions
204 lines (167 loc) · 7.27 KB
/
Copy pathcheck_file_sizes.py
File metadata and controls
204 lines (167 loc) · 7.27 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
"""Block oversized files entering the repository through a pull request.
Scope is the change under review, not the whole tree: paths added, modified,
renamed or copied since the merge base. Files already committed above the
ceiling are left alone, so the gate never fails a pull request for something
it did not introduce.
The same ceiling is enforced locally by the ``check-added-large-files``
pre-commit hook. Keep ``MAX_KB`` and ``--maxkb`` in ``.pre-commit-config.yaml``
in lockstep — a unit test pins them together, because a hook that only runs
locally is not a gate. Note the local hook is weaker by construction: it only
looks at files being *added*, so it cannot catch an existing file that grows.
That is the case this script covers on every pull request.
On a push to the base branch itself the merge base is ``HEAD``, the diff is
empty, and the check is a no-op. This is a pull-request gate.
``EXEMPT_PREFIXES`` carves out directories whose contents are machine-generated
and legitimately large. Keep it as short as possible: every entry is a place
the ceiling no longer protects. A unit test pins the list, so growing it shows
up as a deliberate change in review.
"""
from __future__ import annotations
import argparse
import os
import subprocess
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from pathlib import Path
# Kilobytes, matching the pre-commit hook's ``--maxkb`` semantics exactly:
# a file is oversized when ``size_bytes > MAX_KB * 1024``.
MAX_KB = 640
DEFAULT_BASE = "origin/main"
# Directories exempt from the ceiling, as posix path prefixes.
#
# tests/fixtures/search_seed/ holds embedded search corpora regenerated by
# tests/fixtures/_dump_search_seed.py. Two of them already sit near 1 MB and
# grew ~60% in a single release; a ceiling they have to clear would be raised
# on every refresh, which teaches contributors to edit the gate instead of
# their payload. Outside this directory the largest tracked file is ~300 KB.
EXEMPT_PREFIXES = ("tests/fixtures/search_seed/",)
class BaseRefError(RuntimeError):
"""The comparison base could not be resolved."""
@dataclass(frozen=True)
class Violation:
path: str
size_bytes: int
def _repo_root() -> Path:
return Path(__file__).resolve().parent.parent
def is_exempt(path: str, prefixes: Sequence[str] = EXEMPT_PREFIXES) -> bool:
"""Whether ``path`` sits under an exempt directory prefix."""
normalised = path.replace("\\", "/")
return any(normalised.startswith(prefix) for prefix in prefixes)
def find_violations(
paths: Iterable[str],
*,
root: Path,
max_kb: int = MAX_KB,
exempt_prefixes: Sequence[str] = EXEMPT_PREFIXES,
) -> list[Violation]:
"""Return oversized entries among ``paths``, resolved against ``root``.
Entries under ``exempt_prefixes`` are skipped. So are entries that are not
regular files: a diff can name a path that no longer exists in the working
tree, and a symlink would be measured by its target rather than itself.
"""
limit_bytes = max_kb * 1024
violations: list[Violation] = []
for path in paths:
if is_exempt(path, exempt_prefixes):
continue
candidate = root / path
if candidate.is_symlink() or not candidate.is_file():
continue
size_bytes = candidate.stat().st_size
if size_bytes > limit_bytes:
violations.append(Violation(path=path, size_bytes=size_bytes))
return violations
def default_base_ref() -> str:
"""Resolve the base ref to diff against.
``GITHUB_BASE_REF`` is set by GitHub Actions on ``pull_request`` events
and names the target branch; everywhere else fall back to the default
branch's remote tracking ref.
"""
github_base = os.environ.get("GITHUB_BASE_REF", "").strip()
if github_base:
return f"origin/{github_base}"
return DEFAULT_BASE
def _git(root: Path, *args: str) -> str:
result = subprocess.run(
["git", *args],
cwd=root,
capture_output=True,
text=True,
)
if result.returncode != 0:
raise BaseRefError(
f"`git {' '.join(args)}` failed: {result.stderr.strip() or 'unknown error'}"
)
return result.stdout
def changed_paths(root: Path, base_ref: str) -> list[str]:
"""Paths this change adds or grows, relative to ``base_ref``.
Two sources, unioned:
* ``git diff`` from the merge base to the working tree (no second
revision), covering committed and uncommitted edits to tracked files.
``--diff-filter=ACMR`` keeps additions, copies, modifications and
renames; deletions are excluded.
* untracked, non-ignored files. ``git diff`` cannot see a newly created
file until it is staged, so a local pre-push run would otherwise pass a
brand-new oversized file. In CI the checkout is clean and this source
is empty.
Raises :class:`BaseRefError` when ``base_ref`` cannot be resolved — a
gate that silently passes on a shallow clone is worse than no gate.
"""
merge_base = _git(root, "merge-base", base_ref, "HEAD").strip()
if not merge_base:
raise BaseRefError(f"no merge base between {base_ref} and HEAD")
tracked = _git(
root,
"diff",
"--name-only",
"-z",
"--diff-filter=ACMR",
merge_base,
)
untracked = _git(root, "ls-files", "--others", "--exclude-standard", "-z")
entries = {entry for entry in tracked.split("\0") if entry}
entries.update(entry for entry in untracked.split("\0") if entry)
return sorted(entries)
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--base",
default=None,
help=f"ref to diff against (default: $GITHUB_BASE_REF, else {DEFAULT_BASE})",
)
args = parser.parse_args(argv)
root = _repo_root()
base_ref = args.base or default_base_ref()
try:
paths = changed_paths(root, base_ref)
except BaseRefError as exc:
print(
f"Repository file size check could not run: {exc}\n"
"The gate diffs against the base branch, so it needs that ref and "
"enough history to find a merge base. In CI, set "
"`fetch-depth: 0` on actions/checkout; locally, run "
"`git fetch origin main`."
)
return 1
violations = find_violations(paths, root=root)
if not violations:
print(
f"Repository file size check passed "
f"({len(paths)} changed file(s) vs {base_ref}, ceiling {MAX_KB} KB)."
)
return 0
print(
f"Repository file size check failed: this change adds or grows files "
f"above {MAX_KB} KB.\n"
"Large payloads belong in release artifacts, external hosting, or "
"another approved storage location, then linked from docs. If a "
"generated corpus genuinely has to live in-tree, add its directory "
"to EXEMPT_PREFIXES in this script rather than raising the ceiling "
"for the whole repository — and say why in the commit.\n"
)
for violation in sorted(violations, key=lambda item: -item.size_bytes):
size_kb = violation.size_bytes / 1024
print(f"- {violation.path}: {size_kb:.1f} KB")
return 1
if __name__ == "__main__":
raise SystemExit(main())