Repository navigation
Expand file tree
/
Copy pathcodegen
More file actions
executable file
·363 lines (310 loc) · 13.7 KB
/
Copy pathcodegen
File metadata and controls
executable file
·363 lines (310 loc) · 13.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
#!/usr/bin/env python3
"""Run the nativeapi code generators.
Orchestrates the Rust generators in tools/codegen against the workspace layout:
./codegen # full run: C ABI, then all bindings
./codegen capi # C ABI + umbrella header only
./codegen bindings # bindings only (parses headers, writes no C ABI)
./codegen bindings --lang rust,go,swift
./codegen check # verify everything is up to date (CI mode)
./codegen ffigen # Dart bindings, then ffigen (cnativeapi's raw FFI)
./codegen readme # copy the shared README sections into every repo
./codegen sync # after a core change: regenerate everything,
# rerun bindgen/ffigen, and commit core,
# then the workspace
./codegen sync -m "Add Foo API" --push
./codegen --dump-ir ir.json # also keep the intermediate IR at this path
Every bindings run also writes the API reference the website renders
(website/content/api/: per-header JSON with docs and every binding's
signatures); `check` verifies it like any other generated file.
The C ABI generator parses the C++ headers in core/ with libclang and emits an
IR JSON; the bindings generator consumes that IR. Every binding lives in this
repository under bindings/ and builds against the core/ submodule directly.
"""
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
WORKSPACE = Path(__file__).resolve().parent
CORE = WORKSPACE / "core"
CODEGEN = WORKSPACE / "tools" / "codegen"
DEFAULT_IR = CODEGEN / "target" / "ir.json"
# API reference JSON for the website (every binding's signatures + docs).
API_REFERENCE = WORKSPACE / "website" / "content" / "api"
# lang -> (cli flag of codegen-bindings, binding directory)
BINDINGS = {
"rust": ("--rust", WORKSPACE / "bindings" / "rust"),
"dart": ("--dart", WORKSPACE / "bindings" / "dart"),
"csharp": ("--csharp", WORKSPACE / "bindings" / "csharp"),
"js": ("--js", WORKSPACE / "bindings" / "js"),
"python": ("--python", WORKSPACE / "bindings" / "python"),
"go": ("--go", WORKSPACE / "bindings" / "go"),
"swift": ("--swift", WORKSPACE / "bindings" / "swift"),
}
def run(binary: str, *extra: str) -> None:
cmd = [
"cargo", "run", "--quiet", "--release",
"--manifest-path", str(CODEGEN / "Cargo.toml"),
"-p", binary, "--", *extra,
]
result = subprocess.run(cmd)
if result.returncode != 0:
sys.exit(result.returncode)
def binding_flags(langs: list[str]) -> list[str]:
flags = []
for lang in langs:
flag, repo = BINDINGS[lang]
flags += [flag, str(repo)]
return flags
def run_capi(ir: Path, check: bool, parse_only: bool = False) -> None:
extra = ["--repo", str(CORE), "--emit-ir", str(ir)]
if parse_only:
extra.append("--parse-only")
if check:
extra.append("--check")
run("codegen-capi", *extra)
def run_bindings(ir: Path, langs: list[str], check: bool) -> None:
extra = ["--ir", str(ir), "--repo", str(CORE), *binding_flags(langs),
"--api-reference", str(API_REFERENCE)]
if check:
extra.append("--check")
run("codegen-bindings", *extra)
# ---------------------------------------------------------------------------
# readme: keep the shared README sections identical in every repo
# ---------------------------------------------------------------------------
README_SNIPPETS = WORKSPACE / "tools" / "readme"
# snippet file -> READMEs (relative to the workspace) carrying that section.
# A snippet starts with its own `## Heading` line; the section with the same
# heading in each README is replaced wholesale, the rest is hand-written.
README_SECTIONS = {
"contributing.md": [
"core/README.md",
"bindings/dart/README.md",
"bindings/rust/README.md",
"bindings/csharp/README.md",
"bindings/js/README.md",
"bindings/python/README.md",
"bindings/go/README.md",
"bindings/swift/README.md",
],
"contributing.zh.md": [
"bindings/dart/README-ZH.md",
],
}
def sync_readmes(check: bool) -> bool:
"""Returns True when every README already matched its snippets."""
clean = True
for snippet_name, readmes in README_SECTIONS.items():
snippet = (README_SNIPPETS / snippet_name).read_text().rstrip("\n") + "\n"
heading = snippet.split("\n", 1)[0]
section = re.compile(rf"^{re.escape(heading)}\n.*?(?=^## |\Z)", re.S | re.M)
for rel in readmes:
path = WORKSPACE / rel
if not path.exists():
print(f" {rel:<32} (skipped, not checked out)", file=sys.stderr)
continue
text = path.read_text()
if not section.search(text):
print(f"error: {rel} has no `{heading}` section", file=sys.stderr)
clean = False
continue
# Keep a blank line before the next heading, none at end of file.
updated = section.sub(
lambda m: snippet if m.end() == len(text) else snippet + "\n",
text, count=1)
if updated == text:
continue
clean = False
if check:
print(f"stale: {rel} (`{heading}`); run ./codegen readme",
file=sys.stderr)
else:
path.write_text(updated)
print(f" updated {rel}")
return clean
# ---------------------------------------------------------------------------
# sync: propagate a core change through every binding
# ---------------------------------------------------------------------------
def git(repo: Path, *args: str, capture: bool = False, check: bool = True) -> str:
result = subprocess.run(
["git", "-C", str(repo), *args],
capture_output=capture, text=True,
)
if check and result.returncode != 0:
if capture and result.stderr:
print(result.stderr, file=sys.stderr, end="")
sys.exit(f"error: git {' '.join(args)} failed in {repo}")
return (result.stdout or "").strip() if capture else ""
def dirty(repo: Path) -> bool:
return bool(git(repo, "status", "--porcelain", capture=True))
def current_branch(repo: Path) -> str:
branch = git(repo, "branch", "--show-current", capture=True)
if not branch:
sys.exit(f"error: {repo.relative_to(WORKSPACE)} is on a detached HEAD; "
"check out a branch first")
return branch
def regen_rust_raw_ffi(repo: Path) -> bool:
if shutil.which("bindgen") is None:
print("warning: bindgen not installed, skipped regenerating "
"bindings/rust/cnativeapi/src/bindings.rs", file=sys.stderr)
return False
clang_args = ["-x", "c"]
if sys.platform == "darwin":
sysroot = subprocess.run(
["xcrun", "--show-sdk-path"], capture_output=True, text=True, check=True,
).stdout.strip()
clang_args += ["-isysroot", sysroot]
cmd = [
"bindgen", str(CORE / "include" / "nativeapi.h"),
"--allowlist-function", "native_.*|free_c_str",
"--allowlist-type", "native_.*",
"--allowlist-var", "NATIVE_.*",
"--with-derive-default", "--no-layout-tests", "--no-prepend-enum-name",
"--raw-line", "#![allow(non_upper_case_globals)]",
"--raw-line", "#![allow(non_camel_case_types)]",
"--raw-line", "#![allow(non_snake_case)]",
"-o", str(repo / "cnativeapi" / "src" / "bindings.rs"),
"--", *clang_args,
f"-I{CORE / 'src'}", f"-I{CORE / 'include'}",
]
if subprocess.run(cmd).returncode != 0:
print("warning: bindgen failed; bindings.rs may be stale", file=sys.stderr)
return False
return True
def regen_dart_ffi(repo: Path) -> bool:
"""Run ffigen over bindings/dart/cnativeapi/ffigen.yaml (written by the
bindings generator), refreshing lib/src/bindings_generated.dart."""
package = repo / "cnativeapi"
if shutil.which("dart") is None:
print("warning: dart not installed, skipped ffigen", file=sys.stderr)
return False
if subprocess.run(["dart", "pub", "get"], cwd=package).returncode != 0:
print("warning: dart pub get failed; skipped ffigen", file=sys.stderr)
return False
config = package / "ffigen.yaml"
temporary = None
try:
# LIBCLANG_PATH names libclang or its directory; ffigen wants the LLVM
# root in llvm-path. Keep that machine-specific path out of the
# checked-in configuration by passing it through a temporary copy.
if os.environ.get("LIBCLANG_PATH"):
llvm = Path(os.environ["LIBCLANG_PATH"])
if llvm.is_file():
llvm = llvm.parent
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", dir=package,
encoding="utf-8", delete=False) as out:
out.write(config.read_text(encoding="utf-8"))
out.write("\nllvm-path:\n")
for candidate in (llvm, llvm.parent):
out.write(" - " + json.dumps(str(candidate)) + "\n")
temporary = config = Path(out.name)
result = subprocess.run(["dart", "run", "ffigen", "--config", str(config)],
cwd=package)
finally:
if temporary:
temporary.unlink()
if result.returncode != 0:
print("warning: ffigen failed; bindings_generated.dart may be stale",
file=sys.stderr)
return False
return True
def commit_if_dirty(repo: Path, message: str) -> bool:
if not dirty(repo):
return False
git(repo, "add", "-A")
git(repo, "commit", "-m", message)
return True
def cmd_sync(args, langs: list[str], ir: Path) -> None:
bindings = [BINDINGS[lang][1] for lang in langs]
# Fail early on detached HEADs: sync commits into both repos.
current_branch(CORE)
current_branch(WORKSPACE)
print("==> regenerating C ABI and bindings")
run_capi(ir, check=False)
run_bindings(ir, langs, check=False)
sync_readmes(check=False)
print("==> committing core")
committed = []
if commit_if_dirty(CORE, args.message):
committed.append(CORE)
core_sha = git(CORE, "rev-parse", "HEAD", capture=True)
short = core_sha[:9]
sync_message = f"Sync with core {short}"
for binding in bindings:
print(f"==> syncing {binding.relative_to(WORKSPACE)}")
if binding == BINDINGS["rust"][1]:
regen_rust_raw_ffi(binding)
if binding == BINDINGS["dart"][1]:
regen_dart_ffi(binding)
# The core pointer, everything under the synced bindings, and the API
# reference the website builds from.
print("==> committing workspace")
paths = [str(CORE.relative_to(WORKSPACE))] + [
str(binding.relative_to(WORKSPACE)) for binding in bindings
] + [str(API_REFERENCE.relative_to(WORKSPACE))]
git(WORKSPACE, "add", "--", *paths)
if git(WORKSPACE, "diff", "--cached", "--name-only", capture=True):
git(WORKSPACE, "commit", "-m", sync_message)
committed.append(WORKSPACE)
if not committed:
print("nothing to commit; everything already in sync")
return
names = ", ".join(
"workspace" if repo == WORKSPACE else str(repo.relative_to(WORKSPACE))
for repo in committed
)
print(f"committed: {names}")
if args.push:
# Core first so the workspace's core pointers never dangle remotely.
for repo in committed:
name = "workspace" if repo == WORKSPACE else str(repo.relative_to(WORKSPACE))
print(f"==> pushing {name}")
git(repo, "push")
else:
print("not pushed; rerun with --push, or push manually (core first, "
"then the workspace repo)")
def main() -> None:
cli = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
cli.add_argument("command", nargs="?", default="all",
choices=["all", "capi", "bindings", "check", "ffigen", "readme", "sync"])
cli.add_argument("--lang", default=",".join(BINDINGS),
help="comma-separated subset of: " + ",".join(BINDINGS))
cli.add_argument("--dump-ir", type=Path, metavar="PATH",
help="keep the intermediate IR JSON at this path")
cli.add_argument("-m", "--message", default="Update API",
metavar="MSG", help="sync: commit message for the core repo")
cli.add_argument("--push", action="store_true",
help="sync: also push every repo that received a commit")
args = cli.parse_args()
langs = [lang.strip() for lang in args.lang.split(",") if lang.strip()]
unknown = [lang for lang in langs if lang not in BINDINGS]
if unknown:
cli.error(f"unknown --lang value(s): {', '.join(unknown)}")
ir = args.dump_ir or DEFAULT_IR
check = args.command == "check"
if args.command in ("all", "check"):
readmes_clean = sync_readmes(check)
run_capi(ir, check)
run_bindings(ir, langs, check)
if check and not readmes_clean:
sys.exit(1)
elif args.command == "readme":
sync_readmes(check=False)
elif args.command == "capi":
run_capi(ir, check=False)
elif args.command == "bindings":
run_capi(ir, check=False, parse_only=True)
run_bindings(ir, langs, check=False)
elif args.command == "ffigen":
run_capi(ir, check=False, parse_only=True)
run_bindings(ir, ["dart"], check=False)
if not regen_dart_ffi(BINDINGS["dart"][1]):
sys.exit(1)
elif args.command == "sync":
cmd_sync(args, langs, ir)
if __name__ == "__main__":
main()