#!/usr/bin/env python3 """Generate atom_names.rs and opcodes.rs from the vendored quickjs headers. Reads two files from third_party/quickjs and writes the two Rust tables the parser needs: atom_names.rs the predefined atom strings opcodes.rs the `OpCode` enum and its per-opcode information table Nothing outside the standard library is imported. Usage ----- ./c_parse.py write to ./c_parse ./c_parse.py --out DIR write somewhere else ./c_parse.py --check write nothing; compare and exit 1 on drift """ from __future__ import annotations import argparse import subprocess import sys from pathlib import Path from typing import NamedTuple # --------------------------------------------------------------------------- # C source reading # --------------------------------------------------------------------------- def read_source(path: Path) -> str: """Read a C file and remove its comments. Decoded as latin-1 so that one byte is one character, which reproduces a byte-indexed walk of the source exactly. """ return strip_comments(path.read_bytes().decode("latin-1")) def strip_comments(src: str) -> str: """Remove /* */ and // comments, leaving string and character literals. Each block comment becomes one space so that tokens stay separated. A line comment leaves its newline behind, so line structure survives. """ out: list[str] = [] i, n = 0, len(src) while i < n: c = src[i] if c == "/" and i + 1 < n and src[i + 1] == "*": i += 2 while i + 1 < n and not (src[i] == "*" and src[i + 1] == "/"): i += 1 i = min(i + 2, n) out.append(" ") elif c == "/" and i + 1 < n and src[i + 1] == "/": while i < n and src[i] != "\n": i += 1 elif c in ('"', "'"): quote = c out.append(quote) i += 1 while i < n and src[i] != quote: if src[i] == "\\" and i + 1 < n: out.append(src[i]) i += 1 out.append(src[i]) i += 1 if i < n: out.append(quote) i += 1 else: out.append(c) i += 1 return "".join(out) def paren_body(line: str, start: int = 0) -> str | None: """The text between the parentheses that follow `start`.""" open_i = line.find("(", start) if open_i < 0: return None depth = 0 for i in range(open_i, len(line)): if line[i] == "(": depth += 1 elif line[i] == ")": depth -= 1 if depth == 0: return line[open_i + 1 : i] return None def split_args(body: str) -> list[str]: out: list[str] = [] depth = 0 start = 0 for i, c in enumerate(body): if c in "([": depth += 1 elif c in ")]": depth = max(depth - 1, 0) elif c == "," and depth == 0: out.append(body[start:i].strip()) start = i + 1 out.append(body[start:].strip()) return out def parse_u8(s: str) -> int | None: """Rust's `str::parse::()`: an optional `+`, then digits, 0..=255.""" t = s[1:] if s.startswith("+") else s if not t or not t.isdigit() or not t.isascii(): return None v = int(t) return v if 0 <= v <= 255 else None class OpDef(NamedTuple): name: str size: int n_pop: int n_push: int fmt: str temporary: bool # written as lowercase `def()`: a compiler-only opcode # Opcodes that this compiler adds. QuickJS never emits them, so they come after # every real opcode and `short_opcode_info` never gives them for a byte of # bytecode. Only the transforms emit them. EXTRA_OPS = [ # [obj, value] -> [obj]: like `define_field`, but the field is the location # that `value` was read from, not a copy of it (see transform/closure.rs) OpDef("define_field_ref", 5, 2, 1, "atom", False), # [] -> []: like `close_loc`, for a capture: the binding that capture `i` shares gets a # new location, with the current value (see transform/async.rs) OpDef("close_var_ref", 3, 0, 0, "var_ref", False), # [obj] -> []: field `x` of `obj`, which shares a location, gets a new location, with # the current value. What `close_var_ref` lowers to (see transform/closure.rs) OpDef("close_field", 5, 1, 0, "atom", False), ] def parse_fmt_list(text: str) -> list[str]: """The `FMT(name)` list, in order.""" out = [] for line in text.split("\n"): t = line.lstrip() if t.startswith("#") or not t.startswith("FMT("): continue body = paren_body(t, 0) if body is not None: out.append(body.strip()) return out def parse_opcodes(text: str) -> list[OpDef]: """Every `DEF(...)` and `def(...)` from quickjs-opcode.h, in file order.""" out = [] for line in text.split("\n"): t = line.lstrip() if t.startswith("#"): continue if t.startswith("DEF("): temporary = False elif t.startswith("def("): temporary = True else: continue body = paren_body(t, 0) if body is None: continue args = split_args(body) if len(args) != 5: continue size, n_pop, n_push = (parse_u8(args[1]), parse_u8(args[2]), parse_u8(args[3])) if size is None or n_pop is None or n_push is None: continue out.append(OpDef(args[0], size, n_pop, n_push, args[4], temporary)) return out def parse_atoms(text: str) -> list[str]: """The string of every `DEF(name, "text")` entry in quickjs-atom.h.""" out = [] for line in text.split("\n"): t = line.lstrip() if t.startswith("#") or not t.startswith("DEF("): continue body = paren_body(t, 0) if body is None: continue # Split at the first comma only: the string itself may contain one. comma = body.find(",") if comma < 0: continue s = decode_c_string(body[comma + 1 :].strip()) if s is not None: out.append(s) return out def decode_c_string(lit: str) -> str | None: """Decode a C string literal, including the escapes quickjs-atom.h uses.""" if len(lit) < 2 or not lit.startswith('"') or not lit.endswith('"'): return None inner = lit[1:-1] out: list[str] = [] i, n = 0, len(inner) while i < n: c = inner[i] i += 1 if c != "\\": out.append(c) continue if i >= n: return None esc = inner[i] i += 1 if esc == "n": out.append("\n") elif esc == "t": out.append("\t") elif esc == "r": out.append("\r") elif esc == "0": out.append("\0") elif esc == "x": if i + 1 >= n: return None try: out.append(chr(int(inner[i : i + 2], 16))) except ValueError: return None i += 2 else: # Covers \\, \" and \', and passes anything else through. out.append(esc) return "".join(out) def rust_string_literal(s: str) -> str: """Escape a decoded string back into a Rust literal.""" out = ['"'] for c in s: if c == '"': out.append('\\"') elif c == "\\": out.append("\\\\") elif c == "\n": out.append("\\n") elif c == "\r": out.append("\\r") elif c == "\t": out.append("\\t") elif c == "\0": out.append("\\0") elif ord(c) < 0x20: out.append("\\u{%x}" % ord(c)) else: out.append(c) out.append('"') return "".join(out) # --------------------------------------------------------------------------- # emitters # --------------------------------------------------------------------------- def header(tag: str, frm: str) -> str: return ( "// @generated by c_parse.py. Do not edit by hand.\n" f"// Source: third_party/quickjs ({tag}) {frm}\n\n" ) def camel(s: str) -> str: """`none_int` -> `NoneInt`, `push_i32` -> `PushI32`.""" out = [] for part in s.split("_"): if part: out.append(part[0].upper() + part[1:]) return "".join(out) def gen_opcodes(text: str, tag: str) -> str: fmts = parse_fmt_list(text) if not fmts: raise Fail("no FMT() entries found in quickjs-opcode.h") ops = parse_opcodes(text) if not ops: raise Fail("no DEF() entries found in quickjs-opcode.h") # `OPCodeEnum` numbers only the uppercase `DEF` entries, in file order, so # a byte read from serialized bytecode is one of these and equals its index # here. The lowercase `def()` temporary opcodes are compiler-only and never # reach a file, so dropping them lets the byte index the table directly — # no short-opcode shift, unlike the C `short_opcode_info` macro whose table # still holds them. real = [o for o in ops if not o.temporary] real_count = len(real) real += EXTRA_OPS s = [header(tag, "quickjs-opcode.h")] s.append("/// Operand encoding of an opcode, from the `FMT()` list.\n") s.append("#[derive(Clone, Copy, PartialEq, Eq, Debug)]\npub enum OpFmt {\n") for f in fmts: s.append(f" {camel(f)},\n") s.append("}\n\n") s.append( "/// Every opcode that can appear in serialized bytecode, numbered as\n" "/// `OPCodeEnum`. Compiler-only temporary opcodes are excluded, so a\n" "/// byte read from bytecode is always one of these.\n" "#[derive(Clone, Copy, PartialEq, Eq, Debug)]\n" "#[repr(u8)]\n" "pub enum OpCode {\n" ) for i, o in enumerate(real): s.append(f" {camel(o.name)} = {i},\n") s.append("}\n\n") s.append( "impl OpCode {\n" " /// The C identifier, as it appears in `quickjs-opcode.h`.\n" " pub fn name(self) -> &'static str {\n" " match self {\n" ) for o in real: s.append(f' OpCode::{camel(o.name)} => "{o.name}",\n') s.append( " }\n" " }\n\n" " /// This opcode's row in `OPCODE_INFO`.\n" " pub fn info(self) -> &'static OpInfo {\n" " &OPCODE_INFO[self as usize]\n" " }\n" "}\n\n" ) s.append( "/// One row of the engine's `opcode_info[]` table.\n" "#[derive(Clone, Copy, Debug)]\n" "pub struct OpInfo {\n" " pub op: OpCode,\n" " pub size: u8,\n" " pub n_pop: u8,\n" " pub n_push: u8,\n" " pub fmt: OpFmt,\n" "}\n\n" ) s.append( f"/// Opcode descriptions indexed by opcode byte, all {len(real)} of them.\n" f"pub static OPCODE_INFO: [OpInfo; {len(real)}] = [\n" ) for o in real: s.append( f" OpInfo {{ op: OpCode::{camel(o.name)}, size: {o.size}, " f"n_pop: {o.n_pop}, n_push: {o.n_push}, fmt: OpFmt::{camel(o.fmt)} }},\n" ) s.append("];\n\n") s.append( "/// Look up the description of an opcode read from serialized bytecode.\n" "///\n" "/// The table already excludes the compiler's temporary opcodes, so the\n" "/// byte indexes it directly; there is no short-opcode shift to apply.\n" "/// The opcodes that this compiler adds are not in bytecode, so they\n" "/// are never given.\n" "pub fn short_opcode_info(op: u8) -> Option<&'static OpInfo> {\n" f" OPCODE_INFO[..{real_count}].get(op as usize)\n" "}\n" ) return "".join(s) def gen_atoms(text: str, tag: str) -> str: atoms = parse_atoms(text) if not atoms: raise Fail("no DEF() entries found in quickjs-atom.h") s = [header(tag, "quickjs-atom.h")] s.append( "// Index 0 is `JS_ATOM_NULL`, which the enum defines as `__JS_ATOM_NULL`\n" "// before the header's own entries. The `DEF()` entries therefore start\n" "// at index 1, and `JS_ATOM_END` is the length of this table.\n" ) s.append(f"pub static PREDEFINED_ATOMS: [&str; {len(atoms) + 1}] = [\n") s.append(' "",\n') for a in atoms: s.append(f" {rust_string_literal(a)},\n") s.append("];\n\n") s.append( "/// `JS_ATOM_END`: the first atom index that is not predefined.\n" f"pub const JS_ATOM_END: u32 = {len(atoms) + 1};\n" ) return "".join(s) # --------------------------------------------------------------------------- # driver # --------------------------------------------------------------------------- class Fail(Exception): pass def git_describe(src: Path) -> str: try: r = subprocess.run( ["git", "describe", "--tags", "--always"], cwd=src, capture_output=True, text=True, ) except OSError: return "unknown" if r.returncode != 0: return "unknown" return r.stdout.strip() or "unknown" def build(root: Path) -> dict[str, str]: """Produce every generated file as {filename: contents}.""" src = root / "third_party/quickjs" if not (src / "quickjs-opcode.h").exists(): raise Fail( f"vendored source not found at {src}. Clone quickjs v0.16.1 there first." ) opcode_h = read_source(src / "quickjs-opcode.h") atom_h = read_source(src / "quickjs-atom.h") tag = git_describe(src) return { "opcodes.rs": gen_opcodes(opcode_h, tag), "atom_names.rs": gen_atoms(atom_h, tag), } def main() -> int: root = Path(__file__).resolve().parent ap = argparse.ArgumentParser( description="Generate atom_names.rs and opcodes.rs from the vendored quickjs." ) ap.add_argument( "--out", type=Path, default=root / "c_parse", metavar="DIR", help="output directory (default: c_parse)", ) ap.add_argument( "--check", action="store_true", help="write nothing; compare against --out and exit 1 on any difference", ) args = ap.parse_args() try: files = build(root) except Fail as e: print(f"c_parse.py: {e}", file=sys.stderr) return 1 if args.check: drift = [] for name, body in sorted(files.items()): path = args.out / name if not path.exists(): drift.append(f"{name}: missing") elif path.read_bytes() != body.encode("utf-8"): drift.append(f"{name}: differs") if drift: print("c_parse.py --check found drift:", file=sys.stderr) for d in drift: print(f" {d}", file=sys.stderr) return 1 print(f"c_parse.py: {len(files)} files match {args.out}") return 0 args.out.mkdir(parents=True, exist_ok=True) for name, body in sorted(files.items()): (args.out / name).write_text(body, encoding="utf-8") print(f" {args.out / name} ({len(body)} bytes)") print(f"generated {len(files)} files from third_party/quickjs") return 0 if __name__ == "__main__": sys.exit(main())