496 lines
15 KiB
Python
496 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate atom_names.rs and opcodes.rs from the vendored quickjs headers.
|
|
|
|
Reads two files from third_party/quickjs and writes the two Rust tables the
|
|
parser needs:
|
|
|
|
atom_names.rs the predefined atom strings
|
|
opcodes.rs the `OpCode` enum and its per-opcode information table
|
|
|
|
Nothing outside the standard library is imported.
|
|
|
|
Usage
|
|
-----
|
|
./c_parse.py write to ./c_parse
|
|
./c_parse.py --out DIR write somewhere else
|
|
./c_parse.py --check write nothing; compare and exit 1 on drift
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import NamedTuple
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# C source reading
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def read_source(path: Path) -> str:
|
|
"""Read a C file and remove its comments.
|
|
|
|
Decoded as latin-1 so that one byte is one character, which reproduces a
|
|
byte-indexed walk of the source exactly.
|
|
"""
|
|
return strip_comments(path.read_bytes().decode("latin-1"))
|
|
|
|
|
|
def strip_comments(src: str) -> str:
|
|
"""Remove /* */ and // comments, leaving string and character literals.
|
|
|
|
Each block comment becomes one space so that tokens stay separated. A line
|
|
comment leaves its newline behind, so line structure survives.
|
|
"""
|
|
out: list[str] = []
|
|
i, n = 0, len(src)
|
|
while i < n:
|
|
c = src[i]
|
|
if c == "/" and i + 1 < n and src[i + 1] == "*":
|
|
i += 2
|
|
while i + 1 < n and not (src[i] == "*" and src[i + 1] == "/"):
|
|
i += 1
|
|
i = min(i + 2, n)
|
|
out.append(" ")
|
|
elif c == "/" and i + 1 < n and src[i + 1] == "/":
|
|
while i < n and src[i] != "\n":
|
|
i += 1
|
|
elif c in ('"', "'"):
|
|
quote = c
|
|
out.append(quote)
|
|
i += 1
|
|
while i < n and src[i] != quote:
|
|
if src[i] == "\\" and i + 1 < n:
|
|
out.append(src[i])
|
|
i += 1
|
|
out.append(src[i])
|
|
i += 1
|
|
if i < n:
|
|
out.append(quote)
|
|
i += 1
|
|
else:
|
|
out.append(c)
|
|
i += 1
|
|
return "".join(out)
|
|
|
|
|
|
def paren_body(line: str, start: int = 0) -> str | None:
|
|
"""The text between the parentheses that follow `start`."""
|
|
open_i = line.find("(", start)
|
|
if open_i < 0:
|
|
return None
|
|
depth = 0
|
|
for i in range(open_i, len(line)):
|
|
if line[i] == "(":
|
|
depth += 1
|
|
elif line[i] == ")":
|
|
depth -= 1
|
|
if depth == 0:
|
|
return line[open_i + 1 : i]
|
|
return None
|
|
|
|
|
|
def split_args(body: str) -> list[str]:
|
|
out: list[str] = []
|
|
depth = 0
|
|
start = 0
|
|
for i, c in enumerate(body):
|
|
if c in "([":
|
|
depth += 1
|
|
elif c in ")]":
|
|
depth = max(depth - 1, 0)
|
|
elif c == "," and depth == 0:
|
|
out.append(body[start:i].strip())
|
|
start = i + 1
|
|
out.append(body[start:].strip())
|
|
return out
|
|
|
|
|
|
def parse_u8(s: str) -> int | None:
|
|
"""Rust's `str::parse::<u8>()`: an optional `+`, then digits, 0..=255."""
|
|
t = s[1:] if s.startswith("+") else s
|
|
if not t or not t.isdigit() or not t.isascii():
|
|
return None
|
|
v = int(t)
|
|
return v if 0 <= v <= 255 else None
|
|
|
|
|
|
class OpDef(NamedTuple):
|
|
name: str
|
|
size: int
|
|
n_pop: int
|
|
n_push: int
|
|
fmt: str
|
|
temporary: bool # written as lowercase `def()`: a compiler-only opcode
|
|
|
|
|
|
# Opcodes that this compiler adds. QuickJS never emits them, so they come after
|
|
# every real opcode and `short_opcode_info` never gives them for a byte of
|
|
# bytecode. Only the transforms emit them.
|
|
EXTRA_OPS = [
|
|
# [obj, value] -> [obj]: like `define_field`, but the field is the location
|
|
# that `value` was read from, not a copy of it (see transform/closure.rs)
|
|
OpDef("define_field_ref", 5, 2, 1, "atom", False),
|
|
# [] -> []: like `close_loc`, for a capture: the binding that capture `i` shares gets a
|
|
# new location, with the current value (see transform/async.rs)
|
|
OpDef("close_var_ref", 3, 0, 0, "var_ref", False),
|
|
# [obj] -> []: field `x` of `obj`, which shares a location, gets a new location, with
|
|
# the current value. What `close_var_ref` lowers to (see transform/closure.rs)
|
|
OpDef("close_field", 5, 1, 0, "atom", False),
|
|
]
|
|
|
|
|
|
def parse_fmt_list(text: str) -> list[str]:
|
|
"""The `FMT(name)` list, in order."""
|
|
out = []
|
|
for line in text.split("\n"):
|
|
t = line.lstrip()
|
|
if t.startswith("#") or not t.startswith("FMT("):
|
|
continue
|
|
body = paren_body(t, 0)
|
|
if body is not None:
|
|
out.append(body.strip())
|
|
return out
|
|
|
|
|
|
def parse_opcodes(text: str) -> list[OpDef]:
|
|
"""Every `DEF(...)` and `def(...)` from quickjs-opcode.h, in file order."""
|
|
out = []
|
|
for line in text.split("\n"):
|
|
t = line.lstrip()
|
|
if t.startswith("#"):
|
|
continue
|
|
if t.startswith("DEF("):
|
|
temporary = False
|
|
elif t.startswith("def("):
|
|
temporary = True
|
|
else:
|
|
continue
|
|
body = paren_body(t, 0)
|
|
if body is None:
|
|
continue
|
|
args = split_args(body)
|
|
if len(args) != 5:
|
|
continue
|
|
size, n_pop, n_push = (parse_u8(args[1]), parse_u8(args[2]), parse_u8(args[3]))
|
|
if size is None or n_pop is None or n_push is None:
|
|
continue
|
|
out.append(OpDef(args[0], size, n_pop, n_push, args[4], temporary))
|
|
return out
|
|
|
|
|
|
def parse_atoms(text: str) -> list[str]:
|
|
"""The string of every `DEF(name, "text")` entry in quickjs-atom.h."""
|
|
out = []
|
|
for line in text.split("\n"):
|
|
t = line.lstrip()
|
|
if t.startswith("#") or not t.startswith("DEF("):
|
|
continue
|
|
body = paren_body(t, 0)
|
|
if body is None:
|
|
continue
|
|
# Split at the first comma only: the string itself may contain one.
|
|
comma = body.find(",")
|
|
if comma < 0:
|
|
continue
|
|
s = decode_c_string(body[comma + 1 :].strip())
|
|
if s is not None:
|
|
out.append(s)
|
|
return out
|
|
|
|
|
|
def decode_c_string(lit: str) -> str | None:
|
|
"""Decode a C string literal, including the escapes quickjs-atom.h uses."""
|
|
if len(lit) < 2 or not lit.startswith('"') or not lit.endswith('"'):
|
|
return None
|
|
inner = lit[1:-1]
|
|
out: list[str] = []
|
|
i, n = 0, len(inner)
|
|
while i < n:
|
|
c = inner[i]
|
|
i += 1
|
|
if c != "\\":
|
|
out.append(c)
|
|
continue
|
|
if i >= n:
|
|
return None
|
|
esc = inner[i]
|
|
i += 1
|
|
if esc == "n":
|
|
out.append("\n")
|
|
elif esc == "t":
|
|
out.append("\t")
|
|
elif esc == "r":
|
|
out.append("\r")
|
|
elif esc == "0":
|
|
out.append("\0")
|
|
elif esc == "x":
|
|
if i + 1 >= n:
|
|
return None
|
|
try:
|
|
out.append(chr(int(inner[i : i + 2], 16)))
|
|
except ValueError:
|
|
return None
|
|
i += 2
|
|
else:
|
|
# Covers \\, \" and \', and passes anything else through.
|
|
out.append(esc)
|
|
return "".join(out)
|
|
|
|
|
|
def rust_string_literal(s: str) -> str:
|
|
"""Escape a decoded string back into a Rust literal."""
|
|
out = ['"']
|
|
for c in s:
|
|
if c == '"':
|
|
out.append('\\"')
|
|
elif c == "\\":
|
|
out.append("\\\\")
|
|
elif c == "\n":
|
|
out.append("\\n")
|
|
elif c == "\r":
|
|
out.append("\\r")
|
|
elif c == "\t":
|
|
out.append("\\t")
|
|
elif c == "\0":
|
|
out.append("\\0")
|
|
elif ord(c) < 0x20:
|
|
out.append("\\u{%x}" % ord(c))
|
|
else:
|
|
out.append(c)
|
|
out.append('"')
|
|
return "".join(out)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# emitters
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def header(tag: str, frm: str) -> str:
|
|
return (
|
|
"// @generated by c_parse.py. Do not edit by hand.\n"
|
|
f"// Source: third_party/quickjs ({tag}) {frm}\n\n"
|
|
)
|
|
|
|
|
|
def camel(s: str) -> str:
|
|
"""`none_int` -> `NoneInt`, `push_i32` -> `PushI32`."""
|
|
out = []
|
|
for part in s.split("_"):
|
|
if part:
|
|
out.append(part[0].upper() + part[1:])
|
|
return "".join(out)
|
|
|
|
|
|
def gen_opcodes(text: str, tag: str) -> str:
|
|
fmts = parse_fmt_list(text)
|
|
if not fmts:
|
|
raise Fail("no FMT() entries found in quickjs-opcode.h")
|
|
ops = parse_opcodes(text)
|
|
if not ops:
|
|
raise Fail("no DEF() entries found in quickjs-opcode.h")
|
|
|
|
# `OPCodeEnum` numbers only the uppercase `DEF` entries, in file order, so
|
|
# a byte read from serialized bytecode is one of these and equals its index
|
|
# here. The lowercase `def()` temporary opcodes are compiler-only and never
|
|
# reach a file, so dropping them lets the byte index the table directly —
|
|
# no short-opcode shift, unlike the C `short_opcode_info` macro whose table
|
|
# still holds them.
|
|
real = [o for o in ops if not o.temporary]
|
|
real_count = len(real)
|
|
real += EXTRA_OPS
|
|
|
|
s = [header(tag, "quickjs-opcode.h")]
|
|
|
|
s.append("/// Operand encoding of an opcode, from the `FMT()` list.\n")
|
|
s.append("#[derive(Clone, Copy, PartialEq, Eq, Debug)]\npub enum OpFmt {\n")
|
|
for f in fmts:
|
|
s.append(f" {camel(f)},\n")
|
|
s.append("}\n\n")
|
|
|
|
s.append(
|
|
"/// Every opcode that can appear in serialized bytecode, numbered as\n"
|
|
"/// `OPCodeEnum`. Compiler-only temporary opcodes are excluded, so a\n"
|
|
"/// byte read from bytecode is always one of these.\n"
|
|
"#[derive(Clone, Copy, PartialEq, Eq, Debug)]\n"
|
|
"#[repr(u8)]\n"
|
|
"pub enum OpCode {\n"
|
|
)
|
|
for i, o in enumerate(real):
|
|
s.append(f" {camel(o.name)} = {i},\n")
|
|
s.append("}\n\n")
|
|
|
|
s.append(
|
|
"impl OpCode {\n"
|
|
" /// The C identifier, as it appears in `quickjs-opcode.h`.\n"
|
|
" pub fn name(self) -> &'static str {\n"
|
|
" match self {\n"
|
|
)
|
|
for o in real:
|
|
s.append(f' OpCode::{camel(o.name)} => "{o.name}",\n')
|
|
s.append(
|
|
" }\n"
|
|
" }\n\n"
|
|
" /// This opcode's row in `OPCODE_INFO`.\n"
|
|
" pub fn info(self) -> &'static OpInfo {\n"
|
|
" &OPCODE_INFO[self as usize]\n"
|
|
" }\n"
|
|
"}\n\n"
|
|
)
|
|
|
|
s.append(
|
|
"/// One row of the engine's `opcode_info[]` table.\n"
|
|
"#[derive(Clone, Copy, Debug)]\n"
|
|
"pub struct OpInfo {\n"
|
|
" pub op: OpCode,\n"
|
|
" pub size: u8,\n"
|
|
" pub n_pop: u8,\n"
|
|
" pub n_push: u8,\n"
|
|
" pub fmt: OpFmt,\n"
|
|
"}\n\n"
|
|
)
|
|
|
|
s.append(
|
|
f"/// Opcode descriptions indexed by opcode byte, all {len(real)} of them.\n"
|
|
f"pub static OPCODE_INFO: [OpInfo; {len(real)}] = [\n"
|
|
)
|
|
for o in real:
|
|
s.append(
|
|
f" OpInfo {{ op: OpCode::{camel(o.name)}, size: {o.size}, "
|
|
f"n_pop: {o.n_pop}, n_push: {o.n_push}, fmt: OpFmt::{camel(o.fmt)} }},\n"
|
|
)
|
|
s.append("];\n\n")
|
|
|
|
s.append(
|
|
"/// Look up the description of an opcode read from serialized bytecode.\n"
|
|
"///\n"
|
|
"/// The table already excludes the compiler's temporary opcodes, so the\n"
|
|
"/// byte indexes it directly; there is no short-opcode shift to apply.\n"
|
|
"/// The opcodes that this compiler adds are not in bytecode, so they\n"
|
|
"/// are never given.\n"
|
|
"pub fn short_opcode_info(op: u8) -> Option<&'static OpInfo> {\n"
|
|
f" OPCODE_INFO[..{real_count}].get(op as usize)\n"
|
|
"}\n"
|
|
)
|
|
return "".join(s)
|
|
|
|
|
|
def gen_atoms(text: str, tag: str) -> str:
|
|
atoms = parse_atoms(text)
|
|
if not atoms:
|
|
raise Fail("no DEF() entries found in quickjs-atom.h")
|
|
s = [header(tag, "quickjs-atom.h")]
|
|
s.append(
|
|
"// Index 0 is `JS_ATOM_NULL`, which the enum defines as `__JS_ATOM_NULL`\n"
|
|
"// before the header's own entries. The `DEF()` entries therefore start\n"
|
|
"// at index 1, and `JS_ATOM_END` is the length of this table.\n"
|
|
)
|
|
s.append(f"pub static PREDEFINED_ATOMS: [&str; {len(atoms) + 1}] = [\n")
|
|
s.append(' "<null>",\n')
|
|
for a in atoms:
|
|
s.append(f" {rust_string_literal(a)},\n")
|
|
s.append("];\n\n")
|
|
s.append(
|
|
"/// `JS_ATOM_END`: the first atom index that is not predefined.\n"
|
|
f"pub const JS_ATOM_END: u32 = {len(atoms) + 1};\n"
|
|
)
|
|
return "".join(s)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# driver
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class Fail(Exception):
|
|
pass
|
|
|
|
|
|
def git_describe(src: Path) -> str:
|
|
try:
|
|
r = subprocess.run(
|
|
["git", "describe", "--tags", "--always"],
|
|
cwd=src,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
except OSError:
|
|
return "unknown"
|
|
if r.returncode != 0:
|
|
return "unknown"
|
|
return r.stdout.strip() or "unknown"
|
|
|
|
|
|
def build(root: Path) -> dict[str, str]:
|
|
"""Produce every generated file as {filename: contents}."""
|
|
src = root / "third_party/quickjs"
|
|
if not (src / "quickjs-opcode.h").exists():
|
|
raise Fail(
|
|
f"vendored source not found at {src}. Clone quickjs v0.16.1 there first."
|
|
)
|
|
|
|
opcode_h = read_source(src / "quickjs-opcode.h")
|
|
atom_h = read_source(src / "quickjs-atom.h")
|
|
tag = git_describe(src)
|
|
|
|
return {
|
|
"opcodes.rs": gen_opcodes(opcode_h, tag),
|
|
"atom_names.rs": gen_atoms(atom_h, tag),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
root = Path(__file__).resolve().parent
|
|
ap = argparse.ArgumentParser(
|
|
description="Generate atom_names.rs and opcodes.rs from the vendored quickjs."
|
|
)
|
|
ap.add_argument(
|
|
"--out",
|
|
type=Path,
|
|
default=root / "c_parse",
|
|
metavar="DIR",
|
|
help="output directory (default: c_parse)",
|
|
)
|
|
ap.add_argument(
|
|
"--check",
|
|
action="store_true",
|
|
help="write nothing; compare against --out and exit 1 on any difference",
|
|
)
|
|
args = ap.parse_args()
|
|
|
|
try:
|
|
files = build(root)
|
|
except Fail as e:
|
|
print(f"c_parse.py: {e}", file=sys.stderr)
|
|
return 1
|
|
|
|
if args.check:
|
|
drift = []
|
|
for name, body in sorted(files.items()):
|
|
path = args.out / name
|
|
if not path.exists():
|
|
drift.append(f"{name}: missing")
|
|
elif path.read_bytes() != body.encode("utf-8"):
|
|
drift.append(f"{name}: differs")
|
|
if drift:
|
|
print("c_parse.py --check found drift:", file=sys.stderr)
|
|
for d in drift:
|
|
print(f" {d}", file=sys.stderr)
|
|
return 1
|
|
print(f"c_parse.py: {len(files)} files match {args.out}")
|
|
return 0
|
|
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
for name, body in sorted(files.items()):
|
|
(args.out / name).write_text(body, encoding="utf-8")
|
|
print(f" {args.out / name} ({len(body)} bytes)")
|
|
print(f"generated {len(files)} files from third_party/quickjs")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|