Add files

This commit is contained in:
2026-09-01 20:22:11 +00:00
parent 6dc95c412f
commit a8acb7706c
7 changed files with 1595 additions and 1 deletions
+3
View File
@@ -0,0 +1,3 @@
c_parse
compiler/target
compiler/Cargo.lock
+475
View File
@@ -0,0 +1,475 @@
#!/usr/bin/env python3
"""Generate atom_names.rs and opcodes.rs from the vendored quickjs headers.
Reads two files from third_party/quickjs and writes the two Rust tables the
parser needs:
atom_names.rs the predefined atom strings
opcodes.rs the `OpCode` enum and its per-opcode information table
Nothing outside the standard library is imported.
Usage
-----
./c_parse.py write to ./c_parse
./c_parse.py --out DIR write somewhere else
./c_parse.py --check write nothing; compare and exit 1 on drift
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
from typing import NamedTuple
# ---------------------------------------------------------------------------
# C source reading
# ---------------------------------------------------------------------------
def read_source(path: Path) -> str:
"""Read a C file and remove its comments.
Decoded as latin-1 so that one byte is one character, which reproduces a
byte-indexed walk of the source exactly.
"""
return strip_comments(path.read_bytes().decode("latin-1"))
def strip_comments(src: str) -> str:
"""Remove /* */ and // comments, leaving string and character literals.
Each block comment becomes one space so that tokens stay separated. A line
comment leaves its newline behind, so line structure survives.
"""
out: list[str] = []
i, n = 0, len(src)
while i < n:
c = src[i]
if c == "/" and i + 1 < n and src[i + 1] == "*":
i += 2
while i + 1 < n and not (src[i] == "*" and src[i + 1] == "/"):
i += 1
i = min(i + 2, n)
out.append(" ")
elif c == "/" and i + 1 < n and src[i + 1] == "/":
while i < n and src[i] != "\n":
i += 1
elif c in ('"', "'"):
quote = c
out.append(quote)
i += 1
while i < n and src[i] != quote:
if src[i] == "\\" and i + 1 < n:
out.append(src[i])
i += 1
out.append(src[i])
i += 1
if i < n:
out.append(quote)
i += 1
else:
out.append(c)
i += 1
return "".join(out)
def paren_body(line: str, start: int = 0) -> str | None:
"""The text between the parentheses that follow `start`."""
open_i = line.find("(", start)
if open_i < 0:
return None
depth = 0
for i in range(open_i, len(line)):
if line[i] == "(":
depth += 1
elif line[i] == ")":
depth -= 1
if depth == 0:
return line[open_i + 1 : i]
return None
def split_args(body: str) -> list[str]:
out: list[str] = []
depth = 0
start = 0
for i, c in enumerate(body):
if c in "([":
depth += 1
elif c in ")]":
depth = max(depth - 1, 0)
elif c == "," and depth == 0:
out.append(body[start:i].strip())
start = i + 1
out.append(body[start:].strip())
return out
def parse_u8(s: str) -> int | None:
"""Rust's `str::parse::<u8>()`: an optional `+`, then digits, 0..=255."""
t = s[1:] if s.startswith("+") else s
if not t or not t.isdigit() or not t.isascii():
return None
v = int(t)
return v if 0 <= v <= 255 else None
class OpDef(NamedTuple):
name: str
size: int
n_pop: int
n_push: int
fmt: str
temporary: bool # written as lowercase `def()`: a compiler-only opcode
def parse_fmt_list(text: str) -> list[str]:
"""The `FMT(name)` list, in order."""
out = []
for line in text.split("\n"):
t = line.lstrip()
if t.startswith("#") or not t.startswith("FMT("):
continue
body = paren_body(t, 0)
if body is not None:
out.append(body.strip())
return out
def parse_opcodes(text: str) -> list[OpDef]:
"""Every `DEF(...)` and `def(...)` from quickjs-opcode.h, in file order."""
out = []
for line in text.split("\n"):
t = line.lstrip()
if t.startswith("#"):
continue
if t.startswith("DEF("):
temporary = False
elif t.startswith("def("):
temporary = True
else:
continue
body = paren_body(t, 0)
if body is None:
continue
args = split_args(body)
if len(args) != 5:
continue
size, n_pop, n_push = (parse_u8(args[1]), parse_u8(args[2]), parse_u8(args[3]))
if size is None or n_pop is None or n_push is None:
continue
out.append(OpDef(args[0], size, n_pop, n_push, args[4], temporary))
return out
def parse_atoms(text: str) -> list[str]:
"""The string of every `DEF(name, "text")` entry in quickjs-atom.h."""
out = []
for line in text.split("\n"):
t = line.lstrip()
if t.startswith("#") or not t.startswith("DEF("):
continue
body = paren_body(t, 0)
if body is None:
continue
# Split at the first comma only: the string itself may contain one.
comma = body.find(",")
if comma < 0:
continue
s = decode_c_string(body[comma + 1 :].strip())
if s is not None:
out.append(s)
return out
def decode_c_string(lit: str) -> str | None:
"""Decode a C string literal, including the escapes quickjs-atom.h uses."""
if len(lit) < 2 or not lit.startswith('"') or not lit.endswith('"'):
return None
inner = lit[1:-1]
out: list[str] = []
i, n = 0, len(inner)
while i < n:
c = inner[i]
i += 1
if c != "\\":
out.append(c)
continue
if i >= n:
return None
esc = inner[i]
i += 1
if esc == "n":
out.append("\n")
elif esc == "t":
out.append("\t")
elif esc == "r":
out.append("\r")
elif esc == "0":
out.append("\0")
elif esc == "x":
if i + 1 >= n:
return None
try:
out.append(chr(int(inner[i : i + 2], 16)))
except ValueError:
return None
i += 2
else:
# Covers \\, \" and \', and passes anything else through.
out.append(esc)
return "".join(out)
def rust_string_literal(s: str) -> str:
"""Escape a decoded string back into a Rust literal."""
out = ['"']
for c in s:
if c == '"':
out.append('\\"')
elif c == "\\":
out.append("\\\\")
elif c == "\n":
out.append("\\n")
elif c == "\r":
out.append("\\r")
elif c == "\t":
out.append("\\t")
elif c == "\0":
out.append("\\0")
elif ord(c) < 0x20:
out.append("\\u{%x}" % ord(c))
else:
out.append(c)
out.append('"')
return "".join(out)
# ---------------------------------------------------------------------------
# emitters
# ---------------------------------------------------------------------------
def header(tag: str, frm: str) -> str:
return (
"// @generated by c_parse.py. Do not edit by hand.\n"
f"// Source: third_party/quickjs ({tag}) {frm}\n\n"
)
def camel(s: str) -> str:
"""`none_int` -> `NoneInt`, `push_i32` -> `PushI32`."""
out = []
for part in s.split("_"):
if part:
out.append(part[0].upper() + part[1:])
return "".join(out)
def gen_opcodes(text: str, tag: str) -> str:
fmts = parse_fmt_list(text)
if not fmts:
raise Fail("no FMT() entries found in quickjs-opcode.h")
ops = parse_opcodes(text)
if not ops:
raise Fail("no DEF() entries found in quickjs-opcode.h")
# `OPCodeEnum` numbers only the uppercase `DEF` entries, in file order, so
# a byte read from serialized bytecode is one of these and equals its index
# here. The lowercase `def()` temporary opcodes are compiler-only and never
# reach a file, so dropping them lets the byte index the table directly —
# no short-opcode shift, unlike the C `short_opcode_info` macro whose table
# still holds them.
real = [o for o in ops if not o.temporary]
s = [header(tag, "quickjs-opcode.h")]
s.append("/// Operand encoding of an opcode, from the `FMT()` list.\n")
s.append("#[derive(Clone, Copy, PartialEq, Eq, Debug)]\npub enum OpFmt {\n")
for f in fmts:
s.append(f" {camel(f)},\n")
s.append("}\n\n")
s.append(
"/// Every opcode that can appear in serialized bytecode, numbered as\n"
"/// `OPCodeEnum`. Compiler-only temporary opcodes are excluded, so a\n"
"/// byte read from bytecode is always one of these.\n"
"#[derive(Clone, Copy, PartialEq, Eq, Debug)]\n"
"#[repr(u8)]\n"
"pub enum OpCode {\n"
)
for i, o in enumerate(real):
s.append(f" {camel(o.name)} = {i},\n")
s.append("}\n\n")
s.append(
"impl OpCode {\n"
" /// The C identifier, as it appears in `quickjs-opcode.h`.\n"
" pub fn name(self) -> &'static str {\n"
" match self {\n"
)
for o in real:
s.append(f' OpCode::{camel(o.name)} => "{o.name}",\n')
s.append(
" }\n"
" }\n\n"
" /// This opcode's row in `OPCODE_INFO`.\n"
" pub fn info(self) -> &'static OpInfo {\n"
" &OPCODE_INFO[self as usize]\n"
" }\n"
"}\n\n"
)
s.append(
"/// One row of the engine's `opcode_info[]` table.\n"
"#[derive(Clone, Copy, Debug)]\n"
"pub struct OpInfo {\n"
" pub op: OpCode,\n"
" pub size: u8,\n"
" pub n_pop: u8,\n"
" pub n_push: u8,\n"
" pub fmt: OpFmt,\n"
"}\n\n"
)
s.append(
f"/// Opcode descriptions indexed by opcode byte, all {len(real)} of them.\n"
f"pub static OPCODE_INFO: [OpInfo; {len(real)}] = [\n"
)
for o in real:
s.append(
f" OpInfo {{ op: OpCode::{camel(o.name)}, size: {o.size}, "
f"n_pop: {o.n_pop}, n_push: {o.n_push}, fmt: OpFmt::{camel(o.fmt)} }},\n"
)
s.append("];\n\n")
s.append(
"/// Look up the description of an opcode read from serialized bytecode.\n"
"///\n"
"/// The table already excludes the compiler's temporary opcodes, so the\n"
"/// byte indexes it directly; there is no short-opcode shift to apply.\n"
"pub fn short_opcode_info(op: u8) -> Option<&'static OpInfo> {\n"
" OPCODE_INFO.get(op as usize)\n"
"}\n"
)
return "".join(s)
def gen_atoms(text: str, tag: str) -> str:
atoms = parse_atoms(text)
if not atoms:
raise Fail("no DEF() entries found in quickjs-atom.h")
s = [header(tag, "quickjs-atom.h")]
s.append(
"// Index 0 is `JS_ATOM_NULL`, which the enum defines as `__JS_ATOM_NULL`\n"
"// before the header's own entries. The `DEF()` entries therefore start\n"
"// at index 1, and `JS_ATOM_END` is the length of this table.\n"
)
s.append(f"pub static PREDEFINED_ATOMS: [&str; {len(atoms) + 1}] = [\n")
s.append(' "<null>",\n')
for a in atoms:
s.append(f" {rust_string_literal(a)},\n")
s.append("];\n\n")
s.append(
"/// `JS_ATOM_END`: the first atom index that is not predefined.\n"
f"pub const JS_ATOM_END: u32 = {len(atoms) + 1};\n"
)
return "".join(s)
# ---------------------------------------------------------------------------
# driver
# ---------------------------------------------------------------------------
class Fail(Exception):
pass
def git_describe(src: Path) -> str:
try:
r = subprocess.run(
["git", "describe", "--tags", "--always"],
cwd=src,
capture_output=True,
text=True,
)
except OSError:
return "unknown"
if r.returncode != 0:
return "unknown"
return r.stdout.strip() or "unknown"
def build(root: Path) -> dict[str, str]:
"""Produce every generated file as {filename: contents}."""
src = root / "third_party/quickjs"
if not (src / "quickjs-opcode.h").exists():
raise Fail(
f"vendored source not found at {src}. Clone quickjs v0.16.1 there first."
)
opcode_h = read_source(src / "quickjs-opcode.h")
atom_h = read_source(src / "quickjs-atom.h")
tag = git_describe(src)
return {
"opcodes.rs": gen_opcodes(opcode_h, tag),
"atom_names.rs": gen_atoms(atom_h, tag),
}
def main() -> int:
root = Path(__file__).resolve().parent
ap = argparse.ArgumentParser(
description="Generate atom_names.rs and opcodes.rs from the vendored quickjs."
)
ap.add_argument(
"--out",
type=Path,
default=root / "c_parse",
metavar="DIR",
help="output directory (default: c_parse)",
)
ap.add_argument(
"--check",
action="store_true",
help="write nothing; compare against --out and exit 1 on any difference",
)
args = ap.parse_args()
try:
files = build(root)
except Fail as e:
print(f"c_parse.py: {e}", file=sys.stderr)
return 1
if args.check:
drift = []
for name, body in sorted(files.items()):
path = args.out / name
if not path.exists():
drift.append(f"{name}: missing")
elif path.read_bytes() != body.encode("utf-8"):
drift.append(f"{name}: differs")
if drift:
print("c_parse.py --check found drift:", file=sys.stderr)
for d in drift:
print(f" {d}", file=sys.stderr)
return 1
print(f"c_parse.py: {len(files)} files match {args.out}")
return 0
args.out.mkdir(parents=True, exist_ok=True)
for name, body in sorted(files.items()):
(args.out / name).write_text(body, encoding="utf-8")
print(f" {args.out / name} ({len(body)} bytes)")
print(f"generated {len(files)} files from third_party/quickjs")
return 0
if __name__ == "__main__":
sys.exit(main())
+2
View File
@@ -4,3 +4,5 @@ version = "0.1.0"
edition = "2024" edition = "2024"
[dependencies] [dependencies]
nom = "8.0.0"
num_enum = "0.7.6"
+129
View File
@@ -0,0 +1,129 @@
//! This module takes the nested QuickJS binary structure and separates the functions into a flat indexed namespace
use crate::parse::{
Atom, AtomRef, ClosureKind, Function, Instruction, JsString, JsValue, QuickJSBinary, atom_names,
};
#[derive(Debug, Clone, Copy)]
pub enum CaptureFrom {
/// `parent.var_defs[i]`
ParentArg(usize),
/// `parent.var_defs[parent.arg_count + i]`
ParentLocal(usize),
/// `parent.captures[i]`, itself resolved against the grandparent.
ParentCapture(usize),
/// A module-level binding of the module body.
ModuleBinding(usize),
/// Global or module import; not frame-relative.
NotFrameRelative(&'static str),
}
#[derive(Debug)]
pub struct Capture {
pub name: String,
/// The name as it was on the wire. A field cannot be given a name that is
/// not already an atom, so the operand has to be kept, not rebuilt.
pub atom: AtomRef,
pub from: CaptureFrom,
}
#[derive(Debug)]
pub struct FlatFn {
pub name: String,
pub arg_count: usize,
pub var_count: usize, // num locals
pub captures: Vec<Capture>,
pub cpool: Vec<JsValue>,
pub code: Vec<Instruction>,
}
pub fn flatten(bin: QuickJSBinary) -> (Vec<FlatFn>, Vec<Atom>) {
let mut fns = vec![];
walk(&mut fns, bin.root_mod.root_fn, &bin.atoms);
(fns, bin.atoms)
}
fn walk(flat_fns: &mut Vec<FlatFn>, func: Function, atoms: &[Atom]) -> usize {
let id = flat_fns.len();
let name = atom_str(&func.name, atoms);
// Find vars func captures from its environment
let captures = func
.closure_vars
.iter()
.map(|cv| Capture {
name: atom_str(&cv.name, atoms),
atom: cv.name,
from: match cv.flags.closure_kind {
ClosureKind::Arg => CaptureFrom::ParentArg(cv.var_idx),
ClosureKind::Local => CaptureFrom::ParentLocal(cv.var_idx),
ClosureKind::Ref => CaptureFrom::ParentCapture(cv.var_idx),
ClosureKind::ModuleDecl => CaptureFrom::ModuleBinding(cv.var_idx),
ClosureKind::ModuleImport => todo!(), // CaptureFrom::NotFrameRelative("module_import"),
ClosureKind::Global | ClosureKind::GlobalDecl | ClosureKind::GlobalRef => todo!(), // CaptureFrom::NotFrameRelative("global")
},
})
.collect();
// Reserve this function's slot before recursing into nested functions,
// so it keeps `id` while its children take the ids after it
flat_fns.push(FlatFn {
name,
arg_count: func.arg_count,
var_count: func.var_count,
captures,
cpool: Vec::new(),
code: func.byte_code,
});
// Nested functions live in the cpool; replace each with its flat id.
let mut cpool = Vec::with_capacity(func.cpool.len());
for const_entry in func.cpool {
let entry = match const_entry {
JsValue::Function(child) => JsValue::FnById(walk(flat_fns, child, atoms)),
other => other,
};
cpool.push(entry);
}
flat_fns[id].cpool = cpool;
id
}
// LLM GENERATED
// ─── atom rendering ──────────────────────────────────────────────────────
pub fn atom_str(a: &AtomRef, atoms: &[Atom]) -> String {
match a {
AtomRef::TaggedInt(n) => n.to_string(),
AtomRef::Predefined(i) => atom_names::PREDEFINED_ATOMS
.get(*i as usize)
.copied()
.unwrap_or("<predef?>")
.to_string(),
AtomRef::FileLocal(i) => atoms
.get(*i)
.map(atom_text)
.unwrap_or_else(|| format!("<file_atom {i}>")),
}
}
fn atom_text(a: &Atom) -> String {
match a {
Atom::String(s) | Atom::GlobalSymbol(s) | Atom::Symbol(s) => js_string(s),
Atom::Predefined(n) => format!("<predef {n}>"),
}
}
fn js_string(s: &JsString) -> String {
match s {
JsString::Latin1(b) => b.iter().map(|&c| c as char).collect(),
JsString::UTF16(b) => {
let u: Vec<u16> = b
.chunks_exact(2)
.map(|c| u16::from_le_bytes([c[0], c[1]]))
.collect();
String::from_utf16_lossy(&u)
}
}
}
+28 -1
View File
@@ -1,3 +1,30 @@
mod flatten;
mod parse;
mod ssa;
use std::process::Command;
use flatten::flatten;
use parse::QuickJSBinary;
const QJSC: &str = "../third_party/quickjs/build/qjsc";
const BIN_OUT: &str = "/tmp/qjsrt.bin";
fn main() { fn main() {
println!("Hello, world!"); let args = std::env::args().collect::<Vec<String>>();
let filename = args.get(1).unwrap();
// Compile source file
let output = Command::new(QJSC)
.args(["-b", "-m", "-o", BIN_OUT])
.arg(filename)
.output()
.unwrap();
assert!(output.status.success());
// Parse, then flatten
let bin = std::fs::read(BIN_OUT).unwrap();
let (_, parsed) = QuickJSBinary::parse(&bin).unwrap();
let (flat_fns, atoms) = flatten(parsed);
} }
+898
View File
@@ -0,0 +1,898 @@
//! This module takes the QuickJS produced binary and parses it into an identical structured represention
use std::fmt::Debug;
use std::fmt::Write;
use nom::{
IResult, Parser,
bytes::complete::take,
error::{Error, ErrorKind},
multi::count,
number::complete::{le_u16, le_u32, le_u64, u8},
};
use num_enum::TryFromPrimitive;
pub mod atom_names {
include!("../../c_parse/atom_names.rs");
}
pub mod opcodes {
include!("../../c_parse/opcodes.rs");
}
#[derive(Debug)]
pub struct QuickJSBinary {
pub version: u8,
pub checksum: u32,
pub atoms: Vec<Atom>,
pub root_mod: Module,
}
impl QuickJSBinary {
pub fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, version) = u8(bin)?;
let (bin, checksum) = le_u32(bin)?;
let (bin, num_atoms) = leb128(bin)?;
let (bin, atoms) = count(Atom::parse, num_atoms).parse(bin)?;
let (bin, value) = JsValue::parse(bin)?;
let JsValue::Module(root_mod) = value else {
panic!()
};
Ok((
bin,
QuickJSBinary {
version,
checksum,
atoms,
root_mod,
},
))
}
}
#[derive(Debug)]
pub enum Atom {
Predefined(u32),
String(JsString),
GlobalSymbol(JsString),
Symbol(JsString),
}
impl Atom {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, kind) = u8(bin)?;
let ret = match kind {
0 => {
let (bin, raw) = le_u32(bin)?;
(bin, Self::Predefined(raw))
}
1 => {
let (bin, header) = leb128(bin)?;
let wide_char = header & 1 == 1;
let len = header >> 1;
let (bin, data) = take(len)(bin)?;
let data = data.to_vec();
let string = if wide_char {
JsString::UTF16(data)
} else {
JsString::Latin1(data)
};
(bin, Self::String(string))
}
2 => todo!(),
3 => todo!(),
_ => unreachable!(),
};
Ok(ret)
}
}
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub enum AtomRef {
TaggedInt(u32),
Predefined(u32),
FileLocal(usize),
}
impl AtomRef {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, v) = leb128(bin)?;
if v & 1 == 1 {
return Ok((bin, Self::TaggedInt((v >> 1) as u32)));
}
let i = v >> 1;
let end = atom_names::JS_ATOM_END as usize;
if i < end {
Ok((bin, Self::Predefined(i as u32)))
} else {
Ok((bin, Self::FileLocal(i - end)))
}
}
// LLM GENERATED
//
// Atom operands *inside bytecode* use a different encoding from the
// `AtomRef::parse` above: a bare u32 with no shift, where a tagged
// integer is marked by bit 31 instead of bit 0. Same index space, two
// encodings
fn from_code_u32(v: u32) -> Self {
if v & (1 << 31) != 0 {
return Self::TaggedInt(v & !(1 << 31));
}
let end = atom_names::JS_ATOM_END;
if v < end {
Self::Predefined(v)
} else {
Self::FileLocal((v - end) as usize)
}
}
}
impl Debug for AtomRef {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::TaggedInt(n) => write!(f, "int({n})"),
Self::Predefined(i) => {
let name = atom_names::PREDEFINED_ATOMS[*i as usize];
write!(f, "atom({name:?})")
}
Self::FileLocal(i) => write!(f, "file_atom[{i}]"),
}
}
}
#[derive(Debug)]
pub enum JsValue {
Null,
Undefined,
BoolFalse,
BoolTrue,
Int(i32),
Float(f64),
String(JsString),
Object,
Array(Vec<JsValue>),
/// Two's-complement little-endian bytes. Empty means zero.
BigInt(Vec<u8>),
/// A tagged-template's frozen `strings` array. `cooked` is the
/// array body; `raw` is the extra trailing value stored as `.raw`
/// (`Undefined` when the template has no raw form).
TemplateObject {
cooked: Vec<JsValue>,
raw: Box<JsValue>,
},
Function(Function),
FnById(usize),
Module(Module),
TypedArray,
ArrayBuffer,
SharedArrayBuffer,
RegExp,
Date,
ObjectValue,
ObjectReference,
Map,
Set,
Symbol,
}
impl JsValue {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, tag) = u8(bin)?;
let ret = match tag {
1 => (bin, Self::Null),
2 => (bin, Self::Undefined),
3 => (bin, Self::BoolFalse),
4 => (bin, Self::BoolTrue),
5 => {
let (bin, v) = sleb128(bin)?;
(bin, Self::Int(v))
}
6 => {
let (bin, bits) = le_u64(bin)?;
(bin, Self::Float(f64::from_bits(bits)))
}
7 => {
// String — same JSString encoding as atoms. Tentative.
let (bin, header) = leb128(bin)?;
let wide = header & 1 == 1;
let len = header >> 1;
let byte_len = if wide { len * 2 } else { len };
let (bin, data) = take(byte_len)(bin)?;
let s = if wide {
JsString::UTF16(data.to_vec())
} else {
JsString::Latin1(data.to_vec())
};
(bin, Self::String(s))
}
9 => {
// Tags 9 and 11 are one reader in quickjs (`JS_ReadArray`): a
// leb128 count then that many values. A template object adds one
// trailing value, defined as `.raw` when it is not undefined.
let (bin, len) = leb128(bin)?;
let (bin, items) = count(JsValue::parse, len as usize).parse(bin)?;
(bin, Self::Array(items))
}
10 => {
// Length is a byte count, not a limb count. quickjs reads it
// back as u32 limbs plus a tail of loose bytes, which is the
// same byte sequence either way.
let (bin, len) = leb128(bin)?;
let (bin, data) = take(len)(bin)?;
(bin, Self::BigInt(data.to_vec()))
}
11 => {
let (bin, len) = leb128(bin)?;
let (bin, cooked) = count(JsValue::parse, len as usize).parse(bin)?;
let (bin, raw) = JsValue::parse(bin)?;
(
bin,
Self::TemplateObject {
cooked,
raw: Box::new(raw),
},
)
}
12 => {
let (bin, function) = Function::parse(bin)?;
(bin, Self::Function(function))
}
13 => {
let (bin, module) = Module::parse(bin)?;
(bin, Self::Module(module))
}
_ => todo!("JsValue tag {tag}"),
};
Ok(ret)
}
}
#[derive(Debug, TryFromPrimitive)]
#[repr(u16)]
pub enum FuncKind {
Normal = 0,
Generator = 1,
Async = 2,
AsyncGenerator = 3,
}
#[derive(Debug)]
pub struct FunctionFlags {
pub has_prototype: bool,
pub has_simple_param_list: bool,
pub is_derived_class_constructor: bool,
pub need_home_object: bool,
pub func_kind: FuncKind,
pub new_target_allowed: bool,
pub super_call_allowed: bool,
pub super_allowed: bool,
pub arguments_allowed: bool,
pub backtrace_barrier: bool,
pub has_debug: bool,
}
impl FunctionFlags {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, raw) = le_u16(bin)?;
Ok((
bin,
Self {
has_prototype: raw & (1 << 0) != 0,
has_simple_param_list: raw & (1 << 1) != 0,
is_derived_class_constructor: raw & (1 << 2) != 0,
need_home_object: raw & (1 << 3) != 0,
func_kind: FuncKind::try_from_primitive((raw >> 4) & 0b11).unwrap(),
new_target_allowed: raw & (1 << 6) != 0,
super_call_allowed: raw & (1 << 7) != 0,
super_allowed: raw & (1 << 8) != 0,
arguments_allowed: raw & (1 << 9) != 0,
backtrace_barrier: raw & (1 << 10) != 0,
has_debug: raw & (1 << 11) != 0,
},
))
}
}
#[derive(Debug, PartialEq, Eq, TryFromPrimitive)]
#[repr(u8)]
pub enum VarKind {
Normal = 0,
FunctionDecl = 1,
NewFunctionDecl = 2,
Catch = 3,
FunctionName = 4,
PrivateField = 5,
PrivateMethod = 6,
PrivateGetter = 7,
PrivateSetter = 8,
PrivateGetterSetter = 9,
Using = 10,
UsingMethod = 11,
}
#[derive(Debug)]
pub struct VarDefFlags {
pub kind: VarKind,
pub is_const: bool,
pub is_lexical: bool,
pub is_captured: bool,
}
impl VarDefFlags {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, raw) = u8(bin)?;
let kind = VarKind::try_from_primitive(raw & 0b1111).unwrap();
let is_const = raw & (1 << 4) != 0;
let is_lexical = raw & (1 << 5) != 0;
let is_captured = raw & (1 << 6) != 0;
Ok((
bin,
Self {
kind,
is_const,
is_lexical,
is_captured,
},
))
}
}
#[derive(Debug)]
pub struct VarDef {
pub name: AtomRef,
pub scope_level: usize,
pub scope_next_plus_1: usize,
pub flags: VarDefFlags,
pub var_ref_idx: Option<usize>,
}
impl VarDef {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, name) = AtomRef::parse(bin)?;
let (bin, scope_level) = leb128(bin)?;
let (bin, scope_next_plus_1) = leb128(bin)?;
let (bin, flags) = VarDefFlags::parse(bin)?;
let (bin, var_ref_idx) = if flags.is_captured {
let (bin, idx) = leb128(bin)?;
(bin, Some(idx))
} else {
(bin, None)
};
Ok((
bin,
Self {
name,
scope_level,
scope_next_plus_1,
flags,
var_ref_idx,
},
))
}
}
#[derive(Debug, TryFromPrimitive)]
#[repr(u8)]
pub enum ClosureKind {
Local = 0,
Arg = 1,
Ref = 2,
GlobalRef = 3,
GlobalDecl = 4,
Global = 5,
ModuleDecl = 6,
ModuleImport = 7,
}
#[derive(Debug)]
pub struct ClosureVarFlags {
pub closure_kind: ClosureKind,
pub is_const: bool,
pub is_lexical: bool,
pub var_kind: VarKind,
}
impl ClosureVarFlags {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, raw) = leb128(bin)?;
// bits 0..2 = closure_type, bit 3 = is_const, bit 4 = is_lexical, bits 5..8 = var_kind
let closure_kind = ClosureKind::try_from_primitive((raw & 0b111) as u8).unwrap();
let is_const = raw & (1 << 3) != 0;
let is_lexical = raw & (1 << 4) != 0;
let var_kind = VarKind::try_from_primitive(((raw >> 5) & 0xF) as u8).unwrap();
Ok((
bin,
Self {
closure_kind,
is_const,
is_lexical,
var_kind,
},
))
}
}
#[derive(Debug)]
pub struct ClosureVar {
pub name: AtomRef,
pub var_idx: usize,
pub flags: ClosureVarFlags,
}
impl ClosureVar {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, name) = AtomRef::parse(bin)?;
let (bin, var_idx) = leb128(bin)?;
let (bin, flags) = ClosureVarFlags::parse(bin)?;
Ok((
bin,
Self {
name,
var_idx,
flags,
},
))
}
}
// LLM GENERATED, TENTATIVE
//
// Decoded operands, one variant per `OpFmt` family. Widths come from
// `OPCODE_INFO[op].size`, which counts the opcode byte, so an
// instruction carries `size - 1` operand bytes.
#[derive(Debug, Clone, Copy)]
pub enum ImmOperand {
/// No operand bytes: `none`, `npopx`, and the `none_*` families
/// where the value is folded into the opcode number itself.
None,
U8(u8),
I8(i8),
U16(u16),
I16(i16),
U32(u32),
I32(i32),
U32x2(u32, u32),
/// Local slot index: `loc`, `loc8`.
Loc(u16),
/// Argument slot index: `arg`.
Arg(u16),
/// Closure variable index: `var_ref`.
VarRef(u16),
/// Constant pool index: `const`, `const8`.
Const(u32),
/// Argument count of a call: `npop`.
Npop(u16),
/// `npop_u16`, used by `eval`: the u16 is a local index biased +1
/// (the scope chain entry point, FORMAT.md §3.3).
NpopU16 {
npop: u16,
scope: u16,
},
/// Jump. The wire value is a signed offset from the operand
/// field's own position; `target` is already resolved.
Label {
target: usize,
},
Atom(AtomRef),
AtomU8 {
atom: AtomRef,
arg: u8,
},
AtomU16 {
atom: AtomRef,
arg: u16,
},
AtomLabelU8 {
atom: AtomRef,
target: usize,
arg: u8,
},
AtomLabelU16 {
atom: AtomRef,
target: usize,
arg: u16,
},
LabelU16 {
target: usize,
arg: u16,
},
}
pub struct Instruction {
pub pc: usize,
pub op: opcodes::OpCode,
pub imm_operands: ImmOperand,
}
impl Debug for Instruction {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:04} {}", self.pc, self.op.name())?;
match &self.imm_operands {
ImmOperand::None => Ok(()),
other => write!(f, " {other:?}"),
}
}
}
// LLM GENERATED, TENTATIVE
//
// Walks `code` instruction by instruction. Two things make this more
// than a byte copy (FORMAT.md §2.3 and notes 2 and 5):
//
// - The opcode byte indexes `short_opcode_info()`, giving the opcode,
// its width, and its operand format. The generated table already
// omits the compiler's temporary opcodes, so no short-opcode shift
// applies.
// - Jump operands are offsets from the operand field's own position,
// not from the start or end of the instruction. For the
// `atom_label_*` formats that base sits 4 bytes into the operands.
fn parse_bytecode(code: &[u8]) -> IResult<&[u8], Vec<Instruction>> {
let jump = |base: usize, offset: i64| (base as i64 + offset) as usize;
let mut out = Vec::new();
let mut pc = 0usize;
while pc < code.len() {
let op = code[pc];
let Some(info) = opcodes::short_opcode_info(op) else {
return Err(nom::Err::Failure(Error::new(code, ErrorKind::Tag)));
};
let size = info.size as usize;
if pc + size > code.len() {
return Err(nom::Err::Incomplete(nom::Needed::new(
pc + size - code.len(),
)));
}
let ops = &code[pc + 1..pc + size];
let u8_at = |i: usize| ops[i];
let u16_at = |i: usize| u16::from_le_bytes([ops[i], ops[i + 1]]);
let u32_at = |i: usize| u32::from_le_bytes([ops[i], ops[i + 1], ops[i + 2], ops[i + 3]]);
// The label field of an `atom_label_*` operand starts after the atom.
let atom_label_base = pc + 1 + 4;
let operands = match info.fmt {
opcodes::OpFmt::None
| opcodes::OpFmt::NoneInt
| opcodes::OpFmt::NoneLoc
| opcodes::OpFmt::NoneArg
| opcodes::OpFmt::NoneVarRef
| opcodes::OpFmt::Npopx => ImmOperand::None,
opcodes::OpFmt::U8 => ImmOperand::U8(u8_at(0)),
opcodes::OpFmt::I8 => ImmOperand::I8(u8_at(0) as i8),
opcodes::OpFmt::Loc8 => ImmOperand::Loc(u8_at(0) as u16),
opcodes::OpFmt::Const8 => ImmOperand::Const(u8_at(0) as u32),
opcodes::OpFmt::Label8 => ImmOperand::Label {
target: jump(pc + 1, u8_at(0) as i8 as i64),
},
opcodes::OpFmt::U16 => ImmOperand::U16(u16_at(0)),
opcodes::OpFmt::I16 => ImmOperand::I16(u16_at(0) as i16),
opcodes::OpFmt::Label16 => ImmOperand::Label {
target: jump(pc + 1, u16_at(0) as i16 as i64),
},
opcodes::OpFmt::Npop => ImmOperand::Npop(u16_at(0)),
opcodes::OpFmt::NpopU16 => ImmOperand::NpopU16 {
npop: u16_at(0),
scope: u16_at(2),
},
opcodes::OpFmt::Loc => ImmOperand::Loc(u16_at(0)),
opcodes::OpFmt::Arg => ImmOperand::Arg(u16_at(0)),
opcodes::OpFmt::VarRef => ImmOperand::VarRef(u16_at(0)),
opcodes::OpFmt::U32 => ImmOperand::U32(u32_at(0)),
opcodes::OpFmt::U32x2 => ImmOperand::U32x2(u32_at(0), u32_at(4)),
opcodes::OpFmt::I32 => ImmOperand::I32(u32_at(0) as i32),
opcodes::OpFmt::Const => ImmOperand::Const(u32_at(0)),
opcodes::OpFmt::Label => ImmOperand::Label {
target: jump(pc + 1, u32_at(0) as i32 as i64),
},
opcodes::OpFmt::Atom => ImmOperand::Atom(AtomRef::from_code_u32(u32_at(0))),
opcodes::OpFmt::AtomU8 => ImmOperand::AtomU8 {
atom: AtomRef::from_code_u32(u32_at(0)),
arg: u8_at(4),
},
opcodes::OpFmt::AtomU16 => ImmOperand::AtomU16 {
atom: AtomRef::from_code_u32(u32_at(0)),
arg: u16_at(4),
},
opcodes::OpFmt::AtomLabelU8 => ImmOperand::AtomLabelU8 {
atom: AtomRef::from_code_u32(u32_at(0)),
target: jump(atom_label_base, u32_at(4) as i32 as i64),
arg: u8_at(8),
},
opcodes::OpFmt::AtomLabelU16 => ImmOperand::AtomLabelU16 {
atom: AtomRef::from_code_u32(u32_at(0)),
target: jump(atom_label_base, u32_at(4) as i32 as i64),
arg: u16_at(8),
},
opcodes::OpFmt::LabelU16 => ImmOperand::LabelU16 {
target: jump(pc + 1, u32_at(0) as i32 as i64),
arg: u16_at(4),
},
};
out.push(Instruction {
pc,
op: info.op,
imm_operands: operands,
});
pc += size;
}
Ok((&code[pc..], out))
}
#[derive(Debug)]
pub struct Function {
pub flags: FunctionFlags,
pub is_strict_mode: bool,
pub name: AtomRef,
/// Regular declared params
pub arg_count: usize,
/// Num locals
pub var_count: usize,
/// Parameters before the first default or rest, this is Function.prototype.length
pub defined_arg_count: usize,
pub stack_size: usize,
pub var_ref_count: usize,
/// arguments first, then locals
pub var_defs: Vec<VarDef>,
pub closure_vars: Vec<ClosureVar>,
pub cpool: Vec<JsValue>,
/// LLM GENERATED, TENTATIVE
pub byte_code: Vec<Instruction>,
}
impl Function {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, flags) = FunctionFlags::parse(bin)?;
let (bin, strict_mode) = u8(bin)?;
let (bin, name) = AtomRef::parse(bin)?;
let (bin, arg_count) = leb128(bin)?;
let (bin, var_count) = leb128(bin)?;
let (bin, defined_arg_count) = leb128(bin)?;
let (bin, stack_size) = leb128(bin)?;
let (bin, var_ref_count) = leb128(bin)?;
let (bin, closure_var_count) = leb128(bin)?;
let (bin, cpool_count) = leb128(bin)?;
let (bin, bytecode_len) = leb128(bin)?;
let (bin, var_def_count) = leb128(bin)?;
let (bin, var_defs) = count(VarDef::parse, var_def_count).parse(bin)?;
let (bin, closure_vars) = count(ClosureVar::parse, closure_var_count).parse(bin)?;
let (bin, cpool) = count(JsValue::parse, cpool_count).parse(bin)?;
let (bin, code) = take(bytecode_len)(bin)?;
let (_, byte_code) = parse_bytecode(code)?;
// skip debug block
let bin = if flags.has_debug {
let (bin, _filename) = AtomRef::parse(bin)?;
let (bin, _line_num) = leb128(bin)?;
let (bin, _col_num) = leb128(bin)?;
let (bin, pc2line_len) = leb128(bin)?;
let (bin, _pc2line) = take(pc2line_len)(bin)?;
let (bin, source_len) = leb128(bin)?;
let (bin, _source) = take(source_len)(bin)?;
bin
} else {
bin
};
Ok((
bin,
Self {
flags,
is_strict_mode: strict_mode == 1,
name,
arg_count,
var_count,
defined_arg_count,
stack_size,
var_ref_count,
var_defs,
closure_vars,
cpool,
byte_code,
},
))
}
}
#[derive(Debug)]
pub struct Module {
pub name: AtomRef,
pub import_modules: Vec<AtomRef>,
pub exports: Vec<ExportInfo>,
pub star_exports: Vec<usize>,
pub imports: Vec<ImportInfo>,
pub has_top_level_await: bool,
pub root_fn: Function,
}
impl Module {
pub fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, name) = AtomRef::parse(bin)?;
let (bin, num_import_modules) = leb128(bin)?;
let (bin, import_modules) =
count(AtomRef::parse, num_import_modules as usize).parse(bin)?;
let (bin, num_exports) = leb128(bin)?;
let (bin, exports) = count(ExportInfo::parse, num_exports).parse(bin)?;
let (bin, num_star_exports) = leb128(bin)?;
let (bin, star_exports) = count(leb128, num_star_exports).parse(bin)?;
let (bin, num_imports) = leb128(bin)?;
let (bin, imports) = count(ImportInfo::parse, num_imports).parse(bin)?;
let (bin, has_tla) = u8(bin)?;
let (bin, value) = JsValue::parse(bin)?;
let JsValue::Function(root_fn) = value else {
panic!()
};
Ok((
bin,
Self {
name,
import_modules,
exports,
star_exports,
imports,
has_top_level_await: has_tla == 1,
root_fn,
},
))
}
}
#[derive(Debug)]
pub enum ExportInfo {
/// All exports except aliases and star exports
Local { var_idx: usize, name: AtomRef },
/// Alias exports, has `from`, eg: export { a } from "./dep.js";
Alias,
}
impl ExportInfo {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, kind) = u8(bin)?;
let ret = match kind {
0 => {
let (bin, var_idx) = leb128(bin)?;
let (bin, name) = AtomRef::parse(bin)?;
(bin, Self::Local { var_idx, name })
}
1 => (bin, Self::Alias),
_ => unreachable!(),
};
Ok(ret)
}
}
#[derive(Debug)]
pub struct ImportInfo {
pub var_idx: usize,
pub name: AtomRef,
pub req_module_idx: usize,
}
impl ImportInfo {
fn parse(bin: &[u8]) -> IResult<&[u8], Self> {
let (bin, var_idx) = leb128(bin)?;
let (bin, name) = AtomRef::parse(bin)?;
let (bin, req_module_idx) = leb128(bin)?;
Ok((
bin,
Self {
var_idx,
name,
req_module_idx,
},
))
}
}
/// Signed LEB128 via zigzag over unsigned LEB128. Tentative.
fn sleb128(input: &[u8]) -> IResult<&[u8], i32> {
let (rest, u) = leb128(input)?;
let v = (u >> 1) as i32 ^ -((u & 1) as i32);
Ok((rest, v))
}
/// Parses leb128 into u64 (assumes usize == u64)
fn leb128(input: &[u8]) -> IResult<&[u8], usize> {
let mut result: u64 = 0;
let mut shift: u32 = 0;
let mut pos = 0;
loop {
if pos >= input.len() {
return Err(nom::Err::Incomplete(nom::Needed::new(1)));
}
let byte = input[pos];
pos += 1;
// Guard against overflow (u64 holds 64 bits, so max shift is 63)
if shift >= 64 {
return Err(nom::Err::Failure(Error::new(input, ErrorKind::TooLarge)));
}
result |= ((byte & 0x7F) as u64) << shift;
shift += 7;
// High bit clear means this is the last byte
if byte & 0x80 == 0 {
return Ok((&input[pos..], result as usize));
}
}
}
pub enum JsString {
Latin1(Vec<u8>),
UTF16(Vec<u8>),
}
impl Debug for JsString {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_char('"')?;
match self {
JsString::Latin1(bytes) => {
for &b in bytes {
let ch = b as char;
if ch == '"' || ch == '\\' {
f.write_char('\\')?;
}
f.write_char(ch)?;
}
}
JsString::UTF16(bytes) => {
let u16s: Vec<u16> = bytes
.chunks_exact(2)
.map(|c| u16::from_le_bytes([c[0], c[1]]))
.collect();
for result in char::decode_utf16(u16s.into_iter()) {
match result {
Ok(ch) => {
if ch == '"' || ch == '\\' {
f.write_char('\\')?;
}
f.write_char(ch)?;
}
Err(e) => write!(f, "\\u{{{:04X}}}", e.unpaired_surrogate())?,
}
}
}
}
f.write_char('"')
}
}
+60
View File
@@ -0,0 +1,60 @@
#![allow(non_snake_case)]
use crate::parse::{ImmOperand, Instruction};
struct BlockSpan {
start: usize,
end: usize,
}
fn find_basic_blocks(code: &[Instruction]) -> Vec<BlockSpan> {
// Find block starts: the first instruction, every jump target,
// and whatever follows a jump or a return
let mut starts = vec![0usize];
for (i, instr) in code.iter().enumerate() {
use crate::parse::opcodes::OpCode::*;
match instr.op {
// If cond/uncond jump, jump target is a start, next instr (if exists) is another start
Goto | Goto8 | Goto16 | IfFalse | IfTrue | IfFalse8 | IfTrue8 => {
let jump_target = match instr.imm_operands {
ImmOperand::Label { target } => {
code.iter().position(|x| x.pc == target).unwrap()
}
_ => panic!("branch opcode without a label operand"),
};
starts.push(jump_target);
if i + 1 < code.len() {
starts.push(i + 1);
}
}
// If terminal, next instr if exists is a start
Return | ReturnUndef | ReturnAsync | Throw => {
if i + 1 < code.len() {
starts.push(i + 1);
}
}
_ => {}
}
}
starts.sort_unstable();
starts.dedup();
// Create blocks, one per block start
let blocks = starts
.iter()
.enumerate()
.map(|(i, &start)| BlockSpan {
start,
end: starts.get(i + 1).copied().unwrap_or(code.len()),
})
.collect::<Vec<BlockSpan>>();
blocks
}