From a8acb7706c69967b7a594e5f409ac147f708b35c Mon Sep 17 00:00:00 2001 From: uditdey Date: Tue, 1 Sep 2026 20:22:11 +0000 Subject: [PATCH] Add files --- .gitignore | 3 + c_parse.py | 475 +++++++++++++++++++++ compiler/Cargo.toml | 2 + compiler/src/flatten.rs | 129 ++++++ compiler/src/main.rs | 29 +- compiler/src/parse.rs | 898 ++++++++++++++++++++++++++++++++++++++++ compiler/src/ssa.rs | 60 +++ 7 files changed, 1595 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 c_parse.py create mode 100644 compiler/src/flatten.rs create mode 100644 compiler/src/parse.rs create mode 100644 compiler/src/ssa.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..63cebb3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +c_parse +compiler/target +compiler/Cargo.lock diff --git a/c_parse.py b/c_parse.py new file mode 100644 index 0000000..668ebf5 --- /dev/null +++ b/c_parse.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +"""Generate atom_names.rs and opcodes.rs from the vendored quickjs headers. + +Reads two files from third_party/quickjs and writes the two Rust tables the +parser needs: + + atom_names.rs the predefined atom strings + opcodes.rs the `OpCode` enum and its per-opcode information table + +Nothing outside the standard library is imported. + +Usage +----- + ./c_parse.py write to ./c_parse + ./c_parse.py --out DIR write somewhere else + ./c_parse.py --check write nothing; compare and exit 1 on drift +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path +from typing import NamedTuple + +# --------------------------------------------------------------------------- +# C source reading +# --------------------------------------------------------------------------- + + +def read_source(path: Path) -> str: + """Read a C file and remove its comments. + + Decoded as latin-1 so that one byte is one character, which reproduces a + byte-indexed walk of the source exactly. + """ + return strip_comments(path.read_bytes().decode("latin-1")) + + +def strip_comments(src: str) -> str: + """Remove /* */ and // comments, leaving string and character literals. + + Each block comment becomes one space so that tokens stay separated. A line + comment leaves its newline behind, so line structure survives. + """ + out: list[str] = [] + i, n = 0, len(src) + while i < n: + c = src[i] + if c == "/" and i + 1 < n and src[i + 1] == "*": + i += 2 + while i + 1 < n and not (src[i] == "*" and src[i + 1] == "/"): + i += 1 + i = min(i + 2, n) + out.append(" ") + elif c == "/" and i + 1 < n and src[i + 1] == "/": + while i < n and src[i] != "\n": + i += 1 + elif c in ('"', "'"): + quote = c + out.append(quote) + i += 1 + while i < n and src[i] != quote: + if src[i] == "\\" and i + 1 < n: + out.append(src[i]) + i += 1 + out.append(src[i]) + i += 1 + if i < n: + out.append(quote) + i += 1 + else: + out.append(c) + i += 1 + return "".join(out) + + +def paren_body(line: str, start: int = 0) -> str | None: + """The text between the parentheses that follow `start`.""" + open_i = line.find("(", start) + if open_i < 0: + return None + depth = 0 + for i in range(open_i, len(line)): + if line[i] == "(": + depth += 1 + elif line[i] == ")": + depth -= 1 + if depth == 0: + return line[open_i + 1 : i] + return None + + +def split_args(body: str) -> list[str]: + out: list[str] = [] + depth = 0 + start = 0 + for i, c in enumerate(body): + if c in "([": + depth += 1 + elif c in ")]": + depth = max(depth - 1, 0) + elif c == "," and depth == 0: + out.append(body[start:i].strip()) + start = i + 1 + out.append(body[start:].strip()) + return out + + +def parse_u8(s: str) -> int | None: + """Rust's `str::parse::()`: an optional `+`, then digits, 0..=255.""" + t = s[1:] if s.startswith("+") else s + if not t or not t.isdigit() or not t.isascii(): + return None + v = int(t) + return v if 0 <= v <= 255 else None + + +class OpDef(NamedTuple): + name: str + size: int + n_pop: int + n_push: int + fmt: str + temporary: bool # written as lowercase `def()`: a compiler-only opcode + + +def parse_fmt_list(text: str) -> list[str]: + """The `FMT(name)` list, in order.""" + out = [] + for line in text.split("\n"): + t = line.lstrip() + if t.startswith("#") or not t.startswith("FMT("): + continue + body = paren_body(t, 0) + if body is not None: + out.append(body.strip()) + return out + + +def parse_opcodes(text: str) -> list[OpDef]: + """Every `DEF(...)` and `def(...)` from quickjs-opcode.h, in file order.""" + out = [] + for line in text.split("\n"): + t = line.lstrip() + if t.startswith("#"): + continue + if t.startswith("DEF("): + temporary = False + elif t.startswith("def("): + temporary = True + else: + continue + body = paren_body(t, 0) + if body is None: + continue + args = split_args(body) + if len(args) != 5: + continue + size, n_pop, n_push = (parse_u8(args[1]), parse_u8(args[2]), parse_u8(args[3])) + if size is None or n_pop is None or n_push is None: + continue + out.append(OpDef(args[0], size, n_pop, n_push, args[4], temporary)) + return out + + +def parse_atoms(text: str) -> list[str]: + """The string of every `DEF(name, "text")` entry in quickjs-atom.h.""" + out = [] + for line in text.split("\n"): + t = line.lstrip() + if t.startswith("#") or not t.startswith("DEF("): + continue + body = paren_body(t, 0) + if body is None: + continue + # Split at the first comma only: the string itself may contain one. + comma = body.find(",") + if comma < 0: + continue + s = decode_c_string(body[comma + 1 :].strip()) + if s is not None: + out.append(s) + return out + + +def decode_c_string(lit: str) -> str | None: + """Decode a C string literal, including the escapes quickjs-atom.h uses.""" + if len(lit) < 2 or not lit.startswith('"') or not lit.endswith('"'): + return None + inner = lit[1:-1] + out: list[str] = [] + i, n = 0, len(inner) + while i < n: + c = inner[i] + i += 1 + if c != "\\": + out.append(c) + continue + if i >= n: + return None + esc = inner[i] + i += 1 + if esc == "n": + out.append("\n") + elif esc == "t": + out.append("\t") + elif esc == "r": + out.append("\r") + elif esc == "0": + out.append("\0") + elif esc == "x": + if i + 1 >= n: + return None + try: + out.append(chr(int(inner[i : i + 2], 16))) + except ValueError: + return None + i += 2 + else: + # Covers \\, \" and \', and passes anything else through. + out.append(esc) + return "".join(out) + + +def rust_string_literal(s: str) -> str: + """Escape a decoded string back into a Rust literal.""" + out = ['"'] + for c in s: + if c == '"': + out.append('\\"') + elif c == "\\": + out.append("\\\\") + elif c == "\n": + out.append("\\n") + elif c == "\r": + out.append("\\r") + elif c == "\t": + out.append("\\t") + elif c == "\0": + out.append("\\0") + elif ord(c) < 0x20: + out.append("\\u{%x}" % ord(c)) + else: + out.append(c) + out.append('"') + return "".join(out) + + +# --------------------------------------------------------------------------- +# emitters +# --------------------------------------------------------------------------- + + +def header(tag: str, frm: str) -> str: + return ( + "// @generated by c_parse.py. Do not edit by hand.\n" + f"// Source: third_party/quickjs ({tag}) {frm}\n\n" + ) + + +def camel(s: str) -> str: + """`none_int` -> `NoneInt`, `push_i32` -> `PushI32`.""" + out = [] + for part in s.split("_"): + if part: + out.append(part[0].upper() + part[1:]) + return "".join(out) + + +def gen_opcodes(text: str, tag: str) -> str: + fmts = parse_fmt_list(text) + if not fmts: + raise Fail("no FMT() entries found in quickjs-opcode.h") + ops = parse_opcodes(text) + if not ops: + raise Fail("no DEF() entries found in quickjs-opcode.h") + + # `OPCodeEnum` numbers only the uppercase `DEF` entries, in file order, so + # a byte read from serialized bytecode is one of these and equals its index + # here. The lowercase `def()` temporary opcodes are compiler-only and never + # reach a file, so dropping them lets the byte index the table directly — + # no short-opcode shift, unlike the C `short_opcode_info` macro whose table + # still holds them. + real = [o for o in ops if not o.temporary] + + s = [header(tag, "quickjs-opcode.h")] + + s.append("/// Operand encoding of an opcode, from the `FMT()` list.\n") + s.append("#[derive(Clone, Copy, PartialEq, Eq, Debug)]\npub enum OpFmt {\n") + for f in fmts: + s.append(f" {camel(f)},\n") + s.append("}\n\n") + + s.append( + "/// Every opcode that can appear in serialized bytecode, numbered as\n" + "/// `OPCodeEnum`. Compiler-only temporary opcodes are excluded, so a\n" + "/// byte read from bytecode is always one of these.\n" + "#[derive(Clone, Copy, PartialEq, Eq, Debug)]\n" + "#[repr(u8)]\n" + "pub enum OpCode {\n" + ) + for i, o in enumerate(real): + s.append(f" {camel(o.name)} = {i},\n") + s.append("}\n\n") + + s.append( + "impl OpCode {\n" + " /// The C identifier, as it appears in `quickjs-opcode.h`.\n" + " pub fn name(self) -> &'static str {\n" + " match self {\n" + ) + for o in real: + s.append(f' OpCode::{camel(o.name)} => "{o.name}",\n') + s.append( + " }\n" + " }\n\n" + " /// This opcode's row in `OPCODE_INFO`.\n" + " pub fn info(self) -> &'static OpInfo {\n" + " &OPCODE_INFO[self as usize]\n" + " }\n" + "}\n\n" + ) + + s.append( + "/// One row of the engine's `opcode_info[]` table.\n" + "#[derive(Clone, Copy, Debug)]\n" + "pub struct OpInfo {\n" + " pub op: OpCode,\n" + " pub size: u8,\n" + " pub n_pop: u8,\n" + " pub n_push: u8,\n" + " pub fmt: OpFmt,\n" + "}\n\n" + ) + + s.append( + f"/// Opcode descriptions indexed by opcode byte, all {len(real)} of them.\n" + f"pub static OPCODE_INFO: [OpInfo; {len(real)}] = [\n" + ) + for o in real: + s.append( + f" OpInfo {{ op: OpCode::{camel(o.name)}, size: {o.size}, " + f"n_pop: {o.n_pop}, n_push: {o.n_push}, fmt: OpFmt::{camel(o.fmt)} }},\n" + ) + s.append("];\n\n") + + s.append( + "/// Look up the description of an opcode read from serialized bytecode.\n" + "///\n" + "/// The table already excludes the compiler's temporary opcodes, so the\n" + "/// byte indexes it directly; there is no short-opcode shift to apply.\n" + "pub fn short_opcode_info(op: u8) -> Option<&'static OpInfo> {\n" + " OPCODE_INFO.get(op as usize)\n" + "}\n" + ) + return "".join(s) + + +def gen_atoms(text: str, tag: str) -> str: + atoms = parse_atoms(text) + if not atoms: + raise Fail("no DEF() entries found in quickjs-atom.h") + s = [header(tag, "quickjs-atom.h")] + s.append( + "// Index 0 is `JS_ATOM_NULL`, which the enum defines as `__JS_ATOM_NULL`\n" + "// before the header's own entries. The `DEF()` entries therefore start\n" + "// at index 1, and `JS_ATOM_END` is the length of this table.\n" + ) + s.append(f"pub static PREDEFINED_ATOMS: [&str; {len(atoms) + 1}] = [\n") + s.append(' "",\n') + for a in atoms: + s.append(f" {rust_string_literal(a)},\n") + s.append("];\n\n") + s.append( + "/// `JS_ATOM_END`: the first atom index that is not predefined.\n" + f"pub const JS_ATOM_END: u32 = {len(atoms) + 1};\n" + ) + return "".join(s) + + +# --------------------------------------------------------------------------- +# driver +# --------------------------------------------------------------------------- + + +class Fail(Exception): + pass + + +def git_describe(src: Path) -> str: + try: + r = subprocess.run( + ["git", "describe", "--tags", "--always"], + cwd=src, + capture_output=True, + text=True, + ) + except OSError: + return "unknown" + if r.returncode != 0: + return "unknown" + return r.stdout.strip() or "unknown" + + +def build(root: Path) -> dict[str, str]: + """Produce every generated file as {filename: contents}.""" + src = root / "third_party/quickjs" + if not (src / "quickjs-opcode.h").exists(): + raise Fail( + f"vendored source not found at {src}. Clone quickjs v0.16.1 there first." + ) + + opcode_h = read_source(src / "quickjs-opcode.h") + atom_h = read_source(src / "quickjs-atom.h") + tag = git_describe(src) + + return { + "opcodes.rs": gen_opcodes(opcode_h, tag), + "atom_names.rs": gen_atoms(atom_h, tag), + } + + +def main() -> int: + root = Path(__file__).resolve().parent + ap = argparse.ArgumentParser( + description="Generate atom_names.rs and opcodes.rs from the vendored quickjs." + ) + ap.add_argument( + "--out", + type=Path, + default=root / "c_parse", + metavar="DIR", + help="output directory (default: c_parse)", + ) + ap.add_argument( + "--check", + action="store_true", + help="write nothing; compare against --out and exit 1 on any difference", + ) + args = ap.parse_args() + + try: + files = build(root) + except Fail as e: + print(f"c_parse.py: {e}", file=sys.stderr) + return 1 + + if args.check: + drift = [] + for name, body in sorted(files.items()): + path = args.out / name + if not path.exists(): + drift.append(f"{name}: missing") + elif path.read_bytes() != body.encode("utf-8"): + drift.append(f"{name}: differs") + if drift: + print("c_parse.py --check found drift:", file=sys.stderr) + for d in drift: + print(f" {d}", file=sys.stderr) + return 1 + print(f"c_parse.py: {len(files)} files match {args.out}") + return 0 + + args.out.mkdir(parents=True, exist_ok=True) + for name, body in sorted(files.items()): + (args.out / name).write_text(body, encoding="utf-8") + print(f" {args.out / name} ({len(body)} bytes)") + print(f"generated {len(files)} files from third_party/quickjs") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 1170539..89df6eb 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -4,3 +4,5 @@ version = "0.1.0" edition = "2024" [dependencies] +nom = "8.0.0" +num_enum = "0.7.6" diff --git a/compiler/src/flatten.rs b/compiler/src/flatten.rs new file mode 100644 index 0000000..4350673 --- /dev/null +++ b/compiler/src/flatten.rs @@ -0,0 +1,129 @@ +//! This module takes the nested QuickJS binary structure and separates the functions into a flat indexed namespace + +use crate::parse::{ + Atom, AtomRef, ClosureKind, Function, Instruction, JsString, JsValue, QuickJSBinary, atom_names, +}; + +#[derive(Debug, Clone, Copy)] +pub enum CaptureFrom { + /// `parent.var_defs[i]` + ParentArg(usize), + /// `parent.var_defs[parent.arg_count + i]` + ParentLocal(usize), + /// `parent.captures[i]`, itself resolved against the grandparent. + ParentCapture(usize), + /// A module-level binding of the module body. + ModuleBinding(usize), + /// Global or module import; not frame-relative. + NotFrameRelative(&'static str), +} + +#[derive(Debug)] +pub struct Capture { + pub name: String, + /// The name as it was on the wire. A field cannot be given a name that is + /// not already an atom, so the operand has to be kept, not rebuilt. + pub atom: AtomRef, + pub from: CaptureFrom, +} + +#[derive(Debug)] +pub struct FlatFn { + pub name: String, + pub arg_count: usize, + pub var_count: usize, // num locals + pub captures: Vec, + pub cpool: Vec, + pub code: Vec, +} + +pub fn flatten(bin: QuickJSBinary) -> (Vec, Vec) { + let mut fns = vec![]; + walk(&mut fns, bin.root_mod.root_fn, &bin.atoms); + (fns, bin.atoms) +} + +fn walk(flat_fns: &mut Vec, func: Function, atoms: &[Atom]) -> usize { + let id = flat_fns.len(); + let name = atom_str(&func.name, atoms); + + // Find vars func captures from its environment + let captures = func + .closure_vars + .iter() + .map(|cv| Capture { + name: atom_str(&cv.name, atoms), + atom: cv.name, + from: match cv.flags.closure_kind { + ClosureKind::Arg => CaptureFrom::ParentArg(cv.var_idx), + ClosureKind::Local => CaptureFrom::ParentLocal(cv.var_idx), + ClosureKind::Ref => CaptureFrom::ParentCapture(cv.var_idx), + ClosureKind::ModuleDecl => CaptureFrom::ModuleBinding(cv.var_idx), + ClosureKind::ModuleImport => todo!(), // CaptureFrom::NotFrameRelative("module_import"), + ClosureKind::Global | ClosureKind::GlobalDecl | ClosureKind::GlobalRef => todo!(), // CaptureFrom::NotFrameRelative("global") + }, + }) + .collect(); + + // Reserve this function's slot before recursing into nested functions, + // so it keeps `id` while its children take the ids after it + flat_fns.push(FlatFn { + name, + arg_count: func.arg_count, + var_count: func.var_count, + captures, + cpool: Vec::new(), + code: func.byte_code, + }); + + // Nested functions live in the cpool; replace each with its flat id. + let mut cpool = Vec::with_capacity(func.cpool.len()); + for const_entry in func.cpool { + let entry = match const_entry { + JsValue::Function(child) => JsValue::FnById(walk(flat_fns, child, atoms)), + other => other, + }; + cpool.push(entry); + } + flat_fns[id].cpool = cpool; + + id +} + +// LLM GENERATED +// ─── atom rendering ────────────────────────────────────────────────────── + +pub fn atom_str(a: &AtomRef, atoms: &[Atom]) -> String { + match a { + AtomRef::TaggedInt(n) => n.to_string(), + AtomRef::Predefined(i) => atom_names::PREDEFINED_ATOMS + .get(*i as usize) + .copied() + .unwrap_or("") + .to_string(), + AtomRef::FileLocal(i) => atoms + .get(*i) + .map(atom_text) + .unwrap_or_else(|| format!("")), + } +} + +fn atom_text(a: &Atom) -> String { + match a { + Atom::String(s) | Atom::GlobalSymbol(s) | Atom::Symbol(s) => js_string(s), + Atom::Predefined(n) => format!(""), + } +} + +fn js_string(s: &JsString) -> String { + match s { + JsString::Latin1(b) => b.iter().map(|&c| c as char).collect(), + JsString::UTF16(b) => { + let u: Vec = b + .chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect(); + String::from_utf16_lossy(&u) + } + } +} diff --git a/compiler/src/main.rs b/compiler/src/main.rs index e7a11a9..8bf16b0 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -1,3 +1,30 @@ +mod flatten; +mod parse; +mod ssa; + +use std::process::Command; + +use flatten::flatten; +use parse::QuickJSBinary; + +const QJSC: &str = "../third_party/quickjs/build/qjsc"; +const BIN_OUT: &str = "/tmp/qjsrt.bin"; + fn main() { - println!("Hello, world!"); + let args = std::env::args().collect::>(); + let filename = args.get(1).unwrap(); + + // Compile source file + let output = Command::new(QJSC) + .args(["-b", "-m", "-o", BIN_OUT]) + .arg(filename) + .output() + .unwrap(); + + assert!(output.status.success()); + + // Parse, then flatten + let bin = std::fs::read(BIN_OUT).unwrap(); + let (_, parsed) = QuickJSBinary::parse(&bin).unwrap(); + let (flat_fns, atoms) = flatten(parsed); } diff --git a/compiler/src/parse.rs b/compiler/src/parse.rs new file mode 100644 index 0000000..84408a4 --- /dev/null +++ b/compiler/src/parse.rs @@ -0,0 +1,898 @@ +//! This module takes the QuickJS produced binary and parses it into an identical structured represention + +use std::fmt::Debug; +use std::fmt::Write; + +use nom::{ + IResult, Parser, + bytes::complete::take, + error::{Error, ErrorKind}, + multi::count, + number::complete::{le_u16, le_u32, le_u64, u8}, +}; + +use num_enum::TryFromPrimitive; + +pub mod atom_names { + include!("../../c_parse/atom_names.rs"); +} + +pub mod opcodes { + include!("../../c_parse/opcodes.rs"); +} + +#[derive(Debug)] +pub struct QuickJSBinary { + pub version: u8, + pub checksum: u32, + pub atoms: Vec, + pub root_mod: Module, +} + +impl QuickJSBinary { + pub fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, version) = u8(bin)?; + let (bin, checksum) = le_u32(bin)?; + let (bin, num_atoms) = leb128(bin)?; + let (bin, atoms) = count(Atom::parse, num_atoms).parse(bin)?; + let (bin, value) = JsValue::parse(bin)?; + + let JsValue::Module(root_mod) = value else { + panic!() + }; + + Ok(( + bin, + QuickJSBinary { + version, + checksum, + atoms, + root_mod, + }, + )) + } +} + +#[derive(Debug)] +pub enum Atom { + Predefined(u32), + String(JsString), + GlobalSymbol(JsString), + Symbol(JsString), +} + +impl Atom { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, kind) = u8(bin)?; + + let ret = match kind { + 0 => { + let (bin, raw) = le_u32(bin)?; + (bin, Self::Predefined(raw)) + } + 1 => { + let (bin, header) = leb128(bin)?; + let wide_char = header & 1 == 1; + let len = header >> 1; + let (bin, data) = take(len)(bin)?; + let data = data.to_vec(); + + let string = if wide_char { + JsString::UTF16(data) + } else { + JsString::Latin1(data) + }; + + (bin, Self::String(string)) + } + 2 => todo!(), + 3 => todo!(), + _ => unreachable!(), + }; + + Ok(ret) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub enum AtomRef { + TaggedInt(u32), + Predefined(u32), + FileLocal(usize), +} + +impl AtomRef { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, v) = leb128(bin)?; + if v & 1 == 1 { + return Ok((bin, Self::TaggedInt((v >> 1) as u32))); + } + let i = v >> 1; + let end = atom_names::JS_ATOM_END as usize; + if i < end { + Ok((bin, Self::Predefined(i as u32))) + } else { + Ok((bin, Self::FileLocal(i - end))) + } + } + + // LLM GENERATED + // + // Atom operands *inside bytecode* use a different encoding from the + // `AtomRef::parse` above: a bare u32 with no shift, where a tagged + // integer is marked by bit 31 instead of bit 0. Same index space, two + // encodings + fn from_code_u32(v: u32) -> Self { + if v & (1 << 31) != 0 { + return Self::TaggedInt(v & !(1 << 31)); + } + let end = atom_names::JS_ATOM_END; + if v < end { + Self::Predefined(v) + } else { + Self::FileLocal((v - end) as usize) + } + } +} + +impl Debug for AtomRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TaggedInt(n) => write!(f, "int({n})"), + Self::Predefined(i) => { + let name = atom_names::PREDEFINED_ATOMS[*i as usize]; + write!(f, "atom({name:?})") + } + Self::FileLocal(i) => write!(f, "file_atom[{i}]"), + } + } +} + +#[derive(Debug)] +pub enum JsValue { + Null, + Undefined, + BoolFalse, + BoolTrue, + Int(i32), + Float(f64), + String(JsString), + Object, + Array(Vec), + /// Two's-complement little-endian bytes. Empty means zero. + BigInt(Vec), + /// A tagged-template's frozen `strings` array. `cooked` is the + /// array body; `raw` is the extra trailing value stored as `.raw` + /// (`Undefined` when the template has no raw form). + TemplateObject { + cooked: Vec, + raw: Box, + }, + Function(Function), + FnById(usize), + Module(Module), + TypedArray, + ArrayBuffer, + SharedArrayBuffer, + RegExp, + Date, + ObjectValue, + ObjectReference, + Map, + Set, + Symbol, +} + +impl JsValue { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, tag) = u8(bin)?; + + let ret = match tag { + 1 => (bin, Self::Null), + 2 => (bin, Self::Undefined), + 3 => (bin, Self::BoolFalse), + 4 => (bin, Self::BoolTrue), + 5 => { + let (bin, v) = sleb128(bin)?; + (bin, Self::Int(v)) + } + 6 => { + let (bin, bits) = le_u64(bin)?; + (bin, Self::Float(f64::from_bits(bits))) + } + 7 => { + // String — same JSString encoding as atoms. Tentative. + let (bin, header) = leb128(bin)?; + let wide = header & 1 == 1; + let len = header >> 1; + let byte_len = if wide { len * 2 } else { len }; + let (bin, data) = take(byte_len)(bin)?; + let s = if wide { + JsString::UTF16(data.to_vec()) + } else { + JsString::Latin1(data.to_vec()) + }; + (bin, Self::String(s)) + } + 9 => { + // Tags 9 and 11 are one reader in quickjs (`JS_ReadArray`): a + // leb128 count then that many values. A template object adds one + // trailing value, defined as `.raw` when it is not undefined. + let (bin, len) = leb128(bin)?; + let (bin, items) = count(JsValue::parse, len as usize).parse(bin)?; + (bin, Self::Array(items)) + } + 10 => { + // Length is a byte count, not a limb count. quickjs reads it + // back as u32 limbs plus a tail of loose bytes, which is the + // same byte sequence either way. + let (bin, len) = leb128(bin)?; + let (bin, data) = take(len)(bin)?; + (bin, Self::BigInt(data.to_vec())) + } + 11 => { + let (bin, len) = leb128(bin)?; + let (bin, cooked) = count(JsValue::parse, len as usize).parse(bin)?; + let (bin, raw) = JsValue::parse(bin)?; + ( + bin, + Self::TemplateObject { + cooked, + raw: Box::new(raw), + }, + ) + } + 12 => { + let (bin, function) = Function::parse(bin)?; + (bin, Self::Function(function)) + } + 13 => { + let (bin, module) = Module::parse(bin)?; + (bin, Self::Module(module)) + } + _ => todo!("JsValue tag {tag}"), + }; + + Ok(ret) + } +} + +#[derive(Debug, TryFromPrimitive)] +#[repr(u16)] +pub enum FuncKind { + Normal = 0, + Generator = 1, + Async = 2, + AsyncGenerator = 3, +} + +#[derive(Debug)] +pub struct FunctionFlags { + pub has_prototype: bool, + pub has_simple_param_list: bool, + pub is_derived_class_constructor: bool, + pub need_home_object: bool, + pub func_kind: FuncKind, + pub new_target_allowed: bool, + pub super_call_allowed: bool, + pub super_allowed: bool, + pub arguments_allowed: bool, + pub backtrace_barrier: bool, + pub has_debug: bool, +} + +impl FunctionFlags { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, raw) = le_u16(bin)?; + + Ok(( + bin, + Self { + has_prototype: raw & (1 << 0) != 0, + has_simple_param_list: raw & (1 << 1) != 0, + is_derived_class_constructor: raw & (1 << 2) != 0, + need_home_object: raw & (1 << 3) != 0, + func_kind: FuncKind::try_from_primitive((raw >> 4) & 0b11).unwrap(), + new_target_allowed: raw & (1 << 6) != 0, + super_call_allowed: raw & (1 << 7) != 0, + super_allowed: raw & (1 << 8) != 0, + arguments_allowed: raw & (1 << 9) != 0, + backtrace_barrier: raw & (1 << 10) != 0, + has_debug: raw & (1 << 11) != 0, + }, + )) + } +} + +#[derive(Debug, PartialEq, Eq, TryFromPrimitive)] +#[repr(u8)] +pub enum VarKind { + Normal = 0, + FunctionDecl = 1, + NewFunctionDecl = 2, + Catch = 3, + FunctionName = 4, + PrivateField = 5, + PrivateMethod = 6, + PrivateGetter = 7, + PrivateSetter = 8, + PrivateGetterSetter = 9, + Using = 10, + UsingMethod = 11, +} + +#[derive(Debug)] +pub struct VarDefFlags { + pub kind: VarKind, + pub is_const: bool, + pub is_lexical: bool, + pub is_captured: bool, +} + +impl VarDefFlags { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, raw) = u8(bin)?; + + let kind = VarKind::try_from_primitive(raw & 0b1111).unwrap(); + let is_const = raw & (1 << 4) != 0; + let is_lexical = raw & (1 << 5) != 0; + let is_captured = raw & (1 << 6) != 0; + + Ok(( + bin, + Self { + kind, + is_const, + is_lexical, + is_captured, + }, + )) + } +} + +#[derive(Debug)] +pub struct VarDef { + pub name: AtomRef, + pub scope_level: usize, + pub scope_next_plus_1: usize, + pub flags: VarDefFlags, + pub var_ref_idx: Option, +} + +impl VarDef { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, name) = AtomRef::parse(bin)?; + let (bin, scope_level) = leb128(bin)?; + let (bin, scope_next_plus_1) = leb128(bin)?; + let (bin, flags) = VarDefFlags::parse(bin)?; + + let (bin, var_ref_idx) = if flags.is_captured { + let (bin, idx) = leb128(bin)?; + (bin, Some(idx)) + } else { + (bin, None) + }; + + Ok(( + bin, + Self { + name, + scope_level, + scope_next_plus_1, + flags, + var_ref_idx, + }, + )) + } +} + +#[derive(Debug, TryFromPrimitive)] +#[repr(u8)] +pub enum ClosureKind { + Local = 0, + Arg = 1, + Ref = 2, + GlobalRef = 3, + GlobalDecl = 4, + Global = 5, + ModuleDecl = 6, + ModuleImport = 7, +} + +#[derive(Debug)] +pub struct ClosureVarFlags { + pub closure_kind: ClosureKind, + pub is_const: bool, + pub is_lexical: bool, + pub var_kind: VarKind, +} + +impl ClosureVarFlags { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, raw) = leb128(bin)?; + + // bits 0..2 = closure_type, bit 3 = is_const, bit 4 = is_lexical, bits 5..8 = var_kind + let closure_kind = ClosureKind::try_from_primitive((raw & 0b111) as u8).unwrap(); + let is_const = raw & (1 << 3) != 0; + let is_lexical = raw & (1 << 4) != 0; + let var_kind = VarKind::try_from_primitive(((raw >> 5) & 0xF) as u8).unwrap(); + + Ok(( + bin, + Self { + closure_kind, + is_const, + is_lexical, + var_kind, + }, + )) + } +} + +#[derive(Debug)] +pub struct ClosureVar { + pub name: AtomRef, + pub var_idx: usize, + pub flags: ClosureVarFlags, +} + +impl ClosureVar { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, name) = AtomRef::parse(bin)?; + let (bin, var_idx) = leb128(bin)?; + let (bin, flags) = ClosureVarFlags::parse(bin)?; + + Ok(( + bin, + Self { + name, + var_idx, + flags, + }, + )) + } +} + +// LLM GENERATED, TENTATIVE +// +// Decoded operands, one variant per `OpFmt` family. Widths come from +// `OPCODE_INFO[op].size`, which counts the opcode byte, so an +// instruction carries `size - 1` operand bytes. +#[derive(Debug, Clone, Copy)] +pub enum ImmOperand { + /// No operand bytes: `none`, `npopx`, and the `none_*` families + /// where the value is folded into the opcode number itself. + None, + U8(u8), + I8(i8), + U16(u16), + I16(i16), + U32(u32), + I32(i32), + U32x2(u32, u32), + /// Local slot index: `loc`, `loc8`. + Loc(u16), + /// Argument slot index: `arg`. + Arg(u16), + /// Closure variable index: `var_ref`. + VarRef(u16), + /// Constant pool index: `const`, `const8`. + Const(u32), + /// Argument count of a call: `npop`. + Npop(u16), + /// `npop_u16`, used by `eval`: the u16 is a local index biased +1 + /// (the scope chain entry point, FORMAT.md §3.3). + NpopU16 { + npop: u16, + scope: u16, + }, + /// Jump. The wire value is a signed offset from the operand + /// field's own position; `target` is already resolved. + Label { + target: usize, + }, + Atom(AtomRef), + AtomU8 { + atom: AtomRef, + arg: u8, + }, + AtomU16 { + atom: AtomRef, + arg: u16, + }, + AtomLabelU8 { + atom: AtomRef, + target: usize, + arg: u8, + }, + AtomLabelU16 { + atom: AtomRef, + target: usize, + arg: u16, + }, + LabelU16 { + target: usize, + arg: u16, + }, +} + +pub struct Instruction { + pub pc: usize, + pub op: opcodes::OpCode, + pub imm_operands: ImmOperand, +} + +impl Debug for Instruction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:04} {}", self.pc, self.op.name())?; + match &self.imm_operands { + ImmOperand::None => Ok(()), + other => write!(f, " {other:?}"), + } + } +} + +// LLM GENERATED, TENTATIVE +// +// Walks `code` instruction by instruction. Two things make this more +// than a byte copy (FORMAT.md §2.3 and notes 2 and 5): +// +// - The opcode byte indexes `short_opcode_info()`, giving the opcode, +// its width, and its operand format. The generated table already +// omits the compiler's temporary opcodes, so no short-opcode shift +// applies. +// - Jump operands are offsets from the operand field's own position, +// not from the start or end of the instruction. For the +// `atom_label_*` formats that base sits 4 bytes into the operands. +fn parse_bytecode(code: &[u8]) -> IResult<&[u8], Vec> { + let jump = |base: usize, offset: i64| (base as i64 + offset) as usize; + + let mut out = Vec::new(); + let mut pc = 0usize; + + while pc < code.len() { + let op = code[pc]; + + let Some(info) = opcodes::short_opcode_info(op) else { + return Err(nom::Err::Failure(Error::new(code, ErrorKind::Tag))); + }; + + let size = info.size as usize; + if pc + size > code.len() { + return Err(nom::Err::Incomplete(nom::Needed::new( + pc + size - code.len(), + ))); + } + + let ops = &code[pc + 1..pc + size]; + let u8_at = |i: usize| ops[i]; + let u16_at = |i: usize| u16::from_le_bytes([ops[i], ops[i + 1]]); + let u32_at = |i: usize| u32::from_le_bytes([ops[i], ops[i + 1], ops[i + 2], ops[i + 3]]); + + // The label field of an `atom_label_*` operand starts after the atom. + let atom_label_base = pc + 1 + 4; + + let operands = match info.fmt { + opcodes::OpFmt::None + | opcodes::OpFmt::NoneInt + | opcodes::OpFmt::NoneLoc + | opcodes::OpFmt::NoneArg + | opcodes::OpFmt::NoneVarRef + | opcodes::OpFmt::Npopx => ImmOperand::None, + + opcodes::OpFmt::U8 => ImmOperand::U8(u8_at(0)), + opcodes::OpFmt::I8 => ImmOperand::I8(u8_at(0) as i8), + opcodes::OpFmt::Loc8 => ImmOperand::Loc(u8_at(0) as u16), + opcodes::OpFmt::Const8 => ImmOperand::Const(u8_at(0) as u32), + opcodes::OpFmt::Label8 => ImmOperand::Label { + target: jump(pc + 1, u8_at(0) as i8 as i64), + }, + + opcodes::OpFmt::U16 => ImmOperand::U16(u16_at(0)), + opcodes::OpFmt::I16 => ImmOperand::I16(u16_at(0) as i16), + opcodes::OpFmt::Label16 => ImmOperand::Label { + target: jump(pc + 1, u16_at(0) as i16 as i64), + }, + opcodes::OpFmt::Npop => ImmOperand::Npop(u16_at(0)), + opcodes::OpFmt::NpopU16 => ImmOperand::NpopU16 { + npop: u16_at(0), + scope: u16_at(2), + }, + opcodes::OpFmt::Loc => ImmOperand::Loc(u16_at(0)), + opcodes::OpFmt::Arg => ImmOperand::Arg(u16_at(0)), + opcodes::OpFmt::VarRef => ImmOperand::VarRef(u16_at(0)), + + opcodes::OpFmt::U32 => ImmOperand::U32(u32_at(0)), + opcodes::OpFmt::U32x2 => ImmOperand::U32x2(u32_at(0), u32_at(4)), + opcodes::OpFmt::I32 => ImmOperand::I32(u32_at(0) as i32), + opcodes::OpFmt::Const => ImmOperand::Const(u32_at(0)), + opcodes::OpFmt::Label => ImmOperand::Label { + target: jump(pc + 1, u32_at(0) as i32 as i64), + }, + + opcodes::OpFmt::Atom => ImmOperand::Atom(AtomRef::from_code_u32(u32_at(0))), + opcodes::OpFmt::AtomU8 => ImmOperand::AtomU8 { + atom: AtomRef::from_code_u32(u32_at(0)), + arg: u8_at(4), + }, + opcodes::OpFmt::AtomU16 => ImmOperand::AtomU16 { + atom: AtomRef::from_code_u32(u32_at(0)), + arg: u16_at(4), + }, + opcodes::OpFmt::AtomLabelU8 => ImmOperand::AtomLabelU8 { + atom: AtomRef::from_code_u32(u32_at(0)), + target: jump(atom_label_base, u32_at(4) as i32 as i64), + arg: u8_at(8), + }, + opcodes::OpFmt::AtomLabelU16 => ImmOperand::AtomLabelU16 { + atom: AtomRef::from_code_u32(u32_at(0)), + target: jump(atom_label_base, u32_at(4) as i32 as i64), + arg: u16_at(8), + }, + opcodes::OpFmt::LabelU16 => ImmOperand::LabelU16 { + target: jump(pc + 1, u32_at(0) as i32 as i64), + arg: u16_at(4), + }, + }; + + out.push(Instruction { + pc, + op: info.op, + imm_operands: operands, + }); + + pc += size; + } + + Ok((&code[pc..], out)) +} + +#[derive(Debug)] +pub struct Function { + pub flags: FunctionFlags, + pub is_strict_mode: bool, + pub name: AtomRef, + /// Regular declared params + pub arg_count: usize, + /// Num locals + pub var_count: usize, + /// Parameters before the first default or rest, this is Function.prototype.length + pub defined_arg_count: usize, + pub stack_size: usize, + pub var_ref_count: usize, + /// arguments first, then locals + pub var_defs: Vec, + pub closure_vars: Vec, + pub cpool: Vec, + /// LLM GENERATED, TENTATIVE + pub byte_code: Vec, +} + +impl Function { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, flags) = FunctionFlags::parse(bin)?; + let (bin, strict_mode) = u8(bin)?; + let (bin, name) = AtomRef::parse(bin)?; + let (bin, arg_count) = leb128(bin)?; + let (bin, var_count) = leb128(bin)?; + let (bin, defined_arg_count) = leb128(bin)?; + let (bin, stack_size) = leb128(bin)?; + let (bin, var_ref_count) = leb128(bin)?; + let (bin, closure_var_count) = leb128(bin)?; + let (bin, cpool_count) = leb128(bin)?; + let (bin, bytecode_len) = leb128(bin)?; + let (bin, var_def_count) = leb128(bin)?; + + let (bin, var_defs) = count(VarDef::parse, var_def_count).parse(bin)?; + let (bin, closure_vars) = count(ClosureVar::parse, closure_var_count).parse(bin)?; + let (bin, cpool) = count(JsValue::parse, cpool_count).parse(bin)?; + + let (bin, code) = take(bytecode_len)(bin)?; + let (_, byte_code) = parse_bytecode(code)?; + + // skip debug block + let bin = if flags.has_debug { + let (bin, _filename) = AtomRef::parse(bin)?; + let (bin, _line_num) = leb128(bin)?; + let (bin, _col_num) = leb128(bin)?; + let (bin, pc2line_len) = leb128(bin)?; + let (bin, _pc2line) = take(pc2line_len)(bin)?; + let (bin, source_len) = leb128(bin)?; + let (bin, _source) = take(source_len)(bin)?; + bin + } else { + bin + }; + + Ok(( + bin, + Self { + flags, + is_strict_mode: strict_mode == 1, + name, + arg_count, + var_count, + defined_arg_count, + stack_size, + var_ref_count, + var_defs, + closure_vars, + cpool, + byte_code, + }, + )) + } +} + +#[derive(Debug)] +pub struct Module { + pub name: AtomRef, + pub import_modules: Vec, + pub exports: Vec, + pub star_exports: Vec, + pub imports: Vec, + pub has_top_level_await: bool, + pub root_fn: Function, +} + +impl Module { + pub fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, name) = AtomRef::parse(bin)?; + + let (bin, num_import_modules) = leb128(bin)?; + let (bin, import_modules) = + count(AtomRef::parse, num_import_modules as usize).parse(bin)?; + + let (bin, num_exports) = leb128(bin)?; + let (bin, exports) = count(ExportInfo::parse, num_exports).parse(bin)?; + let (bin, num_star_exports) = leb128(bin)?; + let (bin, star_exports) = count(leb128, num_star_exports).parse(bin)?; + let (bin, num_imports) = leb128(bin)?; + let (bin, imports) = count(ImportInfo::parse, num_imports).parse(bin)?; + let (bin, has_tla) = u8(bin)?; + let (bin, value) = JsValue::parse(bin)?; + + let JsValue::Function(root_fn) = value else { + panic!() + }; + + Ok(( + bin, + Self { + name, + import_modules, + exports, + star_exports, + imports, + has_top_level_await: has_tla == 1, + root_fn, + }, + )) + } +} + +#[derive(Debug)] +pub enum ExportInfo { + /// All exports except aliases and star exports + Local { var_idx: usize, name: AtomRef }, + /// Alias exports, has `from`, eg: export { a } from "./dep.js"; + Alias, +} + +impl ExportInfo { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, kind) = u8(bin)?; + + let ret = match kind { + 0 => { + let (bin, var_idx) = leb128(bin)?; + let (bin, name) = AtomRef::parse(bin)?; + (bin, Self::Local { var_idx, name }) + } + 1 => (bin, Self::Alias), + _ => unreachable!(), + }; + + Ok(ret) + } +} + +#[derive(Debug)] +pub struct ImportInfo { + pub var_idx: usize, + pub name: AtomRef, + pub req_module_idx: usize, +} + +impl ImportInfo { + fn parse(bin: &[u8]) -> IResult<&[u8], Self> { + let (bin, var_idx) = leb128(bin)?; + let (bin, name) = AtomRef::parse(bin)?; + let (bin, req_module_idx) = leb128(bin)?; + + Ok(( + bin, + Self { + var_idx, + name, + req_module_idx, + }, + )) + } +} + +/// Signed LEB128 via zigzag over unsigned LEB128. Tentative. +fn sleb128(input: &[u8]) -> IResult<&[u8], i32> { + let (rest, u) = leb128(input)?; + let v = (u >> 1) as i32 ^ -((u & 1) as i32); + Ok((rest, v)) +} + +/// Parses leb128 into u64 (assumes usize == u64) +fn leb128(input: &[u8]) -> IResult<&[u8], usize> { + let mut result: u64 = 0; + let mut shift: u32 = 0; + let mut pos = 0; + + loop { + if pos >= input.len() { + return Err(nom::Err::Incomplete(nom::Needed::new(1))); + } + + let byte = input[pos]; + pos += 1; + + // Guard against overflow (u64 holds 64 bits, so max shift is 63) + if shift >= 64 { + return Err(nom::Err::Failure(Error::new(input, ErrorKind::TooLarge))); + } + + result |= ((byte & 0x7F) as u64) << shift; + shift += 7; + + // High bit clear means this is the last byte + if byte & 0x80 == 0 { + return Ok((&input[pos..], result as usize)); + } + } +} + +pub enum JsString { + Latin1(Vec), + UTF16(Vec), +} + +impl Debug for JsString { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_char('"')?; + match self { + JsString::Latin1(bytes) => { + for &b in bytes { + let ch = b as char; + if ch == '"' || ch == '\\' { + f.write_char('\\')?; + } + f.write_char(ch)?; + } + } + JsString::UTF16(bytes) => { + let u16s: Vec = bytes + .chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect(); + for result in char::decode_utf16(u16s.into_iter()) { + match result { + Ok(ch) => { + if ch == '"' || ch == '\\' { + f.write_char('\\')?; + } + f.write_char(ch)?; + } + Err(e) => write!(f, "\\u{{{:04X}}}", e.unpaired_surrogate())?, + } + } + } + } + f.write_char('"') + } +} diff --git a/compiler/src/ssa.rs b/compiler/src/ssa.rs new file mode 100644 index 0000000..3967cfa --- /dev/null +++ b/compiler/src/ssa.rs @@ -0,0 +1,60 @@ +#![allow(non_snake_case)] + +use crate::parse::{ImmOperand, Instruction}; + +struct BlockSpan { + start: usize, + end: usize, +} + +fn find_basic_blocks(code: &[Instruction]) -> Vec { + // Find block starts: the first instruction, every jump target, + // and whatever follows a jump or a return + let mut starts = vec![0usize]; + + for (i, instr) in code.iter().enumerate() { + use crate::parse::opcodes::OpCode::*; + + match instr.op { + // If cond/uncond jump, jump target is a start, next instr (if exists) is another start + Goto | Goto8 | Goto16 | IfFalse | IfTrue | IfFalse8 | IfTrue8 => { + let jump_target = match instr.imm_operands { + ImmOperand::Label { target } => { + code.iter().position(|x| x.pc == target).unwrap() + } + _ => panic!("branch opcode without a label operand"), + }; + + starts.push(jump_target); + + if i + 1 < code.len() { + starts.push(i + 1); + } + } + + // If terminal, next instr if exists is a start + Return | ReturnUndef | ReturnAsync | Throw => { + if i + 1 < code.len() { + starts.push(i + 1); + } + } + + _ => {} + } + } + + starts.sort_unstable(); + starts.dedup(); + + // Create blocks, one per block start + let blocks = starts + .iter() + .enumerate() + .map(|(i, &start)| BlockSpan { + start, + end: starts.get(i + 1).copied().unwrap_or(code.len()), + }) + .collect::>(); + + blocks +}