From 1e12e7be2e16255756158356fc92ee57c50f6a51 Mon Sep 17 00:00:00 2001 From: uditdey Date: Mon, 26 Oct 2026 12:39:58 +0000 Subject: [PATCH] Initial ES modules support --- compiler/src/emit/functions.rs | 6 +- compiler/src/emit/js.rs | 19 +- compiler/src/emit/mod.rs | 12 +- compiler/src/emit/objects.rs | 5 +- compiler/src/emit/program.rs | 44 ++- compiler/src/flatten.rs | 168 ++++------- compiler/src/infer/memory.rs | 6 +- compiler/src/infer/mod.rs | 47 ++-- compiler/src/infer/walk.rs | 19 +- compiler/src/libqjs/ffi.rs | 45 ++- compiler/src/libqjs/mod.rs | 143 ++++++++-- compiler/src/libqjs/parse.rs | 107 ++++--- compiler/src/link.rs | 452 ++++++++++++++++++++++++++++++ compiler/src/main.rs | 115 +++++--- compiler/src/native.rs | 33 +-- compiler/src/transform/async.rs | 108 ++----- compiler/src/transform/closure.rs | 5 +- compiler/src/transform/expand.rs | 27 +- compiler/src/transform/mod.rs | 35 +-- compiler/src/transform/modules.rs | 198 +++++++++++++ docs/model.md | 20 +- lib/stdlib/array.js | 4 +- lib/stdlib/boolean.js | 2 +- lib/stdlib/console.js | 4 +- lib/stdlib/coroutine.js | 6 +- lib/stdlib/end.js | 3 - lib/stdlib/globals.js | 25 +- lib/stdlib/io.js | 14 +- lib/stdlib/loop.js | 14 +- lib/stdlib/number.js | 10 +- lib/stdlib/object.js | 2 +- lib/stdlib/prelude.js | 42 +++ lib/stdlib/promise.js | 12 +- lib/stdlib/string.js | 12 +- lib/stdlib/timers.js | 7 +- 35 files changed, 1266 insertions(+), 505 deletions(-) create mode 100644 compiler/src/link.rs create mode 100644 compiler/src/transform/modules.rs delete mode 100644 lib/stdlib/end.js create mode 100644 lib/stdlib/prelude.js diff --git a/compiler/src/emit/functions.rs b/compiler/src/emit/functions.rs index 4487729..1fa0630 100644 --- a/compiler/src/emit/functions.rs +++ b/compiler/src/emit/functions.rs @@ -15,7 +15,7 @@ use super::classes::{CType, ClassKind, Member}; use super::program::Program; use super::{js, objects}; -use crate::{infer::Type, libqjs::parse::AtomRef, regify::Operand}; +use crate::{flatten::ENTRY_FN, infer::Type, libqjs::parse::AtomRef, regify::Operand}; #[derive(Clone, Copy)] pub enum Function { @@ -65,7 +65,7 @@ impl Function { /// Call the entry, with undefined args fn start_body(program: &Program) -> String { - let arg_count = program.fns[program.fns.len() - 1].arg_count; + let arg_count = program.fns[ENTRY_FN].arg_count; let args = vec![js::UNDEFINED.to_string(); arg_count]; format!(" {};\n", host_call(program, program.entry, &args)) } @@ -81,7 +81,7 @@ fn complete_body(program: &Program) -> String { /// The call of `__complete`, with the closure as its own env and `this` undefined. Infer /// made the same call (see `TypeCache::call_host_complete`) fn complete_call(program: &Program) -> Option { - let (callee, ctype) = program.binding("__complete")?; + let (callee, ctype) = program.binding("jsrt:loop", "__complete")?; let CType::One(Member::Obj(class)) = ctype else { return None; }; diff --git a/compiler/src/emit/js.rs b/compiler/src/emit/js.rs index ad018bb..7e50e52 100644 --- a/compiler/src/emit/js.rs +++ b/compiler/src/emit/js.rs @@ -120,10 +120,8 @@ impl JsFunction<'_> { self.program.slot_type(self.walk, slot) } - /// A C local for each local and reg, and a cell for each shared local and arg. A module - /// binding, which is a local of the entry, also puts its cell in its C global + /// A C local for each local and reg, and a cell for each shared local and arg fn declarations(&self) -> String { - let entry = self.walk == self.program.entry; let mut c = String::new(); for slot in slots(self.f) { @@ -134,9 +132,6 @@ impl JsFunction<'_> { c += &match (slot, self.shared.contains(&slot)) { (Operand::Arg(_), false) => continue, (Operand::Arg(k), true) => format!("{ctype} *{name} = {cell}; *{name} = p{k};\n"), - (Operand::Local(k), true) if entry && self.program.globals.contains_key(&k) => { - format!("{ctype} *{name} = g{k} = {cell};\n") - } (_, true) => format!("{ctype} *{name} = {cell};\n"), (_, false) => format!("{ctype} {name} = {{0}};\n"), }; @@ -220,6 +215,10 @@ impl JsFunction<'_> { // Copies. A key stays as it is: a number is an index SetLoc | SetName | ToPropkey => self.set(instr.dsts[0], &get(0)), + // The bindings at the top of a module, which are C globals (see link.rs) + GetVar => self.set(instr.dsts[0], &format!("g{}", global(instr))), + PutVar => format!("SET(g{}, {});\n", global(instr), get(0)), + PostInc | PostDec => { let name = if instr.op == PostInc { "POST_INC" @@ -298,6 +297,14 @@ impl JsFunction<'_> { } } +/// The global that `get_var` or `put_var` reads or writes (see transform/modules.rs) +fn global(instr: &RegInstr) -> u32 { + match instr.imm { + ImmOperand::Global(global) => global, + imm => unreachable!("a global name that is not a global: {imm:?}"), + } +} + /// An int, as C fn int(n: i64) -> String { match n == i32::MIN as i64 { diff --git a/compiler/src/emit/mod.rs b/compiler/src/emit/mod.rs index 0bbd82c..0065d31 100644 --- a/compiler/src/emit/mod.rs +++ b/compiler/src/emit/mod.rs @@ -5,7 +5,7 @@ //! in an order that C can compile: //! //! - **Types:** a struct for each class of objects (see `classes`). -//! - **Statics:** each string literal, and a C global for each module binding. +//! - **Statics:** each string literal, and a C global for each binding at the top of a module. //! - **Functions:** one for each walk of a JS function, one for each lookup, and the host's //! two entry points (see `functions`). //! @@ -19,7 +19,7 @@ mod js; mod objects; mod program; -use crate::{flatten::ModuleBindings, infer::TypeCache, libqjs::parse::Atom, regify::RegFn}; +use crate::{infer::TypeCache, libqjs::parse::Atom, link::Link, regify::RegFn}; use classes::ClassKind; use program::Program; @@ -27,8 +27,8 @@ use program::Program; const HEADER: &str = "// Generated by jsrt\n#define JSRT_IMPLEMENTATION\n#include \"jsrt.h\"\n\n"; /// The C of the program, which includes the runtime's `jsrt.h` -pub fn emit(fns: &[RegFn], cache: &TypeCache, atoms: &[Atom], bindings: &ModuleBindings) -> String { - let program = Program::new(fns, cache, atoms, bindings); +pub fn emit(fns: &[RegFn], cache: &TypeCache, atoms: &[Atom], link: &Link) -> String { + let program = Program::new(fns, cache, atoms, link); let classes = 0..program.classes.classes.len(); let mut c = String::from(HEADER); @@ -48,8 +48,8 @@ pub fn emit(fns: &[RegFn], cache: &TypeCache, atoms: &[Atom], bindings: &ModuleB c += &string_definition(id, text); } - for (&local, &ctype) in &program.globals { - c += &format!("static {} *g{local};\n", ctype.c_name()); + for (g, ctype) in program.globals.iter().enumerate() { + c += &format!("static {} g{g};\n", ctype.c_name()); } // Functions. Each is declared first, because they call each other diff --git a/compiler/src/emit/objects.rs b/compiler/src/emit/objects.rs index 17776fd..550b274 100644 --- a/compiler/src/emit/objects.rs +++ b/compiler/src/emit/objects.rs @@ -51,10 +51,7 @@ impl JsFunction<'_> { /// `[a, b, ...]`. The operands are the elements, last first fn array_from(&self, idx: usize, instr: &RegInstr) -> String { let class = self.class_here(idx); - let mut c = format!( - "struct c{class} *a;\nARRAY_NEW(a, {});\n", - instr.args.len() - ); + let mut c = format!("struct c{class} *a;\nARRAY_NEW(a, {});\n", instr.args.len()); for (i, &elem) in instr.args.iter().rev().enumerate() { c += &format!("SET(a->items[{i}], {});\n", self.get(elem)); diff --git a/compiler/src/emit/program.rs b/compiler/src/emit/program.rs index 29c814c..2f47f9c 100644 --- a/compiler/src/emit/program.rs +++ b/compiler/src/emit/program.rs @@ -3,15 +3,16 @@ //! `Program::new` finds all of them before any C is written. Writing a function only reads //! the program, so the functions can be written in any order. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use super::classes::{CType, ClassField, ClassKind, Classes, Member}; use super::functions::Function; use super::js::indent; use crate::{ - flatten::ModuleBindings, - infer::{LENGTH_FIELD, Type, TypeCache, join, module_binding}, + flatten::ENTRY_FN, + infer::{Addr, LENGTH_FIELD, Type, TypeCache, join}, libqjs::parse::{Atom, AtomRef, JsValue, opcodes::OpCode}, + link::{Link, PRELUDE, Target}, regify::{Operand, RegFn, RegInstr}, transform::{PROTO_FIELD, PROTOTYPE_FIELD}, }; @@ -21,32 +22,27 @@ pub struct Program<'a> { pub fns: &'a [RegFn], pub cache: &'a TypeCache, pub atoms: &'a [Atom], - bindings: &'a ModuleBindings, + pub link: &'a Link, pub entry: usize, // the walk of the entry: the JS function that runs the program // The output pub classes: Classes, // types: a struct for each class of objects pub strings: Vec, // statics: the text of each string literal - pub globals: BTreeMap, // statics: the module bindings, which are locals of the entry, and their C types + pub globals: Vec, // statics: the C type of each binding at the top of a module (see link.rs) pub functions: Vec, } impl<'a> Program<'a> { - pub fn new( - fns: &'a [RegFn], - cache: &'a TypeCache, - atoms: &'a [Atom], - bindings: &'a ModuleBindings, - ) -> Self { + pub fn new(fns: &'a [RegFn], cache: &'a TypeCache, atoms: &'a [Atom], link: &'a Link) -> Self { let mut program = Program { fns, cache, atoms, - bindings, - entry: cache.find(fns.len() - 1, None, &[], fns), + link, + entry: cache.find(ENTRY_FN, None, &[], fns), classes: Classes::new(&cache.memory), strings: vec![], - globals: BTreeMap::new(), + globals: vec![], functions: vec![], }; @@ -142,12 +138,8 @@ impl<'a> Program<'a> { /// Each module binding is a C global, so that any function can use the stdlib's fn add_globals(&mut self) { - let entry = self.fns.len() - 1; - - for &local in self.bindings.values() { - let ty = self.cache.memory.get(module_binding(entry, local)); - self.globals.insert(local, self.classes.ctype(ty)); - } + let types = (0..self.link.globals).map(|g| self.cache.memory.get(Addr::Global(g))); + self.globals = types.map(|ty| self.classes.ctype(ty)).collect(); } /// The instrs of a JS function that can run in a walk @@ -200,10 +192,12 @@ impl Program<'_> { } } - /// The value of the stdlib's module binding `name`, as C, and its C type - pub fn binding(&self, name: &str) -> Option<(String, CType)> { - let local = *self.bindings.get(name)?; - Some((format!("(*g{local})"), self.globals[&local])) + /// The value of the export `name` of the stdlib's module `module`, as C, and its C type + pub fn binding(&self, module: &str, name: &str) -> Option<(String, CType)> { + match self.link.export(module, name)? { + Target::Global(g) => Some((format!("g{g}"), self.globals[g])), + Target::Native(_) => None, + } } } @@ -343,7 +337,7 @@ impl Program<'_> { _ => return None, }; - let (stdlib_class, CType::One(Member::Obj(class))) = self.binding(name)? else { + let (stdlib_class, CType::One(Member::Obj(class))) = self.binding(PRELUDE, name)? else { return None; }; let proto = self.classes.field(class, PROTOTYPE_FIELD)?; diff --git a/compiler/src/flatten.rs b/compiler/src/flatten.rs index e87110c..66555f2 100644 --- a/compiler/src/flatten.rs +++ b/compiler/src/flatten.rs @@ -1,26 +1,21 @@ -//! QuickJS bytecode represents the program as a hierarchy. Each function has a set of child functions -//! that reference captured variables in their parent +//! QuickJS bytecode represents each module as a hierarchy. Each function has a set of child +//! functions that reference captured variables in their parent //! //! This module: -//! - flattens the hierarchy +//! - flattens each module's hierarchy into one list for the program //! - assigns an integer idx to each function //! - converts child ownership to a reference by-idx //! - extracts closure captures into a separate vec //! -//! Also adds a synthetic entry point function that calls the top level fn generated by QuickJS, -//! and makes the module's bindings its locals - -use std::collections::HashMap; +//! Fn 0 is the program's entry, which link.rs writes: it runs every module's body use crate::libqjs::parse::{ - Atom, AtomRef, ClosureKind, ClosureVar, FuncKind, Function, ImmOperand, Instr, JsValue, - QuickJSBinary, opcodes::OpCode, + Atom, AtomRef, ClosureVar, ExportInfo, FuncKind, Function, ImportInfo, Instr, JsValue, + QuickJSBinary, }; -/// The module's bindings, which are locals of the entry fn, by name. The compiler reads the -/// stdlib's by name where it needs one: the classes whose prototypes the primitives use, and -/// what the host calls when an IO op completes -pub type ModuleBindings = HashMap; +/// The entry fn (see link.rs) +pub const ENTRY_FN: usize = 0; #[derive(Debug)] pub struct FlatFn { @@ -32,50 +27,66 @@ pub struct FlatFn { pub code: Vec, } -/// Flattens the binary -/// -/// Returns: -/// - flat_fns[i] = the i'th flattened function -/// - captures[i] = the i'th fn's captures from its parent -/// - the module's bindings, as locals of the entry fn -pub fn flatten( - bin: QuickJSBinary, -) -> (Vec, Vec>, Vec, ModuleBindings) { - let mut fns = vec![]; - let mut captures = vec![]; - - walk(bin.root_mod.root_fn, &mut fns, &mut captures); - add_entry_point(&mut fns, &mut captures); - let bindings = bind_module(&mut fns, &mut captures, &bin.atoms); - - (fns, captures, bin.atoms, bindings) +/// A module, as QuickJS gives it, with its body as a flat fn +#[derive(Debug)] +pub struct FlatModule { + pub name: String, // the name that it was loaded by (see `libqjs::compile`) + pub requests: Vec, // the modules that it imports from, as the source writes them + pub exports: Vec, + pub star_exports: Vec, // `export * from`: each an index into `requests` + pub imports: Vec, + pub has_top_level_await: bool, + pub body: usize, // the fn of its body. Its captures are the module's bindings } -/// Make each binding of the module a local of the entry fn, after its own two. The module fn -/// (``, the first fn) captures them, as any fn captures its parent's locals, so they are -/// cells (see transform/closure.rs) -fn bind_module( - fns: &mut [FlatFn], - captures: &mut [Vec], - atoms: &[Atom], -) -> ModuleBindings { - let entry = fns.len() - 1; - let mut bindings = ModuleBindings::new(); +/// Flattens every module of the binary into one list of fns. Gives the fns, each fn's captures +/// from its parent, the atoms, and the modules +pub fn flatten( + bin: QuickJSBinary, +) -> ( + Vec, + Vec>, + Vec, + Vec, +) { + let text = |atom: AtomRef| atom.text(&bin.atoms); + let mut fns = vec![entry_placeholder()]; + let mut captures = vec![vec![]]; + let mut modules = vec![]; - for capture in &mut captures[0] { - if capture.flags.closure_kind != ClosureKind::ModuleDecl { - continue; + for module in bin.modules { + let has_top_level_await = module.has_top_level_await; + let body = walk(module.root_fn, &mut fns, &mut captures); + + // QuickJS makes every module's body async. It only suspends if it awaits at its top + if !has_top_level_await { + fns[body].kind = FuncKind::Normal; } - let local = fns[entry].local_count; - fns[entry].local_count += 1; - - capture.flags.closure_kind = ClosureKind::Local; - capture.var_idx = local; - bindings.insert(capture.name.text(atoms), local); + modules.push(FlatModule { + name: text(module.name), + requests: module.requests.iter().map(|&r| text(r)).collect(), + exports: module.exports, + star_exports: module.star_exports, + imports: module.imports, + has_top_level_await, + body, + }); } - bindings + (fns, captures, bin.atoms, modules) +} + +/// Fn 0, before link.rs writes it +fn entry_placeholder() -> FlatFn { + FlatFn { + name: AtomRef::Predefined(0), + kind: FuncKind::Normal, + arg_count: 0, + local_count: 0, + cpool: vec![], + code: vec![], + } } fn walk(func: Function, flat_fns: &mut Vec, captures: &mut Vec>) -> usize { @@ -105,60 +116,3 @@ fn walk(func: Function, flat_fns: &mut Vec, captures: &mut Vec() { -/// // Module top level created by quickjs -/// // All other top level decls are inside -/// function () { -/// ... -/// } -/// -/// // Link phase (`this = true`), then run phase (`this = false`) -/// let phase = true; -/// while (true) { -/// .call(phase); -/// if (!phase) break; -/// phase = false; -/// } -/// } -/// ``` -/// -/// Both phases call `` from one call instr. Infer names the objects that a walk makes -/// by its call site, so with two call instrs, each phase would make its own copy of every -/// module-level object, and each module binding would hold one of two copies -/// -/// This gets called by C main() -fn add_entry_point(fns: &mut Vec, captures: &mut Vec>) { - let instr = |pc, op, imm| Instr { pc, op, imm }; - - let code = vec![ - instr(0, OpCode::Fclosure8, ImmOperand::Const(0)), - instr(1, OpCode::PutLoc, ImmOperand::Loc(0)), - instr(2, OpCode::PushTrue, ImmOperand::None), - instr(3, OpCode::PutLoc, ImmOperand::Loc(1)), - // Loop head: one call for both phases - instr(4, OpCode::GetLoc, ImmOperand::Loc(1)), - instr(5, OpCode::GetLoc, ImmOperand::Loc(0)), - instr(6, OpCode::CallMethod, ImmOperand::Npop(0)), - instr(7, OpCode::Drop, ImmOperand::None), - instr(8, OpCode::GetLoc, ImmOperand::Loc(1)), - instr(9, OpCode::IfFalse8, ImmOperand::Label { target: 13 }), - instr(10, OpCode::PushFalse, ImmOperand::None), - instr(11, OpCode::PutLoc, ImmOperand::Loc(1)), - instr(12, OpCode::Goto8, ImmOperand::Label { target: 4 }), - instr(13, OpCode::ReturnUndef, ImmOperand::None), - ]; - - fns.push(FlatFn { - name: AtomRef::Predefined(0), - kind: FuncKind::Normal, - arg_count: 0, - local_count: 2, // , and the phase - cpool: vec![JsValue::FnByIdx(0)], // First fn is the top level - code, - }); - - captures.push(vec![]); -} diff --git a/compiler/src/infer/memory.rs b/compiler/src/infer/memory.rs index 6f0019f..aad49fc 100644 --- a/compiler/src/infer/memory.rs +++ b/compiler/src/infer/memory.rs @@ -39,7 +39,8 @@ pub struct ObjAddr { /// A location in the program, which memory holds a type for /// /// Every location is named by its context and a place in its function, except the ones that -/// only one walk can see, which are named by the walk. A type can hold an object's address, +/// only one walk can see, which are named by the walk, and the bindings at the top of a +/// module, of which there is one each for the whole program. A type can hold an object's address, /// and a walk's arg types can hold that type, so objects are never named by a walk: that /// could make new walks without end. Every part of an address comes from the program text or /// from a walk, so there are finitely many, and infer ends @@ -50,6 +51,7 @@ pub enum Addr { Shared { context: Context, slot: Operand }, // a binding that closures share (see transform/closure.rs). The closures are objects, so it is named like them Private { walk: usize, slot: Operand }, // a binding that only its walk sees Return(usize), // what a walk returns + Global(usize), // a binding at the top of a module: one for the whole program (see link.rs) } /// What each location holds, by its address. A location that nothing has reached holds @@ -64,7 +66,7 @@ pub struct Memory { } /// The stdlib's classes, whose prototypes the primitives and arrays read: the address of each -/// one's module binding (see `flatten::ModuleBindings`). A program without the stdlib has none +/// one's binding, which the prelude exports (see link.rs) #[derive(Debug, Default)] pub struct Builtins { pub string: Option, diff --git a/compiler/src/infer/mod.rs b/compiler/src/infer/mod.rs index 8e8a1ec..e3039f2 100644 --- a/compiler/src/infer/mod.rs +++ b/compiler/src/infer/mod.rs @@ -26,7 +26,8 @@ mod walk; use std::collections::HashMap; use crate::{ - flatten::ModuleBindings, + flatten::ENTRY_FN, + link::{Link, Target}, regify::{Operand, RegFn}, }; @@ -170,12 +171,15 @@ fn binding_addr(w: usize, context: Context, shared: &[Operand], slot: Operand) - /// Walk the program from the entry, in rounds, until a round changes nothing /// -/// The host makes two kinds of call (see lib/rt/jsrt.h): the entry, with no args, which -/// `flatten` appends last, and the stdlib's `__complete`, with an op's handle, when the op -/// completes. Both come from the host, which has no call site -pub fn infer_types(fns: &[RegFn], bindings: &ModuleBindings) -> TypeCache { - let entry = fns.len() - 1; - let binding = |name: &str| bindings.get(name).map(|&k| module_binding(entry, k)); +/// The host makes two kinds of call (see lib/rt/jsrt.h): the entry, with no args, and the +/// stdlib's `__complete`, with an op's handle, when the op completes. Both come from the +/// host, which has no call site +pub fn infer_types(fns: &[RegFn], link: &Link) -> TypeCache { + let global = |name: &str| match link.global(name) { + Some(Target::Global(global)) => Some(Addr::Global(global)), + _ => None, + }; + let complete = Addr::Global(link.stdlib("jsrt:loop", "__complete")); let mut cache = TypeCache { walks: vec![], @@ -185,37 +189,20 @@ pub fn infer_types(fns: &[RegFn], bindings: &ModuleBindings) -> TypeCache { }; cache.memory.builtins = Builtins { - string: binding("String"), - number: binding("Number"), - boolean: binding("Boolean"), - array: binding("Array"), + string: global("String"), + number: global("Number"), + boolean: global("Boolean"), + array: global("Array"), }; loop { let before = cache.progress(); cache.round += 1; - cache.call(entry, None, &[], fns); - - if let Some(complete) = binding("__complete") { - cache.call_host_complete(complete, fns); - } + cache.call(ENTRY_FN, None, &[], fns); + cache.call_host_complete(complete, fns); if cache.progress() == before { return cache; } } } - -/// The address of module binding `local` (see `flatten::ModuleBindings`): a local of the -/// entry, which the module's function shares with its closures -pub fn module_binding(entry: usize, local: usize) -> Addr { - let context = Context { - fn_idx: entry, - call_site: None, - }; - - Addr::Shared { - context, - slot: Operand::Local(local), - } -} diff --git a/compiler/src/infer/walk.rs b/compiler/src/infer/walk.rs index 4b8828e..97f7909 100644 --- a/compiler/src/infer/walk.rs +++ b/compiler/src/infer/walk.rs @@ -15,7 +15,7 @@ use super::{ }, }; use crate::{ - libqjs::parse::{JsValue, opcodes::OpCode}, + libqjs::parse::{ImmOperand, JsValue, opcodes::OpCode}, native, regify::{BasicBlock, Operand, RegFn, RegInstr}, transform::{FIRST_ARG, THIS_ARG}, @@ -217,6 +217,15 @@ impl Walker<'_> { // Moves. A key stays as it is: a number is an index SetLoc | SetName | ToPropkey => args[0].clone(), + // The bindings at the top of a module, which are globals (see link.rs) + GetVar => memory.get(Addr::Global(global(instr))).clone(), + PutVar => { + self.cache + .memory + .join(Addr::Global(global(instr)), &args[0]); + Type::Unknown + } + // Objects Object => self.make(idx, Type::Shape(Shape::Obj(vec![]))), Fclosure => { @@ -452,3 +461,11 @@ fn check_index(index: &Type) { } } } + +/// The global that `get_var` or `put_var` reads or writes (see transform/modules.rs) +fn global(instr: &RegInstr) -> usize { + match instr.imm { + ImmOperand::Global(global) => global as usize, + imm => unreachable!("a global name that is not a global: {imm:?}"), + } +} diff --git a/compiler/src/libqjs/ffi.rs b/compiler/src/libqjs/ffi.rs index 1f05f0f..3937b0f 100644 --- a/compiler/src/libqjs/ffi.rs +++ b/compiler/src/libqjs/ffi.rs @@ -1,6 +1,7 @@ -use std::ffi::{c_int, c_void}; +use std::ffi::{c_char, c_int, c_void}; pub const JS_TAG_EXCEPTION: i64 = 6; +pub const JS_TAG_MODULE: i64 = -3; pub const JS_EVAL_TYPE_MODULE: c_int = 1 << 0; pub const JS_EVAL_FLAG_COMPILE_ONLY: c_int = 1 << 5; pub const JS_WRITE_OBJ_BYTECODE: c_int = 1 << 0; @@ -12,6 +13,13 @@ pub struct JSValue { pub tag: i64, } +impl JSValue { + /// The pointer of a value that has one, such as a module + pub fn ptr(self) -> *mut c_void { + unsafe { self.u.ptr } + } +} + #[repr(C)] #[derive(Clone, Copy)] union JSValueUnion { @@ -22,6 +30,15 @@ union JSValueUnion { pub type JSRuntime = c_void; pub type JSContext = c_void; +pub type JSModuleDef = c_void; + +pub type JSModuleLoaderFunc = unsafe extern "C" fn( + ctx: *mut JSContext, + module_name: *const c_char, + opaque: *mut c_void, +) -> *mut JSModuleDef; + +pub type JSModuleInitFunc = unsafe extern "C" fn(ctx: *mut JSContext, m: *mut JSModuleDef) -> c_int; unsafe extern "C" { pub fn JS_NewRuntime() -> *mut JSRuntime; @@ -48,6 +65,32 @@ unsafe extern "C" { pub fn JS_GetException(ctx: *mut JSContext) -> JSValue; + pub fn JS_ThrowReferenceError(ctx: *mut JSContext, fmt: *const c_char, ...) -> JSValue; + + pub fn JS_SetModuleLoaderFunc( + rt: *mut JSRuntime, + module_normalize: *const c_void, + module_loader: JSModuleLoaderFunc, + opaque: *mut c_void, + ); + + pub fn JS_NewCModule( + ctx: *mut JSContext, + name: *const c_char, + func: JSModuleInitFunc, + ) -> *mut JSModuleDef; + + pub fn JS_AddModuleExport( + ctx: *mut JSContext, + m: *mut JSModuleDef, + name: *const c_char, + ) -> c_int; + + pub fn JS_NewArray(ctx: *mut JSContext) -> JSValue; + + pub fn JS_SetPropertyUint32(ctx: *mut JSContext, obj: JSValue, idx: u32, val: JSValue) + -> c_int; + pub fn JS_ToCStringLen2( ctx: *mut JSContext, plen: *mut usize, diff --git a/compiler/src/libqjs/mod.rs b/compiler/src/libqjs/mod.rs index a9b0796..75895db 100644 --- a/compiler/src/libqjs/mod.rs +++ b/compiler/src/libqjs/mod.rs @@ -1,38 +1,60 @@ mod ffi; pub mod parse; -use std::ffi::{CStr, CString}; +use std::ffi::{CStr, CString, c_char, c_int, c_void}; use parse::QuickJSBinary; -pub fn compile(src: &[u8], filename: &str) -> QuickJSBinary { - unsafe { - let filename = CString::new(filename).unwrap(); +/// Where the compiler gets the modules that QuickJS asks for +pub struct Sources { + pub source: fn(&str) -> Option>, // the source of a module, by its name + pub native: &'static str, // the module of the natives, which has no source + pub native_exports: Vec<&'static str>, // its exports +} +/// Compiles `roots`, in order, and every module that they import. QuickJS finds the imports: +/// it names each one from the specifier and the importing module's name, and asks `sources` +/// for any module that it has not loaded (see `load_module`) +/// +/// Every module is written in one go, as one array, so all of them share one atom table. The +/// module of the natives is a C module: it has exports but no code, so it is not written +pub fn compile(roots: &[String], sources: Sources) -> QuickJSBinary { + unsafe { let rt = ffi::JS_NewRuntime(); let ctx = ffi::JS_NewContext(rt); + // QuickJS calls `load_module` while it compiles a module, so the loader is only ever + // reached through this pointer, and never held across a compile + let loader = Box::into_raw(Box::new(Loader { + sources, + loaded: vec![], + })); + ffi::JS_SetModuleLoaderFunc(rt, std::ptr::null(), load_module, loader.cast()); - // JS_Eval requires a '\0' after the last byte of the source. - let src_z = [src, &[0]].concat(); + for root in roots { + if (*loader).loaded.iter().any(|(name, _)| name == root) { + continue; + } - let obj = ffi::JS_Eval( - ctx, - src_z.as_ptr().cast(), - src.len(), - filename.as_ptr().cast(), - ffi::JS_EVAL_FLAG_COMPILE_ONLY | ffi::JS_EVAL_TYPE_MODULE, - ); + let source = ((*loader).sources.source)(root); + let source = source.unwrap_or_else(|| panic!("cannot read `{root}`")); + let module = eval_module(ctx, &source, root); + assert!(module.tag != ffi::JS_TAG_EXCEPTION, "{}", exception(ctx)); + (*loader).loaded.push((root.clone(), module)); + } - assert!(obj.tag != ffi::JS_TAG_EXCEPTION, "{}", get_exception(ctx)); + let loader = Box::from_raw(loader); + let modules = ffi::JS_NewArray(ctx); + + for (i, (_, module)) in loader.loaded.into_iter().enumerate() { + ffi::JS_SetPropertyUint32(ctx, modules, i as u32, module); + } let mut len = 0; - let buf = ffi::JS_WriteObject(ctx, &mut len, obj, ffi::JS_WRITE_OBJ_BYTECODE); - ffi::JS_FreeValue(ctx, obj); - - assert!(!buf.is_null()); + let buf = ffi::JS_WriteObject(ctx, &mut len, modules, ffi::JS_WRITE_OBJ_BYTECODE); + assert!(!buf.is_null(), "{}", exception(ctx)); let bytes = std::slice::from_raw_parts(buf, len).to_vec(); - // TODO: Are we sure other cleanup is not necessary? + ffi::JS_FreeValue(ctx, modules); ffi::JS_FreeContext(ctx); ffi::JS_FreeRuntime(rt); @@ -40,7 +62,88 @@ pub fn compile(src: &[u8], filename: &str) -> QuickJSBinary { } } -fn get_exception(ctx: *mut ffi::JSContext) -> String { +/// The loader's state, which QuickJS hands back to `load_module` +struct Loader { + sources: Sources, + loaded: Vec<(String, ffi::JSValue)>, // each module that it compiled, by name, in order +} + +/// What QuickJS calls for a module that it has not loaded: `name` is already made from the +/// specifier and the importing module's name. The module is compiled, not run. A failure +/// sets an exception and gives null, and QuickJS passes it on +unsafe extern "C" fn load_module( + ctx: *mut ffi::JSContext, + name: *const c_char, + opaque: *mut c_void, +) -> *mut ffi::JSModuleDef { + unsafe { + let loader = opaque.cast::(); + let text = CStr::from_ptr(name).to_string_lossy().into_owned(); + + if text == (*loader).sources.native { + return native_module(ctx, name, &(*loader).sources.native_exports); + } + + let Some(source) = ((*loader).sources.source)(&text) else { + let fmt = c"cannot find module `%s`".as_ptr(); + ffi::JS_ThrowReferenceError(ctx, fmt, name); + return std::ptr::null_mut(); + }; + + let module = eval_module(ctx, &source, &text); + + if module.tag == ffi::JS_TAG_EXCEPTION { + return std::ptr::null_mut(); + } + + (*loader).loaded.push((text, module)); + module.ptr() + } +} + +/// The module of the natives: a C module, with one export for each native +unsafe fn native_module( + ctx: *mut ffi::JSContext, + name: *const c_char, + exports: &[&str], +) -> *mut ffi::JSModuleDef { + unsafe extern "C" fn init(_: *mut ffi::JSContext, _: *mut ffi::JSModuleDef) -> c_int { + 0 + } + + unsafe { + let module = ffi::JS_NewCModule(ctx, name, init); + + for export in exports { + let export = CString::new(*export).unwrap(); + ffi::JS_AddModuleExport(ctx, module, export.as_ptr()); + } + + module + } +} + +/// Compile `source` as the module `name`, without running it +unsafe fn eval_module(ctx: *mut ffi::JSContext, source: &[u8], name: &str) -> ffi::JSValue { + // JS_Eval requires a '\0' after the last byte of the source + let source_z = [source, &[0]].concat(); + let name = CString::new(name).unwrap(); + let flags = ffi::JS_EVAL_FLAG_COMPILE_ONLY | ffi::JS_EVAL_TYPE_MODULE; + + unsafe { + let module = ffi::JS_Eval( + ctx, + source_z.as_ptr(), + source.len(), + name.as_ptr().cast(), + flags, + ); + debug_assert!(module.tag == ffi::JS_TAG_MODULE || module.tag == ffi::JS_TAG_EXCEPTION); + module + } +} + +fn exception(ctx: *mut ffi::JSContext) -> String { unsafe { let err = ffi::JS_GetException(ctx); let text = ffi::JS_ToCStringLen2(ctx, std::ptr::null_mut(), err, false); diff --git a/compiler/src/libqjs/parse.rs b/compiler/src/libqjs/parse.rs index 7f39ceb..45b0db7 100644 --- a/compiler/src/libqjs/parse.rs +++ b/compiler/src/libqjs/parse.rs @@ -27,8 +27,8 @@ pub mod opcodes { pub struct QuickJSBinary { pub _version: u8, pub _checksum: u32, - pub atoms: Vec, - pub root_mod: Module, + pub atoms: Vec, // one table for every module (see `libqjs::compile`) + pub modules: Vec, // in the order that they were loaded } impl QuickJSBinary { @@ -39,9 +39,16 @@ impl QuickJSBinary { let (bin, atoms) = count(Atom::parse, num_atoms).parse(bin)?; let (bin, value) = JsValue::parse(bin)?; - let JsValue::Module(root_mod) = value else { - panic!() + // An array of the program's modules (see `libqjs::compile`) + let JsValue::Array(items) = value else { + panic!("the binary is not an array of modules") }; + let modules = (items.into_iter()) + .map(|item| match item { + JsValue::Module(module) => module, + _ => panic!("the binary holds a value that is not a module"), + }) + .collect(); Ok(( bin, @@ -49,7 +56,7 @@ impl QuickJSBinary { _version: version, _checksum: checksum, atoms, - root_mod, + modules, }, )) } @@ -533,6 +540,9 @@ pub enum ImmOperand { _target: usize, _arg: u16, }, + /// A module binding, by its index in the program (see link.rs). Only the compiler makes + /// it, for `get_var` and `put_var` + Global(u32), } impl ImmOperand { @@ -763,13 +773,13 @@ impl Function { #[derive(Debug)] pub struct Module { - pub _name: AtomRef, - pub _import_modules: Vec, - pub _exports: Vec, - pub _star_exports: Vec, - pub _imports: Vec, - pub _has_top_level_await: bool, - pub root_fn: Function, + pub name: AtomRef, // the name that it was loaded by (see `libqjs::compile`) + pub requests: Vec, // the modules that it imports from, as the source writes them + pub exports: Vec, + pub star_exports: Vec, // `export * from`: each an index into `requests` + pub imports: Vec, + pub has_top_level_await: bool, + pub root_fn: Function, // the module's body } impl Module { @@ -796,69 +806,76 @@ impl Module { Ok(( bin, Self { - _name: name, - _import_modules: import_modules, - _exports: exports, - _star_exports: star_exports, - _imports: imports, - _has_top_level_await: has_tla == 1, + name, + requests: import_modules, + exports, + star_exports, + imports, + has_top_level_await: has_tla == 1, root_fn, }, )) } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum ExportInfo { - /// All exports except aliases and star exports - Local { _var_idx: usize, _name: AtomRef }, - /// Alias exports, has `from`, eg: export { a } from "./dep.js"; - Alias, + Local { + var_idx: usize, // a captured var of the module's body: its own binding, or an import + name: AtomRef, + }, // `export { x }`, `export let x`, `export function x`, ... + Indirect { + request: usize, // an index into the module's `requests` + local: AtomRef, // the name that the other module exports + name: AtomRef, + }, // `export { x as y } from "..."` } impl ExportInfo { fn parse(bin: &[u8]) -> IResult<&[u8], Self> { let (bin, kind) = u8(bin)?; - let ret = match kind { + match kind { 0 => { let (bin, var_idx) = leb128(bin)?; let (bin, name) = AtomRef::parse(bin)?; - ( - bin, - Self::Local { - _var_idx: var_idx, - _name: name, - }, - ) + Ok((bin, Self::Local { var_idx, name })) } - 1 => (bin, Self::Alias), - _ => unreachable!(), - }; - - Ok(ret) + _ => { + let (bin, request) = leb128(bin)?; + let (bin, local) = AtomRef::parse(bin)?; + let (bin, name) = AtomRef::parse(bin)?; + Ok(( + bin, + Self::Indirect { + request, + local, + name, + }, + )) + } + } } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ImportInfo { - pub _var_idx: usize, - pub _name: AtomRef, - pub _req_module_idx: usize, + pub var_idx: usize, // a captured var of the module's body + pub name: AtomRef, // the name that the other module exports, or `*` for a namespace + pub request: usize, // an index into the module's `requests` } impl ImportInfo { fn parse(bin: &[u8]) -> IResult<&[u8], Self> { let (bin, var_idx) = leb128(bin)?; let (bin, name) = AtomRef::parse(bin)?; - let (bin, req_module_idx) = leb128(bin)?; - + let (bin, request) = leb128(bin)?; Ok(( bin, Self { - _var_idx: var_idx, - _name: name, - _req_module_idx: req_module_idx, + var_idx, + name, + request, }, )) } diff --git a/compiler/src/link.rs b/compiler/src/link.rs new file mode 100644 index 0000000..80a5371 --- /dev/null +++ b/compiler/src/link.rs @@ -0,0 +1,452 @@ +//! Link: what each name in each module refers to, and the program's entry fn +//! +//! Every binding at the top of a module (`let`, `const`, `function`, `class`, ...) is a +//! global: there is one of each for the whole program. A module's body reaches its own +//! bindings and its imports as captured vars, which QuickJS calls module decls and module +//! imports. Link gives each decl a global, and follows each import to the global of the +//! binding that it names (see `Link::resolve`). transform/modules.rs then rewrites the code +//! +//! A name that no module declares, such as `String` or `console`, is a global name: it +//! refers to the export of that name of the prelude, `jsrt:prelude`, as names in JS refer +//! to the global object. A module can declare a binding of the same name, which hides it +//! +//! The natives are the exports of `jsrt:native`, which QuickJS sees as a C module (see +//! `libqjs::compile`) +//! +//! # The entry fn +//! +//! ```js +//! function () { +//! // Phase 1 (`this` is true): each body only makes its top-level function declarations. +//! // Phase 2: each body runs. Every module's phase 1 comes first, so that modules that +//! // import each other see each other's functions +//! let phase = true; +//! while (true) { +//! for (const body of bodies) body.call(phase); +//! if (!phase) break; +//! phase = false; +//! } +//! __drain_microtasks(); +//! } +//! ``` +//! +//! Both phases call each body from one call instr. Infer names the objects that a walk makes +//! by its call site, so with two call instrs, each phase would make its own copy of every +//! object at the top of a module +//! +//! The bodies run in ES's order: each module after the modules that it imports, from the +//! prelude, then each module that the command line names. A body that awaits at its top runs +//! until its first await, so only the last body may await (see transform/async.rs) + +use std::collections::HashMap; + +use crate::{ + flatten::{ENTRY_FN, FlatFn, FlatModule}, + libqjs::parse::{ + Atom, AtomRef, ClosureKind, ClosureVar, ExportInfo, FuncKind, ImmOperand as Imm, Instr, + JsValue, opcodes::OpCode, + }, + native::NATIVES, +}; + +/// The module whose exports are every module's global names +pub const PRELUDE: &str = "jsrt:prelude"; + +/// The module of the natives +pub const NATIVE: &str = "jsrt:native"; + +/// What a name refers to +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Target { + Global(usize), // a binding at the top of a module + Native(usize), // a native, by its index in `native::NATIVES` +} + +pub struct Link { + pub globals: usize, // how many there are + bodies: HashMap)>, // for the fn of each module's body: the module's name, and what each of the body's captured vars refers to + exports: HashMap>, // for each module, by name: each export, by name +} + +impl Link { + /// The name of the module whose body is `f`, if it is a body, and what each captured var + /// of the body refers to + pub fn body(&self, f: usize) -> Option<(&str, &[Target])> { + self.bodies + .get(&f) + .map(|(name, vars)| (&name[..], &vars[..])) + } + + /// What the global name `name` refers to: the prelude's export of that name + pub fn global(&self, name: &str) -> Option { + self.export(PRELUDE, name) + } + + /// The export `name` of the module `module` + pub fn export(&self, module: &str, name: &str) -> Option { + self.exports.get(module)?.get(name).copied() + } + + /// The global of an export that the compiler needs from the stdlib + pub fn stdlib(&self, module: &str, name: &str) -> usize { + match self.export(module, name) { + Some(Target::Global(global)) => global, + _ => panic!("the compiler needs `{name}` from `{module}`"), + } + } +} + +/// Link the modules, and write the entry fn. `roots` are the modules that the command line +/// names, in order +pub fn link( + modules: &[FlatModule], + roots: &[String], + fns: &mut [FlatFn], + captures: &[Vec], + atoms: &[Atom], +) -> Link { + let linker = Linker::new(modules, captures, atoms); + + let link = Link { + globals: linker.globals, + bodies: linker.bodies(), + exports: linker.all_exports(), + }; + + let order = linker.order(roots); + check_top_level_await(&order, modules); + + let drain = link.stdlib("jsrt:loop", "__drain_microtasks"); + fns[ENTRY_FN] = entry(&order, modules, drain); + link +} + +/// A body that awaits returns at its first await, so the bodies after it would run too early. +/// Only the last one may await, until the entry waits for each body's promise +fn check_top_level_await(order: &[usize], modules: &[FlatModule]) { + for &m in &order[..order.len() - 1] { + if modules[m].has_top_level_await { + let name = &modules[m].name; + unimplemented!("{name}: top-level `await` in a module that is not the last to run"); + } + } +} + +/// The natives, as the exports of `jsrt:native` +fn natives() -> HashMap { + let natives = NATIVES.iter().enumerate(); + natives + .map(|(idx, native)| (native.name.to_string(), Target::Native(idx))) + .collect() +} + +struct Linker<'a> { + modules: &'a [FlatModule], + captures: &'a [Vec], + atoms: &'a [Atom], + index: HashMap, // each module, by name + decls: Vec>, // for each module: the global of each of its body's decls, by captured var + globals: usize, +} + +impl<'a> Linker<'a> { + /// The linker, with a global for each decl of each module's body + fn new(modules: &'a [FlatModule], captures: &'a [Vec], atoms: &'a [Atom]) -> Self { + let index = modules.iter().enumerate(); + let mut linker = Linker { + modules, + captures, + atoms, + index: index.map(|(m, module)| (module.name.clone(), m)).collect(), + decls: vec![], + globals: 0, + }; + + linker.add_globals(); + linker + } + + /// For each module's body: the module's name, and what each captured var refers to + fn bodies(&self) -> HashMap)> { + let bodies = self.modules.iter().enumerate(); + bodies + .map(|(m, module)| (module.body, (module.name.clone(), self.body_vars(m)))) + .collect() + } + + /// Every module's exports, by the module's name, with the natives' module + fn all_exports(&self) -> HashMap> { + let modules = self.modules.iter().enumerate(); + let exports = + modules.map(|(m, module)| (module.name.clone(), self.exports(m, &mut vec![]))); + exports.chain([(NATIVE.to_string(), natives())]).collect() + } +} + +impl Linker<'_> { + /// A global for each decl of each module's body. QuickJS makes a namespace import a decl + /// too, whose object its own linker makes at run time, so that is rejected here + fn add_globals(&mut self) { + for module in self.modules { + let atoms = self.atoms; + let namespace = module + .imports + .iter() + .find(|import| import.name.text(atoms) == "*"); + + if namespace.is_some() { + unimplemented!("{}: `import * as` needs namespace objects", module.name); + } + + let vars = self.captures[module.body].iter().enumerate(); + let decls = vars.filter(|(_, var)| var.flags.closure_kind == ClosureKind::ModuleDecl); + let mut globals = HashMap::new(); + + for (i, _) in decls { + globals.insert(i, self.globals); + self.globals += 1; + } + + self.decls.push(globals); + } + } + + /// What each captured var of module `m`'s body refers to + fn body_vars(&self, m: usize) -> Vec { + (0..self.captures[self.modules[m].body].len()) + .map(|i| self.var(m, i, &mut vec![])) + .collect() + } + + /// What captured var `i` of module `m`'s body refers to: its global if it is a decl, else + /// the export that it imports. `seen` guards against imports that go round in a circle + fn var(&self, m: usize, i: usize, seen: &mut Vec<(usize, usize)>) -> Target { + let module = &self.modules[m]; + + if let Some(&global) = self.decls[m].get(&i) { + return Target::Global(global); + } + + assert!( + !seen.contains(&(m, i)), + "{}: an import refers back to itself", + module.name + ); + seen.push((m, i)); + + let import = module.imports.iter().find(|import| import.var_idx == i); + let import = + import.unwrap_or_else(|| panic!("{}: a module var with no import", module.name)); + let name = import.name.text(self.atoms); + let from = self.request(m, import.request); + + match self.resolve(&from, &name, seen) { + Some(target) => target, + None => panic!("{}: `{from}` has no export `{name}`", module.name), + } + } + + /// ES's ResolveExport: what export `name` of module `module` refers to + fn resolve(&self, module: &str, name: &str, seen: &mut Vec<(usize, usize)>) -> Option { + if module == NATIVE { + return natives().get(name).copied(); + } + + let Some(&m) = self.index.get(module) else { + panic!("no module `{module}`"); + }; + + for export in &self.modules[m].exports { + match export { + ExportInfo::Local { var_idx, name: n } if n.text(self.atoms) == name => { + return Some(self.var(m, *var_idx, seen)); + } + ExportInfo::Indirect { + request, + local, + name: n, + } if n.text(self.atoms) == name => { + return self.resolve(&self.request(m, *request), &local.text(self.atoms), seen); + } + _ => {} + } + } + + // `export *` passes on every export but `default` + if name == "default" { + return None; + } + + let mut stars = self.modules[m].star_exports.iter(); + stars.find_map(|&request| self.resolve(&self.request(m, request), name, seen)) + } + + /// Every export of module `m`, by name. `seen` guards against `export *` in a circle + fn exports(&self, m: usize, seen: &mut Vec) -> HashMap { + let module = &self.modules[m]; + let mut exports = HashMap::new(); + + if seen.contains(&m) { + return exports; + } + + seen.push(m); + + for &request in &module.star_exports { + let from = self.request(m, request); + let passed = match self.index.get(&from) { + Some(&other) => self.exports(other, seen), + None if from == NATIVE => natives(), + None => panic!("no module `{from}`"), + }; + + for (name, target) in passed { + if name != "default" { + exports.insert(name, target); + } + } + } + + for export in &module.exports { + let (ExportInfo::Local { name, .. } | ExportInfo::Indirect { name, .. }) = export; + let name = name.text(self.atoms); + let target = self.resolve(&module.name, &name, &mut vec![]); + exports.insert(name, target.unwrap()); + } + + exports + } + + /// The name of request `r` of module `m`. QuickJS made the same name when it loaded it + /// (see `normalize`) + fn request(&self, m: usize, r: usize) -> String { + let module = &self.modules[m]; + normalize(&module.name, &module.requests[r]) + } + + /// The order that the bodies run in: each module after the modules that it imports, from + /// the prelude, then each root + fn order(&self, roots: &[String]) -> Vec { + let mut order = vec![]; + + for root in [PRELUDE.to_string()].iter().chain(roots) { + self.visit(self.index[root], &mut order); + } + + order + } + + fn visit(&self, m: usize, order: &mut Vec) { + if order.contains(&m) { + return; + } + + order.push(m); + let at = order.len() - 1; + + for r in 0..self.modules[m].requests.len() { + if let Some(&other) = self.index.get(&self.request(m, r)) { + self.visit(other, order); + } + } + + // After its imports. It was marked first, so that a circle of imports ends + let m = order.remove(at); + order.push(m); + } +} + +/// The name of the module that `specifier` names, in the module `base`. This is QuickJS's +/// default rule, which QuickJS used to load it: a name that does not start with `.` is as it +/// is. Else it is relative to the folder of `base`, and its leading `./` and `../` are taken +/// out +fn normalize(base: &str, specifier: &str) -> String { + if !specifier.starts_with('.') { + return specifier.to_string(); + } + + let mut folder = match base.rfind('/') { + Some(slash) => base[..slash].to_string(), + None => String::new(), + }; + let mut rest = specifier; + + loop { + if let Some(after) = rest.strip_prefix("./") { + rest = after; + } else if let Some(after) = rest.strip_prefix("../") { + let last = folder.rfind('/').map_or(0, |slash| slash + 1); + + if folder.is_empty() || matches!(&folder[last..], "." | "..") { + break; + } + + folder.truncate(last.saturating_sub(1)); + rest = after; + } else { + break; + } + } + + match folder.is_empty() { + true => rest.to_string(), + false => format!("{folder}/{rest}"), + } +} + +/// The entry fn (see the module doc). Local `k` holds the closure of the `k`th body to run, +/// and the last local is the phase +fn entry(order: &[usize], modules: &[FlatModule], drain: usize) -> FlatFn { + use OpCode::*; + + let n = order.len(); + let phase = Imm::Loc(n as u16); + let body = |k: usize| Imm::Loc(k as u16); + + // The loop starts after the closures and the phase, and ends after a call of each body + // and the phase's test + let head = 2 * n + 2; + let end = head + 4 * n + 5; + + let make = (0..n).flat_map(|k| [(Fclosure, Imm::Const(k as u32)), (PutLoc, body(k))]); + let call = (0..n).flat_map(|k| { + [ + (GetLoc, phase), + (GetLoc, body(k)), + (CallMethod, Imm::Npop(0)), + (Drop, Imm::None), + ] + }); + + let code: Vec<(OpCode, Imm)> = make + .chain([(PushTrue, Imm::None), (PutLoc, phase)]) + .chain(call) + .chain([ + (GetLoc, phase), + (IfFalse, Imm::Label { target: end }), + (PushFalse, Imm::None), + (PutLoc, phase), + (Goto, Imm::Label { target: head }), + (GetVar, Imm::Global(drain as u32)), + (Call0, Imm::None), + (Drop, Imm::None), + (ReturnUndef, Imm::None), + ]) + .collect(); + + let code = code.into_iter().enumerate(); + let code = code.map(|(pc, (op, imm))| Instr { pc, op, imm }).collect(); + let bodies = order + .iter() + .map(|&m| JsValue::FnByIdx(modules[m].body)) + .collect(); + + FlatFn { + name: AtomRef::Predefined(0), + kind: FuncKind::Normal, + arg_count: 0, + local_count: n + 1, + cpool: bodies, + code, + } +} diff --git a/compiler/src/main.rs b/compiler/src/main.rs index bbf7704..c13dbed 100644 --- a/compiler/src/main.rs +++ b/compiler/src/main.rs @@ -2,33 +2,59 @@ mod emit; mod flatten; mod infer; mod libqjs; +mod link; mod native; mod regify; mod transform; -use std::fs::File; -use std::io::Read; +use libqjs::{Sources, compile}; +use link::{NATIVE, PRELUDE}; +use native::NATIVES; -use libqjs::compile; - -/// The stdlib, in the order that it runs in, in front of the program -const STDLIB: &[&[u8]] = &[ - include_bytes!("../../lib/stdlib/globals.js"), - include_bytes!("../../lib/stdlib/object.js"), - include_bytes!("../../lib/stdlib/string.js"), - include_bytes!("../../lib/stdlib/number.js"), - include_bytes!("../../lib/stdlib/boolean.js"), - include_bytes!("../../lib/stdlib/array.js"), - include_bytes!("../../lib/stdlib/loop.js"), - include_bytes!("../../lib/stdlib/promise.js"), - include_bytes!("../../lib/stdlib/coroutine.js"), - include_bytes!("../../lib/stdlib/timers.js"), - include_bytes!("../../lib/stdlib/io.js"), - include_bytes!("../../lib/stdlib/console.js"), +/// The stdlib's modules, by name. The prelude's exports are every module's global names (see +/// link.rs) +const STDLIB: &[(&str, &[u8])] = &[ + ( + "jsrt:prelude", + include_bytes!("../../lib/stdlib/prelude.js"), + ), + ( + "jsrt:globals", + include_bytes!("../../lib/stdlib/globals.js"), + ), + ("jsrt:object", include_bytes!("../../lib/stdlib/object.js")), + ("jsrt:string", include_bytes!("../../lib/stdlib/string.js")), + ("jsrt:number", include_bytes!("../../lib/stdlib/number.js")), + ( + "jsrt:boolean", + include_bytes!("../../lib/stdlib/boolean.js"), + ), + ("jsrt:array", include_bytes!("../../lib/stdlib/array.js")), + ("jsrt:loop", include_bytes!("../../lib/stdlib/loop.js")), + ( + "jsrt:promise", + include_bytes!("../../lib/stdlib/promise.js"), + ), + ( + "jsrt:coroutine", + include_bytes!("../../lib/stdlib/coroutine.js"), + ), + ("jsrt:timers", include_bytes!("../../lib/stdlib/timers.js")), + ("jsrt:io", include_bytes!("../../lib/stdlib/io.js")), + ( + "jsrt:console", + include_bytes!("../../lib/stdlib/console.js"), + ), ]; -/// Runs after the program's top level, or after its first top-level `await` -const STDLIB_END: &[u8] = include_bytes!("../../lib/stdlib/end.js"); +/// The source of a module: a stdlib module by its name, else a file by its path +fn source(name: &str) -> Option> { + match STDLIB.iter().find(|(n, _)| *n == name) { + Some((_, source)) => Some(source.to_vec()), + None if name.starts_with("jsrt:") => None, + None => std::fs::read(name).ok(), + } +} /// The runtime. The emitted C includes `jsrt.h`, which includes the rest const RUNTIME: &[(&str, &[u8])] = &[ @@ -41,34 +67,37 @@ const RUNTIME: &[(&str, &[u8])] = &[ ]; fn main() { - // Load source file - let args = std::env::args().collect::>(); - let filename = args.get(1).unwrap(); + // `compiler ...`: each file is a module, and they run in order. A file's imports + // are found from it. Each is named by its full path, so that its relative imports name + // their files the same way from any folder + let files: Vec = (std::env::args().skip(1)) + .map(|file| match std::fs::canonicalize(&file) { + Ok(path) => path.to_string_lossy().into_owned(), + Err(err) => panic!("cannot read `{file}`: {err}"), + }) + .collect(); - let mut file = File::open(filename).unwrap(); - let metadata = std::fs::metadata(filename).unwrap(); - assert!(metadata.is_file()); + assert!(!files.is_empty(), "usage: compiler ..."); - // The stdlib is compiled with the program, in front of it. The program is the body of an - // async fn of its own, so that its names can shadow the stdlib's, as a script's names - // shadow JS's globals, and so that it can `await` at its top level. `})` is on a line of - // its own, in case the program ends in a comment - let mut src = STDLIB.concat(); - src.reserve(metadata.len() as usize + STDLIB_END.len() + 64); - src.extend_from_slice(b"(async function () {\n"); - file.read_to_end(&mut src).unwrap(); - src.extend_from_slice(b"\n})();\n"); - src.extend_from_slice(STDLIB_END); - - // Compile to bytecode - let bin = compile(&src, filename); + // Compile every module to bytecode: the prelude first, then the files + let roots: Vec = [PRELUDE.to_string()] + .into_iter() + .chain(files.clone()) + .collect(); + let sources = Sources { + source, + native: NATIVE, + native_exports: NATIVES.iter().map(|native| native.name).collect(), + }; + let bin = compile(&roots, sources); // Process it - let (flat_fns, captures, atoms, bindings) = flatten::flatten(bin); - let flat_fns = transform::transform(flat_fns, captures, &atoms); + let (mut flat_fns, captures, atoms, modules) = flatten::flatten(bin); + let link = link::link(&modules, &files, &mut flat_fns, &captures, &atoms); + let flat_fns = transform::transform(flat_fns, captures, &atoms, &link); let fns = regify::regify(flat_fns); - let cache = infer::infer_types(&fns, &bindings); - let c = emit::emit(&fns, &cache, &atoms, &bindings); + let cache = infer::infer_types(&fns, &link); + let c = emit::emit(&fns, &cache, &atoms, &link); // Compile the C, with the runtime, to `a.out` let dir = std::env::temp_dir().join(format!("jsrt-{}", std::process::id())); diff --git a/compiler/src/native.rs b/compiler/src/native.rs index 9a4e7e4..aa864cb 100644 --- a/compiler/src/native.rs +++ b/compiler/src/native.rs @@ -1,18 +1,12 @@ -//! The C fns that JS can call, and the globals +//! The C fns that JS can call //! -//! A native is a C fn of the runtime (lib/rt), which the stdlib calls by name, as a global. Its -//! prefix tells where it comes from (see lib/rt/jsrt.h): +//! A native is a C fn of the runtime (lib/rt). The natives are the exports of the module +//! `jsrt:native`, which the stdlib imports them from (see link.rs). Its prefix tells where it +//! comes from (see lib/rt/jsrt.h): //! - `__native_*`: the platform's IO, which another platform's runtime replaces //! - `__rt_*`: the same on every platform: strings and numbers -//! -//! The only globals are the natives and `undefined`. Everything else that the stdlib defines -//! is a module binding, which the compiler reads by name where it needs one (see -//! `flatten::flatten`) -use crate::{ - infer::Type, - libqjs::parse::{Atom, AtomRef, JsValue}, -}; +use crate::infer::Type; /// A native: one set of param types, and one return type pub struct NativeFn { @@ -80,23 +74,6 @@ pub const NATIVES: &[NativeFn] = natives! { fn __rt_string_from_char_code(Int) -> String; }; -/// The value of the global `name` -pub fn global(name: AtomRef, atoms: &[Atom]) -> JsValue { - let name = name.text(atoms); - - // A global property in JS, but a constant - if name == "undefined" { - return JsValue::Undefined; - } - - match NATIVES.iter().position(|n| name == n.name) { - Some(idx) => JsValue::NativeFn(idx), - None => panic!( - "unknown global `{name}`: the globals are the natives in compiler/src/native.rs, and `undefined`" - ), - } -} - /// Whether a value of type `arg` can be passed for a param of type `param`. JS has one number /// type, so an int and a float both pass for a number pub fn can_pass(arg: &Type, param: &Type) -> bool { diff --git a/compiler/src/transform/async.rs b/compiler/src/transform/async.rs index 5d80a33..0b19499 100644 --- a/compiler/src/transform/async.rs +++ b/compiler/src/transform/async.rs @@ -40,8 +40,10 @@ //! Every kind of fn that suspends lowers the same way: only the driver differs, which is //! stdlib JS (lib/stdlib/coroutine.js). `co` holds what a suspension hands out to it, each //! in a field of its own so that their types stay apart: `value` for a yield, `awaiting` for -//! an await, `result` for the return. The constructor reaches its kind's driver by name, as -//! if the source named it +//! an await, `result` for the return. The constructor reads its kind's driver, an export of +//! `jsrt:coroutine`, as a global (see link.rs) +//! +//! A module's body that awaits at its top is an async fn too (see flatten.rs) //! //! In the step: //! a local, arg or capture the step's capture of it @@ -55,8 +57,7 @@ //! Its label, where the step's first instrs jump for that state, pushes them back. `yield` //! pushes a flag after its value, which is true for `return()`: here it is always false //! -//! Not supported: `yield*`, which QuickJS expands to a loop of iterator ops, and top-level -//! `await`. The module fn is async to QuickJS, but runs as a normal fn (see flatten.rs) +//! Not supported: `yield*`, which QuickJS expands to a loop of iterator ops use super::PROTO_FIELD; use crate::{ @@ -65,19 +66,19 @@ use crate::{ Atom, AtomRef, ClosureKind, ClosureVar, ClosureVarFlags, FuncKind, ImmOperand as Imm, Instr, JsValue, VarKind, atom_names::PREDEFINED_ATOMS, opcodes::OpCode, }, + link::Link, }; -/// The module fn, ``: flatten's first fn -const MODULE_FN: usize = 0; - // ─────────────────────────────────────── The spine ─────────────────────────────────────── -pub fn deasync(fns: &mut Vec, captures: &mut Vec>, atoms: &[Atom]) { - let top_level_await = fns[MODULE_FN].code.iter().any(|i| i.op == OpCode::Await); - assert!(!top_level_await, "top-level `await` is not supported"); - +pub fn deasync( + fns: &mut Vec, + captures: &mut Vec>, + atoms: &[Atom], + link: &Link, +) { let coroutines: Vec = (0..fns.len()) - .filter(|&f| is_coroutine(f, &fns[f])) + .filter(|&f| fns[f].kind != FuncKind::Normal) .collect(); let mut steps = vec![]; @@ -95,9 +96,8 @@ pub fn deasync(fns: &mut Vec, captures: &mut Vec>, atoms let step_captures = step.captures(&own); retarget_children(&cpool, &step, &own, fns, captures); - // 3. Add the step just before the synthetic entry, which stays last (see flatten.rs). - // No fn refers to the entry, so moving it changes no reference - let s = fns.len() - 1; + // 3. Add the step after every other fn + let s = fns.len(); let step_fn = FlatFn { name: fns[f].name, kind: FuncKind::Normal, @@ -106,30 +106,21 @@ pub fn deasync(fns: &mut Vec, captures: &mut Vec>, atoms cpool, code: step.code(), }; - fns.insert(s, step_fn); - captures.insert(s, step_captures); + fns.push(step_fn); + captures.push(step_captures); fns[f].local_count = step.locals; fns[f].cpool = vec![JsValue::FnByIdx(s)]; steps.push(step); } - // 4. Each constructor makes `co`, linked to its kind's driver, and starts it. A link goes - // through the fn's parents, so it waits until every fn is in place - let parents = parents(fns); - + // 4. Each constructor makes `co`, linked to its kind's driver, and starts it for (&f, step) in coroutines.iter().zip(&steps) { - let proto = module_binding(f, driver(fns[f].kind), &parents, captures, atoms); - fns[f].code = constructor(step, proto, atoms); + let driver = link.stdlib("jsrt:coroutine", driver(fns[f].kind)); + fns[f].code = constructor(step, driver, atoms); } } -/// Whether `f` is an async fn or a generator. The module fn is async to QuickJS, but it is -/// not lowered -fn is_coroutine(f: usize, fun: &FlatFn) -> bool { - f != MODULE_FN && fun.kind != FuncKind::Normal -} - /// The stdlib's driver of a kind of fn (see lib/stdlib/coroutine.js) fn driver(kind: FuncKind) -> &'static str { match kind { @@ -531,9 +522,10 @@ fn rename_forwards( // ──────────────────────────────────── The constructor ──────────────────────────────────── -/// The constructor's code: keep `this` and the fn itself, make `co` with the step, set the -/// state to 0, and return what the driver's `start` gives -fn constructor(step: &Step, proto: usize, atoms: &[Atom]) -> Vec { +/// The constructor's code: keep `this` and the fn itself, make `co` with the step and the +/// driver, which is the global `driver`, set the state to 0, and return what the driver's +/// `start` gives +fn constructor(step: &Step, driver: usize, atoms: &[Atom]) -> Vec { use OpCode::*; let field = |text: &str| Imm::Atom(find_atom(text, atoms)); @@ -545,7 +537,7 @@ fn constructor(step: &Step, proto: usize, atoms: &[Atom]) -> Vec { (SpecialObject, Imm::U8(2)), (PutLoc, loc(step.callee)), (Object, Imm::None), - (GetVarRef, Imm::VarRef(proto as u16)), + (GetVar, Imm::Global(driver as u32)), (DefineField, Imm::Atom(PROTO_FIELD)), (Fclosure, Imm::Const(0)), (DefineField, field("step")), @@ -690,53 +682,3 @@ fn successors(code: &[Instr], i: usize) -> Vec { next } - -// ───────────────────────────────── Links to the drivers ────────────────────────────────── - -/// The capture of `f` that reads the module binding `name`, made if needed, with a forward in -/// each fn between the module fn and `f` -fn module_binding( - f: usize, - name: &str, - parents: &[usize], - captures: &mut [Vec], - atoms: &[Atom], -) -> usize { - if f == MODULE_FN { - for (i, c) in captures[MODULE_FN].iter().enumerate() { - // Every capture of the module fn is a module binding (see flatten.rs) - if c.name.text(atoms) == name { - return i; - } - } - - panic!("async fns and generators need `{name}` from the stdlib (lib/stdlib/coroutine.js)"); - } - - let i = module_binding(parents[f], name, parents, captures, atoms); - - for (j, c) in captures[f].iter().enumerate() { - if c.flags.closure_kind == ClosureKind::Ref && c.var_idx == i { - return j; - } - } - - let atom = captures[parents[f]][i].name; - captures[f].push(closure_var(atom, i, ClosureKind::Ref)); - captures[f].len() - 1 -} - -/// The parent of each fn: the fn whose constant pool holds it -fn parents(fns: &[FlatFn]) -> Vec { - let mut parents = vec![usize::MAX; fns.len()]; - - for (p, f) in fns.iter().enumerate() { - for c in &f.cpool { - if let &JsValue::FnByIdx(child) = c { - parents[child] = p; - } - } - } - - parents -} diff --git a/compiler/src/transform/closure.rs b/compiler/src/transform/closure.rs index d975dcb..3af06be 100644 --- a/compiler/src/transform/closure.rs +++ b/compiler/src/transform/closure.rs @@ -93,8 +93,7 @@ //! a capture: transform/async.rs makes it, when a binding that it moves out of a fn's frame //! needs a new location for each loop iteration //! -//! A module binding is a local of the synthetic entry fn, which makes `` once (see -//! flatten.rs), so there is one location for each module binding +//! A module binding is not a capture: modules.rs made it a global first //! //! In the emitted C, every underlying "plain" function is called as: //! `fn(env, this, arg_0, arg_1, ... arg_n)` @@ -192,7 +191,7 @@ pub fn try_replace_closure( /// If `instr` reads or writes capture `i` of its fn: `i`, whether it writes, and whether a /// write also leaves the value on the stack -fn var_ref_access(instr: &Instr) -> Option<(usize, bool, bool)> { +pub(super) fn var_ref_access(instr: &Instr) -> Option<(usize, bool, bool)> { use OpCode::*; let i = match instr.imm { diff --git a/compiler/src/transform/expand.rs b/compiler/src/transform/expand.rs index bf208f4..afc6403 100644 --- a/compiler/src/transform/expand.rs +++ b/compiler/src/transform/expand.rs @@ -4,10 +4,7 @@ use super::{ENV_ARG, PROTO_FIELD}; -use crate::{ - libqjs::parse::{Atom, ImmOperand as Imm, Instr, JsValue, opcodes::OpCode}, - native, -}; +use crate::libqjs::parse::{ImmOperand as Imm, Instr, opcodes::OpCode}; #[allow(non_snake_case)] pub fn try_replace_expansion(instr: &Instr, out: &mut Vec) -> bool { @@ -147,25 +144,3 @@ pub fn try_replace_expansion(instr: &Instr, out: &mut Vec) -> bool { true } - -/// Replace a read of a global with the constant it is (see `native`) -pub fn replace_get_var( - instr: &Instr, - atoms: &[Atom], - cpool: &mut Vec, - out: &mut Vec, -) -> bool { - if instr.op != OpCode::GetVar { - return false; - } - - cpool.push(native::global(instr.imm.atom(), atoms)); - - out.push(Instr { - pc: instr.pc, - op: OpCode::PushConst, - imm: Imm::Const(cpool.len() as u32 - 1), - }); - - true -} diff --git a/compiler/src/transform/mod.rs b/compiler/src/transform/mod.rs index 95deca8..5d2ec78 100644 --- a/compiler/src/transform/mod.rs +++ b/compiler/src/transform/mod.rs @@ -12,7 +12,7 @@ //! - define_method //! - check_ctor //! - special_object -//! - get_var +//! - get_var_undef, put_var_init //! - push_this //! - get_arg{0, 1, 2, 3} //! - put_arg{0, 1, 2, 3} @@ -26,6 +26,8 @@ //! //! //! The following instructions take on a different meaning: +//! - get_var, put_var: a read or write of a binding at the top of a module, which is a +//! global (see modules.rs) //! - call_constructor (see new.rs) //! - close_loc (see closure.rs) //! @@ -33,7 +35,10 @@ //! - define_field_ref (see closure.rs) //! - close_field (see closure.rs) //! -//! Async fns and generators become a constructor and a step closure first (see async.rs), +//! Module bindings become globals first (see modules.rs): `get_var` and `put_var` then read +//! and write one, by an `ImmOperand::Global` +//! +//! Async fns and generators become a constructor and a step closure next (see async.rs), //! so no instr of theirs is left: initial_yield, yield, await, return_async in them, and //! close_var_ref, which deasync makes and closure.rs lowers @@ -42,18 +47,21 @@ mod r#async; mod class; mod closure; mod expand; +mod modules; mod new; use crate::{ flatten::FlatFn, libqjs::parse::{Atom, AtomRef, ClosureVar}, + link::Link, }; use args::try_replace_arg_access; use r#async::deasync; use class::declass; use closure::try_replace_closure; -use expand::{replace_get_var, try_replace_expansion}; +use expand::try_replace_expansion; +use modules::globalize; use new::try_replace_new; // Object Prototype Rules: @@ -85,25 +93,24 @@ pub fn transform( mut flat_fns: Vec, mut captures: Vec>, atoms: &[Atom], + link: &Link, ) -> Vec { - // Async fns and generators become state machines first: this adds fns, and changes the + // Module bindings become globals first: this drops captured vars + globalize(&mut flat_fns, &mut captures, link, atoms); + + // Async fns and generators become state machines next: this adds fns, and changes the // captures that the other transforms read - deasync(&mut flat_fns, &mut captures, atoms); + deasync(&mut flat_fns, &mut captures, atoms, link); flat_fns .into_iter() .enumerate() - .map(|(idx, f)| transform_one(f, &captures[idx], &captures, atoms)) + .map(|(idx, f)| transform_one(f, &captures[idx], &captures)) .collect() } /// `own` is the captures of `f` -fn transform_one( - mut f: FlatFn, - own: &[ClosureVar], - captures: &[Vec], - atoms: &[Atom], -) -> FlatFn { +fn transform_one(f: FlatFn, own: &[ClosureVar], captures: &[Vec]) -> FlatFn { // Transform class declarations let code = declass(&f.code); @@ -132,10 +139,6 @@ fn transform_one( continue; } - if replace_get_var(instr, atoms, &mut f.cpool, &mut out) { - continue; - } - if try_replace_closure(instr, own, captures, &f.cpool, &mut out) { continue; } diff --git a/compiler/src/transform/modules.rs b/compiler/src/transform/modules.rs new file mode 100644 index 0000000..25ee64c --- /dev/null +++ b/compiler/src/transform/modules.rs @@ -0,0 +1,198 @@ +//! Module bindings become globals (see link.rs) +//! +//! A module's body reaches its bindings and imports as captured vars, and so does each fn in +//! it, through its parents. Link found what each of the body's captured vars refers to. Here +//! each read and write of one becomes a read or write of a global, with `get_var` and +//! `put_var` on an `ImmOperand::Global`. A native is a constant +//! +//! get_var_ref k get_var g (or push_const of a native) +//! put_var_ref k put_var g +//! set_var_ref k dup; put_var g +//! +//! The `_check` forms are the same: the TDZ is not checked. Those captured vars are then +//! dropped, so a closure no longer holds module bindings, and the captured vars that are +//! left are numbered again +//! +//! A global name, such as `String`, is a `get_var` of its name. It becomes the global or the +//! native that the prelude exports by that name. `typeof` of a name that nothing declares is +//! a `get_var_undef`, which gives undefined. A write of a global name is not allowed + +use super::closure::var_ref_access; +use crate::{ + flatten::{ENTRY_FN, FlatFn}, + libqjs::parse::{ + Atom, ClosureKind, ClosureVar, ImmOperand as Imm, Instr, JsValue, opcodes::OpCode, + }, + link::{Link, Target}, +}; + +pub fn globalize( + fns: &mut [FlatFn], + captures: &mut [Vec], + link: &Link, + atoms: &[Atom], +) { + // Each fn's parent goes first, because a capture names a captured var of its parent. Each + // fn also gets the name of its module, for errors + let mut stack = vec![(ENTRY_FN, vec![], String::new())]; + + while let Some((f, parent, module)) = stack.pop() { + let module = link.body(f).map_or(module, |(name, _)| name.to_string()); + let targets = targets(f, &captures[f], &parent, link); + let renumber = renumber(&targets); + + captures[f] = keep(&captures[f], &targets, &parent); + fns[f].code = rewrite(&mut fns[f], &targets, &renumber, link, atoms, &module); + + let children = fns[f].cpool.iter().filter_map(|c| match c { + JsValue::FnByIdx(child) => Some(*child), + _ => None, + }); + let own = targets.into_iter().zip(renumber).map(|(t, r)| Slot { + target: t, + index: r, + }); + let own: Vec = own.collect(); + + for child in children.collect::>() { + stack.push((child, own.clone(), module.clone())); + } + } +} + +/// A captured var of a fn, after this pass: the binding that it refers to, if that is a +/// module binding, else its new index +#[derive(Clone, Copy)] +struct Slot { + target: Option, + index: Option, +} + +/// What each captured var of `f` refers to, if a module binding. A module's body's captured +/// vars are all module bindings. Any other fn's capture refers to one if the captured var of +/// its parent does +fn targets(f: usize, own: &[ClosureVar], parent: &[Slot], link: &Link) -> Vec> { + if let Some((_, vars)) = link.body(f) { + return vars.iter().map(|&target| Some(target)).collect(); + } + + (own.iter()) + .map(|var| match var.flags.closure_kind { + ClosureKind::Ref => parent[var.var_idx].target, + _ => None, + }) + .collect() +} + +/// The new index of each captured var that stays +fn renumber(targets: &[Option]) -> Vec> { + let mut next = 0; + + (targets.iter()) + .map(|target| match target { + Some(_) => None, + None => { + next += 1; + Some(next - 1) + } + }) + .collect() +} + +/// The captured vars that stay. A capture of the parent's captured var gets its new index +fn keep(own: &[ClosureVar], targets: &[Option], parent: &[Slot]) -> Vec { + let stay = own + .iter() + .zip(targets) + .filter(|(_, target)| target.is_none()); + + stay.map(|(var, _)| { + let mut var = var.clone(); + + if var.flags.closure_kind == ClosureKind::Ref { + var.var_idx = parent[var.var_idx].index.unwrap(); + } + + var + }) + .collect() +} + +/// The code of `f`, with module bindings as globals, and the captured vars that stay at their +/// new indices +fn rewrite( + f: &mut FlatFn, + targets: &[Option], + renumber: &[Option], + link: &Link, + atoms: &[Atom], + module: &str, +) -> Vec { + use OpCode::*; + + let mut out = vec![]; + let code = std::mem::take(&mut f.code); + + for instr in code { + let pc = instr.pc; + let mut push = |op, imm| out.push(Instr { pc, op, imm }); + + if let Some((i, writes, leaves)) = var_ref_access(&instr) { + match (targets[i], writes) { + (Some(target), false) => read(target, &mut f.cpool, &mut push), + (Some(Target::Global(global)), true) => { + if leaves { + push(Dup, Imm::None); + } + + push(PutVar, Imm::Global(global as u32)); + } + (Some(Target::Native(_)), true) => panic!("a native cannot be assigned to"), + (None, _) => { + let op = match (writes, leaves) { + (false, _) => GetVarRef, + (true, false) => PutVarRef, + (true, true) => SetVarRef, + }; + push(op, Imm::VarRef(renumber[i].unwrap() as u16)); + } + } + + continue; + } + + match (instr.op, instr.imm) { + (GetVar | GetVarUndef, Imm::Atom(name)) => { + let text = name.text(atoms); + + match (link.global(&text), instr.op) { + (Some(target), _) => read(target, &mut f.cpool, &mut push), + (None, GetVarUndef) => push(Undefined, Imm::None), + (None, _) => { + panic!("{module}: `{text}` is not declared, and is not a global name") + } + } + } + (PutVar | PutVarInit, Imm::Atom(name)) => { + panic!( + "{module}: `{}` is a global name, which cannot be assigned to", + name.text(atoms) + ) + } + _ => out.push(instr), + } + } + + out +} + +/// A read of what a module binding refers to: a global, or a native as a constant +fn read(target: Target, cpool: &mut Vec, push: &mut impl FnMut(OpCode, Imm)) { + match target { + Target::Global(global) => push(OpCode::GetVar, Imm::Global(global as u32)), + Target::Native(idx) => { + cpool.push(JsValue::NativeFn(idx)); + push(OpCode::PushConst, Imm::Const(cpool.len() as u32 - 1)); + } + } +} diff --git a/docs/model.md b/docs/model.md index 1c50dc1..ac945e0 100644 --- a/docs/model.md +++ b/docs/model.md @@ -110,24 +110,24 @@ The same thing happens on a large scale with partial types. Inside a walk, a cal ## 5. Representation in C -The C type of a value comes from its type only: +The C type of a value comes from its type only. A type with one member is that member's C type: -| type | C | +| member | C | |---|---| -| `Unknown` | the code cannot run | | `Undefined`, `Null` | `undefined_t`, `null_t`: unit types | | `Bool` | `bool` | | `Int` | `int32_t` | | `Float` | `double` | | `Str` | `string_t` | -| `Obj(cell)` | a pointer to the struct of the cell. The struct starts with a header that holds the cell's number | -| `Str` or one `Obj`, plus `Undefined` and/or `Null` | that pointer. `NULL` is null, and `JSRT_UNDEFINED` is undefined | -| `Obj` of several cells, maybe with `Undefined` and `Null` | a pointer to the header | -| any other sum, and `Any` | `any_t`: a tagged union of every kind | +| `Native` | `native_t`: the native's index | +| `Obj` | a pointer to the struct of the object's class | -- **A field that can be absent** holds a sentinel for absent: `JSRT_ABSENT` in a pointer, a tag in an `any_t`. -- **An op on a sum** tests which member the value is (by kind, by a sentinel, or by the cell's number in the header), and then works on that member. -- **An array's struct** has `length`, `cap` and `items` after its header. An element holds undefined for a hole. +A type with several members is a `jsrt_value`: a tag, which says the kind, and the value. A number in a `jsrt_value` is always a double, and an object's tag also says its class. `Unknown` means that the code cannot run. + +- **The C of an op** is a macro of the runtime (lib/rt/core.h), which picks its code by the C types of its operands. A plain C value becomes a `jsrt_value` with a constant tag, so the C compiler removes the tag and keeps only the code for that kind. +- **A field that can be absent** has a `has_` flag beside it. Getting it calls a lookup, which goes along the prototype chain. +- **An op that depends on an object's class**, such as getting a field or calling a closure, tests the tag of a `jsrt_value` with a `switch`, and has code for each class. +- **An array's struct** has `length`, `cap` and `items`. An element holds undefined for a hole. ## 6. Not in this model yet diff --git a/lib/stdlib/array.js b/lib/stdlib/array.js index 4bddbda..5748477 100644 --- a/lib/stdlib/array.js +++ b/lib/stdlib/array.js @@ -8,7 +8,9 @@ // - Holes are not skipped: a callback sees undefined for a hole // `Array(n)` and `new Array(n)` make an array of `n` holes -function Array(n) { +import { relativeIndex } from "jsrt:string"; + +export function Array(n) { const a = []; if (n !== undefined) a.length = n; return a; diff --git a/lib/stdlib/boolean.js b/lib/stdlib/boolean.js index d5576eb..fb7a061 100644 --- a/lib/stdlib/boolean.js +++ b/lib/stdlib/boolean.js @@ -1,5 +1,5 @@ // Converts any value to a boolean, like the spec's ToBoolean -function Boolean(x) { +export function Boolean(x) { return !!x; } diff --git a/lib/stdlib/console.js b/lib/stdlib/console.js index fdcf2a7..dede52a 100644 --- a/lib/stdlib/console.js +++ b/lib/stdlib/console.js @@ -1,6 +1,8 @@ // Not as the spec says: it prints its first arg only, because more args need // rest params, which need arrays -const console = { +import { __native_print_string } from "jsrt:native"; + +export const console = { log(x) { // As node does: `-0` prints as `-0`, though String(-0) is "0" const negativeZero = typeof x === "number" && x === 0 && 1 / x < 0; diff --git a/lib/stdlib/coroutine.js b/lib/stdlib/coroutine.js index 4f57413..a87a33f 100644 --- a/lib/stdlib/coroutine.js +++ b/lib/stdlib/coroutine.js @@ -13,8 +13,10 @@ // Not supported yet: `throw()` on a generator, and `finally` blocks in // `return()`, which need exceptions +import { __is_promise, __reject, __resolve } from "jsrt:promise"; + // A generator's object is `co` itself -const __GeneratorProto = { +export const __GeneratorProto = { // The body runs up to its `initial_yield`, after its params start() { this.step(undefined); @@ -39,7 +41,7 @@ const __GeneratorProto = { // An async fn returns a promise, which its return fulfils. A rejected await // rejects it, and the rest of the body does not run, as an uncaught throw at the // await would -const __AsyncProto = { +export const __AsyncProto = { // The body runs up to its first await, in the call start() { const promise = new Promise(undefined); diff --git a/lib/stdlib/end.js b/lib/stdlib/end.js deleted file mode 100644 index b212ed7..0000000 --- a/lib/stdlib/end.js +++ /dev/null @@ -1,3 +0,0 @@ -// Runs after the program's top level: its microtasks. The host then runs -// the event loop (see loop.js) -__drain_microtasks(); diff --git a/lib/stdlib/globals.js b/lib/stdlib/globals.js index ccf1c64..40c1bd0 100644 --- a/lib/stdlib/globals.js +++ b/lib/stdlib/globals.js @@ -1,20 +1,7 @@ -// The stdlib, compiled in front of every program. Its files are joined in the -// order of `STDLIB` in compiler/src/main.rs, and this one is first -// -// Built on the natives in compiler/src/native.rs, the common layer of -// operations on primitive values. The compiler's own code for operators (`+` on -// strings and so on) uses those natives directly, not these functions. -// -// Restrictions: -// - The prototypes of `String`, `Number`, `Boolean` and `Array` are fixed: -// the compiler reads them by name, as the prototypes of the primitives and of -// arrays. User code may not change them (no monkey-patching) -// - `new String(x)`, `new Number(x)` and `new Boolean(x)` (wrapper objects) are -// not supported. These functions only convert -// -// Needs `typeof`. `String.prototype.concat` is not here, because it needs rest -// params, which need arrays. `console.log` prints one arg only, for the same -// reason +// The global values that are not functions. `undefined` is a binding like any +// other: a global name refers to the prelude's export of that name (see +// prelude.js) -const NaN = 0 / 0; -const Infinity = 1 / 0; +export const undefined = void 0; +export const NaN = 0 / 0; +export const Infinity = 1 / 0; diff --git a/lib/stdlib/io.js b/lib/stdlib/io.js index 4c12b5f..d094dc5 100644 --- a/lib/stdlib/io.js +++ b/lib/stdlib/io.js @@ -5,6 +5,18 @@ // A failure rejects with the negative errno, not an `Error`. A read or a write // that the kernel does not finish in one op is not continued +import { + __native_io_queue_close, + __native_io_queue_open, + __native_io_queue_read, + __native_io_queue_stat, + __native_io_queue_write, + __native_io_result_number, + __native_io_result_string, + __native_io_status, +} from "jsrt:native"; +import { __pending } from "jsrt:loop"; + // A promise of what op `h` gives when it completes: its status, its data, or // a number. A result can only be read before the next wait, so these read it // when the op completes @@ -39,7 +51,7 @@ function __io_ok(h) { return __io_status(h).then((status) => (status < 0 ? Promise.reject(status) : status)); } -const fs = { +export const fs = { readFile(path) { let fd = -1; let text = ""; diff --git a/lib/stdlib/loop.js b/lib/stdlib/loop.js index defb2d4..0de2501 100644 --- a/lib/stdlib/loop.js +++ b/lib/stdlib/loop.js @@ -1,12 +1,12 @@ // The event loop's JS side. The host owns the loop (see lib/rt/jsrt.h). A // macrotask is an IO op or a timer, which the host runs: it calls // `__complete(h)` when op `h` is done. A microtask is a promise job, in -// `__microtasks`. After the program's top level, end.js runs all microtasks, -// and after each macrotask, `__complete` does +// `__microtasks`. After every module's body, the entry runs all microtasks +// (see compiler/src/link.rs), and after each macrotask, `__complete` does // What to do when IO op `h` completes: `__pending[h]`, a fn that gets `h`. // Handles are small ints, and a handle is used again after its op completes -const __pending = []; +export const __pending = []; // Promise jobs, in order. `head` is the next one, so that taking a job does // not move the others. A job is a closure, which keeps its own data: the @@ -14,11 +14,11 @@ const __pending = []; // values of its own promise (see `docs/model.md`, section 3.3) const __microtasks = { jobs: [], head: 0 }; -function __queue_job(job) { +export function __queue_job(job) { __microtasks.jobs.push(job); } -function __drain_microtasks() { +export function __drain_microtasks() { const q = __microtasks; while (q.head < q.jobs.length) { @@ -33,8 +33,8 @@ function __drain_microtasks() { } // What the host calls when op `h` is done: its callback, then the microtasks. -// The compiler finds it by name -function __complete(h) { +// The compiler finds it as this module's export +export function __complete(h) { const done = __pending[h]; __pending[h] = undefined; if (done !== undefined) done(h); diff --git a/lib/stdlib/number.js b/lib/stdlib/number.js index ceb912d..9f821a8 100644 --- a/lib/stdlib/number.js +++ b/lib/stdlib/number.js @@ -1,5 +1,13 @@ +import { + __rt_number_to_fixed, + __rt_number_to_string, + __rt_string_concat, + __rt_string_from_char_code, + __rt_string_to_number, +} from "jsrt:native"; + // Converts any value to a number, like the spec's ToNumber -function Number(x) { +export function Number(x) { switch (typeof x) { case "number": return x; case "string": return __rt_string_to_number(x); diff --git a/lib/stdlib/object.js b/lib/stdlib/object.js index c71f7f8..109d2c7 100644 --- a/lib/stdlib/object.js +++ b/lib/stdlib/object.js @@ -1,5 +1,5 @@ // `new Object()` makes an empty object. Called without `new`, it does the same -function Object() { +export function Object() { return {}; } diff --git a/lib/stdlib/prelude.js b/lib/stdlib/prelude.js new file mode 100644 index 0000000..2ff7a73 --- /dev/null +++ b/lib/stdlib/prelude.js @@ -0,0 +1,42 @@ +// The prelude: its exports are the global names of every module, such as +// `String` or `console`. A module that does not declare a name, and does not +// import it, refers to the prelude's export of that name. A module can declare +// a binding of the same name, which hides it (see compiler/src/link.rs) +// +// The stdlib is built on the natives in compiler/src/native.rs, the common +// layer of operations on primitive values, which `jsrt:native` exports. The +// compiler's own code for operators (`+` on strings and so on) uses the runtime +// directly, not these functions. +// +// Restrictions: +// - The prototypes of `String`, `Number`, `Boolean` and `Array` are fixed: +// the compiler reads them as the prelude's exports, as the prototypes of the +// primitives and of arrays. User code may not change them (no monkey-patching) +// - `new String(x)`, `new Number(x)` and `new Boolean(x)` (wrapper objects) are +// not supported. These functions only convert +// - A global name cannot be assigned to +// +// `String.prototype.concat` is not here, because it needs rest params, which +// need arrays. `console.log` prints one arg only, for the same reason +// +// The natives are global names too, as the runtime's own fns, with `__pending`, +// where an IO op's callback waits for the host (see loop.js). The compiler +// needs `jsrt:coroutine` for async fns and generators, so the prelude loads it. +// It finds the host's `__complete` in `jsrt:loop` +// +// The modules run in the order of these imports, before any other module + +export { undefined, NaN, Infinity } from "jsrt:globals"; +export { Object } from "jsrt:object"; +export { String } from "jsrt:string"; +export { Number } from "jsrt:number"; +export { Boolean } from "jsrt:boolean"; +export { Array } from "jsrt:array"; +export { Promise, queueMicrotask } from "jsrt:promise"; +export { setTimeout, clearTimeout } from "jsrt:timers"; +export { fs } from "jsrt:io"; +export { console } from "jsrt:console"; +export * from "jsrt:native"; +export { __pending } from "jsrt:loop"; + +import "jsrt:coroutine"; diff --git a/lib/stdlib/promise.js b/lib/stdlib/promise.js index bcbbb94..92998bc 100644 --- a/lib/stdlib/promise.js +++ b/lib/stdlib/promise.js @@ -15,12 +15,14 @@ // - A fn that makes a closure keeps out of the common path: a captured arg is // in a box, which every call from one place shares +import { __queue_job } from "jsrt:loop"; + // The states of a promise const __PENDING = 0; const __FULFILLED = 1; const __REJECTED = 2; -function Promise(executor) { +export function Promise(executor) { const p = { __proto__: this.__proto__, state: __PENDING, value: undefined, reason: undefined, resolved: false, reactions: [] }; if (executor !== undefined) { @@ -122,19 +124,19 @@ Promise.prototype = { }, }; -function __is_promise(x) { +export function __is_promise(x) { return typeof x === "object" && x !== null && x.reactions !== undefined; } // Resolve `p` with `value`, unless it was resolved or rejected before -function __resolve(p, value) { +export function __resolve(p, value) { if (p.resolved) return; p.resolved = true; __fulfil_or_adopt(p, value); } // Reject `p` with `reason`, unless it was resolved or rejected before -function __reject(p, reason) { +export function __reject(p, reason) { if (p.resolved) return; p.resolved = true; __settle_rejected(p, reason); @@ -194,6 +196,6 @@ function __adopt(p, thenable) { ); } -function queueMicrotask(f) { +export function queueMicrotask(f) { __queue_job(f); } diff --git a/lib/stdlib/string.js b/lib/stdlib/string.js index 03be20e..a6895f0 100644 --- a/lib/stdlib/string.js +++ b/lib/stdlib/string.js @@ -1,5 +1,13 @@ +import { + __rt_number_to_string, + __rt_string_char_code_at, + __rt_string_concat, + __rt_string_from_char_code, + __rt_string_slice, +} from "jsrt:native"; + // Converts any value to a string, like the spec's ToString -function String(x) { +export function String(x) { switch (typeof x) { case "string": return x; case "number": return __rt_number_to_string(x); @@ -109,7 +117,7 @@ function clampIndex(i, len) { } // An index where a negative value counts from the end, clamped to [0, len] -function relativeIndex(i, len) { +export function relativeIndex(i, len) { return clampIndex(i < 0 ? i + len : i, len); } diff --git a/lib/stdlib/timers.js b/lib/stdlib/timers.js index 72bb971..9590e91 100644 --- a/lib/stdlib/timers.js +++ b/lib/stdlib/timers.js @@ -4,7 +4,10 @@ // // Not as the spec says: `setTimeout` takes no args for the callback -function setTimeout(f, ms) { +import { __native_io_queue_cancel, __native_io_queue_timeout } from "jsrt:native"; +import { __pending } from "jsrt:loop"; + +export function setTimeout(f, ms) { // As in node, a delay below 1 ms is 1 ms const delay = ms === undefined || !(ms >= 1) ? 1 : ms; const timer = { handle: __native_io_queue_timeout(delay), active: true }; @@ -18,7 +21,7 @@ function setTimeout(f, ms) { return timer; } -function clearTimeout(timer) { +export function clearTimeout(timer) { if (timer === undefined || timer === null || !timer.active) return; timer.active = false; __native_io_queue_cancel(timer.handle);