diff --git a/examples/lisp/CLAUDE.md b/examples/lisp/CLAUDE.md index a71736d4..28029e06 100644 --- a/examples/lisp/CLAUDE.md +++ b/examples/lisp/CLAUDE.md @@ -9,7 +9,7 @@ not as a reference Lisp implementation. eval, apply, printer, REPL. Everything the entry point and the test driver consume is `export`-ed. - `lisp.ww` entry point; `use lispcore;` + `main()`. -- `lisp_test.ww` in-process test driver (66 probes). Built as a +- `lisp_test.ww` in-process test driver (72 probes). Built as a standalone binary, exec'd directly — `ww test` drops `-I` in single-file mode, so the Makefile runs the binary itself. @@ -89,6 +89,17 @@ same bug class shows up in any new code that hits the same pattern. Integer compound assigns work fine, so `acc += i` on `i64` stays as-is. +9. **`let r = call(); foreign_call(); return r?;` corrupts `r` when + the call returned a tagged union.** The (`tag`, `payload1`, + `payload2`) triple sits in AX/DX/CX after the call, and the + foreign call between capture and `?`-unwrap clobbers at least one + register before the cgen has spilled it to the local slot. + Symptom: a `(*value | rterror)` whose `rterror` carries a string + literal prints with a `str.len` of tens of thousands. Workaround: + `match` the union inline before the foreign call and let each arm + return its own typed result. See eval's BUILTIN apply path + (where we free the args slice after `apply_builtin`). + If a new function "should work but acts weird", the bug is almost always one of the above and shows up under valgrind/gdb the same way it did the first time: silently dropped store, missing field @@ -115,17 +126,22 @@ env where `od?` is unbound; the tie-back only adds the self-binding. ## Interpreter limitations (design, not bug) -- **No GC.** Cons / value / env cells come from a single bump- - pointer arena (`arena_alloc`, 64 KiB chunks chained via `os.alloc`). - The arena never reclaims — long REPL sessions still leak — but - cells pack tightly instead of one cell per 4 KiB mmap. test_huge - peaks at ~253 MB under `--pages-as-heap=yes`, down from ~525 MB - pre-arena. The remaining bulk is `append(xs, av)` in eval's arg - loop: every apply allocates a fresh `[]*value` through `rt_ensure`, - which still hits `rt_alloc` page-per-call. Per-top-level-form - arena reset (perm/trans split + Cheney promote on define/set!) is - the next step — see the closing note in `repl()` for the hook - point. +- **Arena, not GC.** Cells live in two bump-pointer arenas: `perm` + (top-level definitions and the value graph each one pins) and + `trans` (parse cells, intermediate evals, the form's printed + result). `repl()` calls `arena_reset_trans()` between top-level + forms; a Cheney-style `promote_v` deep-copies the value graph at + every top-level `define`/`set!` boundary so no perm cell ever + points into trans. Forwarding markers (`pin = -1` plus the new + perm ptr stashed in `.car`/`.val`) break cycles. Detached trans + chunks go onto a per-arena free list, so peak virtual address + space is bounded by the largest form's working set. Builtin + args-slice headers in eval's apply path are released with + `os.free` per dispatch — without that, `rt_ensure`'s page-per- + call allocation dominates the profile. test_huge peaks at ~2.6 MB + under `--pages-as-heap=yes`, down from ~525 MB pre-arena (~200×). + No real GC inside a single form, so a pathological one-shot like + `(fib 25)` would still grow trans linearly until the form returns. - **No bigints.** `i64` wraps silently on overflow. `(fact 21)` rolls over. - **Float printing is fixed `%.6f`.** `1.0` prints as `1.000000`. @@ -155,7 +171,7 @@ env where `od?` is unbound; the tie-back only adds the self-binding. ``` make # build ./lisp -make test # build + run lisp_test (66 in-process probes) +make test # build + run lisp_test (72 in-process probes) make demo # cat each test_*.lisp through ./lisp make clean ``` diff --git a/examples/lisp/lispcore.ww b/examples/lisp/lispcore.ww index 491ae1ed..dd8a75c2 100644 --- a/examples/lisp/lispcore.ww +++ b/examples/lisp/lispcore.ww @@ -20,8 +20,9 @@ // - classic `for (cond)` + `break` // - `defer` to release the slurp buffer on every exit edge // - runes ('a'), str literals, f64 arithmetic, mixed i64/f64 promo -// - a chunked bump-pointer arena (arena_alloc) over os.alloc, so -// every cell isn't paying a 4 KiB-page-per-allocation tax +// - a chunked perm/trans bump arena with Cheney-style promotion +// at top-level define/set! boundaries; trans resets between +// forms so deep eval graphs don't keep growing process RSS // - module imports (use os/fmt/ascii/strconv/strings) // // Build: make @@ -88,38 +89,53 @@ export type value = struct { fval: f64, // FLOAT sid: i32, // SYM — interner id text: str, // STR - car: *value, // CONS car / LAMBDA params head + car: *value, // CONS car / LAMBDA params head; promote fwd cdr: *value, // CONS cdr / LAMBDA body head envp: *env, // LAMBDA captured env + pin: i32, // 0 = trans, 1 = perm, -1 = forwarded (see promote_v) }; -// 4+4+8+8+4+pad+16+8+8+8 = 72 with natural alignment. Round up to 96 -// so any future field growth (or alignment surprise from the wwstage -// cgen) doesn't corrupt the next heap cell. +// 4+4+8+8+4+pad+16+8+8+8+4 = 76 with natural alignment. Round up to +// 96 so any future field growth (or alignment surprise from the +// wwstage cgen) doesn't corrupt the next heap cell. def VALUE_SZ: u64 = 96u64; // env — single linked list of (sym → value) bindings. Adding a new // scope is just a prepend; lookup walks the chain head→tail. export type env = struct { sid: i32, - val: *value, + val: *value, // promote_env stashes the perm-forwarded ptr here next: *env, + pin: i32, // 0 = trans, 1 = perm, -1 = forwarded }; def ENV_SZ: u64 = 32u64; // ---- arena allocator -------------------------------------------------- // -// Bump-pointer arena over 64 KiB chunks from os.alloc. Replaces the -// page-per-cell waste of routing every value/env through os.alloc -// directly — rt_alloc rounds the request up to a 4 KiB page, so each -// 96-byte value used to cost 4032 bytes of slack. test_huge.lisp -// peaked at ~595 MB under massif --pages-as-heap=yes; the same -// allocation pattern now packs into chained 64 KiB chunks. +// Two bump-pointer arenas over 64 KiB chunks: `perm_arena` for cells +// that have to outlive the top-level form that produced them (top- +// level definitions and the value graph they pin), and `trans_arena` +// for everything else (parse cells, intermediate evals, the form's +// result). repl() resets the trans arena between top-level forms. // -// Same lifetime semantics as the old direct-mmap path: never reclaimed -// inside a single process. A follow-up (the closing note in repl()) -// will split this into perm/trans and reset the trans arena between -// top-level forms. +// Pre-arena every cell paid the 4 KiB page-per-call cost of +// rt_alloc; test_huge.lisp peaked at ~525 MB under massif +// --pages-as-heap=yes. With chunking alone Pass A brought that to +// ~253 MB; per-form trans reset (this pass) drops it further by +// reclaiming the trans high-water mark after each form. +// +// Promotion. eval routes every alloc through trans by default; +// `env_define` at the top level (`tailed == false`) calls +// `promote_v` first, which deep-copies the value graph into perm +// using Cheney-style forwarding in the source cells (`pin = -1` plus +// the new perm pointer stashed in `.car`/`.val`). `env_set` does the +// same when the binding it lands on lives in perm. After that, no +// perm cell ever points back into trans, so the reset is safe. +// +// Chunk recycling. arena_reset_trans hands the head chunk's older +// neighbours to a free list inside the same arena; subsequent +// overflows pop from that list before mmap'ing new memory. Peak +// virtual address space is bounded by max-form working set. def ARENA_CHUNK: u64 = 65536u64; // 64 KiB per mmap def CHUNK_HEADER: u64 = 32u64; // header bytes, 8-aligned @@ -129,10 +145,13 @@ type chunk = struct { cur: u64, }; -// Chained list of mmap'd chunks. `arena_head` is the active chunk; -// older chunks reachable via .next stay live for the values that -// landed in them. arena_init() seeds the first chunk. -let arena_head: *chunk = nil; +type arena = struct { + head: *chunk, // active bump chunk; .next chains older chunks + free: *chunk, // recycled chunks (single-linked via .next) +}; + +let perm_arena: arena; +let trans_arena: arena; fn arena_new_chunk(prev: *chunk) *chunk = { let raw: *u8 = os.alloc(ARENA_CHUNK): *u8; @@ -143,26 +162,69 @@ fn arena_new_chunk(prev: *chunk) *chunk = { }; export fn arena_init() void = { - arena_head = arena_new_chunk(nil); + perm_arena.head = arena_new_chunk(nil); + perm_arena.free = nil; + trans_arena.head = arena_new_chunk(nil); + trans_arena.free = nil; }; -// arena_alloc — bump `n` bytes off the head chunk; mmap a new chunk if -// the head can't fit the request. Result is zeroed memory: fresh mmap -// pages start zero, and we don't recycle in this pass. -fn arena_alloc(n: u64) *void = { +// arena_alloc_in — bump `n` bytes off the arena's head chunk; pull +// from the free list (or mmap fresh) if the head is too full. +// Allocated bytes are memzero'd so reused trans chunks start clean +// for the new form. +fn arena_alloc_in(A: *arena, n: u64) *void = { let n8: u64 = (n + 7u64) & ~7u64; - let h: *chunk = arena_head; + let h: *chunk = A.head; if (h.cur + n8 + CHUNK_HEADER > ARENA_CHUNK) { - let c: *chunk = arena_new_chunk(h); - arena_head = c; + let c: *chunk; + if (A.free != nil) { + c = A.free; + A.free = c.next; + c.next = h; + c.cur = 0u64; + } else { + c = arena_new_chunk(h); + }; + A.head = c; h = c; }; let raw: *u8 = h: *u8; let p: *u8 = raw + CHUNK_HEADER + h.cur; h.cur += n8; + let i: u64 = 0u64; + for (i < n8) { p[i] = 0; i += 1u64; }; return p: *void; }; +fn arena_alloc(n: u64) *void = { + return arena_alloc_in(&trans_arena, n); +}; + +fn arena_alloc_perm(n: u64) *void = { + return arena_alloc_in(&perm_arena, n); +}; + +// arena_reset_trans — detach chunks above the bottom of trans, push +// them onto the free list for reuse, memzero the touched portion of +// the head chunk (we kept its memory, not its contents), and reset +// the bump pointer. Called from repl() between top-level forms. +export fn arena_reset_trans() void = { + let h: *chunk = trans_arena.head; + if (h == nil) { return; }; + if (h.next != nil) { + let cur: *chunk = h.next; + let end: *chunk = cur; + for (end.next != nil) { end = end.next; }; + end.next = trans_arena.free; + trans_arena.free = cur; + h.next = nil; + }; + let raw: *u8 = (h: *u8) + CHUNK_HEADER; + let i: u64 = 0u64; + for (i < h.cur) { raw[i] = 0; i += 1u64; }; + h.cur = 0u64; +}; + // ---- error variants --------------------------------------------------- export type rterror = !str; // runtime: division by zero, unbound symbol, … @@ -711,6 +773,18 @@ fn env_define(e: **env, sid: i32, v: *value) void = { *e = f; }; +// env_define_perm — top-level define path. The new env node lives in +// the perm arena (pin=1); the caller must have already promoted `v` +// (see promote_v) so that no perm cell ends up pointing into trans. +fn env_define_perm(e: **env, sid: i32, v: *value) void = { + let f: *env = arena_alloc_perm(ENV_SZ): *env; + f.sid = sid; + f.val = v; + f.next = *e; + f.pin = 1; + *e = f; +}; + fn env_lookup(e: *env, sid: i32) (*value | rterror) = { let cur: *env = e; for (cur != nil) { @@ -720,15 +794,102 @@ fn env_lookup(e: *env, sid: i32) (*value | rterror) = { return "unbound symbol": rterror; }; +// env_set — when the binding we land on lives in perm, the new value +// has to be promoted before we store it, or the perm cell would point +// into trans and dangle on the next reset. fn env_set(e: *env, sid: i32, v: *value) (i32 | rterror) = { let cur: *env = e; for (cur != nil) { - if (cur.sid == sid) { cur.val = v; return 0; }; + if (cur.sid == sid) { + let nv: *value = v; + if (cur.pin == 1) { nv = promote_v(v); }; + cur.val = nv; + return 0; + }; cur = cur.next; }; return "set!: unbound symbol": rterror; }; +// ---- promotion (trans → perm) ---------------------------------------- +// +// Top-level define / set! is the only way a cell can escape its +// originating top-level form. promote_v deep-copies the reachable +// graph from a trans cell into perm, leaving forwarding markers in +// place so cycles (recursive lambdas, the (define) tie-back frame, +// shared sub-trees) don't get cloned twice and so back-edges resolve +// to the new perm cells. +// +// Forwarding encoding. After a trans cell has been promoted, its +// `pin` is set to -1 and the new perm pointer is stashed in `.car` +// (for value) or `.val` (for env). The original kind/fields are +// effectively dead from that point — anyone who reaches the trans +// cell again chases the forwarding pointer instead. The cells stay +// in trans until the next arena_reset_trans wipes them. + +fn promote_v(v: *value) *value = { + if (v == nil) { return nil; }; + let p: *value = v; + if (p.pin == 1) { return v; }; // already perm + if (p.pin == -1) { return p.car; }; // forwarded; chase + + let n: *value = arena_alloc_perm(VALUE_SZ): *value; + n.kind = p.kind; + n.bval = p.bval; + n.ival = p.ival; + n.sid = p.sid; + n.text = p.text; + n.car = p.car; + n.cdr = p.cdr; + n.envp = p.envp; + if (p.kind == valkind.FLOAT) { + // f64 byte copy — `n.fval = p.fval` through *value lowers + // to an integer-reg store at the f64 offset, same trap + // vfloat sidesteps via the `fbuf` scratch global. + let psrc: *u8 = (p: *u8) + FVAL_OFF; + let ndst: *u8 = (n: *u8) + FVAL_OFF; + let i: u64 = 0u64; + for (i < 8u64) { ndst[i] = psrc[i]; i += 1u64; }; + }; + n.pin = 1; + + // Forward before recursing — a sub-pointer that loops back to + // `p` (recursive lambda envp, define tie-back) chases the + // forward instead of allocating a second perm copy. + p.pin = -1; + p.car = n; + + if (n.kind == valkind.CONS) { + n.car = promote_v(n.car); + n.cdr = promote_v(n.cdr); + } else if (n.kind == valkind.LAMBDA) { + n.car = promote_v(n.car); // params list + n.cdr = promote_v(n.cdr); // body list + n.envp = promote_env(n.envp); + }; + return n; +}; + +fn promote_env(e: *env) *env = { + if (e == nil) { return nil; }; + let q: *env = e; + if (q.pin == 1) { return e; }; + if (q.pin == -1) { return q.val: *env; }; // forwarded; chase + + let n: *env = arena_alloc_perm(ENV_SZ): *env; + n.sid = q.sid; + n.val = q.val; + n.next = q.next; + n.pin = 1; + + q.pin = -1; + q.val = n: *value; // reuse .val as the forwarding slot + + n.val = promote_v(n.val); + n.next = promote_env(n.next); + return n; +}; + // ---- helpers on values ------------------------------------------------ fn truthy(v: *value) bool = { @@ -1140,10 +1301,17 @@ export fn eval(v0: *value, ein: **env) (*value | rterror) = { }; let body: *value = rest.cdr.car; let bv = eval(body, &cure)?; + // Top-level define (`tailed == false`, ein points at + // the global env): promote bv into perm and allocate + // the env node in perm too, so this binding survives + // the trans reset that ends every top-level form. + // Nested defines stay trans-local — they're scoped to + // the lambda body that produced them. if (tailed) { env_define(&cure, nameval.sid, bv); } else { - env_define(ein, nameval.sid, bv); + bv = promote_v(bv); + env_define_perm(ein, nameval.sid, bv); cure = *ein; }; // Recursive-lambda tie-back: prepend a self-binding @@ -1153,7 +1321,13 @@ export fn eval(v0: *value, ein: **env) (*value | rterror) = { // env that closures (e.g. `(adder 5)` returning // `(lambda (x) (+ x k))`) had captured. if (bv.kind == valkind.LAMBDA) { - let frame: *env = arena_alloc(ENV_SZ): *env; + let frame: *env; + if (tailed) { + frame = arena_alloc(ENV_SZ): *env; + } else { + frame = arena_alloc_perm(ENV_SZ): *env; + frame.pin = 1; + }; frame.sid = nameval.sid; frame.val = bv; frame.next = bv.envp; @@ -1250,8 +1424,29 @@ export fn eval(v0: *value, ein: **env) (*value | rterror) = { acur = acur.cdr; }; + // Each apply leaks one 4 KiB rt_ensure page if we don't + // release `xs.ptr` — release happens in every exit edge + // below. `xs.cap` was set by the doubling slice grower and + // matches what rt_ensure mmaped. + let xs_cap: u64 = (xs.cap: u64) * 8u64; + let xs_ptr: *void = xs.ptr: *void; if (callee.kind == valkind.BUILTIN) { - return apply_builtin(callee.ival: i32, xs)?; + // Match into a local before freeing: leaving the + // `(*value | rterror)` result inside a `let r = …` + // across the os.free call lets a wwstage cgen quirk + // stomp on the rterror's str.len (the union is spilled + // to AX/DX/CX, and the foreign-call clobber blanks one + // of them before the `?` reads it back). + match (apply_builtin(callee.ival: i32, xs)) { + case let pv: *value => { + if (xs_ptr != nil) { os.free(xs_ptr: *void, xs_cap); }; + return pv; + }; + case let er: rterror => { + if (xs_ptr != nil) { os.free(xs_ptr: *void, xs_cap); }; + return er; + }; + }; }; if (callee.kind == valkind.LAMBDA) { // Extend the lambda's captured env with one frame @@ -1261,10 +1456,12 @@ export fn eval(v0: *value, ein: **env) (*value | rterror) = { let i: i32 = 0; for (p.kind == valkind.CONS) { if (i >= xs.len) { + if (xs_ptr != nil) { os.free(xs_ptr: *void, xs_cap); }; return "lambda: too few args": rterror; }; let nm: *value = p.car; if (nm.kind != valkind.SYM) { + if (xs_ptr != nil) { os.free(xs_ptr: *void, xs_cap); }; return "lambda: bad param": rterror; }; let f: *env = arena_alloc(ENV_SZ): *env; @@ -1276,8 +1473,10 @@ export fn eval(v0: *value, ein: **env) (*value | rterror) = { i += 1; }; if (i != xs.len) { + if (xs_ptr != nil) { os.free(xs_ptr: *void, xs_cap); }; return "lambda: too many args": rterror; }; + if (xs_ptr != nil) { os.free(xs_ptr: *void, xs_cap); }; cure = inner; tailed = true; let bod: *value = callee.cdr; @@ -1288,6 +1487,7 @@ export fn eval(v0: *value, ein: **env) (*value | rterror) = { v = bod.car; continue; }; + if (xs_ptr != nil) { os.free(xs_ptr: *void, xs_cap); }; return "not callable": rterror; }; // Unreachable — every path inside the loop returns or continues. @@ -1394,9 +1594,13 @@ export fn print_value(v: *value, nl: bool) void = { // ---- builtin binding -------------------------------------------------- +// bind_one — initial global-env wire-up. Builtins are part of the +// permanent root env, so both the binding node and the value cell go +// to perm (or the first arena_reset_trans would wipe them). fn bind_one(e: **env, name: str, id: i32) void = { let sid: i32 = intern(name); - env_define(e, sid, vbuiltin(id)); + let bv: *value = promote_v(vbuiltin(id)); + env_define_perm(e, sid, bv); }; export fn initsyms() void = { @@ -1570,6 +1774,12 @@ export fn repl() i32 = { for (j < nleft) { buf[j] = buf[nc + j]; j += 1; }; buf.len = nleft; cont = false; + // Form's parse cells, intermediate eval values, and the + // printed result are all in the trans arena now and + // won't be referenced again — reclaim them for the + // next form. Top-level defines have already been + // promoted into perm via the DEFINE handler. + arena_reset_trans(); }; case let er: parserr => { // "unterminated list" / "unterminated string" means we @@ -1591,6 +1801,7 @@ export fn repl() i32 = { print_err("parse error: ", er: str); buf.len = 0; cont = false; + arena_reset_trans(); }; case eof => { // next() succeeded but parse_expr ran out: the form was