// shlex — POSIX shell tokenizer + quoter. Port of Hare's lib/shlex // (ref/hare/shlex/{split.ha,escape.ha}) using ww byte-indexed cursors // in place of Hare's strings::iterator. // // Surface today: // // shlex.syntaxerr — !void; bad shell syntax // shlex.strerror(syntaxerr) — "Invalid shell syntax" // shlex.split(in: str) — tokenize; returns ([]str | syntaxerr) // shlex.quote(*io.stream, s) — write `s` shell-quoted to a sink // shlex.quotestr(s: str) str — quote into a fresh str // // Owning model: split() returns a fresh `[]str` of strings.dup'd // elements; release with [[strings.freeall]] (Hare's natural disposer // shape). quotestr() returns an os.alloc'd str; release via // `os.free(r.ptr, r.len: u64)` exactly like [[strings.dup]]. // // Quoting rules (POSIX shell, byte-wise): // // - whitespace separators ' ' / '\t' / '\n' (collapse runs) // - '\\' outside quotes: literal ; '\\' deleted; // trailing bare '\\' → syntaxerr // - '"..."': '\\' processed inside (any ); unterminated → syntaxerr // - "'...'": literal until closing "'"; no escapes inside; // unterminated → syntaxerr // - "Empty ''" yields a literal empty token (preserves quoting // intent — distinguishes `cmd ''` from `cmd`) // // Divergences from Hare: // // - Drop nomem: ww os.alloc has no recoverable failure path (OOM // yields a poisonous pointer that faults on deref; see lib/os.ww // comment). Same precedent as strings.dup, getopt.appendoption. // Hare's (...|syntaxerr|nomem) collapses to (...|syntaxerr). // // - Byte-wise iteration via i32 cursor instead of Hare's // strings::iterator (no UTF-8 rune iteration in the language // stack yet; same precedent as fnmatch). scan_double / scan_single // / scan_backslash work on bytes; an escaped multibyte sequence // round-trips as the byte sequence it appears in the input. // // - memio glue uses ww's caller-supplies-state shape: // let mst: memio.state; let s: io.stream; // memio.dynamic(&mst, &s); // ... io.write(&s, ...) ... // let view: str = memio.string(&mst); // borrowed // instead of Hare's `let s = memio::dynamic()` return-by-value. // Locked in by lib/memio's commit until cgen ships >24B return- // by-value (graduates "in one go" per lib/CLAUDE.md). // // - quote() takes `*io.stream` (not Hare's `io::handle`) — that's // already the ww-divergent shape lib/io ships. Returns // `(i32 | io.closed)` instead of Hare's wider io::error: lib/io's // stream vtable carries only `closed` on the write arm. // // - quote() emits raw bytes directly into the sink; without // [[memio.appendrune]] / [[memio.concat]] in lib/memio yet, the // "write one byte" path goes through `io.write(s, buf[0:1])` // against a stack `[N]u8`. Same shape as fmt's rune-arm. // // - `[]str` grown via direct rt_ensure rather than the `append` // builtin: cgen lowers `append(slice, element)` to a single MOVQ // of the first 8 bytes, which drops the `len` half of a 16B str // element. [[appendstr]] writes both halves through `*str`. // Same gap, same workaround, same comment shape as // [[getopt.appendoption]] (lib/getopt/getopt.ww:96-106) — kept // greppable across modules. When the append-builtin is fixed, // all three (getopt, shlex, anywhere else) collapse in one go. // // - Internal [[dupstr]] inlined rather than `use strings;`: keeps // the dep surface small for a 6-line helper. shlex doesn't reach // for any other strings:: routine, and callers free split()'s // result via [[strings.freeall]] directly. // // Caller layout — split: // // match (shlex.split("ls -l '/etc/passwd'")) { // case let toks: []str => { // let i: i32 = 0; // for (i < toks.len) { /* toks[i] */ i += 1; }; // strings.freeall(toks); // }; // case shlex.syntaxerr => { /* bad input */ }; // }; // // Caller layout — quote into a fixed buffer: // // let buf: [128]u8; // let mst: memio.state; // let s: io.stream; // memio.fixed(&mst, &s, buf[0:128]); // shlex.quote(&s, "hello world"); // writes 'hello world' // let view: str = memio.string(&mst); package shlex; import io; import memio; import os; import strings; // rt_ensure is the runtime slice-growth helper invoked by the // `append(s, v)` builtin. We bind it directly because the builtin's // expansion stores only 8 bytes of the new element (cgen emits a // single MOVQ), losing the `len` half of a `str` (16B). No public // stdlib facade exposes it, hence the direct @symbol. @symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; // syntaxerr — the input wasn't a valid shell-tokenizable string // (unterminated quote / bare trailing backslash). Mirrors Hare's // shlex::syntaxerr; a `!void` so it can ride a tagged-union return // alongside `[]str`. export type syntaxerr = !void; // strerror — human-friendly rendering of [[syntaxerr]]. Mirrors // Hare's shlex::strerror(syntaxerr) str. export fn strerror(err: syntaxerr) str = { return "Invalid shell syntax"; }; // dupstr — local strings.dup. Inlined (same algorithm and same // {nil, 0} handling for empty input). fn dupstr(s: str) str = { let r: str; r.ptr = nil; r.len = 0; if (s.len == 0) { return r; }; let buf: []u8 = alloc([], s.len: u64)!; let i: i32 = 0; for (i < s.len) { buf[i] = s[i]; i += 1; }; buf.len = s.len; return strings.frombytes(buf); }; // appendstr — grow `*slice` by one and store `item` (16B). Bypasses // the `append` builtin's first-8B-only store gap; mirror of // [[getopt.appendoption]] for the `[]option` case. fn appendstr(slice: *[]str, item: str) void = { let newlen: i32 = slice.len + 1; slice.len = newlen; rtensure(slice: *void, size(str): u64); let dst: *str = &slice.ptr[newlen - 1]; dst.ptr = item.ptr; dst.len = item.len; }; // freepartial — drop a partially built [[split]] result on the // syntaxerr path. Hare uses a `defer if (!ok)` guard; ww has no defer, // so we hand-roll the cleanup at every error return. Mirror of // [[strings.freeall]] (skip empty {nil, 0} elements; skip the header // free if cap == 0). fn freepartial(slice: []str) void = { let i: i32 = 0; for (i < slice.len) { if (slice[i].len > 0) { os.free(slice[i].ptr: *void, slice[i].len: u64); }; i += 1; }; if (slice.cap > 0) { os.free(slice.ptr: *void, (slice.cap: u64) * size(str): u64); }; }; // writebyte — send one byte to `s` via [[io.write]]. The `[1]u8` slice // dance mirrors fmt's rune-arm (lib/fmt/fmt.ww:210-218); memio doesn't // ship an appendbyte/appendrune in the ww port yet, so this is the // uniform write-one-byte primitive across the module. // // The dynamic memio sink never returns `io.closed`; the [[split]] + // [[quotestr]] callers ignore the error arm because their stream is // always memio.dynamic. [[quote]] (caller-supplied sink) propagates // it through its `(i32 | io.closed)` return. fn writebyte(s: *io.stream, b: u8) (i32 | io.closed) = { let buf: [1]u8; buf[0] = b; return io.write(s, buf[0:1]); }; // scan_backslash — consume one `\` escape at `*pos`. POSIX: // " followed by shall be removed" — we eat the // newline silently. A trailing bare backslash (`*pos == in.len`) is // `syntaxerr`. Otherwise the next byte is appended verbatim. Mirrors // Hare's scan_backslash (split.ha:86). fn scan_backslash(out: *io.stream, in: str, pos: *i32) (void | syntaxerr) = { if (*pos >= in.len) { let e: syntaxerr; return e; }; let r: u8 = in[*pos]; *pos += 1; if (r == 10u8) { return; }; // '\n' deleted let _w = writebyte(out, r); return; }; // scan_double — consume a `"..."` group at `*pos`. Closes on '"'; // '\' inside processed by [[scan_backslash]]; unterminated → err. // Mirrors Hare's scan_double (split.ha:110). fn scan_double(out: *io.stream, in: str, pos: *i32) (void | syntaxerr) = { let loop: bool = true; for (loop) { if (*pos >= in.len) { let e: syntaxerr; return e; }; let r: u8 = in[*pos]; *pos += 1; if (r == 34u8) { loop = false; } // '"' else { if (r == 92u8) { // '\\' let er = scan_backslash(out, in, pos); match (er) { case let e: syntaxerr => return e; case void => {}; }; } else { let _w = writebyte(out, r); }; }; }; return; }; // scan_single — consume a `'...'` group at `*pos`. Closes on "'"; // no escapes inside (POSIX: single-quoted text is fully literal); // unterminated → err. Mirrors Hare's scan_single (split.ha:135). fn scan_single(out: *io.stream, in: str, pos: *i32) (void | syntaxerr) = { let loop: bool = true; for (loop) { if (*pos >= in.len) { let e: syntaxerr; return e; }; let r: u8 = in[*pos]; *pos += 1; if (r == 39u8) { loop = false; } // "'" else { let _w = writebyte(out, r); }; }; return; }; // split — tokenize `in` according to POSIX shell quoting rules. // Returns a fresh `[]str` of [[dupstr]]'d elements; on success, free // with [[strings.freeall]]. Empty input returns an empty slice (not // an error). Mirrors Hare's split (split.ha:16). // // Algorithm: a single forward pass over the input, accumulating bytes // into a memio.dynamic sink. Whitespace runs flush the buffer as one // token (skipping the flush before the first token, so leading // whitespace doesn't push an empty entry). The `dirty` flag tracks // whether the current iteration began processing input — it's the // signal for "an empty quote group was the only content of this // token", which still pushes a literal "" (e.g. `cmd ''`). export fn split(in: str) ([]str | syntaxerr) = { let mst: memio.state; let snk: io.stream; memio.dynamic(&mst, &snk); let slice: []str; slice.ptr = nil: *str; slice.len = 0; slice.cap = 0; let first: bool = true; let dirty: bool = false; let pos: i32 = 0; for (pos < in.len) { let r: u8 = in[pos]; pos += 1; dirty = true; if (r == 32u8 || r == 9u8 || r == 10u8) { // Collapse a run of whitespace. let inner: bool = true; for (inner) { if (pos >= in.len) { inner = false; } else { let r2: u8 = in[pos]; if (r2 != 32u8 && r2 != 9u8 && r2 != 10u8) { inner = false; } else { pos += 1; }; }; }; if (!first) { let view: str = memio.string(&mst); let owned: str = dupstr(view); appendstr(&slice, owned); memio.reset(&mst); }; dirty = false; } else { if (r == 92u8) { // '\\' let er = scan_backslash(&snk, in, &pos); match (er) { case let e: syntaxerr => { let _c = io.close(&snk); freepartial(slice); return e; }; case void => {}; }; } else { if (r == 34u8) { // '"' let er = scan_double(&snk, in, &pos); match (er) { case let e: syntaxerr => { let _c = io.close(&snk); freepartial(slice); return e; }; case void => {}; }; } else { if (r == 39u8) { // "'" let er = scan_single(&snk, in, &pos); match (er) { case let e: syntaxerr => { let _c = io.close(&snk); freepartial(slice); return e; }; case void => {}; }; } else { let _w = writebyte(&snk, r); }; }; }; }; if (first) { first = false; }; }; if (dirty) { let view: str = memio.string(&mst); let owned: str = dupstr(view); appendstr(&slice, owned); }; let _c = io.close(&snk); return slice; }; // issafe — true if `s` is composed entirely of bytes that pass through // shell tokenisation unmodified (alnum + `@%+=:,./-`). Used by // [[quote]] to skip emitting quotes around boring strings. Mirrors // Hare's is_safe (escape.ha:9), narrowed to ASCII byte tests because // ww has no rune iteration. fn issafe(s: str) bool = { let i: i32 = 0; for (i < s.len) { let c: u8 = s[i]; // Hare's switch list: '@', '%', '+', '=', ':', ',', '.', '/', '-'. if (c == 64u8 || c == 37u8 || c == 43u8 || c == 61u8 || c == 58u8 || c == 44u8 || c == 46u8 || c == 47u8 || c == 45u8) { i += 1; } else { let alnum: bool = false; if (c >= 48u8 && c <= 57u8) { alnum = true; }; // 0-9 if (c >= 65u8 && c <= 90u8) { alnum = true; }; // A-Z if (c >= 97u8 && c <= 122u8) { alnum = true; }; // a-z if (!alnum) { return false; }; i += 1; }; }; return true; }; // quote — emit `s` shell-quoted to `sink`. Returns total bytes written, // or `io.closed` if the sink rejects mid-write. Mirrors Hare's // shlex::quote (escape.ha:25), modulo the narrower `io.closed`-only // error set on lib/io's stream vtable (Hare's `io::error` would carry // underread/etc., which lib/io doesn't yet model). // // Strategy: // - empty `s` → `''` // - is_safe(s) → emit raw, no quoting // - otherwise → wrap in single quotes; an embedded `'` // becomes `'"'"'` (close, double-quoted // literal `'`, reopen). export fn quote(sink: *io.stream, s: str) (i32 | io.closed) = { if (s.len == 0) { let buf: [2]u8; buf[0] = 39u8; buf[1] = 39u8; let r = io.write(sink, buf[0:2]); match (r) { case let n: i32 => return n; case let c: io.closed => return c; }; }; if (issafe(s)) { let raw: []u8; raw.ptr = s.ptr; raw.len = s.len; let r = io.write(sink, raw); match (r) { case let n: i32 => return n; case let c: io.closed => return c; }; }; let total: i32 = 0; let r1 = writebyte(sink, 39u8); // "'" match (r1) { case let n: i32 => { total += n; }; case let c: io.closed => return c; }; let i: i32 = 0; for (i < s.len) { let c: u8 = s[i]; if (c == 39u8) { // "'" → '"'"' let pat: [5]u8; pat[0] = 39u8; pat[1] = 34u8; pat[2] = 39u8; pat[3] = 34u8; pat[4] = 39u8; let rr = io.write(sink, pat[0:5]); match (rr) { case let n: i32 => { total += n; }; case let c2: io.closed => return c2; }; } else { let rr = writebyte(sink, c); match (rr) { case let n: i32 => { total += n; }; case let c2: io.closed => return c2; }; }; i += 1; }; let r2 = writebyte(sink, 39u8); // "'" match (r2) { case let n: i32 => { total += n; }; case let c: io.closed => return c; }; return total; }; // quotestr — convenience: [[quote]] into a fresh memio.dynamic sink // and return the dup'd result as an owned `str`. Caller releases via // `os.free(r.ptr, r.len: u64)`. Mirrors Hare's shlex::quotestr // (escape.ha:50). // // memio.dynamic's writes never return io.closed, so the `match` on // `quote(...)` discards the error arm — it can't reach this caller's // sink. quote()'s `(i32 | io.closed)` shape is preserved for the // caller-supplied-sink path. export fn quotestr(s: str) str = { let mst: memio.state; let snk: io.stream; memio.dynamic(&mst, &snk); let _r = quote(&snk, s); let view: str = memio.string(&mst); let owned: str = dupstr(view); let _c = io.close(&snk); return owned; };