ww/lib/shlex/shlex.ww

// shlex — POSIX shell tokenizer + quoter. Port of Hare's lib/shlex
// (ref/hare/shlex/{split.ha,escape.ha}) using ww byte-indexed cursors
// in place of Hare's strings::iterator.
//
// Surface today:
//
//   shlex.syntaxerr            — !void; bad shell syntax
//   shlex.strerror(syntaxerr)  — "Invalid shell syntax"
//   shlex.split(in: str)       — tokenize; returns ([]str | syntaxerr)
//   shlex.quote(*io.stream, s) — write `s` shell-quoted to a sink
//   shlex.quotestr(s: str) str — quote into a fresh str
//
// Owning model: split() returns a fresh `[]str` of strings.dup'd
// elements; release with [[strings.freeall]] (Hare's natural disposer
// shape). quotestr() returns an os.alloc'd str; release via
// `os.free(r.ptr, r.len: u64)` exactly like [[strings.dup]].
//
// Quoting rules (POSIX shell, byte-wise):
//
//   - whitespace separators ' ' / '\t' / '\n' (collapse runs)
//   - '\\<c>' outside quotes: literal <c>; '\\<newline>' deleted;
//     trailing bare '\\' → syntaxerr
//   - '"..."': '\\<c>' processed inside (any <c>); unterminated → syntaxerr
//   - "'...'": literal until closing "'"; no escapes inside;
//     unterminated → syntaxerr
//   - "Empty ''" yields a literal empty token (preserves quoting
//     intent — distinguishes `cmd ''` from `cmd`)
//
// Divergences from Hare:
//
//   - Drop nomem: ww os.alloc has no recoverable failure path (OOM
//     yields a poisonous pointer that faults on deref; see lib/os.ww
//     comment). Same precedent as strings.dup, getopt.appendoption.
//     Hare's (...|syntaxerr|nomem) collapses to (...|syntaxerr).
//
//   - Byte-wise iteration via i32 cursor instead of Hare's
//     strings::iterator (no UTF-8 rune iteration in the language
//     stack yet; same precedent as fnmatch). scan_double / scan_single
//     / scan_backslash work on bytes; an escaped multibyte sequence
//     round-trips as the byte sequence it appears in the input.
//
//   - memio glue uses ww's caller-supplies-state shape:
//        let mst: memio.state; let s: io.stream;
//        memio.dynamic(&mst, &s);
//        ... io.write(&s, ...) ...
//        let view: str = memio.string(&mst);   // borrowed
//     instead of Hare's `let s = memio::dynamic()` return-by-value.
//     Locked in by lib/memio's commit until cgen ships >24B return-
//     by-value (graduates "in one go" per lib/CLAUDE.md).
//
//   - quote() takes `*io.stream` (not Hare's `io::handle`) — that's
//     already the ww-divergent shape lib/io ships. Returns
//     `(i32 | io.closed)` instead of Hare's wider io::error: lib/io's
//     stream vtable carries only `closed` on the write arm.
//
//   - quote() emits raw bytes directly into the sink; without
//     [[memio.appendrune]] / [[memio.concat]] in lib/memio yet, the
//     "write one byte" path goes through `io.write(s, buf[0:1])`
//     against a stack `[N]u8`. Same shape as fmt's rune-arm.
//
//   - `[]str` grown via direct rt_ensure rather than the `append`
//     builtin: cgen lowers `append(slice, element)` to a single MOVQ
//     of the first 8 bytes, which drops the `len` half of a 16B str
//     element. [[appendstr]] writes both halves through `*str`.
//     Same gap, same workaround, same comment shape as
//     [[getopt.appendoption]] (lib/getopt/getopt.ww:96-106) — kept
//     greppable across modules. When the append-builtin is fixed,
//     all three (getopt, shlex, anywhere else) collapse in one go.
//
//   - Internal [[dupstr]] inlined rather than `use strings;`: keeps
//     the dep surface small for a 6-line helper. shlex doesn't reach
//     for any other strings:: routine, and callers free split()'s
//     result via [[strings.freeall]] directly.
//
// Caller layout — split:
//
//   match (shlex.split("ls -l '/etc/passwd'")) {
//     case let toks: []str => {
//       let i: i32 = 0;
//       for (i < toks.len) { /* toks[i] */ i += 1; };
//       strings.freeall(toks);
//     };
//     case shlex.syntaxerr => { /* bad input */ };
//   };
//
// Caller layout — quote into a fixed buffer:
//
//   let buf: [128]u8;
//   let mst: memio.state;
//   let s: io.stream;
//   memio.fixed(&mst, &s, buf[0:128]);
//   shlex.quote(&s, "hello world");          // writes 'hello world'
//   let view: str = memio.string(&mst);

package shlex;

import io;
import memio;
import os;

// rt_ensure is the runtime slice-growth helper invoked by the
// `append(s, v)` builtin. We bind it directly because the builtin's
// expansion stores only 8 bytes of the new element (cgen emits a
// single MOVQ), losing the `len` half of a `str` (16B). No public
// stdlib facade exposes it, hence the direct @symbol.
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;

// syntaxerr — the input wasn't a valid shell-tokenizable string
// (unterminated quote / bare trailing backslash). Mirrors Hare's
// shlex::syntaxerr; a `!void` so it can ride a tagged-union return
// alongside `[]str`.
export type syntaxerr = !void;

// strerror — human-friendly rendering of [[syntaxerr]]. Mirrors
// Hare's shlex::strerror(syntaxerr) str.
export fn strerror(err: syntaxerr) str = {
	return "Invalid shell syntax";
};

// dupstr — local strings.dup. Inlined to avoid a `use strings;` dep
// on this small site; same algorithm and same {nil, 0} handling for
// empty input.
fn dupstr(s: str) str = {
	let r: str;
	r.ptr = nil;
	r.len = 0;
	if (s.len == 0) { return r; };
	let buf: *u8 = os.alloc(s.len: u64): *u8;
	let i: i32 = 0;
	for (i < s.len) { buf[i] = s[i]; i += 1; };
	r.ptr = buf;
	r.len = s.len;
	return r;
};

// appendstr — grow `*slice` by one and store `item` (16B). Bypasses
// the `append` builtin's first-8B-only store gap; mirror of
// [[getopt.appendoption]] for the `[]option` case.
fn appendstr(slice: *[]str, item: str) void = {
	let newlen: i32 = slice.len + 1;
	slice.len = newlen;
	rtensure(slice: *void, size(str): u64);
	let dst: *str = &slice.ptr[newlen - 1];
	dst.ptr = item.ptr;
	dst.len = item.len;
};

// freepartial — drop a partially built [[split]] result on the
// syntaxerr path. Hare uses a `defer if (!ok)` guard; ww has no defer,
// so we hand-roll the cleanup at every error return. Mirror of
// [[strings.freeall]] (skip empty {nil, 0} elements; skip the header
// free if cap == 0).
fn freepartial(slice: []str) void = {
	let i: i32 = 0;
	for (i < slice.len) {
		if (slice[i].len > 0) {
			os.free(slice[i].ptr: *void, slice[i].len: u64);
		};
		i += 1;
	};
	if (slice.cap > 0) {
		os.free(slice.ptr: *void, (slice.cap: u64) * size(str): u64);
	};
};

// writebyte — send one byte to `s` via [[io.write]]. The `[1]u8` slice
// dance mirrors fmt's rune-arm (lib/fmt/fmt.ww:210-218); memio doesn't
// ship an appendbyte/appendrune in the ww port yet, so this is the
// uniform write-one-byte primitive across the module.
//
// The dynamic memio sink never returns `io.closed`; the [[split]] +
// [[quotestr]] callers ignore the error arm because their stream is
// always memio.dynamic. [[quote]] (caller-supplied sink) propagates
// it through its `(i32 | io.closed)` return.
fn writebyte(s: *io.stream, b: u8) (i32 | io.closed) = {
	let buf: [1]u8;
	buf[0] = b;
	return io.write(s, buf[0:1]);
};

// scan_backslash — consume one `\<c>` escape at `*pos`. POSIX:
// "<backslash> followed by <newline> shall be removed" — we eat the
// newline silently. A trailing bare backslash (`*pos == in.len`) is
// `syntaxerr`. Otherwise the next byte is appended verbatim. Mirrors
// Hare's scan_backslash (split.ha:86).
fn scan_backslash(out: *io.stream, in: str, pos: *i32) (void | syntaxerr) = {
	if (*pos >= in.len) { let e: syntaxerr; return e; };
	let r: u8 = in[*pos];
	*pos += 1;
	if (r == 10u8) { return; };	// '\n' deleted
	let _w = writebyte(out, r);
	return;
};

// scan_double — consume a `"..."` group at `*pos`. Closes on '"';
// '\<c>' inside processed by [[scan_backslash]]; unterminated → err.
// Mirrors Hare's scan_double (split.ha:110).
fn scan_double(out: *io.stream, in: str, pos: *i32) (void | syntaxerr) = {
	let loop: bool = true;
	for (loop) {
		if (*pos >= in.len) { let e: syntaxerr; return e; };
		let r: u8 = in[*pos];
		*pos += 1;
		if (r == 34u8) { loop = false; }		// '"'
		else { if (r == 92u8) {				// '\\'
			let er = scan_backslash(out, in, pos);
			match (er) {
			case let e: syntaxerr => return e;
			case void => {};
			};
		} else {
			let _w = writebyte(out, r);
		}; };
	};
	return;
};

// scan_single — consume a `'...'` group at `*pos`. Closes on "'";
// no escapes inside (POSIX: single-quoted text is fully literal);
// unterminated → err. Mirrors Hare's scan_single (split.ha:135).
fn scan_single(out: *io.stream, in: str, pos: *i32) (void | syntaxerr) = {
	let loop: bool = true;
	for (loop) {
		if (*pos >= in.len) { let e: syntaxerr; return e; };
		let r: u8 = in[*pos];
		*pos += 1;
		if (r == 39u8) { loop = false; }		// "'"
		else { let _w = writebyte(out, r); };
	};
	return;
};

// split — tokenize `in` according to POSIX shell quoting rules.
// Returns a fresh `[]str` of [[dupstr]]'d elements; on success, free
// with [[strings.freeall]]. Empty input returns an empty slice (not
// an error). Mirrors Hare's split (split.ha:16).
//
// Algorithm: a single forward pass over the input, accumulating bytes
// into a memio.dynamic sink. Whitespace runs flush the buffer as one
// token (skipping the flush before the first token, so leading
// whitespace doesn't push an empty entry). The `dirty` flag tracks
// whether the current iteration began processing input — it's the
// signal for "an empty quote group was the only content of this
// token", which still pushes a literal "" (e.g. `cmd ''`).
export fn split(in: str) ([]str | syntaxerr) = {
	let mst: memio.state;
	let snk: io.stream;
	memio.dynamic(&mst, &snk);

	let slice: []str;
	slice.ptr = nil: *str;
	slice.len = 0;
	slice.cap = 0;

	let first: bool = true;
	let dirty: bool = false;
	let pos: i32 = 0;

	for (pos < in.len) {
		let r: u8 = in[pos];
		pos += 1;
		dirty = true;

		if (r == 32u8 || r == 9u8 || r == 10u8) {
			// Collapse a run of whitespace.
			let inner: bool = true;
			for (inner) {
				if (pos >= in.len) { inner = false; }
				else {
					let r2: u8 = in[pos];
					if (r2 != 32u8 && r2 != 9u8 && r2 != 10u8) {
						inner = false;
					} else {
						pos += 1;
					};
				};
			};
			if (!first) {
				let view: str = memio.string(&mst);
				let owned: str = dupstr(view);
				appendstr(&slice, owned);
				memio.reset(&mst);
			};
			dirty = false;
		} else { if (r == 92u8) {				// '\\'
			let er = scan_backslash(&snk, in, &pos);
			match (er) {
			case let e: syntaxerr => {
				let _c = io.close(&snk);
				freepartial(slice);
				return e;
			};
			case void => {};
			};
		} else { if (r == 34u8) {				// '"'
			let er = scan_double(&snk, in, &pos);
			match (er) {
			case let e: syntaxerr => {
				let _c = io.close(&snk);
				freepartial(slice);
				return e;
			};
			case void => {};
			};
		} else { if (r == 39u8) {				// "'"
			let er = scan_single(&snk, in, &pos);
			match (er) {
			case let e: syntaxerr => {
				let _c = io.close(&snk);
				freepartial(slice);
				return e;
			};
			case void => {};
			};
		} else {
			let _w = writebyte(&snk, r);
		}; }; }; };

		if (first) { first = false; };
	};

	if (dirty) {
		let view: str = memio.string(&mst);
		let owned: str = dupstr(view);
		appendstr(&slice, owned);
	};

	let _c = io.close(&snk);
	return slice;
};

// issafe — true if `s` is composed entirely of bytes that pass through
// shell tokenisation unmodified (alnum + `@%+=:,./-`). Used by
// [[quote]] to skip emitting quotes around boring strings. Mirrors
// Hare's is_safe (escape.ha:9), narrowed to ASCII byte tests because
// ww has no rune iteration.
fn issafe(s: str) bool = {
	let i: i32 = 0;
	for (i < s.len) {
		let c: u8 = s[i];
		// Hare's switch list: '@', '%', '+', '=', ':', ',', '.', '/', '-'.
		if (c == 64u8 || c == 37u8 || c == 43u8 || c == 61u8
		    || c == 58u8 || c == 44u8 || c == 46u8 || c == 47u8
		    || c == 45u8) {
			i += 1;
		} else {
			let alnum: bool = false;
			if (c >= 48u8 && c <= 57u8)  { alnum = true; };	// 0-9
			if (c >= 65u8 && c <= 90u8)  { alnum = true; };	// A-Z
			if (c >= 97u8 && c <= 122u8) { alnum = true; };	// a-z
			if (!alnum) { return false; };
			i += 1;
		};
	};
	return true;
};

// quote — emit `s` shell-quoted to `sink`. Returns total bytes written,
// or `io.closed` if the sink rejects mid-write. Mirrors Hare's
// shlex::quote (escape.ha:25), modulo the narrower `io.closed`-only
// error set on lib/io's stream vtable (Hare's `io::error` would carry
// underread/etc., which lib/io doesn't yet model).
//
// Strategy:
//   - empty `s`           → `''`
//   - is_safe(s)          → emit raw, no quoting
//   - otherwise           → wrap in single quotes; an embedded `'`
//                           becomes `'"'"'` (close, double-quoted
//                           literal `'`, reopen).
export fn quote(sink: *io.stream, s: str) (i32 | io.closed) = {
	if (s.len == 0) {
		let buf: [2]u8;
		buf[0] = 39u8; buf[1] = 39u8;
		let r = io.write(sink, buf[0:2]);
		match (r) {
		case let n: i32 => return n;
		case let c: io.closed => return c;
		};
	};
	if (issafe(s)) {
		let raw: []u8;
		raw.ptr = s.ptr;
		raw.len = s.len;
		let r = io.write(sink, raw);
		match (r) {
		case let n: i32 => return n;
		case let c: io.closed => return c;
		};
	};

	let total: i32 = 0;
	let r1 = writebyte(sink, 39u8);				// "'"
	match (r1) {
	case let n: i32 => { total += n; };
	case let c: io.closed => return c;
	};
	let i: i32 = 0;
	for (i < s.len) {
		let c: u8 = s[i];
		if (c == 39u8) {				// "'" → '"'"'
			let pat: [5]u8;
			pat[0] = 39u8; pat[1] = 34u8; pat[2] = 39u8;
			pat[3] = 34u8; pat[4] = 39u8;
			let rr = io.write(sink, pat[0:5]);
			match (rr) {
			case let n: i32 => { total += n; };
			case let c2: io.closed => return c2;
			};
		} else {
			let rr = writebyte(sink, c);
			match (rr) {
			case let n: i32 => { total += n; };
			case let c2: io.closed => return c2;
			};
		};
		i += 1;
	};
	let r2 = writebyte(sink, 39u8);				// "'"
	match (r2) {
	case let n: i32 => { total += n; };
	case let c: io.closed => return c;
	};
	return total;
};

// quotestr — convenience: [[quote]] into a fresh memio.dynamic sink
// and return the dup'd result as an owned `str`. Caller releases via
// `os.free(r.ptr, r.len: u64)`. Mirrors Hare's shlex::quotestr
// (escape.ha:50).
//
// memio.dynamic's writes never return io.closed, so the `match` on
// `quote(...)` discards the error arm — it can't reach this caller's
// sink. quote()'s `(i32 | io.closed)` shape is preserved for the
// caller-supplied-sink path.
export fn quotestr(s: str) str = {
	let mst: memio.state;
	let snk: io.stream;
	memio.dynamic(&mst, &snk);

	let _r = quote(&snk, s);
	let view: str = memio.string(&mst);
	let owned: str = dupstr(view);

	let _c = io.close(&snk);
	return owned;
};