diff --git a/lib/bufio/scanner.ww b/lib/bufio/scanner.ww new file mode 100644 index 00000000..9adeb942 --- /dev/null +++ b/lib/bufio/scanner.ww @@ -0,0 +1,254 @@ +package bufio; + +import encoding.utf8; +import io; + +// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth +// increment. i32 (not Hare's size) per the lib-wide index-type +// convention (lib/CLAUDE.md). +def BUFSZ: i32 = 4096; + +// overflow — the scanner buffer filled before the delimiter (or +// underlying EOF) was hit. With a caller-supplied buffer we can't +// grow; bumping the budget is on the caller. Mirrors Hare bufio's +// use of errors::overflow for the same condition. +export type overflow = !void; + +// Plain tokenizer — no embedded vtable; driven directly via scanbyte / +// scanbytes / scanline, not through io dispatch. Its `src` is an +// io.stream and refills go through io.read. Value-return constructor +// (Hare newscanner_buf, scanner.ha:92). +// +// ptr/cap kept flat (no `buf: []u8`); the scanner predates struct-held +// slice support. + +export type scanner = struct { + src: io.stream, + ptr: *u8, + cap: i32, + start: i32, // index where the pending region starts in ptr + avail: i32, // pending byte count; pending = ptr[start..start+avail] + maxread: i32, // growth ceiling; == cap for newscannerbuf (scanner.ha:101) +}; + +// newscanner — wire a scanner that allocates and grows its own +// read-ahead buffer, by BUFSZ per refill up to `maxread`. Returns the +// scanner BY VALUE. This is Hare's newscanner +// (ref/hare/bufio/scanner.ha:72) modulo two parameter drops: ww has no +// default arguments, so Hare's `maxread: size = types::SIZE_MAX` is a +// required i32 (callers wanting "no limit" pass types.I32_MAX), and the +// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD +// behavior is the shipped default). +export fn newscanner(src: io.stream, maxread: i32) scanner = { + let r: scanner; + r.src = src; + r.ptr = nil; + r.cap = 0; + r.start = 0; + r.avail = 0; + r.maxread = maxread; + return r; +}; + +// newscannerbuf — wire a scanner to read through `src` using `buf` as +// the fixed read-ahead window (maxread == buf.len, so the buffer never +// grows — scanner.ha:101). Returns the scanner BY VALUE. This is +// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92). +export fn newscannerbuf(src: io.stream, buf: []u8) scanner = { + let r: scanner; + r.src = src; + r.ptr = buf.ptr; + r.cap = buf.len; + r.start = 0; + r.avail = 0; + r.maxread = buf.len; + return r; +}; + +// finish — release scanner-owned resources; src isn't closed. Mirrors +// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the +// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a +// newscannerbuf caller's buffer is untouched either way. +export fn finish(s: *scanner) void = { + free(s.ptr); +}; + +// readahead — make room and read once from src into the back of the +// pending region. Returns bytes newly buffered (>=0), io.eof/io.error +// from src, or overflow when the buffer is full at the maxread ceiling. +// The size from io.read narrows to i32 (buffer-length type). +// +// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer +// first shifts pending left, then — when start == 0 — grows, or returns +// overflow when `avail >= want` (Hare's `pending >= readahead` ceiling, +// scanner.ha:179-181, BEFORE the append). Hare grows via +// `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner +// allocates a fresh backing and copies (the old block is left to +// process-exit reclaim, ww no-free; `!` not `?` per the #36 +// nomem-propagation gap). overflow is bufio-local (scanner.ww:15), NOT +// errors.overflow: ww's io.error is a closed enumerated union with no +// overflow member (lib/io/types.ww), so the can't-grow signal rides a +// distinct arm that scanbyte / scanbytes / scanrune match-forward — +// the single choke-point (drew ruling, drain F-B). Pre-fix this case +// fell through silently to a zero-length io.read, spinning scanbyte +// (catB-144) and nil-derefing scanrune (catB-145). +fn readahead(s: *scanner) (i32 | io.eof | io.error | overflow) = { + if (s.start + s.avail == s.cap) { + if (s.start > 0) { + let i: i32 = 0; + for (i < s.avail) { + s.ptr[i] = s.ptr[s.start + i]; + i += 1; + }; + s.start = 0; + } else { + let want: i32 = s.avail + BUFSZ; + if (want > s.maxread) { want = s.maxread; }; + if (s.avail >= want) { + let e: overflow; return e; + }; + let ncap: i32 = s.avail + want; + let nbuf: []u8 = alloc([], ncap: u64)!; + let np: *u8 = nbuf.ptr; + let i: i32 = 0; + for (i < s.avail) { + np[i] = s.ptr[i]; + i += 1; + }; + s.ptr = np; + s.cap = ncap; + }; + }; + let off: i32 = s.start + s.avail; + let v: []u8; + v.ptr = s.ptr + (off: u64); + v.len = s.cap - off; + let r: (size | io.eof | io.error) = io.read(s.src, v); + match (r) { + case let n: size => { + s.avail += n: i32; + return n: i32; + }; + case io.eof => { let e: io.eof; return e; }; + case let e: io.error => return e; + }; +}; + +// scanbyte — pop one byte, refilling from src on demand. Mirrors +// ref/hare/bufio/scanner.ha:204. +export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = { + for (s.avail == 0) { + let r: (i32 | io.eof | io.error | overflow) = readahead(s); + match (r) { + case let n: i32 => { }; + case io.eof => { let e: io.eof; return e; }; + case let e: io.error => return e; + case overflow => { let e: overflow; return e; }; + }; + }; + let b: u8 = s.ptr[s.start]; + s.start += 1; + s.avail -= 1; + return b; +}; + +// scanbytes — read up to (and not including) the next byte equal to +// `delim`. The delim is consumed but not returned. The returned slice +// borrows from the scanner buffer and is invalidated by the next scan. +// EOF without delim discards the trailing fragment and returns io.eof +// (Hare EOF_DISCARD default); buffer-full without delim returns +// overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte +// form is #217. Mirrors ref/hare/bufio/scanner.ha:220 (narrowed). +export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow) = { + let i: i32 = 0; + for (true) { + for (i < s.avail) { + if (s.ptr[s.start + i] == delim) { + let v: []u8; + v.ptr = s.ptr + (s.start: u64); + v.len = i; + s.start += i + 1; + s.avail -= i + 1; + return v; + }; + i += 1; + }; + // overflow now surfaces from readahead's single choke-point + // (avail >= want == Hare's `pending >= readahead`, + // scanner.ha:179); no separate pre-check (drew ruling, F-B). + let r: (i32 | io.eof | io.error | overflow) = readahead(s); + match (r) { + case let n: i32 => { }; + case io.eof => { let e: io.eof; return e; }; + case let e: io.error => return e; + case overflow => { let e: overflow; return e; }; + }; + }; + let e: io.eof; return e; +}; + +// scanrune — pop one UTF-8-encoded rune, refilling from src on demand. +// EOF mid-codepoint (fewer pending bytes than the initial byte +// announces) is utf8.invalid; a clean EOF before any byte is io.eof. +// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead +// when fewer than 4 bytes (the longest codepoint) are pending, then +// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` propagates +// errors::overflow (through io::error); ww forwards the bufio-local +// overflow as a distinct arm — a zero-cap / at-ceiling scanner cannot +// buffer the first byte, so without this arm the readahead fall-through +// would nil-deref s.ptr[s.start] (catB-145). +export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overflow) = { + if (s.avail < 4) { + let ra: (i32 | io.eof | io.error | overflow) = readahead(s); + match (ra) { + case let n: i32 => { }; + case io.eof => { + if (s.avail == 0) { let e: io.eof; return e; }; + }; + case let e: io.error => return e; + case overflow => { let e: overflow; return e; }; + }; + }; + let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]); + let sz: i32 = 0; + match (szr) { + case let n: i32 => { sz = n; }; + case utf8.invalid => { let e: utf8.invalid; return e; }; + }; + if (s.avail < sz) { + let e: utf8.invalid; return e; + }; + let v: []u8; + v.ptr = s.ptr + (s.start: u64); + v.len = sz; + s.start += sz; + s.avail -= sz; + let dec: utf8.decoder = utf8.decode(v); + let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec); + match (nr) { + case let r: rune => return r; + case utf8.done => { let e: io.eof; return e; }; + case utf8.more => { let e: utf8.invalid; return e; }; + case utf8.invalid => { let e: utf8.invalid; return e; }; + }; +}; + +// scanline — read up to (and not including) the next '\n'. The newline +// is consumed; the returned str view borrows from the scanner buffer. +// Single-byte route (Hare's scan_line = scan_string(s, "\n"); the +// arbitrary multibyte-delim scan_string is #217). Mirrors +// ref/hare/bufio/scanner.ha:307. +export fn scanline(s: *scanner) (str | io.eof | io.error | overflow) = { + let r: ([]u8 | io.eof | io.error | overflow) = scanbytes(s, 10u8); + match (r) { + case let bs: []u8 => { + let v: str; + v.ptr = bs.ptr; + v.len = bs.len; + return v; + }; + case io.eof => { let e: io.eof; return e; }; + case let e: io.error => return e; + case overflow => { let e: overflow; return e; }; + }; +}; diff --git a/lib/bufio/bufio.ww b/lib/bufio/stream.ww similarity index 51% rename from lib/bufio/bufio.ww rename to lib/bufio/stream.ww index 5b55ac6a..b6edb302 100644 --- a/lib/bufio/bufio.ww +++ b/lib/bufio/stream.ww @@ -53,14 +53,8 @@ package bufio; -import encoding.utf8; import io; -// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth -// increment. i32 (not Hare's size) per the lib-wide index-type -// convention (lib/CLAUDE.md). -def BUFSZ: i32 = 4096; - // flushdefault — backing storage for the default flush byte-set // ("\n"). Hare scopes it inside `init` as `static let // flush_default = ['\n': u8]` (ref/hare/bufio/stream.ha:75); ww @@ -72,14 +66,6 @@ let flushdefault: [1]u8 = [10u8]; // Hare expresses with `assert`. @symbol("rt_abort") fn rtabort(msg: str) void; -// overflow — the scanner buffer filled before the delimiter (or -// underlying EOF) was hit. With a caller-supplied buffer we can't -// grow; bumping the budget is on the caller. Mirrors Hare bufio's -// use of errors::overflow for the same condition. -export type overflow = !void; - -// ---- buffered stream ------------------------------------------------------ - // stream — heap-free buffered read+write over an underlying io.stream. // `vt` at offset 0 for the intrusive io.stream→*stream cast. `src` is // an io.stream (the vtable-native underlying handle). Mirrors @@ -192,8 +178,6 @@ export fn isbuffered(s: io.stream) bool = { return false; }; -// ---- buffered-stream vtable callbacks ------------------------------------ - // bread — buffered read. Recover stream via the intrusive cast; top up // rbuf from src via io.read whenever the pending region holds fewer // bytes than requested AND rbuf has spare capacity (Hare's short-read @@ -202,7 +186,7 @@ export fn isbuffered(s: io.stream) bool = { // pending. Mirrors ref/hare/bufio/stream.ha:205 (stream_read). fn bread(s: io.stream, buf: []u8) (size | io.eof | io.error) = { let b: *stream = s: *stream; - // Degenerate read-disabled mode (zero-length rbuf, bufio.ww:87-89); + // Degenerate read-disabled mode (zero-length rbuf, stream.ww:73-75); // ww's contract returns eof where Hare (always given a buffer) would // serve 0 — a separate concern from the top-up below. if (b.rbuf.len == 0) { @@ -299,249 +283,8 @@ fn bwrite(s: io.stream, buf: []u8) (size | io.error) = { // (ref/hare/bufio/stream.ha:194); init's default flag::NONE // (stream.ha:73) flushes and leaves src to its owner. Close-propagation // rides the io fold-2 MANAGED_HANDLE machinery (#5), per the ownership -// header (bufio.ww:46-50) — caller owns src, bclose frees/closes nothing. +// header (stream.ww:46-50) — caller owns src, bclose frees/closes nothing. fn bclose(s: io.stream) (void | io.error) = { let b: *stream = s: *stream; return flush(b); }; - -// ---- scanner (read-ahead tokenizer over an io.stream src) ---------------- -// -// Plain tokenizer — no embedded vtable; driven directly via scanbyte / -// scanbytes / scanline, not through io dispatch. Its `src` is an -// io.stream and refills go through io.read. Value-return constructor -// (Hare newscanner_buf, scanner.ha:92). -// -// ptr/cap kept flat (no `buf: []u8`); the scanner predates struct-held -// slice support. - -export type scanner = struct { - src: io.stream, - ptr: *u8, - cap: i32, - start: i32, // index where the pending region starts in ptr - avail: i32, // pending byte count; pending = ptr[start..start+avail] - maxread: i32, // growth ceiling; == cap for newscannerbuf (scanner.ha:101) -}; - -// newscanner — wire a scanner that allocates and grows its own -// read-ahead buffer, by BUFSZ per refill up to `maxread`. Returns the -// scanner BY VALUE. This is Hare's newscanner -// (ref/hare/bufio/scanner.ha:72) modulo two parameter drops: ww has no -// default arguments, so Hare's `maxread: size = types::SIZE_MAX` is a -// required i32 (callers wanting "no limit" pass types.I32_MAX), and the -// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD -// behavior is the shipped default). -export fn newscanner(src: io.stream, maxread: i32) scanner = { - let r: scanner; - r.src = src; - r.ptr = nil; - r.cap = 0; - r.start = 0; - r.avail = 0; - r.maxread = maxread; - return r; -}; - -// newscannerbuf — wire a scanner to read through `src` using `buf` as -// the fixed read-ahead window (maxread == buf.len, so the buffer never -// grows — scanner.ha:101). Returns the scanner BY VALUE. This is -// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92). -export fn newscannerbuf(src: io.stream, buf: []u8) scanner = { - let r: scanner; - r.src = src; - r.ptr = buf.ptr; - r.cap = buf.len; - r.start = 0; - r.avail = 0; - r.maxread = buf.len; - return r; -}; - -// finish — release scanner-owned resources; src isn't closed. Mirrors -// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the -// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a -// newscannerbuf caller's buffer is untouched either way. -export fn finish(s: *scanner) void = { - free(s.ptr); -}; - -// readahead — make room and read once from src into the back of the -// pending region. Returns bytes newly buffered (>=0), io.eof/io.error -// from src, or overflow when the buffer is full at the maxread ceiling. -// The size from io.read narrows to i32 (buffer-length type). -// -// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer -// first shifts pending left, then — when start == 0 — grows, or returns -// overflow when `avail >= want` (Hare's `pending >= readahead` ceiling, -// scanner.ha:179-181, BEFORE the append). Hare grows via -// `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner -// allocates a fresh backing and copies (the old block is left to -// process-exit reclaim, ww no-free; `!` not `?` per the #36 -// nomem-propagation gap). overflow is bufio-local (bufio.ww:77), NOT -// errors.overflow: ww's io.error is a closed enumerated union with no -// overflow member (lib/io/types.ww), so the can't-grow signal rides a -// distinct arm that scanbyte / scanbytes / scanrune match-forward — -// the single choke-point (drew ruling, drain F-B). Pre-fix this case -// fell through silently to a zero-length io.read, spinning scanbyte -// (catB-144) and nil-derefing scanrune (catB-145). -fn readahead(s: *scanner) (i32 | io.eof | io.error | overflow) = { - if (s.start + s.avail == s.cap) { - if (s.start > 0) { - let i: i32 = 0; - for (i < s.avail) { - s.ptr[i] = s.ptr[s.start + i]; - i += 1; - }; - s.start = 0; - } else { - let want: i32 = s.avail + BUFSZ; - if (want > s.maxread) { want = s.maxread; }; - if (s.avail >= want) { - let e: overflow; return e; - }; - let ncap: i32 = s.avail + want; - let nbuf: []u8 = alloc([], ncap: u64)!; - let np: *u8 = nbuf.ptr; - let i: i32 = 0; - for (i < s.avail) { - np[i] = s.ptr[i]; - i += 1; - }; - s.ptr = np; - s.cap = ncap; - }; - }; - let off: i32 = s.start + s.avail; - let v: []u8; - v.ptr = s.ptr + (off: u64); - v.len = s.cap - off; - let r: (size | io.eof | io.error) = io.read(s.src, v); - match (r) { - case let n: size => { - s.avail += n: i32; - return n: i32; - }; - case io.eof => { let e: io.eof; return e; }; - case let e: io.error => return e; - }; -}; - -// scanbyte — pop one byte, refilling from src on demand. Mirrors -// ref/hare/bufio/scanner.ha:204. -export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = { - for (s.avail == 0) { - let r: (i32 | io.eof | io.error | overflow) = readahead(s); - match (r) { - case let n: i32 => { }; - case io.eof => { let e: io.eof; return e; }; - case let e: io.error => return e; - case overflow => { let e: overflow; return e; }; - }; - }; - let b: u8 = s.ptr[s.start]; - s.start += 1; - s.avail -= 1; - return b; -}; - -// scanbytes — read up to (and not including) the next byte equal to -// `delim`. The delim is consumed but not returned. The returned slice -// borrows from the scanner buffer and is invalidated by the next scan. -// EOF without delim discards the trailing fragment and returns io.eof -// (Hare EOF_DISCARD default); buffer-full without delim returns -// overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte -// form is #217. Mirrors ref/hare/bufio/scanner.ha:220 (narrowed). -export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow) = { - let i: i32 = 0; - for (true) { - for (i < s.avail) { - if (s.ptr[s.start + i] == delim) { - let v: []u8; - v.ptr = s.ptr + (s.start: u64); - v.len = i; - s.start += i + 1; - s.avail -= i + 1; - return v; - }; - i += 1; - }; - // overflow now surfaces from readahead's single choke-point - // (avail >= want == Hare's `pending >= readahead`, - // scanner.ha:179); no separate pre-check (drew ruling, F-B). - let r: (i32 | io.eof | io.error | overflow) = readahead(s); - match (r) { - case let n: i32 => { }; - case io.eof => { let e: io.eof; return e; }; - case let e: io.error => return e; - case overflow => { let e: overflow; return e; }; - }; - }; - let e: io.eof; return e; -}; - -// scanrune — pop one UTF-8-encoded rune, refilling from src on demand. -// EOF mid-codepoint (fewer pending bytes than the initial byte -// announces) is utf8.invalid; a clean EOF before any byte is io.eof. -// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead -// when fewer than 4 bytes (the longest codepoint) are pending, then -// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` propagates -// errors::overflow (through io::error); ww forwards the bufio-local -// overflow as a distinct arm — a zero-cap / at-ceiling scanner cannot -// buffer the first byte, so without this arm the readahead fall-through -// would nil-deref s.ptr[s.start] (catB-145). -export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overflow) = { - if (s.avail < 4) { - let ra: (i32 | io.eof | io.error | overflow) = readahead(s); - match (ra) { - case let n: i32 => { }; - case io.eof => { - if (s.avail == 0) { let e: io.eof; return e; }; - }; - case let e: io.error => return e; - case overflow => { let e: overflow; return e; }; - }; - }; - let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]); - let sz: i32 = 0; - match (szr) { - case let n: i32 => { sz = n; }; - case utf8.invalid => { let e: utf8.invalid; return e; }; - }; - if (s.avail < sz) { - let e: utf8.invalid; return e; - }; - let v: []u8; - v.ptr = s.ptr + (s.start: u64); - v.len = sz; - s.start += sz; - s.avail -= sz; - let dec: utf8.decoder = utf8.decode(v); - let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec); - match (nr) { - case let r: rune => return r; - case utf8.done => { let e: io.eof; return e; }; - case utf8.more => { let e: utf8.invalid; return e; }; - case utf8.invalid => { let e: utf8.invalid; return e; }; - }; -}; - -// scanline — read up to (and not including) the next '\n'. The newline -// is consumed; the returned str view borrows from the scanner buffer. -// Single-byte route (Hare's scan_line = scan_string(s, "\n"); the -// arbitrary multibyte-delim scan_string is #217). Mirrors -// ref/hare/bufio/scanner.ha:307. -export fn scanline(s: *scanner) (str | io.eof | io.error | overflow) = { - let r: ([]u8 | io.eof | io.error | overflow) = scanbytes(s, 10u8); - match (r) { - case let bs: []u8 => { - let v: str; - v.ptr = bs.ptr; - v.len = bs.len; - return v; - }; - case io.eof => { let e: io.eof; return e; }; - case let e: io.error => return e; - case overflow => { let e: overflow; return e; }; - }; -}; diff --git a/lib/regex/regex.ww b/lib/regex/regex.ww index e14d21f9..7589bbb9 100644 --- a/lib/regex/regex.ww +++ b/lib/regex/regex.ww @@ -1114,7 +1114,7 @@ fn search( let last_bytesize: size = 0; // ww has no default arguments: Hare's newscanner(handle) maxread - // default spells types.I32_MAX explicitly (lib/bufio/bufio.ww). + // default spells types.I32_MAX explicitly (lib/bufio/scanner.ww). let scan: bufio.scanner = bufio.newscanner(h, types.I32_MAX); defer bufio.finish(&scan); for (true) {