From b7a4eda40af31dfb61d9fe02af0760d582b4797a Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Thu, 4 Jun 2026 07:07:02 +0900 Subject: [PATCH] lib/bufio: newscanner + scanrune (regex fold-2b prereq) Port Hare's auto-grow newscanner (scanner.ha:72) and scan_rune (scanner.ha:259). scanner gains a maxread field (== cap for newscannerbuf, scanner.ha:101); readahead grows by BUFSZ up to maxread via alloc+copy (Hare appends; ww flat ptr/cap scanner, old block left to process-exit reclaim). scanbytes' overflow test gains the avail >= maxread leg (Hare's pending >= readahead predicate) so a growable scanner refills instead of overflowing. finish ports the free(scan.buffer) verbatim per the regex #27 precedent. Tests: rune scan over 1/2/3/4-byte UTF-8 + EOF, invalid initial/truncated/surrogate sequences, newscanner grow round-trip + scanrune-over-newscanner, maxread overflow. --- lib/bufio/bufio.ww | 160 ++++++++++++++++++++++++++++------ lib/bufio/bufiotest.ww | 191 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 326 insertions(+), 25 deletions(-) diff --git a/lib/bufio/bufio.ww b/lib/bufio/bufio.ww index 06979c4d..df9867dc 100644 --- a/lib/bufio/bufio.ww +++ b/lib/bufio/bufio.ww @@ -4,11 +4,14 @@ // // Surface: // +// bufio.newscanner (src: io.stream, maxread: i32) scanner // bufio.newscannerbuf(src: io.stream, buf: []u8) scanner // bufio.finish (s: *scanner) void // bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error) // bufio.scanbytes (s: *scanner, delim: u8) // ([]u8 | io.eof | io.error | overflow) +// bufio.scanrune (s: *scanner) +// (rune | io.eof | io.error | utf8.invalid) // bufio.scanline (s: *scanner) (str | io.eof | io.error | overflow) // // bufio.init (src: io.stream, rbuf: []u8, wbuf: []u8) stream @@ -26,12 +29,13 @@ // scanbyte / scanbytes / scanline. // // SCOPE: this is the surface ww ships, NOT a port of Hare's full -// scanner. The features ww never implemented — auto-grow newscanner, -// multibyte-delim scanbytes / scanstring, scanrune / readbyte / readtok -// / readline / readrune / unreadrune — are a separate future feature -// fold (#217). The ported set: the fixed-buffer newscannerbuf, -// single-byte scanbytes, single-byte scanline, scanbyte, finish; and -// the buffered-stream init / setflush / flush / unread / isbuffered. +// scanner. The features ww never implemented — multibyte-delim +// scanbytes / scanstring, readbyte / readtok / readline / readrune / +// unreadrune — are a separate future feature fold (#217). The ported +// set: the auto-grow newscanner, the fixed-buffer newscannerbuf, +// single-byte scanbytes, single-byte scanline, scanbyte, scanrune, +// finish; and the buffered-stream init / setflush / flush / unread / +// isbuffered. // // Cast workaround per #206-payoff (ken: KEEP the explicit casts; they // are cgen-neutral and sidestep the #214 over-acceptance surface). @@ -47,8 +51,14 @@ package bufio; +import encoding.utf8; import io; +// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth +// increment. i32 (not Hare's size) per the lib-wide index-type +// convention (lib/CLAUDE.md). +def BUFSZ: i32 = 4096; + // flushdefault — backing storage for the default flush byte-set // ("\n"). Hare scopes it inside `init` as `static let // flush_default = ['\n': u8]` (ref/hare/bufio/stream.ha:75); ww @@ -295,17 +305,37 @@ fn bclose(s: io.stream) (void | io.error) = { // slice support. export type scanner = struct { - src: io.stream, - ptr: *u8, - cap: i32, - start: i32, // index where the pending region starts in ptr - avail: i32, // pending byte count; pending = ptr[start..start+avail] + src: io.stream, + ptr: *u8, + cap: i32, + start: i32, // index where the pending region starts in ptr + avail: i32, // pending byte count; pending = ptr[start..start+avail] + maxread: i32, // growth ceiling; == cap for newscannerbuf (scanner.ha:101) +}; + +// newscanner — wire a scanner that allocates and grows its own +// read-ahead buffer, by BUFSZ per refill up to `maxread`. Returns the +// scanner BY VALUE. This is Hare's newscanner +// (ref/hare/bufio/scanner.ha:72) modulo two parameter drops: ww has no +// default arguments, so Hare's `maxread: size = types::SIZE_MAX` is a +// required i32 (callers wanting "no limit" pass types.I32_MAX), and the +// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD +// behavior is the shipped default). +export fn newscanner(src: io.stream, maxread: i32) scanner = { + let r: scanner; + r.src = src; + r.ptr = nil; + r.cap = 0; + r.start = 0; + r.avail = 0; + r.maxread = maxread; + return r; }; // newscannerbuf — wire a scanner to read through `src` using `buf` as -// the fixed read-ahead window. Returns the scanner BY VALUE. This is -// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92); the auto-grow -// newscanner is a separate feature (#217). +// the fixed read-ahead window (maxread == buf.len, so the buffer never +// grows — scanner.ha:101). Returns the scanner BY VALUE. This is +// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92). export fn newscannerbuf(src: io.stream, buf: []u8) scanner = { let r: scanner; r.src = src; @@ -313,25 +343,55 @@ export fn newscannerbuf(src: io.stream, buf: []u8) scanner = { r.cap = buf.len; r.start = 0; r.avail = 0; + r.maxread = buf.len; return r; }; -// finish — release scanner-owned resources. No-op (buffer is -// caller-owned, src isn't closed); kept on the surface so callers won't -// churn. Mirrors ref/hare/bufio/scanner.ha:110. -export fn finish(s: *scanner) void = { }; +// finish — release scanner-owned resources; src isn't closed. Mirrors +// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the +// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a +// newscannerbuf caller's buffer is untouched either way. +export fn finish(s: *scanner) void = { + free(s.ptr); +}; // readahead — make room and read once from src into the back of the // pending region. Returns bytes newly buffered (>=0), or io.eof/io.error // from src. The size from io.read narrows to i32 (buffer-length type). +// +// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer +// first shifts pending left, then — when start == 0 and maxread allows +// — grows. Hare grows via `append(scan.buffer, [0...], readahead)?`; +// ww's flat ptr/cap scanner allocates a fresh backing and copies (the +// old block is left to process-exit reclaim, ww no-free; `!` not `?` +// per the #36 nomem-propagation gap). The can't-grow case (avail >= +// maxread) is Hare's errors::overflow return — ww's io.error has no +// overflow member (lib/io/types.ww:40 enumerated union), so callers +// that can overflow (scanbytes) detect that state themselves before +// calling, against `maxread`. fn readahead(s: *scanner) (i32 | io.eof | io.error) = { - if (s.start + s.avail == s.cap && s.start > 0) { - let i: i32 = 0; - for (i < s.avail) { - s.ptr[i] = s.ptr[s.start + i]; - i += 1; + if (s.start + s.avail == s.cap) { + if (s.start > 0) { + let i: i32 = 0; + for (i < s.avail) { + s.ptr[i] = s.ptr[s.start + i]; + i += 1; + }; + s.start = 0; + } else if (s.avail < s.maxread) { + let want: i32 = s.avail + BUFSZ; + if (want > s.maxread) { want = s.maxread; }; + let ncap: i32 = s.avail + want; + let nbuf: []u8 = alloc([], ncap: u64)!; + let np: *u8 = nbuf.ptr; + let i: i32 = 0; + for (i < s.avail) { + np[i] = s.ptr[i]; + i += 1; + }; + s.ptr = np; + s.cap = ncap; }; - s.start = 0; }; let off: i32 = s.start + s.avail; let v: []u8; @@ -386,7 +446,12 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow }; i += 1; }; - if (s.start + s.avail == s.cap && s.start == 0) { + // full + unshiftable + ungrowable (avail >= maxread is + // Hare's `pending >= readahead` overflow predicate, + // scanner.ha:179; for a newscannerbuf scanner maxread == + // cap so this is the old fixed-buffer-full test). + if (s.start + s.avail == s.cap && s.start == 0 + && s.avail >= s.maxread) { let e: overflow; return e; }; let r: (i32 | io.eof | io.error) = readahead(s); @@ -399,6 +464,51 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow let e: io.eof; return e; }; +// scanrune — pop one UTF-8-encoded rune, refilling from src on demand. +// EOF mid-codepoint (fewer pending bytes than the initial byte +// announces) is utf8.invalid; a clean EOF before any byte is io.eof. +// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead +// when fewer than 4 bytes (the longest codepoint) are pending, then +// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` also +// propagates errors::overflow through io::error; ww's io.error has no +// overflow member (see readahead) and a <4-byte refill cannot overflow +// a scanner that can hold a codepoint, so only io.error proper +// propagates here. +export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid) = { + if (s.avail < 4) { + let ra: (i32 | io.eof | io.error) = readahead(s); + match (ra) { + case let n: i32 => { }; + case io.eof => { + if (s.avail == 0) { let e: io.eof; return e; }; + }; + case let e: io.error => return e; + }; + }; + let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]); + let sz: i32 = 0; + match (szr) { + case let n: i32 => { sz = n; }; + case utf8.invalid => { let e: utf8.invalid; return e; }; + }; + if (s.avail < sz) { + let e: utf8.invalid; return e; + }; + let v: []u8; + v.ptr = s.ptr + (s.start: u64); + v.len = sz; + s.start += sz; + s.avail -= sz; + let dec: utf8.decoder = utf8.decode(v); + let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec); + match (nr) { + case let r: rune => return r; + case utf8.done => { let e: io.eof; return e; }; + case utf8.more => { let e: utf8.invalid; return e; }; + case utf8.invalid => { let e: utf8.invalid; return e; }; + }; +}; + // scanline — read up to (and not including) the next '\n'. The newline // is consumed; the returned str view borrows from the scanner buffer. // Single-byte route (Hare's scan_line = scan_string(s, "\n"); the diff --git a/lib/bufio/bufiotest.ww b/lib/bufio/bufiotest.ww index 90356b30..8d2714f3 100644 --- a/lib/bufio/bufiotest.ww +++ b/lib/bufio/bufiotest.ww @@ -10,6 +10,7 @@ package bufio; import bufio; import bytes; +import encoding.utf8; import io; import memio; @@ -687,6 +688,192 @@ fn errsource() io.stream = { if (raw[4] != 100u8) { fail(); }; // 'd' }; +// ---- scanrune: ASCII + 2/3/4-byte UTF-8 + idempotent EOF -------------- + +@test fn scanrunecases() void = { + // "a" U+0061; "é" U+00E9 (C3 A9); "한" U+D55C (ED 95 9C); + // "😀" U+1F600 (F0 9F 98 80). + let raw: [10]u8; + raw[0] = 97u8; + raw[1] = 0xC3u8; raw[2] = 0xA9u8; + raw[3] = 0xEDu8; raw[4] = 0x95u8; raw[5] = 0x9Cu8; + raw[6] = 0xF0u8; raw[7] = 0x9Fu8; raw[8] = 0x98u8; raw[9] = 0x80u8; + + let mem: memio.stream = memio.fixed(raw[0:10]); + let m: io.stream = &mem.vt; + // 8B window: the trailing 4-byte rune straddles a refill (shift path). + let buf: [8]u8; + let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]); + + // (wantrune, weof) + let want: [6]u32; + let weof: [6]i32; + want[0] = 97u32; weof[0] = 0; + want[1] = 233u32; weof[1] = 0; + want[2] = 54620u32; weof[2] = 0; + want[3] = 128512u32; weof[3] = 0; + want[4] = 0u32; weof[4] = 1; // past end → eof + want[5] = 0u32; weof[5] = 1; // stay at eof + + let i: i32 = 0; + for (i < 6) { + let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc); + match (r) { + case let rn: rune => { + if (weof[i] != 0) { fail(); }; + if (rn: u32 != want[i]) { fail(); }; + }; + case io.eof => { if (weof[i] == 0) { fail(); }; }; + case let e: io.error => fail(); + case utf8.invalid => fail(); + }; + i += 1; + }; + + bufio.finish(&sc); +}; + +// ---- scanrune: invalid sequences -------------------------------------- + +@test fn scanruneinvalid() void = { + // Bare continuation byte: utf8sz rejects the initial byte. + let r1raw: [1]u8; + r1raw[0] = 0x80u8; + let mem1: memio.stream = memio.fixed(r1raw[0:1]); + let m1: io.stream = &mem1.vt; + let b1: [8]u8; + let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]); + let r1: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc1); + match (r1) { + case let rn: rune => fail(); + case io.eof => fail(); + case let e: io.error => fail(); + case utf8.invalid => { }; + }; + bufio.finish(&sc1); + + // Truncated 2-byte sequence at EOF: pending < utf8sz → invalid + // (Hare scanner.ha:272-274). + let r2raw: [1]u8; + r2raw[0] = 0xC3u8; + let mem2: memio.stream = memio.fixed(r2raw[0:1]); + let m2: io.stream = &mem2.vt; + let b2: [8]u8; + let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]); + let r2: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2); + match (r2) { + case let rn: rune => fail(); + case io.eof => fail(); + case let e: io.error => fail(); + case utf8.invalid => { }; + }; + bufio.finish(&sc2); + + // UTF-16 surrogate encoding (ED A0 80 = U+D800): utf8sz accepts the + // initial byte, the decode DFA rejects the continuation. + let r3raw: [3]u8; + r3raw[0] = 0xEDu8; r3raw[1] = 0xA0u8; r3raw[2] = 0x80u8; + let mem3: memio.stream = memio.fixed(r3raw[0:3]); + let m3: io.stream = &mem3.vt; + let b3: [8]u8; + let sc3: bufio.scanner = bufio.newscannerbuf(m3, b3[0:8]); + let r3: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc3); + match (r3) { + case let rn: rune => fail(); + case io.eof => fail(); + case let e: io.error => fail(); + case utf8.invalid => { }; + }; + bufio.finish(&sc3); +}; + +// ---- newscanner: auto-grow from the empty ctor state ------------------ + +@test fn newscannergrow() void = { + let src: [16]u8; + let n: i32 = putstr("hello\nworld\n", src[0:16], 0); + + let mem: memio.stream = memio.fixed(src[0:n]); + let m: io.stream = &mem.vt; + // cap starts at 0; the first readahead grows to min(BUFSZ, maxread). + let sc: bufio.scanner = bufio.newscanner(m, 1024); + + // (wantlen, wantfirst, weof) + let wl: [3]i32; + let wf: [3]u8; + let weof: [3]i32; + wl[0]=5; wf[0]=104u8; weof[0]=0; // hello + wl[1]=5; wf[1]=119u8; weof[1]=0; // world + wl[2]=0; wf[2]=0u8; weof[2]=1; // eof + + let i: i32 = 0; + for (i < 3) { + let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + match (r) { + case let v: str => { + if (weof[i] != 0) { fail(); }; + if (v.len != wl[i]) { fail(); }; + if (v.len > 0 && v[0] != wf[i]) { fail(); }; + }; + case io.eof => { if (weof[i] == 0) { fail(); }; }; + case let e: io.error => fail(); + case bufio.overflow => fail(); + }; + i += 1; + }; + bufio.finish(&sc); + + // scanrune over a fresh newscanner (the regex search() shape). + let r2raw: [3]u8; + r2raw[0] = 0xC3u8; r2raw[1] = 0xA9u8; // é + r2raw[2] = 122u8; // 'z' + let mem2: memio.stream = memio.fixed(r2raw[0:3]); + let m2: io.stream = &mem2.vt; + let sc2: bufio.scanner = bufio.newscanner(m2, 1024); + + let want: [3]u32; + let weof2: [3]i32; + want[0]=233u32; weof2[0]=0; + want[1]=122u32; weof2[1]=0; + want[2]=0u32; weof2[2]=1; + + i = 0; + for (i < 3) { + let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2); + match (r) { + case let rn: rune => { + if (weof2[i] != 0) { fail(); }; + if (rn: u32 != want[i]) { fail(); }; + }; + case io.eof => { if (weof2[i] == 0) { fail(); }; }; + case let e: io.error => fail(); + case utf8.invalid => fail(); + }; + i += 1; + }; + bufio.finish(&sc2); +}; + +// ---- newscanner: maxread reached without a delimiter → overflow ------- + +@test fn newscanneroverflow() void = { + let src: [16]u8; + let n: i32 = putstr("ABCDEFGH\n", src[0:16], 0); + + let mem: memio.stream = memio.fixed(src[0:n]); + let m: io.stream = &mem.vt; + let sc: bufio.scanner = bufio.newscanner(m, 4); // token (8B) > maxread + + let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + match (r) { + case let v: str => fail(); + case io.eof => fail(); + case let e: io.error => fail(); + case bufio.overflow => { }; + }; + bufio.finish(&sc); +}; + export fn main() i32 = { signalled = 1; scanbytecases(); signalled = 2; scanlinecases(); @@ -706,5 +893,9 @@ export fn main() i32 = { signalled = 16; streamflushempty(); signalled = 17; streamcloseflushes(); signalled = 18; streamsetflushcustom(); + signalled = 19; scanrunecases(); + signalled = 20; scanruneinvalid(); + signalled = 21; newscannergrow(); + signalled = 22; newscanneroverflow(); return 0; };