From 851915e6fe29b86e2c43477190b0666ff304c4cd Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Sun, 14 Jun 2026 11:54:20 +0900 Subject: [PATCH] bufio: readahead signals overflow when it cannot grow (F-B) The scanner readahead silently fell through when start==0 and the buffer was full at maxread, producing no bytes and no error. scanbyte then spun forever re-requesting bytes that never came (catB-144) and scanrune nil-dereferenced s.ptr[s.start] (catB-145). Make readahead the single overflow choke-point: at the ceiling it returns a bufio-local `overflow` before the grow, propagated through scanbyte/scanrune/scanbytes (scanbytes drops its now-redundant manual pre-check). Mirrors ref/hare/bufio/scanner.ha:174-182, which returns errors::overflow there; ww uses bufio-local overflow because io.error is a closed enum without an overflow member. Consumers (regex, the 778 embedded source) gain the totality arm. Table-driven @test crosses {nil-ptr, zero-len} x {scanbyte, scanrune}; neutralizing the overflow return reproduces the catB-144 hang. --- lib/bufio/bufio.ww | 72 ++++++++++++++------------ lib/bufio/bufiotest.ww | 89 +++++++++++++++++++++++++++++--- lib/regex/regex.ww | 4 ++ test/wcc/778_bufio_vstream_run.c | 1 + 4 files changed, 125 insertions(+), 41 deletions(-) diff --git a/lib/bufio/bufio.ww b/lib/bufio/bufio.ww index df9867dc..e3066daa 100644 --- a/lib/bufio/bufio.ww +++ b/lib/bufio/bufio.ww @@ -7,11 +7,11 @@ // bufio.newscanner (src: io.stream, maxread: i32) scanner // bufio.newscannerbuf(src: io.stream, buf: []u8) scanner // bufio.finish (s: *scanner) void -// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error) +// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error | overflow) // bufio.scanbytes (s: *scanner, delim: u8) // ([]u8 | io.eof | io.error | overflow) // bufio.scanrune (s: *scanner) -// (rune | io.eof | io.error | utf8.invalid) +// (rune | io.eof | io.error | utf8.invalid | overflow) // bufio.scanline (s: *scanner) (str | io.eof | io.error | overflow) // // bufio.init (src: io.stream, rbuf: []u8, wbuf: []u8) stream @@ -356,20 +356,25 @@ export fn finish(s: *scanner) void = { }; // readahead — make room and read once from src into the back of the -// pending region. Returns bytes newly buffered (>=0), or io.eof/io.error -// from src. The size from io.read narrows to i32 (buffer-length type). +// pending region. Returns bytes newly buffered (>=0), io.eof/io.error +// from src, or overflow when the buffer is full at the maxread ceiling. +// The size from io.read narrows to i32 (buffer-length type). // // Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer -// first shifts pending left, then — when start == 0 and maxread allows -// — grows. Hare grows via `append(scan.buffer, [0...], readahead)?`; -// ww's flat ptr/cap scanner allocates a fresh backing and copies (the -// old block is left to process-exit reclaim, ww no-free; `!` not `?` -// per the #36 nomem-propagation gap). The can't-grow case (avail >= -// maxread) is Hare's errors::overflow return — ww's io.error has no -// overflow member (lib/io/types.ww:40 enumerated union), so callers -// that can overflow (scanbytes) detect that state themselves before -// calling, against `maxread`. -fn readahead(s: *scanner) (i32 | io.eof | io.error) = { +// first shifts pending left, then — when start == 0 — grows, or returns +// overflow when `avail >= want` (Hare's `pending >= readahead` ceiling, +// scanner.ha:179-181, BEFORE the append). Hare grows via +// `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner +// allocates a fresh backing and copies (the old block is left to +// process-exit reclaim, ww no-free; `!` not `?` per the #36 +// nomem-propagation gap). overflow is bufio-local (bufio.ww:77), NOT +// errors.overflow: ww's io.error is a closed enumerated union with no +// overflow member (lib/io/types.ww), so the can't-grow signal rides a +// distinct arm that scanbyte / scanbytes / scanrune match-forward — +// the single choke-point (drew ruling, drain F-B). Pre-fix this case +// fell through silently to a zero-length io.read, spinning scanbyte +// (catB-144) and nil-derefing scanrune (catB-145). +fn readahead(s: *scanner) (i32 | io.eof | io.error | overflow) = { if (s.start + s.avail == s.cap) { if (s.start > 0) { let i: i32 = 0; @@ -378,9 +383,12 @@ fn readahead(s: *scanner) (i32 | io.eof | io.error) = { i += 1; }; s.start = 0; - } else if (s.avail < s.maxread) { + } else { let want: i32 = s.avail + BUFSZ; if (want > s.maxread) { want = s.maxread; }; + if (s.avail >= want) { + let e: overflow; return e; + }; let ncap: i32 = s.avail + want; let nbuf: []u8 = alloc([], ncap: u64)!; let np: *u8 = nbuf.ptr; @@ -410,13 +418,14 @@ fn readahead(s: *scanner) (i32 | io.eof | io.error) = { // scanbyte — pop one byte, refilling from src on demand. Mirrors // ref/hare/bufio/scanner.ha:204. -export fn scanbyte(s: *scanner) (u8 | io.eof | io.error) = { +export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = { for (s.avail == 0) { - let r: (i32 | io.eof | io.error) = readahead(s); + let r: (i32 | io.eof | io.error | overflow) = readahead(s); match (r) { case let n: i32 => { }; case io.eof => { let e: io.eof; return e; }; case let e: io.error => return e; + case overflow => { let e: overflow; return e; }; }; }; let b: u8 = s.ptr[s.start]; @@ -446,19 +455,15 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow }; i += 1; }; - // full + unshiftable + ungrowable (avail >= maxread is - // Hare's `pending >= readahead` overflow predicate, - // scanner.ha:179; for a newscannerbuf scanner maxread == - // cap so this is the old fixed-buffer-full test). - if (s.start + s.avail == s.cap && s.start == 0 - && s.avail >= s.maxread) { - let e: overflow; return e; - }; - let r: (i32 | io.eof | io.error) = readahead(s); + // overflow now surfaces from readahead's single choke-point + // (avail >= want == Hare's `pending >= readahead`, + // scanner.ha:179); no separate pre-check (drew ruling, F-B). + let r: (i32 | io.eof | io.error | overflow) = readahead(s); match (r) { case let n: i32 => { }; case io.eof => { let e: io.eof; return e; }; case let e: io.error => return e; + case overflow => { let e: overflow; return e; }; }; }; let e: io.eof; return e; @@ -469,20 +474,21 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow // announces) is utf8.invalid; a clean EOF before any byte is io.eof. // Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead // when fewer than 4 bytes (the longest codepoint) are pending, then -// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` also -// propagates errors::overflow through io::error; ww's io.error has no -// overflow member (see readahead) and a <4-byte refill cannot overflow -// a scanner that can hold a codepoint, so only io.error proper -// propagates here. -export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid) = { +// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` propagates +// errors::overflow (through io::error); ww forwards the bufio-local +// overflow as a distinct arm — a zero-cap / at-ceiling scanner cannot +// buffer the first byte, so without this arm the readahead fall-through +// would nil-deref s.ptr[s.start] (catB-145). +export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overflow) = { if (s.avail < 4) { - let ra: (i32 | io.eof | io.error) = readahead(s); + let ra: (i32 | io.eof | io.error | overflow) = readahead(s); match (ra) { case let n: i32 => { }; case io.eof => { if (s.avail == 0) { let e: io.eof; return e; }; }; case let e: io.error => return e; + case overflow => { let e: overflow; return e; }; }; }; let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]); diff --git a/lib/bufio/bufiotest.ww b/lib/bufio/bufiotest.ww index c104d512..9a826877 100644 --- a/lib/bufio/bufiotest.ww +++ b/lib/bufio/bufiotest.ww @@ -66,7 +66,7 @@ fn errsource() io.stream = { let i: i32 = 0; for (i < 6) { - let r: (u8 | io.eof | io.error) = bufio.scanbyte(&sc); + let r: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc); match (r) { case let b: u8 => { assert(!(weof[i] != 0)); @@ -74,6 +74,7 @@ fn errsource() io.stream = { }; case io.eof => { assert(!(weof[i] == 0)); }; case let e: io.error => abort(); + case bufio.overflow => abort(); }; i += 1; }; @@ -197,11 +198,12 @@ fn errsource() io.stream = { let buf: [8]u8; let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]); - let r1: (u8 | io.eof | io.error) = bufio.scanbyte(&sc); + let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc); match (r1) { case let b: u8 => abort(); case io.eof => { }; case let e: io.error => abort(); + case bufio.overflow => abort(); }; let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); @@ -230,11 +232,12 @@ fn errsource() io.stream = { let buf: [8]u8; let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]); - let r1: (u8 | io.eof | io.error) = bufio.scanbyte(&sc); + let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc); match (r1) { case let b: u8 => abort(); case io.eof => abort(); case let e: io.error => { }; + case bufio.overflow => abort(); }; let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); @@ -703,7 +706,7 @@ fn errsource() io.stream = { let i: i32 = 0; for (i < 6) { - let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc); + let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc); match (r) { case let rn: rune => { assert(!(weof[i] != 0)); @@ -712,6 +715,7 @@ fn errsource() io.stream = { case io.eof => { assert(!(weof[i] == 0)); }; case let e: io.error => abort(); case utf8.invalid => abort(); + case bufio.overflow => abort(); }; i += 1; }; @@ -729,12 +733,13 @@ fn errsource() io.stream = { let m1: io.stream = &mem1.vt; let b1: [8]u8; let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]); - let r1: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc1); + let r1: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc1); match (r1) { case let rn: rune => abort(); case io.eof => abort(); case let e: io.error => abort(); case utf8.invalid => { }; + case bufio.overflow => abort(); }; bufio.finish(&sc1); @@ -746,12 +751,13 @@ fn errsource() io.stream = { let m2: io.stream = &mem2.vt; let b2: [8]u8; let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]); - let r2: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2); + let r2: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2); match (r2) { case let rn: rune => abort(); case io.eof => abort(); case let e: io.error => abort(); case utf8.invalid => { }; + case bufio.overflow => abort(); }; bufio.finish(&sc2); @@ -763,12 +769,13 @@ fn errsource() io.stream = { let m3: io.stream = &mem3.vt; let b3: [8]u8; let sc3: bufio.scanner = bufio.newscannerbuf(m3, b3[0:8]); - let r3: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc3); + let r3: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc3); match (r3) { case let rn: rune => abort(); case io.eof => abort(); case let e: io.error => abort(); case utf8.invalid => { }; + case bufio.overflow => abort(); }; bufio.finish(&sc3); }; @@ -825,7 +832,7 @@ fn errsource() io.stream = { i = 0; for (i < 3) { - let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2); + let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2); match (r) { case let rn: rune => { assert(!(weof2[i] != 0)); @@ -834,6 +841,7 @@ fn errsource() io.stream = { case io.eof => { assert(!(weof2[i] == 0)); }; case let e: io.error => abort(); case utf8.invalid => abort(); + case bufio.overflow => abort(); }; i += 1; }; @@ -859,3 +867,68 @@ fn errsource() io.stream = { }; bufio.finish(&sc); }; + +// ---- readahead can't-grow → overflow, not spin / nil-deref ----------- +// +// drain F-B: a scanner that cannot buffer the next byte (zero-cap) makes +// readahead unable to make progress. Pre-fix it fell through silently to +// a zero-length io.read returning 0, so scanbyte spun forever (catB-144) +// and scanrune dereferenced the nil / zero-cap s.ptr[s.start] +// (catB-145). Both entry points must now surface bufio.overflow +// (ref/hare/bufio/scanner.ha:179-181). The table crosses the two +// zero-cap constructions — nil-ptr newscanner(,0) and zero-length +// newscannerbuf — with the two single-item entry points; every row is an +// overflow. Because the new code returns promptly, this test terminates +// where the pre-fix scanbyte path would hang. +@test fn readaheadcantgrow() void = { + let src: [4]u8; + src[0]=65u8; src[1]=66u8; src[2]=67u8; src[3]=68u8; // "ABCD" + + // (ctor, entry): ctor 0=newscanner(,0) (nil ptr), 1=newscannerbuf + // (,[0:0]) (zero-len buffer); entry 0=scanbyte, 1=scanrune. + let ctor: [4]i32; + let entry: [4]i32; + ctor[0]=0; entry[0]=0; // newscanner + scanbyte + ctor[1]=0; entry[1]=1; // newscanner + scanrune + ctor[2]=1; entry[2]=0; // newscannerbuf + scanbyte + ctor[3]=1; entry[3]=1; // newscannerbuf + scanrune + + let zbuf: [1]u8; // sliced to [0:0] for the zero-len construction + + let i: i32 = 0; + for (i < 4) { + let mem: memio.stream = memio.fixed(src[0:4]); + let m: io.stream = &mem.vt; + + let sc: bufio.scanner; + if (ctor[i] == 0) { + sc = bufio.newscanner(m, 0); + } else { + sc = bufio.newscannerbuf(m, zbuf[0:0]); + }; + + if (entry[i] == 0) { + let rb: (u8 | io.eof | io.error | bufio.overflow) = + bufio.scanbyte(&sc); + match (rb) { + case let b: u8 => abort(); + case io.eof => abort(); + case let e: io.error => abort(); + case bufio.overflow => { }; + }; + } else { + let rr: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = + bufio.scanrune(&sc); + match (rr) { + case let rn: rune => abort(); + case io.eof => abort(); + case let e: io.error => abort(); + case utf8.invalid => abort(); + case bufio.overflow => { }; + }; + }; + + bufio.finish(&sc); + i += 1; + }; +}; diff --git a/lib/regex/regex.ww b/lib/regex/regex.ww index 2ca875dc..e14d21f9 100644 --- a/lib/regex/regex.ww +++ b/lib/regex/regex.ww @@ -1206,6 +1206,10 @@ fn search( }; case let e: io.error => abort("regex: scanrune io error"); case utf8.invalid => abort("regex: scanrune invalid utf8"); + // Unreachable: this scanner is newscanner(h, I32_MAX), so the + // can't-grow ceiling is never hit; the arm keeps the match + // total over scanrune's widened overflow surface (drain F-B). + case bufio.overflow => abort("regex: scanrune overflow"); }; if (r_or_end is rune) { last_bytesize = (utf8.runesz(r_or_end as rune): size); diff --git a/test/wcc/778_bufio_vstream_run.c b/test/wcc/778_bufio_vstream_run.c index a5cbd677..2a41baf2 100644 --- a/test/wcc/778_bufio_vstream_run.c +++ b/test/wcc/778_bufio_vstream_run.c @@ -326,6 +326,7 @@ static const struct row rows[] = { " case let b: u8 => { fb = b; };\n" " case io.eof => { return 81; };\n" " case let e: io.error => { return 82; };\n" + " case bufio.overflow => { return 92; };\n" " };\n" " if (fb != 65u8) { return 83; };\n" " match (bufio.scanbytes(&sc, 44u8)) {\n"