Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bufio: readahead signals overflow when it cannot grow (F-B)

Browse Source 
The scanner readahead silently fell through when start==0 and the
buffer was full at maxread, producing no bytes and no error. scanbyte
then spun forever re-requesting bytes that never came (catB-144) and
scanrune nil-dereferenced s.ptr[s.start] (catB-145).

Make readahead the single overflow choke-point: at the ceiling it
returns a bufio-local `overflow` before the grow, propagated through
scanbyte/scanrune/scanbytes (scanbytes drops its now-redundant manual
pre-check). Mirrors ref/hare/bufio/scanner.ha:174-182, which returns
errors::overflow there; ww uses bufio-local overflow because io.error
is a closed enum without an overflow member. Consumers (regex, the 778
embedded source) gain the totality arm.

Table-driven @test crosses {nil-ptr, zero-len} x {scanbyte, scanrune};
neutralizing the overflow return reproduces the catB-144 hang.
This commit is contained in:
Hojun-Cho
2026-06-14 11:54:20 +09:00
parent 1752305be7
commit 851915e6fe
4 changed files with 125 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
lib/bufio/bufio.ww
View File

@@ -7,11 +7,11 @@
// bufio.newscanner (src: io.stream, maxread: i32) scanner
// bufio.newscannerbuf(src: io.stream, buf: []u8) scanner
// bufio.finish (s: *scanner) void
// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error)
// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error | overflow)
// bufio.scanbytes (s: *scanner, delim: u8)
// ([]u8 | io.eof | io.error | overflow)
// bufio.scanrune (s: *scanner)
// (rune | io.eof | io.error | utf8.invalid)
// (rune | io.eof | io.error | utf8.invalid | overflow)
// bufio.scanline (s: *scanner) (str | io.eof | io.error | overflow)
//
// bufio.init (src: io.stream, rbuf: []u8, wbuf: []u8) stream
@@ -356,20 +356,25 @@ export fn finish(s: *scanner) void = {
};
// readahead — make room and read once from src into the back of the
// pending region. Returns bytes newly buffered (>=0), or io.eof/io.error
// from src. The size from io.read narrows to i32 (buffer-length type).
// pending region. Returns bytes newly buffered (>=0), io.eof/io.error
// from src, or overflow when the buffer is full at the maxread ceiling.
// The size from io.read narrows to i32 (buffer-length type).
//
// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer
// first shifts pending left, then — when start == 0 and maxread allows
// — grows. Hare grows via `append(scan.buffer, [0...], readahead)?`;
// ww's flat ptr/cap scanner allocates a fresh backing and copies (the
// old block is left to process-exit reclaim, ww no-free; `!` not `?`
// per the #36 nomem-propagation gap). The can't-grow case (avail >=
// maxread) is Hare's errors::overflow return — ww's io.error has no
// overflow member (lib/io/types.ww:40 enumerated union), so callers
// that can overflow (scanbytes) detect that state themselves before
// calling, against `maxread`.
fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
// first shifts pending left, then — when start == 0 — grows, or returns
// overflow when `avail >= want` (Hare's `pending >= readahead` ceiling,
// scanner.ha:179-181, BEFORE the append). Hare grows via
// `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner
// allocates a fresh backing and copies (the old block is left to
// process-exit reclaim, ww no-free; `!` not `?` per the #36
// nomem-propagation gap). overflow is bufio-local (bufio.ww:77), NOT
// errors.overflow: ww's io.error is a closed enumerated union with no
// overflow member (lib/io/types.ww), so the can't-grow signal rides a
// distinct arm that scanbyte / scanbytes / scanrune match-forward —
// the single choke-point (drew ruling, drain F-B). Pre-fix this case
// fell through silently to a zero-length io.read, spinning scanbyte
// (catB-144) and nil-derefing scanrune (catB-145).
fn readahead(s: *scanner) (i32 | io.eof | io.error | overflow) = {
if (s.start + s.avail == s.cap) {
if (s.start > 0) {
let i: i32 = 0;
@@ -378,9 +383,12 @@ fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
i += 1;
};
s.start = 0;
} else if (s.avail < s.maxread) {
} else {
let want: i32 = s.avail + BUFSZ;
if (want > s.maxread) { want = s.maxread; };
if (s.avail >= want) {
let e: overflow; return e;
};
let ncap: i32 = s.avail + want;
let nbuf: []u8 = alloc([], ncap: u64)!;
let np: *u8 = nbuf.ptr;
@@ -410,13 +418,14 @@ fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
// scanbyte — pop one byte, refilling from src on demand. Mirrors
// ref/hare/bufio/scanner.ha:204.
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error) = {
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = {
for (s.avail == 0) {
let r: (i32 | io.eof | io.error) = readahead(s);
let r: (i32 | io.eof | io.error | overflow) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
};
};
let b: u8 = s.ptr[s.start];
@@ -446,19 +455,15 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
};
i += 1;
};
// full + unshiftable + ungrowable (avail >= maxread is
// Hare's `pending >= readahead` overflow predicate,
// scanner.ha:179; for a newscannerbuf scanner maxread ==
// cap so this is the old fixed-buffer-full test).
if (s.start + s.avail == s.cap && s.start == 0
&& s.avail >= s.maxread) {
let e: overflow; return e;
};
let r: (i32 | io.eof | io.error) = readahead(s);
// overflow now surfaces from readahead's single choke-point
// (avail >= want == Hare's `pending >= readahead`,
// scanner.ha:179); no separate pre-check (drew ruling, F-B).
let r: (i32 | io.eof | io.error | overflow) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
};
};
let e: io.eof; return e;
@@ -469,20 +474,21 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
// announces) is utf8.invalid; a clean EOF before any byte is io.eof.
// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead
// when fewer than 4 bytes (the longest codepoint) are pending, then
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` also
// propagates errors::overflow through io::error; ww's io.error has no
// overflow member (see readahead) and a <4-byte refill cannot overflow
// a scanner that can hold a codepoint, so only io.error proper
// propagates here.
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid) = {
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` propagates
// errors::overflow (through io::error); ww forwards the bufio-local
// overflow as a distinct arm — a zero-cap / at-ceiling scanner cannot
// buffer the first byte, so without this arm the readahead fall-through
// would nil-deref s.ptr[s.start] (catB-145).
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overflow) = {
if (s.avail < 4) {
let ra: (i32 | io.eof | io.error) = readahead(s);
let ra: (i32 | io.eof | io.error | overflow) = readahead(s);
match (ra) {
case let n: i32 => { };
case io.eof => {
if (s.avail == 0) { let e: io.eof; return e; };
};
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
};
};
let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]);

89
lib/bufio/bufiotest.ww
View File

@@ -66,7 +66,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 6) {
let r: (u8 | io.eof | io.error) = bufio.scanbyte(&sc);
let r: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
match (r) {
case let b: u8 => {
assert(!(weof[i] != 0));
@@ -74,6 +74,7 @@ fn errsource() io.stream = {
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
};
i += 1;
};
@@ -197,11 +198,12 @@ fn errsource() io.stream = {
let buf: [8]u8;
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]);
let r1: (u8 | io.eof | io.error) = bufio.scanbyte(&sc);
let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
match (r1) {
case let b: u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
};
let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
@@ -230,11 +232,12 @@ fn errsource() io.stream = {
let buf: [8]u8;
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]);
let r1: (u8 | io.eof | io.error) = bufio.scanbyte(&sc);
let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
match (r1) {
case let b: u8 => abort();
case io.eof => abort();
case let e: io.error => { };
case bufio.overflow => abort();
};
let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
@@ -703,7 +706,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 6) {
let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc);
let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc);
match (r) {
case let rn: rune => {
assert(!(weof[i] != 0));
@@ -712,6 +715,7 @@ fn errsource() io.stream = {
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => abort();
};
i += 1;
};
@@ -729,12 +733,13 @@ fn errsource() io.stream = {
let m1: io.stream = &mem1.vt;
let b1: [8]u8;
let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]);
let r1: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc1);
let r1: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc1);
match (r1) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
};
bufio.finish(&sc1);
@@ -746,12 +751,13 @@ fn errsource() io.stream = {
let m2: io.stream = &mem2.vt;
let b2: [8]u8;
let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]);
let r2: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2);
let r2: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2);
match (r2) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
};
bufio.finish(&sc2);
@@ -763,12 +769,13 @@ fn errsource() io.stream = {
let m3: io.stream = &mem3.vt;
let b3: [8]u8;
let sc3: bufio.scanner = bufio.newscannerbuf(m3, b3[0:8]);
let r3: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc3);
let r3: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc3);
match (r3) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
};
bufio.finish(&sc3);
};
@@ -825,7 +832,7 @@ fn errsource() io.stream = {
i = 0;
for (i < 3) {
let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2);
let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2);
match (r) {
case let rn: rune => {
assert(!(weof2[i] != 0));
@@ -834,6 +841,7 @@ fn errsource() io.stream = {
case io.eof => { assert(!(weof2[i] == 0)); };
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => abort();
};
i += 1;
};
@@ -859,3 +867,68 @@ fn errsource() io.stream = {
};
bufio.finish(&sc);
};
// ---- readahead can't-grow → overflow, not spin / nil-deref -----------
//
// drain F-B: a scanner that cannot buffer the next byte (zero-cap) makes
// readahead unable to make progress. Pre-fix it fell through silently to
// a zero-length io.read returning 0, so scanbyte spun forever (catB-144)
// and scanrune dereferenced the nil / zero-cap s.ptr[s.start]
// (catB-145). Both entry points must now surface bufio.overflow
// (ref/hare/bufio/scanner.ha:179-181). The table crosses the two
// zero-cap constructions — nil-ptr newscanner(,0) and zero-length
// newscannerbuf — with the two single-item entry points; every row is an
// overflow. Because the new code returns promptly, this test terminates
// where the pre-fix scanbyte path would hang.
@test fn readaheadcantgrow() void = {
let src: [4]u8;
src[0]=65u8; src[1]=66u8; src[2]=67u8; src[3]=68u8; // "ABCD"
// (ctor, entry): ctor 0=newscanner(,0) (nil ptr), 1=newscannerbuf
// (,[0:0]) (zero-len buffer); entry 0=scanbyte, 1=scanrune.
let ctor: [4]i32;
let entry: [4]i32;
ctor[0]=0; entry[0]=0; // newscanner + scanbyte
ctor[1]=0; entry[1]=1; // newscanner + scanrune
ctor[2]=1; entry[2]=0; // newscannerbuf + scanbyte
ctor[3]=1; entry[3]=1; // newscannerbuf + scanrune
let zbuf: [1]u8; // sliced to [0:0] for the zero-len construction
let i: i32 = 0;
for (i < 4) {
let mem: memio.stream = memio.fixed(src[0:4]);
let m: io.stream = &mem.vt;
let sc: bufio.scanner;
if (ctor[i] == 0) {
sc = bufio.newscanner(m, 0);
} else {
sc = bufio.newscannerbuf(m, zbuf[0:0]);
};
if (entry[i] == 0) {
let rb: (u8 | io.eof | io.error | bufio.overflow) =
bufio.scanbyte(&sc);
match (rb) {
case let b: u8 => abort();
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => { };
};
} else {
let rr: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) =
bufio.scanrune(&sc);
match (rr) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => { };
};
};
bufio.finish(&sc);
i += 1;
};
};

4
lib/regex/regex.ww
View File

@@ -1206,6 +1206,10 @@ fn search(
};
case let e: io.error => abort("regex: scanrune io error");
case utf8.invalid => abort("regex: scanrune invalid utf8");
// Unreachable: this scanner is newscanner(h, I32_MAX), so the
// can't-grow ceiling is never hit; the arm keeps the match
// total over scanrune's widened overflow surface (drain F-B).
case bufio.overflow => abort("regex: scanrune overflow");
};
if (r_or_end is rune) {
last_bytesize = (utf8.runesz(r_or_end as rune): size);

1
test/wcc/778_bufio_vstream_run.c
View File

@@ -326,6 +326,7 @@ static const struct row rows[] = {
" case let b: u8 => { fb = b; };\n"
" case io.eof => { return 81; };\n"
" case let e: io.error => { return 82; };\n"
" case bufio.overflow => { return 92; };\n"
" };\n"
" if (fb != 65u8) { return 83; };\n"
" match (bufio.scanbytes(&sc, 44u8)) {\n"