Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bufio: readahead signals overflow when it cannot grow (F-B)

Browse Source 
The scanner readahead silently fell through when start==0 and the
buffer was full at maxread, producing no bytes and no error. scanbyte
then spun forever re-requesting bytes that never came (catB-144) and
scanrune nil-dereferenced s.ptr[s.start] (catB-145).

Make readahead the single overflow choke-point: at the ceiling it
returns a bufio-local `overflow` before the grow, propagated through
scanbyte/scanrune/scanbytes (scanbytes drops its now-redundant manual
pre-check). Mirrors ref/hare/bufio/scanner.ha:174-182, which returns
errors::overflow there; ww uses bufio-local overflow because io.error
is a closed enum without an overflow member. Consumers (regex, the 778
embedded source) gain the totality arm.

Table-driven @test crosses {nil-ptr, zero-len} x {scanbyte, scanrune};
neutralizing the overflow return reproduces the catB-144 hang.
This commit is contained in:
Hojun-Cho
2026-06-14 11:54:20 +09:00
parent 1752305be7
commit 851915e6fe
4 changed files with 125 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
lib/bufio/bufio.ww
View File

@@ -7,11 +7,11 @@
// bufio.newscanner (src: io.stream, maxread: i32) scanner
// bufio.newscannerbuf(src: io.stream, buf: []u8) scanner
// bufio.finish (s: *scanner) void
// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error)
// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error | overflow)
// bufio.scanbytes (s: *scanner, delim: u8)
// ([]u8 | io.eof | io.error | overflow)
// bufio.scanrune (s: *scanner)
// (rune | io.eof | io.error | utf8.invalid)
// (rune | io.eof | io.error | utf8.invalid | overflow)
// bufio.scanline (s: *scanner) (str | io.eof | io.error | overflow)
//
// bufio.init (src: io.stream, rbuf: []u8, wbuf: []u8) stream
@@ -356,20 +356,25 @@ export fn finish(s: *scanner) void = {
};
// readahead — make room and read once from src into the back of the
// pending region. Returns bytes newly buffered (>=0), or io.eof/io.error
// from src. The size from io.read narrows to i32 (buffer-length type).
// pending region. Returns bytes newly buffered (>=0), io.eof/io.error
// from src, or overflow when the buffer is full at the maxread ceiling.
// The size from io.read narrows to i32 (buffer-length type).
//
// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer
// first shifts pending left, then — when start == 0 and maxread allows
// — grows. Hare grows via `append(scan.buffer, [0...], readahead)?`;
// ww's flat ptr/cap scanner allocates a fresh backing and copies (the
// old block is left to process-exit reclaim, ww no-free; `!` not `?`
// per the #36 nomem-propagation gap). The can't-grow case (avail >=
// maxread) is Hare's errors::overflow return — ww's io.error has no
// overflow member (lib/io/types.ww:40 enumerated union), so callers
// that can overflow (scanbytes) detect that state themselves before
// calling, against `maxread`.
fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
// first shifts pending left, then — when start == 0 — grows, or returns
// overflow when `avail >= want` (Hare's `pending >= readahead` ceiling,
// scanner.ha:179-181, BEFORE the append). Hare grows via
// `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner
// allocates a fresh backing and copies (the old block is left to
// process-exit reclaim, ww no-free; `!` not `?` per the #36
// nomem-propagation gap). overflow is bufio-local (bufio.ww:77), NOT
// errors.overflow: ww's io.error is a closed enumerated union with no
// overflow member (lib/io/types.ww), so the can't-grow signal rides a
// distinct arm that scanbyte / scanbytes / scanrune match-forward —
// the single choke-point (drew ruling, drain F-B). Pre-fix this case
// fell through silently to a zero-length io.read, spinning scanbyte
// (catB-144) and nil-derefing scanrune (catB-145).
fn readahead(s: *scanner) (i32 | io.eof | io.error | overflow) = {
if (s.start + s.avail == s.cap) {
if (s.start > 0) {
let i: i32 = 0;
@@ -378,9 +383,12 @@ fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
i += 1;
};
s.start = 0;
} else if (s.avail < s.maxread) {
} else {
let want: i32 = s.avail + BUFSZ;
if (want > s.maxread) { want = s.maxread; };
if (s.avail >= want) {
let e: overflow; return e;
};
let ncap: i32 = s.avail + want;
let nbuf: []u8 = alloc([], ncap: u64)!;
let np: *u8 = nbuf.ptr;
@@ -410,13 +418,14 @@ fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
// scanbyte — pop one byte, refilling from src on demand. Mirrors
// ref/hare/bufio/scanner.ha:204.
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error) = {
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = {
for (s.avail == 0) {
let r: (i32 | io.eof | io.error) = readahead(s);
let r: (i32 | io.eof | io.error | overflow) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
};
};
let b: u8 = s.ptr[s.start];
@@ -446,19 +455,15 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
};
i += 1;
};
// full + unshiftable + ungrowable (avail >= maxread is
// Hare's `pending >= readahead` overflow predicate,
// scanner.ha:179; for a newscannerbuf scanner maxread ==
// cap so this is the old fixed-buffer-full test).
if (s.start + s.avail == s.cap && s.start == 0
&& s.avail >= s.maxread) {
let e: overflow; return e;
};
let r: (i32 | io.eof | io.error) = readahead(s);
// overflow now surfaces from readahead's single choke-point
// (avail >= want == Hare's `pending >= readahead`,
// scanner.ha:179); no separate pre-check (drew ruling, F-B).
let r: (i32 | io.eof | io.error | overflow) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
};
};
let e: io.eof; return e;
@@ -469,20 +474,21 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
// announces) is utf8.invalid; a clean EOF before any byte is io.eof.
// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead
// when fewer than 4 bytes (the longest codepoint) are pending, then
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` also
// propagates errors::overflow through io::error; ww's io.error has no
// overflow member (see readahead) and a <4-byte refill cannot overflow
// a scanner that can hold a codepoint, so only io.error proper
// propagates here.
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid) = {
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` propagates
// errors::overflow (through io::error); ww forwards the bufio-local
// overflow as a distinct arm — a zero-cap / at-ceiling scanner cannot
// buffer the first byte, so without this arm the readahead fall-through
// would nil-deref s.ptr[s.start] (catB-145).
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overflow) = {
if (s.avail < 4) {
let ra: (i32 | io.eof | io.error) = readahead(s);
let ra: (i32 | io.eof | io.error | overflow) = readahead(s);
match (ra) {
case let n: i32 => { };
case io.eof => {
if (s.avail == 0) { let e: io.eof; return e; };
};
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
};
};
let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]);