diff --git a/lib/bufio/scanner.ww b/lib/bufio/scanner.ww index 9adeb942..636cca23 100644 --- a/lib/bufio/scanner.ww +++ b/lib/bufio/scanner.ww @@ -1,19 +1,15 @@ package bufio; import encoding.utf8; +import errors; import io; +import os; // ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth // increment. i32 (not Hare's size) per the lib-wide index-type // convention (lib/CLAUDE.md). def BUFSZ: i32 = 4096; -// overflow — the scanner buffer filled before the delimiter (or -// underlying EOF) was hit. With a caller-supplied buffer we can't -// grow; bumping the budget is on the caller. Mirrors Hare bufio's -// use of errors::overflow for the same condition. -export type overflow = !void; - // Plain tokenizer — no embedded vtable; driven directly via scanbyte / // scanbytes / scanline, not through io dispatch. Its `src` is an // io.stream and refills go through io.read. Value-return constructor @@ -29,6 +25,8 @@ export type scanner = struct { start: i32, // index where the pending region starts in ptr avail: i32, // pending byte count; pending = ptr[start..start+avail] maxread: i32, // growth ceiling; == cap for newscannerbuf (scanner.ha:101) + owned: bool, // ptr is scanner-owned and finish must release it + eof: bool, // source reached EOF or returned no progress }; // newscanner — wire a scanner that allocates and grows its own @@ -40,6 +38,8 @@ export type scanner = struct { // defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD // behavior is the shipped default). export fn newscanner(src: io.stream, maxread: i32) scanner = { + assert(maxread >= 0, + "bufio.newscanner: maximum read size must not be negative"); let r: scanner; r.src = src; r.ptr = nil; @@ -47,6 +47,8 @@ export fn newscanner(src: io.stream, maxread: i32) scanner = { r.start = 0; r.avail = 0; r.maxread = maxread; + r.owned = true; + r.eof = false; return r; }; @@ -62,88 +64,103 @@ export fn newscannerbuf(src: io.stream, buf: []u8) scanner = { r.start = 0; r.avail = 0; r.maxread = buf.len; + r.owned = false; + r.eof = false; return r; }; -// finish — release scanner-owned resources; src isn't closed. Mirrors -// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the -// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a -// newscannerbuf caller's buffer is untouched either way. +// finish — release scanner-owned resources; src isn't closed. A fixed +// newscannerbuf window is borrowed and remains the caller's property. +// Mirrors ref/hare/bufio/scanner.ha:110. export fn finish(s: *scanner) void = { - free(s.ptr); + if (s.owned && s.cap > 0) { + os.free(s.ptr: *void, s.cap: u64); + }; + s.ptr = nil; + s.cap = 0; + s.start = 0; + s.avail = 0; + s.maxread = 0; + s.owned = false; + s.eof = true; }; // readahead — make room and read once from src into the back of the -// pending region. Returns bytes newly buffered (>=0), io.eof/io.error -// from src, or overflow when the buffer is full at the maxread ceiling. +// pending region. Returns bytes newly buffered (>0), io.eof/io.error +// from src, or errors.overflow at the maxread ceiling. A zero-byte read +// is terminal EOF: retryable sources must return an explicit error. // The size from io.read narrows to i32 (buffer-length type). // // Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer // first shifts pending left, then — when start == 0 — grows, or returns -// overflow when `avail >= want` (Hare's `pending >= readahead` ceiling, +// overflow when no capacity remains (Hare's `pending >= readahead` ceiling, // scanner.ha:179-181, BEFORE the append). Hare grows via // `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner -// allocates a fresh backing and copies (the old block is left to -// process-exit reclaim, ww no-free; `!` not `?` per the #36 -// nomem-propagation gap). overflow is bufio-local (scanner.ww:15), NOT -// errors.overflow: ww's io.error is a closed enumerated union with no -// overflow member (lib/io/types.ww), so the can't-grow signal rides a -// distinct arm that scanbyte / scanbytes / scanrune match-forward — -// the single choke-point (drew ruling, drain F-B). Pre-fix this case -// fell through silently to a zero-length io.read, spinning scanbyte -// (catB-144) and nil-derefing scanrune (catB-145). -fn readahead(s: *scanner) (i32 | io.eof | io.error | overflow) = { +// allocates a fresh backing, copies pending bytes, and releases the +// replaced owned block. Allocation remains `!` rather than `?` per the +// #36 nomem-propagation gap. The generic overflow condition rides a +// distinct arm because io.error is closed (lib/io/types.ww). +fn readahead(s: *scanner) (i32 | io.eof | io.error | errors.overflow) = { + if (s.eof) { let e: io.eof; return e; }; if (s.start + s.avail == s.cap) { if (s.start > 0) { - let i: i32 = 0; - for (i < s.avail) { - s.ptr[i] = s.ptr[s.start + i]; - i += 1; - }; + bufiomove(s.ptr, s.ptr + (s.start: u64), s.avail); s.start = 0; } else { - let want: i32 = s.avail + BUFSZ; - if (want > s.maxread) { want = s.maxread; }; - if (s.avail >= want) { - let e: overflow; return e; + let room: i32 = s.maxread - s.avail; + if (room <= 0) { + let e: errors.overflow; return e; }; - let ncap: i32 = s.avail + want; + let ncap: i32 = s.maxread; + if (room > BUFSZ) { ncap = s.avail + BUFSZ; }; let nbuf: []u8 = alloc([], ncap: u64)!; let np: *u8 = nbuf.ptr; - let i: i32 = 0; - for (i < s.avail) { - np[i] = s.ptr[i]; - i += 1; + bufiomove(np, s.ptr, s.avail); + if (s.owned && s.cap > 0) { + os.free(s.ptr: *void, s.cap: u64); }; s.ptr = np; s.cap = ncap; + s.owned = true; }; }; let off: i32 = s.start + s.avail; let v: []u8; v.ptr = s.ptr + (off: u64); v.len = s.cap - off; + v.cap = v.len; let r: (size | io.eof | io.error) = io.read(s.src, v); match (r) { case let n: size => { + assert(n <= v.len: size, + "bufio scanner: reader returned an oversized count"); + if (n == 0: size) { + s.eof = true; + let e: io.eof; + return e; + }; s.avail += n: i32; return n: i32; }; - case io.eof => { let e: io.eof; return e; }; + case io.eof => { + s.eof = true; + let e: io.eof; + return e; + }; case let e: io.error => return e; }; }; // scanbyte — pop one byte, refilling from src on demand. Mirrors // ref/hare/bufio/scanner.ha:204. -export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = { +export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | errors.overflow) = { for (s.avail == 0) { - let r: (i32 | io.eof | io.error | overflow) = readahead(s); + let r: (i32 | io.eof | io.error | errors.overflow) = readahead(s); match (r) { case let n: i32 => { }; case io.eof => { let e: io.eof; return e; }; case let e: io.error => return e; - case overflow => { let e: overflow; return e; }; + case errors.overflow => { let e: errors.overflow; return e; }; }; }; let b: u8 = s.ptr[s.start]; @@ -157,9 +174,9 @@ export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = { // borrows from the scanner buffer and is invalidated by the next scan. // EOF without delim discards the trailing fragment and returns io.eof // (Hare EOF_DISCARD default); buffer-full without delim returns -// overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte +// errors.overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte // form is #217. Mirrors ref/hare/bufio/scanner.ha:220 (narrowed). -export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow) = { +export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | errors.overflow) = { let i: i32 = 0; for (true) { for (i < s.avail) { @@ -167,21 +184,20 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow let v: []u8; v.ptr = s.ptr + (s.start: u64); v.len = i; + v.cap = v.len; s.start += i + 1; s.avail -= i + 1; return v; }; i += 1; }; - // overflow now surfaces from readahead's single choke-point - // (avail >= want == Hare's `pending >= readahead`, - // scanner.ha:179); no separate pre-check (drew ruling, F-B). - let r: (i32 | io.eof | io.error | overflow) = readahead(s); + // Capacity failure surfaces from readahead's single choke-point. + let r: (i32 | io.eof | io.error | errors.overflow) = readahead(s); match (r) { case let n: i32 => { }; case io.eof => { let e: io.eof; return e; }; case let e: io.error => return e; - case overflow => { let e: overflow; return e; }; + case errors.overflow => { let e: errors.overflow; return e; }; }; }; let e: io.eof; return e; @@ -190,46 +206,73 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow // scanrune — pop one UTF-8-encoded rune, refilling from src on demand. // EOF mid-codepoint (fewer pending bytes than the initial byte // announces) is utf8.invalid; a clean EOF before any byte is io.eof. -// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead -// when fewer than 4 bytes (the longest codepoint) are pending, then -// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` propagates -// errors::overflow (through io::error); ww forwards the bufio-local -// overflow as a distinct arm — a zero-cap / at-ceiling scanner cannot -// buffer the first byte, so without this arm the readahead fall-through -// would nil-deref s.ptr[s.start] (catB-145). -export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overflow) = { - if (s.avail < 4) { - let ra: (i32 | io.eof | io.error | overflow) = readahead(s); +// Partial reads are refilled until the announced width is present. +// Decode happens before consumption: malformed input consumes only the +// bytes visited by utf8.next, preserving any later pending byte. +export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | errors.overflow) = { + for (s.avail == 0) { + let ra: (i32 | io.eof | io.error | errors.overflow) = readahead(s); match (ra) { case let n: i32 => { }; - case io.eof => { - if (s.avail == 0) { let e: io.eof; return e; }; - }; + case io.eof => { let e: io.eof; return e; }; case let e: io.error => return e; - case overflow => { let e: overflow; return e; }; + case errors.overflow => { let e: errors.overflow; return e; }; }; }; let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]); let sz: i32 = 0; match (szr) { case let n: i32 => { sz = n; }; - case utf8.invalid => { let e: utf8.invalid; return e; }; + case utf8.invalid => { + s.start += 1; + s.avail -= 1; + let e: utf8.invalid; + return e; }; - if (s.avail < sz) { - let e: utf8.invalid; return e; + }; + for (s.avail < sz) { + let ra: (i32 | io.eof | io.error | errors.overflow) = readahead(s); + match (ra) { + case let n: i32 => { }; + case io.eof => { + s.start += s.avail; + s.avail = 0; + let e: utf8.invalid; + return e; + }; + case let e: io.error => return e; + case errors.overflow => { let e: errors.overflow; return e; }; + }; }; let v: []u8; v.ptr = s.ptr + (s.start: u64); v.len = sz; - s.start += sz; - s.avail -= sz; + v.cap = v.len; let dec: utf8.decoder = utf8.decode(v); let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec); + let used: i32 = utf8.position(&dec): i32; match (nr) { - case let r: rune => return r; - case utf8.done => { let e: io.eof; return e; }; - case utf8.more => { let e: utf8.invalid; return e; }; - case utf8.invalid => { let e: utf8.invalid; return e; }; + case let r: rune => { + assert(used == sz, "bufio.scanrune: decoder width mismatch"); + s.start += used; + s.avail -= used; + return r; + }; + case utf8.done => abort("bufio.scanrune: decoder made no progress"); + case utf8.more => { + s.start += used; + s.avail -= used; + let e: utf8.invalid; + return e; + }; + case utf8.invalid => { + assert(used > 0 && used <= sz, + "bufio.scanrune: invalid decoder position"); + s.start += used; + s.avail -= used; + let e: utf8.invalid; + return e; + }; }; }; @@ -238,17 +281,18 @@ export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overf // Single-byte route (Hare's scan_line = scan_string(s, "\n"); the // arbitrary multibyte-delim scan_string is #217). Mirrors // ref/hare/bufio/scanner.ha:307. -export fn scanline(s: *scanner) (str | io.eof | io.error | overflow) = { - let r: ([]u8 | io.eof | io.error | overflow) = scanbytes(s, 10u8); +export fn scanline(s: *scanner) (str | io.eof | io.error | errors.overflow) = { + let r: ([]u8 | io.eof | io.error | errors.overflow) = scanbytes(s, 10u8); match (r) { case let bs: []u8 => { let v: str; v.ptr = bs.ptr; v.len = bs.len; + v.cap = v.len; return v; }; case io.eof => { let e: io.eof; return e; }; case let e: io.error => return e; - case overflow => { let e: overflow; return e; }; + case errors.overflow => { let e: errors.overflow; return e; }; }; }; diff --git a/lib/bufio/scanner_test.ww b/lib/bufio/scanner_test.ww index 7bd0de3b..f73a9792 100644 --- a/lib/bufio/scanner_test.ww +++ b/lib/bufio/scanner_test.ww @@ -6,8 +6,10 @@ package bufio_test; import bufio; import encoding.utf8; +import errors; import io; import memio; +import test; fn putstr(s: str, into: []u8, off: i32) i32 = { let i: i32 = 0; @@ -32,6 +34,66 @@ fn errsource() io.stream = { return &errvt; }; +// stepstream is a deliberately short-reading source. `zero` makes its +// first call return no progress so scanner EOF handling is observable. +type stepstream = struct { + vt: io.vtable, + src: []u8, + pos: i32, + calls: i32, + step: i32, + zero: bool, + badcap: bool, +}; + +fn stepread(s: io.stream, buf: []u8) (size | io.eof | io.error) = { + let r: *stepstream = s: *stepstream; + r.calls += 1; + if (buf.cap != buf.len) { r.badcap = true; }; + if (r.zero) { + r.zero = false; + return 0: size; + }; + if (r.pos == r.src.len) { let e: io.eof; return e; }; + let n: i32 = r.step; + if (n > buf.len) { n = buf.len; }; + if (n > r.src.len - r.pos) { n = r.src.len - r.pos; }; + let i: i32 = 0; + for (i < n) { + buf[i] = r.src[r.pos + i]; + i += 1; + }; + r.pos += n; + return n: size; +}; + +fn newstepstream(src: []u8, step: i32, zero: bool) stepstream = { + let r: stepstream; + r.vt.reader = (&stepread): *io.reader; + r.src = src; + r.pos = 0; + r.calls = 0; + r.step = step; + r.zero = zero; + r.badcap = false; + return r; +}; + +def WANT_INVALID: i32 = -1; +def WANT_EOF: i32 = -2; + +fn checkrune(sc: *bufio.scanner, want: i32) void = { + let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = + bufio.scanrune(sc); + match (r) { + case let got: rune => assert(want >= 0 && got: i32 == want); + case io.eof => assert(want == WANT_EOF); + case let e: io.error => abort(); + case utf8.invalid => assert(want == WANT_INVALID); + case errors.overflow => abort(); + }; +}; + @test fn scanbytecases() void = { let raw: [4]u8; raw[0] = 11u8; raw[1] = 22u8; raw[2] = 33u8; raw[3] = 44u8; @@ -53,7 +115,7 @@ fn errsource() io.stream = { let i: i32 = 0; for (i < 6) { - let r: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc); + let r: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc); match (r) { case let b: u8 => { assert(!(weof[i] != 0)); @@ -61,7 +123,7 @@ fn errsource() io.stream = { }; case io.eof => { assert(!(weof[i] == 0)); }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; i += 1; }; @@ -91,17 +153,18 @@ fn errsource() io.stream = { let i: i32 = 0; for (i < 5) { - let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (r) { case let v: str => { assert(!(weof[i] != 0)); assert(!(wovr[i] != 0)); assert(!(v.len != wl[i])); + assert(v.cap == v.len); assert(!(v.len > 0 && v[0] != wf[i])); }; case io.eof => { assert(!(weof[i] == 0)); }; case let e: io.error => abort(); - case bufio.overflow => { assert(!(wovr[i] == 0)); }; + case errors.overflow => { assert(!(wovr[i] == 0)); }; }; i += 1; }; @@ -120,12 +183,12 @@ fn errsource() io.stream = { let buf: [4]u8; let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:4]); - let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (r) { case let v: str => abort(); case io.eof => abort(); case let e: io.error => abort(); - case bufio.overflow => { }; + case errors.overflow => { }; }; bufio.finish(&sc); @@ -153,16 +216,17 @@ fn errsource() io.stream = { let i: i32 = 0; for (i < 5) { - let r: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 44u8); // ',' + let r: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 44u8); // ',' match (r) { case let v: []u8 => { assert(!(weof[i] != 0)); assert(!(v.len != wl[i])); + assert(v.cap == v.len); assert(!(v.len > 0 && v[0] != wf[i])); }; case io.eof => { assert(!(weof[i] == 0)); }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; i += 1; }; @@ -177,28 +241,28 @@ fn errsource() io.stream = { let buf: [8]u8; let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]); - let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc); + let r1: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc); match (r1) { case let b: u8 => abort(); case io.eof => { }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; - let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (r2) { case let v: str => abort(); case io.eof => { }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; - let r3: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 10u8); + let r3: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 10u8); match (r3) { case let v: []u8 => abort(); case io.eof => { }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; bufio.finish(&sc); @@ -209,28 +273,28 @@ fn errsource() io.stream = { let buf: [8]u8; let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]); - let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc); + let r1: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc); match (r1) { case let b: u8 => abort(); case io.eof => abort(); case let e: io.error => { }; - case bufio.overflow => abort(); + case errors.overflow => abort(); }; - let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (r2) { case let v: str => abort(); case io.eof => abort(); case let e: io.error => { }; - case bufio.overflow => abort(); + case errors.overflow => abort(); }; - let r3: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 32u8); + let r3: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 32u8); match (r3) { case let v: []u8 => abort(); case io.eof => abort(); case let e: io.error => { }; - case bufio.overflow => abort(); + case errors.overflow => abort(); }; bufio.finish(&sc); @@ -244,14 +308,14 @@ fn errsource() io.stream = { let b1: [8]u8; let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]); - let r1: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc1); + let r1: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc1); match (r1) { case let v: str => { assert(!(v.len != 0)); }; case io.eof => abort(); case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; - let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc1); + let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc1); match (r2) { case let v: str => { assert(!(v.len != 3)); @@ -259,7 +323,7 @@ fn errsource() io.stream = { }; case io.eof => abort(); case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; bufio.finish(&sc1); @@ -270,19 +334,19 @@ fn errsource() io.stream = { let b2: [8]u8; let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]); - let t1: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc2, 44u8); + let t1: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc2, 44u8); match (t1) { case let v: []u8 => abort(); case io.eof => { }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; - let t2: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc2, 44u8); + let t2: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc2, 44u8); match (t2) { case let v: []u8 => abort(); case io.eof => { }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; bufio.finish(&sc2); }; @@ -306,7 +370,7 @@ fn errsource() io.stream = { let i: i32 = 0; for (i < 3) { - let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (r) { case let v: str => { assert(!(weof[i] != 0)); @@ -315,7 +379,7 @@ fn errsource() io.stream = { }; case io.eof => { assert(!(weof[i] == 0)); }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; i += 1; }; @@ -350,7 +414,7 @@ fn errsource() io.stream = { let i: i32 = 0; for (i < 6) { - let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc); + let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc); match (r) { case let rn: rune => { assert(!(weof[i] != 0)); @@ -359,7 +423,7 @@ fn errsource() io.stream = { case io.eof => { assert(!(weof[i] == 0)); }; case let e: io.error => abort(); case utf8.invalid => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; i += 1; }; @@ -375,13 +439,13 @@ fn errsource() io.stream = { let m1: io.stream = &mem1.vt; let b1: [8]u8; let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]); - let r1: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc1); + let r1: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc1); match (r1) { case let rn: rune => abort(); case io.eof => abort(); case let e: io.error => abort(); case utf8.invalid => { }; - case bufio.overflow => abort(); + case errors.overflow => abort(); }; bufio.finish(&sc1); @@ -393,13 +457,13 @@ fn errsource() io.stream = { let m2: io.stream = &mem2.vt; let b2: [8]u8; let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]); - let r2: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2); + let r2: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc2); match (r2) { case let rn: rune => abort(); case io.eof => abort(); case let e: io.error => abort(); case utf8.invalid => { }; - case bufio.overflow => abort(); + case errors.overflow => abort(); }; bufio.finish(&sc2); @@ -411,17 +475,127 @@ fn errsource() io.stream = { let m3: io.stream = &mem3.vt; let b3: [8]u8; let sc3: bufio.scanner = bufio.newscannerbuf(m3, b3[0:8]); - let r3: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc3); + let r3: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc3); match (r3) { case let rn: rune => abort(); case io.eof => abort(); case let e: io.error => abort(); case utf8.invalid => { }; - case bufio.overflow => abort(); + case errors.overflow => abort(); }; bufio.finish(&sc3); }; +@test fn scanrunepartialreads() void = { + // Each read yields one byte, including across a shifted 4-byte rune. + let raw: [6]u8; + raw[0] = 0xC3u8; raw[1] = 0xA9u8; + raw[2] = 0xF0u8; raw[3] = 0x9Fu8; + raw[4] = 0x98u8; raw[5] = 0x80u8; + let src: stepstream = newstepstream(raw[0:6], 1, false); + let buf: [4]u8; + let sc: bufio.scanner = bufio.newscannerbuf(&src.vt, buf[0:4]); + checkrune(&sc, 233); + checkrune(&sc, 128512); + checkrune(&sc, WANT_EOF); + assert(!src.badcap); + assert(src.calls == 7); + bufio.finish(&sc); +}; + +@test fn scanrunemalformedprogress() void = { + // The decoder rejects E0 80 at position 2; the pending ASCII byte + // beyond that position must remain for the next scan. + let raw1: [3]u8; + raw1[0] = 0xE0u8; raw1[1] = 0x80u8; raw1[2] = 65u8; + let mem1: memio.stream = memio.fixed(raw1[0:3]); + let buf1: [8]u8; + let sc1: bufio.scanner = bufio.newscannerbuf(&mem1.vt, buf1[0:8]); + checkrune(&sc1, WANT_INVALID); + checkrune(&sc1, 65); + bufio.finish(&sc1); + + // Impossible F5-F7 leads consume one byte, not their following ASCII. + let raw2: [4]u8; + raw2[0] = 0xF5u8; raw2[1] = 65u8; + raw2[2] = 66u8; raw2[3] = 67u8; + let mem2: memio.stream = memio.fixed(raw2[0:4]); + let buf2: [8]u8; + let sc2: bufio.scanner = bufio.newscannerbuf(&mem2.vt, buf2[0:8]); + checkrune(&sc2, WANT_INVALID); + let want: [3]u32; + want[0] = 65u32; want[1] = 66u32; want[2] = 67u32; + let i: i32 = 0; + for (i < 3) { + checkrune(&sc2, want[i]: i32); + i += 1; + }; + bufio.finish(&sc2); + + // A truncated prefix is consumed once; a following call is EOF. + let raw3: [1]u8; + raw3[0] = 0xC3u8; + let mem3: memio.stream = memio.fixed(raw3[0:1]); + let buf3: [8]u8; + let sc3: bufio.scanner = bufio.newscannerbuf(&mem3.vt, buf3[0:8]); + checkrune(&sc3, WANT_INVALID); + checkrune(&sc3, WANT_EOF); + bufio.finish(&sc3); +}; + +@test fn scannerzeroreadiseof() void = { + let raw: [1]u8; + raw[0] = 65u8; + let src: stepstream = newstepstream(raw[0:1], 1, true); + let buf: [4]u8; + let sc: bufio.scanner = bufio.newscannerbuf(&src.vt, buf[0:4]); + let i: i32 = 0; + for (i < 2) { + let r: (u8 | io.eof | io.error | errors.overflow) = + bufio.scanbyte(&sc); + match (r) { + case let b: u8 => abort(); + case io.eof => { }; + case let e: io.error => abort(); + case errors.overflow => abort(); + }; + i += 1; + }; + assert(src.calls == 1); + assert(src.pos == 0); + bufio.finish(&sc); +}; + +@test fn scannermaxreadbound() void = { + // The token and delimiter exceed maxread. The old growth expression + // allocated avail+want (8193 here) and incorrectly accepted them. + let raw: [4099]u8; + let i: i32 = 0; + for (i < 4098) { raw[i] = 65u8; i += 1; }; + raw[4098] = 10u8; + let mem: memio.stream = memio.fixed(raw[0:4099]); + let sc: bufio.scanner = bufio.newscanner(&mem.vt, 4097); + let r: (str | io.eof | io.error | errors.overflow) = + bufio.scanline(&sc); + match (r) { + case let s: str => abort(); + case io.eof => abort(); + case let e: io.error => abort(); + case errors.overflow => { }; + }; + assert(sc.cap == 4097); + assert(sc.avail == 4097); + assert(sc.owned); + bufio.finish(&sc); + assert(sc.ptr == nil); + assert(sc.cap == 0 && sc.avail == 0 && !sc.owned && sc.eof); +}; + +@test fn newscannernegativeaborts() void = { + test.expectabort(); + let sc: bufio.scanner = bufio.newscanner(errsource(), -1); +}; + @test fn newscannergrow() void = { let src: [16]u8; let n: i32 = putstr("hello\nworld\n", src[0:16], 0); @@ -441,7 +615,7 @@ fn errsource() io.stream = { let i: i32 = 0; for (i < 3) { - let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (r) { case let v: str => { assert(!(weof[i] != 0)); @@ -450,7 +624,7 @@ fn errsource() io.stream = { }; case io.eof => { assert(!(weof[i] == 0)); }; case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; i += 1; }; @@ -472,7 +646,7 @@ fn errsource() io.stream = { i = 0; for (i < 3) { - let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2); + let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc2); match (r) { case let rn: rune => { assert(!(weof2[i] != 0)); @@ -481,7 +655,7 @@ fn errsource() io.stream = { case io.eof => { assert(!(weof2[i] == 0)); }; case let e: io.error => abort(); case utf8.invalid => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; i += 1; }; @@ -496,12 +670,12 @@ fn errsource() io.stream = { let m: io.stream = &mem.vt; let sc: bufio.scanner = bufio.newscanner(m, 4); // token (8B) > maxread - let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (r) { case let v: str => abort(); case io.eof => abort(); case let e: io.error => abort(); - case bufio.overflow => { }; + case errors.overflow => { }; }; bufio.finish(&sc); }; @@ -510,7 +684,7 @@ fn errsource() io.stream = { // readahead unable to make progress. Pre-fix it fell through silently to // a zero-length io.read returning 0, so scanbyte spun forever (catB-144) // and scanrune dereferenced the nil / zero-cap s.ptr[s.start] -// (catB-145). Both entry points must now surface bufio.overflow +// (catB-145). Both entry points must now surface errors.overflow // (ref/hare/bufio/scanner.ha:179-181). The table crosses the two // zero-cap constructions — nil-ptr newscanner(,0) and zero-length // newscannerbuf — with the two single-item entry points; every row is an @@ -544,23 +718,23 @@ fn errsource() io.stream = { }; if (entry[i] == 0) { - let rb: (u8 | io.eof | io.error | bufio.overflow) = + let rb: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc); match (rb) { case let b: u8 => abort(); case io.eof => abort(); case let e: io.error => abort(); - case bufio.overflow => { }; + case errors.overflow => { }; }; } else { - let rr: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = + let rr: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc); match (rr) { case let rn: rune => abort(); case io.eof => abort(); case let e: io.error => abort(); case utf8.invalid => abort(); - case bufio.overflow => { }; + case errors.overflow => { }; }; }; diff --git a/lib/bufio/stream_test.ww b/lib/bufio/stream_test.ww index 7516fd32..79483550 100644 --- a/lib/bufio/stream_test.ww +++ b/lib/bufio/stream_test.ww @@ -5,6 +5,7 @@ package bufio_test; import bufio; +import errors; import io; import memio; @@ -415,7 +416,7 @@ fn checkpartialflush(zero: bool) void = { let sbuf: [32]u8; let sc: bufio.scanner = bufio.newscannerbuf(p, sbuf[0:32]); - let lr: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let lr: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (lr) { case let v: str => { assert(!(v.len != 8)); // "XYZhello" @@ -424,7 +425,7 @@ fn checkpartialflush(zero: bool) void = { }; case io.eof => abort(); case let e: io.error => abort(); - case bufio.overflow => abort(); + case errors.overflow => abort(); }; bufio.finish(&sc); }; diff --git a/lib/regex/regex.ww b/lib/regex/regex.ww index 7589bbb9..b9294151 100644 --- a/lib/regex/regex.ww +++ b/lib/regex/regex.ww @@ -23,6 +23,7 @@ package regex; import ascii; import bufio; +import errors; import io; import memio; import strconv; @@ -1209,7 +1210,7 @@ fn search( // Unreachable: this scanner is newscanner(h, I32_MAX), so the // can't-grow ceiling is never hit; the arm keeps the match // total over scanrune's widened overflow surface (drain F-B). - case bufio.overflow => abort("regex: scanrune overflow"); + case errors.overflow => abort("regex: scanrune overflow"); }; if (r_or_end is rune) { last_bytesize = (utf8.runesz(r_or_end as rune): size); diff --git a/test/wcc/data/r700_bufio_scanline_discard/case.ww b/test/wcc/data/r700_bufio_scanline_discard/case.ww index 34f8f4f1..e1c9efbc 100644 --- a/test/wcc/data/r700_bufio_scanline_discard/case.ww +++ b/test/wcc/data/r700_bufio_scanline_discard/case.ww @@ -2,6 +2,7 @@ // Migrated from 700_e2e row 104. package main; import bufio; +import errors; import io; import memio; fn main() i32 = { @@ -14,26 +15,26 @@ fn main() i32 = { let buf: [16]u8; let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:16]); let acc: i32 = 0; - let l1: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let l1: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (l1) { case let s: str => acc += s.len; case io.eof => acc += -100; case let _e: io.error => acc += -1000; - case bufio.overflow => acc += -10000; + case errors.overflow => acc += -10000; }; - let l2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let l2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (l2) { case let s: str => acc += s.len; case io.eof => acc += -100; case let _e: io.error => acc += -1000; - case bufio.overflow => acc += -10000; + case errors.overflow => acc += -10000; }; - let l3: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + let l3: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc); match (l3) { case let s: str => acc += -100; case io.eof => acc += 7; case let _e: io.error => acc += -1000; - case bufio.overflow => acc += -10000; + case errors.overflow => acc += -10000; }; return acc; }; diff --git a/test/wcc/data/r778_scanner_byte_bytes/case.ww b/test/wcc/data/r778_scanner_byte_bytes/case.ww index 9cef4493..13be133a 100644 --- a/test/wcc/data/r778_scanner_byte_bytes/case.ww +++ b/test/wcc/data/r778_scanner_byte_bytes/case.ww @@ -3,6 +3,7 @@ package main; import os; import bufio; +import errors; import memio; import io; export fn main() i32 = { @@ -18,20 +19,20 @@ export fn main() i32 = { case let b: u8 => { fb = b; }; case io.eof => { return 81; }; case let e: io.error => { return 82; }; - case bufio.overflow => { return 92; }; + case errors.overflow => { return 92; }; }; if (fb != 65u8) { return 83; }; match (bufio.scanbytes(&sc, 44u8)) { case let bs: []u8 => { if (bs.len != 1 || bs[0] != 66u8) { return 84; }; }; case io.eof => { return 85; }; case let e: io.error => { return 86; }; - case bufio.overflow => { return 87; }; + case errors.overflow => { return 87; }; }; match (bufio.scanbytes(&sc, 44u8)) { case let bs: []u8 => { if (bs.len != 2 || bs[0] != 67u8 || bs[1] != 68u8) { return 88; }; }; case io.eof => { return 89; }; case let e: io.error => { return 90; }; - case bufio.overflow => { return 91; }; + case errors.overflow => { return 91; }; }; return 53; }; diff --git a/test/wcc/data/r778_scanner_lines/case.ww b/test/wcc/data/r778_scanner_lines/case.ww index c0888548..26fce831 100644 --- a/test/wcc/data/r778_scanner_lines/case.ww +++ b/test/wcc/data/r778_scanner_lines/case.ww @@ -3,6 +3,7 @@ package main; import os; import bufio; +import errors; import memio; import io; export fn main() i32 = { @@ -19,19 +20,19 @@ export fn main() i32 = { case let s: str => { if (s.len == 2 && s[0] == 97u8 && s[1] == 98u8) { ok += 1; }; }; case io.eof => { return 81; }; case let e: io.error => { return 82; }; - case bufio.overflow => { return 83; }; + case errors.overflow => { return 83; }; }; match (bufio.scanline(&sc)) { case let s: str => { if (s.len == 1 && s[0] == 99u8) { ok += 1; }; }; case io.eof => { return 84; }; case let e: io.error => { return 85; }; - case bufio.overflow => { return 86; }; + case errors.overflow => { return 86; }; }; match (bufio.scanline(&sc)) { case let s: str => { return 87; }; case io.eof => { ok += 1; }; case let e: io.error => { return 88; }; - case bufio.overflow => { return 89; }; + case errors.overflow => { return 89; }; }; bufio.finish(&sc); if (ok != 3) { return 90; };