Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bufio: make scanning bounded and incremental

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-09 18:17:42 +09:00
parent 09b6f22e85
commit a04de3e02b
7 changed files with 362 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

272
lib/bufio/scanner_test.ww
View File

@@ -6,8 +6,10 @@ package bufio_test;
import bufio;
import encoding.utf8;
import errors;
import io;
import memio;
import test;
fn putstr(s: str, into: []u8, off: i32) i32 = {
let i: i32 = 0;
@@ -32,6 +34,66 @@ fn errsource() io.stream = {
return &errvt;
};
// stepstream is a deliberately short-reading source. `zero` makes its
// first call return no progress so scanner EOF handling is observable.
type stepstream = struct {
vt: io.vtable,
src: []u8,
pos: i32,
calls: i32,
step: i32,
zero: bool,
badcap: bool,
};
fn stepread(s: io.stream, buf: []u8) (size | io.eof | io.error) = {
let r: *stepstream = s: *stepstream;
r.calls += 1;
if (buf.cap != buf.len) { r.badcap = true; };
if (r.zero) {
r.zero = false;
return 0: size;
};
if (r.pos == r.src.len) { let e: io.eof; return e; };
let n: i32 = r.step;
if (n > buf.len) { n = buf.len; };
if (n > r.src.len - r.pos) { n = r.src.len - r.pos; };
let i: i32 = 0;
for (i < n) {
buf[i] = r.src[r.pos + i];
i += 1;
};
r.pos += n;
return n: size;
};
fn newstepstream(src: []u8, step: i32, zero: bool) stepstream = {
let r: stepstream;
r.vt.reader = (&stepread): *io.reader;
r.src = src;
r.pos = 0;
r.calls = 0;
r.step = step;
r.zero = zero;
r.badcap = false;
return r;
};
def WANT_INVALID: i32 = -1;
def WANT_EOF: i32 = -2;
fn checkrune(sc: *bufio.scanner, want: i32) void = {
let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) =
bufio.scanrune(sc);
match (r) {
case let got: rune => assert(want >= 0 && got: i32 == want);
case io.eof => assert(want == WANT_EOF);
case let e: io.error => abort();
case utf8.invalid => assert(want == WANT_INVALID);
case errors.overflow => abort();
};
};
@test fn scanbytecases() void = {
let raw: [4]u8;
raw[0] = 11u8; raw[1] = 22u8; raw[2] = 33u8; raw[3] = 44u8;
@@ -53,7 +115,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 6) {
let r: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
let r: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc);
match (r) {
case let b: u8 => {
assert(!(weof[i] != 0));
@@ -61,7 +123,7 @@ fn errsource() io.stream = {
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -91,17 +153,18 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 5) {
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => {
assert(!(weof[i] != 0));
assert(!(wovr[i] != 0));
assert(!(v.len != wl[i]));
assert(v.cap == v.len);
assert(!(v.len > 0 && v[0] != wf[i]));
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => { assert(!(wovr[i] == 0)); };
case errors.overflow => { assert(!(wovr[i] == 0)); };
};
i += 1;
};
@@ -120,12 +183,12 @@ fn errsource() io.stream = {
let buf: [4]u8;
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:4]);
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => abort();
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => { };
case errors.overflow => { };
};
bufio.finish(&sc);
@@ -153,16 +216,17 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 5) {
let r: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 44u8); // ','
let r: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 44u8); // ','
match (r) {
case let v: []u8 => {
assert(!(weof[i] != 0));
assert(!(v.len != wl[i]));
assert(v.cap == v.len);
assert(!(v.len > 0 && v[0] != wf[i]));
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -177,28 +241,28 @@ fn errsource() io.stream = {
let buf: [8]u8;
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]);
let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
let r1: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc);
match (r1) {
case let b: u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r2) {
case let v: str => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r3: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 10u8);
let r3: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 10u8);
match (r3) {
case let v: []u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc);
@@ -209,28 +273,28 @@ fn errsource() io.stream = {
let buf: [8]u8;
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]);
let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
let r1: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc);
match (r1) {
case let b: u8 => abort();
case io.eof => abort();
case let e: io.error => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r2) {
case let v: str => abort();
case io.eof => abort();
case let e: io.error => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r3: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 32u8);
let r3: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 32u8);
match (r3) {
case let v: []u8 => abort();
case io.eof => abort();
case let e: io.error => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc);
@@ -244,14 +308,14 @@ fn errsource() io.stream = {
let b1: [8]u8;
let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]);
let r1: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc1);
let r1: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc1);
match (r1) {
case let v: str => { assert(!(v.len != 0)); };
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc1);
let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc1);
match (r2) {
case let v: str => {
assert(!(v.len != 3));
@@ -259,7 +323,7 @@ fn errsource() io.stream = {
};
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc1);
@@ -270,19 +334,19 @@ fn errsource() io.stream = {
let b2: [8]u8;
let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]);
let t1: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc2, 44u8);
let t1: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc2, 44u8);
match (t1) {
case let v: []u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
let t2: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc2, 44u8);
let t2: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc2, 44u8);
match (t2) {
case let v: []u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc2);
};
@@ -306,7 +370,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 3) {
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => {
assert(!(weof[i] != 0));
@@ -315,7 +379,7 @@ fn errsource() io.stream = {
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -350,7 +414,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 6) {
let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc);
let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc);
match (r) {
case let rn: rune => {
assert(!(weof[i] != 0));
@@ -359,7 +423,7 @@ fn errsource() io.stream = {
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -375,13 +439,13 @@ fn errsource() io.stream = {
let m1: io.stream = &mem1.vt;
let b1: [8]u8;
let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]);
let r1: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc1);
let r1: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc1);
match (r1) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc1);
@@ -393,13 +457,13 @@ fn errsource() io.stream = {
let m2: io.stream = &mem2.vt;
let b2: [8]u8;
let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]);
let r2: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2);
let r2: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc2);
match (r2) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc2);
@@ -411,17 +475,127 @@ fn errsource() io.stream = {
let m3: io.stream = &mem3.vt;
let b3: [8]u8;
let sc3: bufio.scanner = bufio.newscannerbuf(m3, b3[0:8]);
let r3: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc3);
let r3: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc3);
match (r3) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc3);
};
@test fn scanrunepartialreads() void = {
// Each read yields one byte, including across a shifted 4-byte rune.
let raw: [6]u8;
raw[0] = 0xC3u8; raw[1] = 0xA9u8;
raw[2] = 0xF0u8; raw[3] = 0x9Fu8;
raw[4] = 0x98u8; raw[5] = 0x80u8;
let src: stepstream = newstepstream(raw[0:6], 1, false);
let buf: [4]u8;
let sc: bufio.scanner = bufio.newscannerbuf(&src.vt, buf[0:4]);
checkrune(&sc, 233);
checkrune(&sc, 128512);
checkrune(&sc, WANT_EOF);
assert(!src.badcap);
assert(src.calls == 7);
bufio.finish(&sc);
};
@test fn scanrunemalformedprogress() void = {
// The decoder rejects E0 80 at position 2; the pending ASCII byte
// beyond that position must remain for the next scan.
let raw1: [3]u8;
raw1[0] = 0xE0u8; raw1[1] = 0x80u8; raw1[2] = 65u8;
let mem1: memio.stream = memio.fixed(raw1[0:3]);
let buf1: [8]u8;
let sc1: bufio.scanner = bufio.newscannerbuf(&mem1.vt, buf1[0:8]);
checkrune(&sc1, WANT_INVALID);
checkrune(&sc1, 65);
bufio.finish(&sc1);
// Impossible F5-F7 leads consume one byte, not their following ASCII.
let raw2: [4]u8;
raw2[0] = 0xF5u8; raw2[1] = 65u8;
raw2[2] = 66u8; raw2[3] = 67u8;
let mem2: memio.stream = memio.fixed(raw2[0:4]);
let buf2: [8]u8;
let sc2: bufio.scanner = bufio.newscannerbuf(&mem2.vt, buf2[0:8]);
checkrune(&sc2, WANT_INVALID);
let want: [3]u32;
want[0] = 65u32; want[1] = 66u32; want[2] = 67u32;
let i: i32 = 0;
for (i < 3) {
checkrune(&sc2, want[i]: i32);
i += 1;
};
bufio.finish(&sc2);
// A truncated prefix is consumed once; a following call is EOF.
let raw3: [1]u8;
raw3[0] = 0xC3u8;
let mem3: memio.stream = memio.fixed(raw3[0:1]);
let buf3: [8]u8;
let sc3: bufio.scanner = bufio.newscannerbuf(&mem3.vt, buf3[0:8]);
checkrune(&sc3, WANT_INVALID);
checkrune(&sc3, WANT_EOF);
bufio.finish(&sc3);
};
@test fn scannerzeroreadiseof() void = {
let raw: [1]u8;
raw[0] = 65u8;
let src: stepstream = newstepstream(raw[0:1], 1, true);
let buf: [4]u8;
let sc: bufio.scanner = bufio.newscannerbuf(&src.vt, buf[0:4]);
let i: i32 = 0;
for (i < 2) {
let r: (u8 | io.eof | io.error | errors.overflow) =
bufio.scanbyte(&sc);
match (r) {
case let b: u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case errors.overflow => abort();
};
i += 1;
};
assert(src.calls == 1);
assert(src.pos == 0);
bufio.finish(&sc);
};
@test fn scannermaxreadbound() void = {
// The token and delimiter exceed maxread. The old growth expression
// allocated avail+want (8193 here) and incorrectly accepted them.
let raw: [4099]u8;
let i: i32 = 0;
for (i < 4098) { raw[i] = 65u8; i += 1; };
raw[4098] = 10u8;
let mem: memio.stream = memio.fixed(raw[0:4099]);
let sc: bufio.scanner = bufio.newscanner(&mem.vt, 4097);
let r: (str | io.eof | io.error | errors.overflow) =
bufio.scanline(&sc);
match (r) {
case let s: str => abort();
case io.eof => abort();
case let e: io.error => abort();
case errors.overflow => { };
};
assert(sc.cap == 4097);
assert(sc.avail == 4097);
assert(sc.owned);
bufio.finish(&sc);
assert(sc.ptr == nil);
assert(sc.cap == 0 && sc.avail == 0 && !sc.owned && sc.eof);
};
@test fn newscannernegativeaborts() void = {
test.expectabort();
let sc: bufio.scanner = bufio.newscanner(errsource(), -1);
};
@test fn newscannergrow() void = {
let src: [16]u8;
let n: i32 = putstr("hello\nworld\n", src[0:16], 0);
@@ -441,7 +615,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 3) {
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => {
assert(!(weof[i] != 0));
@@ -450,7 +624,7 @@ fn errsource() io.stream = {
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -472,7 +646,7 @@ fn errsource() io.stream = {
i = 0;
for (i < 3) {
let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2);
let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc2);
match (r) {
case let rn: rune => {
assert(!(weof2[i] != 0));
@@ -481,7 +655,7 @@ fn errsource() io.stream = {
case io.eof => { assert(!(weof2[i] == 0)); };
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -496,12 +670,12 @@ fn errsource() io.stream = {
let m: io.stream = &mem.vt;
let sc: bufio.scanner = bufio.newscanner(m, 4); // token (8B) > maxread
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => abort();
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => { };
case errors.overflow => { };
};
bufio.finish(&sc);
};
@@ -510,7 +684,7 @@ fn errsource() io.stream = {
// readahead unable to make progress. Pre-fix it fell through silently to
// a zero-length io.read returning 0, so scanbyte spun forever (catB-144)
// and scanrune dereferenced the nil / zero-cap s.ptr[s.start]
// (catB-145). Both entry points must now surface bufio.overflow
// (catB-145). Both entry points must now surface errors.overflow
// (ref/hare/bufio/scanner.ha:179-181). The table crosses the two
// zero-cap constructions — nil-ptr newscanner(,0) and zero-length
// newscannerbuf — with the two single-item entry points; every row is an
@@ -544,23 +718,23 @@ fn errsource() io.stream = {
};
if (entry[i] == 0) {
let rb: (u8 | io.eof | io.error | bufio.overflow) =
let rb: (u8 | io.eof | io.error | errors.overflow) =
bufio.scanbyte(&sc);
match (rb) {
case let b: u8 => abort();
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => { };
case errors.overflow => { };
};
} else {
let rr: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) =
let rr: (rune | io.eof | io.error | utf8.invalid | errors.overflow) =
bufio.scanrune(&sc);
match (rr) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => { };
case errors.overflow => { };
};
};