Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bufio: make scanning bounded and incremental

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-09 18:17:42 +09:00
parent 09b6f22e85
commit a04de3e02b
7 changed files with 362 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
lib/bufio/scanner.ww
View File

@@ -1,19 +1,15 @@
package bufio;
import encoding.utf8;
import errors;
import io;
import os;
// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth
// increment. i32 (not Hare's size) per the lib-wide index-type
// convention (lib/CLAUDE.md).
def BUFSZ: i32 = 4096;
// overflow — the scanner buffer filled before the delimiter (or
// underlying EOF) was hit. With a caller-supplied buffer we can't
// grow; bumping the budget is on the caller. Mirrors Hare bufio's
// use of errors::overflow for the same condition.
export type overflow = !void;
// Plain tokenizer — no embedded vtable; driven directly via scanbyte /
// scanbytes / scanline, not through io dispatch. Its `src` is an
// io.stream and refills go through io.read. Value-return constructor
@@ -29,6 +25,8 @@ export type scanner = struct {
start: i32, // index where the pending region starts in ptr
avail: i32, // pending byte count; pending = ptr[start..start+avail]
maxread: i32, // growth ceiling; == cap for newscannerbuf (scanner.ha:101)
owned: bool, // ptr is scanner-owned and finish must release it
eof: bool, // source reached EOF or returned no progress
};
// newscanner — wire a scanner that allocates and grows its own
@@ -40,6 +38,8 @@ export type scanner = struct {
// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD
// behavior is the shipped default).
export fn newscanner(src: io.stream, maxread: i32) scanner = {
assert(maxread >= 0,
"bufio.newscanner: maximum read size must not be negative");
let r: scanner;
r.src = src;
r.ptr = nil;
@@ -47,6 +47,8 @@ export fn newscanner(src: io.stream, maxread: i32) scanner = {
r.start = 0;
r.avail = 0;
r.maxread = maxread;
r.owned = true;
r.eof = false;
return r;
};
@@ -62,88 +64,103 @@ export fn newscannerbuf(src: io.stream, buf: []u8) scanner = {
r.start = 0;
r.avail = 0;
r.maxread = buf.len;
r.owned = false;
r.eof = false;
return r;
};
// finish — release scanner-owned resources; src isn't closed. Mirrors
// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the
// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a
// newscannerbuf caller's buffer is untouched either way.
// finish — release scanner-owned resources; src isn't closed. A fixed
// newscannerbuf window is borrowed and remains the caller's property.
// Mirrors ref/hare/bufio/scanner.ha:110.
export fn finish(s: *scanner) void = {
free(s.ptr);
if (s.owned && s.cap > 0) {
os.free(s.ptr: *void, s.cap: u64);
};
s.ptr = nil;
s.cap = 0;
s.start = 0;
s.avail = 0;
s.maxread = 0;
s.owned = false;
s.eof = true;
};
// readahead — make room and read once from src into the back of the
// pending region. Returns bytes newly buffered (>=0), io.eof/io.error
// from src, or overflow when the buffer is full at the maxread ceiling.
// pending region. Returns bytes newly buffered (>0), io.eof/io.error
// from src, or errors.overflow at the maxread ceiling. A zero-byte read
// is terminal EOF: retryable sources must return an explicit error.
// The size from io.read narrows to i32 (buffer-length type).
//
// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer
// first shifts pending left, then — when start == 0 — grows, or returns
// overflow when `avail >= want` (Hare's `pending >= readahead` ceiling,
// overflow when no capacity remains (Hare's `pending >= readahead` ceiling,
// scanner.ha:179-181, BEFORE the append). Hare grows via
// `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner
// allocates a fresh backing and copies (the old block is left to
// process-exit reclaim, ww no-free; `!` not `?` per the #36
// nomem-propagation gap). overflow is bufio-local (scanner.ww:15), NOT
// errors.overflow: ww's io.error is a closed enumerated union with no
// overflow member (lib/io/types.ww), so the can't-grow signal rides a
// distinct arm that scanbyte / scanbytes / scanrune match-forward —
// the single choke-point (drew ruling, drain F-B). Pre-fix this case
// fell through silently to a zero-length io.read, spinning scanbyte
// (catB-144) and nil-derefing scanrune (catB-145).
fn readahead(s: *scanner) (i32 | io.eof | io.error | overflow) = {
// allocates a fresh backing, copies pending bytes, and releases the
// replaced owned block. Allocation remains `!` rather than `?` per the
// #36 nomem-propagation gap. The generic overflow condition rides a
// distinct arm because io.error is closed (lib/io/types.ww).
fn readahead(s: *scanner) (i32 | io.eof | io.error | errors.overflow) = {
if (s.eof) { let e: io.eof; return e; };
if (s.start + s.avail == s.cap) {
if (s.start > 0) {
let i: i32 = 0;
for (i < s.avail) {
s.ptr[i] = s.ptr[s.start + i];
i += 1;
};
bufiomove(s.ptr, s.ptr + (s.start: u64), s.avail);
s.start = 0;
} else {
let want: i32 = s.avail + BUFSZ;
if (want > s.maxread) { want = s.maxread; };
if (s.avail >= want) {
let e: overflow; return e;
let room: i32 = s.maxread - s.avail;
if (room <= 0) {
let e: errors.overflow; return e;
};
let ncap: i32 = s.avail + want;
let ncap: i32 = s.maxread;
if (room > BUFSZ) { ncap = s.avail + BUFSZ; };
let nbuf: []u8 = alloc([], ncap: u64)!;
let np: *u8 = nbuf.ptr;
let i: i32 = 0;
for (i < s.avail) {
np[i] = s.ptr[i];
i += 1;
bufiomove(np, s.ptr, s.avail);
if (s.owned && s.cap > 0) {
os.free(s.ptr: *void, s.cap: u64);
};
s.ptr = np;
s.cap = ncap;
s.owned = true;
};
};
let off: i32 = s.start + s.avail;
let v: []u8;
v.ptr = s.ptr + (off: u64);
v.len = s.cap - off;
v.cap = v.len;
let r: (size | io.eof | io.error) = io.read(s.src, v);
match (r) {
case let n: size => {
assert(n <= v.len: size,
"bufio scanner: reader returned an oversized count");
if (n == 0: size) {
s.eof = true;
let e: io.eof;
return e;
};
s.avail += n: i32;
return n: i32;
};
case io.eof => { let e: io.eof; return e; };
case io.eof => {
s.eof = true;
let e: io.eof;
return e;
};
case let e: io.error => return e;
};
};
// scanbyte — pop one byte, refilling from src on demand. Mirrors
// ref/hare/bufio/scanner.ha:204.
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = {
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | errors.overflow) = {
for (s.avail == 0) {
let r: (i32 | io.eof | io.error | overflow) = readahead(s);
let r: (i32 | io.eof | io.error | errors.overflow) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
case errors.overflow => { let e: errors.overflow; return e; };
};
};
let b: u8 = s.ptr[s.start];
@@ -157,9 +174,9 @@ export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = {
// borrows from the scanner buffer and is invalidated by the next scan.
// EOF without delim discards the trailing fragment and returns io.eof
// (Hare EOF_DISCARD default); buffer-full without delim returns
// overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte
// errors.overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte
// form is #217. Mirrors ref/hare/bufio/scanner.ha:220 (narrowed).
export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow) = {
export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | errors.overflow) = {
let i: i32 = 0;
for (true) {
for (i < s.avail) {
@@ -167,21 +184,20 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
let v: []u8;
v.ptr = s.ptr + (s.start: u64);
v.len = i;
v.cap = v.len;
s.start += i + 1;
s.avail -= i + 1;
return v;
};
i += 1;
};
// overflow now surfaces from readahead's single choke-point
// (avail >= want == Hare's `pending >= readahead`,
// scanner.ha:179); no separate pre-check (drew ruling, F-B).
let r: (i32 | io.eof | io.error | overflow) = readahead(s);
// Capacity failure surfaces from readahead's single choke-point.
let r: (i32 | io.eof | io.error | errors.overflow) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
case errors.overflow => { let e: errors.overflow; return e; };
};
};
let e: io.eof; return e;
@@ -190,46 +206,73 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
// scanrune — pop one UTF-8-encoded rune, refilling from src on demand.
// EOF mid-codepoint (fewer pending bytes than the initial byte
// announces) is utf8.invalid; a clean EOF before any byte is io.eof.
// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead
// when fewer than 4 bytes (the longest codepoint) are pending, then
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` propagates
// errors::overflow (through io::error); ww forwards the bufio-local
// overflow as a distinct arm — a zero-cap / at-ceiling scanner cannot
// buffer the first byte, so without this arm the readahead fall-through
// would nil-deref s.ptr[s.start] (catB-145).
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overflow) = {
if (s.avail < 4) {
let ra: (i32 | io.eof | io.error | overflow) = readahead(s);
// Partial reads are refilled until the announced width is present.
// Decode happens before consumption: malformed input consumes only the
// bytes visited by utf8.next, preserving any later pending byte.
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | errors.overflow) = {
for (s.avail == 0) {
let ra: (i32 | io.eof | io.error | errors.overflow) = readahead(s);
match (ra) {
case let n: i32 => { };
case io.eof => {
if (s.avail == 0) { let e: io.eof; return e; };
};
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
case errors.overflow => { let e: errors.overflow; return e; };
};
};
let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]);
let sz: i32 = 0;
match (szr) {
case let n: i32 => { sz = n; };
case utf8.invalid => { let e: utf8.invalid; return e; };
case utf8.invalid => {
s.start += 1;
s.avail -= 1;
let e: utf8.invalid;
return e;
};
if (s.avail < sz) {
let e: utf8.invalid; return e;
};
for (s.avail < sz) {
let ra: (i32 | io.eof | io.error | errors.overflow) = readahead(s);
match (ra) {
case let n: i32 => { };
case io.eof => {
s.start += s.avail;
s.avail = 0;
let e: utf8.invalid;
return e;
};
case let e: io.error => return e;
case errors.overflow => { let e: errors.overflow; return e; };
};
};
let v: []u8;
v.ptr = s.ptr + (s.start: u64);
v.len = sz;
s.start += sz;
s.avail -= sz;
v.cap = v.len;
let dec: utf8.decoder = utf8.decode(v);
let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec);
let used: i32 = utf8.position(&dec): i32;
match (nr) {
case let r: rune => return r;
case utf8.done => { let e: io.eof; return e; };
case utf8.more => { let e: utf8.invalid; return e; };
case utf8.invalid => { let e: utf8.invalid; return e; };
case let r: rune => {
assert(used == sz, "bufio.scanrune: decoder width mismatch");
s.start += used;
s.avail -= used;
return r;
};
case utf8.done => abort("bufio.scanrune: decoder made no progress");
case utf8.more => {
s.start += used;
s.avail -= used;
let e: utf8.invalid;
return e;
};
case utf8.invalid => {
assert(used > 0 && used <= sz,
"bufio.scanrune: invalid decoder position");
s.start += used;
s.avail -= used;
let e: utf8.invalid;
return e;
};
};
};
@@ -238,17 +281,18 @@ export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overf
// Single-byte route (Hare's scan_line = scan_string(s, "\n"); the
// arbitrary multibyte-delim scan_string is #217). Mirrors
// ref/hare/bufio/scanner.ha:307.
export fn scanline(s: *scanner) (str | io.eof | io.error | overflow) = {
let r: ([]u8 | io.eof | io.error | overflow) = scanbytes(s, 10u8);
export fn scanline(s: *scanner) (str | io.eof | io.error | errors.overflow) = {
let r: ([]u8 | io.eof | io.error | errors.overflow) = scanbytes(s, 10u8);
match (r) {
case let bs: []u8 => {
let v: str;
v.ptr = bs.ptr;
v.len = bs.len;
v.cap = v.len;
return v;
};
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
case errors.overflow => { let e: errors.overflow; return e; };
};
};

272
lib/bufio/scanner_test.ww
View File

@@ -6,8 +6,10 @@ package bufio_test;
import bufio;
import encoding.utf8;
import errors;
import io;
import memio;
import test;
fn putstr(s: str, into: []u8, off: i32) i32 = {
let i: i32 = 0;
@@ -32,6 +34,66 @@ fn errsource() io.stream = {
return &errvt;
};
// stepstream is a deliberately short-reading source. `zero` makes its
// first call return no progress so scanner EOF handling is observable.
type stepstream = struct {
vt: io.vtable,
src: []u8,
pos: i32,
calls: i32,
step: i32,
zero: bool,
badcap: bool,
};
fn stepread(s: io.stream, buf: []u8) (size | io.eof | io.error) = {
let r: *stepstream = s: *stepstream;
r.calls += 1;
if (buf.cap != buf.len) { r.badcap = true; };
if (r.zero) {
r.zero = false;
return 0: size;
};
if (r.pos == r.src.len) { let e: io.eof; return e; };
let n: i32 = r.step;
if (n > buf.len) { n = buf.len; };
if (n > r.src.len - r.pos) { n = r.src.len - r.pos; };
let i: i32 = 0;
for (i < n) {
buf[i] = r.src[r.pos + i];
i += 1;
};
r.pos += n;
return n: size;
};
fn newstepstream(src: []u8, step: i32, zero: bool) stepstream = {
let r: stepstream;
r.vt.reader = (&stepread): *io.reader;
r.src = src;
r.pos = 0;
r.calls = 0;
r.step = step;
r.zero = zero;
r.badcap = false;
return r;
};
def WANT_INVALID: i32 = -1;
def WANT_EOF: i32 = -2;
fn checkrune(sc: *bufio.scanner, want: i32) void = {
let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) =
bufio.scanrune(sc);
match (r) {
case let got: rune => assert(want >= 0 && got: i32 == want);
case io.eof => assert(want == WANT_EOF);
case let e: io.error => abort();
case utf8.invalid => assert(want == WANT_INVALID);
case errors.overflow => abort();
};
};
@test fn scanbytecases() void = {
let raw: [4]u8;
raw[0] = 11u8; raw[1] = 22u8; raw[2] = 33u8; raw[3] = 44u8;
@@ -53,7 +115,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 6) {
let r: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
let r: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc);
match (r) {
case let b: u8 => {
assert(!(weof[i] != 0));
@@ -61,7 +123,7 @@ fn errsource() io.stream = {
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -91,17 +153,18 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 5) {
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => {
assert(!(weof[i] != 0));
assert(!(wovr[i] != 0));
assert(!(v.len != wl[i]));
assert(v.cap == v.len);
assert(!(v.len > 0 && v[0] != wf[i]));
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => { assert(!(wovr[i] == 0)); };
case errors.overflow => { assert(!(wovr[i] == 0)); };
};
i += 1;
};
@@ -120,12 +183,12 @@ fn errsource() io.stream = {
let buf: [4]u8;
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:4]);
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => abort();
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => { };
case errors.overflow => { };
};
bufio.finish(&sc);
@@ -153,16 +216,17 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 5) {
let r: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 44u8); // ','
let r: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 44u8); // ','
match (r) {
case let v: []u8 => {
assert(!(weof[i] != 0));
assert(!(v.len != wl[i]));
assert(v.cap == v.len);
assert(!(v.len > 0 && v[0] != wf[i]));
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -177,28 +241,28 @@ fn errsource() io.stream = {
let buf: [8]u8;
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]);
let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
let r1: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc);
match (r1) {
case let b: u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r2) {
case let v: str => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r3: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 10u8);
let r3: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 10u8);
match (r3) {
case let v: []u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc);
@@ -209,28 +273,28 @@ fn errsource() io.stream = {
let buf: [8]u8;
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]);
let r1: (u8 | io.eof | io.error | bufio.overflow) = bufio.scanbyte(&sc);
let r1: (u8 | io.eof | io.error | errors.overflow) = bufio.scanbyte(&sc);
match (r1) {
case let b: u8 => abort();
case io.eof => abort();
case let e: io.error => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r2) {
case let v: str => abort();
case io.eof => abort();
case let e: io.error => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r3: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc, 32u8);
let r3: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc, 32u8);
match (r3) {
case let v: []u8 => abort();
case io.eof => abort();
case let e: io.error => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc);
@@ -244,14 +308,14 @@ fn errsource() io.stream = {
let b1: [8]u8;
let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]);
let r1: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc1);
let r1: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc1);
match (r1) {
case let v: str => { assert(!(v.len != 0)); };
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
let r2: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc1);
let r2: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc1);
match (r2) {
case let v: str => {
assert(!(v.len != 3));
@@ -259,7 +323,7 @@ fn errsource() io.stream = {
};
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc1);
@@ -270,19 +334,19 @@ fn errsource() io.stream = {
let b2: [8]u8;
let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]);
let t1: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc2, 44u8);
let t1: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc2, 44u8);
match (t1) {
case let v: []u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
let t2: ([]u8 | io.eof | io.error | bufio.overflow) = bufio.scanbytes(&sc2, 44u8);
let t2: ([]u8 | io.eof | io.error | errors.overflow) = bufio.scanbytes(&sc2, 44u8);
match (t2) {
case let v: []u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc2);
};
@@ -306,7 +370,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 3) {
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => {
assert(!(weof[i] != 0));
@@ -315,7 +379,7 @@ fn errsource() io.stream = {
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -350,7 +414,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 6) {
let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc);
let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc);
match (r) {
case let rn: rune => {
assert(!(weof[i] != 0));
@@ -359,7 +423,7 @@ fn errsource() io.stream = {
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -375,13 +439,13 @@ fn errsource() io.stream = {
let m1: io.stream = &mem1.vt;
let b1: [8]u8;
let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]);
let r1: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc1);
let r1: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc1);
match (r1) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc1);
@@ -393,13 +457,13 @@ fn errsource() io.stream = {
let m2: io.stream = &mem2.vt;
let b2: [8]u8;
let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]);
let r2: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2);
let r2: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc2);
match (r2) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc2);
@@ -411,17 +475,127 @@ fn errsource() io.stream = {
let m3: io.stream = &mem3.vt;
let b3: [8]u8;
let sc3: bufio.scanner = bufio.newscannerbuf(m3, b3[0:8]);
let r3: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc3);
let r3: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc3);
match (r3) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => { };
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc3);
};
@test fn scanrunepartialreads() void = {
// Each read yields one byte, including across a shifted 4-byte rune.
let raw: [6]u8;
raw[0] = 0xC3u8; raw[1] = 0xA9u8;
raw[2] = 0xF0u8; raw[3] = 0x9Fu8;
raw[4] = 0x98u8; raw[5] = 0x80u8;
let src: stepstream = newstepstream(raw[0:6], 1, false);
let buf: [4]u8;
let sc: bufio.scanner = bufio.newscannerbuf(&src.vt, buf[0:4]);
checkrune(&sc, 233);
checkrune(&sc, 128512);
checkrune(&sc, WANT_EOF);
assert(!src.badcap);
assert(src.calls == 7);
bufio.finish(&sc);
};
@test fn scanrunemalformedprogress() void = {
// The decoder rejects E0 80 at position 2; the pending ASCII byte
// beyond that position must remain for the next scan.
let raw1: [3]u8;
raw1[0] = 0xE0u8; raw1[1] = 0x80u8; raw1[2] = 65u8;
let mem1: memio.stream = memio.fixed(raw1[0:3]);
let buf1: [8]u8;
let sc1: bufio.scanner = bufio.newscannerbuf(&mem1.vt, buf1[0:8]);
checkrune(&sc1, WANT_INVALID);
checkrune(&sc1, 65);
bufio.finish(&sc1);
// Impossible F5-F7 leads consume one byte, not their following ASCII.
let raw2: [4]u8;
raw2[0] = 0xF5u8; raw2[1] = 65u8;
raw2[2] = 66u8; raw2[3] = 67u8;
let mem2: memio.stream = memio.fixed(raw2[0:4]);
let buf2: [8]u8;
let sc2: bufio.scanner = bufio.newscannerbuf(&mem2.vt, buf2[0:8]);
checkrune(&sc2, WANT_INVALID);
let want: [3]u32;
want[0] = 65u32; want[1] = 66u32; want[2] = 67u32;
let i: i32 = 0;
for (i < 3) {
checkrune(&sc2, want[i]: i32);
i += 1;
};
bufio.finish(&sc2);
// A truncated prefix is consumed once; a following call is EOF.
let raw3: [1]u8;
raw3[0] = 0xC3u8;
let mem3: memio.stream = memio.fixed(raw3[0:1]);
let buf3: [8]u8;
let sc3: bufio.scanner = bufio.newscannerbuf(&mem3.vt, buf3[0:8]);
checkrune(&sc3, WANT_INVALID);
checkrune(&sc3, WANT_EOF);
bufio.finish(&sc3);
};
@test fn scannerzeroreadiseof() void = {
let raw: [1]u8;
raw[0] = 65u8;
let src: stepstream = newstepstream(raw[0:1], 1, true);
let buf: [4]u8;
let sc: bufio.scanner = bufio.newscannerbuf(&src.vt, buf[0:4]);
let i: i32 = 0;
for (i < 2) {
let r: (u8 | io.eof | io.error | errors.overflow) =
bufio.scanbyte(&sc);
match (r) {
case let b: u8 => abort();
case io.eof => { };
case let e: io.error => abort();
case errors.overflow => abort();
};
i += 1;
};
assert(src.calls == 1);
assert(src.pos == 0);
bufio.finish(&sc);
};
@test fn scannermaxreadbound() void = {
// The token and delimiter exceed maxread. The old growth expression
// allocated avail+want (8193 here) and incorrectly accepted them.
let raw: [4099]u8;
let i: i32 = 0;
for (i < 4098) { raw[i] = 65u8; i += 1; };
raw[4098] = 10u8;
let mem: memio.stream = memio.fixed(raw[0:4099]);
let sc: bufio.scanner = bufio.newscanner(&mem.vt, 4097);
let r: (str | io.eof | io.error | errors.overflow) =
bufio.scanline(&sc);
match (r) {
case let s: str => abort();
case io.eof => abort();
case let e: io.error => abort();
case errors.overflow => { };
};
assert(sc.cap == 4097);
assert(sc.avail == 4097);
assert(sc.owned);
bufio.finish(&sc);
assert(sc.ptr == nil);
assert(sc.cap == 0 && sc.avail == 0 && !sc.owned && sc.eof);
};
@test fn newscannernegativeaborts() void = {
test.expectabort();
let sc: bufio.scanner = bufio.newscanner(errsource(), -1);
};
@test fn newscannergrow() void = {
let src: [16]u8;
let n: i32 = putstr("hello\nworld\n", src[0:16], 0);
@@ -441,7 +615,7 @@ fn errsource() io.stream = {
let i: i32 = 0;
for (i < 3) {
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => {
assert(!(weof[i] != 0));
@@ -450,7 +624,7 @@ fn errsource() io.stream = {
};
case io.eof => { assert(!(weof[i] == 0)); };
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -472,7 +646,7 @@ fn errsource() io.stream = {
i = 0;
for (i < 3) {
let r: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) = bufio.scanrune(&sc2);
let r: (rune | io.eof | io.error | utf8.invalid | errors.overflow) = bufio.scanrune(&sc2);
match (r) {
case let rn: rune => {
assert(!(weof2[i] != 0));
@@ -481,7 +655,7 @@ fn errsource() io.stream = {
case io.eof => { assert(!(weof2[i] == 0)); };
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
i += 1;
};
@@ -496,12 +670,12 @@ fn errsource() io.stream = {
let m: io.stream = &mem.vt;
let sc: bufio.scanner = bufio.newscanner(m, 4); // token (8B) > maxread
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let r: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (r) {
case let v: str => abort();
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => { };
case errors.overflow => { };
};
bufio.finish(&sc);
};
@@ -510,7 +684,7 @@ fn errsource() io.stream = {
// readahead unable to make progress. Pre-fix it fell through silently to
// a zero-length io.read returning 0, so scanbyte spun forever (catB-144)
// and scanrune dereferenced the nil / zero-cap s.ptr[s.start]
// (catB-145). Both entry points must now surface bufio.overflow
// (catB-145). Both entry points must now surface errors.overflow
// (ref/hare/bufio/scanner.ha:179-181). The table crosses the two
// zero-cap constructions — nil-ptr newscanner(,0) and zero-length
// newscannerbuf — with the two single-item entry points; every row is an
@@ -544,23 +718,23 @@ fn errsource() io.stream = {
};
if (entry[i] == 0) {
let rb: (u8 | io.eof | io.error | bufio.overflow) =
let rb: (u8 | io.eof | io.error | errors.overflow) =
bufio.scanbyte(&sc);
match (rb) {
case let b: u8 => abort();
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => { };
case errors.overflow => { };
};
} else {
let rr: (rune | io.eof | io.error | utf8.invalid | bufio.overflow) =
let rr: (rune | io.eof | io.error | utf8.invalid | errors.overflow) =
bufio.scanrune(&sc);
match (rr) {
case let rn: rune => abort();
case io.eof => abort();
case let e: io.error => abort();
case utf8.invalid => abort();
case bufio.overflow => { };
case errors.overflow => { };
};
};

5
lib/bufio/stream_test.ww
View File

@@ -5,6 +5,7 @@
package bufio_test;
import bufio;
import errors;
import io;
import memio;
@@ -415,7 +416,7 @@ fn checkpartialflush(zero: bool) void = {
let sbuf: [32]u8;
let sc: bufio.scanner = bufio.newscannerbuf(p, sbuf[0:32]);
let lr: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
let lr: (str | io.eof | io.error | errors.overflow) = bufio.scanline(&sc);
match (lr) {
case let v: str => {
assert(!(v.len != 8)); // "XYZhello"
@@ -424,7 +425,7 @@ fn checkpartialflush(zero: bool) void = {
};
case io.eof => abort();
case let e: io.error => abort();
case bufio.overflow => abort();
case errors.overflow => abort();
};
bufio.finish(&sc);
};

3
lib/regex/regex.ww
View File

@@ -23,6 +23,7 @@ package regex;
import ascii;
import bufio;
import errors;
import io;
import memio;
import strconv;
@@ -1209,7 +1210,7 @@ fn search(
// Unreachable: this scanner is newscanner(h, I32_MAX), so the
// can't-grow ceiling is never hit; the arm keeps the match
// total over scanrune's widened overflow surface (drain F-B).
case bufio.overflow => abort("regex: scanrune overflow");
case errors.overflow => abort("regex: scanrune overflow");
};
if (r_or_end is rune) {
last_bytesize = (utf8.runesz(r_or_end as rune): size);