lib/bufio: newscanner + scanrune (regex fold-2b prereq)
Port Hare's auto-grow newscanner (scanner.ha:72) and scan_rune (scanner.ha:259). scanner gains a maxread field (== cap for newscannerbuf, scanner.ha:101); readahead grows by BUFSZ up to maxread via alloc+copy (Hare appends; ww flat ptr/cap scanner, old block left to process-exit reclaim). scanbytes' overflow test gains the avail >= maxread leg (Hare's pending >= readahead predicate) so a growable scanner refills instead of overflowing. finish ports the free(scan.buffer) verbatim per the regex #27 precedent. Tests: rune scan over 1/2/3/4-byte UTF-8 + EOF, invalid initial/truncated/surrogate sequences, newscanner grow round-trip + scanrune-over-newscanner, maxread overflow.
This commit is contained in:
@@ -4,11 +4,14 @@
|
||||
//
|
||||
// Surface:
|
||||
//
|
||||
// bufio.newscanner (src: io.stream, maxread: i32) scanner
|
||||
// bufio.newscannerbuf(src: io.stream, buf: []u8) scanner
|
||||
// bufio.finish (s: *scanner) void
|
||||
// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error)
|
||||
// bufio.scanbytes (s: *scanner, delim: u8)
|
||||
// ([]u8 | io.eof | io.error | overflow)
|
||||
// bufio.scanrune (s: *scanner)
|
||||
// (rune | io.eof | io.error | utf8.invalid)
|
||||
// bufio.scanline (s: *scanner) (str | io.eof | io.error | overflow)
|
||||
//
|
||||
// bufio.init (src: io.stream, rbuf: []u8, wbuf: []u8) stream
|
||||
@@ -26,12 +29,13 @@
|
||||
// scanbyte / scanbytes / scanline.
|
||||
//
|
||||
// SCOPE: this is the surface ww ships, NOT a port of Hare's full
|
||||
// scanner. The features ww never implemented — auto-grow newscanner,
|
||||
// multibyte-delim scanbytes / scanstring, scanrune / readbyte / readtok
|
||||
// / readline / readrune / unreadrune — are a separate future feature
|
||||
// fold (#217). The ported set: the fixed-buffer newscannerbuf,
|
||||
// single-byte scanbytes, single-byte scanline, scanbyte, finish; and
|
||||
// the buffered-stream init / setflush / flush / unread / isbuffered.
|
||||
// scanner. The features ww never implemented — multibyte-delim
|
||||
// scanbytes / scanstring, readbyte / readtok / readline / readrune /
|
||||
// unreadrune — are a separate future feature fold (#217). The ported
|
||||
// set: the auto-grow newscanner, the fixed-buffer newscannerbuf,
|
||||
// single-byte scanbytes, single-byte scanline, scanbyte, scanrune,
|
||||
// finish; and the buffered-stream init / setflush / flush / unread /
|
||||
// isbuffered.
|
||||
//
|
||||
// Cast workaround per #206-payoff (ken: KEEP the explicit casts; they
|
||||
// are cgen-neutral and sidestep the #214 over-acceptance surface).
|
||||
@@ -47,8 +51,14 @@
|
||||
|
||||
package bufio;
|
||||
|
||||
import encoding.utf8;
|
||||
import io;
|
||||
|
||||
// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth
|
||||
// increment. i32 (not Hare's size) per the lib-wide index-type
|
||||
// convention (lib/CLAUDE.md).
|
||||
def BUFSZ: i32 = 4096;
|
||||
|
||||
// flushdefault — backing storage for the default flush byte-set
|
||||
// ("\n"). Hare scopes it inside `init` as `static let
|
||||
// flush_default = ['\n': u8]` (ref/hare/bufio/stream.ha:75); ww
|
||||
@@ -295,17 +305,37 @@ fn bclose(s: io.stream) (void | io.error) = {
|
||||
// slice support.
|
||||
|
||||
export type scanner = struct {
|
||||
src: io.stream,
|
||||
ptr: *u8,
|
||||
cap: i32,
|
||||
start: i32, // index where the pending region starts in ptr
|
||||
avail: i32, // pending byte count; pending = ptr[start..start+avail]
|
||||
src: io.stream,
|
||||
ptr: *u8,
|
||||
cap: i32,
|
||||
start: i32, // index where the pending region starts in ptr
|
||||
avail: i32, // pending byte count; pending = ptr[start..start+avail]
|
||||
maxread: i32, // growth ceiling; == cap for newscannerbuf (scanner.ha:101)
|
||||
};
|
||||
|
||||
// newscanner — wire a scanner that allocates and grows its own
|
||||
// read-ahead buffer, by BUFSZ per refill up to `maxread`. Returns the
|
||||
// scanner BY VALUE. This is Hare's newscanner
|
||||
// (ref/hare/bufio/scanner.ha:72) modulo two parameter drops: ww has no
|
||||
// default arguments, so Hare's `maxread: size = types::SIZE_MAX` is a
|
||||
// required i32 (callers wanting "no limit" pass types.I32_MAX), and the
|
||||
// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD
|
||||
// behavior is the shipped default).
|
||||
export fn newscanner(src: io.stream, maxread: i32) scanner = {
|
||||
let r: scanner;
|
||||
r.src = src;
|
||||
r.ptr = nil;
|
||||
r.cap = 0;
|
||||
r.start = 0;
|
||||
r.avail = 0;
|
||||
r.maxread = maxread;
|
||||
return r;
|
||||
};
|
||||
|
||||
// newscannerbuf — wire a scanner to read through `src` using `buf` as
|
||||
// the fixed read-ahead window. Returns the scanner BY VALUE. This is
|
||||
// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92); the auto-grow
|
||||
// newscanner is a separate feature (#217).
|
||||
// the fixed read-ahead window (maxread == buf.len, so the buffer never
|
||||
// grows — scanner.ha:101). Returns the scanner BY VALUE. This is
|
||||
// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92).
|
||||
export fn newscannerbuf(src: io.stream, buf: []u8) scanner = {
|
||||
let r: scanner;
|
||||
r.src = src;
|
||||
@@ -313,25 +343,55 @@ export fn newscannerbuf(src: io.stream, buf: []u8) scanner = {
|
||||
r.cap = buf.len;
|
||||
r.start = 0;
|
||||
r.avail = 0;
|
||||
r.maxread = buf.len;
|
||||
return r;
|
||||
};
|
||||
|
||||
// finish — release scanner-owned resources. No-op (buffer is
|
||||
// caller-owned, src isn't closed); kept on the surface so callers won't
|
||||
// churn. Mirrors ref/hare/bufio/scanner.ha:110.
|
||||
export fn finish(s: *scanner) void = { };
|
||||
// finish — release scanner-owned resources; src isn't closed. Mirrors
|
||||
// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the
|
||||
// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a
|
||||
// newscannerbuf caller's buffer is untouched either way.
|
||||
export fn finish(s: *scanner) void = {
|
||||
free(s.ptr);
|
||||
};
|
||||
|
||||
// readahead — make room and read once from src into the back of the
|
||||
// pending region. Returns bytes newly buffered (>=0), or io.eof/io.error
|
||||
// from src. The size from io.read narrows to i32 (buffer-length type).
|
||||
//
|
||||
// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer
|
||||
// first shifts pending left, then — when start == 0 and maxread allows
|
||||
// — grows. Hare grows via `append(scan.buffer, [0...], readahead)?`;
|
||||
// ww's flat ptr/cap scanner allocates a fresh backing and copies (the
|
||||
// old block is left to process-exit reclaim, ww no-free; `!` not `?`
|
||||
// per the #36 nomem-propagation gap). The can't-grow case (avail >=
|
||||
// maxread) is Hare's errors::overflow return — ww's io.error has no
|
||||
// overflow member (lib/io/types.ww:40 enumerated union), so callers
|
||||
// that can overflow (scanbytes) detect that state themselves before
|
||||
// calling, against `maxread`.
|
||||
fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
|
||||
if (s.start + s.avail == s.cap && s.start > 0) {
|
||||
let i: i32 = 0;
|
||||
for (i < s.avail) {
|
||||
s.ptr[i] = s.ptr[s.start + i];
|
||||
i += 1;
|
||||
if (s.start + s.avail == s.cap) {
|
||||
if (s.start > 0) {
|
||||
let i: i32 = 0;
|
||||
for (i < s.avail) {
|
||||
s.ptr[i] = s.ptr[s.start + i];
|
||||
i += 1;
|
||||
};
|
||||
s.start = 0;
|
||||
} else if (s.avail < s.maxread) {
|
||||
let want: i32 = s.avail + BUFSZ;
|
||||
if (want > s.maxread) { want = s.maxread; };
|
||||
let ncap: i32 = s.avail + want;
|
||||
let nbuf: []u8 = alloc([], ncap: u64)!;
|
||||
let np: *u8 = nbuf.ptr;
|
||||
let i: i32 = 0;
|
||||
for (i < s.avail) {
|
||||
np[i] = s.ptr[i];
|
||||
i += 1;
|
||||
};
|
||||
s.ptr = np;
|
||||
s.cap = ncap;
|
||||
};
|
||||
s.start = 0;
|
||||
};
|
||||
let off: i32 = s.start + s.avail;
|
||||
let v: []u8;
|
||||
@@ -386,7 +446,12 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
if (s.start + s.avail == s.cap && s.start == 0) {
|
||||
// full + unshiftable + ungrowable (avail >= maxread is
|
||||
// Hare's `pending >= readahead` overflow predicate,
|
||||
// scanner.ha:179; for a newscannerbuf scanner maxread ==
|
||||
// cap so this is the old fixed-buffer-full test).
|
||||
if (s.start + s.avail == s.cap && s.start == 0
|
||||
&& s.avail >= s.maxread) {
|
||||
let e: overflow; return e;
|
||||
};
|
||||
let r: (i32 | io.eof | io.error) = readahead(s);
|
||||
@@ -399,6 +464,51 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
|
||||
let e: io.eof; return e;
|
||||
};
|
||||
|
||||
// scanrune — pop one UTF-8-encoded rune, refilling from src on demand.
|
||||
// EOF mid-codepoint (fewer pending bytes than the initial byte
|
||||
// announces) is utf8.invalid; a clean EOF before any byte is io.eof.
|
||||
// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead
|
||||
// when fewer than 4 bytes (the longest codepoint) are pending, then
|
||||
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` also
|
||||
// propagates errors::overflow through io::error; ww's io.error has no
|
||||
// overflow member (see readahead) and a <4-byte refill cannot overflow
|
||||
// a scanner that can hold a codepoint, so only io.error proper
|
||||
// propagates here.
|
||||
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid) = {
|
||||
if (s.avail < 4) {
|
||||
let ra: (i32 | io.eof | io.error) = readahead(s);
|
||||
match (ra) {
|
||||
case let n: i32 => { };
|
||||
case io.eof => {
|
||||
if (s.avail == 0) { let e: io.eof; return e; };
|
||||
};
|
||||
case let e: io.error => return e;
|
||||
};
|
||||
};
|
||||
let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]);
|
||||
let sz: i32 = 0;
|
||||
match (szr) {
|
||||
case let n: i32 => { sz = n; };
|
||||
case utf8.invalid => { let e: utf8.invalid; return e; };
|
||||
};
|
||||
if (s.avail < sz) {
|
||||
let e: utf8.invalid; return e;
|
||||
};
|
||||
let v: []u8;
|
||||
v.ptr = s.ptr + (s.start: u64);
|
||||
v.len = sz;
|
||||
s.start += sz;
|
||||
s.avail -= sz;
|
||||
let dec: utf8.decoder = utf8.decode(v);
|
||||
let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec);
|
||||
match (nr) {
|
||||
case let r: rune => return r;
|
||||
case utf8.done => { let e: io.eof; return e; };
|
||||
case utf8.more => { let e: utf8.invalid; return e; };
|
||||
case utf8.invalid => { let e: utf8.invalid; return e; };
|
||||
};
|
||||
};
|
||||
|
||||
// scanline — read up to (and not including) the next '\n'. The newline
|
||||
// is consumed; the returned str view borrows from the scanner buffer.
|
||||
// Single-byte route (Hare's scan_line = scan_string(s, "\n"); the
|
||||
|
||||
@@ -10,6 +10,7 @@ package bufio;
|
||||
|
||||
import bufio;
|
||||
import bytes;
|
||||
import encoding.utf8;
|
||||
import io;
|
||||
import memio;
|
||||
|
||||
@@ -687,6 +688,192 @@ fn errsource() io.stream = {
|
||||
if (raw[4] != 100u8) { fail(); }; // 'd'
|
||||
};
|
||||
|
||||
// ---- scanrune: ASCII + 2/3/4-byte UTF-8 + idempotent EOF --------------
|
||||
|
||||
@test fn scanrunecases() void = {
|
||||
// "a" U+0061; "é" U+00E9 (C3 A9); "한" U+D55C (ED 95 9C);
|
||||
// "😀" U+1F600 (F0 9F 98 80).
|
||||
let raw: [10]u8;
|
||||
raw[0] = 97u8;
|
||||
raw[1] = 0xC3u8; raw[2] = 0xA9u8;
|
||||
raw[3] = 0xEDu8; raw[4] = 0x95u8; raw[5] = 0x9Cu8;
|
||||
raw[6] = 0xF0u8; raw[7] = 0x9Fu8; raw[8] = 0x98u8; raw[9] = 0x80u8;
|
||||
|
||||
let mem: memio.stream = memio.fixed(raw[0:10]);
|
||||
let m: io.stream = &mem.vt;
|
||||
// 8B window: the trailing 4-byte rune straddles a refill (shift path).
|
||||
let buf: [8]u8;
|
||||
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]);
|
||||
|
||||
// (wantrune, weof)
|
||||
let want: [6]u32;
|
||||
let weof: [6]i32;
|
||||
want[0] = 97u32; weof[0] = 0;
|
||||
want[1] = 233u32; weof[1] = 0;
|
||||
want[2] = 54620u32; weof[2] = 0;
|
||||
want[3] = 128512u32; weof[3] = 0;
|
||||
want[4] = 0u32; weof[4] = 1; // past end → eof
|
||||
want[5] = 0u32; weof[5] = 1; // stay at eof
|
||||
|
||||
let i: i32 = 0;
|
||||
for (i < 6) {
|
||||
let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc);
|
||||
match (r) {
|
||||
case let rn: rune => {
|
||||
if (weof[i] != 0) { fail(); };
|
||||
if (rn: u32 != want[i]) { fail(); };
|
||||
};
|
||||
case io.eof => { if (weof[i] == 0) { fail(); }; };
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => fail();
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
|
||||
bufio.finish(&sc);
|
||||
};
|
||||
|
||||
// ---- scanrune: invalid sequences --------------------------------------
|
||||
|
||||
@test fn scanruneinvalid() void = {
|
||||
// Bare continuation byte: utf8sz rejects the initial byte.
|
||||
let r1raw: [1]u8;
|
||||
r1raw[0] = 0x80u8;
|
||||
let mem1: memio.stream = memio.fixed(r1raw[0:1]);
|
||||
let m1: io.stream = &mem1.vt;
|
||||
let b1: [8]u8;
|
||||
let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]);
|
||||
let r1: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc1);
|
||||
match (r1) {
|
||||
case let rn: rune => fail();
|
||||
case io.eof => fail();
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => { };
|
||||
};
|
||||
bufio.finish(&sc1);
|
||||
|
||||
// Truncated 2-byte sequence at EOF: pending < utf8sz → invalid
|
||||
// (Hare scanner.ha:272-274).
|
||||
let r2raw: [1]u8;
|
||||
r2raw[0] = 0xC3u8;
|
||||
let mem2: memio.stream = memio.fixed(r2raw[0:1]);
|
||||
let m2: io.stream = &mem2.vt;
|
||||
let b2: [8]u8;
|
||||
let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]);
|
||||
let r2: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2);
|
||||
match (r2) {
|
||||
case let rn: rune => fail();
|
||||
case io.eof => fail();
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => { };
|
||||
};
|
||||
bufio.finish(&sc2);
|
||||
|
||||
// UTF-16 surrogate encoding (ED A0 80 = U+D800): utf8sz accepts the
|
||||
// initial byte, the decode DFA rejects the continuation.
|
||||
let r3raw: [3]u8;
|
||||
r3raw[0] = 0xEDu8; r3raw[1] = 0xA0u8; r3raw[2] = 0x80u8;
|
||||
let mem3: memio.stream = memio.fixed(r3raw[0:3]);
|
||||
let m3: io.stream = &mem3.vt;
|
||||
let b3: [8]u8;
|
||||
let sc3: bufio.scanner = bufio.newscannerbuf(m3, b3[0:8]);
|
||||
let r3: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc3);
|
||||
match (r3) {
|
||||
case let rn: rune => fail();
|
||||
case io.eof => fail();
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => { };
|
||||
};
|
||||
bufio.finish(&sc3);
|
||||
};
|
||||
|
||||
// ---- newscanner: auto-grow from the empty ctor state ------------------
|
||||
|
||||
@test fn newscannergrow() void = {
|
||||
let src: [16]u8;
|
||||
let n: i32 = putstr("hello\nworld\n", src[0:16], 0);
|
||||
|
||||
let mem: memio.stream = memio.fixed(src[0:n]);
|
||||
let m: io.stream = &mem.vt;
|
||||
// cap starts at 0; the first readahead grows to min(BUFSZ, maxread).
|
||||
let sc: bufio.scanner = bufio.newscanner(m, 1024);
|
||||
|
||||
// (wantlen, wantfirst, weof)
|
||||
let wl: [3]i32;
|
||||
let wf: [3]u8;
|
||||
let weof: [3]i32;
|
||||
wl[0]=5; wf[0]=104u8; weof[0]=0; // hello
|
||||
wl[1]=5; wf[1]=119u8; weof[1]=0; // world
|
||||
wl[2]=0; wf[2]=0u8; weof[2]=1; // eof
|
||||
|
||||
let i: i32 = 0;
|
||||
for (i < 3) {
|
||||
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
|
||||
match (r) {
|
||||
case let v: str => {
|
||||
if (weof[i] != 0) { fail(); };
|
||||
if (v.len != wl[i]) { fail(); };
|
||||
if (v.len > 0 && v[0] != wf[i]) { fail(); };
|
||||
};
|
||||
case io.eof => { if (weof[i] == 0) { fail(); }; };
|
||||
case let e: io.error => fail();
|
||||
case bufio.overflow => fail();
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
bufio.finish(&sc);
|
||||
|
||||
// scanrune over a fresh newscanner (the regex search() shape).
|
||||
let r2raw: [3]u8;
|
||||
r2raw[0] = 0xC3u8; r2raw[1] = 0xA9u8; // é
|
||||
r2raw[2] = 122u8; // 'z'
|
||||
let mem2: memio.stream = memio.fixed(r2raw[0:3]);
|
||||
let m2: io.stream = &mem2.vt;
|
||||
let sc2: bufio.scanner = bufio.newscanner(m2, 1024);
|
||||
|
||||
let want: [3]u32;
|
||||
let weof2: [3]i32;
|
||||
want[0]=233u32; weof2[0]=0;
|
||||
want[1]=122u32; weof2[1]=0;
|
||||
want[2]=0u32; weof2[2]=1;
|
||||
|
||||
i = 0;
|
||||
for (i < 3) {
|
||||
let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2);
|
||||
match (r) {
|
||||
case let rn: rune => {
|
||||
if (weof2[i] != 0) { fail(); };
|
||||
if (rn: u32 != want[i]) { fail(); };
|
||||
};
|
||||
case io.eof => { if (weof2[i] == 0) { fail(); }; };
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => fail();
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
bufio.finish(&sc2);
|
||||
};
|
||||
|
||||
// ---- newscanner: maxread reached without a delimiter → overflow -------
|
||||
|
||||
@test fn newscanneroverflow() void = {
|
||||
let src: [16]u8;
|
||||
let n: i32 = putstr("ABCDEFGH\n", src[0:16], 0);
|
||||
|
||||
let mem: memio.stream = memio.fixed(src[0:n]);
|
||||
let m: io.stream = &mem.vt;
|
||||
let sc: bufio.scanner = bufio.newscanner(m, 4); // token (8B) > maxread
|
||||
|
||||
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
|
||||
match (r) {
|
||||
case let v: str => fail();
|
||||
case io.eof => fail();
|
||||
case let e: io.error => fail();
|
||||
case bufio.overflow => { };
|
||||
};
|
||||
bufio.finish(&sc);
|
||||
};
|
||||
|
||||
export fn main() i32 = {
|
||||
signalled = 1; scanbytecases();
|
||||
signalled = 2; scanlinecases();
|
||||
@@ -706,5 +893,9 @@ export fn main() i32 = {
|
||||
signalled = 16; streamflushempty();
|
||||
signalled = 17; streamcloseflushes();
|
||||
signalled = 18; streamsetflushcustom();
|
||||
signalled = 19; scanrunecases();
|
||||
signalled = 20; scanruneinvalid();
|
||||
signalled = 21; newscannergrow();
|
||||
signalled = 22; newscanneroverflow();
|
||||
return 0;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user