Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/bufio: newscanner + scanrune (regex fold-2b prereq)

Browse Source 
Port Hare's auto-grow newscanner (scanner.ha:72) and scan_rune
(scanner.ha:259). scanner gains a maxread field (== cap for
newscannerbuf, scanner.ha:101); readahead grows by BUFSZ up to
maxread via alloc+copy (Hare appends; ww flat ptr/cap scanner,
old block left to process-exit reclaim). scanbytes' overflow
test gains the avail >= maxread leg (Hare's pending >= readahead
predicate) so a growable scanner refills instead of overflowing.
finish ports the free(scan.buffer) verbatim per the regex #27
precedent. Tests: rune scan over 1/2/3/4-byte UTF-8 + EOF,
invalid initial/truncated/surrogate sequences, newscanner grow
round-trip + scanrune-over-newscanner, maxread overflow.
This commit is contained in:
Hojun-Cho
2026-06-04 07:07:02 +09:00
parent 9ec72b7895
commit b7a4eda40a
2 changed files with 326 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
lib/bufio/bufio.ww
View File

@@ -4,11 +4,14 @@
//
// Surface:
//
// bufio.newscanner (src: io.stream, maxread: i32) scanner
// bufio.newscannerbuf(src: io.stream, buf: []u8) scanner
// bufio.finish (s: *scanner) void
// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error)
// bufio.scanbytes (s: *scanner, delim: u8)
// ([]u8 | io.eof | io.error | overflow)
// bufio.scanrune (s: *scanner)
// (rune | io.eof | io.error | utf8.invalid)
// bufio.scanline (s: *scanner) (str | io.eof | io.error | overflow)
//
// bufio.init (src: io.stream, rbuf: []u8, wbuf: []u8) stream
@@ -26,12 +29,13 @@
// scanbyte / scanbytes / scanline.
//
// SCOPE: this is the surface ww ships, NOT a port of Hare's full
// scanner. The features ww never implemented — auto-grow newscanner,
// multibyte-delim scanbytes / scanstring, scanrune / readbyte / readtok
// / readline / readrune / unreadrune — are a separate future feature
// fold (#217). The ported set: the fixed-buffer newscannerbuf,
// single-byte scanbytes, single-byte scanline, scanbyte, finish; and
// the buffered-stream init / setflush / flush / unread / isbuffered.
// scanner. The features ww never implemented — multibyte-delim
// scanbytes / scanstring, readbyte / readtok / readline / readrune /
// unreadrune — are a separate future feature fold (#217). The ported
// set: the auto-grow newscanner, the fixed-buffer newscannerbuf,
// single-byte scanbytes, single-byte scanline, scanbyte, scanrune,
// finish; and the buffered-stream init / setflush / flush / unread /
// isbuffered.
//
// Cast workaround per #206-payoff (ken: KEEP the explicit casts; they
// are cgen-neutral and sidestep the #214 over-acceptance surface).
@@ -47,8 +51,14 @@
package bufio;
import encoding.utf8;
import io;
// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth
// increment. i32 (not Hare's size) per the lib-wide index-type
// convention (lib/CLAUDE.md).
def BUFSZ: i32 = 4096;
// flushdefault — backing storage for the default flush byte-set
// ("\n"). Hare scopes it inside `init` as `static let
// flush_default = ['\n': u8]` (ref/hare/bufio/stream.ha:75); ww
@@ -295,17 +305,37 @@ fn bclose(s: io.stream) (void | io.error) = {
// slice support.
export type scanner = struct {
src: io.stream,
ptr: *u8,
cap: i32,
start: i32, // index where the pending region starts in ptr
avail: i32, // pending byte count; pending = ptr[start..start+avail]
src: io.stream,
ptr: *u8,
cap: i32,
start: i32, // index where the pending region starts in ptr
avail: i32, // pending byte count; pending = ptr[start..start+avail]
maxread: i32, // growth ceiling; == cap for newscannerbuf (scanner.ha:101)
};
// newscanner — wire a scanner that allocates and grows its own
// read-ahead buffer, by BUFSZ per refill up to `maxread`. Returns the
// scanner BY VALUE. This is Hare's newscanner
// (ref/hare/bufio/scanner.ha:72) modulo two parameter drops: ww has no
// default arguments, so Hare's `maxread: size = types::SIZE_MAX` is a
// required i32 (callers wanting "no limit" pass types.I32_MAX), and the
// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD
// behavior is the shipped default).
export fn newscanner(src: io.stream, maxread: i32) scanner = {
let r: scanner;
r.src = src;
r.ptr = nil;
r.cap = 0;
r.start = 0;
r.avail = 0;
r.maxread = maxread;
return r;
};
// newscannerbuf — wire a scanner to read through `src` using `buf` as
// the fixed read-ahead window. Returns the scanner BY VALUE. This is
// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92); the auto-grow
// newscanner is a separate feature (#217).
// the fixed read-ahead window (maxread == buf.len, so the buffer never
// grows — scanner.ha:101). Returns the scanner BY VALUE. This is
// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92).
export fn newscannerbuf(src: io.stream, buf: []u8) scanner = {
let r: scanner;
r.src = src;
@@ -313,25 +343,55 @@ export fn newscannerbuf(src: io.stream, buf: []u8) scanner = {
r.cap = buf.len;
r.start = 0;
r.avail = 0;
r.maxread = buf.len;
return r;
};
// finish — release scanner-owned resources. No-op (buffer is
// caller-owned, src isn't closed); kept on the surface so callers won't
// churn. Mirrors ref/hare/bufio/scanner.ha:110.
export fn finish(s: *scanner) void = { };
// finish — release scanner-owned resources; src isn't closed. Mirrors
// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the
// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a
// newscannerbuf caller's buffer is untouched either way.
export fn finish(s: *scanner) void = {
free(s.ptr);
};
// readahead — make room and read once from src into the back of the
// pending region. Returns bytes newly buffered (>=0), or io.eof/io.error
// from src. The size from io.read narrows to i32 (buffer-length type).
//
// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer
// first shifts pending left, then — when start == 0 and maxread allows
// — grows. Hare grows via `append(scan.buffer, [0...], readahead)?`;
// ww's flat ptr/cap scanner allocates a fresh backing and copies (the
// old block is left to process-exit reclaim, ww no-free; `!` not `?`
// per the #36 nomem-propagation gap). The can't-grow case (avail >=
// maxread) is Hare's errors::overflow return — ww's io.error has no
// overflow member (lib/io/types.ww:40 enumerated union), so callers
// that can overflow (scanbytes) detect that state themselves before
// calling, against `maxread`.
fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
if (s.start + s.avail == s.cap && s.start > 0) {
let i: i32 = 0;
for (i < s.avail) {
s.ptr[i] = s.ptr[s.start + i];
i += 1;
if (s.start + s.avail == s.cap) {
if (s.start > 0) {
let i: i32 = 0;
for (i < s.avail) {
s.ptr[i] = s.ptr[s.start + i];
i += 1;
};
s.start = 0;
} else if (s.avail < s.maxread) {
let want: i32 = s.avail + BUFSZ;
if (want > s.maxread) { want = s.maxread; };
let ncap: i32 = s.avail + want;
let nbuf: []u8 = alloc([], ncap: u64)!;
let np: *u8 = nbuf.ptr;
let i: i32 = 0;
for (i < s.avail) {
np[i] = s.ptr[i];
i += 1;
};
s.ptr = np;
s.cap = ncap;
};
s.start = 0;
};
let off: i32 = s.start + s.avail;
let v: []u8;
@@ -386,7 +446,12 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
};
i += 1;
};
if (s.start + s.avail == s.cap && s.start == 0) {
// full + unshiftable + ungrowable (avail >= maxread is
// Hare's `pending >= readahead` overflow predicate,
// scanner.ha:179; for a newscannerbuf scanner maxread ==
// cap so this is the old fixed-buffer-full test).
if (s.start + s.avail == s.cap && s.start == 0
&& s.avail >= s.maxread) {
let e: overflow; return e;
};
let r: (i32 | io.eof | io.error) = readahead(s);
@@ -399,6 +464,51 @@ export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow
let e: io.eof; return e;
};
// scanrune — pop one UTF-8-encoded rune, refilling from src on demand.
// EOF mid-codepoint (fewer pending bytes than the initial byte
// announces) is utf8.invalid; a clean EOF before any byte is io.eof.
// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead
// when fewer than 4 bytes (the longest codepoint) are pending, then
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` also
// propagates errors::overflow through io::error; ww's io.error has no
// overflow member (see readahead) and a <4-byte refill cannot overflow
// a scanner that can hold a codepoint, so only io.error proper
// propagates here.
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid) = {
if (s.avail < 4) {
let ra: (i32 | io.eof | io.error) = readahead(s);
match (ra) {
case let n: i32 => { };
case io.eof => {
if (s.avail == 0) { let e: io.eof; return e; };
};
case let e: io.error => return e;
};
};
let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]);
let sz: i32 = 0;
match (szr) {
case let n: i32 => { sz = n; };
case utf8.invalid => { let e: utf8.invalid; return e; };
};
if (s.avail < sz) {
let e: utf8.invalid; return e;
};
let v: []u8;
v.ptr = s.ptr + (s.start: u64);
v.len = sz;
s.start += sz;
s.avail -= sz;
let dec: utf8.decoder = utf8.decode(v);
let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec);
match (nr) {
case let r: rune => return r;
case utf8.done => { let e: io.eof; return e; };
case utf8.more => { let e: utf8.invalid; return e; };
case utf8.invalid => { let e: utf8.invalid; return e; };
};
};
// scanline — read up to (and not including) the next '\n'. The newline
// is consumed; the returned str view borrows from the scanner buffer.
// Single-byte route (Hare's scan_line = scan_string(s, "\n"); the