Files
ww/lib/bufio/bufio.ww
Hojun-Cho b7a4eda40a lib/bufio: newscanner + scanrune (regex fold-2b prereq)
Port Hare's auto-grow newscanner (scanner.ha:72) and scan_rune
(scanner.ha:259). scanner gains a maxread field (== cap for
newscannerbuf, scanner.ha:101); readahead grows by BUFSZ up to
maxread via alloc+copy (Hare appends; ww flat ptr/cap scanner,
old block left to process-exit reclaim). scanbytes' overflow
test gains the avail >= maxread leg (Hare's pending >= readahead
predicate) so a growable scanner refills instead of overflowing.
finish ports the free(scan.buffer) verbatim per the regex #27
precedent. Tests: rune scan over 1/2/3/4-byte UTF-8 + EOF,
invalid initial/truncated/surrogate sequences, newscanner grow
round-trip + scanrune-over-newscanner, maxread overflow.
2026-06-04 07:07:02 +09:00

531 lines
18 KiB
Plaintext

// bufio — buffered I/O over [[io.stream]] (= `*io.vtable`). Subset of
// Hare's bufio:: surface (ref/hare/bufio/{scanner,stream}.ha). Project
// #94 fold-eFinal.
//
// Surface:
//
// bufio.newscanner (src: io.stream, maxread: i32) scanner
// bufio.newscannerbuf(src: io.stream, buf: []u8) scanner
// bufio.finish (s: *scanner) void
// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.error)
// bufio.scanbytes (s: *scanner, delim: u8)
// ([]u8 | io.eof | io.error | overflow)
// bufio.scanrune (s: *scanner)
// (rune | io.eof | io.error | utf8.invalid)
// bufio.scanline (s: *scanner) (str | io.eof | io.error | overflow)
//
// bufio.init (src: io.stream, rbuf: []u8, wbuf: []u8) stream
// bufio.flush (b: *stream) (void | io.error)
// bufio.setflush (b: *stream, bs: []u8) void
// bufio.unread (b: *stream, buf: []u8) void
// bufio.isbuffered (s: io.stream) bool
//
// VALUE-RETURN (Hare ref/hare/bufio/stream.ha:69 init, scanner.ha:92
// newscanner_buf): each constructor builds in a local `let r: T;`,
// field-assigns every slot, and `return r;`. The caller owns the
// returned struct (stack ownership, no-GC). For the buffered stream
// the caller passes `&b.vt` to the io dispatchers; the scanner is a
// plain read-ahead tokenizer (no embedded vtable) driven directly via
// scanbyte / scanbytes / scanline.
//
// SCOPE: this is the surface ww ships, NOT a port of Hare's full
// scanner. The features ww never implemented — multibyte-delim
// scanbytes / scanstring, readbyte / readtok / readline / readrune /
// unreadrune — are a separate future feature fold (#217). The ported
// set: the auto-grow newscanner, the fixed-buffer newscannerbuf,
// single-byte scanbytes, single-byte scanline, scanbyte, scanrune,
// finish; and the buffered-stream init / setflush / flush / unread /
// isbuffered.
//
// Cast workaround per #206-payoff (ken: KEEP the explicit casts; they
// are cgen-neutral and sidestep the #214 over-acceptance surface).
// `(&fn_name): *io.<role>` at each vtable store + the isbuffered
// fn-ptr-equality comparand — the #206 cast-drop is gated on #214.
//
// Mode discrimination (drew-deferred): Hare uses three vtable
// singletons (vtable_r / vtable_w / vtable_rw); ww's vtable always
// carries all three callbacks (a zero-length rbuf/wbuf degenerates the
// matching callback in-cb). Defer-handle (MANAGED_* ownership bits)
// also deferred — caller owns rbuf/wbuf/src; bclose flushes + forwards
// close but frees nothing. Both graduate with io fold-2 (#5).
package bufio;
import encoding.utf8;
import io;
// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth
// increment. i32 (not Hare's size) per the lib-wide index-type
// convention (lib/CLAUDE.md).
def BUFSZ: i32 = 4096;
// flushdefault — backing storage for the default flush byte-set
// ("\n"). Hare scopes it inside `init` as `static let
// flush_default = ['\n': u8]` (ref/hare/bufio/stream.ha:75); ww
// has no function-scope statics, so it lives at module scope.
let flushdefault: [1]u8 = [10u8];
// rt_abort — terminate on a precondition violation. Used by
// [[unread]] for the "buf fits in front of rbuf" assertion that
// Hare expresses with `assert`.
@symbol("rt_abort") fn rtabort(msg: str) void;
// overflow — the scanner buffer filled before the delimiter (or
// underlying EOF) was hit. With a caller-supplied buffer we can't
// grow; bumping the budget is on the caller. Mirrors Hare bufio's
// use of errors::overflow for the same condition.
export type overflow = !void;
// ---- buffered stream ------------------------------------------------------
// stream — heap-free buffered read+write over an underlying io.stream.
// `vt` at offset 0 for the intrusive io.stream→*stream cast. `src` is
// an io.stream (the vtable-native underlying handle). Mirrors
// ref/hare/bufio/stream.ha:18.
//
// rbuf[rstart..rend] is pending read data; wbuf[0..wend] is pending
// write data. rbuf or wbuf may be zero-length: read-empty makes bread
// return io.eof immediately, write-empty makes bwrite a pass-through to
// src (no buffering).
export type stream = struct {
vt: io.vtable,
src: io.stream,
rbuf: []u8,
rstart: i32,
rend: i32,
wbuf: []u8,
wend: i32,
flush: []u8,
};
// init — wire a buffered stream over an underlying io.stream with
// caller-supplied read/write buffers. Both rbuf and wbuf may be empty;
// bread / bwrite degenerate (see [[stream]]). The flush byte-set
// defaults to "\n" (line-buffered writes); [[setflush]] swaps it.
//
// Returns the stream BY VALUE; the caller passes `&b.vt` to the io
// dispatchers. Mirrors ref/hare/bufio/stream.ha:69.
export fn init(src: io.stream, rbuf: []u8, wbuf: []u8) stream = {
let r: stream;
r.vt.reader = (&bread): *io.reader;
r.vt.writer = (&bwrite): *io.writer;
r.vt.closer = (&bclose): *io.closer;
r.src = src;
// rstart=rbuf.len, rend=rbuf.len: pre-read unread budget = rbuf.len
// (Hare bufio/stream.ha:101).
r.rstart = rbuf.len;
r.rend = rbuf.len;
r.rbuf = rbuf;
r.wbuf = wbuf;
r.wend = 0;
r.flush = flushdefault[0:1];
return r;
};
// setflush — install a new flush byte-set. Any byte from `bs` appearing
// in a write payload triggers an automatic flush after the write copies
// into wbuf. Mirrors ref/hare/bufio/stream.ha:128.
export fn setflush(b: *stream, bs: []u8) void = {
b.flush = bs;
};
// flush — drain any pending wbuf data to src. Public API AND the
// internal drain called by bwrite / bclose. A 0-byte non-error write
// means the sink made no progress (memio.fixed full) — surfaced as a
// nomem-carried io.error rather than spinning, matching what io.writeall
// would do in Hare. Mirrors ref/hare/bufio/stream.ha.
export fn flush(b: *stream) (void | io.error) = {
if (b.wend == 0) { return; };
let off: i32 = 0;
for (off < b.wend) {
let r: (size | io.error) = io.write(b.src, b.wbuf[off:b.wend]);
match (r) {
case let n: size => {
if (n == 0: size) {
let nm: nomem;
let e: io.error = nm;
return e;
};
off += n: i32;
};
case let e: io.error => return e;
};
};
b.wend = 0;
return;
};
// unread — push `buf` back into the read buffer so the next bread
// returns it first. The bytes must fit in front of the pending region
// (rstart >= buf.len); Hare aborts on overflow, ww does the same via
// rtabort. Mirrors ref/hare/bufio/stream.ha:164.
export fn unread(b: *stream, buf: []u8) void = {
if (b.rstart < buf.len) {
rtabort("bufio.unread: more data than rbuf has room for");
};
let i: i32 = 0;
for (i < buf.len) {
b.rbuf[b.rstart - buf.len + i] = buf[i];
i += 1;
};
b.rstart -= buf.len;
};
// isbuffered — true when `s` was returned by [[init]]. Hare's
// discriminator is callback identity (ref/hare/bufio/stream.ha:179).
// Either reader or writer matching is sufficient.
export fn isbuffered(s: io.stream) bool = {
match (s.reader) {
case let r: *io.reader => {
if (r == (&bread): *io.reader) { return true; };
};
case void => { };
};
match (s.writer) {
case let w: *io.writer => {
if (w == (&bwrite): *io.writer) { return true; };
};
case void => { };
};
return false;
};
// ---- buffered-stream vtable callbacks ------------------------------------
// bread — buffered read. Recover stream via the intrusive cast; refill
// rbuf from src via io.read on empty pending region. Mirrors
// ref/hare/bufio/stream.ha (stream_read).
fn bread(s: io.stream, buf: []u8) (size | io.eof | io.error) = {
let b: *stream = s: *stream;
if (b.rbuf.len == 0) {
let e: io.eof; return e;
};
if (b.rstart >= b.rend) {
b.rstart = 0;
b.rend = 0;
let r: (size | io.eof | io.error) = io.read(b.src, b.rbuf);
match (r) {
case let n: size => { b.rend = n: i32; };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
};
};
let avail: i32 = b.rend - b.rstart;
let n: i32 = buf.len;
if (avail < n) { n = avail; };
let i: i32 = 0;
for (i < n) {
buf[i] = b.rbuf[b.rstart + i];
i += 1;
};
b.rstart += n;
return n: size;
};
// bwrite — buffered write. wbuf-empty passes through to src; otherwise
// default-flush scan + per-batch copy + post-write conditional flush.
// Labeled-break inlined (ww has no labeled break). Mirrors
// ref/hare/bufio/stream.ha (stream_write).
fn bwrite(s: io.stream, buf: []u8) (size | io.error) = {
let b: *stream = s: *stream;
if (b.wbuf.len == 0) {
return io.write(b.src, buf);
};
let doflush: bool = false;
if (b.flush.len != 0) {
let i: i32 = 0;
for (i < buf.len) {
let j: i32 = 0;
for (j < b.flush.len) {
if (buf[i] == b.flush[j]) {
doflush = true;
i = buf.len;
j = b.flush.len;
};
j += 1;
};
i += 1;
};
};
let z: i32 = 0;
for (z < buf.len) {
let avail: i32 = b.wbuf.len - b.wend;
if (avail == 0) {
let fr: (void | io.error) = flush(b);
match (fr) {
case void => { };
case let e: io.error => return e;
};
avail = b.wbuf.len;
};
let n: i32 = buf.len - z;
if (avail < n) { n = avail; };
let i: i32 = 0;
for (i < n) {
b.wbuf[b.wend + i] = buf[z + i];
i += 1;
};
b.wend += n;
z += n;
};
if (doflush) {
let fr: (void | io.error) = flush(b);
match (fr) {
case void => { };
case let e: io.error => return e;
};
};
return buf.len: size;
};
// bclose — flush pending wbuf, forward close to src via io.close.
// Caller-owned buffers/src are not freed (drew defer-handle deferral).
fn bclose(s: io.stream) (void | io.error) = {
let b: *stream = s: *stream;
let fr: (void | io.error) = flush(b);
match (fr) {
case void => { };
case let e: io.error => return e;
};
let cr: (void | io.error) = io.close(b.src);
match (cr) {
case void => return void;
case let e: io.error => return e;
};
};
// ---- scanner (read-ahead tokenizer over an io.stream src) ----------------
//
// Plain tokenizer — no embedded vtable; driven directly via scanbyte /
// scanbytes / scanline, not through io dispatch. Its `src` is an
// io.stream and refills go through io.read. Value-return constructor
// (Hare newscanner_buf, scanner.ha:92).
//
// ptr/cap kept flat (no `buf: []u8`); the scanner predates struct-held
// slice support.
export type scanner = struct {
src: io.stream,
ptr: *u8,
cap: i32,
start: i32, // index where the pending region starts in ptr
avail: i32, // pending byte count; pending = ptr[start..start+avail]
maxread: i32, // growth ceiling; == cap for newscannerbuf (scanner.ha:101)
};
// newscanner — wire a scanner that allocates and grows its own
// read-ahead buffer, by BUFSZ per refill up to `maxread`. Returns the
// scanner BY VALUE. This is Hare's newscanner
// (ref/hare/bufio/scanner.ha:72) modulo two parameter drops: ww has no
// default arguments, so Hare's `maxread: size = types::SIZE_MAX` is a
// required i32 (callers wanting "no limit" pass types.I32_MAX), and the
// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD
// behavior is the shipped default).
export fn newscanner(src: io.stream, maxread: i32) scanner = {
let r: scanner;
r.src = src;
r.ptr = nil;
r.cap = 0;
r.start = 0;
r.avail = 0;
r.maxread = maxread;
return r;
};
// newscannerbuf — wire a scanner to read through `src` using `buf` as
// the fixed read-ahead window (maxread == buf.len, so the buffer never
// grows — scanner.ha:101). Returns the scanner BY VALUE. This is
// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92).
export fn newscannerbuf(src: io.stream, buf: []u8) scanner = {
let r: scanner;
r.src = src;
r.ptr = buf.ptr;
r.cap = buf.len;
r.start = 0;
r.avail = 0;
r.maxread = buf.len;
return r;
};
// finish — release scanner-owned resources; src isn't closed. Mirrors
// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the
// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a
// newscannerbuf caller's buffer is untouched either way.
export fn finish(s: *scanner) void = {
free(s.ptr);
};
// readahead — make room and read once from src into the back of the
// pending region. Returns bytes newly buffered (>=0), or io.eof/io.error
// from src. The size from io.read narrows to i32 (buffer-length type).
//
// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer
// first shifts pending left, then — when start == 0 and maxread allows
// — grows. Hare grows via `append(scan.buffer, [0...], readahead)?`;
// ww's flat ptr/cap scanner allocates a fresh backing and copies (the
// old block is left to process-exit reclaim, ww no-free; `!` not `?`
// per the #36 nomem-propagation gap). The can't-grow case (avail >=
// maxread) is Hare's errors::overflow return — ww's io.error has no
// overflow member (lib/io/types.ww:40 enumerated union), so callers
// that can overflow (scanbytes) detect that state themselves before
// calling, against `maxread`.
fn readahead(s: *scanner) (i32 | io.eof | io.error) = {
if (s.start + s.avail == s.cap) {
if (s.start > 0) {
let i: i32 = 0;
for (i < s.avail) {
s.ptr[i] = s.ptr[s.start + i];
i += 1;
};
s.start = 0;
} else if (s.avail < s.maxread) {
let want: i32 = s.avail + BUFSZ;
if (want > s.maxread) { want = s.maxread; };
let ncap: i32 = s.avail + want;
let nbuf: []u8 = alloc([], ncap: u64)!;
let np: *u8 = nbuf.ptr;
let i: i32 = 0;
for (i < s.avail) {
np[i] = s.ptr[i];
i += 1;
};
s.ptr = np;
s.cap = ncap;
};
};
let off: i32 = s.start + s.avail;
let v: []u8;
v.ptr = s.ptr + (off: u64);
v.len = s.cap - off;
let r: (size | io.eof | io.error) = io.read(s.src, v);
match (r) {
case let n: size => {
s.avail += n: i32;
return n: i32;
};
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
};
};
// scanbyte — pop one byte, refilling from src on demand. Mirrors
// ref/hare/bufio/scanner.ha:204.
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error) = {
for (s.avail == 0) {
let r: (i32 | io.eof | io.error) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
};
};
let b: u8 = s.ptr[s.start];
s.start += 1;
s.avail -= 1;
return b;
};
// scanbytes — read up to (and not including) the next byte equal to
// `delim`. The delim is consumed but not returned. The returned slice
// borrows from the scanner buffer and is invalidated by the next scan.
// EOF without delim discards the trailing fragment and returns io.eof
// (Hare EOF_DISCARD default); buffer-full without delim returns
// overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte
// form is #217. Mirrors ref/hare/bufio/scanner.ha:220 (narrowed).
export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow) = {
let i: i32 = 0;
for (true) {
for (i < s.avail) {
if (s.ptr[s.start + i] == delim) {
let v: []u8;
v.ptr = s.ptr + (s.start: u64);
v.len = i;
s.start += i + 1;
s.avail -= i + 1;
return v;
};
i += 1;
};
// full + unshiftable + ungrowable (avail >= maxread is
// Hare's `pending >= readahead` overflow predicate,
// scanner.ha:179; for a newscannerbuf scanner maxread ==
// cap so this is the old fixed-buffer-full test).
if (s.start + s.avail == s.cap && s.start == 0
&& s.avail >= s.maxread) {
let e: overflow; return e;
};
let r: (i32 | io.eof | io.error) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
};
};
let e: io.eof; return e;
};
// scanrune — pop one UTF-8-encoded rune, refilling from src on demand.
// EOF mid-codepoint (fewer pending bytes than the initial byte
// announces) is utf8.invalid; a clean EOF before any byte is io.eof.
// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead
// when fewer than 4 bytes (the longest codepoint) are pending, then
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` also
// propagates errors::overflow through io::error; ww's io.error has no
// overflow member (see readahead) and a <4-byte refill cannot overflow
// a scanner that can hold a codepoint, so only io.error proper
// propagates here.
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid) = {
if (s.avail < 4) {
let ra: (i32 | io.eof | io.error) = readahead(s);
match (ra) {
case let n: i32 => { };
case io.eof => {
if (s.avail == 0) { let e: io.eof; return e; };
};
case let e: io.error => return e;
};
};
let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]);
let sz: i32 = 0;
match (szr) {
case let n: i32 => { sz = n; };
case utf8.invalid => { let e: utf8.invalid; return e; };
};
if (s.avail < sz) {
let e: utf8.invalid; return e;
};
let v: []u8;
v.ptr = s.ptr + (s.start: u64);
v.len = sz;
s.start += sz;
s.avail -= sz;
let dec: utf8.decoder = utf8.decode(v);
let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec);
match (nr) {
case let r: rune => return r;
case utf8.done => { let e: io.eof; return e; };
case utf8.more => { let e: utf8.invalid; return e; };
case utf8.invalid => { let e: utf8.invalid; return e; };
};
};
// scanline — read up to (and not including) the next '\n'. The newline
// is consumed; the returned str view borrows from the scanner buffer.
// Single-byte route (Hare's scan_line = scan_string(s, "\n"); the
// arbitrary multibyte-delim scan_string is #217). Mirrors
// ref/hare/bufio/scanner.ha:307.
export fn scanline(s: *scanner) (str | io.eof | io.error | overflow) = {
let r: ([]u8 | io.eof | io.error | overflow) = scanbytes(s, 10u8);
match (r) {
case let bs: []u8 => {
let v: str;
v.ptr = bs.ptr;
v.len = bs.len;
return v;
};
case io.eof => { let e: io.eof; return e; };
case let e: io.error => return e;
case overflow => { let e: overflow; return e; };
};
};