Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib+test: graduate bufio to Hare scanner on io.stream

Browse Source 
Scanner subset: newscanner, finish, scanbyte, scanline, scantok,
overflow (named-void). Buffer caller-owned; EOF_DISCARD default.
Skips Hare's scanner-as-io.stream embed pending task #6 (chained
N_DOT-of-N_DOT miscompile).
This commit is contained in:
Hojun-Cho
2026-05-13 20:38:55 +09:00
parent 2243849855
commit 036b2c851f
4 changed files with 640 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

259
lib/bufio/bufio.ww
View File

@@ -1,72 +1,197 @@
// bufio — buffered reader/writer over io.stream. Plan 9 'bio'
// analogue, lowered to ww. The buffer is owned by the caller and
// passed in at init time; we don't allocate.
// bufio — buffered scanner over [[io.stream]]. Subset of Hare's
// bufio:: scanner shape (ref/hare/bufio/scanner.ha).
//
// We keep the stream pointer as *void here to avoid a cross-module
// type name (the module-import system isn't online yet); a later
// revision will replace it with *io.stream once `use` resolves
// types from imported modules.
type streamp = *void;
type buf = struct {
s: streamp,
data: *u8,
cap: i32,
r: i32, // read cursor
w: i32, // write cursor (for writers)
};
export fn init(b: *buf, s: streamp, data: *u8, cap: i32) void = {
b.s = s;
b.data = data;
b.cap = cap;
b.r = 0;
b.w = 0;
};
// readbyte — pop one byte. void variant signals EOF (empty buffer).
// Hare name; the (i32 | void) shape is a subset of Hare's full
// (u8 | EOF | io::error) — error reporting from the underlying
// stream will arrive when bufio actually wires up to io::stream.
export fn readbyte(b: *buf) (i32 | void) = {
if (b.r < b.w) {
let c: u8 = b.data[b.r];
b.r += 1;
return c: i32;
};
return;
};
// Distinct alias so `(str | linerr)` has two variant types the
// tagged-union machinery can keep apart at the tag level. The error
// variant carries a short description; callers inspect by length.
// Placeholder until bufio graduates to io.stream + named-void tags
// (eof / closed / underread).
type linerr = str;
// readline — Hare-style fallible line read (name transliterated from
// `read_line`). Drains the buffer up to (but not including) the next
// '\n' and advances the cursor past the newline. Returns the line as
// a borrowed str on success, or a linerr describing why no line was
// available:
// - "eof" when the buffer is empty
// - "no newline" when the buffer contains data but no '\n'
// Surface today:
//
// The returned str borrows from the underlying buffer; callers must
// consume it (or copy) before refilling.
export fn readline(b: *buf) (str | linerr) = {
if (b.r >= b.w) { return "eof": linerr; };
let i: i32 = b.r;
for (i < b.w) {
if (b.data[i] == 10u8) {
let s: str;
s.ptr = b.data + b.r;
s.len = i - b.r;
b.r = i + 1;
return s;
// bufio.newscanner(s: *scanner, src: *io.stream, buf: []u8) void
// bufio.finish (s: *scanner) void
// bufio.scanbyte (s: *scanner) (u8 | io.eof | io.closed)
// bufio.scantok (s: *scanner, delim: u8)
// ([]u8 | io.eof | io.closed | overflow)
// bufio.scanline (s: *scanner) (str | io.eof | io.closed | overflow)
//
// Divergence from Hare:
//
// • Hare returns the scanner by value; ww cgen can't return
// structs wider than 16B by value yet, so `newscanner` is an
// out-parameter (`*scanner`). Same workaround memio.fixed uses.
//
// • Hare embeds an io::stream as the scanner's first field so the
// scanner doubles as a reader for higher layers. ww cgen
// currently miscompiles chained dotted reads through nested
// value-struct fields (`o.i.a` lowers to an undefined symbol
// `a`); the embedded-stream slot would hit that path on every
// callback. Drop the embed until the cgen fix lands; the
// scanner is consumed via scanbyte / scantok / scanline today.
//
// • The buffer is held flat as `(ptr, cap)` rather than a `[]u8`
// field — chained dot through a slice field (`s.buf.len`) is
// still broken. Mirrors lib/memio's flat-field workaround.
//
// • Hare's `newscanner` allocates and grows the buffer up to
// `maxread`; we ship only the caller-supplied shape (Hare's
// `newscanner_buf`) and pick up the auto-grow variant when an
// append over a struct-held slice works.
//
// • EOF-handling defaults to Hare's `EOF_DISCARD`: bytes between
// the last delimiter and EOF are dropped and io.eof is returned
// on the call that would have read them. Hare's EOF_GREEDY mode
// isn't shipped (no caller needs it yet).
//
// Owning model: caller owns the scanner state, the byte buffer, and
// the source stream. `finish` doesn't free the buffer and doesn't
// close src; it's a no-op today but stays on the surface so callers
// don't churn when bufio grows internal allocations.
//
// let mem: memio.state;
// let m: io.stream;
// memio.fixed(&mem, &m, raw[0:N]);
// let buf: [128]u8;
// let sc: bufio.scanner;
// bufio.newscanner(&sc, &m, buf[0:128]);
// match (bufio.scanline(&sc)) { ... };
// bufio.finish(&sc);
use io;
// overflow — the scanner buffer filled before the delimiter (or
// underlying EOF) was hit. With a caller-supplied buffer we can't
// grow; bumping the budget is on the caller. Mirrors Hare bufio's
// use of errors::overflow for the same condition.
export type overflow = !void;
export type scanner = struct {
src: *io.stream,
ptr: *u8,
cap: i32,
start: i32, // index where the pending region starts in ptr
avail: i32, // pending byte count; pending = ptr[start..start+avail]
};
// newscanner — wire `s` to read through `src` using `buf` as the
// read-ahead window. Hare returns the scanner by value
// (newscanner_buf); we take an out-parameter pending cgen support
// for wide-struct return.
export fn newscanner(s: *scanner, src: *io.stream, buf: []u8) void = {
s.src = src;
s.ptr = buf.ptr;
s.cap = buf.len;
s.start = 0;
s.avail = 0;
};
// finish — release scanner-owned resources. No-op today (buffer is
// caller-owned, src isn't closed); kept on the surface so callers
// won't churn when bufio later grows internal allocations.
export fn finish(s: *scanner) void = { };
// readahead — make room and read once from src into the back of the
// pending region. Returns the number of bytes newly buffered (≥0,
// can be 0 if the underlying stream made no progress), or
// io.eof/io.closed propagated from src.
fn readahead(s: *scanner) (i32 | io.eof | io.closed) = {
if (s.start + s.avail == s.cap && s.start > 0) {
// Shift pending region to the front of the buffer.
let i: i32 = 0;
for (i < s.avail) {
s.ptr[i] = s.ptr[s.start + i];
i += 1;
};
i += 1;
s.start = 0;
};
let off: i32 = s.start + s.avail;
let v: []u8;
v.ptr = s.ptr + (off: u64);
v.len = s.cap - off;
let r: (i32 | io.eof | io.closed) = io.read(s.src, v);
match (r) {
case let n: i32 => {
s.avail += n;
return n;
};
case io.eof => { let e: io.eof; return e; };
case io.closed => { let e: io.closed; return e; };
};
};
// scanbyte — pop one byte from the scanner, refilling from src on
// demand. Mirrors Hare's `scan_byte` (ref/hare/bufio/scanner.ha:204).
export fn scanbyte(s: *scanner) (u8 | io.eof | io.closed) = {
for (s.avail == 0) {
let r: (i32 | io.eof | io.closed) = readahead(s);
match (r) {
case let n: i32 => { };
case io.eof => { let e: io.eof; return e; };
case io.closed => { let e: io.closed; return e; };
};
};
let b: u8 = s.ptr[s.start];
s.start += 1;
s.avail -= 1;
return b;
};
// scantok — read up to (and not including) the next byte equal to
// `delim`. The delim byte is consumed from the stream but not
// returned. The returned slice borrows from the scanner's internal
// buffer and is invalidated by the next scan call.
//
// EOF without finding delim discards the trailing fragment and
// returns io.eof (Hare's EOF_DISCARD default). Buffer-full without
// delim returns overflow.
//
// Mirrors Hare's `scan_bytes` (ref/hare/bufio/scanner.ha:220),
// narrowed to a single-byte delimiter.
export fn scantok(s: *scanner, delim: u8) ([]u8 | io.eof | io.closed | overflow) = {
let i: i32 = 0;
for (true) {
for (i < s.avail) {
if (s.ptr[s.start + i] == delim) {
let v: []u8;
v.ptr = s.ptr + (s.start: u64);
v.len = i;
s.start += i + 1;
s.avail -= i + 1;
return v;
};
i += 1;
};
// No delim in pending. If the buffer is full with nowhere
// to shift, the caller's budget is too small — overflow.
if (s.start + s.avail == s.cap && s.start == 0) {
let e: overflow; return e;
};
let r: (i32 | io.eof | io.closed) = readahead(s);
match (r) {
case let n: i32 => {
// readahead may shift start to 0; the searched bytes
// move with it, so `i` (count from start) is still
// accurate. Resume scanning where we left off.
};
case io.eof => { let e: io.eof; return e; };
case io.closed => { let e: io.closed; return e; };
};
};
let e: io.eof; return e;
};
// scanline — read up to (and not including) the next '\n'. The
// newline is consumed; the returned str view borrows from the
// scanner buffer and is invalidated by the next scan call.
// Hare's `scan_line` is `scan_string(s, "\n")`; we route through
// scantok directly since lib/strings doesn't yet ship `toutf8` over
// a multi-byte delim.
export fn scanline(s: *scanner) (str | io.eof | io.closed | overflow) = {
let r: ([]u8 | io.eof | io.closed | overflow) = scantok(s, 10u8);
match (r) {
case let bs: []u8 => {
let v: str;
v.ptr = bs.ptr;
v.len = bs.len;
return v;
};
case io.eof => { let e: io.eof; return e; };
case io.closed => { let e: io.closed; return e; };
case overflow => { let e: overflow; return e; };
};
return "no newline": linerr;
};