ww/lib/bufio/scanner.ww

package bufio;

import encoding.utf8;
import io;

// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth
// increment. i32 (not Hare's size) per the lib-wide index-type
// convention (lib/CLAUDE.md).
def BUFSZ: i32 = 4096;

// overflow — the scanner buffer filled before the delimiter (or
// underlying EOF) was hit. With a caller-supplied buffer we can't
// grow; bumping the budget is on the caller. Mirrors Hare bufio's
// use of errors::overflow for the same condition.
export type overflow = !void;

// Plain tokenizer — no embedded vtable; driven directly via scanbyte /
// scanbytes / scanline, not through io dispatch. Its `src` is an
// io.stream and refills go through io.read. Value-return constructor
// (Hare newscanner_buf, scanner.ha:92).
//
// ptr/cap kept flat (no `buf: []u8`); the scanner predates struct-held
// slice support.

export type scanner = struct {
	src:     io.stream,
	ptr:     *u8,
	cap:     i32,
	start:   i32,	// index where the pending region starts in ptr
	avail:   i32,	// pending byte count; pending = ptr[start..start+avail]
	maxread: i32,	// growth ceiling; == cap for newscannerbuf (scanner.ha:101)
};

// newscanner — wire a scanner that allocates and grows its own
// read-ahead buffer, by BUFSZ per refill up to `maxread`. Returns the
// scanner BY VALUE. This is Hare's newscanner
// (ref/hare/bufio/scanner.ha:72) modulo two parameter drops: ww has no
// default arguments, so Hare's `maxread: size = types::SIZE_MAX` is a
// required i32 (callers wanting "no limit" pass types.I32_MAX), and the
// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD
// behavior is the shipped default).
export fn newscanner(src: io.stream, maxread: i32) scanner = {
	let r: scanner;
	r.src = src;
	r.ptr = nil;
	r.cap = 0;
	r.start = 0;
	r.avail = 0;
	r.maxread = maxread;
	return r;
};

// newscannerbuf — wire a scanner to read through `src` using `buf` as
// the fixed read-ahead window (maxread == buf.len, so the buffer never
// grows — scanner.ha:101). Returns the scanner BY VALUE. This is
// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92).
export fn newscannerbuf(src: io.stream, buf: []u8) scanner = {
	let r: scanner;
	r.src = src;
	r.ptr = buf.ptr;
	r.cap = buf.len;
	r.start = 0;
	r.avail = 0;
	r.maxread = buf.len;
	return r;
};

// finish — release scanner-owned resources; src isn't closed. Mirrors
// ref/hare/bufio/scanner.ha:110 (free(scan.buffer)); ww's free() is the
// no-op builtin (lib/regex/regex.ww finish precedent, #27), so a
// newscannerbuf caller's buffer is untouched either way.
export fn finish(s: *scanner) void = {
	free(s.ptr);
};

// readahead — make room and read once from src into the back of the
// pending region. Returns bytes newly buffered (>=0), io.eof/io.error
// from src, or overflow when the buffer is full at the maxread ceiling.
// The size from io.read narrows to i32 (buffer-length type).
//
// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer
// first shifts pending left, then — when start == 0 — grows, or returns
// overflow when `avail >= want` (Hare's `pending >= readahead` ceiling,
// scanner.ha:179-181, BEFORE the append). Hare grows via
// `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner
// allocates a fresh backing and copies (the old block is left to
// process-exit reclaim, ww no-free; `!` not `?` per the #36
// nomem-propagation gap). overflow is bufio-local (scanner.ww:15), NOT
// errors.overflow: ww's io.error is a closed enumerated union with no
// overflow member (lib/io/types.ww), so the can't-grow signal rides a
// distinct arm that scanbyte / scanbytes / scanrune match-forward —
// the single choke-point (drew ruling, drain F-B). Pre-fix this case
// fell through silently to a zero-length io.read, spinning scanbyte
// (catB-144) and nil-derefing scanrune (catB-145).
fn readahead(s: *scanner) (i32 | io.eof | io.error | overflow) = {
	if (s.start + s.avail == s.cap) {
		if (s.start > 0) {
			let i: i32 = 0;
			for (i < s.avail) {
				s.ptr[i] = s.ptr[s.start + i];
				i += 1;
			};
			s.start = 0;
		} else {
			let want: i32 = s.avail + BUFSZ;
			if (want > s.maxread) { want = s.maxread; };
			if (s.avail >= want) {
				let e: overflow; return e;
			};
			let ncap: i32 = s.avail + want;
			let nbuf: []u8 = alloc([], ncap: u64)!;
			let np: *u8 = nbuf.ptr;
			let i: i32 = 0;
			for (i < s.avail) {
				np[i] = s.ptr[i];
				i += 1;
			};
			s.ptr = np;
			s.cap = ncap;
		};
	};
	let off: i32 = s.start + s.avail;
	let v: []u8;
	v.ptr = s.ptr + (off: u64);
	v.len = s.cap - off;
	let r: (size | io.eof | io.error) = io.read(s.src, v);
	match (r) {
	case let n: size => {
		s.avail += n: i32;
		return n: i32;
	};
	case io.eof => { let e: io.eof; return e; };
	case let e: io.error => return e;
	};
};

// scanbyte — pop one byte, refilling from src on demand. Mirrors
// ref/hare/bufio/scanner.ha:204.
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | overflow) = {
	for (s.avail == 0) {
		let r: (i32 | io.eof | io.error | overflow) = readahead(s);
		match (r) {
		case let n: i32 => { };
		case io.eof => { let e: io.eof; return e; };
		case let e: io.error => return e;
		case overflow => { let e: overflow; return e; };
		};
	};
	let b: u8 = s.ptr[s.start];
	s.start += 1;
	s.avail -= 1;
	return b;
};

// scanbytes — read up to (and not including) the next byte equal to
// `delim`. The delim is consumed but not returned. The returned slice
// borrows from the scanner buffer and is invalidated by the next scan.
// EOF without delim discards the trailing fragment and returns io.eof
// (Hare EOF_DISCARD default); buffer-full without delim returns
// overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte
// form is #217. Mirrors ref/hare/bufio/scanner.ha:220 (narrowed).
export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | overflow) = {
	let i: i32 = 0;
	for (true) {
		for (i < s.avail) {
			if (s.ptr[s.start + i] == delim) {
				let v: []u8;
				v.ptr = s.ptr + (s.start: u64);
				v.len = i;
				s.start += i + 1;
				s.avail -= i + 1;
				return v;
			};
			i += 1;
		};
		// overflow now surfaces from readahead's single choke-point
		// (avail >= want == Hare's `pending >= readahead`,
		// scanner.ha:179); no separate pre-check (drew ruling, F-B).
		let r: (i32 | io.eof | io.error | overflow) = readahead(s);
		match (r) {
		case let n: i32 => { };
		case io.eof => { let e: io.eof; return e; };
		case let e: io.error => return e;
		case overflow => { let e: overflow; return e; };
		};
	};
	let e: io.eof; return e;
};

// scanrune — pop one UTF-8-encoded rune, refilling from src on demand.
// EOF mid-codepoint (fewer pending bytes than the initial byte
// announces) is utf8.invalid; a clean EOF before any byte is io.eof.
// Mirrors ref/hare/bufio/scanner.ha:259 (scan_rune): one readahead
// when fewer than 4 bytes (the longest codepoint) are pending, then
// utf8sz / consume / decode. Hare's `scan_readahead(scan)?` propagates
// errors::overflow (through io::error); ww forwards the bufio-local
// overflow as a distinct arm — a zero-cap / at-ceiling scanner cannot
// buffer the first byte, so without this arm the readahead fall-through
// would nil-deref s.ptr[s.start] (catB-145).
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | overflow) = {
	if (s.avail < 4) {
		let ra: (i32 | io.eof | io.error | overflow) = readahead(s);
		match (ra) {
		case let n: i32 => { };
		case io.eof => {
			if (s.avail == 0) { let e: io.eof; return e; };
		};
		case let e: io.error => return e;
		case overflow => { let e: overflow; return e; };
		};
	};
	let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]);
	let sz: i32 = 0;
	match (szr) {
	case let n: i32 => { sz = n; };
	case utf8.invalid => { let e: utf8.invalid; return e; };
	};
	if (s.avail < sz) {
		let e: utf8.invalid; return e;
	};
	let v: []u8;
	v.ptr = s.ptr + (s.start: u64);
	v.len = sz;
	s.start += sz;
	s.avail -= sz;
	let dec: utf8.decoder = utf8.decode(v);
	let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec);
	match (nr) {
	case let r: rune => return r;
	case utf8.done => { let e: io.eof; return e; };
	case utf8.more => { let e: utf8.invalid; return e; };
	case utf8.invalid => { let e: utf8.invalid; return e; };
	};
};

// scanline — read up to (and not including) the next '\n'. The newline
// is consumed; the returned str view borrows from the scanner buffer.
// Single-byte route (Hare's scan_line = scan_string(s, "\n"); the
// arbitrary multibyte-delim scan_string is #217). Mirrors
// ref/hare/bufio/scanner.ha:307.
export fn scanline(s: *scanner) (str | io.eof | io.error | overflow) = {
	let r: ([]u8 | io.eof | io.error | overflow) = scanbytes(s, 10u8);
	match (r) {
	case let bs: []u8 => {
		let v: str;
		v.ptr = bs.ptr;
		v.len = bs.len;
		return v;
	};
	case io.eof => { let e: io.eof; return e; };
	case let e: io.error => return e;
	case overflow => { let e: overflow; return e; };
	};
};