ww/lib/bufio/scanner.ww

package bufio;

import encoding.utf8;
import errors;
import io;
import os;

// ref/hare/bufio/scanner.ha:11 — the auto-grow scanner's growth
// increment. i32 (not Hare's size) per the lib-wide index-type
// convention (lib/CLAUDE.md).
def BUFSZ: i32 = 4096;

// Plain tokenizer — no embedded vtable; driven directly via scanbyte /
// scanbytes / scanline, not through io dispatch. Its `src` is an
// io.stream and refills go through io.read. Value-return constructor
// (Hare newscanner_buf, scanner.ha:92).
//
// ptr/cap kept flat (no `buf: []u8`); the scanner predates struct-held
// slice support.

export type scanner = struct {
	src:     io.stream,
	ptr:     *u8,
	cap:     i32,
	start:   i32,	// index where the pending region starts in ptr
	avail:   i32,	// pending byte count; pending = ptr[start..start+avail]
	maxread: i32,	// growth ceiling; == cap for newscannerbuf (scanner.ha:101)
	owned:   bool,	// ptr is scanner-owned and finish must release it
	eof:     bool,	// source reached EOF or returned no progress
};

// newscanner — wire a scanner that allocates and grows its own
// read-ahead buffer, by BUFSZ per refill up to `maxread`. Returns the
// scanner BY VALUE. This is Hare's newscanner
// (ref/hare/bufio/scanner.ha:72) modulo two parameter drops: ww has no
// default arguments, so Hare's `maxread: size = types::SIZE_MAX` is a
// required i32 (callers wanting "no limit" pass types.I32_MAX), and the
// defaulted `opts` is dropped like newscannerbuf's (EOF_DISCARD
// behavior is the shipped default).
export fn newscanner(src: io.stream, maxread: i32) scanner = {
	assert(maxread >= 0,
		"bufio.newscanner: maximum read size must not be negative");
	let r: scanner;
	r.src = src;
	r.ptr = nil;
	r.cap = 0;
	r.start = 0;
	r.avail = 0;
	r.maxread = maxread;
	r.owned = true;
	r.eof = false;
	return r;
};

// newscannerbuf — wire a scanner to read through `src` using `buf` as
// the fixed read-ahead window (maxread == buf.len, so the buffer never
// grows — scanner.ha:101). Returns the scanner BY VALUE. This is
// Hare's newscanner_buf (ref/hare/bufio/scanner.ha:92).
export fn newscannerbuf(src: io.stream, buf: []u8) scanner = {
	let r: scanner;
	r.src = src;
	r.ptr = buf.ptr;
	r.cap = buf.len;
	r.start = 0;
	r.avail = 0;
	r.maxread = buf.len;
	r.owned = false;
	r.eof = false;
	return r;
};

// finish — release scanner-owned resources; src isn't closed. A fixed
// newscannerbuf window is borrowed and remains the caller's property.
// Mirrors ref/hare/bufio/scanner.ha:110.
export fn finish(s: *scanner) void = {
	if (s.owned && s.cap > 0) {
		os.free(s.ptr: *void, s.cap: u64);
	};
	s.ptr = nil;
	s.cap = 0;
	s.start = 0;
	s.avail = 0;
	s.maxread = 0;
	s.owned = false;
	s.eof = true;
};

// readahead — make room and read once from src into the back of the
// pending region. Returns bytes newly buffered (>0), io.eof/io.error
// from src, or errors.overflow at the maxread ceiling. A zero-byte read
// is terminal EOF: retryable sources must return an explicit error.
// The size from io.read narrows to i32 (buffer-length type).
//
// Mirrors ref/hare/bufio/scanner.ha:162 (scan_readahead): full buffer
// first shifts pending left, then — when start == 0 — grows, or returns
// overflow when no capacity remains (Hare's `pending >= readahead` ceiling,
// scanner.ha:179-181, BEFORE the append). Hare grows via
// `append(scan.buffer, [0...], readahead)?`; ww's flat ptr/cap scanner
// allocates a fresh backing, copies pending bytes, and releases the
// replaced owned block. Allocation remains `!` rather than `?` per the
// #36 nomem-propagation gap. The generic overflow condition rides a
// distinct arm because io.error is closed (lib/io/types.ww).
fn readahead(s: *scanner) (i32 | io.eof | io.error | errors.overflow) = {
	if (s.eof) { let e: io.eof; return e; };
	if (s.start + s.avail == s.cap) {
		if (s.start > 0) {
			bufiomove(s.ptr, s.ptr + (s.start: u64), s.avail);
			s.start = 0;
		} else {
			let room: i32 = s.maxread - s.avail;
			if (room <= 0) {
				let e: errors.overflow; return e;
			};
			let ncap: i32 = s.maxread;
			if (room > BUFSZ) { ncap = s.avail + BUFSZ; };
			let nbuf: []u8 = alloc([], ncap: u64)!;
			let np: *u8 = nbuf.ptr;
			bufiomove(np, s.ptr, s.avail);
			if (s.owned && s.cap > 0) {
				os.free(s.ptr: *void, s.cap: u64);
			};
			s.ptr = np;
			s.cap = ncap;
			s.owned = true;
		};
	};
	let off: i32 = s.start + s.avail;
	let v: []u8;
	v.ptr = s.ptr + (off: u64);
	v.len = s.cap - off;
	v.cap = v.len;
	let r: (size | io.eof | io.error) = io.read(s.src, v);
	match (r) {
	case let n: size => {
		assert(n <= v.len: size,
			"bufio scanner: reader returned an oversized count");
		if (n == 0: size) {
			s.eof = true;
			let e: io.eof;
			return e;
		};
		s.avail += n: i32;
		return n: i32;
	};
	case io.eof => {
		s.eof = true;
		let e: io.eof;
		return e;
	};
	case let e: io.error => return e;
	};
};

// scanbyte — pop one byte, refilling from src on demand. Mirrors
// ref/hare/bufio/scanner.ha:204.
export fn scanbyte(s: *scanner) (u8 | io.eof | io.error | errors.overflow) = {
	for (s.avail == 0) {
		let r: (i32 | io.eof | io.error | errors.overflow) = readahead(s);
		match (r) {
		case let n: i32 => { };
		case io.eof => { let e: io.eof; return e; };
		case let e: io.error => return e;
		case errors.overflow => { let e: errors.overflow; return e; };
		};
	};
	let b: u8 = s.ptr[s.start];
	s.start += 1;
	s.avail -= 1;
	return b;
};

// scanbytes — read up to (and not including) the next byte equal to
// `delim`. The delim is consumed but not returned. The returned slice
// borrows from the scanner buffer and is invalidated by the next scan.
// EOF without delim discards the trailing fragment and returns io.eof
// (Hare EOF_DISCARD default); buffer-full without delim returns
// errors.overflow. Single-byte delim only — Hare's `(u8 | []u8)` multibyte
// form is #217. Mirrors ref/hare/bufio/scanner.ha:220 (narrowed).
export fn scanbytes(s: *scanner, delim: u8) ([]u8 | io.eof | io.error | errors.overflow) = {
	let i: i32 = 0;
	for (true) {
		for (i < s.avail) {
			if (s.ptr[s.start + i] == delim) {
				let v: []u8;
				v.ptr = s.ptr + (s.start: u64);
				v.len = i;
				v.cap = v.len;
				s.start += i + 1;
				s.avail -= i + 1;
				return v;
			};
			i += 1;
		};
		// Capacity failure surfaces from readahead's single choke-point.
		let r: (i32 | io.eof | io.error | errors.overflow) = readahead(s);
		match (r) {
		case let n: i32 => { };
		case io.eof => { let e: io.eof; return e; };
		case let e: io.error => return e;
		case errors.overflow => { let e: errors.overflow; return e; };
		};
	};
	let e: io.eof; return e;
};

// scanrune — pop one UTF-8-encoded rune, refilling from src on demand.
// EOF mid-codepoint (fewer pending bytes than the initial byte
// announces) is utf8.invalid; a clean EOF before any byte is io.eof.
// Partial reads are refilled until the announced width is present.
// Decode happens before consumption: malformed input consumes only the
// bytes visited by utf8.next, preserving any later pending byte.
export fn scanrune(s: *scanner) (rune | io.eof | io.error | utf8.invalid | errors.overflow) = {
	for (s.avail == 0) {
		let ra: (i32 | io.eof | io.error | errors.overflow) = readahead(s);
		match (ra) {
		case let n: i32 => { };
		case io.eof => { let e: io.eof; return e; };
		case let e: io.error => return e;
		case errors.overflow => { let e: errors.overflow; return e; };
		};
	};
	let szr: (i32 | utf8.invalid) = utf8.utf8sz(s.ptr[s.start]);
	let sz: i32 = 0;
	match (szr) {
	case let n: i32 => { sz = n; };
	case utf8.invalid => {
		s.start += 1;
		s.avail -= 1;
		let e: utf8.invalid;
		return e;
	};
	};
	for (s.avail < sz) {
		let ra: (i32 | io.eof | io.error | errors.overflow) = readahead(s);
		match (ra) {
		case let n: i32 => { };
		case io.eof => {
			s.start += s.avail;
			s.avail = 0;
			let e: utf8.invalid;
			return e;
		};
		case let e: io.error => return e;
		case errors.overflow => { let e: errors.overflow; return e; };
		};
	};
	let v: []u8;
	v.ptr = s.ptr + (s.start: u64);
	v.len = sz;
	v.cap = v.len;
	let dec: utf8.decoder = utf8.decode(v);
	let nr: (rune | utf8.done | utf8.more | utf8.invalid) = utf8.next(&dec);
	let used: i32 = utf8.position(&dec): i32;
	match (nr) {
	case let r: rune => {
		assert(used == sz, "bufio.scanrune: decoder width mismatch");
		s.start += used;
		s.avail -= used;
		return r;
	};
	case utf8.done => abort("bufio.scanrune: decoder made no progress");
	case utf8.more => {
		s.start += used;
		s.avail -= used;
		let e: utf8.invalid;
		return e;
	};
	case utf8.invalid => {
		assert(used > 0 && used <= sz,
			"bufio.scanrune: invalid decoder position");
		s.start += used;
		s.avail -= used;
		let e: utf8.invalid;
		return e;
	};
	};
};

// scanline — read up to (and not including) the next '\n'. The newline
// is consumed; the returned str view borrows from the scanner buffer.
// Single-byte route (Hare's scan_line = scan_string(s, "\n"); the
// arbitrary multibyte-delim scan_string is #217). Mirrors
// ref/hare/bufio/scanner.ha:307.
export fn scanline(s: *scanner) (str | io.eof | io.error | errors.overflow) = {
	let r: ([]u8 | io.eof | io.error | errors.overflow) = scanbytes(s, 10u8);
	match (r) {
	case let bs: []u8 => {
		let v: str;
		v.ptr = bs.ptr;
		v.len = bs.len;
		v.cap = v.len;
		return v;
	};
	case io.eof => { let e: io.eof; return e; };
	case let e: io.error => return e;
	case errors.overflow => { let e: errors.overflow; return e; };
	};
};