lib/bufio: newscanner + scanrune (regex fold-2b prereq)
Port Hare's auto-grow newscanner (scanner.ha:72) and scan_rune (scanner.ha:259). scanner gains a maxread field (== cap for newscannerbuf, scanner.ha:101); readahead grows by BUFSZ up to maxread via alloc+copy (Hare appends; ww flat ptr/cap scanner, old block left to process-exit reclaim). scanbytes' overflow test gains the avail >= maxread leg (Hare's pending >= readahead predicate) so a growable scanner refills instead of overflowing. finish ports the free(scan.buffer) verbatim per the regex #27 precedent. Tests: rune scan over 1/2/3/4-byte UTF-8 + EOF, invalid initial/truncated/surrogate sequences, newscanner grow round-trip + scanrune-over-newscanner, maxread overflow.
This commit is contained in:
@@ -10,6 +10,7 @@ package bufio;
|
||||
|
||||
import bufio;
|
||||
import bytes;
|
||||
import encoding.utf8;
|
||||
import io;
|
||||
import memio;
|
||||
|
||||
@@ -687,6 +688,192 @@ fn errsource() io.stream = {
|
||||
if (raw[4] != 100u8) { fail(); }; // 'd'
|
||||
};
|
||||
|
||||
// ---- scanrune: ASCII + 2/3/4-byte UTF-8 + idempotent EOF --------------
|
||||
|
||||
@test fn scanrunecases() void = {
|
||||
// "a" U+0061; "é" U+00E9 (C3 A9); "한" U+D55C (ED 95 9C);
|
||||
// "😀" U+1F600 (F0 9F 98 80).
|
||||
let raw: [10]u8;
|
||||
raw[0] = 97u8;
|
||||
raw[1] = 0xC3u8; raw[2] = 0xA9u8;
|
||||
raw[3] = 0xEDu8; raw[4] = 0x95u8; raw[5] = 0x9Cu8;
|
||||
raw[6] = 0xF0u8; raw[7] = 0x9Fu8; raw[8] = 0x98u8; raw[9] = 0x80u8;
|
||||
|
||||
let mem: memio.stream = memio.fixed(raw[0:10]);
|
||||
let m: io.stream = &mem.vt;
|
||||
// 8B window: the trailing 4-byte rune straddles a refill (shift path).
|
||||
let buf: [8]u8;
|
||||
let sc: bufio.scanner = bufio.newscannerbuf(m, buf[0:8]);
|
||||
|
||||
// (wantrune, weof)
|
||||
let want: [6]u32;
|
||||
let weof: [6]i32;
|
||||
want[0] = 97u32; weof[0] = 0;
|
||||
want[1] = 233u32; weof[1] = 0;
|
||||
want[2] = 54620u32; weof[2] = 0;
|
||||
want[3] = 128512u32; weof[3] = 0;
|
||||
want[4] = 0u32; weof[4] = 1; // past end → eof
|
||||
want[5] = 0u32; weof[5] = 1; // stay at eof
|
||||
|
||||
let i: i32 = 0;
|
||||
for (i < 6) {
|
||||
let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc);
|
||||
match (r) {
|
||||
case let rn: rune => {
|
||||
if (weof[i] != 0) { fail(); };
|
||||
if (rn: u32 != want[i]) { fail(); };
|
||||
};
|
||||
case io.eof => { if (weof[i] == 0) { fail(); }; };
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => fail();
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
|
||||
bufio.finish(&sc);
|
||||
};
|
||||
|
||||
// ---- scanrune: invalid sequences --------------------------------------
|
||||
|
||||
@test fn scanruneinvalid() void = {
|
||||
// Bare continuation byte: utf8sz rejects the initial byte.
|
||||
let r1raw: [1]u8;
|
||||
r1raw[0] = 0x80u8;
|
||||
let mem1: memio.stream = memio.fixed(r1raw[0:1]);
|
||||
let m1: io.stream = &mem1.vt;
|
||||
let b1: [8]u8;
|
||||
let sc1: bufio.scanner = bufio.newscannerbuf(m1, b1[0:8]);
|
||||
let r1: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc1);
|
||||
match (r1) {
|
||||
case let rn: rune => fail();
|
||||
case io.eof => fail();
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => { };
|
||||
};
|
||||
bufio.finish(&sc1);
|
||||
|
||||
// Truncated 2-byte sequence at EOF: pending < utf8sz → invalid
|
||||
// (Hare scanner.ha:272-274).
|
||||
let r2raw: [1]u8;
|
||||
r2raw[0] = 0xC3u8;
|
||||
let mem2: memio.stream = memio.fixed(r2raw[0:1]);
|
||||
let m2: io.stream = &mem2.vt;
|
||||
let b2: [8]u8;
|
||||
let sc2: bufio.scanner = bufio.newscannerbuf(m2, b2[0:8]);
|
||||
let r2: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2);
|
||||
match (r2) {
|
||||
case let rn: rune => fail();
|
||||
case io.eof => fail();
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => { };
|
||||
};
|
||||
bufio.finish(&sc2);
|
||||
|
||||
// UTF-16 surrogate encoding (ED A0 80 = U+D800): utf8sz accepts the
|
||||
// initial byte, the decode DFA rejects the continuation.
|
||||
let r3raw: [3]u8;
|
||||
r3raw[0] = 0xEDu8; r3raw[1] = 0xA0u8; r3raw[2] = 0x80u8;
|
||||
let mem3: memio.stream = memio.fixed(r3raw[0:3]);
|
||||
let m3: io.stream = &mem3.vt;
|
||||
let b3: [8]u8;
|
||||
let sc3: bufio.scanner = bufio.newscannerbuf(m3, b3[0:8]);
|
||||
let r3: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc3);
|
||||
match (r3) {
|
||||
case let rn: rune => fail();
|
||||
case io.eof => fail();
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => { };
|
||||
};
|
||||
bufio.finish(&sc3);
|
||||
};
|
||||
|
||||
// ---- newscanner: auto-grow from the empty ctor state ------------------
|
||||
|
||||
@test fn newscannergrow() void = {
|
||||
let src: [16]u8;
|
||||
let n: i32 = putstr("hello\nworld\n", src[0:16], 0);
|
||||
|
||||
let mem: memio.stream = memio.fixed(src[0:n]);
|
||||
let m: io.stream = &mem.vt;
|
||||
// cap starts at 0; the first readahead grows to min(BUFSZ, maxread).
|
||||
let sc: bufio.scanner = bufio.newscanner(m, 1024);
|
||||
|
||||
// (wantlen, wantfirst, weof)
|
||||
let wl: [3]i32;
|
||||
let wf: [3]u8;
|
||||
let weof: [3]i32;
|
||||
wl[0]=5; wf[0]=104u8; weof[0]=0; // hello
|
||||
wl[1]=5; wf[1]=119u8; weof[1]=0; // world
|
||||
wl[2]=0; wf[2]=0u8; weof[2]=1; // eof
|
||||
|
||||
let i: i32 = 0;
|
||||
for (i < 3) {
|
||||
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
|
||||
match (r) {
|
||||
case let v: str => {
|
||||
if (weof[i] != 0) { fail(); };
|
||||
if (v.len != wl[i]) { fail(); };
|
||||
if (v.len > 0 && v[0] != wf[i]) { fail(); };
|
||||
};
|
||||
case io.eof => { if (weof[i] == 0) { fail(); }; };
|
||||
case let e: io.error => fail();
|
||||
case bufio.overflow => fail();
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
bufio.finish(&sc);
|
||||
|
||||
// scanrune over a fresh newscanner (the regex search() shape).
|
||||
let r2raw: [3]u8;
|
||||
r2raw[0] = 0xC3u8; r2raw[1] = 0xA9u8; // é
|
||||
r2raw[2] = 122u8; // 'z'
|
||||
let mem2: memio.stream = memio.fixed(r2raw[0:3]);
|
||||
let m2: io.stream = &mem2.vt;
|
||||
let sc2: bufio.scanner = bufio.newscanner(m2, 1024);
|
||||
|
||||
let want: [3]u32;
|
||||
let weof2: [3]i32;
|
||||
want[0]=233u32; weof2[0]=0;
|
||||
want[1]=122u32; weof2[1]=0;
|
||||
want[2]=0u32; weof2[2]=1;
|
||||
|
||||
i = 0;
|
||||
for (i < 3) {
|
||||
let r: (rune | io.eof | io.error | utf8.invalid) = bufio.scanrune(&sc2);
|
||||
match (r) {
|
||||
case let rn: rune => {
|
||||
if (weof2[i] != 0) { fail(); };
|
||||
if (rn: u32 != want[i]) { fail(); };
|
||||
};
|
||||
case io.eof => { if (weof2[i] == 0) { fail(); }; };
|
||||
case let e: io.error => fail();
|
||||
case utf8.invalid => fail();
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
bufio.finish(&sc2);
|
||||
};
|
||||
|
||||
// ---- newscanner: maxread reached without a delimiter → overflow -------
|
||||
|
||||
@test fn newscanneroverflow() void = {
|
||||
let src: [16]u8;
|
||||
let n: i32 = putstr("ABCDEFGH\n", src[0:16], 0);
|
||||
|
||||
let mem: memio.stream = memio.fixed(src[0:n]);
|
||||
let m: io.stream = &mem.vt;
|
||||
let sc: bufio.scanner = bufio.newscanner(m, 4); // token (8B) > maxread
|
||||
|
||||
let r: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc);
|
||||
match (r) {
|
||||
case let v: str => fail();
|
||||
case io.eof => fail();
|
||||
case let e: io.error => fail();
|
||||
case bufio.overflow => { };
|
||||
};
|
||||
bufio.finish(&sc);
|
||||
};
|
||||
|
||||
export fn main() i32 = {
|
||||
signalled = 1; scanbytecases();
|
||||
signalled = 2; scanlinecases();
|
||||
@@ -706,5 +893,9 @@ export fn main() i32 = {
|
||||
signalled = 16; streamflushempty();
|
||||
signalled = 17; streamcloseflushes();
|
||||
signalled = 18; streamsetflushcustom();
|
||||
signalled = 19; scanrunecases();
|
||||
signalled = 20; scanruneinvalid();
|
||||
signalled = 21; newscannergrow();
|
||||
signalled = 22; newscanneroverflow();
|
||||
return 0;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user