From 291860ddefaf0e4a74f7b58df3f3f012d1dc5f1a Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Sat, 8 Aug 2026 17:32:16 +0900 Subject: [PATCH] lib: split bufio tests into stream+scanner per ref/hare/bufio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure move: bufio_test.ww dissolves along its banner seams into stream_test.ww (13 stream* rows) and scanner_test.ww (13 scan*/ newscanner*/emptystream/errorsource/boundarycases/multifill/ readaheadcantgrow rows), mirroring ref/hare/bufio/{stream_test+test.ha, scanner_test+test.ha} ownership. streamscannerunread stays stream-side: Hare's stream-level unread test lives in stream_test+test.ha:86. Moved blocks are byte-identical; the 28 file-scope banner lines (and their // continuation lines) are deleted, the 2 interior banners inside boundarycases stay. Imports minimized per file (the unused `import bytes;` drops; utf8 is scanner-only). Helper inventory: putstr spans both halves (7 scanner rows / 8 call sites; 8 stream rows / 8 call sites), so it is duplicated under per-file names per the sanctioned strconv streq/fstreq precedent — scanner_test.ww keeps `putstr`, the stream copy is `sputstr` (decl, head comment, 8 call sites; body otherwise byte-identical). The error-returning source (errvt/errread/errwrite/errsource) moves with its sole consumer errorsource into scanner_test.ww; the close-recording source (srcclosed/closevt/closesink/closewrite/closeclose/closesource) moves with streamclosenopropagate into stream_test.ww. Line-multiset diff old-vs-concat (heads/banners excluded, sputstr normalized): the only residue is the 10-line putstr helper counted twice. Consumers: Makefile LIBRARY_TESTS replaces the bufio_test.ww entry with the two new entries in place; test/byteid/libbyteid_test.ww roster row becomes two fx rows, NENTEXPECT 58->59. Validation: standalone ww test lib/bufio/scanner_test.ww (13 passed) and lib/bufio/stream_test.ww (13 passed); dir mode ww test lib/bufio (26 passed, no duplicate symbols); byteid compile-only build exit 0, 59 roster appends == NENTEXPECT. --- Makefile | 3 +- lib/bufio/{bufio_test.ww => scanner_test.ww} | 512 +------------------ lib/bufio/stream_test.ww | 464 +++++++++++++++++ test/byteid/libbyteid_test.ww | 7 +- 4 files changed, 476 insertions(+), 510 deletions(-) rename lib/bufio/{bufio_test.ww => scanner_test.ww} (51%) create mode 100644 lib/bufio/stream_test.ww diff --git a/Makefile b/Makefile index ab0feb3d..a9015cbe 100644 --- a/Makefile +++ b/Makefile @@ -493,7 +493,8 @@ LIBRARY_TESTS = lib/errors/errno_test.ww lib/ascii/ascii_test.ww \ lib/bytes/trim_test.ww lib/encoding/utf8/rune_test.ww \ lib/encoding/utf8/encode_test.ww lib/encoding/utf8/decode_test.ww \ lib/encoding/utf8/types_test.ww \ - lib/bufio/bufio_test.ww lib/math/random/random_test.ww \ + lib/bufio/stream_test.ww lib/bufio/scanner_test.ww \ + lib/math/random/random_test.ww \ lib/math/checked/checked_test.ww lib/fmt/fmt_test.ww \ lib/log/log_test.ww lib/log/silent_test.ww lib/fnmatch/fnmatch_test.ww \ lib/shlex/shlex_test.ww lib/time/arithm_test.ww \ diff --git a/lib/bufio/bufio_test.ww b/lib/bufio/scanner_test.ww similarity index 51% rename from lib/bufio/bufio_test.ww rename to lib/bufio/scanner_test.ww index 16ad6ed1..8c4ee388 100644 --- a/lib/bufio/bufio_test.ww +++ b/lib/bufio/scanner_test.ww @@ -1,15 +1,13 @@ -// bufiotest — exercises lib/bufio. Run with `out/bin/ww run lib/bufio/bufiotest.ww`. -// -// The scanner @tests enumerate parallel `[N]T` arrays of inputs and -// expectations, then iterate one body across them. The stream @tests -// are one scenario per fn. #94 fold-eFinal: the unified value-return -// surface — `let st = memio.fixed(buf); &st.vt` into the scanner/stream -// init, io.read/write/close return size/io.error. +// scannertest — exercises the lib/bufio scanner surface. The @tests +// enumerate parallel `[N]T` arrays of inputs and expectations, then +// iterate one body across them. #94 fold-eFinal: the unified +// value-return surface — `let st = memio.fixed(buf); &st.vt` into the +// scanner init, io.read/write/close return size/io.error. Row +// ownership mirrors ref/hare/bufio/scanner_test+test.ha. package bufio_test; import bufio; -import bytes; import encoding.utf8; import io; import memio; @@ -25,8 +23,6 @@ fn putstr(s: str, into: []u8, off: i32) i32 = { return off + s.len; }; -// ---- an error-returning stream for the io.error surfacing test -------- -// // Replaces the OLD io.closed source. errvt is module-static; its // reader/writer return a nomem-widened io.error. let errvt: io.vtable; @@ -43,29 +39,6 @@ fn errsource() io.stream = { return &errvt; }; -// ---- a close-recording source for the no-propagate test --------------- -// -// closevt forwards writes to closesink (so a flush is observable) and -// records whether its closer fired in srcclosed. -let srcclosed: bool = false; -let closevt: io.vtable; -let closesink: memio.stream; - -fn closewrite(s: io.stream, buf: []u8) (size | io.error) = { - return io.write(&closesink.vt, buf); -}; -fn closeclose(s: io.stream) (void | io.error) = { - srcclosed = true; - return void; -}; -fn closesource() io.stream = { - closevt.writer = (&closewrite): *io.writer; - closevt.closer = (&closeclose): *io.closer; - return &closevt; -}; - -// ---- scanbyte: drain four bytes through a 2-byte window --------------- - @test fn scanbytecases() void = { let raw: [4]u8; raw[0] = 11u8; raw[1] = 22u8; raw[2] = 33u8; raw[3] = 44u8; @@ -103,8 +76,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- scanline: 3 lines + tail fragment + extra calls stay at EOF ------ - @test fn scanlinecases() void = { let src: [32]u8; let n: i32 = putstr("foo\nbar\nbaz\ntrailing", src[0:32], 0); @@ -145,8 +116,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- scanline: line longer than buffer → overflow -------------------- - @test fn scanlineoverflow() void = { let src: [32]u8; let n: i32 = putstr("ABCDEFGHI\n", src[0:32], 0); @@ -169,8 +138,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- scanbytes: multi-delim hit, delim-at-start, EOF before delim ---- - @test fn scanbytescases() void = { let src: [16]u8; // ",a,,b,c" — leading delim, empty token, multi-hit, trailing fragment. @@ -210,8 +177,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- empty stream: every variant returns eof immediately -------------- - @test fn emptystream() void = { let raw: [1]u8; let mem: memio.stream = memio.fixed(raw[0:0]); @@ -246,8 +211,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- io.error surfacing on a stream that returns error ---------------- - @test fn errorsource() void = { let m: io.stream = errsource(); let buf: [8]u8; @@ -280,8 +243,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- boundary: empty line + idempotent EOF on scanbytes -------------- - @test fn boundarycases() void = { // ---- empty line via scanline ----------------------------------- let s1: [16]u8; @@ -335,8 +296,6 @@ fn closesource() io.stream = { bufio.finish(&sc2); }; -// ---- multi-fill: window smaller than longest token, shift path -------- - @test fn multifill() void = { let src: [32]u8; let n: i32 = putstr("abcd\nxy\n", src[0:32], 0); @@ -373,457 +332,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- stream init + flush round-trip --------------------------------- - -@test fn streamsmallwrite() void = { - let raw: [16]u8; - let mem: memio.stream = memio.fixed(raw[0:16]); - let m: io.stream = &mem.vt; - - let rb: [8]u8; - let wb: [8]u8; - let b: bufio.stream = bufio.init(m, rb[0:8], wb[0:8]); - - // Write under the buffer limit — nothing reaches src. - let buf: [5]u8; - let z: i32 = putstr("hello", buf[0:5], 0); - let p: io.stream = &b.vt; - let r: (size | io.error) = io.write(p, buf[0:5]); - match (r) { - case let n: size => { assert(!(n: i32 != 5)); }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 0)); - - let fr: (void | io.error) = bufio.flush(&b); - match (fr) { - case void => { }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 5)); - assert(!(raw[0] != 104u8)); - assert(!(raw[4] != 111u8)); -}; - -// ---- write larger than wbuf → auto-flush ----------------------------- - -@test fn streamautoflush() void = { - let raw: [32]u8; - let mem: memio.stream = memio.fixed(raw[0:32]); - let m: io.stream = &mem.vt; - - let rb: [4]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:4]); - - // 10B payload through a 4B wbuf: bwrite must flush twice mid-write. - let buf: [10]u8; - let z: i32 = putstr("abcdefghij", buf[0:10], 0); - let p: io.stream = &b.vt; - let r: (size | io.error) = io.write(p, buf[0:10]); - match (r) { - case let n: size => { assert(!(n: i32 != 10)); }; - case let e: io.error => abort(); - }; - // Two full flushes happened (8B); 2B still pending. - assert(!(mem.pos != 8)); - - let fr: (void | io.error) = bufio.flush(&b); - match (fr) { - case void => { }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 10)); - assert(!(raw[0] != 97u8)); // 'a' - assert(!(raw[9] != 106u8)); // 'j' -}; - -// ---- default flush byte-set: '\n' in payload triggers flush --------- - -@test fn streamlineflush() void = { - let raw: [32]u8; - let mem: memio.stream = memio.fixed(raw[0:32]); - let m: io.stream = &mem.vt; - - let rb: [4]u8; - let wb: [16]u8; - let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:16]); - - let p: io.stream = &b.vt; - - // First write: no '\n', stays buffered. - let h: [5]u8; - let zh: i32 = putstr("hello", h[0:5], 0); - let r1: (size | io.error) = io.write(p, h[0:5]); - match (r1) { - case let n: size => { assert(!(n: i32 != 5)); }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 0)); - - // Second write: '\n' present, triggers flush of everything (6B). - let nl: [1]u8; - nl[0] = 10u8; - let r2: (size | io.error) = io.write(p, nl[0:1]); - match (r2) { - case let n: size => { assert(!(n: i32 != 1)); }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 6)); - assert(!(raw[0] != 104u8)); - assert(!(raw[5] != 10u8)); -}; - -// ---- explicit-flush-per-write: caller drives the drain --------------- - -@test fn streamflushonwrite() void = { - let raw: [32]u8; - let mem: memio.stream = memio.fixed(raw[0:32]); - let m: io.stream = &mem.vt; - - let rb: [4]u8; - let wb: [16]u8; - let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:16]); - let nilbs: [1]u8; - bufio.setflush(&b, nilbs[0:0]); - - let p: io.stream = &b.vt; - let h: [2]u8; - h[0] = 65u8; h[1] = 66u8; // "AB" - let r1: (size | io.error) = io.write(p, h[0:2]); - match (r1) { - case let n: size => { assert(!(n: i32 != 2)); }; - case let e: io.error => abort(); - }; - let f1: (void | io.error) = bufio.flush(&b); - match (f1) { - case void => { }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 2)); - assert(!(raw[0] != 65u8)); - - let r2: (size | io.error) = io.write(p, h[0:1]); - match (r2) { - case let n: size => { assert(!(n: i32 != 1)); }; - case let e: io.error => abort(); - }; - let f2: (void | io.error) = bufio.flush(&b); - match (f2) { - case void => { }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 3)); - assert(!(raw[2] != 65u8)); -}; - -// ---- isbuffered: stream → true, plain memio → false ----------------- - -@test fn streamisbuffered() void = { - let raw: [8]u8; - let mem: memio.stream = memio.fixed(raw[0:8]); - let m: io.stream = &mem.vt; - - let rb: [4]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:4]); - - assert(!(!bufio.isbuffered(&b.vt))); - assert(!(bufio.isbuffered(m))); -}; - -// ---- bread tops up a partial pending buffer (Hare short-read) -------- -// -// rbuf holds SOME pending bytes (fewer than requested) AND has spare -// capacity: Hare shifts the pending region to the front and reads more -// into the tail before serving (ref/hare/bufio/stream.ha:209). Pre-fix -// ww served only the short pending count and never topped up. -@test fn streamreadtopup() void = { - let raw: [16]u8; - let n: i32 = putstr("ABCDEFGHIJKLMNOP", raw[0:16], 0); - - let mem: memio.stream = memio.fixed(raw[0:n]); - let m: io.stream = &mem.vt; - - let rb: [8]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(m, rb[0:8], wb[0:4]); - - let p: io.stream = &b.vt; - - // First read fills rbuf (8B "ABCDEFGH"), serves 3 — 5 pending remain. - let out: [10]u8; - let r1: (size | io.eof | io.error) = io.read(p, out[0:3]); - match (r1) { - case let z: size => { assert(!(z: i32 != 3)); }; - case io.eof => abort(); - case let e: io.error => abort(); - }; - - // Second read wants 10 > 5 pending: Hare shifts "DEFGH" to the front, - // reads 3 more ("IJK") into the tail, serves 8. Pre-fix ww returned 5. - let r2: (size | io.eof | io.error) = io.read(p, out[0:10]); - match (r2) { - case let z: size => { assert(!(z: i32 != 8)); }; - case io.eof => abort(); - case let e: io.error => abort(); - }; - assert(!(out[0] != 68u8)); // 'D' - assert(!(out[7] != 75u8)); // 'K' -}; - -// ---- bread serves pending bytes when the top-up read hits EOF -------- -// -// The load-bearing "EOF fatal only when avail==0" path -// (ref/hare/bufio/stream.ha:216): rbuf still holds pending bytes when -// the top-up io.read returns EOF — Hare drains the pending region, it -// does NOT propagate eof. Source is shorter than two reads so the -// second read's top-up io.read EOFs with bytes still buffered. -@test fn streamreadtopupeof() void = { - let raw: [5]u8; - let n: i32 = putstr("ABCDE", raw[0:5], 0); - - let mem: memio.stream = memio.fixed(raw[0:n]); - let m: io.stream = &mem.vt; - - let rb: [8]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(m, rb[0:8], wb[0:4]); - - let p: io.stream = &b.vt; - - // First read fills rbuf (5B "ABCDE", source drained), serves 3 — - // 2 pending remain. - let out: [10]u8; - let r1: (size | io.eof | io.error) = io.read(p, out[0:3]); - match (r1) { - case let z: size => { assert(!(z: i32 != 3)); }; - case io.eof => abort(); - case let e: io.error => abort(); - }; - - // Second read wants 10 > 2 pending: shift "DE" to the front, top-up - // io.read hits EOF (source drained). avail==2 != 0 so Hare drains the - // pending region — serves 2, NOT eof. "always fatal" would return eof. - let r2: (size | io.eof | io.error) = io.read(p, out[0:10]); - match (r2) { - case let z: size => { assert(!(z: i32 != 2)); }; - case io.eof => abort(); - case let e: io.error => abort(); - }; - assert(!(out[0] != 68u8)); // 'D' - assert(!(out[1] != 69u8)); // 'E' -}; - -// ---- bread + unread: pushed-back bytes come out first ---------------- - -@test fn streamunread() void = { - let raw: [8]u8; - let n: i32 = putstr("ABCDEFGH", raw[0:8], 0); - - let mem: memio.stream = memio.fixed(raw[0:n]); - let m: io.stream = &mem.vt; - - let rb: [8]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(m, rb[0:8], wb[0:4]); - - let p: io.stream = &b.vt; - - // Pull 3 bytes through bread — fills rbuf from src (8B), serves 3. - let out: [4]u8; - let r1: (size | io.eof | io.error) = io.read(p, out[0:3]); - match (r1) { - case let z: size => { assert(!(z: i32 != 3)); }; - case io.eof => abort(); - case let e: io.error => abort(); - }; - assert(!(out[0] != 65u8)); // 'A' - assert(!(out[2] != 67u8)); // 'C' - - // Push back 2 bytes; next read returns them first. - let push: [2]u8; - push[0] = 88u8; push[1] = 89u8; // "XY" - bufio.unread(&b, push[0:2]); - - let r2: (size | io.eof | io.error) = io.read(p, out[0:2]); - match (r2) { - case let z: size => { assert(!(z: i32 != 2)); }; - case io.eof => abort(); - case let e: io.error => abort(); - }; - assert(!(out[0] != 88u8)); // 'X' - assert(!(out[1] != 89u8)); // 'Y' - - // Subsequent read returns the original tail. - let r3: (size | io.eof | io.error) = io.read(p, out[0:4]); - match (r3) { - case let z: size => { assert(!(z: i32 != 4)); }; - case io.eof => abort(); - case let e: io.error => abort(); - }; - assert(!(out[0] != 68u8)); // 'D' - assert(!(out[3] != 71u8)); // 'G' -}; - -// ---- scanner over a stream-wrapped src: pre-read unread + scanline -- - -@test fn streamscannerunread() void = { - let raw: [16]u8; - let n: i32 = putstr("hello\n", raw[0:16], 0); - - let mem: memio.stream = memio.fixed(raw[0:n]); - let m: io.stream = &mem.vt; - - let rb: [16]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(m, rb[0:16], wb[0:4]); - - let push: [3]u8; - push[0] = 88u8; push[1] = 89u8; push[2] = 90u8; // "XYZ" - bufio.unread(&b, push[0:3]); - - let p: io.stream = &b.vt; - let sbuf: [32]u8; - let sc: bufio.scanner = bufio.newscannerbuf(p, sbuf[0:32]); - - let lr: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); - match (lr) { - case let v: str => { - assert(!(v.len != 8)); // "XYZhello" - assert(!(v[0] != 88u8)); // 'X' - assert(!(v[7] != 111u8)); // 'o' - }; - case io.eof => abort(); - case let e: io.error => abort(); - case bufio.overflow => abort(); - }; - bufio.finish(&sc); -}; - -// ---- flush() on empty wbuf is a no-op -------------------------------- - -@test fn streamflushempty() void = { - let raw: [8]u8; - let mem: memio.stream = memio.fixed(raw[0:8]); - let m: io.stream = &mem.vt; - - let rb: [4]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:4]); - - let r: (void | io.error) = bufio.flush(&b); - match (r) { - case void => { }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 0)); -}; - -// ---- bclose drains pending wbuf then forwards close to src ---------- - -@test fn streamcloseflushes() void = { - let raw: [8]u8; - let mem: memio.stream = memio.fixed(raw[0:8]); - let m: io.stream = &mem.vt; - - let rb: [4]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:4]); - let nilbs: [1]u8; - bufio.setflush(&b, nilbs[0:0]); // suppress the default '\n' detector - - let p: io.stream = &b.vt; - let h: [2]u8; - h[0] = 80u8; h[1] = 81u8; // "PQ" - let r: (size | io.error) = io.write(p, h[0:2]); - match (r) { - case let n: size => { assert(!(n: i32 != 2)); }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 0)); // still buffered - - let cr: (void | io.error) = io.close(p); - match (cr) { - case void => { }; - case let e: io.error => abort(); - }; - assert(!(mem.pos != 2)); - assert(!(raw[0] != 80u8)); - assert(!(raw[1] != 81u8)); -}; - -// ---- bclose flushes but does NOT close the underlying ---------------- -// -// Hare's close_buffered closes src only under flag::MANAGED_HANDLE -// (ref/hare/bufio/stream.ha:194); init's default flag::NONE flushes -// only. The close-recording closesource lets us assert both: the -// buffered byte reaches the sink (flush) AND the closer never fires. -@test fn streamclosenopropagate() void = { - srcclosed = false; - let raw: [8]u8; - closesink = memio.fixed(raw[0:8]); - let src: io.stream = closesource(); - - let rb: [4]u8; - let wb: [4]u8; - let b: bufio.stream = bufio.init(src, rb[0:4], wb[0:4]); - let nilbs: [1]u8; - bufio.setflush(&b, nilbs[0:0]); // suppress the default '\n' detector - - let p: io.stream = &b.vt; - let h: [2]u8; - h[0] = 80u8; h[1] = 81u8; // "PQ" - let r: (size | io.error) = io.write(p, h[0:2]); - match (r) { - case let n: size => { assert(!(n: i32 != 2)); }; - case let e: io.error => abort(); - }; - assert(!(closesink.pos != 0)); // still buffered - - let cr: (void | io.error) = io.close(p); - match (cr) { - case void => { }; - case let e: io.error => abort(); - }; - // Flush happened: the 2 buffered bytes reached the sink. - assert(!(closesink.pos != 2)); - assert(!(raw[0] != 80u8)); - assert(!(raw[1] != 81u8)); - // But the underlying was NOT closed (Hare flag::NONE default). - assert(!srcclosed); -}; - -// ---- setflush with a custom non-empty byte-set ----------------------- - -@test fn streamsetflushcustom() void = { - let raw: [16]u8; - let mem: memio.stream = memio.fixed(raw[0:16]); - let m: io.stream = &mem.vt; - - let rb: [4]u8; - let wb: [16]u8; - let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:16]); - let bs: [1]u8; - bs[0] = 32u8; // ' ' - bufio.setflush(&b, bs[0:1]); - - let p: io.stream = &b.vt; - let buf: [5]u8; - let z: i32 = putstr("ab cd", buf[0:5], 0); - let r: (size | io.error) = io.write(p, buf[0:5]); - match (r) { - case let n: size => { assert(!(n: i32 != 5)); }; - case let e: io.error => abort(); - }; - // Space at index 2 triggers post-copy flush of all 5 bytes. - assert(!(mem.pos != 5)); - assert(!(raw[2] != 32u8)); - assert(!(raw[4] != 100u8)); // 'd' -}; - -// ---- scanrune: ASCII + 2/3/4-byte UTF-8 + idempotent EOF -------------- - @test fn scanrunecases() void = { // "a" U+0061; "é" U+00E9 (C3 A9); "한" U+D55C (ED 95 9C); // "😀" U+1F600 (F0 9F 98 80). @@ -868,8 +376,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- scanrune: invalid sequences -------------------------------------- - @test fn scanruneinvalid() void = { // Bare continuation byte: utf8sz rejects the initial byte. let r1raw: [1]u8; @@ -925,8 +431,6 @@ fn closesource() io.stream = { bufio.finish(&sc3); }; -// ---- newscanner: auto-grow from the empty ctor state ------------------ - @test fn newscannergrow() void = { let src: [16]u8; let n: i32 = putstr("hello\nworld\n", src[0:16], 0); @@ -993,8 +497,6 @@ fn closesource() io.stream = { bufio.finish(&sc2); }; -// ---- newscanner: maxread reached without a delimiter → overflow ------- - @test fn newscanneroverflow() void = { let src: [16]u8; let n: i32 = putstr("ABCDEFGH\n", src[0:16], 0); @@ -1013,8 +515,6 @@ fn closesource() io.stream = { bufio.finish(&sc); }; -// ---- readahead can't-grow → overflow, not spin / nil-deref ----------- -// // drain F-B: a scanner that cannot buffer the next byte (zero-cap) makes // readahead unable to make progress. Pre-fix it fell through silently to // a zero-length io.read returning 0, so scanbyte spun forever (catB-144) diff --git a/lib/bufio/stream_test.ww b/lib/bufio/stream_test.ww new file mode 100644 index 00000000..779730d8 --- /dev/null +++ b/lib/bufio/stream_test.ww @@ -0,0 +1,464 @@ +// streamtest — exercises the lib/bufio buffered stream surface. The +// stream @tests are one scenario per fn. #94 fold-eFinal: the unified +// value-return surface — `let st = memio.fixed(buf); &st.vt` into the +// stream init, io.read/write/close return size/io.error. Row +// ownership mirrors ref/hare/bufio/stream_test+test.ha. + +package bufio_test; + +import bufio; +import io; +import memio; + +// sputstr — copy the bytes of `s` into `into` starting at `off`, +// returning the new offset. +fn sputstr(s: str, into: []u8, off: i32) i32 = { + let i: i32 = 0; + for (i < s.len) { + into[off + i] = s[i]; + i += 1; + }; + return off + s.len; +}; + +// closevt forwards writes to closesink (so a flush is observable) and +// records whether its closer fired in srcclosed. +let srcclosed: bool = false; +let closevt: io.vtable; +let closesink: memio.stream; + +fn closewrite(s: io.stream, buf: []u8) (size | io.error) = { + return io.write(&closesink.vt, buf); +}; +fn closeclose(s: io.stream) (void | io.error) = { + srcclosed = true; + return void; +}; +fn closesource() io.stream = { + closevt.writer = (&closewrite): *io.writer; + closevt.closer = (&closeclose): *io.closer; + return &closevt; +}; + +@test fn streamsmallwrite() void = { + let raw: [16]u8; + let mem: memio.stream = memio.fixed(raw[0:16]); + let m: io.stream = &mem.vt; + + let rb: [8]u8; + let wb: [8]u8; + let b: bufio.stream = bufio.init(m, rb[0:8], wb[0:8]); + + // Write under the buffer limit — nothing reaches src. + let buf: [5]u8; + let z: i32 = sputstr("hello", buf[0:5], 0); + let p: io.stream = &b.vt; + let r: (size | io.error) = io.write(p, buf[0:5]); + match (r) { + case let n: size => { assert(!(n: i32 != 5)); }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 0)); + + let fr: (void | io.error) = bufio.flush(&b); + match (fr) { + case void => { }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 5)); + assert(!(raw[0] != 104u8)); + assert(!(raw[4] != 111u8)); +}; + +@test fn streamautoflush() void = { + let raw: [32]u8; + let mem: memio.stream = memio.fixed(raw[0:32]); + let m: io.stream = &mem.vt; + + let rb: [4]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:4]); + + // 10B payload through a 4B wbuf: bwrite must flush twice mid-write. + let buf: [10]u8; + let z: i32 = sputstr("abcdefghij", buf[0:10], 0); + let p: io.stream = &b.vt; + let r: (size | io.error) = io.write(p, buf[0:10]); + match (r) { + case let n: size => { assert(!(n: i32 != 10)); }; + case let e: io.error => abort(); + }; + // Two full flushes happened (8B); 2B still pending. + assert(!(mem.pos != 8)); + + let fr: (void | io.error) = bufio.flush(&b); + match (fr) { + case void => { }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 10)); + assert(!(raw[0] != 97u8)); // 'a' + assert(!(raw[9] != 106u8)); // 'j' +}; + +@test fn streamlineflush() void = { + let raw: [32]u8; + let mem: memio.stream = memio.fixed(raw[0:32]); + let m: io.stream = &mem.vt; + + let rb: [4]u8; + let wb: [16]u8; + let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:16]); + + let p: io.stream = &b.vt; + + // First write: no '\n', stays buffered. + let h: [5]u8; + let zh: i32 = sputstr("hello", h[0:5], 0); + let r1: (size | io.error) = io.write(p, h[0:5]); + match (r1) { + case let n: size => { assert(!(n: i32 != 5)); }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 0)); + + // Second write: '\n' present, triggers flush of everything (6B). + let nl: [1]u8; + nl[0] = 10u8; + let r2: (size | io.error) = io.write(p, nl[0:1]); + match (r2) { + case let n: size => { assert(!(n: i32 != 1)); }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 6)); + assert(!(raw[0] != 104u8)); + assert(!(raw[5] != 10u8)); +}; + +@test fn streamflushonwrite() void = { + let raw: [32]u8; + let mem: memio.stream = memio.fixed(raw[0:32]); + let m: io.stream = &mem.vt; + + let rb: [4]u8; + let wb: [16]u8; + let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:16]); + let nilbs: [1]u8; + bufio.setflush(&b, nilbs[0:0]); + + let p: io.stream = &b.vt; + let h: [2]u8; + h[0] = 65u8; h[1] = 66u8; // "AB" + let r1: (size | io.error) = io.write(p, h[0:2]); + match (r1) { + case let n: size => { assert(!(n: i32 != 2)); }; + case let e: io.error => abort(); + }; + let f1: (void | io.error) = bufio.flush(&b); + match (f1) { + case void => { }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 2)); + assert(!(raw[0] != 65u8)); + + let r2: (size | io.error) = io.write(p, h[0:1]); + match (r2) { + case let n: size => { assert(!(n: i32 != 1)); }; + case let e: io.error => abort(); + }; + let f2: (void | io.error) = bufio.flush(&b); + match (f2) { + case void => { }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 3)); + assert(!(raw[2] != 65u8)); +}; + +@test fn streamisbuffered() void = { + let raw: [8]u8; + let mem: memio.stream = memio.fixed(raw[0:8]); + let m: io.stream = &mem.vt; + + let rb: [4]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:4]); + + assert(!(!bufio.isbuffered(&b.vt))); + assert(!(bufio.isbuffered(m))); +}; + +// rbuf holds SOME pending bytes (fewer than requested) AND has spare +// capacity: Hare shifts the pending region to the front and reads more +// into the tail before serving (ref/hare/bufio/stream.ha:209). Pre-fix +// ww served only the short pending count and never topped up. +@test fn streamreadtopup() void = { + let raw: [16]u8; + let n: i32 = sputstr("ABCDEFGHIJKLMNOP", raw[0:16], 0); + + let mem: memio.stream = memio.fixed(raw[0:n]); + let m: io.stream = &mem.vt; + + let rb: [8]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(m, rb[0:8], wb[0:4]); + + let p: io.stream = &b.vt; + + // First read fills rbuf (8B "ABCDEFGH"), serves 3 — 5 pending remain. + let out: [10]u8; + let r1: (size | io.eof | io.error) = io.read(p, out[0:3]); + match (r1) { + case let z: size => { assert(!(z: i32 != 3)); }; + case io.eof => abort(); + case let e: io.error => abort(); + }; + + // Second read wants 10 > 5 pending: Hare shifts "DEFGH" to the front, + // reads 3 more ("IJK") into the tail, serves 8. Pre-fix ww returned 5. + let r2: (size | io.eof | io.error) = io.read(p, out[0:10]); + match (r2) { + case let z: size => { assert(!(z: i32 != 8)); }; + case io.eof => abort(); + case let e: io.error => abort(); + }; + assert(!(out[0] != 68u8)); // 'D' + assert(!(out[7] != 75u8)); // 'K' +}; + +// The load-bearing "EOF fatal only when avail==0" path +// (ref/hare/bufio/stream.ha:216): rbuf still holds pending bytes when +// the top-up io.read returns EOF — Hare drains the pending region, it +// does NOT propagate eof. Source is shorter than two reads so the +// second read's top-up io.read EOFs with bytes still buffered. +@test fn streamreadtopupeof() void = { + let raw: [5]u8; + let n: i32 = sputstr("ABCDE", raw[0:5], 0); + + let mem: memio.stream = memio.fixed(raw[0:n]); + let m: io.stream = &mem.vt; + + let rb: [8]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(m, rb[0:8], wb[0:4]); + + let p: io.stream = &b.vt; + + // First read fills rbuf (5B "ABCDE", source drained), serves 3 — + // 2 pending remain. + let out: [10]u8; + let r1: (size | io.eof | io.error) = io.read(p, out[0:3]); + match (r1) { + case let z: size => { assert(!(z: i32 != 3)); }; + case io.eof => abort(); + case let e: io.error => abort(); + }; + + // Second read wants 10 > 2 pending: shift "DE" to the front, top-up + // io.read hits EOF (source drained). avail==2 != 0 so Hare drains the + // pending region — serves 2, NOT eof. "always fatal" would return eof. + let r2: (size | io.eof | io.error) = io.read(p, out[0:10]); + match (r2) { + case let z: size => { assert(!(z: i32 != 2)); }; + case io.eof => abort(); + case let e: io.error => abort(); + }; + assert(!(out[0] != 68u8)); // 'D' + assert(!(out[1] != 69u8)); // 'E' +}; + +@test fn streamunread() void = { + let raw: [8]u8; + let n: i32 = sputstr("ABCDEFGH", raw[0:8], 0); + + let mem: memio.stream = memio.fixed(raw[0:n]); + let m: io.stream = &mem.vt; + + let rb: [8]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(m, rb[0:8], wb[0:4]); + + let p: io.stream = &b.vt; + + // Pull 3 bytes through bread — fills rbuf from src (8B), serves 3. + let out: [4]u8; + let r1: (size | io.eof | io.error) = io.read(p, out[0:3]); + match (r1) { + case let z: size => { assert(!(z: i32 != 3)); }; + case io.eof => abort(); + case let e: io.error => abort(); + }; + assert(!(out[0] != 65u8)); // 'A' + assert(!(out[2] != 67u8)); // 'C' + + // Push back 2 bytes; next read returns them first. + let push: [2]u8; + push[0] = 88u8; push[1] = 89u8; // "XY" + bufio.unread(&b, push[0:2]); + + let r2: (size | io.eof | io.error) = io.read(p, out[0:2]); + match (r2) { + case let z: size => { assert(!(z: i32 != 2)); }; + case io.eof => abort(); + case let e: io.error => abort(); + }; + assert(!(out[0] != 88u8)); // 'X' + assert(!(out[1] != 89u8)); // 'Y' + + // Subsequent read returns the original tail. + let r3: (size | io.eof | io.error) = io.read(p, out[0:4]); + match (r3) { + case let z: size => { assert(!(z: i32 != 4)); }; + case io.eof => abort(); + case let e: io.error => abort(); + }; + assert(!(out[0] != 68u8)); // 'D' + assert(!(out[3] != 71u8)); // 'G' +}; + +@test fn streamscannerunread() void = { + let raw: [16]u8; + let n: i32 = sputstr("hello\n", raw[0:16], 0); + + let mem: memio.stream = memio.fixed(raw[0:n]); + let m: io.stream = &mem.vt; + + let rb: [16]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(m, rb[0:16], wb[0:4]); + + let push: [3]u8; + push[0] = 88u8; push[1] = 89u8; push[2] = 90u8; // "XYZ" + bufio.unread(&b, push[0:3]); + + let p: io.stream = &b.vt; + let sbuf: [32]u8; + let sc: bufio.scanner = bufio.newscannerbuf(p, sbuf[0:32]); + + let lr: (str | io.eof | io.error | bufio.overflow) = bufio.scanline(&sc); + match (lr) { + case let v: str => { + assert(!(v.len != 8)); // "XYZhello" + assert(!(v[0] != 88u8)); // 'X' + assert(!(v[7] != 111u8)); // 'o' + }; + case io.eof => abort(); + case let e: io.error => abort(); + case bufio.overflow => abort(); + }; + bufio.finish(&sc); +}; + +@test fn streamflushempty() void = { + let raw: [8]u8; + let mem: memio.stream = memio.fixed(raw[0:8]); + let m: io.stream = &mem.vt; + + let rb: [4]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:4]); + + let r: (void | io.error) = bufio.flush(&b); + match (r) { + case void => { }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 0)); +}; + +@test fn streamcloseflushes() void = { + let raw: [8]u8; + let mem: memio.stream = memio.fixed(raw[0:8]); + let m: io.stream = &mem.vt; + + let rb: [4]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:4]); + let nilbs: [1]u8; + bufio.setflush(&b, nilbs[0:0]); // suppress the default '\n' detector + + let p: io.stream = &b.vt; + let h: [2]u8; + h[0] = 80u8; h[1] = 81u8; // "PQ" + let r: (size | io.error) = io.write(p, h[0:2]); + match (r) { + case let n: size => { assert(!(n: i32 != 2)); }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 0)); // still buffered + + let cr: (void | io.error) = io.close(p); + match (cr) { + case void => { }; + case let e: io.error => abort(); + }; + assert(!(mem.pos != 2)); + assert(!(raw[0] != 80u8)); + assert(!(raw[1] != 81u8)); +}; + +// Hare's close_buffered closes src only under flag::MANAGED_HANDLE +// (ref/hare/bufio/stream.ha:194); init's default flag::NONE flushes +// only. The close-recording closesource lets us assert both: the +// buffered byte reaches the sink (flush) AND the closer never fires. +@test fn streamclosenopropagate() void = { + srcclosed = false; + let raw: [8]u8; + closesink = memio.fixed(raw[0:8]); + let src: io.stream = closesource(); + + let rb: [4]u8; + let wb: [4]u8; + let b: bufio.stream = bufio.init(src, rb[0:4], wb[0:4]); + let nilbs: [1]u8; + bufio.setflush(&b, nilbs[0:0]); // suppress the default '\n' detector + + let p: io.stream = &b.vt; + let h: [2]u8; + h[0] = 80u8; h[1] = 81u8; // "PQ" + let r: (size | io.error) = io.write(p, h[0:2]); + match (r) { + case let n: size => { assert(!(n: i32 != 2)); }; + case let e: io.error => abort(); + }; + assert(!(closesink.pos != 0)); // still buffered + + let cr: (void | io.error) = io.close(p); + match (cr) { + case void => { }; + case let e: io.error => abort(); + }; + // Flush happened: the 2 buffered bytes reached the sink. + assert(!(closesink.pos != 2)); + assert(!(raw[0] != 80u8)); + assert(!(raw[1] != 81u8)); + // But the underlying was NOT closed (Hare flag::NONE default). + assert(!srcclosed); +}; + +@test fn streamsetflushcustom() void = { + let raw: [16]u8; + let mem: memio.stream = memio.fixed(raw[0:16]); + let m: io.stream = &mem.vt; + + let rb: [4]u8; + let wb: [16]u8; + let b: bufio.stream = bufio.init(m, rb[0:4], wb[0:16]); + let bs: [1]u8; + bs[0] = 32u8; // ' ' + bufio.setflush(&b, bs[0:1]); + + let p: io.stream = &b.vt; + let buf: [5]u8; + let z: i32 = sputstr("ab cd", buf[0:5], 0); + let r: (size | io.error) = io.write(p, buf[0:5]); + match (r) { + case let n: size => { assert(!(n: i32 != 5)); }; + case let e: io.error => abort(); + }; + // Space at index 2 triggers post-copy flush of all 5 bytes. + assert(!(mem.pos != 5)); + assert(!(raw[2] != 32u8)); + assert(!(raw[4] != 100u8)); // 'd' +}; diff --git a/test/byteid/libbyteid_test.ww b/test/byteid/libbyteid_test.ww index 54557b7c..00e5f351 100644 --- a/test/byteid/libbyteid_test.ww +++ b/test/byteid/libbyteid_test.ww @@ -43,7 +43,7 @@ import time; def MID: i32 = 0; def MDIVERGE: i32 = 1; def MWWREJECT: i32 = 2; -def NENTEXPECT: i32 = 58; +def NENTEXPECT: i32 = 59; type ent = struct { fixture: str, // repo-relative .ww; "" -> probe entry @@ -83,7 +83,7 @@ fn pri(p: str, inc: str, sentinel: str, moddir: str) ent = { return e; }; -// The 58-unit roster. Graduation history lives in git (the retired C +// The 59-unit roster. Graduation history lives in git (the retired C // carrier's table comments); cites are kept only where a non-ID pin // would need them. fn corpus() []ent = { @@ -125,7 +125,8 @@ fn corpus() []ent = { append(es, fx("lib/time/arithm_test.ww")); append(es, fx("lib/time/duration_test.ww")); append(es, fx("lib/time/instant_test.ww")); - append(es, fx("lib/bufio/bufio_test.ww")); + append(es, fx("lib/bufio/stream_test.ww")); + append(es, fx("lib/bufio/scanner_test.ww")); append(es, fx("lib/fmt/fmt_test.ww")); append(es, pr("package main;\nimport sort;\nfn main() i32 = { return 0; };\n", "package sort;", "lib/sort"));