From d09197af8e1373b319ab8eca9801613918f670d5 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Mon, 18 May 2026 12:16:22 +0900 Subject: [PATCH] lib/strings+test: Hare port (iterator + next) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forward UTF-8 rune cursor per ref/hare/strings/iter.ha; iterator flattens Hare's anon-embedded utf8::decoder to explicit offs/src/reverse fields, next() copy-in/copy-out a local decoder and aborts on more/invalid per move()'s discipline. The iterator flattens Hare's anonymous-embedded utf8::decoder (ref/hare/strings/iter.ha:6-9) to explicit offs/src/reverse fields because ww has no anon-embed syntax. reverse is retained on the struct so riter populates it once utf8.prev (reverse DFA) and strings.prev land. next() copies the iterator's offs/src into a local utf8.decoder, delegates to utf8.next, then writes offs back; copy-in/copy-out is the cost of the flattened layout. more/invalid arms abort with "strings.next: invalid UTF-8", mirroring Hare's move() (ref/hare/strings/iter.ha:51-58) which aborts unconditionally on those arms. Deferred surface (no in-tree caller; follow-up tasks): prev, riter, iterstr, slice, position, move. prev specifically needs utf8.prev (reverse DFA), which isn't on the lib/encoding/utf8 surface yet. lib/strings/strings.ww moves off test/wcc/900_stdlib.c's standalone-compile list per the existing bufio/fmt/os precedent: the iterator's (rune | utf8.done) return type and the local utf8.decoder reference need cross-module type resolution, which the standalone w6c path doesn't do. Runtime coverage stays at 966_strings_run, which now exercises 21 signalled cases (was 15). The iterator's Hare-faithful `reverse: bool` field surfaced #33 (cstage cgen narrow-sret-field copy mis-width) during this port's pre-flight; that fix landed at 0d96196 ahead of this commit. Future bisecters tracking a narrow-field-related regression in lib/strings or sret-aware stdlib growth should consult #33 + this commit's bracket. The 4-arm match on utf8.next return surfaced #31 (wwstage fnretlookupmod) earlier in the same chain (b787641). Tests: - 6 new @test fns in stringstest.ww (signalled 16-21): empty, ASCII, 2-byte (café), 3-byte (こんにちは), 4-byte (🦀rust), mixed-width ("Hello, 世界! 🌍"). Each verifies forward iteration, done@EOI, and repeated next-after-done stays done (iter_empty). Multibyte literal limitation handled via `0xE9u32: rune` cast per existing pattern at stringstest.ww:82. 104/104 ok. 995_self_rebuild stays green (ww2==ww3==ww4 byte-id). --- lib/strings/strings.ww | 47 ++++++++++ lib/strings/stringstest.ww | 131 +++++++++++++++++++++++++++ selfhost/cmd/w6c/main.combined.ww | 47 ++++++++++ selfhost/cmd/wwdump/main.combined.ww | 47 ++++++++++ selfhost/test/smoke.combined.ww | 47 ++++++++++ test/wcc/900_stdlib.c | 21 +++-- 6 files changed, 330 insertions(+), 10 deletions(-) diff --git a/lib/strings/strings.ww b/lib/strings/strings.ww index 0cd5a4b6..0dd1f084 100644 --- a/lib/strings/strings.ww +++ b/lib/strings/strings.ww @@ -22,6 +22,13 @@ // `os.alloc` aborts on OOM (no `nomem` type), so we return plain // `str`. Empty input returns `{nil, 0}`; Hare returns the static // empty string — same observable result. +// - `iterator` is flattened (`offs`, `src`, `reverse` fields). +// Hare uses anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9); ww has no anonymous-embed +// syntax. `next` copies the iterator's `offs`/`src` into a local +// `utf8.decoder` for the call, then writes `offs` back. `prev` / +// `riter` / `iterstr` / `slice` / `position` are deferred — no +// in-tree caller; `prev` needs `utf8.prev` (reverse DFA). use bytes; use utf8; @@ -268,3 +275,43 @@ export fn rtrim(input: str, exclude: rune) str = { export fn trim(input: str, exclude: rune) str = { return ltrim(rtrim(input, exclude), exclude); }; + +// iterator — forward UTF-8 rune cursor over a `str`. Layout flattens +// Hare's anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9) to explicit fields; `reverse` is +// retained on the type because `riter` will populate it once `prev` / +// `utf8.prev` land. May be copied to save state. +export type iterator = struct { + offs: i32, + src: []u8, + reverse: bool, +}; + +// iter — initialize a forward iterator at the start of `src`. +// ref/hare/strings/iter.ha:24. +export fn iter(src: str) iterator = { + let r: iterator; + r.src = toutf8(src); + r.offs = 0; + r.reverse = false; + return r; +}; + +// next — advance the iterator one rune. Returns `utf8.done` at end +// of input. Aborts on `more` / `invalid` — mirrors Hare's +// ref/hare/strings/iter.ha:51-58 `move()`, which aborts unconditionally +// on those arms ("Invalid UTF-8 string (this should not happen)"). +// +// Copy-in / copy-out is the cost of flattening the embedded decoder; +// see the iterator divergence note at the top of the file. +export fn next(it: *iterator) (rune | utf8.done) = { + let d: utf8.decoder; + d.src = it.src; + d.offs = it.offs; + match (utf8.next(&d)) { + case let r: rune => { it.offs = d.offs; return r; }; + case let dn: utf8.done => return dn; + case let m: utf8.more => abort("strings.next: invalid UTF-8"); + case let e: utf8.invalid => abort("strings.next: invalid UTF-8"); + }; +}; diff --git a/lib/strings/stringstest.ww b/lib/strings/stringstest.ww index ac0f22fb..d1feb5cb 100644 --- a/lib/strings/stringstest.ww +++ b/lib/strings/stringstest.ww @@ -7,6 +7,7 @@ // suffix,compare}.ha where ww can express them. use strings; +use utf8; use os; let signalled: i32 = 0; @@ -270,6 +271,130 @@ fn streq(a: str, b: str) bool = { if (r.ptr != s.ptr) { fail(); }; // borrowed, not copied }; +// ---- iter / next ------------------------------------------------------ +// ref/hare/strings/iter.ha:84-108. Hare's @test fn iter() uses prev + +// riter heavily; both are deferred (no `utf8.prev`). Rebuild forward- +// only here: empty / ASCII / 2-byte / 3-byte / 4-byte / done@EOI / +// mixed-width. + +@test fn iter_empty_cases() void = { + let it: strings.iterator = strings.iter(""); + match (strings.next(&it)) { + case let r: rune => { fail(); }; + case utf8.done => void; + }; + // Repeated next after done stays done. + match (strings.next(&it)) { + case let r: rune => { fail(); }; + case utf8.done => void; + }; +}; + +@test fn iter_ascii_cases() void = { + let it: strings.iterator = strings.iter("hi!"); + match (strings.next(&it)) { + case let r: rune => { if (r != 'h') { fail(); }; }; + case utf8.done => { fail(); }; + }; + match (strings.next(&it)) { + case let r: rune => { if (r != 'i') { fail(); }; }; + case utf8.done => { fail(); }; + }; + match (strings.next(&it)) { + case let r: rune => { if (r != '!') { fail(); }; }; + case utf8.done => { fail(); }; + }; + match (strings.next(&it)) { + case let r: rune => { fail(); }; + case utf8.done => void; + }; +}; + +@test fn iter_twobyte_cases() void = { + let it: strings.iterator = strings.iter("café"); + let i: i32 = 0; + let expect: [4]rune; + expect[0] = 'c'; expect[1] = 'a'; expect[2] = 'f'; + expect[3] = 0xE9u32: rune; // 'é' U+00E9 + for (i < 4) { + match (strings.next(&it)) { + case let r: rune => { if (r != expect[i]) { fail(); }; }; + case utf8.done => { fail(); }; + }; + i += 1; + }; + match (strings.next(&it)) { + case let r: rune => { fail(); }; + case utf8.done => void; + }; +}; + +@test fn iter_threebyte_cases() void = { + let it: strings.iterator = strings.iter("こんにちは"); + let i: i32 = 0; + let expect: [5]rune; + expect[0] = 0x3053u32: rune; // 'こ' + expect[1] = 0x3093u32: rune; // 'ん' + expect[2] = 0x306Bu32: rune; // 'に' + expect[3] = 0x3061u32: rune; // 'ち' + expect[4] = 0x306Fu32: rune; // 'は' + for (i < 5) { + match (strings.next(&it)) { + case let r: rune => { if (r != expect[i]) { fail(); }; }; + case utf8.done => { fail(); }; + }; + i += 1; + }; + match (strings.next(&it)) { + case let r: rune => { fail(); }; + case utf8.done => void; + }; +}; + +@test fn iter_fourbyte_cases() void = { + let it: strings.iterator = strings.iter("🦀rust"); + let i: i32 = 0; + let expect: [5]rune; + expect[0] = 0x1F980u32: rune; // '🦀' + expect[1] = 'r'; expect[2] = 'u'; expect[3] = 's'; expect[4] = 't'; + for (i < 5) { + match (strings.next(&it)) { + case let r: rune => { if (r != expect[i]) { fail(); }; }; + case utf8.done => { fail(); }; + }; + i += 1; + }; + match (strings.next(&it)) { + case let r: rune => { fail(); }; + case utf8.done => void; + }; +}; + +@test fn iter_mixed_cases() void = { + // "Hello, 世界! 🌍" — 1+1+1+1+1+1+1+3+3+1+1+4 = 12 runes, + // widths 1/3/4 mixed. + let it: strings.iterator = strings.iter("Hello, 世界! 🌍"); + let i: i32 = 0; + let expect: [12]rune; + expect[0] = 'H'; expect[1] = 'e'; expect[2] = 'l'; expect[3] = 'l'; + expect[4] = 'o'; expect[5] = ','; expect[6] = ' '; + expect[7] = 0x4E16u32: rune; // '世' + expect[8] = 0x754Cu32: rune; // '界' + expect[9] = '!'; expect[10] = ' '; + expect[11] = 0x1F30Du32: rune; // '🌍' + for (i < 12) { + match (strings.next(&it)) { + case let r: rune => { if (r != expect[i]) { fail(); }; }; + case utf8.done => { fail(); }; + }; + i += 1; + }; + match (strings.next(&it)) { + case let r: rune => { fail(); }; + case utf8.done => void; + }; +}; + export fn main() i32 = { signalled = 1; dup_cases(); signalled = 2; concat_cases(); @@ -286,5 +411,11 @@ export fn main() i32 = { signalled = 13; trim_cases(); signalled = 14; compare_cases(); signalled = 15; utf8_roundtrip_cases(); + signalled = 16; iter_empty_cases(); + signalled = 17; iter_ascii_cases(); + signalled = 18; iter_twobyte_cases(); + signalled = 19; iter_threebyte_cases(); + signalled = 20; iter_fourbyte_cases(); + signalled = 21; iter_mixed_cases(); return 0; }; diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index 0df3e14f..3f9e5b68 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -1371,6 +1371,13 @@ export fn encoderune(out: []u8, r: rune) i32 = { // `os.alloc` aborts on OOM (no `nomem` type), so we return plain // `str`. Empty input returns `{nil, 0}`; Hare returns the static // empty string — same observable result. +// - `iterator` is flattened (`offs`, `src`, `reverse` fields). +// Hare uses anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9); ww has no anonymous-embed +// syntax. `next` copies the iterator's `offs`/`src` into a local +// `utf8.decoder` for the call, then writes `offs` back. `prev` / +// `riter` / `iterstr` / `slice` / `position` are deferred — no +// in-tree caller; `prev` needs `utf8.prev` (reverse DFA). use bytes; use utf8; @@ -1618,6 +1625,46 @@ export fn trim(input: str, exclude: rune) str = { return ltrim(rtrim(input, exclude), exclude); }; +// iterator — forward UTF-8 rune cursor over a `str`. Layout flattens +// Hare's anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9) to explicit fields; `reverse` is +// retained on the type because `riter` will populate it once `prev` / +// `utf8.prev` land. May be copied to save state. +export type iterator = struct { + offs: i32, + src: []u8, + reverse: bool, +}; + +// iter — initialize a forward iterator at the start of `src`. +// ref/hare/strings/iter.ha:24. +export fn iter(src: str) iterator = { + let r: iterator; + r.src = toutf8(src); + r.offs = 0; + r.reverse = false; + return r; +}; + +// next — advance the iterator one rune. Returns `utf8.done` at end +// of input. Aborts on `more` / `invalid` — mirrors Hare's +// ref/hare/strings/iter.ha:51-58 `move()`, which aborts unconditionally +// on those arms ("Invalid UTF-8 string (this should not happen)"). +// +// Copy-in / copy-out is the cost of flattening the embedded decoder; +// see the iterator divergence note at the top of the file. +export fn next(it: *iterator) (rune | utf8.done) = { + let d: utf8.decoder; + d.src = it.src; + d.offs = it.offs; + match (utf8.next(&d)) { + case let r: rune => { it.offs = d.offs; return r; }; + case let dn: utf8.done => return dn; + case let m: utf8.more => abort("strings.next: invalid UTF-8"); + case let e: utf8.invalid => abort("strings.next: invalid UTF-8"); + }; +}; + // MODULE: strconv // strconv — number↔string conversions. // diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index 6bac16ab..d7d599de 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -1371,6 +1371,13 @@ export fn encoderune(out: []u8, r: rune) i32 = { // `os.alloc` aborts on OOM (no `nomem` type), so we return plain // `str`. Empty input returns `{nil, 0}`; Hare returns the static // empty string — same observable result. +// - `iterator` is flattened (`offs`, `src`, `reverse` fields). +// Hare uses anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9); ww has no anonymous-embed +// syntax. `next` copies the iterator's `offs`/`src` into a local +// `utf8.decoder` for the call, then writes `offs` back. `prev` / +// `riter` / `iterstr` / `slice` / `position` are deferred — no +// in-tree caller; `prev` needs `utf8.prev` (reverse DFA). use bytes; use utf8; @@ -1618,6 +1625,46 @@ export fn trim(input: str, exclude: rune) str = { return ltrim(rtrim(input, exclude), exclude); }; +// iterator — forward UTF-8 rune cursor over a `str`. Layout flattens +// Hare's anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9) to explicit fields; `reverse` is +// retained on the type because `riter` will populate it once `prev` / +// `utf8.prev` land. May be copied to save state. +export type iterator = struct { + offs: i32, + src: []u8, + reverse: bool, +}; + +// iter — initialize a forward iterator at the start of `src`. +// ref/hare/strings/iter.ha:24. +export fn iter(src: str) iterator = { + let r: iterator; + r.src = toutf8(src); + r.offs = 0; + r.reverse = false; + return r; +}; + +// next — advance the iterator one rune. Returns `utf8.done` at end +// of input. Aborts on `more` / `invalid` — mirrors Hare's +// ref/hare/strings/iter.ha:51-58 `move()`, which aborts unconditionally +// on those arms ("Invalid UTF-8 string (this should not happen)"). +// +// Copy-in / copy-out is the cost of flattening the embedded decoder; +// see the iterator divergence note at the top of the file. +export fn next(it: *iterator) (rune | utf8.done) = { + let d: utf8.decoder; + d.src = it.src; + d.offs = it.offs; + match (utf8.next(&d)) { + case let r: rune => { it.offs = d.offs; return r; }; + case let dn: utf8.done => return dn; + case let m: utf8.more => abort("strings.next: invalid UTF-8"); + case let e: utf8.invalid => abort("strings.next: invalid UTF-8"); + }; +}; + // MODULE: strconv // strconv — number↔string conversions. // diff --git a/selfhost/test/smoke.combined.ww b/selfhost/test/smoke.combined.ww index 0a1400a3..d1e7d855 100644 --- a/selfhost/test/smoke.combined.ww +++ b/selfhost/test/smoke.combined.ww @@ -1263,6 +1263,13 @@ export fn encoderune(out: []u8, r: rune) i32 = { // `os.alloc` aborts on OOM (no `nomem` type), so we return plain // `str`. Empty input returns `{nil, 0}`; Hare returns the static // empty string — same observable result. +// - `iterator` is flattened (`offs`, `src`, `reverse` fields). +// Hare uses anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9); ww has no anonymous-embed +// syntax. `next` copies the iterator's `offs`/`src` into a local +// `utf8.decoder` for the call, then writes `offs` back. `prev` / +// `riter` / `iterstr` / `slice` / `position` are deferred — no +// in-tree caller; `prev` needs `utf8.prev` (reverse DFA). use bytes; use utf8; @@ -1510,6 +1517,46 @@ export fn trim(input: str, exclude: rune) str = { return ltrim(rtrim(input, exclude), exclude); }; +// iterator — forward UTF-8 rune cursor over a `str`. Layout flattens +// Hare's anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9) to explicit fields; `reverse` is +// retained on the type because `riter` will populate it once `prev` / +// `utf8.prev` land. May be copied to save state. +export type iterator = struct { + offs: i32, + src: []u8, + reverse: bool, +}; + +// iter — initialize a forward iterator at the start of `src`. +// ref/hare/strings/iter.ha:24. +export fn iter(src: str) iterator = { + let r: iterator; + r.src = toutf8(src); + r.offs = 0; + r.reverse = false; + return r; +}; + +// next — advance the iterator one rune. Returns `utf8.done` at end +// of input. Aborts on `more` / `invalid` — mirrors Hare's +// ref/hare/strings/iter.ha:51-58 `move()`, which aborts unconditionally +// on those arms ("Invalid UTF-8 string (this should not happen)"). +// +// Copy-in / copy-out is the cost of flattening the embedded decoder; +// see the iterator divergence note at the top of the file. +export fn next(it: *iterator) (rune | utf8.done) = { + let d: utf8.decoder; + d.src = it.src; + d.offs = it.offs; + match (utf8.next(&d)) { + case let r: rune => { it.offs = d.offs; return r; }; + case let dn: utf8.done => return dn; + case let m: utf8.more => abort("strings.next: invalid UTF-8"); + case let e: utf8.invalid => abort("strings.next: invalid UTF-8"); + }; +}; + // MODULE: strconv // strconv — number↔string conversions. // diff --git a/test/wcc/900_stdlib.c b/test/wcc/900_stdlib.c index f58479ec..e5e0fe35 100644 --- a/test/wcc/900_stdlib.c +++ b/test/wcc/900_stdlib.c @@ -13,7 +13,6 @@ static const char *modules[] = { "lib/types/types.ww", "lib/ascii/ascii.ww", "lib/bytes/bytes.ww", - "lib/strings/strings.ww", "lib/io/io.ww", "lib/errors/errors.ww", "lib/strconv/strconv.ww", @@ -32,15 +31,17 @@ static const char *modules[] = { "lib/math/random/random.ww", "lib/time/time.ww", "lib/c/libc/libc.ww", - /* lib/bufio/bufio.ww, lib/fmt/fmt.ww, and lib/os/os.ww moved - * off this list: each has cross-module type refs that only - * resolve once the driver concatenates `use`d modules. bufio / - * fmt graduated to io.stream-based sinks (*io.stream, io.closed, - * io.eof); lib/os carries time.instant in filestat post-Commit B. - * Coverage lives at lib/bufio/bufiotest.ww + lib/fmt/fmttest.ww + - * lib/os/stattest.ww (wired at 998_bufio_run.c, 970_fmt_run.c, - * 976_stat_run.c), plus the bufio.scanline / fmt.println e2e rows - * in test/wcc/700_e2e.c. */ + /* lib/bufio/bufio.ww, lib/fmt/fmt.ww, lib/os/os.ww, and + * lib/strings/strings.ww moved off this list: each has cross- + * module type refs that only resolve once the driver concatenates + * `use`d modules. bufio / fmt graduated to io.stream-based sinks + * (*io.stream, io.closed, io.eof); lib/os carries time.instant in + * filestat post-Commit B; lib/strings.iterator + lib/strings.next + * reference utf8.decoder / utf8.done. Coverage lives at + * lib/bufio/bufiotest.ww + lib/fmt/fmttest.ww + lib/os/stattest.ww + * + lib/strings/stringstest.ww (wired at 998_bufio_run.c, + * 970_fmt_run.c, 976_stat_run.c, 966_strings_run.c), plus the + * bufio.scanline / fmt.println e2e rows in test/wcc/700_e2e.c. */ "lib/net/net.ww", NULL };