From 616027709892f2e426ef774aa7337fa73db20c25 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Thu, 4 Jun 2026 14:51:49 +0900 Subject: [PATCH] =?UTF-8?q?lib/regex:=20fold-2b=20tranche=20C=20=E2=80=94?= =?UTF-8?q?=20search=20(first=20end-to-end=20match)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of search (ref/hare/regex/regex.ha:746-898) per the drew §9c map: the thread-machine driver over bufio.scanrune — per-rune dispatch, all_matched best-pick (leftmost-longest), need_captures early-exit, first-match leftmost trim, same-pc dedup, failed sweep. compile()'s literal programs now match end to end; test/find (the exec surface) ride tranche D behind the C6 multi-success `?` gate. Spelling divergences, each documented at site with its ha cite: io::handle param → io.stream; alloc([thread{...}])? → decl + append; defer-block cleanup omitted (single-expr defer, no-op frees, #27); rep_counters prefill → ratified loud n_reps>0 abort (no 2b program can set it); newscanner default maxread → types.I32_MAX; scanrune nomem arm dropped (no such member) and multi-type arms split (#13); `return [];` → bind-first zero header (#25/#31 ruling); `result` internals spelled []capture (#20/#38 alias family — reverts with #47); ha:820's indexed capture spread loud-bounded provably-empty (#35); `&..` by-ref ranges → index loops; the ha:821 sized fill-append → count loop (self-activates with the group fold). Two checker findings surfaced mid-port, probe-isolated, dodged at site and FILED: #51 (cs≠ww — the cstage checker types a match-EXPRESSION by its first arm's yield and rejects the io.eof arm against rune; w6c_ww accepts the expression form and emits runtime-correct code, review-verified on scratch/r51.ww), so the scanrune receive is a statement match assigning into a pre-declared (rune | io.eof); #52 (cs≠ww, wwstage only) — a same-name let in a CLOSED sibling scope poisons a later for-init rhs (`let j: i64 = i + 1` resolves i against the dead `let i: size`), so the ha:872 dedup counters are di/dj at site (scratch probes p51/p51b/c/d isolate the trigger and prove the rename byte-identical). add_thread's dedup bound reverts to len(*threads) — the FB1/#41 dodge, fix landed at 796d41b. Imports grow bufio + types. regex_test: +3 @test fns (signalled 20-22) driving private search directly over memio.fixed streams. search_matches = 6 struct-row table rows: full match mid-string, mismatch-restart bcd/abcd, leftmost-longest aa/aaa, zero-length ""/"" (the all_matched path with matchlen 0 must NOT take the need_captures=false early-exit), multibyte b.d over "aßbxd" (root 2/3..5/6 — every idx differs from its bytesize), dedup-heavy aa/aaaa (stable across >=3 same-pc passes). search_early_exit pins ha:845-847 (empty result, len 0); search_no_match pins thread-drain void + EOF-mid-pattern void. Every match row checks all four root indices plus content and result_frees its result (including the early-exit empty one). Coverage limit, mutation-verified and documented at the dedup row: in 2a's fixed-length program space every match ties on match_len, so the leftmost trim (ha:860-866) and the dedup sweep (ha:872-889) are result-invisible — disabling either still passes the table; disabling the failed sweep hangs (caught). Both turn result- and termination-visible with the split/star fold; result stability is the only external pin available today (threads is search-local). Both drivers run the fixture exit 0; w6c vs w6c_ww on the regenerated combined are byte-identical (FC0). PC1-PC4 + P12 probed at base; PC2's blocker fix is the separate #49 commit (c34a48a). --- lib/regex/regex.ww | 248 +++++++++++++++++++++++++++++++++++++--- lib/regex/regex_test.ww | 134 +++++++++++++++++++++- 2 files changed, 365 insertions(+), 17 deletions(-) diff --git a/lib/regex/regex.ww b/lib/regex/regex.ww index 8b4c692c..2d7fe170 100644 --- a/lib/regex/regex.ww +++ b/lib/regex/regex.ww @@ -3,12 +3,12 @@ // compile() literal core (lit/any/match + the leading skip); fold 2b // tranche A = the thread-machine scaffolding (thread/newmatch types, // result_free, strerror); fold 2b tranche B = the engine -// (delete_thread/is_consuming_inst/add_thread/run_thread — dead -// until search lands, fold-1 precedent). Every other metacharacter -// arm, search (F5 element-copy + F2 len-builtin, tranche C) and the -// exec surface (test/find on the C6 multi-success `?` gate, tranche -// D; replace) are DEFERRED behind compiler fixes — probed pre-port, -// pA*/pB*/PB* probes. They land with those fixes. +// (delete_thread/is_consuming_inst/add_thread/run_thread); fold 2b +// tranche C = search, the first end-to-end match. Every other +// metacharacter arm and the exec surface (test/find on the C6 +// multi-success `?` gate, tranche D; replace) are DEFERRED behind +// compiler fixes — probed pre-port, pA*/pB*/PB*/PC* probes. They +// land with those fixes. // // One fold-1 construct is held back behind a filed compiler/fidelity // gap (see the charclass_map site below): @@ -18,8 +18,10 @@ // uses exact type_eq, no element decay). package regex; +import bufio; import io; import strings; +import types; import encoding.utf8; // ref/hare/regex/regex.ha:14 — an error string describing a compilation @@ -225,17 +227,14 @@ fn is_consuming_inst(a: inst) bool = { // ref/hare/regex/regex.ha:557-587. The dedup scan (ha:560-566) is an // index loop: ww has no by-ref `&..` range and a by-value range over -// `*threads` miscompiles (F1, ww-core #11). Its bound reads the -// `.len` pseudo-field, not the len() builtin — len(*threads) -// mis-reads the data pointer as the length (FB1, ww-core #41; the -// F2/#10 deref sibling). Hare's `append(...)?` nomem propagation and the -// ok/defer-if unwind (ha:568/573/586) drop together: ww append -// returns void (#36 filed) and free() reclaims nothing (#27), so -// there is nothing to propagate or unwind. +// `*threads` miscompiles (F1, ww-core #11). Hare's `append(...)?` +// nomem propagation and the ok/defer-if unwind (ha:568/573/586) drop +// together: ww append returns void (#36 filed) and free() reclaims +// nothing (#27), so there is nothing to propagate or unwind. fn add_thread(threads: *[]thread, parent_idx: size, new_pc: size) (void | nomem) = { // Do not add this thread if there is already another thread with // the same PC - for (let k: size = 0; k < ((*threads).len: size); k += 1) { + for (let k: size = 0; k < (len(*threads): size); k += 1) { if ((*threads)[k].pc == new_pc && !(*threads)[k].matched && (*threads)[k].start_idx @@ -377,6 +376,227 @@ fn run_thread( return; }; +// Attempts to match a regular expression against a string and returns +// either the longest leftmost match or all matches. +// +// ref/hare/regex/regex.ha:746-898. Hare's io::handle param is ww's +// io.stream (callers pass &strm.vt over a memio.fixed). The +// (void | []capture | nomem) signature keeps nomem for Hare API +// parity even though nothing in this fold can produce it: ww append +// returns void (#36 filed) and scanrune has no nomem member. +fn search( + re: *regex, + string: str, + h: io.stream, + need_captures: bool, +) (void | []capture | nomem) = { + // Hare builds the initial thread list with `alloc([thread { + // captures = [], ... }])?` (ha:752-754); ww has per-value alloc + // only, so the list is a bare decl plus one appended + // fully-defaulted thread (the `...` fill zeroes captures too). + let threads: []thread; + append(threads, thread { ... }); + // Hare's defer-block cleanup (ha:755-761) is omitted: ww defer + // takes a single expression (cf compile()'s omitted `defer if`) + // and every free in the block is the documented no-op (#27). + + // ha:763-765 prefills threads[0].rep_counters via + // `alloc([0...], re.n_reps)?`. No program this fold can compile + // has n_reps > 0 (the inst_repeat arm is loud), and the sized + // fill form has no ww spelling yet — loud, like the inst_repeat + // arm; the real prefill lands with the repeat fold. + if (re.n_reps > 0) { + abort("regex: repetitions not yet ported"); + }; + + let str_idx: size = 0; + // Hare writes `= void` (ha:768); a ww bare tagged decl zeroes + // the tag, which IS the void member here (PC4-pinned). + let first_match_idx: (void | size); + let str_bytesize: size = 0; + let last_bytesize: size = 0; + + // ww has no default arguments: Hare's newscanner(handle) maxread + // default spells types.I32_MAX explicitly (lib/bufio/bufio.ww). + let scan: bufio.scanner = bufio.newscanner(h, types.I32_MAX); + defer bufio.finish(&scan); + for (true) { + str_bytesize += last_bytesize; + + if (len(threads) == 0) { + return; + }; + + let all_matched: bool = true; + for (let i: size = 0; i < (len(threads): size); i += 1) { + if (!threads[i].matched) { + all_matched = false; + break; + }; + }; + + if (all_matched) { + let best_len: size = 0; + let best_n_captures: size = 0; + let best_idx: size = 0; + for (let i: size = 0; i < (len(threads): size); i += 1) { + let match_len: size = + threads[i].root_capture.end + - threads[i].root_capture.start; + let is_better: bool = match_len > best_len + || match_len == best_len + && (len(threads[i].captures): size) + > best_n_captures; + if (is_better) { + best_len = match_len; + best_idx = i; + best_n_captures = + (len(threads[i].captures): size); + }; + }; + + // length = number of captures (index of final group + + // 1) + root capture + let length: size = 1; + for (let i: size = (len(re.insts): size); i > 0; i -= 1) { + // match-on-indexed scrutinee binds through a + // typed let (the run_thread spelling). + let cur: inst = re.insts[i - 1]; + match (cur) { + case let z: inst_groupstart => { + length = (z: size) + 2; + break; + }; + case => void; + }; + }; + // Hare types this `result` with a capacity hint + // (ha:818). A named-alias local trips the #20/#38 + // alias-value family, so the internal spelling is the + // structural twin []capture (reverts with ww-core + // #47); the capacity hint drops with per-value alloc. + let res: []capture; + append(res, threads[best_idx].root_capture); + // ha:820 spreads threads[best_idx].captures into res. + // The indexed spread source is #35-blocked, but + // captures only populate via the loud-aborted + // groupstart arm or add_thread's loud-bounded dup — + // provably empty in every program this fold can run. + // Loud-bound like add_thread's dup; the verbatim + // spread lands with the group fold. + if (threads[best_idx].captures.len != 0) { + abort("regex: capture copy not yet ported (#35)"); + }; + // ha:821-824's sized fill-append `[capture { ... }...]` + // has no ww spelling; the count loop is shape-correct + // and self-activates with the group fold (length stays + // 1 until then). + for (length != (len(res): size)) { + append(res, capture { ... }); + }; + return res; + }; + + // ww scanrune has no nomem member, so Hare's `case nomem => + // return nomem;` (ha:831-832) drops; the multi-type arms + // (ha:829/833) split per member (the #13 membership-test + // gap); the aborts carry messages (the #45 builtin-shadow + // dodge, cf run_thread). Statement match assigning into the + // declared local, not Hare's expression match: the cstage + // checker types a match-expr by its first arm's yield and + // rejects the io.eof arm against rune — wwstage accepts the + // expression form, a cs≠ww divergence (filed, ww-core #51; + // repro scratch/r51.ww). + let r_or_end: (rune | io.eof); + match (bufio.scanrune(&scan)) { + case let r: rune => { r_or_end = r; }; + case io.eof => { + let e: io.eof; + r_or_end = e; + }; + case let e: io.error => abort("regex: scanrune io error"); + case utf8.invalid => abort("regex: scanrune invalid utf8"); + }; + if (r_or_end is rune) { + last_bytesize = (utf8.runesz(r_or_end as rune): size); + }; + + for (let i: size = 0; i < (len(threads): size); i += 1) { + // Hare itself binds run_thread without `?` (ha:841) — + // its nomem drops on the floor here too. + let res: (void | newmatch | nomem) = run_thread(i, re, + string, &threads, r_or_end, str_idx, + str_bytesize); + let matchlen: size = threads[i].root_capture.end + - threads[i].root_capture.start; + if (res is newmatch && matchlen > 0 && !need_captures) { + // `return [];` (ha:845) — the empty-slice + // literal in return position has no ww spelling + // (#25/#31 ruling); a zero header is a valid + // empty result. + let none: []capture; + return none; + }; + let is_better: bool = res is newmatch && matchlen > 0 + && (first_match_idx is void + || threads[i].start_idx + < first_match_idx as size); + if (is_better) { + first_match_idx = threads[i].start_idx; + }; + }; + str_idx += 1; + + // When we only want the leftmost match, delete all threads + // that start after the earliest non-zero-length matched + // thread. (ww has no by-ref `&..` range — index loop, the + // add_thread divergence.) + if (first_match_idx is size) { + for (let k: size = 0; k < (len(threads): size); k += 1) { + if (threads[k].start_idx + > first_match_idx as size) { + threads[k].failed = true; + }; + }; + }; + + // Delete threads that have a PC that has already been + // encountered in previous threads. Prioritise threads that + // have an earlier start_idx, and threads that were added + // earlier. Hare names these counters i/j (ha:872); they are + // di/dj here because the wwstage checker resolves the inner + // init's `i + 1` against a dead same-name `let i: size` + // sibling scope and rejects size → i64 (cs≠ww, filed + // ww-core #52). + for (let di: i64 = 0; di < (len(threads): i64) - 1; di += 1) { + for (let dj: i64 = di + 1; dj < (len(threads): i64); dj += 1) { + let same_pc: bool = + threads[di].pc == threads[dj].pc; + let none_matched: bool = !threads[dj].matched + && !threads[di].matched; + if (same_pc && none_matched) { + if (threads[di].start_idx + <= threads[dj].start_idx) { + delete_thread((dj: size), &threads); + dj -= 1; + } else { + delete_thread((di: size), &threads); + di -= 1; + break; + }; + }; + }; + }; + + for (let i: size = 0; i < (len(threads): size); i += 1) { + if (threads[i].failed) { + delete_thread(i, &threads); + i -= 1; + }; + }; + }; +}; + // Frees a [[result]]. // // ref/hare/regex/regex.ha:1113-1116, verbatim — the free() builtin is diff --git a/lib/regex/regex_test.ww b/lib/regex/regex_test.ww index a7e1753d..733eedeb 100644 --- a/lib/regex/regex_test.ww +++ b/lib/regex/regex_test.ww @@ -1,8 +1,9 @@ // regex_test — exercises the lib/regex fold-1 data model (the type -// model + finish()), the fold-2a compile() literal core, and the +// model + finish()), the fold-2a compile() literal core, the // fold-2b tranche-A/B thread machine (thread/newmatch + result_free -// + strerror; delete_thread/is_consuming_inst/add_thread/run_thread; -// search and the exec surface are deferred — see regex.ww). Run with +// + strerror; delete_thread/is_consuming_inst/add_thread/run_thread), +// and the tranche-C search end-to-end matches (the exec surface +// test/find is tranche D — see regex.ww). Run with // `out/bin/ww run lib/regex/regex_test.ww`. // // Private symbols (thread, newmatch) are reached unqualified: this @@ -31,6 +32,7 @@ package regex; import regex; import io; +import memio; import os; import strings; @@ -664,6 +666,129 @@ fn ic_one(v: regex.inst, want: bool) void = { if (ts2[0].root_capture.end != (0: size)) { fail(); }; }; +// search (regex.ha:746-898) driven DIRECTLY (private fn, package-regex +// test) over memio-backed streams — the exec surface (test/find) is +// tranche D. Each match row pins the root capture's four indices plus +// content; the multibyte row keeps idx != bytesize honest. Rows share +// (expr, input, need_captures, want) shape — the P12 struct-row table. +type scase = struct { + expr: str, + input: str, + nc: bool, + start: size, + sb: size, + end: size, + eb: size, + content: str, +}; + +@test fn search_matches() void = { + let rows: [6]scase = [ + // full match mid-string: skip-respawn + dispatch + + // all_matched exit + scase { expr = "ab", input = "xab", nc = true, + start = 1, sb = 1, end = 3, eb = 3, content = "ab" }, + // mismatch-restart: the idx-0 child fails and is swept; the + // restarted thread wins (failed-sweep interplay) + scase { expr = "bcd", input = "abcd", nc = true, + start = 1, sb = 1, end = 4, eb = 4, content = "bcd" }, + // leftmost-longest best-pick + first_match_idx trim + scase { expr = "aa", input = "aaa", nc = true, + start = 0, sb = 0, end = 2, eb = 2, content = "aa" }, + // zero-length: the all_matched path with matchlen 0 must + // NOT take the need_captures=false early-exit (ha:845 + // requires matchlen > 0) — hence nc=false expecting the + // FULL one-capture result, not the empty early-exit slice + scase { expr = "", input = "", nc = false, + start = 0, sb = 0, end = 0, eb = 0, content = "" }, + // multibyte: the 2-byte ß before the match start splits + // every idx from its bytesize; inst_any consumes 'x' + scase { expr = "b.d", input = "aßbxd", nc = true, + start = 2, sb = 3, end = 5, eb = 6, content = "bxd" }, + // dedup-heavy: same-pc threads spawn on every step across + // >=3 passes (ha:872-889); the pick must stay stable. + // Result stability is the only external pin available this + // fold: 2a programs are all fixed-length, every match ties + // on match_len, and best-pick's insertion-order tiebreak + // alone yields leftmost — so the dedup sweep and the + // leftmost trim are result-invisible (mutation-verified: + // disabling either still passes this table; disabling the + // failed sweep hangs). Both turn result- and + // termination-visible with the split/star fold. + scase { expr = "aa", input = "aaaa", nc = true, + start = 0, sb = 0, end = 2, eb = 2, content = "aa" }, + ]; + let i: i32 = 0; + for (i < len(rows)) { + let ex: str = rows[i].expr; + let inp: str = rows[i].input; + let c: (regex.regex | regex.error | nomem) = regex.compile(ex); + match (c) { + case let re: regex.regex => { + let strm: memio.stream = + memio.fixed(strings.toutf8(inp)); + let r: (void | []capture | nomem) = + search(&re, inp, &strm.vt, rows[i].nc); + if (!(r is []capture)) { fail(); }; + let caps: []capture = r as []capture; + if (len(caps) != 1) { fail(); }; + if (caps[0].start != rows[i].start) { fail(); }; + if (caps[0].start_bytesize != rows[i].sb) { fail(); }; + if (caps[0].end != rows[i].end) { fail(); }; + if (caps[0].end_bytesize != rows[i].eb) { fail(); }; + let wc: str = rows[i].content; + if (strings.compare(caps[0].content, wc) != 0) { + fail(); + }; + regex.result_free(caps); + regex.finish(&re); + }; + case => fail(); + }; + i += 1; + }; +}; + +// ha:845-847: a non-zero-length newmatch with need_captures=false +// returns the empty result immediately, skipping the best-pick pass. +@test fn search_early_exit() void = { + let c: (regex.regex | regex.error | nomem) = regex.compile("ab"); + match (c) { + case let re: regex.regex => { + let strm: memio.stream = memio.fixed(strings.toutf8("xab")); + let r: (void | []capture | nomem) = + search(&re, "xab", &strm.vt, false); + if (!(r is []capture)) { fail(); }; + let caps: []capture = r as []capture; + if (len(caps) != 0) { fail(); }; + regex.result_free(caps); + regex.finish(&re); + }; + case => fail(); + }; +}; + +// void rows: no match anywhere ("ab" over "xyz" — every thread fails, +// the list drains, ha:777-779) and EOF mid-pattern ("ab" over "a" — +// the consuming-inst EOF fail). +@test fn search_no_match() void = { + let c: (regex.regex | regex.error | nomem) = regex.compile("ab"); + match (c) { + case let re: regex.regex => { + let strm: memio.stream = memio.fixed(strings.toutf8("xyz")); + let r: (void | []capture | nomem) = + search(&re, "xyz", &strm.vt, true); + if (!(r is void)) { fail(); }; + let strm2: memio.stream = memio.fixed(strings.toutf8("a")); + let r2: (void | []capture | nomem) = + search(&re, "a", &strm2.vt, true); + if (!(r2 is void)) { fail(); }; + regex.finish(&re); + }; + case => fail(); + }; +}; + export fn main() i32 = { signalled = 1; lit_and_match(); signalled = 2; size_aliases_distinct(); @@ -684,5 +809,8 @@ export fn main() i32 = { signalled = 17; add_thread_dedup_inherit(); signalled = 18; run_thread_literal_program(); signalled = 19; run_thread_anchored_route(); + signalled = 20; search_matches(); + signalled = 21; search_early_exit(); + signalled = 22; search_no_match(); return 0; };