Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wcc/ww: drop underscores from next/peek/remaining-tokens (F-Z)

Browse Source
This commit is contained in:
Hojun-Cho
2026-06-15 04:31:16 +09:00
parent 5ae3787cb9
commit 04d35c25c3
10 changed files with 278 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
lib/bytes/bytes.ww
View File

@@ -7,7 +7,7 @@
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent. // Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse` // - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings // ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical. // `move`. Outwardly identical.
@@ -23,7 +23,7 @@ package bytes;
import os; import os;
import types; import types;
// done — iteration sentinel returned by next_token / peek_token at // done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does // token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. // (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -242,8 +242,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// rtokenize — reverse-direction tokenize. First next_token yields the // rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. // last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = { export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX, assert((in.len: i64) < types.I64_MAX,
@@ -259,10 +259,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// peek_token — next token without advancing the cursor. Returns done // peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token. // once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91. // ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = { export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) { if (s.delim.len == 0) {
let d: done; return d; let d: done; return d;
}; };
@@ -338,12 +338,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r; return r;
}; };
// next_token — current token, then advance past it and the delim. // nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so // Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = { export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8; let b: []u8;
match (peek_token(s)) { match (peektoken(s)) {
case let v: []u8 => { b = v; }; case let v: []u8 => { b = v; };
case done => { let d: done; return d; }; case done => { let d: done; return d; };
}; };
@@ -380,9 +380,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b; return b;
}; };
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. // remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145. // ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = { export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in; return s.in;
}; };
@@ -409,16 +409,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...); let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -447,16 +447,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...); let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };

40
lib/bytes/bytestest.ww
View File

@@ -237,7 +237,7 @@ import os;
assert(!(!bytes.hassuffix(a4[0:4], s234[0:3]))); assert(!(!bytes.hassuffix(a4[0:4], s234[0:3])));
}; };
// ---- tokenize / rtokenize / peek_token / remaining_tokens ----------- // ---- tokenize / rtokenize / peektoken / remainingtokens -----------
// ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize // ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize
// drives the iterator through an expected-token sequence and asserts // drives the iterator through an expected-token sequence and asserts
// `equal(p, n)` (peek == next), `equal(n, want)` (next == expected). // `equal(p, n)` (peek == next), `equal(n, want)` (next == expected).
@@ -245,13 +245,13 @@ import os;
// expect_token — table row driver. Advances `t` once, asserts the // expect_token — table row driver. Advances `t` once, asserts the
// returned token matches `want`. peek invariant: peek must equal next. // returned token matches `want`. peek invariant: peek must equal next.
fn expect_token(t: *bytes.tokenizer, want: []u8) void = { fn expect_token(t: *bytes.tokenizer, want: []u8) void = {
match (bytes.peek_token(t)) { match (bytes.peektoken(t)) {
case let p: []u8 => { case let p: []u8 => {
assert(!(!bytes.equal(p, want))); assert(!(!bytes.equal(p, want)));
}; };
case bytes.done => abort(); case bytes.done => abort();
}; };
match (bytes.next_token(t)) { match (bytes.nexttoken(t)) {
case let n: []u8 => { case let n: []u8 => {
assert(!(!bytes.equal(n, want))); assert(!(!bytes.equal(n, want)));
}; };
@@ -261,11 +261,11 @@ fn expect_token(t: *bytes.tokenizer, want: []u8) void = {
// expect_done — table-row driver. peek and next must both be done. // expect_done — table-row driver. peek and next must both be done.
fn expect_done(t: *bytes.tokenizer) void = { fn expect_done(t: *bytes.tokenizer) void = {
match (bytes.peek_token(t)) { match (bytes.peektoken(t)) {
case let p: []u8 => abort(); case let p: []u8 => abort();
case bytes.done => void; case bytes.done => void;
}; };
match (bytes.next_token(t)) { match (bytes.nexttoken(t)) {
case let n: []u8 => abort(); case let n: []u8 => abort();
case bytes.done => void; case bytes.done => void;
}; };
@@ -381,63 +381,63 @@ fn expect_done(t: *bytes.tokenizer) void = {
expect_done(&t6); expect_done(&t6);
}; };
@test fn peek_token_cases() void = { @test fn peektoken_cases() void = {
// Peeking twice without advancing returns the same token. // Peeking twice without advancing returns the same token.
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.peek_token(&t)) { match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); }; case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); };
case bytes.done => abort(); case bytes.done => abort();
}; };
match (bytes.peek_token(&t)) { match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); }; case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); };
case bytes.done => abort(); case bytes.done => abort();
}; };
// Then advance once — peek-after-next is the second token. // Then advance once — peek-after-next is the second token.
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
match (bytes.next_token(&t)) { match (bytes.nexttoken(&t)) {
case let n: []u8 => { assert(!(!bytes.equal(n, e_12[0:2]))); }; case let n: []u8 => { assert(!(!bytes.equal(n, e_12[0:2]))); };
case bytes.done => abort(); case bytes.done => abort();
}; };
match (bytes.peek_token(&t)) { match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort(); case bytes.done => abort();
}; };
// Reverse peek symmetry — first peek is last token. // Reverse peek symmetry — first peek is last token.
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.peek_token(&t2)) { match (bytes.peektoken(&t2)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort(); case bytes.done => abort();
}; };
match (bytes.peek_token(&t2)) { match (bytes.peektoken(&t2)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort(); case bytes.done => abort();
}; };
}; };
@test fn remaining_tokens_cases() void = { @test fn remainingtokens_cases() void = {
// After one next_token, remaining is bytes past the consumed delim. // After one nexttoken, remaining is bytes past the consumed delim.
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.next_token(&t)) { match (bytes.nexttoken(&t)) {
case let n: []u8 => void; case let n: []u8 => void;
case bytes.done => abort(); case bytes.done => abort();
}; };
let r: []u8 = bytes.remaining_tokens(&t); let r: []u8 = bytes.remainingtokens(&t);
assert(!(!bytes.equal(r, e_34[0:2]))); assert(!(!bytes.equal(r, e_34[0:2])));
// Reverse — after one next_token, remaining is bytes before the // Reverse — after one nexttoken, remaining is bytes before the
// consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2]. // consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2].
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.next_token(&t2)) { match (bytes.nexttoken(&t2)) {
case let n: []u8 => void; case let n: []u8 => void;
case bytes.done => abort(); case bytes.done => abort();
}; };
let r2: []u8 = bytes.remaining_tokens(&t2); let r2: []u8 = bytes.remainingtokens(&t2);
assert(!(!bytes.equal(r2, e_12[0:2]))); assert(!(!bytes.equal(r2, e_12[0:2])));
}; };
@@ -655,7 +655,7 @@ fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = {
// delim absent — done short-circuits immediately at iter 0; // delim absent — done short-circuits immediately at iter 0;
// toks ends up holding only the eventual remainder if n=1 // toks ends up holding only the eventual remainder if n=1
// (the loop never runs and peek picks up the full input). // (the loop never runs and peek picks up the full input).
// With n>1 the loop's first next_token sees done (no delim // With n>1 the loop's first nexttoken sees done (no delim
// match anywhere) and returns toks={} per Hare's early-exit. // match anywhere) and returns toks={} per Hare's early-exit.
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8; let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5); let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5);

32
lib/strings/strings.ww
View File

@@ -679,7 +679,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
}; };
// rtokenize — reverse-direction counterpart to [[tokenize]]. First // rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first. // nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44. // ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = { export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim); let d: []u8 = toutf8(delim);
@@ -692,31 +692,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...); return bytes.rtokenize(toutf8(s), d...);
}; };
// next_token — current token, advancing the cursor. // nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62. // ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = { export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) { match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// peek_token — current token without advancing. // peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71. // ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = { export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) { match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// remaining_tokens — unconsumed portion of the input ahead of the // remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79. // cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = { export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b)); return frombytes(bytes.remainingtokens(b));
}; };
// cut — split `in` along the first instance of `delim`, returning the // cut — split `in` along the first instance of `delim`, returning the
@@ -757,16 +757,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim); let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -791,16 +791,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim); let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };

40
lib/strings/stringstest.ww
View File

@@ -1096,28 +1096,28 @@ fn streq(a: str, b: str) bool = {
assert(!(!streq(strings.iterstr(&rit), "he"))); assert(!(!streq(strings.iterstr(&rit), "he")));
}; };
// ---- tokenize / rtokenize / peek_token / remaining_tokens ----------- // ---- tokenize / rtokenize / peektoken / remainingtokens -----------
// ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers // ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers
// mirror lib/bytes/bytestest expect_token / expect_done but compare // mirror lib/bytes/bytestest expect_token / expect_done but compare
// str via streq. // str via streq.
fn expect_str_token(t: *strings.tokenizer, want: str) void = { fn expect_str_token(t: *strings.tokenizer, want: str) void = {
match (strings.peek_token(t)) { match (strings.peektoken(t)) {
case let p: str => { assert(!(!streq(p, want))); }; case let p: str => { assert(!(!streq(p, want))); };
case bytes.done => { abort(); }; case bytes.done => { abort(); };
}; };
match (strings.next_token(t)) { match (strings.nexttoken(t)) {
case let n: str => { assert(!(!streq(n, want))); }; case let n: str => { assert(!(!streq(n, want))); };
case bytes.done => { abort(); }; case bytes.done => { abort(); };
}; };
}; };
fn expect_str_done(t: *strings.tokenizer) void = { fn expect_str_done(t: *strings.tokenizer) void = {
match (strings.peek_token(t)) { match (strings.peektoken(t)) {
case let p: str => { abort(); }; case let p: str => { abort(); };
case bytes.done => void; case bytes.done => void;
}; };
match (strings.next_token(t)) { match (strings.nexttoken(t)) {
case let n: str => { abort(); }; case let n: str => { abort(); };
case bytes.done => void; case bytes.done => void;
}; };
@@ -1177,7 +1177,7 @@ fn expect_str_done(t: *strings.tokenizer) void = {
}; };
@test fn rtokenize_cases() void = { @test fn rtokenize_cases() void = {
// Reverse direction — first next_token is the last token. // Reverse direction — first nexttoken is the last token.
let t: strings.tokenizer = strings.rtokenize( let t: strings.tokenizer = strings.rtokenize(
"Hello world! My name is Harriet.", " "); "Hello world! My name is Harriet.", " ");
expect_str_token(&t, "Harriet."); expect_str_token(&t, "Harriet.");
@@ -1200,55 +1200,55 @@ fn expect_str_done(t: *strings.tokenizer) void = {
expect_str_done(&t3); expect_str_done(&t3);
}; };
@test fn peek_token_cases() void = { @test fn peektoken_cases() void = {
// Two peeks without advancing return the same token. // Two peeks without advancing return the same token.
let t: strings.tokenizer = strings.tokenize("a b c", " "); let t: strings.tokenizer = strings.tokenize("a b c", " ");
match (strings.peek_token(&t)) { match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "a"))); }; case let p: str => { assert(!(!streq(p, "a"))); };
case bytes.done => { abort(); }; case bytes.done => { abort(); };
}; };
match (strings.peek_token(&t)) { match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "a"))); }; case let p: str => { assert(!(!streq(p, "a"))); };
case bytes.done => { abort(); }; case bytes.done => { abort(); };
}; };
// Advance once — peek then returns "b". // Advance once — peek then returns "b".
match (strings.next_token(&t)) { match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "a"))); }; case let n: str => { assert(!(!streq(n, "a"))); };
case bytes.done => { abort(); }; case bytes.done => { abort(); };
}; };
match (strings.peek_token(&t)) { match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "b"))); }; case let p: str => { assert(!(!streq(p, "b"))); };
case bytes.done => { abort(); }; case bytes.done => { abort(); };
}; };
// Empty input — peek is done. // Empty input — peek is done.
let t2: strings.tokenizer = strings.tokenize("", " "); let t2: strings.tokenizer = strings.tokenize("", " ");
match (strings.peek_token(&t2)) { match (strings.peektoken(&t2)) {
case let p: str => { abort(); }; case let p: str => { abort(); };
case bytes.done => void; case bytes.done => void;
}; };
}; };
@test fn remaining_tokens_cases() void = { @test fn remainingtokens_cases() void = {
// ref/hare/strings/tokenize.ha:157. After 2 next_tokens, remaining // ref/hare/strings/tokenize.ha:157. After 2 nexttokens, remaining
// is "My name is Harriet.". // is "My name is Harriet.".
let t: strings.tokenizer = strings.tokenize( let t: strings.tokenizer = strings.tokenize(
"Hello world! My name is Harriet.", " "); "Hello world! My name is Harriet.", " ");
match (strings.next_token(&t)) { match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "Hello"))); }; case let n: str => { assert(!(!streq(n, "Hello"))); };
case bytes.done => { abort(); }; case bytes.done => { abort(); };
}; };
match (strings.next_token(&t)) { match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "world!"))); }; case let n: str => { assert(!(!streq(n, "world!"))); };
case bytes.done => { abort(); }; case bytes.done => { abort(); };
}; };
if (!streq(strings.remaining_tokens(&t), "My name is Harriet.")) { if (!streq(strings.remainingtokens(&t), "My name is Harriet.")) {
abort(); abort();
}; };
// Fresh tokenizer — remaining_tokens is the whole input. // Fresh tokenizer — remainingtokens is the whole input.
let t2: strings.tokenizer = strings.tokenize("a b c", " "); let t2: strings.tokenizer = strings.tokenize("a b c", " ");
assert(!(!streq(strings.remaining_tokens(&t2), "a b c"))); assert(!(!streq(strings.remainingtokens(&t2), "a b c")));
}; };
// ---- splitn / rsplitn / split ---------------------------------------- // ---- splitn / rsplitn / split ----------------------------------------
@@ -1338,7 +1338,7 @@ fn expect_str(toks: []str, i: i32, want: str) void = {
expect_str(t3, 0, "a b c"); expect_str(t3, 0, "a b c");
os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64); os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64);
// delim absent — first next_token yields the entire input as the // delim absent — first nexttoken yields the entire input as the
// sole token; second iter sees done and short-circuits with the // sole token; second iter sees done and short-circuits with the
// 1-elem toks un-reversed (single element, reverse is a no-op). // 1-elem toks un-reversed (single element, reverse is a no-op).
let t4: []str = strings.rsplitn("abc", "=", 5); let t4: []str = strings.rsplitn("abc", "=", 5);

68
selfhost/cmd/w6a/main.combined.ww
View File

@@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent. // Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse` // - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings // ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical. // `move`. Outwardly identical.
@@ -925,7 +925,7 @@ package bytes;
import os; import os;
import types; import types;
// done — iteration sentinel returned by next_token / peek_token at // done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does // token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. // (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// rtokenize — reverse-direction tokenize. First next_token yields the // rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. // last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = { export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX, assert((in.len: i64) < types.I64_MAX,
@@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// peek_token — next token without advancing the cursor. Returns done // peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token. // once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91. // ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = { export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) { if (s.delim.len == 0) {
let d: done; return d; let d: done; return d;
}; };
@@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r; return r;
}; };
// next_token — current token, then advance past it and the delim. // nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so // Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = { export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8; let b: []u8;
match (peek_token(s)) { match (peektoken(s)) {
case let v: []u8 => { b = v; }; case let v: []u8 => { b = v; };
case done => { let d: done; return d; }; case done => { let d: done; return d; };
}; };
@@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b; return b;
}; };
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. // remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145. // ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = { export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in; return s.in;
}; };
@@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...); let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...); let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
}; };
// rtokenize — reverse-direction counterpart to [[tokenize]]. First // rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first. // nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44. // ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = { export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim); let d: []u8 = toutf8(delim);
@@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...); return bytes.rtokenize(toutf8(s), d...);
}; };
// next_token — current token, advancing the cursor. // nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62. // ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = { export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) { match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// peek_token — current token without advancing. // peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71. // ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = { export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) { match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// remaining_tokens — unconsumed portion of the input ahead of the // remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79. // cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = { export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b)); return frombytes(bytes.remainingtokens(b));
}; };
// cut — split `in` along the first instance of `delim`, returning the // cut — split `in` along the first instance of `delim`, returning the
@@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim); let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim); let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };

68
selfhost/cmd/w6c/main.combined.ww
View File

@@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent. // Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse` // - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings // ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical. // `move`. Outwardly identical.
@@ -925,7 +925,7 @@ package bytes;
import os; import os;
import types; import types;
// done — iteration sentinel returned by next_token / peek_token at // done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does // token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. // (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// rtokenize — reverse-direction tokenize. First next_token yields the // rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. // last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = { export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX, assert((in.len: i64) < types.I64_MAX,
@@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// peek_token — next token without advancing the cursor. Returns done // peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token. // once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91. // ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = { export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) { if (s.delim.len == 0) {
let d: done; return d; let d: done; return d;
}; };
@@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r; return r;
}; };
// next_token — current token, then advance past it and the delim. // nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so // Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = { export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8; let b: []u8;
match (peek_token(s)) { match (peektoken(s)) {
case let v: []u8 => { b = v; }; case let v: []u8 => { b = v; };
case done => { let d: done; return d; }; case done => { let d: done; return d; };
}; };
@@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b; return b;
}; };
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. // remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145. // ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = { export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in; return s.in;
}; };
@@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...); let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...); let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
}; };
// rtokenize — reverse-direction counterpart to [[tokenize]]. First // rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first. // nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44. // ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = { export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim); let d: []u8 = toutf8(delim);
@@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...); return bytes.rtokenize(toutf8(s), d...);
}; };
// next_token — current token, advancing the cursor. // nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62. // ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = { export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) { match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// peek_token — current token without advancing. // peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71. // ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = { export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) { match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// remaining_tokens — unconsumed portion of the input ahead of the // remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79. // cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = { export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b)); return frombytes(bytes.remainingtokens(b));
}; };
// cut — split `in` along the first instance of `delim`, returning the // cut — split `in` along the first instance of `delim`, returning the
@@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim); let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim); let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };

68
selfhost/cmd/w6l/main.combined.ww
View File

@@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent. // Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse` // - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings // ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical. // `move`. Outwardly identical.
@@ -925,7 +925,7 @@ package bytes;
import os; import os;
import types; import types;
// done — iteration sentinel returned by next_token / peek_token at // done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does // token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. // (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// rtokenize — reverse-direction tokenize. First next_token yields the // rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. // last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = { export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX, assert((in.len: i64) < types.I64_MAX,
@@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// peek_token — next token without advancing the cursor. Returns done // peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token. // once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91. // ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = { export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) { if (s.delim.len == 0) {
let d: done; return d; let d: done; return d;
}; };
@@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r; return r;
}; };
// next_token — current token, then advance past it and the delim. // nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so // Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = { export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8; let b: []u8;
match (peek_token(s)) { match (peektoken(s)) {
case let v: []u8 => { b = v; }; case let v: []u8 => { b = v; };
case done => { let d: done; return d; }; case done => { let d: done; return d; };
}; };
@@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b; return b;
}; };
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. // remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145. // ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = { export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in; return s.in;
}; };
@@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...); let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...); let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
}; };
// rtokenize — reverse-direction counterpart to [[tokenize]]. First // rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first. // nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44. // ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = { export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim); let d: []u8 = toutf8(delim);
@@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...); return bytes.rtokenize(toutf8(s), d...);
}; };
// next_token — current token, advancing the cursor. // nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62. // ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = { export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) { match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// peek_token — current token without advancing. // peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71. // ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = { export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) { match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// remaining_tokens — unconsumed portion of the input ahead of the // remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79. // cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = { export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b)); return frombytes(bytes.remainingtokens(b));
}; };
// cut — split `in` along the first instance of `delim`, returning the // cut — split `in` along the first instance of `delim`, returning the
@@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim); let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim); let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };

68
selfhost/cmd/ww/main.combined.ww
View File

@@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent. // Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse` // - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings // ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical. // `move`. Outwardly identical.
@@ -925,7 +925,7 @@ package bytes;
import os; import os;
import types; import types;
// done — iteration sentinel returned by next_token / peek_token at // done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does // token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. // (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// rtokenize — reverse-direction tokenize. First next_token yields the // rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. // last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = { export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX, assert((in.len: i64) < types.I64_MAX,
@@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// peek_token — next token without advancing the cursor. Returns done // peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token. // once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91. // ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = { export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) { if (s.delim.len == 0) {
let d: done; return d; let d: done; return d;
}; };
@@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r; return r;
}; };
// next_token — current token, then advance past it and the delim. // nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so // Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = { export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8; let b: []u8;
match (peek_token(s)) { match (peektoken(s)) {
case let v: []u8 => { b = v; }; case let v: []u8 => { b = v; };
case done => { let d: done; return d; }; case done => { let d: done; return d; };
}; };
@@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b; return b;
}; };
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. // remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145. // ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = { export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in; return s.in;
}; };
@@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...); let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...); let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
}; };
// rtokenize — reverse-direction counterpart to [[tokenize]]. First // rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first. // nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44. // ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = { export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim); let d: []u8 = toutf8(delim);
@@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...); return bytes.rtokenize(toutf8(s), d...);
}; };
// next_token — current token, advancing the cursor. // nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62. // ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = { export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) { match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// peek_token — current token without advancing. // peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71. // ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = { export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) { match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// remaining_tokens — unconsumed portion of the input ahead of the // remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79. // cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = { export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b)); return frombytes(bytes.remainingtokens(b));
}; };
// cut — split `in` along the first instance of `delim`, returning the // cut — split `in` along the first instance of `delim`, returning the
@@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim); let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim); let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };

68
selfhost/cmd/wwdump/main.combined.ww
View File

@@ -2426,7 +2426,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent. // Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse` // - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings // ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical. // `move`. Outwardly identical.
@@ -2442,7 +2442,7 @@ package bytes;
import os; import os;
import types; import types;
// done — iteration sentinel returned by next_token / peek_token at // done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does // token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. // (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -2661,8 +2661,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// rtokenize — reverse-direction tokenize. First next_token yields the // rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. // last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = { export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX, assert((in.len: i64) < types.I64_MAX,
@@ -2678,10 +2678,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// peek_token — next token without advancing the cursor. Returns done // peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token. // once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91. // ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = { export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) { if (s.delim.len == 0) {
let d: done; return d; let d: done; return d;
}; };
@@ -2757,12 +2757,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r; return r;
}; };
// next_token — current token, then advance past it and the delim. // nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so // Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = { export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8; let b: []u8;
match (peek_token(s)) { match (peektoken(s)) {
case let v: []u8 => { b = v; }; case let v: []u8 => { b = v; };
case done => { let d: done; return d; }; case done => { let d: done; return d; };
}; };
@@ -2799,9 +2799,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b; return b;
}; };
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. // remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145. // ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = { export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in; return s.in;
}; };
@@ -2828,16 +2828,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...); let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2866,16 +2866,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...); let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -4106,7 +4106,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
}; };
// rtokenize — reverse-direction counterpart to [[tokenize]]. First // rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first. // nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44. // ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = { export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim); let d: []u8 = toutf8(delim);
@@ -4119,31 +4119,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...); return bytes.rtokenize(toutf8(s), d...);
}; };
// next_token — current token, advancing the cursor. // nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62. // ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = { export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) { match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// peek_token — current token without advancing. // peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71. // ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = { export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) { match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// remaining_tokens — unconsumed portion of the input ahead of the // remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79. // cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = { export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b)); return frombytes(bytes.remainingtokens(b));
}; };
// cut — split `in` along the first instance of `delim`, returning the // cut — split `in` along the first instance of `delim`, returning the
@@ -4184,16 +4184,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim); let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -4218,16 +4218,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim); let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };

68
selfhost/test/smoke.combined.ww
View File

@@ -2426,7 +2426,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent. // Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse` // - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings // ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical. // `move`. Outwardly identical.
@@ -2442,7 +2442,7 @@ package bytes;
import os; import os;
import types; import types;
// done — iteration sentinel returned by next_token / peek_token at // done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does // token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. // (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -2661,8 +2661,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// rtokenize — reverse-direction tokenize. First next_token yields the // rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. // last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = { export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX, assert((in.len: i64) < types.I64_MAX,
@@ -2678,10 +2678,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t; return t;
}; };
// peek_token — next token without advancing the cursor. Returns done // peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token. // once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91. // ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = { export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) { if (s.delim.len == 0) {
let d: done; return d; let d: done; return d;
}; };
@@ -2757,12 +2757,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r; return r;
}; };
// next_token — current token, then advance past it and the delim. // nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so // Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = { export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8; let b: []u8;
match (peek_token(s)) { match (peektoken(s)) {
case let v: []u8 => { b = v; }; case let v: []u8 => { b = v; };
case done => { let d: done; return d; }; case done => { let d: done; return d; };
}; };
@@ -2799,9 +2799,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b; return b;
}; };
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. // remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145. // ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = { export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in; return s.in;
}; };
@@ -2828,16 +2828,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...); let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -2866,16 +2866,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...); let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); }; case let s: []u8 => { append(toks, s); };
case done => { return toks; }; case done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case done => void; case done => void;
case let pk: []u8 => { case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok); let r: []u8 = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -4106,7 +4106,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
}; };
// rtokenize — reverse-direction counterpart to [[tokenize]]. First // rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first. // nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44. // ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = { export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim); let d: []u8 = toutf8(delim);
@@ -4119,31 +4119,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...); return bytes.rtokenize(toutf8(s), d...);
}; };
// next_token — current token, advancing the cursor. // nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62. // ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = { export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) { match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// peek_token — current token without advancing. // peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71. // ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = { export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) { match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v); case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; }; case bytes.done => { let d: bytes.done; return d; };
}; };
}; };
// remaining_tokens — unconsumed portion of the input ahead of the // remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79. // cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = { export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer; let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b)); return frombytes(bytes.remainingtokens(b));
}; };
// cut — split `in` along the first instance of `delim`, returning the // cut — split `in` along the first instance of `delim`, returning the
@@ -4184,16 +4184,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim); let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };
@@ -4218,16 +4218,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim); let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0; let i: i32 = 0;
for (i < n - 1) { for (i < n - 1) {
match (next_token(&tok)) { match (nexttoken(&tok)) {
case let s: str => { append(toks, s); }; case let s: str => { append(toks, s); };
case bytes.done => { return toks; }; case bytes.done => { return toks; };
}; };
i += 1; i += 1;
}; };
match (peek_token(&tok)) { match (peektoken(&tok)) {
case bytes.done => void; case bytes.done => void;
case let pk: str => { case let pk: str => {
let r: str = remaining_tokens(&tok); let r: str = remainingtokens(&tok);
append(toks, r); append(toks, r);
}; };
}; };