Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wcc/ww: drop underscores from next/peek/remaining-tokens (F-Z)

Browse Source
This commit is contained in:
Hojun-Cho
2026-06-15 04:31:16 +09:00
parent 5ae3787cb9
commit 04d35c25c3
10 changed files with 278 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
lib/bytes/bytes.ww
View File

@@ -7,7 +7,7 @@
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
@@ -23,7 +23,7 @@ package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -242,8 +242,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
// rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX,
@@ -259,10 +259,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
@@ -338,12 +338,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r;
};
// next_token — current token, then advance past it and the delim.
// nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
match (peektoken(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
@@ -380,9 +380,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in;
};
@@ -409,16 +409,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -447,16 +447,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};

40
lib/bytes/bytestest.ww
View File

@@ -237,7 +237,7 @@ import os;
assert(!(!bytes.hassuffix(a4[0:4], s234[0:3])));
};
// ---- tokenize / rtokenize / peek_token / remaining_tokens -----------
// ---- tokenize / rtokenize / peektoken / remainingtokens -----------
// ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize
// drives the iterator through an expected-token sequence and asserts
// `equal(p, n)` (peek == next), `equal(n, want)` (next == expected).
@@ -245,13 +245,13 @@ import os;
// expect_token — table row driver. Advances `t` once, asserts the
// returned token matches `want`. peek invariant: peek must equal next.
fn expect_token(t: *bytes.tokenizer, want: []u8) void = {
match (bytes.peek_token(t)) {
match (bytes.peektoken(t)) {
case let p: []u8 => {
assert(!(!bytes.equal(p, want)));
};
case bytes.done => abort();
};
match (bytes.next_token(t)) {
match (bytes.nexttoken(t)) {
case let n: []u8 => {
assert(!(!bytes.equal(n, want)));
};
@@ -261,11 +261,11 @@ fn expect_token(t: *bytes.tokenizer, want: []u8) void = {
// expect_done — table-row driver. peek and next must both be done.
fn expect_done(t: *bytes.tokenizer) void = {
match (bytes.peek_token(t)) {
match (bytes.peektoken(t)) {
case let p: []u8 => abort();
case bytes.done => void;
};
match (bytes.next_token(t)) {
match (bytes.nexttoken(t)) {
case let n: []u8 => abort();
case bytes.done => void;
};
@@ -381,63 +381,63 @@ fn expect_done(t: *bytes.tokenizer) void = {
expect_done(&t6);
};
@test fn peek_token_cases() void = {
@test fn peektoken_cases() void = {
// Peeking twice without advancing returns the same token.
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.peek_token(&t)) {
match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); };
case bytes.done => abort();
};
match (bytes.peek_token(&t)) {
match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); };
case bytes.done => abort();
};
// Then advance once — peek-after-next is the second token.
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
match (bytes.next_token(&t)) {
match (bytes.nexttoken(&t)) {
case let n: []u8 => { assert(!(!bytes.equal(n, e_12[0:2]))); };
case bytes.done => abort();
};
match (bytes.peek_token(&t)) {
match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort();
};
// Reverse peek symmetry — first peek is last token.
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.peek_token(&t2)) {
match (bytes.peektoken(&t2)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort();
};
match (bytes.peek_token(&t2)) {
match (bytes.peektoken(&t2)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort();
};
};
@test fn remaining_tokens_cases() void = {
// After one next_token, remaining is bytes past the consumed delim.
@test fn remainingtokens_cases() void = {
// After one nexttoken, remaining is bytes past the consumed delim.
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.next_token(&t)) {
match (bytes.nexttoken(&t)) {
case let n: []u8 => void;
case bytes.done => abort();
};
let r: []u8 = bytes.remaining_tokens(&t);
let r: []u8 = bytes.remainingtokens(&t);
assert(!(!bytes.equal(r, e_34[0:2])));
// Reverse — after one next_token, remaining is bytes before the
// Reverse — after one nexttoken, remaining is bytes before the
// consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2].
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.next_token(&t2)) {
match (bytes.nexttoken(&t2)) {
case let n: []u8 => void;
case bytes.done => abort();
};
let r2: []u8 = bytes.remaining_tokens(&t2);
let r2: []u8 = bytes.remainingtokens(&t2);
assert(!(!bytes.equal(r2, e_12[0:2])));
};
@@ -655,7 +655,7 @@ fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = {
// delim absent — done short-circuits immediately at iter 0;
// toks ends up holding only the eventual remainder if n=1
// (the loop never runs and peek picks up the full input).
// With n>1 the loop's first next_token sees done (no delim
// With n>1 the loop's first nexttoken sees done (no delim
// match anywhere) and returns toks={} per Hare's early-exit.
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5);

32
lib/strings/strings.ww
View File

@@ -679,7 +679,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -692,31 +692,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -757,16 +757,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -791,16 +791,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};

40
lib/strings/stringstest.ww
View File

@@ -1096,28 +1096,28 @@ fn streq(a: str, b: str) bool = {
assert(!(!streq(strings.iterstr(&rit), "he")));
};
// ---- tokenize / rtokenize / peek_token / remaining_tokens -----------
// ---- tokenize / rtokenize / peektoken / remainingtokens -----------
// ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers
// mirror lib/bytes/bytestest expect_token / expect_done but compare
// str via streq.
fn expect_str_token(t: *strings.tokenizer, want: str) void = {
match (strings.peek_token(t)) {
match (strings.peektoken(t)) {
case let p: str => { assert(!(!streq(p, want))); };
case bytes.done => { abort(); };
};
match (strings.next_token(t)) {
match (strings.nexttoken(t)) {
case let n: str => { assert(!(!streq(n, want))); };
case bytes.done => { abort(); };
};
};
fn expect_str_done(t: *strings.tokenizer) void = {
match (strings.peek_token(t)) {
match (strings.peektoken(t)) {
case let p: str => { abort(); };
case bytes.done => void;
};
match (strings.next_token(t)) {
match (strings.nexttoken(t)) {
case let n: str => { abort(); };
case bytes.done => void;
};
@@ -1177,7 +1177,7 @@ fn expect_str_done(t: *strings.tokenizer) void = {
};
@test fn rtokenize_cases() void = {
// Reverse direction — first next_token is the last token.
// Reverse direction — first nexttoken is the last token.
let t: strings.tokenizer = strings.rtokenize(
"Hello world! My name is Harriet.", " ");
expect_str_token(&t, "Harriet.");
@@ -1200,55 +1200,55 @@ fn expect_str_done(t: *strings.tokenizer) void = {
expect_str_done(&t3);
};
@test fn peek_token_cases() void = {
@test fn peektoken_cases() void = {
// Two peeks without advancing return the same token.
let t: strings.tokenizer = strings.tokenize("a b c", " ");
match (strings.peek_token(&t)) {
match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "a"))); };
case bytes.done => { abort(); };
};
match (strings.peek_token(&t)) {
match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "a"))); };
case bytes.done => { abort(); };
};
// Advance once — peek then returns "b".
match (strings.next_token(&t)) {
match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "a"))); };
case bytes.done => { abort(); };
};
match (strings.peek_token(&t)) {
match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "b"))); };
case bytes.done => { abort(); };
};
// Empty input — peek is done.
let t2: strings.tokenizer = strings.tokenize("", " ");
match (strings.peek_token(&t2)) {
match (strings.peektoken(&t2)) {
case let p: str => { abort(); };
case bytes.done => void;
};
};
@test fn remaining_tokens_cases() void = {
// ref/hare/strings/tokenize.ha:157. After 2 next_tokens, remaining
@test fn remainingtokens_cases() void = {
// ref/hare/strings/tokenize.ha:157. After 2 nexttokens, remaining
// is "My name is Harriet.".
let t: strings.tokenizer = strings.tokenize(
"Hello world! My name is Harriet.", " ");
match (strings.next_token(&t)) {
match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "Hello"))); };
case bytes.done => { abort(); };
};
match (strings.next_token(&t)) {
match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "world!"))); };
case bytes.done => { abort(); };
};
if (!streq(strings.remaining_tokens(&t), "My name is Harriet.")) {
if (!streq(strings.remainingtokens(&t), "My name is Harriet.")) {
abort();
};
// Fresh tokenizer — remaining_tokens is the whole input.
// Fresh tokenizer — remainingtokens is the whole input.
let t2: strings.tokenizer = strings.tokenize("a b c", " ");
assert(!(!streq(strings.remaining_tokens(&t2), "a b c")));
assert(!(!streq(strings.remainingtokens(&t2), "a b c")));
};
// ---- splitn / rsplitn / split ----------------------------------------
@@ -1338,7 +1338,7 @@ fn expect_str(toks: []str, i: i32, want: str) void = {
expect_str(t3, 0, "a b c");
os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64);
// delim absent — first next_token yields the entire input as the
// delim absent — first nexttoken yields the entire input as the
// sole token; second iter sees done and short-circuits with the
// 1-elem toks un-reversed (single element, reverse is a no-op).
let t4: []str = strings.rsplitn("abc", "=", 5);

68
selfhost/cmd/w6a/main.combined.ww
View File

@@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
@@ -925,7 +925,7 @@ package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
// rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX,
@@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
@@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r;
};
// next_token — current token, then advance past it and the delim.
// nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
match (peektoken(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
@@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in;
};
@@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};

68
selfhost/cmd/w6c/main.combined.ww
View File

@@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
@@ -925,7 +925,7 @@ package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
// rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX,
@@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
@@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r;
};
// next_token — current token, then advance past it and the delim.
// nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
match (peektoken(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
@@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in;
};
@@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};

68
selfhost/cmd/w6l/main.combined.ww
View File

@@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
@@ -925,7 +925,7 @@ package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
// rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX,
@@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
@@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r;
};
// next_token — current token, then advance past it and the delim.
// nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
match (peektoken(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
@@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in;
};
@@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};

68
selfhost/cmd/ww/main.combined.ww
View File

@@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
@@ -925,7 +925,7 @@ package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
// rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX,
@@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
@@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r;
};
// next_token — current token, then advance past it and the delim.
// nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
match (peektoken(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
@@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in;
};
@@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};

68
selfhost/cmd/wwdump/main.combined.ww
View File

@@ -2426,7 +2426,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
@@ -2442,7 +2442,7 @@ package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -2661,8 +2661,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
// rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX,
@@ -2678,10 +2678,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
@@ -2757,12 +2757,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r;
};
// next_token — current token, then advance past it and the delim.
// nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
match (peektoken(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
@@ -2799,9 +2799,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in;
};
@@ -2828,16 +2828,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2866,16 +2866,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -4106,7 +4106,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -4119,31 +4119,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -4184,16 +4184,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -4218,16 +4218,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};

68
selfhost/test/smoke.combined.ww
View File

@@ -2426,7 +2426,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
@@ -2442,7 +2442,7 @@ package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -2661,8 +2661,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
// rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX,
@@ -2678,10 +2678,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
@@ -2757,12 +2757,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r;
};
// next_token — current token, then advance past it and the delim.
// nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
match (peektoken(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
@@ -2799,9 +2799,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in;
};
@@ -2828,16 +2828,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2866,16 +2866,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -4106,7 +4106,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -4119,31 +4119,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -4184,16 +4184,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -4218,16 +4218,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};