From 04d35c25c359c5268a13540b1987ba674fb571df Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Mon, 15 Jun 2026 04:31:16 +0900 Subject: [PATCH] wcc/ww: drop underscores from next/peek/remaining-tokens (F-Z) --- lib/bytes/bytes.ww | 36 +++++++-------- lib/bytes/bytestest.ww | 40 ++++++++-------- lib/strings/strings.ww | 32 ++++++------- lib/strings/stringstest.ww | 40 ++++++++-------- selfhost/cmd/w6a/main.combined.ww | 68 ++++++++++++++-------------- selfhost/cmd/w6c/main.combined.ww | 68 ++++++++++++++-------------- selfhost/cmd/w6l/main.combined.ww | 68 ++++++++++++++-------------- selfhost/cmd/ww/main.combined.ww | 68 ++++++++++++++-------------- selfhost/cmd/wwdump/main.combined.ww | 68 ++++++++++++++-------------- selfhost/test/smoke.combined.ww | 68 ++++++++++++++-------------- 10 files changed, 278 insertions(+), 278 deletions(-) diff --git a/lib/bytes/bytes.ww b/lib/bytes/bytes.ww index e72fc793..a9029d5b 100644 --- a/lib/bytes/bytes.ww +++ b/lib/bytes/bytes.ww @@ -7,7 +7,7 @@ // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. -// - peek_token dispatches index/rindex by branching on `reverse` +// - peektoken dispatches index/rindex by branching on `reverse` // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // ww has no fn pointers in scope yet — same pattern as lib/strings // `move`. Outwardly identical. @@ -23,7 +23,7 @@ package bytes; import os; import types; -// done — iteration sentinel returned by next_token / peek_token at +// done — iteration sentinel returned by nexttoken / peektoken at // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // token; ww spells it per-package the same way lib/encoding/utf8 does // (utf8.ww:36). Plain `void` (not `!void`): continuation signal. @@ -242,8 +242,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// rtokenize — reverse-direction tokenize. First next_token yields the -// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +// rtokenize — reverse-direction tokenize. First nexttoken yields the +// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40. export fn rtokenize(in: []u8, delim: u8...) tokenizer = { assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert((in.len: i64) < types.I64_MAX, @@ -259,10 +259,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// peek_token — next token without advancing the cursor. Returns done -// once `s.delim` has been zeroed by a prior past-end next_token. +// peektoken — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end nexttoken. // ref/hare/bytes/tokenize.ha:91. -export fn peek_token(s: *tokenizer) ([]u8 | done) = { +export fn peektoken(s: *tokenizer) ([]u8 | done) = { if (s.delim.len == 0) { let d: done; return d; }; @@ -338,12 +338,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = { return r; }; -// next_token — current token, then advance past it and the delim. +// nexttoken — current token, then advance past it and the delim. // Once the input is exhausted, returns done and zeros `s.delim` so // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. -export fn next_token(s: *tokenizer) ([]u8 | done) = { +export fn nexttoken(s: *tokenizer) ([]u8 | done) = { let b: []u8; - match (peek_token(s)) { + match (peektoken(s)) { case let v: []u8 => { b = v; }; case done => { let d: done; return d; }; }; @@ -380,9 +380,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = { return b; }; -// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// remainingtokens — the unconsumed portion of `s.in`. Read-only view. // ref/hare/bytes/tokenize.ha:145. -export fn remaining_tokens(s: *tokenizer) []u8 = { +export fn remainingtokens(s: *tokenizer) []u8 = { return s.in; }; @@ -409,16 +409,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = tokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -447,16 +447,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = rtokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; diff --git a/lib/bytes/bytestest.ww b/lib/bytes/bytestest.ww index ef1da824..89f43eaa 100644 --- a/lib/bytes/bytestest.ww +++ b/lib/bytes/bytestest.ww @@ -237,7 +237,7 @@ import os; assert(!(!bytes.hassuffix(a4[0:4], s234[0:3]))); }; -// ---- tokenize / rtokenize / peek_token / remaining_tokens ----------- +// ---- tokenize / rtokenize / peektoken / remainingtokens ----------- // ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize // drives the iterator through an expected-token sequence and asserts // `equal(p, n)` (peek == next), `equal(n, want)` (next == expected). @@ -245,13 +245,13 @@ import os; // expect_token — table row driver. Advances `t` once, asserts the // returned token matches `want`. peek invariant: peek must equal next. fn expect_token(t: *bytes.tokenizer, want: []u8) void = { - match (bytes.peek_token(t)) { + match (bytes.peektoken(t)) { case let p: []u8 => { assert(!(!bytes.equal(p, want))); }; case bytes.done => abort(); }; - match (bytes.next_token(t)) { + match (bytes.nexttoken(t)) { case let n: []u8 => { assert(!(!bytes.equal(n, want))); }; @@ -261,11 +261,11 @@ fn expect_token(t: *bytes.tokenizer, want: []u8) void = { // expect_done — table-row driver. peek and next must both be done. fn expect_done(t: *bytes.tokenizer) void = { - match (bytes.peek_token(t)) { + match (bytes.peektoken(t)) { case let p: []u8 => abort(); case bytes.done => void; }; - match (bytes.next_token(t)) { + match (bytes.nexttoken(t)) { case let n: []u8 => abort(); case bytes.done => void; }; @@ -381,63 +381,63 @@ fn expect_done(t: *bytes.tokenizer) void = { expect_done(&t6); }; -@test fn peek_token_cases() void = { +@test fn peektoken_cases() void = { // Peeking twice without advancing returns the same token. let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); - match (bytes.peek_token(&t)) { + match (bytes.peektoken(&t)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); }; case bytes.done => abort(); }; - match (bytes.peek_token(&t)) { + match (bytes.peektoken(&t)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); }; case bytes.done => abort(); }; // Then advance once — peek-after-next is the second token. let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; - match (bytes.next_token(&t)) { + match (bytes.nexttoken(&t)) { case let n: []u8 => { assert(!(!bytes.equal(n, e_12[0:2]))); }; case bytes.done => abort(); }; - match (bytes.peek_token(&t)) { + match (bytes.peektoken(&t)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case bytes.done => abort(); }; // Reverse peek symmetry — first peek is last token. let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); - match (bytes.peek_token(&t2)) { + match (bytes.peektoken(&t2)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case bytes.done => abort(); }; - match (bytes.peek_token(&t2)) { + match (bytes.peektoken(&t2)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case bytes.done => abort(); }; }; -@test fn remaining_tokens_cases() void = { - // After one next_token, remaining is bytes past the consumed delim. +@test fn remainingtokens_cases() void = { + // After one nexttoken, remaining is bytes past the consumed delim. let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); - match (bytes.next_token(&t)) { + match (bytes.nexttoken(&t)) { case let n: []u8 => void; case bytes.done => abort(); }; - let r: []u8 = bytes.remaining_tokens(&t); + let r: []u8 = bytes.remainingtokens(&t); assert(!(!bytes.equal(r, e_34[0:2]))); - // Reverse — after one next_token, remaining is bytes before the + // Reverse — after one nexttoken, remaining is bytes before the // consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2]. let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); - match (bytes.next_token(&t2)) { + match (bytes.nexttoken(&t2)) { case let n: []u8 => void; case bytes.done => abort(); }; - let r2: []u8 = bytes.remaining_tokens(&t2); + let r2: []u8 = bytes.remainingtokens(&t2); assert(!(!bytes.equal(r2, e_12[0:2]))); }; @@ -655,7 +655,7 @@ fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = { // delim absent — done short-circuits immediately at iter 0; // toks ends up holding only the eventual remainder if n=1 // (the loop never runs and peek picks up the full input). - // With n>1 the loop's first next_token sees done (no delim + // With n>1 the loop's first nexttoken sees done (no delim // match anywhere) and returns toks={} per Hare's early-exit. let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8; let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5); diff --git a/lib/strings/strings.ww b/lib/strings/strings.ww index f5440e03..7977f8c0 100644 --- a/lib/strings/strings.ww +++ b/lib/strings/strings.ww @@ -679,7 +679,7 @@ export fn tokenize(s: str, delim: str) tokenizer = { }; // rtokenize — reverse-direction counterpart to [[tokenize]]. First -// next_token yields the last token, last yields the first. +// nexttoken yields the last token, last yields the first. // ref/hare/strings/tokenize.ha:44. export fn rtokenize(s: str, delim: str) tokenizer = { let d: []u8 = toutf8(delim); @@ -692,31 +692,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = { return bytes.rtokenize(toutf8(s), d...); }; -// next_token — current token, advancing the cursor. +// nexttoken — current token, advancing the cursor. // ref/hare/strings/tokenize.ha:62. -export fn next_token(s: *tokenizer) (str | bytes.done) = { +export fn nexttoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.next_token(b)) { + match (bytes.nexttoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// peek_token — current token without advancing. +// peektoken — current token without advancing. // ref/hare/strings/tokenize.ha:71. -export fn peek_token(s: *tokenizer) (str | bytes.done) = { +export fn peektoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.peek_token(b)) { + match (bytes.peektoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// remaining_tokens — unconsumed portion of the input ahead of the +// remainingtokens — unconsumed portion of the input ahead of the // cursor. ref/hare/strings/tokenize.ha:79. -export fn remaining_tokens(s: *tokenizer) str = { +export fn remainingtokens(s: *tokenizer) str = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - return frombytes(bytes.remaining_tokens(b)); + return frombytes(bytes.remainingtokens(b)); }; // cut — split `in` along the first instance of `delim`, returning the @@ -757,16 +757,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = tokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; @@ -791,16 +791,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = rtokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; diff --git a/lib/strings/stringstest.ww b/lib/strings/stringstest.ww index 01f8f4fb..e8a46740 100644 --- a/lib/strings/stringstest.ww +++ b/lib/strings/stringstest.ww @@ -1096,28 +1096,28 @@ fn streq(a: str, b: str) bool = { assert(!(!streq(strings.iterstr(&rit), "he"))); }; -// ---- tokenize / rtokenize / peek_token / remaining_tokens ----------- +// ---- tokenize / rtokenize / peektoken / remainingtokens ----------- // ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers // mirror lib/bytes/bytestest expect_token / expect_done but compare // str via streq. fn expect_str_token(t: *strings.tokenizer, want: str) void = { - match (strings.peek_token(t)) { + match (strings.peektoken(t)) { case let p: str => { assert(!(!streq(p, want))); }; case bytes.done => { abort(); }; }; - match (strings.next_token(t)) { + match (strings.nexttoken(t)) { case let n: str => { assert(!(!streq(n, want))); }; case bytes.done => { abort(); }; }; }; fn expect_str_done(t: *strings.tokenizer) void = { - match (strings.peek_token(t)) { + match (strings.peektoken(t)) { case let p: str => { abort(); }; case bytes.done => void; }; - match (strings.next_token(t)) { + match (strings.nexttoken(t)) { case let n: str => { abort(); }; case bytes.done => void; }; @@ -1177,7 +1177,7 @@ fn expect_str_done(t: *strings.tokenizer) void = { }; @test fn rtokenize_cases() void = { - // Reverse direction — first next_token is the last token. + // Reverse direction — first nexttoken is the last token. let t: strings.tokenizer = strings.rtokenize( "Hello world! My name is Harriet.", " "); expect_str_token(&t, "Harriet."); @@ -1200,55 +1200,55 @@ fn expect_str_done(t: *strings.tokenizer) void = { expect_str_done(&t3); }; -@test fn peek_token_cases() void = { +@test fn peektoken_cases() void = { // Two peeks without advancing return the same token. let t: strings.tokenizer = strings.tokenize("a b c", " "); - match (strings.peek_token(&t)) { + match (strings.peektoken(&t)) { case let p: str => { assert(!(!streq(p, "a"))); }; case bytes.done => { abort(); }; }; - match (strings.peek_token(&t)) { + match (strings.peektoken(&t)) { case let p: str => { assert(!(!streq(p, "a"))); }; case bytes.done => { abort(); }; }; // Advance once — peek then returns "b". - match (strings.next_token(&t)) { + match (strings.nexttoken(&t)) { case let n: str => { assert(!(!streq(n, "a"))); }; case bytes.done => { abort(); }; }; - match (strings.peek_token(&t)) { + match (strings.peektoken(&t)) { case let p: str => { assert(!(!streq(p, "b"))); }; case bytes.done => { abort(); }; }; // Empty input — peek is done. let t2: strings.tokenizer = strings.tokenize("", " "); - match (strings.peek_token(&t2)) { + match (strings.peektoken(&t2)) { case let p: str => { abort(); }; case bytes.done => void; }; }; -@test fn remaining_tokens_cases() void = { - // ref/hare/strings/tokenize.ha:157. After 2 next_tokens, remaining +@test fn remainingtokens_cases() void = { + // ref/hare/strings/tokenize.ha:157. After 2 nexttokens, remaining // is "My name is Harriet.". let t: strings.tokenizer = strings.tokenize( "Hello world! My name is Harriet.", " "); - match (strings.next_token(&t)) { + match (strings.nexttoken(&t)) { case let n: str => { assert(!(!streq(n, "Hello"))); }; case bytes.done => { abort(); }; }; - match (strings.next_token(&t)) { + match (strings.nexttoken(&t)) { case let n: str => { assert(!(!streq(n, "world!"))); }; case bytes.done => { abort(); }; }; - if (!streq(strings.remaining_tokens(&t), "My name is Harriet.")) { + if (!streq(strings.remainingtokens(&t), "My name is Harriet.")) { abort(); }; - // Fresh tokenizer — remaining_tokens is the whole input. + // Fresh tokenizer — remainingtokens is the whole input. let t2: strings.tokenizer = strings.tokenize("a b c", " "); - assert(!(!streq(strings.remaining_tokens(&t2), "a b c"))); + assert(!(!streq(strings.remainingtokens(&t2), "a b c"))); }; // ---- splitn / rsplitn / split ---------------------------------------- @@ -1338,7 +1338,7 @@ fn expect_str(toks: []str, i: i32, want: str) void = { expect_str(t3, 0, "a b c"); os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64); - // delim absent — first next_token yields the entire input as the + // delim absent — first nexttoken yields the entire input as the // sole token; second iter sees done and short-circuits with the // 1-elem toks un-reversed (single element, reverse is a no-op). let t4: []str = strings.rsplitn("abc", "=", 5); diff --git a/selfhost/cmd/w6a/main.combined.ww b/selfhost/cmd/w6a/main.combined.ww index 70782b03..643d77ec 100644 --- a/selfhost/cmd/w6a/main.combined.ww +++ b/selfhost/cmd/w6a/main.combined.ww @@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0'; // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. -// - peek_token dispatches index/rindex by branching on `reverse` +// - peektoken dispatches index/rindex by branching on `reverse` // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // ww has no fn pointers in scope yet — same pattern as lib/strings // `move`. Outwardly identical. @@ -925,7 +925,7 @@ package bytes; import os; import types; -// done — iteration sentinel returned by next_token / peek_token at +// done — iteration sentinel returned by nexttoken / peektoken at // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // token; ww spells it per-package the same way lib/encoding/utf8 does // (utf8.ww:36). Plain `void` (not `!void`): continuation signal. @@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// rtokenize — reverse-direction tokenize. First next_token yields the -// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +// rtokenize — reverse-direction tokenize. First nexttoken yields the +// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40. export fn rtokenize(in: []u8, delim: u8...) tokenizer = { assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert((in.len: i64) < types.I64_MAX, @@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// peek_token — next token without advancing the cursor. Returns done -// once `s.delim` has been zeroed by a prior past-end next_token. +// peektoken — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end nexttoken. // ref/hare/bytes/tokenize.ha:91. -export fn peek_token(s: *tokenizer) ([]u8 | done) = { +export fn peektoken(s: *tokenizer) ([]u8 | done) = { if (s.delim.len == 0) { let d: done; return d; }; @@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = { return r; }; -// next_token — current token, then advance past it and the delim. +// nexttoken — current token, then advance past it and the delim. // Once the input is exhausted, returns done and zeros `s.delim` so // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. -export fn next_token(s: *tokenizer) ([]u8 | done) = { +export fn nexttoken(s: *tokenizer) ([]u8 | done) = { let b: []u8; - match (peek_token(s)) { + match (peektoken(s)) { case let v: []u8 => { b = v; }; case done => { let d: done; return d; }; }; @@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = { return b; }; -// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// remainingtokens — the unconsumed portion of `s.in`. Read-only view. // ref/hare/bytes/tokenize.ha:145. -export fn remaining_tokens(s: *tokenizer) []u8 = { +export fn remainingtokens(s: *tokenizer) []u8 = { return s.in; }; @@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = tokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = rtokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = { }; // rtokenize — reverse-direction counterpart to [[tokenize]]. First -// next_token yields the last token, last yields the first. +// nexttoken yields the last token, last yields the first. // ref/hare/strings/tokenize.ha:44. export fn rtokenize(s: str, delim: str) tokenizer = { let d: []u8 = toutf8(delim); @@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = { return bytes.rtokenize(toutf8(s), d...); }; -// next_token — current token, advancing the cursor. +// nexttoken — current token, advancing the cursor. // ref/hare/strings/tokenize.ha:62. -export fn next_token(s: *tokenizer) (str | bytes.done) = { +export fn nexttoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.next_token(b)) { + match (bytes.nexttoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// peek_token — current token without advancing. +// peektoken — current token without advancing. // ref/hare/strings/tokenize.ha:71. -export fn peek_token(s: *tokenizer) (str | bytes.done) = { +export fn peektoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.peek_token(b)) { + match (bytes.peektoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// remaining_tokens — unconsumed portion of the input ahead of the +// remainingtokens — unconsumed portion of the input ahead of the // cursor. ref/hare/strings/tokenize.ha:79. -export fn remaining_tokens(s: *tokenizer) str = { +export fn remainingtokens(s: *tokenizer) str = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - return frombytes(bytes.remaining_tokens(b)); + return frombytes(bytes.remainingtokens(b)); }; // cut — split `in` along the first instance of `delim`, returning the @@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = tokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; @@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = rtokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index 3c8e21bb..43cce039 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0'; // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. -// - peek_token dispatches index/rindex by branching on `reverse` +// - peektoken dispatches index/rindex by branching on `reverse` // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // ww has no fn pointers in scope yet — same pattern as lib/strings // `move`. Outwardly identical. @@ -925,7 +925,7 @@ package bytes; import os; import types; -// done — iteration sentinel returned by next_token / peek_token at +// done — iteration sentinel returned by nexttoken / peektoken at // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // token; ww spells it per-package the same way lib/encoding/utf8 does // (utf8.ww:36). Plain `void` (not `!void`): continuation signal. @@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// rtokenize — reverse-direction tokenize. First next_token yields the -// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +// rtokenize — reverse-direction tokenize. First nexttoken yields the +// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40. export fn rtokenize(in: []u8, delim: u8...) tokenizer = { assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert((in.len: i64) < types.I64_MAX, @@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// peek_token — next token without advancing the cursor. Returns done -// once `s.delim` has been zeroed by a prior past-end next_token. +// peektoken — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end nexttoken. // ref/hare/bytes/tokenize.ha:91. -export fn peek_token(s: *tokenizer) ([]u8 | done) = { +export fn peektoken(s: *tokenizer) ([]u8 | done) = { if (s.delim.len == 0) { let d: done; return d; }; @@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = { return r; }; -// next_token — current token, then advance past it and the delim. +// nexttoken — current token, then advance past it and the delim. // Once the input is exhausted, returns done and zeros `s.delim` so // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. -export fn next_token(s: *tokenizer) ([]u8 | done) = { +export fn nexttoken(s: *tokenizer) ([]u8 | done) = { let b: []u8; - match (peek_token(s)) { + match (peektoken(s)) { case let v: []u8 => { b = v; }; case done => { let d: done; return d; }; }; @@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = { return b; }; -// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// remainingtokens — the unconsumed portion of `s.in`. Read-only view. // ref/hare/bytes/tokenize.ha:145. -export fn remaining_tokens(s: *tokenizer) []u8 = { +export fn remainingtokens(s: *tokenizer) []u8 = { return s.in; }; @@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = tokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = rtokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = { }; // rtokenize — reverse-direction counterpart to [[tokenize]]. First -// next_token yields the last token, last yields the first. +// nexttoken yields the last token, last yields the first. // ref/hare/strings/tokenize.ha:44. export fn rtokenize(s: str, delim: str) tokenizer = { let d: []u8 = toutf8(delim); @@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = { return bytes.rtokenize(toutf8(s), d...); }; -// next_token — current token, advancing the cursor. +// nexttoken — current token, advancing the cursor. // ref/hare/strings/tokenize.ha:62. -export fn next_token(s: *tokenizer) (str | bytes.done) = { +export fn nexttoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.next_token(b)) { + match (bytes.nexttoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// peek_token — current token without advancing. +// peektoken — current token without advancing. // ref/hare/strings/tokenize.ha:71. -export fn peek_token(s: *tokenizer) (str | bytes.done) = { +export fn peektoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.peek_token(b)) { + match (bytes.peektoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// remaining_tokens — unconsumed portion of the input ahead of the +// remainingtokens — unconsumed portion of the input ahead of the // cursor. ref/hare/strings/tokenize.ha:79. -export fn remaining_tokens(s: *tokenizer) str = { +export fn remainingtokens(s: *tokenizer) str = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - return frombytes(bytes.remaining_tokens(b)); + return frombytes(bytes.remainingtokens(b)); }; // cut — split `in` along the first instance of `delim`, returning the @@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = tokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; @@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = rtokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; diff --git a/selfhost/cmd/w6l/main.combined.ww b/selfhost/cmd/w6l/main.combined.ww index 9e6120f2..3b4dad5c 100644 --- a/selfhost/cmd/w6l/main.combined.ww +++ b/selfhost/cmd/w6l/main.combined.ww @@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0'; // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. -// - peek_token dispatches index/rindex by branching on `reverse` +// - peektoken dispatches index/rindex by branching on `reverse` // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // ww has no fn pointers in scope yet — same pattern as lib/strings // `move`. Outwardly identical. @@ -925,7 +925,7 @@ package bytes; import os; import types; -// done — iteration sentinel returned by next_token / peek_token at +// done — iteration sentinel returned by nexttoken / peektoken at // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // token; ww spells it per-package the same way lib/encoding/utf8 does // (utf8.ww:36). Plain `void` (not `!void`): continuation signal. @@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// rtokenize — reverse-direction tokenize. First next_token yields the -// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +// rtokenize — reverse-direction tokenize. First nexttoken yields the +// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40. export fn rtokenize(in: []u8, delim: u8...) tokenizer = { assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert((in.len: i64) < types.I64_MAX, @@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// peek_token — next token without advancing the cursor. Returns done -// once `s.delim` has been zeroed by a prior past-end next_token. +// peektoken — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end nexttoken. // ref/hare/bytes/tokenize.ha:91. -export fn peek_token(s: *tokenizer) ([]u8 | done) = { +export fn peektoken(s: *tokenizer) ([]u8 | done) = { if (s.delim.len == 0) { let d: done; return d; }; @@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = { return r; }; -// next_token — current token, then advance past it and the delim. +// nexttoken — current token, then advance past it and the delim. // Once the input is exhausted, returns done and zeros `s.delim` so // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. -export fn next_token(s: *tokenizer) ([]u8 | done) = { +export fn nexttoken(s: *tokenizer) ([]u8 | done) = { let b: []u8; - match (peek_token(s)) { + match (peektoken(s)) { case let v: []u8 => { b = v; }; case done => { let d: done; return d; }; }; @@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = { return b; }; -// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// remainingtokens — the unconsumed portion of `s.in`. Read-only view. // ref/hare/bytes/tokenize.ha:145. -export fn remaining_tokens(s: *tokenizer) []u8 = { +export fn remainingtokens(s: *tokenizer) []u8 = { return s.in; }; @@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = tokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = rtokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = { }; // rtokenize — reverse-direction counterpart to [[tokenize]]. First -// next_token yields the last token, last yields the first. +// nexttoken yields the last token, last yields the first. // ref/hare/strings/tokenize.ha:44. export fn rtokenize(s: str, delim: str) tokenizer = { let d: []u8 = toutf8(delim); @@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = { return bytes.rtokenize(toutf8(s), d...); }; -// next_token — current token, advancing the cursor. +// nexttoken — current token, advancing the cursor. // ref/hare/strings/tokenize.ha:62. -export fn next_token(s: *tokenizer) (str | bytes.done) = { +export fn nexttoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.next_token(b)) { + match (bytes.nexttoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// peek_token — current token without advancing. +// peektoken — current token without advancing. // ref/hare/strings/tokenize.ha:71. -export fn peek_token(s: *tokenizer) (str | bytes.done) = { +export fn peektoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.peek_token(b)) { + match (bytes.peektoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// remaining_tokens — unconsumed portion of the input ahead of the +// remainingtokens — unconsumed portion of the input ahead of the // cursor. ref/hare/strings/tokenize.ha:79. -export fn remaining_tokens(s: *tokenizer) str = { +export fn remainingtokens(s: *tokenizer) str = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - return frombytes(bytes.remaining_tokens(b)); + return frombytes(bytes.remainingtokens(b)); }; // cut — split `in` along the first instance of `delim`, returning the @@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = tokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; @@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = rtokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; diff --git a/selfhost/cmd/ww/main.combined.ww b/selfhost/cmd/ww/main.combined.ww index b165c381..aac394e4 100644 --- a/selfhost/cmd/ww/main.combined.ww +++ b/selfhost/cmd/ww/main.combined.ww @@ -909,7 +909,7 @@ def RUNE_MIN: rune = '\0'; // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. -// - peek_token dispatches index/rindex by branching on `reverse` +// - peektoken dispatches index/rindex by branching on `reverse` // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // ww has no fn pointers in scope yet — same pattern as lib/strings // `move`. Outwardly identical. @@ -925,7 +925,7 @@ package bytes; import os; import types; -// done — iteration sentinel returned by next_token / peek_token at +// done — iteration sentinel returned by nexttoken / peektoken at // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // token; ww spells it per-package the same way lib/encoding/utf8 does // (utf8.ww:36). Plain `void` (not `!void`): continuation signal. @@ -1144,8 +1144,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// rtokenize — reverse-direction tokenize. First next_token yields the -// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +// rtokenize — reverse-direction tokenize. First nexttoken yields the +// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40. export fn rtokenize(in: []u8, delim: u8...) tokenizer = { assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert((in.len: i64) < types.I64_MAX, @@ -1161,10 +1161,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// peek_token — next token without advancing the cursor. Returns done -// once `s.delim` has been zeroed by a prior past-end next_token. +// peektoken — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end nexttoken. // ref/hare/bytes/tokenize.ha:91. -export fn peek_token(s: *tokenizer) ([]u8 | done) = { +export fn peektoken(s: *tokenizer) ([]u8 | done) = { if (s.delim.len == 0) { let d: done; return d; }; @@ -1240,12 +1240,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = { return r; }; -// next_token — current token, then advance past it and the delim. +// nexttoken — current token, then advance past it and the delim. // Once the input is exhausted, returns done and zeros `s.delim` so // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. -export fn next_token(s: *tokenizer) ([]u8 | done) = { +export fn nexttoken(s: *tokenizer) ([]u8 | done) = { let b: []u8; - match (peek_token(s)) { + match (peektoken(s)) { case let v: []u8 => { b = v; }; case done => { let d: done; return d; }; }; @@ -1282,9 +1282,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = { return b; }; -// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// remainingtokens — the unconsumed portion of `s.in`. Read-only view. // ref/hare/bytes/tokenize.ha:145. -export fn remaining_tokens(s: *tokenizer) []u8 = { +export fn remainingtokens(s: *tokenizer) []u8 = { return s.in; }; @@ -1311,16 +1311,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = tokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -1349,16 +1349,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = rtokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -2589,7 +2589,7 @@ export fn tokenize(s: str, delim: str) tokenizer = { }; // rtokenize — reverse-direction counterpart to [[tokenize]]. First -// next_token yields the last token, last yields the first. +// nexttoken yields the last token, last yields the first. // ref/hare/strings/tokenize.ha:44. export fn rtokenize(s: str, delim: str) tokenizer = { let d: []u8 = toutf8(delim); @@ -2602,31 +2602,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = { return bytes.rtokenize(toutf8(s), d...); }; -// next_token — current token, advancing the cursor. +// nexttoken — current token, advancing the cursor. // ref/hare/strings/tokenize.ha:62. -export fn next_token(s: *tokenizer) (str | bytes.done) = { +export fn nexttoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.next_token(b)) { + match (bytes.nexttoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// peek_token — current token without advancing. +// peektoken — current token without advancing. // ref/hare/strings/tokenize.ha:71. -export fn peek_token(s: *tokenizer) (str | bytes.done) = { +export fn peektoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.peek_token(b)) { + match (bytes.peektoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// remaining_tokens — unconsumed portion of the input ahead of the +// remainingtokens — unconsumed portion of the input ahead of the // cursor. ref/hare/strings/tokenize.ha:79. -export fn remaining_tokens(s: *tokenizer) str = { +export fn remainingtokens(s: *tokenizer) str = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - return frombytes(bytes.remaining_tokens(b)); + return frombytes(bytes.remainingtokens(b)); }; // cut — split `in` along the first instance of `delim`, returning the @@ -2667,16 +2667,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = tokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; @@ -2701,16 +2701,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = rtokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index 6b5bad14..d14a8a54 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -2426,7 +2426,7 @@ def RUNE_MIN: rune = '\0'; // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. -// - peek_token dispatches index/rindex by branching on `reverse` +// - peektoken dispatches index/rindex by branching on `reverse` // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // ww has no fn pointers in scope yet — same pattern as lib/strings // `move`. Outwardly identical. @@ -2442,7 +2442,7 @@ package bytes; import os; import types; -// done — iteration sentinel returned by next_token / peek_token at +// done — iteration sentinel returned by nexttoken / peektoken at // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // token; ww spells it per-package the same way lib/encoding/utf8 does // (utf8.ww:36). Plain `void` (not `!void`): continuation signal. @@ -2661,8 +2661,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// rtokenize — reverse-direction tokenize. First next_token yields the -// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +// rtokenize — reverse-direction tokenize. First nexttoken yields the +// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40. export fn rtokenize(in: []u8, delim: u8...) tokenizer = { assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert((in.len: i64) < types.I64_MAX, @@ -2678,10 +2678,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// peek_token — next token without advancing the cursor. Returns done -// once `s.delim` has been zeroed by a prior past-end next_token. +// peektoken — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end nexttoken. // ref/hare/bytes/tokenize.ha:91. -export fn peek_token(s: *tokenizer) ([]u8 | done) = { +export fn peektoken(s: *tokenizer) ([]u8 | done) = { if (s.delim.len == 0) { let d: done; return d; }; @@ -2757,12 +2757,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = { return r; }; -// next_token — current token, then advance past it and the delim. +// nexttoken — current token, then advance past it and the delim. // Once the input is exhausted, returns done and zeros `s.delim` so // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. -export fn next_token(s: *tokenizer) ([]u8 | done) = { +export fn nexttoken(s: *tokenizer) ([]u8 | done) = { let b: []u8; - match (peek_token(s)) { + match (peektoken(s)) { case let v: []u8 => { b = v; }; case done => { let d: done; return d; }; }; @@ -2799,9 +2799,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = { return b; }; -// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// remainingtokens — the unconsumed portion of `s.in`. Read-only view. // ref/hare/bytes/tokenize.ha:145. -export fn remaining_tokens(s: *tokenizer) []u8 = { +export fn remainingtokens(s: *tokenizer) []u8 = { return s.in; }; @@ -2828,16 +2828,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = tokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -2866,16 +2866,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = rtokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -4106,7 +4106,7 @@ export fn tokenize(s: str, delim: str) tokenizer = { }; // rtokenize — reverse-direction counterpart to [[tokenize]]. First -// next_token yields the last token, last yields the first. +// nexttoken yields the last token, last yields the first. // ref/hare/strings/tokenize.ha:44. export fn rtokenize(s: str, delim: str) tokenizer = { let d: []u8 = toutf8(delim); @@ -4119,31 +4119,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = { return bytes.rtokenize(toutf8(s), d...); }; -// next_token — current token, advancing the cursor. +// nexttoken — current token, advancing the cursor. // ref/hare/strings/tokenize.ha:62. -export fn next_token(s: *tokenizer) (str | bytes.done) = { +export fn nexttoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.next_token(b)) { + match (bytes.nexttoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// peek_token — current token without advancing. +// peektoken — current token without advancing. // ref/hare/strings/tokenize.ha:71. -export fn peek_token(s: *tokenizer) (str | bytes.done) = { +export fn peektoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.peek_token(b)) { + match (bytes.peektoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// remaining_tokens — unconsumed portion of the input ahead of the +// remainingtokens — unconsumed portion of the input ahead of the // cursor. ref/hare/strings/tokenize.ha:79. -export fn remaining_tokens(s: *tokenizer) str = { +export fn remainingtokens(s: *tokenizer) str = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - return frombytes(bytes.remaining_tokens(b)); + return frombytes(bytes.remainingtokens(b)); }; // cut — split `in` along the first instance of `delim`, returning the @@ -4184,16 +4184,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = tokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; @@ -4218,16 +4218,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = rtokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; diff --git a/selfhost/test/smoke.combined.ww b/selfhost/test/smoke.combined.ww index f8269e1d..000f681c 100644 --- a/selfhost/test/smoke.combined.ww +++ b/selfhost/test/smoke.combined.ww @@ -2426,7 +2426,7 @@ def RUNE_MIN: rune = '\0'; // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. -// - peek_token dispatches index/rindex by branching on `reverse` +// - peektoken dispatches index/rindex by branching on `reverse` // rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). // ww has no fn pointers in scope yet — same pattern as lib/strings // `move`. Outwardly identical. @@ -2442,7 +2442,7 @@ package bytes; import os; import types; -// done — iteration sentinel returned by next_token / peek_token at +// done — iteration sentinel returned by nexttoken / peektoken at // end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` // token; ww spells it per-package the same way lib/encoding/utf8 does // (utf8.ww:36). Plain `void` (not `!void`): continuation signal. @@ -2661,8 +2661,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// rtokenize — reverse-direction tokenize. First next_token yields the -// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +// rtokenize — reverse-direction tokenize. First nexttoken yields the +// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40. export fn rtokenize(in: []u8, delim: u8...) tokenizer = { assert(delim.len > 0, "bytes.rtokenize called with empty slice"); assert((in.len: i64) < types.I64_MAX, @@ -2678,10 +2678,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = { return t; }; -// peek_token — next token without advancing the cursor. Returns done -// once `s.delim` has been zeroed by a prior past-end next_token. +// peektoken — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end nexttoken. // ref/hare/bytes/tokenize.ha:91. -export fn peek_token(s: *tokenizer) ([]u8 | done) = { +export fn peektoken(s: *tokenizer) ([]u8 | done) = { if (s.delim.len == 0) { let d: done; return d; }; @@ -2757,12 +2757,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = { return r; }; -// next_token — current token, then advance past it and the delim. +// nexttoken — current token, then advance past it and the delim. // Once the input is exhausted, returns done and zeros `s.delim` so // subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. -export fn next_token(s: *tokenizer) ([]u8 | done) = { +export fn nexttoken(s: *tokenizer) ([]u8 | done) = { let b: []u8; - match (peek_token(s)) { + match (peektoken(s)) { case let v: []u8 => { b = v; }; case done => { let d: done; return d; }; }; @@ -2799,9 +2799,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = { return b; }; -// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// remainingtokens — the unconsumed portion of `s.in`. Read-only view. // ref/hare/bytes/tokenize.ha:145. -export fn remaining_tokens(s: *tokenizer) []u8 = { +export fn remainingtokens(s: *tokenizer) []u8 = { return s.in; }; @@ -2828,16 +2828,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = tokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -2866,16 +2866,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { let tok: tokenizer = rtokenize(in, delim...); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: []u8 => { append(toks, s); }; case done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case done => void; case let pk: []u8 => { - let r: []u8 = remaining_tokens(&tok); + let r: []u8 = remainingtokens(&tok); append(toks, r); }; }; @@ -4106,7 +4106,7 @@ export fn tokenize(s: str, delim: str) tokenizer = { }; // rtokenize — reverse-direction counterpart to [[tokenize]]. First -// next_token yields the last token, last yields the first. +// nexttoken yields the last token, last yields the first. // ref/hare/strings/tokenize.ha:44. export fn rtokenize(s: str, delim: str) tokenizer = { let d: []u8 = toutf8(delim); @@ -4119,31 +4119,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = { return bytes.rtokenize(toutf8(s), d...); }; -// next_token — current token, advancing the cursor. +// nexttoken — current token, advancing the cursor. // ref/hare/strings/tokenize.ha:62. -export fn next_token(s: *tokenizer) (str | bytes.done) = { +export fn nexttoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.next_token(b)) { + match (bytes.nexttoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// peek_token — current token without advancing. +// peektoken — current token without advancing. // ref/hare/strings/tokenize.ha:71. -export fn peek_token(s: *tokenizer) (str | bytes.done) = { +export fn peektoken(s: *tokenizer) (str | bytes.done) = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - match (bytes.peek_token(b)) { + match (bytes.peektoken(b)) { case let v: []u8 => return frombytes(v); case bytes.done => { let d: bytes.done; return d; }; }; }; -// remaining_tokens — unconsumed portion of the input ahead of the +// remainingtokens — unconsumed portion of the input ahead of the // cursor. ref/hare/strings/tokenize.ha:79. -export fn remaining_tokens(s: *tokenizer) str = { +export fn remainingtokens(s: *tokenizer) str = { let b: *bytes.tokenizer = s: *bytes.tokenizer; - return frombytes(bytes.remaining_tokens(b)); + return frombytes(bytes.remainingtokens(b)); }; // cut — split `in` along the first instance of `delim`, returning the @@ -4184,16 +4184,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = tokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; }; @@ -4218,16 +4218,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { let tok: tokenizer = rtokenize(in, delim); let i: i32 = 0; for (i < n - 1) { - match (next_token(&tok)) { + match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; case bytes.done => { return toks; }; }; i += 1; }; - match (peek_token(&tok)) { + match (peektoken(&tok)) { case bytes.done => void; case let pk: str => { - let r: str = remaining_tokens(&tok); + let r: str = remainingtokens(&tok); append(toks, r); }; };