Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wcc/ww: drop underscores from next/peek/remaining-tokens (F-Z)

Browse Source
This commit is contained in:
Hojun-Cho
2026-06-15 04:31:16 +09:00
parent 5ae3787cb9
commit 04d35c25c3
10 changed files with 278 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
lib/strings/strings.ww
View File

@@ -679,7 +679,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -692,31 +692,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -757,16 +757,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -791,16 +791,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};

40
lib/strings/stringstest.ww
View File

@@ -1096,28 +1096,28 @@ fn streq(a: str, b: str) bool = {
assert(!(!streq(strings.iterstr(&rit), "he")));
};
// ---- tokenize / rtokenize / peek_token / remaining_tokens -----------
// ---- tokenize / rtokenize / peektoken / remainingtokens -----------
// ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers
// mirror lib/bytes/bytestest expect_token / expect_done but compare
// str via streq.
fn expect_str_token(t: *strings.tokenizer, want: str) void = {
match (strings.peek_token(t)) {
match (strings.peektoken(t)) {
case let p: str => { assert(!(!streq(p, want))); };
case bytes.done => { abort(); };
};
match (strings.next_token(t)) {
match (strings.nexttoken(t)) {
case let n: str => { assert(!(!streq(n, want))); };
case bytes.done => { abort(); };
};
};
fn expect_str_done(t: *strings.tokenizer) void = {
match (strings.peek_token(t)) {
match (strings.peektoken(t)) {
case let p: str => { abort(); };
case bytes.done => void;
};
match (strings.next_token(t)) {
match (strings.nexttoken(t)) {
case let n: str => { abort(); };
case bytes.done => void;
};
@@ -1177,7 +1177,7 @@ fn expect_str_done(t: *strings.tokenizer) void = {
};
@test fn rtokenize_cases() void = {
// Reverse direction — first next_token is the last token.
// Reverse direction — first nexttoken is the last token.
let t: strings.tokenizer = strings.rtokenize(
"Hello world! My name is Harriet.", " ");
expect_str_token(&t, "Harriet.");
@@ -1200,55 +1200,55 @@ fn expect_str_done(t: *strings.tokenizer) void = {
expect_str_done(&t3);
};
@test fn peek_token_cases() void = {
@test fn peektoken_cases() void = {
// Two peeks without advancing return the same token.
let t: strings.tokenizer = strings.tokenize("a b c", " ");
match (strings.peek_token(&t)) {
match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "a"))); };
case bytes.done => { abort(); };
};
match (strings.peek_token(&t)) {
match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "a"))); };
case bytes.done => { abort(); };
};
// Advance once — peek then returns "b".
match (strings.next_token(&t)) {
match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "a"))); };
case bytes.done => { abort(); };
};
match (strings.peek_token(&t)) {
match (strings.peektoken(&t)) {
case let p: str => { assert(!(!streq(p, "b"))); };
case bytes.done => { abort(); };
};
// Empty input — peek is done.
let t2: strings.tokenizer = strings.tokenize("", " ");
match (strings.peek_token(&t2)) {
match (strings.peektoken(&t2)) {
case let p: str => { abort(); };
case bytes.done => void;
};
};
@test fn remaining_tokens_cases() void = {
// ref/hare/strings/tokenize.ha:157. After 2 next_tokens, remaining
@test fn remainingtokens_cases() void = {
// ref/hare/strings/tokenize.ha:157. After 2 nexttokens, remaining
// is "My name is Harriet.".
let t: strings.tokenizer = strings.tokenize(
"Hello world! My name is Harriet.", " ");
match (strings.next_token(&t)) {
match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "Hello"))); };
case bytes.done => { abort(); };
};
match (strings.next_token(&t)) {
match (strings.nexttoken(&t)) {
case let n: str => { assert(!(!streq(n, "world!"))); };
case bytes.done => { abort(); };
};
if (!streq(strings.remaining_tokens(&t), "My name is Harriet.")) {
if (!streq(strings.remainingtokens(&t), "My name is Harriet.")) {
abort();
};
// Fresh tokenizer — remaining_tokens is the whole input.
// Fresh tokenizer — remainingtokens is the whole input.
let t2: strings.tokenizer = strings.tokenize("a b c", " ");
assert(!(!streq(strings.remaining_tokens(&t2), "a b c")));
assert(!(!streq(strings.remainingtokens(&t2), "a b c")));
};
// ---- splitn / rsplitn / split ----------------------------------------
@@ -1338,7 +1338,7 @@ fn expect_str(toks: []str, i: i32, want: str) void = {
expect_str(t3, 0, "a b c");
os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64);
// delim absent — first next_token yields the entire input as the
// delim absent — first nexttoken yields the entire input as the
// sole token; second iter sees done and short-circuits with the
// 1-elem toks un-reversed (single element, reverse is a no-op).
let t4: []str = strings.rsplitn("abc", "=", 5);