Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wcc/ww: drop underscores from next/peek/remaining-tokens (F-Z)

Browse Source
This commit is contained in:
Hojun-Cho
2026-06-15 04:31:16 +09:00
parent 5ae3787cb9
commit 04d35c25c3
10 changed files with 278 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
lib/bytes/bytestest.ww
View File

@@ -237,7 +237,7 @@ import os;
assert(!(!bytes.hassuffix(a4[0:4], s234[0:3])));
};
// ---- tokenize / rtokenize / peek_token / remaining_tokens -----------
// ---- tokenize / rtokenize / peektoken / remainingtokens -----------
// ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize
// drives the iterator through an expected-token sequence and asserts
// `equal(p, n)` (peek == next), `equal(n, want)` (next == expected).
@@ -245,13 +245,13 @@ import os;
// expect_token — table row driver. Advances `t` once, asserts the
// returned token matches `want`. peek invariant: peek must equal next.
fn expect_token(t: *bytes.tokenizer, want: []u8) void = {
match (bytes.peek_token(t)) {
match (bytes.peektoken(t)) {
case let p: []u8 => {
assert(!(!bytes.equal(p, want)));
};
case bytes.done => abort();
};
match (bytes.next_token(t)) {
match (bytes.nexttoken(t)) {
case let n: []u8 => {
assert(!(!bytes.equal(n, want)));
};
@@ -261,11 +261,11 @@ fn expect_token(t: *bytes.tokenizer, want: []u8) void = {
// expect_done — table-row driver. peek and next must both be done.
fn expect_done(t: *bytes.tokenizer) void = {
match (bytes.peek_token(t)) {
match (bytes.peektoken(t)) {
case let p: []u8 => abort();
case bytes.done => void;
};
match (bytes.next_token(t)) {
match (bytes.nexttoken(t)) {
case let n: []u8 => abort();
case bytes.done => void;
};
@@ -381,63 +381,63 @@ fn expect_done(t: *bytes.tokenizer) void = {
expect_done(&t6);
};
@test fn peek_token_cases() void = {
@test fn peektoken_cases() void = {
// Peeking twice without advancing returns the same token.
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.peek_token(&t)) {
match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); };
case bytes.done => abort();
};
match (bytes.peek_token(&t)) {
match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); };
case bytes.done => abort();
};
// Then advance once — peek-after-next is the second token.
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
match (bytes.next_token(&t)) {
match (bytes.nexttoken(&t)) {
case let n: []u8 => { assert(!(!bytes.equal(n, e_12[0:2]))); };
case bytes.done => abort();
};
match (bytes.peek_token(&t)) {
match (bytes.peektoken(&t)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort();
};
// Reverse peek symmetry — first peek is last token.
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.peek_token(&t2)) {
match (bytes.peektoken(&t2)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort();
};
match (bytes.peek_token(&t2)) {
match (bytes.peektoken(&t2)) {
case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); };
case bytes.done => abort();
};
};
@test fn remaining_tokens_cases() void = {
// After one next_token, remaining is bytes past the consumed delim.
@test fn remainingtokens_cases() void = {
// After one nexttoken, remaining is bytes past the consumed delim.
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.next_token(&t)) {
match (bytes.nexttoken(&t)) {
case let n: []u8 => void;
case bytes.done => abort();
};
let r: []u8 = bytes.remaining_tokens(&t);
let r: []u8 = bytes.remainingtokens(&t);
assert(!(!bytes.equal(r, e_34[0:2])));
// Reverse — after one next_token, remaining is bytes before the
// Reverse — after one nexttoken, remaining is bytes before the
// consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2].
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.next_token(&t2)) {
match (bytes.nexttoken(&t2)) {
case let n: []u8 => void;
case bytes.done => abort();
};
let r2: []u8 = bytes.remaining_tokens(&t2);
let r2: []u8 = bytes.remainingtokens(&t2);
assert(!(!bytes.equal(r2, e_12[0:2])));
};
@@ -655,7 +655,7 @@ fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = {
// delim absent — done short-circuits immediately at iter 0;
// toks ends up holding only the eventual remainder if n=1
// (the loop never runs and peek picks up the full input).
// With n>1 the loop's first next_token sees done (no delim
// With n>1 the loop's first nexttoken sees done (no delim
// match anywhere) and returns toks={} per Hare's early-exit.
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5);