Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wcc/ww: drop underscores from next/peek/remaining-tokens (F-Z)

Browse Source
This commit is contained in:
Hojun-Cho
2026-06-15 04:31:16 +09:00
parent 5ae3787cb9
commit 04d35c25c3
10 changed files with 278 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
selfhost/test/smoke.combined.ww
View File

@@ -2426,7 +2426,7 @@ def RUNE_MIN: rune = '\0';
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// - peektoken dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
@@ -2442,7 +2442,7 @@ package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// done — iteration sentinel returned by nexttoken / peektoken at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
@@ -2661,8 +2661,8 @@ export fn tokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
// rtokenize — reverse-direction tokenize. First nexttoken yields the
// last token, last nexttoken yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
assert(delim.len > 0, "bytes.rtokenize called with empty slice");
assert((in.len: i64) < types.I64_MAX,
@@ -2678,10 +2678,10 @@ export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// peektoken — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end nexttoken.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
export fn peektoken(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
@@ -2757,12 +2757,12 @@ export fn peek_token(s: *tokenizer) ([]u8 | done) = {
return r;
};
// next_token — current token, then advance past it and the delim.
// nexttoken — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn nexttoken(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
match (peektoken(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
@@ -2799,9 +2799,9 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// remainingtokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
export fn remainingtokens(s: *tokenizer) []u8 = {
return s.in;
};
@@ -2828,16 +2828,16 @@ export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -2866,16 +2866,16 @@ export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: []u8 => { append(toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
let r: []u8 = remainingtokens(&tok);
append(toks, r);
};
};
@@ -4106,7 +4106,7 @@ export fn tokenize(s: str, delim: str) tokenizer = {
};
// rtokenize — reverse-direction counterpart to [[tokenize]]. First
// next_token yields the last token, last yields the first.
// nexttoken yields the last token, last yields the first.
// ref/hare/strings/tokenize.ha:44.
export fn rtokenize(s: str, delim: str) tokenizer = {
let d: []u8 = toutf8(delim);
@@ -4119,31 +4119,31 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
return bytes.rtokenize(toutf8(s), d...);
};
// next_token — current token, advancing the cursor.
// nexttoken — current token, advancing the cursor.
// ref/hare/strings/tokenize.ha:62.
export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn nexttoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
match (bytes.nexttoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// peek_token — current token without advancing.
// peektoken — current token without advancing.
// ref/hare/strings/tokenize.ha:71.
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
export fn peektoken(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
match (bytes.peektoken(b)) {
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
// remaining_tokens — unconsumed portion of the input ahead of the
// remainingtokens — unconsumed portion of the input ahead of the
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
export fn remainingtokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return frombytes(bytes.remaining_tokens(b));
return frombytes(bytes.remainingtokens(b));
};
// cut — split `in` along the first instance of `delim`, returning the
@@ -4184,16 +4184,16 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};
@@ -4218,16 +4218,16 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
match (peektoken(&tok)) {
case bytes.done => void;
case let pk: str => {
let r: str = remaining_tokens(&tok);
let r: str = remainingtokens(&tok);
append(toks, r);
};
};