Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
e3f49234f9c9e922c184542304187a0b2364cd83
ww/lib/encoding/utf8/utf8.ww
Hojun-Cho e3f49234f9 lib/encoding/hex: align to Hare io-streaming surface 
The old buffer surface (encodedsize/decodedsize + encode(dst,src) i32 +
decode(dst,src) (i32|invalid)) does not exist in Hare — it predates the
#94 io vtable and mis-cited hex.ha:175 while implementing a different
signature. Replace it with Hare's real surface
(ref/hare/encoding/hex/hex.ha):

  - newencoder(out: io.handle) (:28) — write-only encoder stream.
  - encode(out: io.handle, in) (size | io.error) (:91).
  - encodestr(in) str (:68).
  - decodestr(s) ([]u8 | errors.invalid) (:175).

Divergences (documented at-site):

  - The streaming DECODER (newdecoder/decode_reader, :120,:129) is
    DEFERRED to #247, blocked on #199b: Hare's decode_reader returns
    errors::invalid, which fits Hare's io::error (spreads
    ...errors::error). ww's io.error (lib/io/types.ww:55-62) does not
    carry errors.invalid, and io.read's (size|eof|error) can't propagate
    it, so a hex decoder *stream* can't faithfully report invalid hex
    through io.read yet. decodestr ships as a direct transform meanwhile.
  - nomem dropped from encodestr/decodestr returns (ww memio.dynamic has
    no failure path — same memio.string rule-9 carve-out, memio.ww:208).
  - The local hex.invalid type is deleted in favor of errors.invalid
    (that was the original divergence).
  - encode uses a single io.write rather than Hare's io::writeall (ww has
    none — fmt.fprint:498-501: callers drive write-all over raw io.write;
    encode_writer is whole-slice so a single write is equivalent).
  - dump (:212) deferred: ww has no default-arg support and fmt's
    formattable lacks u64 (#209), so the address column can't be ported
    faithfully yet.

hex is now import-bearing, so it moves off the 900_stdlib standalone-
compile list (like fmt/os/strings/bufio/bytes/errors before it); coverage
stays at 979_hex_run.c. The stale "mirrors lib/encoding/hex.encode"
comments in lib/encoding/utf8/utf8.ww are updated, which regenerates the
6 selfhost combined.ww (5 cmd + test/smoke) (comment-only, byte-id-neutral).
2026-06-02 00:40:04 +09:00

436 lines
22 KiB
Plaintext
Raw Blame History

// encoding/utf8 — UTF-8 encode/decode. Hare port; see
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
//
// The decoder is Hoehrmann's branchless DFA, originally published
// at <https://bjoern.hoehrmann.de/utf-8/decoder/dfa/>. Hare's
// ref/hare/encoding/utf8/decodetable.ha:4 restructures Hoehrmann's
// flat table to 2D `[8][256]i8`; we flatten back to 1D `[2048]i8`
// because ww cgen does not yet ship 2D arrays (task #20).
//
// Surface deviation from ref/hare/encoding/utf8:
//
// - `encoderune` takes a caller-supplied `out: []u8` and returns
// the byte count. Hare returns a slice into a `static let buf`;
// the caller-buffer form skips the static-buffer/slice-return pair.
//
// Deferred (no in-tree caller, follow-up tasks): `appendrune`,
// `strencode`, `strdecode`. Hare's string-iteration surface
// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha)
// lives under lib/strings, not here.
// ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence.
// Plain `void` (not `!void`): a truncated tail is a control-flow
// signal, not an error caller can ignore.
package utf8;
export type more = void;
// ref/hare/encoding/utf8/types.ha:9 — invalid UTF-8 sequence.
export type invalid = !void;
// ref/hare/encoding/utf8/types.ha:12 — fixed message; `invalid` carries
// no payload, so the rendering is constant.
export fn strerror(err: invalid) str = {
return "Invalid UTF-8";
};
// `done` is not a built-in singleton in ww (Hare ships it as part of
// the type system). Plain `void` (not `!void`): end-of-input is a
// continuation signal, not an error. lib/io spells its EOF the same
// way (lib/io/io.ww:8-11).
export type done = void;
// ref/hare/encoding/utf8/decodetable.ha:4 — Hoehrmann's UTF-8 DFA,
// flat 1D `[2048]i8`. Layout: dfa[state*256 + byte] gives the next
// state (>0), the accept transition (0 — emit rune), or invalid (-1).
// Values match ref/hare/encoding/utf8/decodetable.ha verbatim.
let dfa: [2048]i8 = [
// state 0 — initial byte: ASCII accepts (0), continuation/illegal
// byte rejects (-1), legal multibyte start emits a state.
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
3i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 4i8, 2i8, 2i8,
5i8, 6i8, 6i8, 6i8, 7i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
// state 1 — expecting one continuation byte (0x80..0xBF).
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
// state 2 — expecting one continuation byte (full 0x80..0xBF range).
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
// state 3 — first byte was 0xE0; continuation byte must be 0xA0..0xBF
// (rejects overlong 3-byte encodings).
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
// state 4 — first byte was 0xED; continuation byte must be 0x80..0x9F
// (rejects UTF-16 surrogate codepoints U+D800..U+DFFF).
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
// state 5 — first byte was 0xF0; continuation byte must be 0x90..0xBF
// (rejects overlong 4-byte encodings).
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
// state 6 — middle continuation byte of a 4-byte sequence (0x80..0xBF).
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
// state 7 — first byte was 0xF4; continuation byte must be 0x80..0x8F
// (rejects codepoints above U+10FFFF).
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
];
// ref/hare/encoding/utf8/decode.ha:17 — payload-bit masks. Hare's
// [2][8]u8 flattened to 1D [16]u8; row 0 (offsets 0..7) is the
// continuation-byte mask (always 0x3F), row 1 (offsets 8..15) is the
// initial-byte payload mask indexed by the transition class.
let masks: [16]u8 = [
0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8,
0x7fu8, 0x1fu8, 0x0fu8, 0x0fu8, 0x0fu8, 0x07u8, 0x07u8, 0x07u8,
];
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
export type decoder = struct {
offs: i32,
src: []u8,
};
// ref/hare/encoding/utf8/decode.ha:12.
export fn decode(src: []u8) decoder = {
let d: decoder;
d.src = src;
d.offs = 0;
return d;
};
// ref/hare/encoding/utf8/decode.ha:27. Returns the next rune from a
// decoder, `done` at end-of-input, `more` on truncated trailing
// sequence, `invalid` on malformed input (overlong, surrogate,
// out-of-range, bad continuation).
//
// Algorithm is verbatim Hoehrmann (see file header). One structural
// rewrite: Hare encodes the "initial vs continuation byte" decision
// as the branchless `(state - 1): uint >> 31`, which assumes a 32-bit
// uint. ww's uint is 64-bit (cmd/wcc/type.c:58), so the shift answer
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
// with an explicit conditional.
export fn next(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == d.src.len) {
let dn: done; return dn;
};
let nx: i32 = 0;
let state: i32 = 0;
let r: u32 = 0u32;
for (d.offs < d.src.len) {
let b: u8 = d.src[d.offs];
let bi: i32 = b: i32;
let row: i32 = state * 256 + bi;
let cell: i8 = dfa[row];
nx = cell: i32;
let mi: i32 = 0;
if (state == 0) { mi = 1; };
let m: u8 = masks[mi * 8 + (nx & 7)];
r = (r << 6u32) | ((b & m): u32);
if (nx <= 0) {
d.offs += 1;
if (nx == 0) { return r: rune; };
let e: invalid; return e;
};
state = nx;
d.offs += 1;
};
let mr: more; return mr;
};
// ref/hare/encoding/utf8/decode.ha:207. Strict whole-input check.
// The hot path: tight DFA loop, no rune assembly. Bails the moment
// the table returns -1 so malformed inputs don't pay for the rest
// of the buffer.
export fn validate(src: []u8) (void | invalid) = {
let state: i32 = 0;
let i: i32 = 0;
for (i < src.len) {
if (state < 0) { break; };
let bi: i32 = src[i]: i32;
let cell: i8 = dfa[state * 256 + bi];
state = cell: i32;
i += 1;
};
if (state == 0) { return; };
let e: invalid; return e;
};
// ref/hare/encoding/utf8/rune.ha:5. Encoded byte length of `r` as
// UTF-8. Callers in ww use this to size the buffer they hand to
// [[encoderune]]; values >0x10FFFF or negative are not legal Unicode
// codepoints and Hare aborts on them in `encoderune` itself, so we
// keep `runesz` infallible (matches Hare).
export fn runesz(r: rune) i32 = {
let ch: u32 = r: u32;
if (ch < 128u32) { return 1; };
if (ch < 2048u32) { return 2; };
if (ch < 65536u32) { return 3; };
return 4;
};
// ref/hare/encoding/utf8/rune.ha:15. Expected byte length of the
// codepoint that starts with `c`, or `invalid` if `c` cannot start
// a legal UTF-8 sequence. Constants written in decimal because ww
// doesn't accept Hare's `0b1000_0000` binary syntax: 0x80=128,
// 0xC2=194, 0xE0=224, 0xF0=240, 0xF8=248.
export fn utf8sz(c: u8) (i32 | invalid) = {
if (c < 128u8) { return 1; };
if (c < 194u8) { let e: invalid; return e; };
if (c >= 248u8) { let e: invalid; return e; };
if (c < 224u8) { return 2; };
if (c < 240u8) { return 3; };
return 4;
};
// ref/hare/encoding/utf8/encode.ha:7. Encode `r` into `out` (caller-
// supplied; must hold at least [[runesz]](r) bytes) and return the
// byte count. ABORT if `r` is a UTF-16 surrogate or above U+10FFFF —
// same precondition Hare asserts at ref/hare/encoding/utf8/encode.ha:9.
//
// Surface deviation: Hare returns `[]u8` (slice into a static buf).
// ww uses the caller-buffer form; caller can reuse a [4]u8 stack
// scratch across encodes.
export fn encoderune(out: []u8, r: rune) i32 = {
let ch: u32 = r: u32;
if (ch >= 0xD800u32) {
if (ch <= 0xDFFFu32) {
abort("utf8.encoderune: surrogate codepoint");
};
};
if (ch > 0x10FFFFu32) {
abort("utf8.encoderune: codepoint > U+10FFFF");
};
let n: i32 = 0;
let first: u8 = 0u8;
if (ch < 0x80u32) {
first = 0u8; n = 1;
} else if (ch < 0x800u32) {
first = 0xC0u8; n = 2;
} else if (ch < 0x10000u32) {
first = 0xE0u8; n = 3;
} else {
first = 0xF0u8; n = 4;
};
let v: u32 = ch;
let i: i32 = n - 1;
for (i > 0) {
out[i] = ((v: u8) & 0x3Fu8) | 0x80u8;
v = v >> 6u32;
i -= 1;
};
out[0] = (v: u8) | first;
return n;
};
// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a
// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes
// forward from there, and confirms the forward decode lands back at the
// original offset. Returns `done` at start-of-input; `invalid` if no
// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds
// 4 bytes), if the forward decode returns `more`/`invalid`, or if it
// lands at a different offset than expected. Returns `more` when the
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
};
return r;
};
case let dn: done => {
d.offs = t;
let e: invalid; return e;
};
case let m: more => {
d.offs = t;
let e: invalid; return e;
};
case let e: invalid => {
d.offs = t;
let e2: invalid; return e2;
};
};
};
if (n - d.offs == 4) {
let e: invalid; return e;
};
d.offs -= 1;
};
let mr: more; return mr;
};
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
return r;
};
// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes
// between two decoders' positions. Precondition (Hare asserts both):
// the decoders share the same source, and `begin.offs <= end.offs`.
export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.src.ptr != end.src.ptr) {
abort("utf8.slice: decoders from different sources");
};
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
};
Reference in New Issue View Git Blame Copy Permalink