Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/encoding/utf8: decoder offs i32->size, closing prev/next OOB (#70)

Browse Source 
prev()'s walk-back decremented offs (i32) past 0 to -1 and returned
`more`; a subsequent next() then passed the signed `-1 < len` guard and
read d.src[-1] — a silent OOB decode of a garbage rune (no runtime
bounds net). Hare's decoder.offs is `size`: the underflow wraps to
SIZE_MAX so every `offs < len` guard exits safely (next returns more,
not a rune). Change offs to size and spell prev's loop as the Hare-form
`offs < len` guard; index sites take an i32 temp (ww's slice index is
i32 and `[...]` reads ':' as the slice separator).

No-runtime-net residual: remaining() would silently build a ptr-1/len+1
OOB view when called in the post-`more` state; guard it with a loud
abort (caller contract: don't call after `more`). The offs type ripples
into strings.ww's iterator<->decoder bridge (move/slice) — cast at the
four sites, safe on the rune-return path where offs is in range.

utf8/strings embed into all five selfhost combined.ww snapshots plus the
smoke.combined.ww test amalgamation; all regen'd. utf8test gains
prev_more_then_next_no_oob pinning the closed OOB.
This commit is contained in:
Hojun-Cho
2026-06-13 10:45:06 +09:00
parent 427b67f656
commit a9dcea70ed
9 changed files with 348 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
lib/encoding/utf8/utf8.ww
View File

@@ -205,8 +205,15 @@ let masks: [16]u8 = [
];
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
// offs is `size` (unsigned), matching the Hare field: prev()'s walk-back
// relies on the underflow past 0 wrapping to SIZE_MAX so the `offs < len`
// guards in next()/prev() exit safely. An i32 offs went to -1 and the
// signed `-1 < len` guard then read src[-1] (#70). Index sites cast to
// i32 (ww's slice index is i32 and `[...]` reads ':' as the slice
// separator, so the cast can't be inline) and are only reached when
// offs is in [0, len).
export type decoder = struct {
offs: i32,
offs: size,
src: []u8,
};
@@ -230,14 +237,15 @@ export fn decode(src: []u8) decoder = {
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
// with an explicit conditional.
export fn next(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == d.src.len) {
if (d.offs == d.src.len: size) {
let dn: done; return dn;
};
let nx: i32 = 0;
let state: i32 = 0;
let r: u32 = 0u32;
for (d.offs < d.src.len) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let row: i32 = state * 256 + bi;
let cell: i8 = dfa[row];
@@ -354,24 +362,27 @@ export fn encoderune(out: []u8, r: rune) i32 = {
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
// wrap-around to exit when offs underflows past 0; offs is `size` here
// too, so the same `d.offs < d.src.len` guard exits and leaves offs at
// SIZE_MAX on the more-path (a subsequent next() then returns more, not
// an OOB read — #70). Hare's `defer d.offs = t` is inlined in each
// match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
let n: size = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
let t: size = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
let landed: size = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
@@ -403,10 +414,19 @@ export fn prev(d: *decoder) (rune | done | more | invalid) = {
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
// No-runtime-net residual (#70): Hare's d.src[d.offs..] bounds-asserts
// loudly when offs is out of range (e.g. SIZE_MAX after prev() returned
// `more`). ww has no such net, so a stale offs would silently build a
// ptr-1/len+1 OOB view. Guard it loudly instead: callers must not call
// remaining() after next()/prev() returned `more`.
if (d.offs > d.src.len: size) {
abort("utf8.remaining: decoder offset past end of source");
};
let oi: i32 = d.offs: i32;
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
r.len = d.src.len - oi;
r.cap = d.src.len - oi;
return r;
};
@@ -420,16 +440,17 @@ export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let span: i32 = (end.offs - begin.offs): i32;
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
r.len = span;
r.cap = span;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
return d.offs: i32;
};

26
lib/encoding/utf8/utf8test.ww
View File

@@ -437,6 +437,32 @@ fn streq(a: str, b: str) bool = {
};
};
// #70: after prev() returns `more` it leaves offs out of range (SIZE_MAX,
// via the unsigned underflow). A subsequent next() must see offs >= len
// and return `more` — NOT read one byte before the buffer and decode a
// garbage rune. The byte before the slice (buf[2] = 0x41 = 'A') is the
// planted sentinel the old i32 offs=-1 path returned as rune 65.
@test fn prev_more_then_next_no_oob() void = {
let buf: [4]u8;
buf[0] = 0x58u8; buf[1] = 0x59u8; buf[2] = 0x41u8; // 'A' sentinel
buf[3] = 0xA0u8; // lone continuation
let d: utf8.decoder = utf8.decode(buf[3:4]);
match (utf8.next(&d)) {
case let e: utf8.invalid => void;
case => { abort(); };
};
match (utf8.prev(&d)) {
case utf8.more => void;
case => { abort(); };
};
match (utf8.next(&d)) {
case utf8.more => void; // safe: not src[-1]'s 'A'
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case let e: utf8.invalid => { abort(); };
};
};
@test fn prev_incomplete_invalid() void = {
let src: [2]u8;
src[0] = 0xE3u8; src[1] = 0x81u8;

13
lib/strings/strings.ww
View File

@@ -586,17 +586,20 @@ export fn riter(src: str) iterator = {
fn move(forward: bool, it: *iterator) (rune | utf8.done) = {
let d: utf8.decoder;
d.src = it.src;
d.offs = it.offs;
// utf8.decoder.offs is `size` (#70); the iterator carries i32. The
// rune-return path keeps offs in [0, len), so the narrowing cast back
// is safe.
d.offs = it.offs: size;
if (forward) {
match (utf8.next(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
};
} else {
match (utf8.prev(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
@@ -639,10 +642,10 @@ export fn iterstr(it: *iterator) str = {
export fn slice(begin: *iterator, end: *iterator) str = {
let b: utf8.decoder;
b.src = begin.src;
b.offs = begin.offs;
b.offs = begin.offs: size; // decoder.offs is size (#70)
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
e.offs = end.offs: size;
return frombytes(utf8.slice(&b, &e));
};

68
selfhost/cmd/w6a/main.combined.ww
View File

@@ -1658,8 +1658,15 @@ let masks: [16]u8 = [
];
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
// offs is `size` (unsigned), matching the Hare field: prev()'s walk-back
// relies on the underflow past 0 wrapping to SIZE_MAX so the `offs < len`
// guards in next()/prev() exit safely. An i32 offs went to -1 and the
// signed `-1 < len` guard then read src[-1] (#70). Index sites cast to
// i32 (ww's slice index is i32 and `[...]` reads ':' as the slice
// separator, so the cast can't be inline) and are only reached when
// offs is in [0, len).
export type decoder = struct {
offs: i32,
offs: size,
src: []u8,
};
@@ -1683,14 +1690,15 @@ export fn decode(src: []u8) decoder = {
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
// with an explicit conditional.
export fn next(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == d.src.len) {
if (d.offs == d.src.len: size) {
let dn: done; return dn;
};
let nx: i32 = 0;
let state: i32 = 0;
let r: u32 = 0u32;
for (d.offs < d.src.len) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let row: i32 = state * 256 + bi;
let cell: i8 = dfa[row];
@@ -1807,24 +1815,27 @@ export fn encoderune(out: []u8, r: rune) i32 = {
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
// wrap-around to exit when offs underflows past 0; offs is `size` here
// too, so the same `d.offs < d.src.len` guard exits and leaves offs at
// SIZE_MAX on the more-path (a subsequent next() then returns more, not
// an OOB read — #70). Hare's `defer d.offs = t` is inlined in each
// match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
let n: size = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
let t: size = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
let landed: size = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
@@ -1856,10 +1867,19 @@ export fn prev(d: *decoder) (rune | done | more | invalid) = {
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
// No-runtime-net residual (#70): Hare's d.src[d.offs..] bounds-asserts
// loudly when offs is out of range (e.g. SIZE_MAX after prev() returned
// `more`). ww has no such net, so a stale offs would silently build a
// ptr-1/len+1 OOB view. Guard it loudly instead: callers must not call
// remaining() after next()/prev() returned `more`.
if (d.offs > d.src.len: size) {
abort("utf8.remaining: decoder offset past end of source");
};
let oi: i32 = d.offs: i32;
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
r.len = d.src.len - oi;
r.cap = d.src.len - oi;
return r;
};
@@ -1873,17 +1893,18 @@ export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let span: i32 = (end.offs - begin.offs): i32;
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
r.len = span;
r.cap = span;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
return d.offs: i32;
};
@@ -2475,17 +2496,20 @@ export fn riter(src: str) iterator = {
fn move(forward: bool, it: *iterator) (rune | utf8.done) = {
let d: utf8.decoder;
d.src = it.src;
d.offs = it.offs;
// utf8.decoder.offs is `size` (#70); the iterator carries i32. The
// rune-return path keeps offs in [0, len), so the narrowing cast back
// is safe.
d.offs = it.offs: size;
if (forward) {
match (utf8.next(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
};
} else {
match (utf8.prev(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
@@ -2528,10 +2552,10 @@ export fn iterstr(it: *iterator) str = {
export fn slice(begin: *iterator, end: *iterator) str = {
let b: utf8.decoder;
b.src = begin.src;
b.offs = begin.offs;
b.offs = begin.offs: size; // decoder.offs is size (#70)
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
e.offs = end.offs: size;
return frombytes(utf8.slice(&b, &e));
};

68
selfhost/cmd/w6c/main.combined.ww
View File

@@ -1658,8 +1658,15 @@ let masks: [16]u8 = [
];
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
// offs is `size` (unsigned), matching the Hare field: prev()'s walk-back
// relies on the underflow past 0 wrapping to SIZE_MAX so the `offs < len`
// guards in next()/prev() exit safely. An i32 offs went to -1 and the
// signed `-1 < len` guard then read src[-1] (#70). Index sites cast to
// i32 (ww's slice index is i32 and `[...]` reads ':' as the slice
// separator, so the cast can't be inline) and are only reached when
// offs is in [0, len).
export type decoder = struct {
offs: i32,
offs: size,
src: []u8,
};
@@ -1683,14 +1690,15 @@ export fn decode(src: []u8) decoder = {
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
// with an explicit conditional.
export fn next(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == d.src.len) {
if (d.offs == d.src.len: size) {
let dn: done; return dn;
};
let nx: i32 = 0;
let state: i32 = 0;
let r: u32 = 0u32;
for (d.offs < d.src.len) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let row: i32 = state * 256 + bi;
let cell: i8 = dfa[row];
@@ -1807,24 +1815,27 @@ export fn encoderune(out: []u8, r: rune) i32 = {
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
// wrap-around to exit when offs underflows past 0; offs is `size` here
// too, so the same `d.offs < d.src.len` guard exits and leaves offs at
// SIZE_MAX on the more-path (a subsequent next() then returns more, not
// an OOB read — #70). Hare's `defer d.offs = t` is inlined in each
// match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
let n: size = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
let t: size = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
let landed: size = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
@@ -1856,10 +1867,19 @@ export fn prev(d: *decoder) (rune | done | more | invalid) = {
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
// No-runtime-net residual (#70): Hare's d.src[d.offs..] bounds-asserts
// loudly when offs is out of range (e.g. SIZE_MAX after prev() returned
// `more`). ww has no such net, so a stale offs would silently build a
// ptr-1/len+1 OOB view. Guard it loudly instead: callers must not call
// remaining() after next()/prev() returned `more`.
if (d.offs > d.src.len: size) {
abort("utf8.remaining: decoder offset past end of source");
};
let oi: i32 = d.offs: i32;
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
r.len = d.src.len - oi;
r.cap = d.src.len - oi;
return r;
};
@@ -1873,17 +1893,18 @@ export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let span: i32 = (end.offs - begin.offs): i32;
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
r.len = span;
r.cap = span;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
return d.offs: i32;
};
@@ -2475,17 +2496,20 @@ export fn riter(src: str) iterator = {
fn move(forward: bool, it: *iterator) (rune | utf8.done) = {
let d: utf8.decoder;
d.src = it.src;
d.offs = it.offs;
// utf8.decoder.offs is `size` (#70); the iterator carries i32. The
// rune-return path keeps offs in [0, len), so the narrowing cast back
// is safe.
d.offs = it.offs: size;
if (forward) {
match (utf8.next(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
};
} else {
match (utf8.prev(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
@@ -2528,10 +2552,10 @@ export fn iterstr(it: *iterator) str = {
export fn slice(begin: *iterator, end: *iterator) str = {
let b: utf8.decoder;
b.src = begin.src;
b.offs = begin.offs;
b.offs = begin.offs: size; // decoder.offs is size (#70)
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
e.offs = end.offs: size;
return frombytes(utf8.slice(&b, &e));
};

68
selfhost/cmd/w6l/main.combined.ww
View File

@@ -1658,8 +1658,15 @@ let masks: [16]u8 = [
];
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
// offs is `size` (unsigned), matching the Hare field: prev()'s walk-back
// relies on the underflow past 0 wrapping to SIZE_MAX so the `offs < len`
// guards in next()/prev() exit safely. An i32 offs went to -1 and the
// signed `-1 < len` guard then read src[-1] (#70). Index sites cast to
// i32 (ww's slice index is i32 and `[...]` reads ':' as the slice
// separator, so the cast can't be inline) and are only reached when
// offs is in [0, len).
export type decoder = struct {
offs: i32,
offs: size,
src: []u8,
};
@@ -1683,14 +1690,15 @@ export fn decode(src: []u8) decoder = {
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
// with an explicit conditional.
export fn next(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == d.src.len) {
if (d.offs == d.src.len: size) {
let dn: done; return dn;
};
let nx: i32 = 0;
let state: i32 = 0;
let r: u32 = 0u32;
for (d.offs < d.src.len) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let row: i32 = state * 256 + bi;
let cell: i8 = dfa[row];
@@ -1807,24 +1815,27 @@ export fn encoderune(out: []u8, r: rune) i32 = {
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
// wrap-around to exit when offs underflows past 0; offs is `size` here
// too, so the same `d.offs < d.src.len` guard exits and leaves offs at
// SIZE_MAX on the more-path (a subsequent next() then returns more, not
// an OOB read — #70). Hare's `defer d.offs = t` is inlined in each
// match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
let n: size = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
let t: size = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
let landed: size = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
@@ -1856,10 +1867,19 @@ export fn prev(d: *decoder) (rune | done | more | invalid) = {
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
// No-runtime-net residual (#70): Hare's d.src[d.offs..] bounds-asserts
// loudly when offs is out of range (e.g. SIZE_MAX after prev() returned
// `more`). ww has no such net, so a stale offs would silently build a
// ptr-1/len+1 OOB view. Guard it loudly instead: callers must not call
// remaining() after next()/prev() returned `more`.
if (d.offs > d.src.len: size) {
abort("utf8.remaining: decoder offset past end of source");
};
let oi: i32 = d.offs: i32;
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
r.len = d.src.len - oi;
r.cap = d.src.len - oi;
return r;
};
@@ -1873,17 +1893,18 @@ export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let span: i32 = (end.offs - begin.offs): i32;
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
r.len = span;
r.cap = span;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
return d.offs: i32;
};
@@ -2475,17 +2496,20 @@ export fn riter(src: str) iterator = {
fn move(forward: bool, it: *iterator) (rune | utf8.done) = {
let d: utf8.decoder;
d.src = it.src;
d.offs = it.offs;
// utf8.decoder.offs is `size` (#70); the iterator carries i32. The
// rune-return path keeps offs in [0, len), so the narrowing cast back
// is safe.
d.offs = it.offs: size;
if (forward) {
match (utf8.next(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
};
} else {
match (utf8.prev(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
@@ -2528,10 +2552,10 @@ export fn iterstr(it: *iterator) str = {
export fn slice(begin: *iterator, end: *iterator) str = {
let b: utf8.decoder;
b.src = begin.src;
b.offs = begin.offs;
b.offs = begin.offs: size; // decoder.offs is size (#70)
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
e.offs = end.offs: size;
return frombytes(utf8.slice(&b, &e));
};

68
selfhost/cmd/ww/main.combined.ww
View File

@@ -1658,8 +1658,15 @@ let masks: [16]u8 = [
];
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
// offs is `size` (unsigned), matching the Hare field: prev()'s walk-back
// relies on the underflow past 0 wrapping to SIZE_MAX so the `offs < len`
// guards in next()/prev() exit safely. An i32 offs went to -1 and the
// signed `-1 < len` guard then read src[-1] (#70). Index sites cast to
// i32 (ww's slice index is i32 and `[...]` reads ':' as the slice
// separator, so the cast can't be inline) and are only reached when
// offs is in [0, len).
export type decoder = struct {
offs: i32,
offs: size,
src: []u8,
};
@@ -1683,14 +1690,15 @@ export fn decode(src: []u8) decoder = {
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
// with an explicit conditional.
export fn next(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == d.src.len) {
if (d.offs == d.src.len: size) {
let dn: done; return dn;
};
let nx: i32 = 0;
let state: i32 = 0;
let r: u32 = 0u32;
for (d.offs < d.src.len) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let row: i32 = state * 256 + bi;
let cell: i8 = dfa[row];
@@ -1807,24 +1815,27 @@ export fn encoderune(out: []u8, r: rune) i32 = {
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
// wrap-around to exit when offs underflows past 0; offs is `size` here
// too, so the same `d.offs < d.src.len` guard exits and leaves offs at
// SIZE_MAX on the more-path (a subsequent next() then returns more, not
// an OOB read — #70). Hare's `defer d.offs = t` is inlined in each
// match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
let n: size = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
let t: size = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
let landed: size = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
@@ -1856,10 +1867,19 @@ export fn prev(d: *decoder) (rune | done | more | invalid) = {
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
// No-runtime-net residual (#70): Hare's d.src[d.offs..] bounds-asserts
// loudly when offs is out of range (e.g. SIZE_MAX after prev() returned
// `more`). ww has no such net, so a stale offs would silently build a
// ptr-1/len+1 OOB view. Guard it loudly instead: callers must not call
// remaining() after next()/prev() returned `more`.
if (d.offs > d.src.len: size) {
abort("utf8.remaining: decoder offset past end of source");
};
let oi: i32 = d.offs: i32;
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
r.len = d.src.len - oi;
r.cap = d.src.len - oi;
return r;
};
@@ -1873,17 +1893,18 @@ export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let span: i32 = (end.offs - begin.offs): i32;
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
r.len = span;
r.cap = span;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
return d.offs: i32;
};
@@ -2475,17 +2496,20 @@ export fn riter(src: str) iterator = {
fn move(forward: bool, it: *iterator) (rune | utf8.done) = {
let d: utf8.decoder;
d.src = it.src;
d.offs = it.offs;
// utf8.decoder.offs is `size` (#70); the iterator carries i32. The
// rune-return path keeps offs in [0, len), so the narrowing cast back
// is safe.
d.offs = it.offs: size;
if (forward) {
match (utf8.next(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
};
} else {
match (utf8.prev(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
@@ -2528,10 +2552,10 @@ export fn iterstr(it: *iterator) str = {
export fn slice(begin: *iterator, end: *iterator) str = {
let b: utf8.decoder;
b.src = begin.src;
b.offs = begin.offs;
b.offs = begin.offs: size; // decoder.offs is size (#70)
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
e.offs = end.offs: size;
return frombytes(utf8.slice(&b, &e));
};

68
selfhost/cmd/wwdump/main.combined.ww
View File

@@ -3175,8 +3175,15 @@ let masks: [16]u8 = [
];
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
// offs is `size` (unsigned), matching the Hare field: prev()'s walk-back
// relies on the underflow past 0 wrapping to SIZE_MAX so the `offs < len`
// guards in next()/prev() exit safely. An i32 offs went to -1 and the
// signed `-1 < len` guard then read src[-1] (#70). Index sites cast to
// i32 (ww's slice index is i32 and `[...]` reads ':' as the slice
// separator, so the cast can't be inline) and are only reached when
// offs is in [0, len).
export type decoder = struct {
offs: i32,
offs: size,
src: []u8,
};
@@ -3200,14 +3207,15 @@ export fn decode(src: []u8) decoder = {
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
// with an explicit conditional.
export fn next(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == d.src.len) {
if (d.offs == d.src.len: size) {
let dn: done; return dn;
};
let nx: i32 = 0;
let state: i32 = 0;
let r: u32 = 0u32;
for (d.offs < d.src.len) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let row: i32 = state * 256 + bi;
let cell: i8 = dfa[row];
@@ -3324,24 +3332,27 @@ export fn encoderune(out: []u8, r: rune) i32 = {
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
// wrap-around to exit when offs underflows past 0; offs is `size` here
// too, so the same `d.offs < d.src.len` guard exits and leaves offs at
// SIZE_MAX on the more-path (a subsequent next() then returns more, not
// an OOB read — #70). Hare's `defer d.offs = t` is inlined in each
// match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
let n: size = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
let t: size = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
let landed: size = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
@@ -3373,10 +3384,19 @@ export fn prev(d: *decoder) (rune | done | more | invalid) = {
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
// No-runtime-net residual (#70): Hare's d.src[d.offs..] bounds-asserts
// loudly when offs is out of range (e.g. SIZE_MAX after prev() returned
// `more`). ww has no such net, so a stale offs would silently build a
// ptr-1/len+1 OOB view. Guard it loudly instead: callers must not call
// remaining() after next()/prev() returned `more`.
if (d.offs > d.src.len: size) {
abort("utf8.remaining: decoder offset past end of source");
};
let oi: i32 = d.offs: i32;
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
r.len = d.src.len - oi;
r.cap = d.src.len - oi;
return r;
};
@@ -3390,17 +3410,18 @@ export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let span: i32 = (end.offs - begin.offs): i32;
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
r.len = span;
r.cap = span;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
return d.offs: i32;
};
@@ -3992,17 +4013,20 @@ export fn riter(src: str) iterator = {
fn move(forward: bool, it: *iterator) (rune | utf8.done) = {
let d: utf8.decoder;
d.src = it.src;
d.offs = it.offs;
// utf8.decoder.offs is `size` (#70); the iterator carries i32. The
// rune-return path keeps offs in [0, len), so the narrowing cast back
// is safe.
d.offs = it.offs: size;
if (forward) {
match (utf8.next(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
};
} else {
match (utf8.prev(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
@@ -4045,10 +4069,10 @@ export fn iterstr(it: *iterator) str = {
export fn slice(begin: *iterator, end: *iterator) str = {
let b: utf8.decoder;
b.src = begin.src;
b.offs = begin.offs;
b.offs = begin.offs: size; // decoder.offs is size (#70)
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
e.offs = end.offs: size;
return frombytes(utf8.slice(&b, &e));
};

68
selfhost/test/smoke.combined.ww
View File

@@ -3175,8 +3175,15 @@ let masks: [16]u8 = [
];
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
// offs is `size` (unsigned), matching the Hare field: prev()'s walk-back
// relies on the underflow past 0 wrapping to SIZE_MAX so the `offs < len`
// guards in next()/prev() exit safely. An i32 offs went to -1 and the
// signed `-1 < len` guard then read src[-1] (#70). Index sites cast to
// i32 (ww's slice index is i32 and `[...]` reads ':' as the slice
// separator, so the cast can't be inline) and are only reached when
// offs is in [0, len).
export type decoder = struct {
offs: i32,
offs: size,
src: []u8,
};
@@ -3200,14 +3207,15 @@ export fn decode(src: []u8) decoder = {
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
// with an explicit conditional.
export fn next(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == d.src.len) {
if (d.offs == d.src.len: size) {
let dn: done; return dn;
};
let nx: i32 = 0;
let state: i32 = 0;
let r: u32 = 0u32;
for (d.offs < d.src.len) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let row: i32 = state * 256 + bi;
let cell: i8 = dfa[row];
@@ -3324,24 +3332,27 @@ export fn encoderune(out: []u8, r: rune) i32 = {
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
// wrap-around to exit when offs underflows past 0; offs is `size` here
// too, so the same `d.offs < d.src.len` guard exits and leaves offs at
// SIZE_MAX on the more-path (a subsequent next() then returns more, not
// an OOB read — #70). Hare's `defer d.offs = t` is inlined in each
// match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
let n: size = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
for (d.offs < d.src.len: size) {
let oi: i32 = d.offs: i32;
let b: u8 = d.src[oi];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
let t: size = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
let landed: size = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
@@ -3373,10 +3384,19 @@ export fn prev(d: *decoder) (rune | done | more | invalid) = {
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
// No-runtime-net residual (#70): Hare's d.src[d.offs..] bounds-asserts
// loudly when offs is out of range (e.g. SIZE_MAX after prev() returned
// `more`). ww has no such net, so a stale offs would silently build a
// ptr-1/len+1 OOB view. Guard it loudly instead: callers must not call
// remaining() after next()/prev() returned `more`.
if (d.offs > d.src.len: size) {
abort("utf8.remaining: decoder offset past end of source");
};
let oi: i32 = d.offs: i32;
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
r.len = d.src.len - oi;
r.cap = d.src.len - oi;
return r;
};
@@ -3390,17 +3410,18 @@ export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let span: i32 = (end.offs - begin.offs): i32;
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
r.len = span;
r.cap = span;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
return d.offs: i32;
};
@@ -3992,17 +4013,20 @@ export fn riter(src: str) iterator = {
fn move(forward: bool, it: *iterator) (rune | utf8.done) = {
let d: utf8.decoder;
d.src = it.src;
d.offs = it.offs;
// utf8.decoder.offs is `size` (#70); the iterator carries i32. The
// rune-return path keeps offs in [0, len), so the narrowing cast back
// is safe.
d.offs = it.offs: size;
if (forward) {
match (utf8.next(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
};
} else {
match (utf8.prev(&d)) {
case let r: rune => { it.offs = d.offs; return r; };
case let r: rune => { it.offs = d.offs: i32; return r; };
case let dn: utf8.done => return dn;
case let m: utf8.more => abort("strings.move: invalid UTF-8");
case let e: utf8.invalid => abort("strings.move: invalid UTF-8");
@@ -4045,10 +4069,10 @@ export fn iterstr(it: *iterator) str = {
export fn slice(begin: *iterator, end: *iterator) str = {
let b: utf8.decoder;
b.src = begin.src;
b.offs = begin.offs;
b.offs = begin.offs: size; // decoder.offs is size (#70)
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
e.offs = end.offs: size;
return frombytes(utf8.slice(&b, &e));
};