ww source: reject malformed UTF-8

cmd/wcc/lex.c

@@ -16,6 +16,61 @@ bomat(const char *src, u64 len, u64 pos)
 	    && (unsigned char)src[pos + 2] == 0xbf;
 }
 
+/* Return the width of the valid UTF-8 sequence beginning at pos, or zero.
+ * This is the same scalar-value partition used by unicode/utf8.DecodeRune:
+ * overlong encodings, surrogates, values above U+10FFFF, stray continuation
+ * bytes, and truncated sequences are invalid. */
+static int
+utf8seqwidth(const char *src, u64 len, u64 pos)
+{
+	if (pos >= len)
+		return 0;
+	const unsigned char *s = (const unsigned char *)src;
+	unsigned char c = s[pos];
+	if (c < 0x80)
+		return 1;
+	if (c >= 0xc2 && c <= 0xdf && len - pos >= 2
+	    && s[pos + 1] >= 0x80 && s[pos + 1] <= 0xbf)
+		return 2;
+	if (len - pos >= 3 && s[pos + 2] >= 0x80
+	    && s[pos + 2] <= 0xbf) {
+		unsigned char c1 = s[pos + 1];
+		if ((c == 0xe0 && c1 >= 0xa0 && c1 <= 0xbf)
+		    || (c >= 0xe1 && c <= 0xec && c1 >= 0x80 && c1 <= 0xbf)
+		    || (c == 0xed && c1 >= 0x80 && c1 <= 0x9f)
+		    || (c >= 0xee && c <= 0xef && c1 >= 0x80 && c1 <= 0xbf))
+			return 3;
+	}
+	if (len - pos >= 4 && s[pos + 2] >= 0x80
+	    && s[pos + 2] <= 0xbf && s[pos + 3] >= 0x80
+	    && s[pos + 3] <= 0xbf) {
+		unsigned char c1 = s[pos + 1];
+		if ((c == 0xf0 && c1 >= 0x90 && c1 <= 0xbf)
+		    || (c >= 0xf1 && c <= 0xf3 && c1 >= 0x80 && c1 <= 0xbf)
+		    || (c == 0xf4 && c1 >= 0x80 && c1 <= 0x8f))
+			return 4;
+	}
+	return 0;
+}
+
+/* The lexer deliberately keeps valid non-ASCII source byte-oriented. Decide
+ * whether one raw byte belongs to a complete valid UTF-8 sequence without
+ * changing that established token model. A continuation byte is valid only
+ * when a valid sequence beginning at most three bytes earlier contains it. */
+static int
+utf8bytevalid(const char *src, u64 len, u64 pos)
+{
+	if (utf8seqwidth(src, len, pos) != 0)
+		return 1;
+	unsigned char c = (unsigned char)src[pos];
+	if (c < 0x80 || c > 0xbf)
+		return 0;
+	for (u64 back = 1; back <= 3 && back <= pos; back++)
+		if (utf8seqwidth(src, len, pos - back) > (int)back)
+			return 1;
+	return 0;
+}
+
 void
 lexinit(Lex *l, Arena *a, const char *file, const char *src, u64 len)
 {
@@ -47,16 +102,39 @@ lskipnul(Lex *l)
 	}
 }
 
-/* Return the raw offset of a logical byte lookahead. Raw NUL bytes do not
- * occupy a slot in the token stream: Go's source.nextch diagnoses them and
- * immediately resumes decoding at the following character. */
+/* Go 1.26.5 syntax.source.nextch reports and discards one byte whenever
+ * utf8.DecodeRune returns RuneError with width one. Drain the same malformed
+ * bytes before they can affect token recovery. */
+static void
+lskiputf8(Lex *l)
+{
+	while (l->pos < l->srclen
+	    && (unsigned char)l->src[l->pos] >= 0x80
+	    && !utf8bytevalid(l->src, l->srclen, l->pos)) {
+		Pos p = { l->file, l->line, l->col };
+		l->pos++;
+		l->col++;
+		errorf(p, "invalid UTF-8 encoding");
+		l->errs++;
+		l->utf8count++;
+	}
+}
+
+/* Return the raw offset of a logical byte lookahead. Raw NUL and malformed
+ * UTF-8 bytes do not occupy a slot in the token stream: Go's source.nextch
+ * diagnoses them and immediately resumes at the following byte. */
 static u64
 lrawoff(Lex *l, u64 ahead)
 {
 	u64 p = l->pos;
 	for (;;) {
-		while (p < l->srclen && l->src[p] == '\0')
+		while (p < l->srclen) {
+			unsigned char c = (unsigned char)l->src[p];
+			if (c != 0 && (c < 0x80
+			    || utf8bytevalid(l->src, l->srclen, p)))
+				break;
 			p++;
+		}
 		if (ahead == 0 || p >= l->srclen)
 			return p;
 		p++;
@@ -77,6 +155,11 @@ lpeek(Lex *l, u64 ahead)
 			lskipnul(l);
 			continue;
 		}
+		if (c >= 0x80
+		    && !utf8bytevalid(l->src, l->srclen, l->pos)) {
+			lskiputf8(l);
+			continue;
+		}
 		if (ahead == 0) {
 			if (bomat(l->src, l->srclen, l->pos))
 				return 0xfeff;
@@ -102,6 +185,11 @@ lget(Lex *l)
 			lskipnul(l);
 			continue;
 		}
+		if (c >= 0x80
+		    && !utf8bytevalid(l->src, l->srclen, l->pos)) {
+			lskiputf8(l);
+			continue;
+		}
 		if (bomat(l->src, l->srclen, l->pos)) {
 			Pos p = { l->file, l->line, l->col };
 			l->pos += 3;
@@ -128,17 +216,20 @@ lpos(Lex *l)
 	return p;
 }
 
-/* Copy one raw source span into token text while omitting diagnosed NUL
- * bytes. This keeps keyword, numeric, suffix, and directive recovery on the
- * same logical character stream as lpeek/lget. */
+/* Copy one raw source span into token text while omitting diagnosed NUL and
+ * malformed UTF-8 bytes. This keeps keyword, numeric, suffix, and directive
+ * recovery on the same logical character stream as lpeek/lget. */
 static char *
 lexspan(Lex *l, u64 begin, u64 end, u64 *len)
 {
 	char *s = amalloc(l->a, end - begin + 1);
 	u64 j = 0;
-	for (u64 i = begin; i < end; i++)
-		if (l->src[i] != '\0')
+	for (u64 i = begin; i < end; i++) {
+		unsigned char c = (unsigned char)l->src[i];
+		if (c != 0 && (c < 0x80
+		    || utf8bytevalid(l->src, l->srclen, i)))
 			s[j++] = l->src[i];
+	}
 	s[j] = '\0';
 	*len = j;
 	return s;
@@ -173,13 +264,14 @@ linecomment(Lex *l)
 {
 	u64 begin = l->pos;
 	u64 nulbegin = l->nulcount;
+	u64 utf8begin = l->utf8count;
 	int c;
 	while ((c = lpeek(l, 0)) >= 0 && c != '\n')
 		lget(l);
 	u64 end = l->pos;
 	u64 n;
 	const char *body;
-	if (l->nulcount == nulbegin) {
+	if (l->nulcount == nulbegin && l->utf8count == utf8begin) {
 		n = end - begin;
 		body = l->src + begin;
 	} else {
@@ -414,6 +506,7 @@ lexnum(Lex *l, Pos start)
 	Tok t = (Tok){ TK_INT, start, NULL, 0, {0}, TK_NONE };
 	u64 begin = l->pos;
 	u64 nulbegin = l->nulcount;
+	u64 utf8begin = l->utf8count;
 	int base = 10;
 	int isfloat = 0;
 	int c = lpeek(l, 0);
@@ -454,7 +547,7 @@ lexnum(Lex *l, Pos start)
 
 	u64 rawend = l->pos;
 	u64 n;
-	if (l->nulcount == nulbegin) {
+	if (l->nulcount == nulbegin && l->utf8count == utf8begin) {
 		n = rawend - begin;
 		t.text = astrndup(l->a, l->src + begin, n);
 	} else {
@@ -511,11 +604,12 @@ lexident(Lex *l, Pos start)
 {
 	u64 begin = l->pos;
 	u64 nulbegin = l->nulcount;
+	u64 utf8begin = l->utf8count;
 	while (isidcont(lpeek(l, 0)))
 		lget(l);
 	u64 n;
 	const char *p;
-	int filtered = l->nulcount != nulbegin;
+	int filtered = l->nulcount != nulbegin || l->utf8count != utf8begin;
 	if (!filtered) {
 		n = l->pos - begin;
 		p = l->src + begin;

cmd/wcc/ww.h

@@ -203,6 +203,7 @@ struct Lex {
 	Arena	*a;		/* token-text arena */
 	int	errs;
 	u64	nulcount;	/* raw NUL bytes diagnosed by the source decoder */
+	u64	utf8count;	/* malformed UTF-8 bytes diagnosed/filtered */
 	int	modreset;	/* a `//ww:module-reset` directive was seen in
 				 * the last skipped run; lexnext emits TK_MODRESET
 				 * before the next real token. */