ww source: align UTF-8 BOM placement

lib/ww/syntax/lex.ww

@@ -65,6 +65,14 @@ export type lex = struct {
 	modresetpath: str,
 };
 
+fn bomat(src: *u8, len: u64, off: u64) bool = {
+	if (off > len || len - off < 3u64) { return false; };
+	let i: i32 = off: i32;
+	return src[i] == 0xefu8
+		&& src[i + 1] == 0xbbu8
+		&& src[i + 2] == 0xbfu8;
+};
+
 export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
 	l.file = file;
 	l.src = src;
@@ -76,6 +84,9 @@ export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
 	l.modreset = 0;
 	l.modpathset = 0;
 	l.modresetpathset = 0;
+	// Go 1.26.5 syntax.source.nextch ignores U+FEFF only at the first
+	// source position but counts its three bytes for the next column.
+	if (bomat(src, len, 0u64)) { l.lpos = 3u64; l.col = 4; };
 };
 
 fn srcb(l: *lex, off: u64) i32 = {
@@ -87,11 +98,22 @@ fn srcb(l: *lex, off: u64) i32 = {
 fn lpeek(l: *lex, ahead: u64) i32 = {
 	let p: u64 = l.lpos + ahead;
 	if (p >= l.srclen) { return -1; };
+	if (bomat(l.src, l.srclen, p)) { return 0xFEFF; };
 	return srcb(l, p);
 };
 
 fn lget(l: *lex) i32 = {
 	if (l.lpos >= l.srclen) { return -1; };
+	if (bomat(l.src, l.srclen, l.lpos)) {
+		let bp: pos;
+		bp.file = l.file;
+		bp.line = l.line;
+		bp.col = l.col;
+		l.lpos += 3u64;
+		l.col += 3;
+		errat(l, &bp, "invalid BOM in the middle of the file");
+		return 0xFEFF;
+	};
 	let c: i32 = srcb(l, l.lpos);
 	l.lpos += 1u64;
 	if (c == '\n') {
@@ -801,6 +823,11 @@ export fn lexnext(l: *lex, out: *tok) void = {
 		return;
 	};
 	let c: i32 = lpeek(l, 0u64);
+	if (c == 0xFEFF) {
+		lget(l);
+		emitsimple(&start, tkind.TK_ERR, out);
+		return;
+	};
 
 	if (c >= 0) {
 		if (isidstart(c: rune)) { lexident(l, &start, out); return; };

lib/ww/syntax/tok_test.ww

@@ -338,3 +338,78 @@ fn checkfloat(src: str, want: u64) void = {
 	assert(!(t.col != 6));
 	assert(!(t.text != "bar"));
 };
+
+fn bomsource(dst: *u8, before: str, after: str) u64 = {
+	let n: u64 = 0u64;
+	let i: i32 = 0;
+	for (i < before.len) {
+		let ni: i32 = n: i32;
+		dst[ni] = before[i];
+		n += 1u64;
+		i += 1;
+	};
+	let bi: i32 = n: i32;
+	dst[bi] = 0xefu8;
+	dst[bi + 1] = 0xbbu8;
+	dst[bi + 2] = 0xbfu8;
+	n += 3u64;
+	i = 0;
+	for (i < after.len) {
+		let ni: i32 = n: i32;
+		dst[ni] = after[i];
+		n += 1u64;
+		i += 1;
+	};
+	return n;
+};
+
+fn bomerror(before: str, after: str) bool = {
+	let src: [128]u8;
+	let n: u64 = bomsource(src.ptr, before, after);
+	let l: syntax.lex;
+	syntax.lexinit(&l, "t", src.ptr, n);
+	let t: syntax.tok;
+	for (true) {
+		syntax.lexnext(&l, &t);
+		if (t.kind == syntax.tkind.TK_EOF) { break; };
+	};
+	return l.errs > 0;
+};
+
+fn doublebomerror() bool = {
+	let src: [6]u8;
+	let i: i32 = 0;
+	for (i < 6) {
+		if (i % 3 == 0) { src[i] = 0xefu8; };
+		if (i % 3 == 1) { src[i] = 0xbbu8; };
+		if (i % 3 == 2) { src[i] = 0xbfu8; };
+		i += 1;
+	};
+	let l: syntax.lex;
+	syntax.lexinit(&l, "t", src.ptr, 6u64);
+	let t: syntax.tok;
+	for (true) {
+		syntax.lexnext(&l, &t);
+		if (t.kind == syntax.tkind.TK_EOF) { break; };
+	};
+	return l.errs > 0;
+};
+
+@test fn utf8_bom_is_ignored_only_at_source_start() void = {
+	let src: [128]u8;
+	let n: u64 = bomsource(src.ptr, "", "package main;");
+	let l: syntax.lex;
+	syntax.lexinit(&l, "t", src.ptr, n);
+	let t: syntax.tok;
+	syntax.lexnext(&l, &t);
+	assert(t.kind == syntax.tkind.TK_MODULE);
+	assert(t.line == 1 && t.col == 4);
+	assert(l.errs == 0);
+
+	assert(doublebomerror());
+	assert(bomerror("package ", "main;"));
+	assert(bomerror("// ", "\n"));
+	assert(bomerror("/* ", " */"));
+	assert(bomerror("\"", "\""));
+	assert(bomerror("'", "'"));
+};