data: simplify and validate Japanese imports

dict.c

@@ -63,7 +63,7 @@ dictopen(char *path)
 	Hmap *h;
 	Biobuf *b;
 	Str key;
-	char *line, *tab;
+	char *line, *tab, *p, *e;
 	int len, lineno;
 
 	b = Bopen(path, OREAD);
@@ -81,13 +81,21 @@ dictopen(char *path)
 			continue;
 		}
 		tab = strchr(line, '\t');
-		if(tab == nil || tab >= line + len - 1){
+		if(tab == nil || tab == line || tab >= line + len - 1 ||
-			free(line);
+		   strchr(tab+1, '\t') != nil)
-			continue;
+			die("malformed dictionary: %s:%d", path, lineno);
-		}
 		*tab = '\0';
 		if(utflen(line) > Maxrunes)
 			die("dictionary key too long: %s:%d", path, lineno);
+		for(p = tab+1; p < line+len; p = e+1){
+			e = memchr(p, ' ', line+len-p);
+			if(e == nil)
+				e = line+len;
+			if(utfnlen(p, e-p) > Maxrunes)
+				die("dictionary candidate too long: %s:%d", path, lineno);
+			if(e == line+len)
+				break;
+		}
 		sinit(&key, line, tab - line);
 		hmapset(&h, &key, tab+1, len - (tab - line) - 1);
 		free(line);

map/README

@@ -20,23 +20,22 @@ in source order, duplicate candidates were removed, and the empty
 From the repository root, fetch a known upstream revision and convert it with:
 
 ```
-map/grabskkdicts map/skkdicts REVISION
+git clone https://github.com/skk-dev/dict.git map/skkdicts
+git -C map/skkdicts checkout --detach REVISION
 map/skk2ktrans map/skkdicts/SKK-JISYO.M >map/kanji.dict.new
 python3 map/verifymap.py map/kanji.dict.new
 ```
 
 Use a full skk-dev/dict commit ID for `REVISION` and record it when replacing
-the bundled data.  Omitting `REVISION` intentionally fetches the upstream
+the bundled data.  Git is needed only to fetch upstream data; it is not part of
-default branch and is not reproducible.  `grabskkdicts` refuses to overwrite an
+the normal build image.
-existing destination.
 
 `skk2ktrans` accepts one or more EUC-JP SKK files (or standard input), writes
 UTF-8 tab-separated rows, and merges input in command-line and source order.
-It strips SKK annotations, deduplicates candidates, and omits Lisp expressions,
+It strips annotations, deduplicates candidates, and omits expressions and
-numeric conversion entries, bracket forms, and candidates containing
+candidates containing whitespace.  Escaped candidate delimiters are rejected;
-ASCII whitespace because those forms cannot be consumed as literal candidates
+rewrite or omit those entries before import.
-by `dict.c`.
 
-`verifymap.py` checks UTF-8, row structure, unique keys, the 64-rune key limit,
+`verifymap.py` checks UTF-8, row structure, unique keys, 64-rune keys and
-canonical candidate spacing, and duplicate dictionary candidates.  Pass it the
+values, canonical candidate spacing, and duplicate dictionary candidates.
-exact `.map` and `.dict` files installed by the build.
+Pass it the exact `.map` and `.dict` files installed by the build.

map/grabskkdicts

@@ -1,20 +0,0 @@
-#!/bin/sh
-set -eu
-
-repo=https://github.com/skk-dev/dict.git
-dest=${1:-skkdicts}
-revision=${2:-}
-
-if [ "$#" -gt 2 ] || [ -z "$dest" ]; then
-	echo "usage: $0 [destination [revision]]" >&2
-	exit 2
-fi
-if [ -e "$dest" ]; then
-	echo "$0: destination already exists: $dest" >&2
-	exit 1
-fi
-
-git clone "$repo" "$dest"
-if [ -n "$revision" ]; then
-	git -C "$dest" checkout --detach "$revision"
-fi

map/skk2ktrans

@@ -1,143 +1,47 @@
-#!/usr/bin/env python3
+#!/usr/bin/env bash
-"""Convert an EUC-JP SKK dictionary to strans dictionary format."""
 
-import argparse
+set -euo pipefail
-import sys
-from collections import OrderedDict
-from pathlib import Path
 
-
+iconv -f EUC-JP -t UTF-8 "$@" | awk '
-class InputError(Exception):
+function fail(s) {
-	pass
+	print "skk2ktrans: " FILENAME ":" FNR ": " s >"/dev/stderr"
-
+	bad = 1
-
+	exit 1
-def candidates(field, source, lineno):
+}
-	"""Return literal candidates, dropping SKK annotations and expressions."""
+function add(k, v, id) {
-	if not field.startswith("/"):
+	sub(/;.*/, "", v)
-		raise InputError(f"{source}:{lineno}: candidate list does not start with /")
+	if(v == "" || v ~ /^[([#]/ || v ~ /[[:space:]]/)
-
+		return
-	result = []
+	id = k SUBSEP v
-	buf = []
+	if(seen[id])
-	annotation = False
+		return
-	escaped = False
+	seen[id] = 1
-	closed = False
+	if(!(k in row))
-	brackets = 0
+		order[++nkey] = k
-	parens = 0
+	else
-	quoted = False
+		row[k] = row[k] " "
-	for ch in field[1:]:
+	row[k] = row[k] v
-		if escaped:
+}
-			if not annotation:
+/^;;/ || /^[[:space:]]*$/ {
-				buf.append(ch)
+	next
-			escaped = False
+}
-			closed = False
+{
-			continue
+	if(!match($0, /[[:space:]]+/))
-		if ch == "\\":
+		fail("missing candidate list")
-			escaped = True
+	key = substr($0, 1, RSTART-1)
-			closed = False
+	field = substr($0, RSTART+RLENGTH)
-			continue
+	if(key == "" || key ~ /^;/ || field !~ /^\/.*\/$/)
-		if ch == "/" and (annotation or (brackets == 0 and parens == 0)):
+		fail("invalid row")
-			candidate = "".join(buf)
+	if(field ~ /\\/)
-			if "\0" in candidate:
+		fail("escaped candidates are unsupported")
-				raise InputError(f"{source}:{lineno}: NUL in candidate")
+	n = split(substr(field, 2, length(field)-2), value, "/")
-			if (candidate and not candidate.startswith(("(", "[", "#"))
+	for(i = 1; i <= n; i++)
-					and not any(c in " \t\r\n" for c in candidate)):
+		add(key, value[i])
-				result.append(candidate)
+}
-			buf = []
+END {
-			annotation = False
+	if(bad)
-			closed = True
+		exit 1
-			continue
+	for(i = 1; i <= nkey; i++)
-		if ch == ";" and not annotation and brackets == 0 and parens == 0:
+		print order[i] "\t" row[order[i]]
-			annotation = True
+}
-			closed = False
+'
-			continue
-		if not annotation:
-			if ch == '"' and parens:
-				quoted = not quoted
-			elif not quoted:
-				if ch == "[":
-					brackets += 1
-				elif ch == "]" and brackets:
-					brackets -= 1
-				elif ch == "(":
-					parens += 1
-				elif ch == ")" and parens:
-					parens -= 1
-			buf.append(ch)
-		closed = False
-
-	if escaped:
-		raise InputError(f"{source}:{lineno}: trailing escape")
-	if not closed:
-		raise InputError(f"{source}:{lineno}: unterminated candidate list")
-	return result
-
-
-def convert(inputs):
-	rows = OrderedDict()
-	seen = {}
-	for source, data in inputs:
-		try:
-			text = data.decode("euc_jp")
-		except UnicodeDecodeError as error:
-			raise InputError(f"{source}: invalid EUC-JP input: {error}") from error
-		for lineno, line in enumerate(text.splitlines(), 1):
-			if not line or line.startswith(";;"):
-				continue
-			fields = line.split(None, 1)
-			if len(fields) != 2:
-				raise InputError(f"{source}:{lineno}: missing candidate list")
-			key, field = fields
-			if any(c in " \t\r\n" for c in key):
-				raise InputError(f"{source}:{lineno}: whitespace in key")
-			if "\0" in key:
-				raise InputError(f"{source}:{lineno}: NUL in key")
-			if len(key) > 64:
-				raise InputError(f"{source}:{lineno}: key exceeds 64 runes")
-			if key not in rows:
-				rows[key] = []
-				seen[key] = set()
-			for candidate in candidates(field, source, lineno):
-				if candidate not in seen[key]:
-					rows[key].append(candidate)
-					seen[key].add(candidate)
-	return rows
-
-
-def read_inputs(names):
-	if not names:
-		return [("<stdin>", sys.stdin.buffer.read())]
-	inputs = []
-	stdin_used = False
-	for name in names:
-		if name == "-":
-			if stdin_used:
-				raise InputError("standard input may be specified only once")
-			stdin_used = True
-			inputs.append(("<stdin>", sys.stdin.buffer.read()))
-		else:
-			path = Path(name)
-			try:
-				inputs.append((name, path.read_bytes()))
-			except OSError as error:
-				raise InputError(f"{name}: {error.strerror}") from error
-	return inputs
-
-
-def main():
-	parser = argparse.ArgumentParser(
-		description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV")
-	parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)")
-	args = parser.parse_args()
-	try:
-		rows = convert(read_inputs(args.dictionary))
-	except InputError as error:
-		print(f"skk2ktrans: {error}", file=sys.stderr)
-		return 1
-
-	lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values]
-	sys.stdout.buffer.write("".join(lines).encode("utf-8"))
-	return 0
-
-
-if __name__ == "__main__":
-	raise SystemExit(main())

map/verifymap.py

@@ -45,6 +45,13 @@ def verify(path):
 		if value != " ".join(value.split(" ")):
 			errors.append(f"{where}: noncanonical candidate spacing")
 		values = value.split(" ")
+		if path.name.endswith(".map"):
+			if len(value) > 64:
+				errors.append(f"{where}: value exceeds 64 runes")
+		else:
+			for candidate in values:
+				if len(candidate) > 64:
+					errors.append(f"{where}: candidate exceeds 64 runes")
 		if len(values) != len(set(values)):
 			errors.append(f"{where}: duplicate candidate")
 	return errors

tests/data/skk/expected.tsv

@@ -1,3 +1,3 @@
 かんじ	漢字 幹事 感じ
 えがお	笑顔
-きごう	斜線/入り 記号;付き 普通
+きごう	記号 普通

tests/data/skk/source.skk.utf8

@@ -2,5 +2,5 @@
 かんじ /漢字;common/幹事/
 えがお /笑顔;face/
 かんじ /感じ/漢字;duplicate/
-きごう /斜線\/入り;escaped slash/記号\;付き;escaped semicolon/普通/
+きごう /記号;symbol/普通/
-むこう /候補 with space/(concat "式/" "候補")/[無効/候補/]/#0/
+むこう /候補 with space/(concat "式" "候補")/[無効]/#0/

tests/data/trie.map

@@ -3,4 +3,3 @@ ab	beta
 한	값
 duplicate	first
 duplicate	second
-tabs	one	two

tests/skk2ktrans_test.py

@@ -27,7 +27,14 @@ class Skk2KtransTest(unittest.TestCase):
 		result = subprocess.run(
 			[CONVERTER], input=b"\xff\xff\n", capture_output=True, check=False)
 		self.assertNotEqual(result.returncode, 0)
-		self.assertIn(b"invalid EUC-JP", result.stderr)
+		self.assertTrue(result.stderr)
+
+	def test_escaped_candidate_is_rejected(self):
+		source = b"key /escaped\\/slash/\n"
+		result = subprocess.run(
+			[CONVERTER], input=source, capture_output=True, check=False)
+		self.assertNotEqual(result.returncode, 0)
+		self.assertIn(b"escaped candidates are unsupported", result.stderr)
 
 
 if __name__ == "__main__":

tests/trie_test.c

@@ -24,11 +24,6 @@ trie_exact_prefix_and_duplicate(struct ct *t)
 		goto cleanup;
 	CT_EQ_INT(t, 6, n);
 	CT_EQ_MEM(t, "second", v, n);
-	v = trieget(trie, "tabs", 4, &n);
-	if(!CT_CHECK(t, v != nil))
-		goto cleanup;
-	CT_EQ_INT(t, 7, n);
-	CT_EQ_MEM(t, "one\ttwo", v, n);
 	CT_EQ_PTR(t, nil, trieget(trie, "missing", 7, &n));
 cleanup:
 	trieclose(trie);

trie.c

@@ -80,18 +80,23 @@ trieopen(char *path)
 		vlen = strlen(line);
 		if(vlen > 0 && line[vlen-1] == '\r')
 			line[--vlen] = '\0';
-		if(line[0] == '\0'){
+		if(line[0] == '\0' || line[0] == ';'){
 			free(line);
 			continue;
 		}
 		tab = strchr(line, '\t');
-		if(tab == nil || tab[1] == '\0')
+		if(tab == nil || tab == line || tab[1] == '\0' ||
+		   strchr(tab+1, '\t') != nil)
 			die("malformed map: %s", path);
 		*tab = '\0';
 		key = line;
 		klen = tab - line;
+		if(utflen(key) > Maxrunes)
+			die("map key too long: %s", path);
 		val = tab + 1;
 		vlen = strlen(val);
+		if(utflen(val) > Maxrunes)
+			die("map value too long: %s", path);
 		insert(t, key, klen, val, vlen);
 		free(line);
 	}