data: simplify and validate Japanese imports

map/README

@@ -20,23 +20,22 @@ in source order, duplicate candidates were removed, and the empty
 From the repository root, fetch a known upstream revision and convert it with:
 
 ```
-map/grabskkdicts map/skkdicts REVISION
+git clone https://github.com/skk-dev/dict.git map/skkdicts
+git -C map/skkdicts checkout --detach REVISION
 map/skk2ktrans map/skkdicts/SKK-JISYO.M >map/kanji.dict.new
 python3 map/verifymap.py map/kanji.dict.new
 ```
 
 Use a full skk-dev/dict commit ID for `REVISION` and record it when replacing
-the bundled data.  Omitting `REVISION` intentionally fetches the upstream
-default branch and is not reproducible.  `grabskkdicts` refuses to overwrite an
-existing destination.
+the bundled data.  Git is needed only to fetch upstream data; it is not part of
+the normal build image.
 
 `skk2ktrans` accepts one or more EUC-JP SKK files (or standard input), writes
 UTF-8 tab-separated rows, and merges input in command-line and source order.
-It strips SKK annotations, deduplicates candidates, and omits Lisp expressions,
-numeric conversion entries, bracket forms, and candidates containing
-ASCII whitespace because those forms cannot be consumed as literal candidates
-by `dict.c`.
+It strips annotations, deduplicates candidates, and omits expressions and
+candidates containing whitespace.  Escaped candidate delimiters are rejected;
+rewrite or omit those entries before import.
 
-`verifymap.py` checks UTF-8, row structure, unique keys, the 64-rune key limit,
-canonical candidate spacing, and duplicate dictionary candidates.  Pass it the
-exact `.map` and `.dict` files installed by the build.
+`verifymap.py` checks UTF-8, row structure, unique keys, 64-rune keys and
+values, canonical candidate spacing, and duplicate dictionary candidates.
+Pass it the exact `.map` and `.dict` files installed by the build.

map/grabskkdicts

@@ -1,20 +0,0 @@
-#!/bin/sh
-set -eu
-
-repo=https://github.com/skk-dev/dict.git
-dest=${1:-skkdicts}
-revision=${2:-}
-
-if [ "$#" -gt 2 ] || [ -z "$dest" ]; then
-	echo "usage: $0 [destination [revision]]" >&2
-	exit 2
-fi
-if [ -e "$dest" ]; then
-	echo "$0: destination already exists: $dest" >&2
-	exit 1
-fi
-
-git clone "$repo" "$dest"
-if [ -n "$revision" ]; then
-	git -C "$dest" checkout --detach "$revision"
-fi

map/skk2ktrans

@@ -1,143 +1,47 @@
-#!/usr/bin/env python3
-"""Convert an EUC-JP SKK dictionary to strans dictionary format."""
+#!/usr/bin/env bash
 
-import argparse
-import sys
-from collections import OrderedDict
-from pathlib import Path
+set -euo pipefail
 
-
-class InputError(Exception):
-	pass
-
-
-def candidates(field, source, lineno):
-	"""Return literal candidates, dropping SKK annotations and expressions."""
-	if not field.startswith("/"):
-		raise InputError(f"{source}:{lineno}: candidate list does not start with /")
-
-	result = []
-	buf = []
-	annotation = False
-	escaped = False
-	closed = False
-	brackets = 0
-	parens = 0
-	quoted = False
-	for ch in field[1:]:
-		if escaped:
-			if not annotation:
-				buf.append(ch)
-			escaped = False
-			closed = False
-			continue
-		if ch == "\\":
-			escaped = True
-			closed = False
-			continue
-		if ch == "/" and (annotation or (brackets == 0 and parens == 0)):
-			candidate = "".join(buf)
-			if "\0" in candidate:
-				raise InputError(f"{source}:{lineno}: NUL in candidate")
-			if (candidate and not candidate.startswith(("(", "[", "#"))
-					and not any(c in " \t\r\n" for c in candidate)):
-				result.append(candidate)
-			buf = []
-			annotation = False
-			closed = True
-			continue
-		if ch == ";" and not annotation and brackets == 0 and parens == 0:
-			annotation = True
-			closed = False
-			continue
-		if not annotation:
-			if ch == '"' and parens:
-				quoted = not quoted
-			elif not quoted:
-				if ch == "[":
-					brackets += 1
-				elif ch == "]" and brackets:
-					brackets -= 1
-				elif ch == "(":
-					parens += 1
-				elif ch == ")" and parens:
-					parens -= 1
-			buf.append(ch)
-		closed = False
-
-	if escaped:
-		raise InputError(f"{source}:{lineno}: trailing escape")
-	if not closed:
-		raise InputError(f"{source}:{lineno}: unterminated candidate list")
-	return result
-
-
-def convert(inputs):
-	rows = OrderedDict()
-	seen = {}
-	for source, data in inputs:
-		try:
-			text = data.decode("euc_jp")
-		except UnicodeDecodeError as error:
-			raise InputError(f"{source}: invalid EUC-JP input: {error}") from error
-		for lineno, line in enumerate(text.splitlines(), 1):
-			if not line or line.startswith(";;"):
-				continue
-			fields = line.split(None, 1)
-			if len(fields) != 2:
-				raise InputError(f"{source}:{lineno}: missing candidate list")
-			key, field = fields
-			if any(c in " \t\r\n" for c in key):
-				raise InputError(f"{source}:{lineno}: whitespace in key")
-			if "\0" in key:
-				raise InputError(f"{source}:{lineno}: NUL in key")
-			if len(key) > 64:
-				raise InputError(f"{source}:{lineno}: key exceeds 64 runes")
-			if key not in rows:
-				rows[key] = []
-				seen[key] = set()
-			for candidate in candidates(field, source, lineno):
-				if candidate not in seen[key]:
-					rows[key].append(candidate)
-					seen[key].add(candidate)
-	return rows
-
-
-def read_inputs(names):
-	if not names:
-		return [("<stdin>", sys.stdin.buffer.read())]
-	inputs = []
-	stdin_used = False
-	for name in names:
-		if name == "-":
-			if stdin_used:
-				raise InputError("standard input may be specified only once")
-			stdin_used = True
-			inputs.append(("<stdin>", sys.stdin.buffer.read()))
-		else:
-			path = Path(name)
-			try:
-				inputs.append((name, path.read_bytes()))
-			except OSError as error:
-				raise InputError(f"{name}: {error.strerror}") from error
-	return inputs
-
-
-def main():
-	parser = argparse.ArgumentParser(
-		description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV")
-	parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)")
-	args = parser.parse_args()
-	try:
-		rows = convert(read_inputs(args.dictionary))
-	except InputError as error:
-		print(f"skk2ktrans: {error}", file=sys.stderr)
-		return 1
-
-	lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values]
-	sys.stdout.buffer.write("".join(lines).encode("utf-8"))
-	return 0
-
-
-if __name__ == "__main__":
-	raise SystemExit(main())
+iconv -f EUC-JP -t UTF-8 "$@" | awk '
+function fail(s) {
+	print "skk2ktrans: " FILENAME ":" FNR ": " s >"/dev/stderr"
+	bad = 1
+	exit 1
+}
+function add(k, v, id) {
+	sub(/;.*/, "", v)
+	if(v == "" || v ~ /^[([#]/ || v ~ /[[:space:]]/)
+		return
+	id = k SUBSEP v
+	if(seen[id])
+		return
+	seen[id] = 1
+	if(!(k in row))
+		order[++nkey] = k
+	else
+		row[k] = row[k] " "
+	row[k] = row[k] v
+}
+/^;;/ || /^[[:space:]]*$/ {
+	next
+}
+{
+	if(!match($0, /[[:space:]]+/))
+		fail("missing candidate list")
+	key = substr($0, 1, RSTART-1)
+	field = substr($0, RSTART+RLENGTH)
+	if(key == "" || key ~ /^;/ || field !~ /^\/.*\/$/)
+		fail("invalid row")
+	if(field ~ /\\/)
+		fail("escaped candidates are unsupported")
+	n = split(substr(field, 2, length(field)-2), value, "/")
+	for(i = 1; i <= n; i++)
+		add(key, value[i])
+}
+END {
+	if(bad)
+		exit 1
+	for(i = 1; i <= nkey; i++)
+		print order[i] "\t" row[order[i]]
+}
+'

map/verifymap.py

@@ -45,6 +45,13 @@ def verify(path):
 		if value != " ".join(value.split(" ")):
 			errors.append(f"{where}: noncanonical candidate spacing")
 		values = value.split(" ")
+		if path.name.endswith(".map"):
+			if len(value) > 64:
+				errors.append(f"{where}: value exceeds 64 runes")
+		else:
+			for candidate in values:
+				if len(candidate) > 64:
+					errors.append(f"{where}: candidate exceeds 64 runes")
 		if len(values) != len(set(values)):
 			errors.append(f"{where}: duplicate candidate")
 	return errors