data: validate and repair Japanese dictionaries

map/skk2ktrans

@@ -1,2 +1,143 @@
-#!/bin/rc
-tcs -sf ujis | awk '$1 !~ /;;/ {gsub("(^\/|\/$)", "", $2); gsub(" ", "	"); gsub("\/", " ", $2);} {print}'
+#!/usr/bin/env python3
+"""Convert an EUC-JP SKK dictionary to strans dictionary format."""
+
+import argparse
+import sys
+from collections import OrderedDict
+from pathlib import Path
+
+
+class InputError(Exception):
+	pass
+
+
+def candidates(field, source, lineno):
+	"""Return literal candidates, dropping SKK annotations and expressions."""
+	if not field.startswith("/"):
+		raise InputError(f"{source}:{lineno}: candidate list does not start with /")
+
+	result = []
+	buf = []
+	annotation = False
+	escaped = False
+	closed = False
+	brackets = 0
+	parens = 0
+	quoted = False
+	for ch in field[1:]:
+		if escaped:
+			if not annotation:
+				buf.append(ch)
+			escaped = False
+			closed = False
+			continue
+		if ch == "\\":
+			escaped = True
+			closed = False
+			continue
+		if ch == "/" and (annotation or (brackets == 0 and parens == 0)):
+			candidate = "".join(buf)
+			if "\0" in candidate:
+				raise InputError(f"{source}:{lineno}: NUL in candidate")
+			if (candidate and not candidate.startswith(("(", "[", "#"))
+					and not any(c in " \t\r\n" for c in candidate)):
+				result.append(candidate)
+			buf = []
+			annotation = False
+			closed = True
+			continue
+		if ch == ";" and not annotation and brackets == 0 and parens == 0:
+			annotation = True
+			closed = False
+			continue
+		if not annotation:
+			if ch == '"' and parens:
+				quoted = not quoted
+			elif not quoted:
+				if ch == "[":
+					brackets += 1
+				elif ch == "]" and brackets:
+					brackets -= 1
+				elif ch == "(":
+					parens += 1
+				elif ch == ")" and parens:
+					parens -= 1
+			buf.append(ch)
+		closed = False
+
+	if escaped:
+		raise InputError(f"{source}:{lineno}: trailing escape")
+	if not closed:
+		raise InputError(f"{source}:{lineno}: unterminated candidate list")
+	return result
+
+
+def convert(inputs):
+	rows = OrderedDict()
+	seen = {}
+	for source, data in inputs:
+		try:
+			text = data.decode("euc_jp")
+		except UnicodeDecodeError as error:
+			raise InputError(f"{source}: invalid EUC-JP input: {error}") from error
+		for lineno, line in enumerate(text.splitlines(), 1):
+			if not line or line.startswith(";;"):
+				continue
+			fields = line.split(None, 1)
+			if len(fields) != 2:
+				raise InputError(f"{source}:{lineno}: missing candidate list")
+			key, field = fields
+			if any(c in " \t\r\n" for c in key):
+				raise InputError(f"{source}:{lineno}: whitespace in key")
+			if "\0" in key:
+				raise InputError(f"{source}:{lineno}: NUL in key")
+			if len(key) > 64:
+				raise InputError(f"{source}:{lineno}: key exceeds 64 runes")
+			if key not in rows:
+				rows[key] = []
+				seen[key] = set()
+			for candidate in candidates(field, source, lineno):
+				if candidate not in seen[key]:
+					rows[key].append(candidate)
+					seen[key].add(candidate)
+	return rows
+
+
+def read_inputs(names):
+	if not names:
+		return [("<stdin>", sys.stdin.buffer.read())]
+	inputs = []
+	stdin_used = False
+	for name in names:
+		if name == "-":
+			if stdin_used:
+				raise InputError("standard input may be specified only once")
+			stdin_used = True
+			inputs.append(("<stdin>", sys.stdin.buffer.read()))
+		else:
+			path = Path(name)
+			try:
+				inputs.append((name, path.read_bytes()))
+			except OSError as error:
+				raise InputError(f"{name}: {error.strerror}") from error
+	return inputs
+
+
+def main():
+	parser = argparse.ArgumentParser(
+		description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV")
+	parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)")
+	args = parser.parse_args()
+	try:
+		rows = convert(read_inputs(args.dictionary))
+	except InputError as error:
+		print(f"skk2ktrans: {error}", file=sys.stderr)
+		return 1
+
+	lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values]
+	sys.stdout.buffer.write("".join(lines).encode("utf-8"))
+	return 0
+
+
+if __name__ == "__main__":
+	raise SystemExit(main())