data: validate and repair Japanese dictionaries

map/verifymap.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""Verify the text maps and dictionaries consumed by strans."""
+
+import argparse
+import sys
+from pathlib import Path
+
+
+def verify(path):
+	errors = []
+	try:
+		data = path.read_bytes()
+	except OSError as error:
+		return [f"{path}: {error.strerror}"]
+	try:
+		text = data.decode("utf-8")
+	except UnicodeDecodeError as error:
+		return [f"{path}: invalid UTF-8: {error}"]
+
+	keys = {}
+	for lineno, line in enumerate(text.split("\n"), 1):
+		where = f"{path}:{lineno}"
+		if "\r" in line:
+			errors.append(f"{where}: carriage return is not canonical")
+			line = line.replace("\r", "")
+		if not line or line.startswith(";"):
+			continue
+		if "\0" in line:
+			errors.append(f"{where}: embedded NUL")
+		if line.count("\t") != 1:
+			errors.append(f"{where}: expected exactly one tab")
+			continue
+		key, value = line.split("\t")
+		if not key:
+			errors.append(f"{where}: empty key")
+		if len(key) > 64:
+			errors.append(f"{where}: key has {len(key)} runes; maximum is 64")
+		if key in keys:
+			errors.append(f"{where}: duplicate key; first defined on line {keys[key]}")
+		else:
+			keys[key] = lineno
+		if not value:
+			errors.append(f"{where}: empty value")
+			continue
+		if value != " ".join(value.split(" ")):
+			errors.append(f"{where}: noncanonical candidate spacing")
+		values = value.split(" ")
+		if len(values) != len(set(values)):
+			errors.append(f"{where}: duplicate candidate")
+	return errors
+
+
+def main():
+	parser = argparse.ArgumentParser(description=__doc__)
+	parser.add_argument("file", nargs="+", type=Path)
+	args = parser.parse_args()
+	errors = []
+	for path in args.file:
+		errors.extend(verify(path))
+	for error in errors:
+		print(error, file=sys.stderr)
+	return bool(errors)
+
+
+if __name__ == "__main__":
+	raise SystemExit(main())