data: validate and repair Japanese dictionaries

Makefile

@@ -56,6 +56,8 @@ test:
 verify-map:
 	python3 map/mktelex.py | cmp - map/telex.map
 	python3 -B map/mkemoji | cmp - map/emoji.dict
+	python3 -B map/verifymap.py map/*.map map/*.dict
 	python3 -B tests/mkemoji_test.py
+	python3 -B tests/skk2ktrans_test.py
 
 .PHONY: all check test verify-map clean xim gtk bench docker docker-check

map/README

@@ -1,12 +1,42 @@
-#
-# The following are a set of tools to obtain and process dictionaries from the SKK project in order to use them with ktrans(1).
-#
-# grabskkdicts pulls the skk kana-kanji conversion dictionaries from the skk-dev/dict repo.
-# skk2ktrans takes an skk dictionary and converts it into a kanji jisho suitable to be used with ktrans(1).
-#
-# You can fetch and convert all the dictionaries by running this file.
-#
+# Japanese dictionary data
 
-grabskkdicts
-for(d in skkdicts/SKK-JISYO.*)
-	<$d skk2ktrans >$d.jisho
+`kanji.dict` is the historical dictionary bundled with strans.  Its own header
+identifies it as the SKK Medium dictionary, version 8.1 of May 24, 1995,
+rearranged for Plan 9 ktrans by Kenji Okamoto on February 17, 2000.  It was
+already present when this repository was created in commit
+`dcd1147638f908eca7e4b7616815e215022cc99d` on December 23, 2025.  No original
+SKK input file or upstream revision was committed, so the current file cannot
+honestly be regenerated byte-for-byte from repository artifacts.  The license
+notice in its header permits redistribution and modification under GPL version
+2 or later.
+
+The repository copy keeps that historical candidate order.  Duplicate keys
+were merged at their first occurrence, later unseen candidates were appended
+in source order, duplicate candidates were removed, and the empty
+`ようたつ` row was removed.
+
+## Importing a current SKK dictionary
+
+From the repository root, fetch a known upstream revision and convert it with:
+
+```
+map/grabskkdicts map/skkdicts REVISION
+map/skk2ktrans map/skkdicts/SKK-JISYO.M >map/kanji.dict.new
+python3 map/verifymap.py map/kanji.dict.new
+```
+
+Use a full skk-dev/dict commit ID for `REVISION` and record it when replacing
+the bundled data.  Omitting `REVISION` intentionally fetches the upstream
+default branch and is not reproducible.  `grabskkdicts` refuses to overwrite an
+existing destination.
+
+`skk2ktrans` accepts one or more EUC-JP SKK files (or standard input), writes
+UTF-8 tab-separated rows, and merges input in command-line and source order.
+It strips SKK annotations, deduplicates candidates, and omits Lisp expressions,
+numeric conversion entries, bracket forms, and candidates containing
+ASCII whitespace because those forms cannot be consumed as literal candidates
+by `dict.c`.
+
+`verifymap.py` checks UTF-8, row structure, unique keys, the 64-rune key limit,
+canonical candidate spacing, and duplicate dictionary candidates.  Pass it the
+exact `.map` and `.dict` files installed by the build.

map/grabskkdicts

@@ -1,2 +1,20 @@
-#!/bin/rc
-git/clone https://github.com/skk-dev/dict skkdicts
+#!/bin/sh
+set -eu
+
+repo=https://github.com/skk-dev/dict.git
+dest=${1:-skkdicts}
+revision=${2:-}
+
+if [ "$#" -gt 2 ] || [ -z "$dest" ]; then
+	echo "usage: $0 [destination [revision]]" >&2
+	exit 2
+fi
+if [ -e "$dest" ]; then
+	echo "$0: destination already exists: $dest" >&2
+	exit 1
+fi
+
+git clone "$repo" "$dest"
+if [ -n "$revision" ]; then
+	git -C "$dest" checkout --detach "$revision"
+fi

map/kanji.dict

@@ -60,7 +60,6 @@
 あたらs	新
 あつi	熱 暑 厚
 あつk	厚 熱 暑
-あつk	厚 熱 暑
 あつs	暑
 あつm	集 厚
 あつかe	扱
@@ -350,7 +349,6 @@
 かわいr	可愛
 かんj	感
 かんs	関
-かんj	感
 かんz	感
 かんがe	考
 かんしゃs	感謝
@@ -455,7 +453,7 @@
 こu	乞 請
 こy	来
 こうj	高
-こうしょう	鉱床 高尚
+こうしょう	鉱床 高尚 交渉
 こうぶつ	鉱物 好物
 こうりょs	考慮
 こえt	超
@@ -665,8 +663,6 @@
 ただc	直
 ただt	直
 たとe	例
-たおr	倒
-たおr	倒
 たのm	頼
 たのn	頼
 たのs	楽
@@ -742,7 +738,6 @@
 つみあげr	積み上げ
 つみあげt	積み上げ
 つめt	冷
-つづr	綴
 つよi	強
 つよk	強
 つらi	辛
@@ -793,7 +788,6 @@
 とt	取 撮 採
 とu	問
 とw	問
-とj	閉
 といあw	問い合
 とうj	投
 とおi	遠
@@ -913,13 +907,12 @@
 ねらt	狙
 ねらu	狙
 のb	述 伸 延
-のk	退
+のk	退 乗
 のm	飲 呑
 のn	飲
 のr	乗 載
 のs	載 乗
 のt	乗 載
-のk	乗
 のこr	残
 のこs	残
 のこt	残
@@ -1059,12 +1052,11 @@
 ぼうえいs	防衛
 まc	待
 まi	舞
-まj	交
+まj	交 混
 まk	負 巻
 まs	増
 まt	待
 まu	舞
-まj	混
 まz	混 先
 まいr	参 詣
 まいt	参 詣
@@ -1241,8 +1233,6 @@
 よわm	弱
 よわs	弱
 よわt	弱
-よろこb	喜 慶
-よろこb	喜 慶
 ろんj	論
 ろんz	論
 わk	分 湧
@@ -2267,7 +2257,6 @@
 がくわり	学割
 がけ	崖
 がし	樫
-かせい	火星
 がそ	画素
 がそかん	画素間
 がそごと	画素毎
@@ -2993,7 +2982,6 @@
 こうしゃ	後者 公社 校舎
 こうしゅう	講習 公衆
 こうしゅうかい	講習会
-こうしょう	交渉 鉱床
 こうしん	更新 交信 後身
 こうじ	麹 公示 高次 工事 孝二
 こうじつ	口実
@@ -4802,7 +4790,7 @@
 たいきゅうりょく	耐久力
 たいきょく	対極
 たいきん	大金
-たいく	体躯
+たいく	体躯 体育
 たいくつ	退屈
 たいけい	体系 体型
 たいけん	体験
@@ -4949,8 +4937,6 @@
 たんらく	短絡
 だ	騨 駄 陀 楕 舵 柁 打 惰 妥 堕 唾 田 朶
 だい	大 第 内 代 台 題 醍
-たいく	体育
-たいいく	体育
 だいいち	第一
 だいがく	大学
 だいがくいん	大学院
@@ -7314,7 +7300,6 @@
 よこう	予行 予稿
 よこうち	横内
 よこく	予告
-ようたつ	
 よこて	横手
 よこはま	横浜
 よこもじ	横文字
@@ -7426,7 +7411,7 @@
 りゅうど	粒度
 りゆう	理由
 りょ	虜 旅 慮 侶
-りょう	量 寮 両 領 陵 遼 諒 良 糧 稜 瞭 療 猟 涼 梁 料 凌 僚 亮 了 漁 糧 粮 霊
+りょう	量 寮 両 領 陵 遼 諒 良 糧 稜 瞭 療 猟 涼 梁 料 凌 僚 亮 了 漁 粮 霊
 りょうあし	両足
 りょういき	領域
 りょうか	量化

map/skk2ktrans

@@ -1,2 +1,143 @@
-#!/bin/rc
-tcs -sf ujis | awk '$1 !~ /;;/ {gsub("(^\/|\/$)", "", $2); gsub(" ", "	"); gsub("\/", " ", $2);} {print}'
+#!/usr/bin/env python3
+"""Convert an EUC-JP SKK dictionary to strans dictionary format."""
+
+import argparse
+import sys
+from collections import OrderedDict
+from pathlib import Path
+
+
+class InputError(Exception):
+	pass
+
+
+def candidates(field, source, lineno):
+	"""Return literal candidates, dropping SKK annotations and expressions."""
+	if not field.startswith("/"):
+		raise InputError(f"{source}:{lineno}: candidate list does not start with /")
+
+	result = []
+	buf = []
+	annotation = False
+	escaped = False
+	closed = False
+	brackets = 0
+	parens = 0
+	quoted = False
+	for ch in field[1:]:
+		if escaped:
+			if not annotation:
+				buf.append(ch)
+			escaped = False
+			closed = False
+			continue
+		if ch == "\\":
+			escaped = True
+			closed = False
+			continue
+		if ch == "/" and (annotation or (brackets == 0 and parens == 0)):
+			candidate = "".join(buf)
+			if "\0" in candidate:
+				raise InputError(f"{source}:{lineno}: NUL in candidate")
+			if (candidate and not candidate.startswith(("(", "[", "#"))
+					and not any(c in " \t\r\n" for c in candidate)):
+				result.append(candidate)
+			buf = []
+			annotation = False
+			closed = True
+			continue
+		if ch == ";" and not annotation and brackets == 0 and parens == 0:
+			annotation = True
+			closed = False
+			continue
+		if not annotation:
+			if ch == '"' and parens:
+				quoted = not quoted
+			elif not quoted:
+				if ch == "[":
+					brackets += 1
+				elif ch == "]" and brackets:
+					brackets -= 1
+				elif ch == "(":
+					parens += 1
+				elif ch == ")" and parens:
+					parens -= 1
+			buf.append(ch)
+		closed = False
+
+	if escaped:
+		raise InputError(f"{source}:{lineno}: trailing escape")
+	if not closed:
+		raise InputError(f"{source}:{lineno}: unterminated candidate list")
+	return result
+
+
+def convert(inputs):
+	rows = OrderedDict()
+	seen = {}
+	for source, data in inputs:
+		try:
+			text = data.decode("euc_jp")
+		except UnicodeDecodeError as error:
+			raise InputError(f"{source}: invalid EUC-JP input: {error}") from error
+		for lineno, line in enumerate(text.splitlines(), 1):
+			if not line or line.startswith(";;"):
+				continue
+			fields = line.split(None, 1)
+			if len(fields) != 2:
+				raise InputError(f"{source}:{lineno}: missing candidate list")
+			key, field = fields
+			if any(c in " \t\r\n" for c in key):
+				raise InputError(f"{source}:{lineno}: whitespace in key")
+			if "\0" in key:
+				raise InputError(f"{source}:{lineno}: NUL in key")
+			if len(key) > 64:
+				raise InputError(f"{source}:{lineno}: key exceeds 64 runes")
+			if key not in rows:
+				rows[key] = []
+				seen[key] = set()
+			for candidate in candidates(field, source, lineno):
+				if candidate not in seen[key]:
+					rows[key].append(candidate)
+					seen[key].add(candidate)
+	return rows
+
+
+def read_inputs(names):
+	if not names:
+		return [("<stdin>", sys.stdin.buffer.read())]
+	inputs = []
+	stdin_used = False
+	for name in names:
+		if name == "-":
+			if stdin_used:
+				raise InputError("standard input may be specified only once")
+			stdin_used = True
+			inputs.append(("<stdin>", sys.stdin.buffer.read()))
+		else:
+			path = Path(name)
+			try:
+				inputs.append((name, path.read_bytes()))
+			except OSError as error:
+				raise InputError(f"{name}: {error.strerror}") from error
+	return inputs
+
+
+def main():
+	parser = argparse.ArgumentParser(
+		description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV")
+	parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)")
+	args = parser.parse_args()
+	try:
+		rows = convert(read_inputs(args.dictionary))
+	except InputError as error:
+		print(f"skk2ktrans: {error}", file=sys.stderr)
+		return 1
+
+	lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values]
+	sys.stdout.buffer.write("".join(lines).encode("utf-8"))
+	return 0
+
+
+if __name__ == "__main__":
+	raise SystemExit(main())

map/verifymap.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""Verify the text maps and dictionaries consumed by strans."""
+
+import argparse
+import sys
+from pathlib import Path
+
+
+def verify(path):
+	errors = []
+	try:
+		data = path.read_bytes()
+	except OSError as error:
+		return [f"{path}: {error.strerror}"]
+	try:
+		text = data.decode("utf-8")
+	except UnicodeDecodeError as error:
+		return [f"{path}: invalid UTF-8: {error}"]
+
+	keys = {}
+	for lineno, line in enumerate(text.split("\n"), 1):
+		where = f"{path}:{lineno}"
+		if "\r" in line:
+			errors.append(f"{where}: carriage return is not canonical")
+			line = line.replace("\r", "")
+		if not line or line.startswith(";"):
+			continue
+		if "\0" in line:
+			errors.append(f"{where}: embedded NUL")
+		if line.count("\t") != 1:
+			errors.append(f"{where}: expected exactly one tab")
+			continue
+		key, value = line.split("\t")
+		if not key:
+			errors.append(f"{where}: empty key")
+		if len(key) > 64:
+			errors.append(f"{where}: key has {len(key)} runes; maximum is 64")
+		if key in keys:
+			errors.append(f"{where}: duplicate key; first defined on line {keys[key]}")
+		else:
+			keys[key] = lineno
+		if not value:
+			errors.append(f"{where}: empty value")
+			continue
+		if value != " ".join(value.split(" ")):
+			errors.append(f"{where}: noncanonical candidate spacing")
+		values = value.split(" ")
+		if len(values) != len(set(values)):
+			errors.append(f"{where}: duplicate candidate")
+	return errors
+
+
+def main():
+	parser = argparse.ArgumentParser(description=__doc__)
+	parser.add_argument("file", nargs="+", type=Path)
+	args = parser.parse_args()
+	errors = []
+	for path in args.file:
+		errors.extend(verify(path))
+	for error in errors:
+		print(error, file=sys.stderr)
+	return bool(errors)
+
+
+if __name__ == "__main__":
+	raise SystemExit(main())

tests/data/skk/expected.tsv

@@ -0,0 +1,3 @@
+かんじ	漢字 幹事 感じ
+えがお	笑顔
+きごう	斜線/入り 記号;付き 普通

tests/data/skk/source.skk.utf8

@@ -0,0 +1,6 @@
+;; converter fixture; the test transcodes this file to EUC-JP
+かんじ /漢字;common/幹事/
+えがお /笑顔;face/
+かんじ /感じ/漢字;duplicate/
+きごう /斜線\/入り;escaped slash/記号\;付き;escaped semicolon/普通/
+むこう /候補 with space/(concat "式/" "候補")/[無効/候補/]/#0/

tests/skk2ktrans_test.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+CONVERTER = ROOT / "map" / "skk2ktrans"
+FIXTURES = Path(__file__).parent / "data" / "skk"
+
+
+class Skk2KtransTest(unittest.TestCase):
+	def test_euc_jp_conversion_is_stable_and_literal(self):
+		source = (FIXTURES / "source.skk.utf8").read_text(encoding="utf-8")
+		expected = (FIXTURES / "expected.tsv").read_bytes()
+		with tempfile.TemporaryDirectory() as directory:
+			input_path = Path(directory) / "source.skk"
+			input_path.write_bytes(source.encode("euc_jp"))
+			result = subprocess.run(
+				[CONVERTER, input_path], capture_output=True, check=False)
+		self.assertEqual(result.returncode, 0, result.stderr.decode())
+		self.assertEqual(result.stdout, expected)
+
+	def test_invalid_euc_jp_is_rejected(self):
+		result = subprocess.run(
+			[CONVERTER], input=b"\xff\xff\n", capture_output=True, check=False)
+		self.assertNotEqual(result.returncode, 0)
+		self.assertIn(b"invalid EUC-JP", result.stderr)
+
+
+if __name__ == "__main__":
+	unittest.main()