data: validate and repair Japanese dictionaries

2026-08-12 15:34:00 +09:00
parent f0ba7fc5f6
commit d2fa51ceba
9 changed files with 324 additions and 39 deletions
--- a/map/README
+++ b/map/README
@@ -1,12 +1,42 @@
-#
-# The following are a set of tools to obtain and process dictionaries from the SKK project in order to use them with ktrans(1).
-#
-# grabskkdicts pulls the skk kana-kanji conversion dictionaries from the skk-dev/dict repo.
-# skk2ktrans takes an skk dictionary and converts it into a kanji jisho suitable to be used with ktrans(1).
-#
-# You can fetch and convert all the dictionaries by running this file.
-#
+# Japanese dictionary data

-grabskkdicts
-for(d in skkdicts/SKK-JISYO.*)
-	<$d skk2ktrans >$d.jisho
+`kanji.dict` is the historical dictionary bundled with strans.  Its own header
+identifies it as the SKK Medium dictionary, version 8.1 of May 24, 1995,
+rearranged for Plan 9 ktrans by Kenji Okamoto on February 17, 2000.  It was
+already present when this repository was created in commit
+`dcd1147638f908eca7e4b7616815e215022cc99d` on December 23, 2025.  No original
+SKK input file or upstream revision was committed, so the current file cannot
+honestly be regenerated byte-for-byte from repository artifacts.  The license
+notice in its header permits redistribution and modification under GPL version
+2 or later.
+
+The repository copy keeps that historical candidate order.  Duplicate keys
+were merged at their first occurrence, later unseen candidates were appended
+in source order, duplicate candidates were removed, and the empty
+`ようたつ` row was removed.
+
+## Importing a current SKK dictionary
+
+From the repository root, fetch a known upstream revision and convert it with:
+
+```
+map/grabskkdicts map/skkdicts REVISION
+map/skk2ktrans map/skkdicts/SKK-JISYO.M >map/kanji.dict.new
+python3 map/verifymap.py map/kanji.dict.new
+```
+
+Use a full skk-dev/dict commit ID for `REVISION` and record it when replacing
+the bundled data.  Omitting `REVISION` intentionally fetches the upstream
+default branch and is not reproducible.  `grabskkdicts` refuses to overwrite an
+existing destination.
+
+`skk2ktrans` accepts one or more EUC-JP SKK files (or standard input), writes
+UTF-8 tab-separated rows, and merges input in command-line and source order.
+It strips SKK annotations, deduplicates candidates, and omits Lisp expressions,
+numeric conversion entries, bracket forms, and candidates containing
+ASCII whitespace because those forms cannot be consumed as literal candidates
+by `dict.c`.
+
+`verifymap.py` checks UTF-8, row structure, unique keys, the 64-rune key limit,
+canonical candidate spacing, and duplicate dictionary candidates.  Pass it the
+exact `.map` and `.dict` files installed by the build.
--- a/map/grabskkdicts
+++ b/map/grabskkdicts
@@ -1,2 +1,20 @@
-#!/bin/rc
-git/clone https://github.com/skk-dev/dict skkdicts
+#!/bin/sh
+set -eu
+
+repo=https://github.com/skk-dev/dict.git
+dest=${1:-skkdicts}
+revision=${2:-}
+
+if [ "$#" -gt 2 ] || [ -z "$dest" ]; then
+	echo "usage: $0 [destination [revision]]" >&2
+	exit 2
+fi
+if [ -e "$dest" ]; then
+	echo "$0: destination already exists: $dest" >&2
+	exit 1
+fi
+
+git clone "$repo" "$dest"
+if [ -n "$revision" ]; then
+	git -C "$dest" checkout --detach "$revision"
+fi
--- a/map/kanji.dict
+++ b/map/kanji.dict
@@ -60,7 +60,6 @@
 あたらs	新
 あつi	熱 暑 厚
 あつk	厚 熱 暑
-あつk	厚 熱 暑
 あつs	暑
 あつm	集 厚
 あつかe	扱
@@ -350,7 +349,6 @@
 かわいr	可愛
 かんj	感
 かんs	関
-かんj	感
 かんz	感
 かんがe	考
 かんしゃs	感謝
@@ -455,7 +453,7 @@
 こu	乞 請
 こy	来
 こうj	高
-こうしょう	鉱床 高尚
+こうしょう	鉱床 高尚 交渉
 こうぶつ	鉱物 好物
 こうりょs	考慮
 こえt	超
@@ -665,8 +663,6 @@
 ただc	直
 ただt	直
 たとe	例
-たおr	倒
-たおr	倒
 たのm	頼
 たのn	頼
 たのs	楽
@@ -742,7 +738,6 @@
 つみあげr	積み上げ
 つみあげt	積み上げ
 つめt	冷
-つづr	綴
 つよi	強
 つよk	強
 つらi	辛
@@ -793,7 +788,6 @@
 とt	取 撮 採
 とu	問
 とw	問
-とj	閉
 といあw	問い合
 とうj	投
 とおi	遠
@@ -913,13 +907,12 @@
 ねらt	狙
 ねらu	狙
 のb	述 伸 延
-のk	退
+のk	退 乗
 のm	飲 呑
 のn	飲
 のr	乗 載
 のs	載 乗
 のt	乗 載
-のk	乗
 のこr	残
 のこs	残
 のこt	残
@@ -1059,12 +1052,11 @@
 ぼうえいs	防衛
 まc	待
 まi	舞
-まj	交
+まj	交 混
 まk	負 巻
 まs	増
 まt	待
 まu	舞
-まj	混
 まz	混 先
 まいr	参 詣
 まいt	参 詣
@@ -1241,8 +1233,6 @@
 よわm	弱
 よわs	弱
 よわt	弱
-よろこb	喜 慶
-よろこb	喜 慶
 ろんj	論
 ろんz	論
 わk	分 湧
@@ -1938,7 +1928,7 @@
 かいもの	買物
 かいよう	海洋 潰瘍
 かいらん	回覧
-かいり	解離 乖離 海里 浬 
+かいり	解離 乖離 海里 浬
 かいりょう	改良
 かいれい	海嶺
 かいろ	回路
@@ -2267,7 +2257,6 @@
 がくわり	学割
 がけ	崖
 がし	樫
-かせい	火星
 がそ	画素
 がそかん	画素間
 がそごと	画素毎
@@ -2993,7 +2982,6 @@
 こうしゃ	後者 公社 校舎
 こうしゅう	講習 公衆
 こうしゅうかい	講習会
-こうしょう	交渉 鉱床
 こうしん	更新 交信 後身
 こうじ	麹 公示 高次 工事 孝二
 こうじつ	口実
@@ -3509,7 +3497,7 @@
 しこく	四国
 しご	死後
 しごと	仕事
-しごとば	仕事場 堺市学園町1ー1 
+しごとば	仕事場 堺市学園町1ー1
 しさ	示唆 視差
 しさい	仔細 子細
 しさく	試作
@@ -4802,7 +4790,7 @@
 たいきゅうりょく	耐久力
 たいきょく	対極
 たいきん	大金
-たいく	体躯
+たいく	体躯 体育
 たいくつ	退屈
 たいけい	体系 体型
 たいけん	体験
@@ -4949,8 +4937,6 @@
 たんらく	短絡
 だ	騨 駄 陀 楕 舵 柁 打 惰 妥 堕 唾 田 朶
 だい	大 第 内 代 台 題 醍
-たいく	体育
-たいいく	体育
 だいいち	第一
 だいがく	大学
 だいがくいん	大学院
@@ -5673,7 +5659,7 @@
 どんき	鈍器
 どんてん	曇天
 どんよく	貪欲
-な	名 七 那 奈 魚 菜 字 肴 納 無 儺 己 汝 何 
+な	名 七 那 奈 魚 菜 字 肴 納 無 儺 己 汝 何
 ない	内 無
 ないがい	内外
 ないき	内記 内規
@@ -6816,7 +6802,7 @@
 ぼうし	防止
 ぼうそう	暴走
 ぼうだい	膨大
-ぼうちょう	膨張 防潮 傍聴 
+ぼうちょう	膨張 防潮 傍聴
 ぼうとう	冒頭
 ぼうどう	暴動
 ぼうねんかい	忘年会
@@ -7314,7 +7300,6 @@
 よこう	予行 予稿
 よこうち	横内
 よこく	予告
-ようたつ	
 よこて	横手
 よこはま	横浜
 よこもじ	横文字
@@ -7426,7 +7411,7 @@
 りゅうど	粒度
 りゆう	理由
 りょ	虜 旅 慮 侶
-りょう	量 寮 両 領 陵 遼 諒 良 糧 稜 瞭 療 猟 涼 梁 料 凌 僚 亮 了 漁 糧 粮 霊
+りょう	量 寮 両 領 陵 遼 諒 良 糧 稜 瞭 療 猟 涼 梁 料 凌 僚 亮 了 漁 粮 霊
 りょうあし	両足
 りょういき	領域
 りょうか	量化
--- a/map/skk2ktrans
+++ b/map/skk2ktrans
@@ -1,2 +1,143 @@
-#!/bin/rc
-tcs -sf ujis | awk '$1 !~ /;;/ {gsub("(^\/|\/$)", "", $2); gsub(" ", "	"); gsub("\/", " ", $2);} {print}'
+#!/usr/bin/env python3
+"""Convert an EUC-JP SKK dictionary to strans dictionary format."""
+
+import argparse
+import sys
+from collections import OrderedDict
+from pathlib import Path
+
+
+class InputError(Exception):
+	pass
+
+
+def candidates(field, source, lineno):
+	"""Return literal candidates, dropping SKK annotations and expressions."""
+	if not field.startswith("/"):
+		raise InputError(f"{source}:{lineno}: candidate list does not start with /")
+
+	result = []
+	buf = []
+	annotation = False
+	escaped = False
+	closed = False
+	brackets = 0
+	parens = 0
+	quoted = False
+	for ch in field[1:]:
+		if escaped:
+			if not annotation:
+				buf.append(ch)
+			escaped = False
+			closed = False
+			continue
+		if ch == "\\":
+			escaped = True
+			closed = False
+			continue
+		if ch == "/" and (annotation or (brackets == 0 and parens == 0)):
+			candidate = "".join(buf)
+			if "\0" in candidate:
+				raise InputError(f"{source}:{lineno}: NUL in candidate")
+			if (candidate and not candidate.startswith(("(", "[", "#"))
+					and not any(c in " \t\r\n" for c in candidate)):
+				result.append(candidate)
+			buf = []
+			annotation = False
+			closed = True
+			continue
+		if ch == ";" and not annotation and brackets == 0 and parens == 0:
+			annotation = True
+			closed = False
+			continue
+		if not annotation:
+			if ch == '"' and parens:
+				quoted = not quoted
+			elif not quoted:
+				if ch == "[":
+					brackets += 1
+				elif ch == "]" and brackets:
+					brackets -= 1
+				elif ch == "(":
+					parens += 1
+				elif ch == ")" and parens:
+					parens -= 1
+			buf.append(ch)
+		closed = False
+
+	if escaped:
+		raise InputError(f"{source}:{lineno}: trailing escape")
+	if not closed:
+		raise InputError(f"{source}:{lineno}: unterminated candidate list")
+	return result
+
+
+def convert(inputs):
+	rows = OrderedDict()
+	seen = {}
+	for source, data in inputs:
+		try:
+			text = data.decode("euc_jp")
+		except UnicodeDecodeError as error:
+			raise InputError(f"{source}: invalid EUC-JP input: {error}") from error
+		for lineno, line in enumerate(text.splitlines(), 1):
+			if not line or line.startswith(";;"):
+				continue
+			fields = line.split(None, 1)
+			if len(fields) != 2:
+				raise InputError(f"{source}:{lineno}: missing candidate list")
+			key, field = fields
+			if any(c in " \t\r\n" for c in key):
+				raise InputError(f"{source}:{lineno}: whitespace in key")
+			if "\0" in key:
+				raise InputError(f"{source}:{lineno}: NUL in key")
+			if len(key) > 64:
+				raise InputError(f"{source}:{lineno}: key exceeds 64 runes")
+			if key not in rows:
+				rows[key] = []
+				seen[key] = set()
+			for candidate in candidates(field, source, lineno):
+				if candidate not in seen[key]:
+					rows[key].append(candidate)
+					seen[key].add(candidate)
+	return rows
+
+
+def read_inputs(names):
+	if not names:
+		return [("<stdin>", sys.stdin.buffer.read())]
+	inputs = []
+	stdin_used = False
+	for name in names:
+		if name == "-":
+			if stdin_used:
+				raise InputError("standard input may be specified only once")
+			stdin_used = True
+			inputs.append(("<stdin>", sys.stdin.buffer.read()))
+		else:
+			path = Path(name)
+			try:
+				inputs.append((name, path.read_bytes()))
+			except OSError as error:
+				raise InputError(f"{name}: {error.strerror}") from error
+	return inputs
+
+
+def main():
+	parser = argparse.ArgumentParser(
+		description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV")
+	parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)")
+	args = parser.parse_args()
+	try:
+		rows = convert(read_inputs(args.dictionary))
+	except InputError as error:
+		print(f"skk2ktrans: {error}", file=sys.stderr)
+		return 1
+
+	lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values]
+	sys.stdout.buffer.write("".join(lines).encode("utf-8"))
+	return 0
+
+
+if __name__ == "__main__":
+	raise SystemExit(main())
--- a/map/verifymap.py
+++ b/map/verifymap.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""Verify the text maps and dictionaries consumed by strans."""
+
+import argparse
+import sys
+from pathlib import Path
+
+
+def verify(path):
+	errors = []
+	try:
+		data = path.read_bytes()
+	except OSError as error:
+		return [f"{path}: {error.strerror}"]
+	try:
+		text = data.decode("utf-8")
+	except UnicodeDecodeError as error:
+		return [f"{path}: invalid UTF-8: {error}"]
+
+	keys = {}
+	for lineno, line in enumerate(text.split("\n"), 1):
+		where = f"{path}:{lineno}"
+		if "\r" in line:
+			errors.append(f"{where}: carriage return is not canonical")
+			line = line.replace("\r", "")
+		if not line or line.startswith(";"):
+			continue
+		if "\0" in line:
+			errors.append(f"{where}: embedded NUL")
+		if line.count("\t") != 1:
+			errors.append(f"{where}: expected exactly one tab")
+			continue
+		key, value = line.split("\t")
+		if not key:
+			errors.append(f"{where}: empty key")
+		if len(key) > 64:
+			errors.append(f"{where}: key has {len(key)} runes; maximum is 64")
+		if key in keys:
+			errors.append(f"{where}: duplicate key; first defined on line {keys[key]}")
+		else:
+			keys[key] = lineno
+		if not value:
+			errors.append(f"{where}: empty value")
+			continue
+		if value != " ".join(value.split(" ")):
+			errors.append(f"{where}: noncanonical candidate spacing")
+		values = value.split(" ")
+		if len(values) != len(set(values)):
+			errors.append(f"{where}: duplicate candidate")
+	return errors
+
+
+def main():
+	parser = argparse.ArgumentParser(description=__doc__)
+	parser.add_argument("file", nargs="+", type=Path)
+	args = parser.parse_args()
+	errors = []
+	for path in args.file:
+		errors.extend(verify(path))
+	for error in errors:
+		print(error, file=sys.stderr)
+	return bool(errors)
+
+
+if __name__ == "__main__":
+	raise SystemExit(main())