diff --git a/Makefile b/Makefile index 03598c5..be8ab4f 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,8 @@ test: verify-map: python3 map/mktelex.py | cmp - map/telex.map python3 -B map/mkemoji | cmp - map/emoji.dict + python3 -B map/verifymap.py map/*.map map/*.dict python3 -B tests/mkemoji_test.py + python3 -B tests/skk2ktrans_test.py .PHONY: all check test verify-map clean xim gtk bench docker docker-check diff --git a/map/README b/map/README index bcc40b5..828a41e 100644 --- a/map/README +++ b/map/README @@ -1,12 +1,42 @@ -# -# The following are a set of tools to obtain and process dictionaries from the SKK project in order to use them with ktrans(1). -# -# grabskkdicts pulls the skk kana-kanji conversion dictionaries from the skk-dev/dict repo. -# skk2ktrans takes an skk dictionary and converts it into a kanji jisho suitable to be used with ktrans(1). -# -# You can fetch and convert all the dictionaries by running this file. -# +# Japanese dictionary data -grabskkdicts -for(d in skkdicts/SKK-JISYO.*) - <$d skk2ktrans >$d.jisho +`kanji.dict` is the historical dictionary bundled with strans. Its own header +identifies it as the SKK Medium dictionary, version 8.1 of May 24, 1995, +rearranged for Plan 9 ktrans by Kenji Okamoto on February 17, 2000. It was +already present when this repository was created in commit +`dcd1147638f908eca7e4b7616815e215022cc99d` on December 23, 2025. No original +SKK input file or upstream revision was committed, so the current file cannot +honestly be regenerated byte-for-byte from repository artifacts. The license +notice in its header permits redistribution and modification under GPL version +2 or later. + +The repository copy keeps that historical candidate order. Duplicate keys +were merged at their first occurrence, later unseen candidates were appended +in source order, duplicate candidates were removed, and the empty +`ようたつ` row was removed. + +## Importing a current SKK dictionary + +From the repository root, fetch a known upstream revision and convert it with: + +``` +map/grabskkdicts map/skkdicts REVISION +map/skk2ktrans map/skkdicts/SKK-JISYO.M >map/kanji.dict.new +python3 map/verifymap.py map/kanji.dict.new +``` + +Use a full skk-dev/dict commit ID for `REVISION` and record it when replacing +the bundled data. Omitting `REVISION` intentionally fetches the upstream +default branch and is not reproducible. `grabskkdicts` refuses to overwrite an +existing destination. + +`skk2ktrans` accepts one or more EUC-JP SKK files (or standard input), writes +UTF-8 tab-separated rows, and merges input in command-line and source order. +It strips SKK annotations, deduplicates candidates, and omits Lisp expressions, +numeric conversion entries, bracket forms, and candidates containing +ASCII whitespace because those forms cannot be consumed as literal candidates +by `dict.c`. + +`verifymap.py` checks UTF-8, row structure, unique keys, the 64-rune key limit, +canonical candidate spacing, and duplicate dictionary candidates. Pass it the +exact `.map` and `.dict` files installed by the build. diff --git a/map/grabskkdicts b/map/grabskkdicts index 3ce0bfb..6447127 100755 --- a/map/grabskkdicts +++ b/map/grabskkdicts @@ -1,2 +1,20 @@ -#!/bin/rc -git/clone https://github.com/skk-dev/dict skkdicts +#!/bin/sh +set -eu + +repo=https://github.com/skk-dev/dict.git +dest=${1:-skkdicts} +revision=${2:-} + +if [ "$#" -gt 2 ] || [ -z "$dest" ]; then + echo "usage: $0 [destination [revision]]" >&2 + exit 2 +fi +if [ -e "$dest" ]; then + echo "$0: destination already exists: $dest" >&2 + exit 1 +fi + +git clone "$repo" "$dest" +if [ -n "$revision" ]; then + git -C "$dest" checkout --detach "$revision" +fi diff --git a/map/kanji.dict b/map/kanji.dict index cdb84c2..00ae470 100644 --- a/map/kanji.dict +++ b/map/kanji.dict @@ -60,7 +60,6 @@ あたらs 新 あつi 熱 暑 厚 あつk 厚 熱 暑 -あつk 厚 熱 暑 あつs 暑 あつm 集 厚 あつかe 扱 @@ -350,7 +349,6 @@ かわいr 可愛 かんj 感 かんs 関 -かんj 感 かんz 感 かんがe 考 かんしゃs 感謝 @@ -455,7 +453,7 @@ こu 乞 請 こy 来 こうj 高 -こうしょう 鉱床 高尚 +こうしょう 鉱床 高尚 交渉 こうぶつ 鉱物 好物 こうりょs 考慮 こえt 超 @@ -665,8 +663,6 @@ ただc 直 ただt 直 たとe 例 -たおr 倒 -たおr 倒 たのm 頼 たのn 頼 たのs 楽 @@ -742,7 +738,6 @@ つみあげr 積み上げ つみあげt 積み上げ つめt 冷 -つづr 綴 つよi 強 つよk 強 つらi 辛 @@ -793,7 +788,6 @@ とt 取 撮 採 とu 問 とw 問 -とj 閉 といあw 問い合 とうj 投 とおi 遠 @@ -913,13 +907,12 @@ ねらt 狙 ねらu 狙 のb 述 伸 延 -のk 退 +のk 退 乗 のm 飲 呑 のn 飲 のr 乗 載 のs 載 乗 のt 乗 載 -のk 乗 のこr 残 のこs 残 のこt 残 @@ -1059,12 +1052,11 @@ ぼうえいs 防衛 まc 待 まi 舞 -まj 交 +まj 交 混 まk 負 巻 まs 増 まt 待 まu 舞 -まj 混 まz 混 先 まいr 参 詣 まいt 参 詣 @@ -1241,8 +1233,6 @@ よわm 弱 よわs 弱 よわt 弱 -よろこb 喜 慶 -よろこb 喜 慶 ろんj 論 ろんz 論 わk 分 湧 @@ -1938,7 +1928,7 @@ かいもの 買物 かいよう 海洋 潰瘍 かいらん 回覧 -かいり 解離 乖離 海里 浬 +かいり 解離 乖離 海里 浬 かいりょう 改良 かいれい 海嶺 かいろ 回路 @@ -2267,7 +2257,6 @@ がくわり 学割 がけ 崖 がし 樫 -かせい 火星 がそ 画素 がそかん 画素間 がそごと 画素毎 @@ -2993,7 +2982,6 @@ こうしゃ 後者 公社 校舎 こうしゅう 講習 公衆 こうしゅうかい 講習会 -こうしょう 交渉 鉱床 こうしん 更新 交信 後身 こうじ 麹 公示 高次 工事 孝二 こうじつ 口実 @@ -3509,7 +3497,7 @@ しこく 四国 しご 死後 しごと 仕事 -しごとば 仕事場 堺市学園町1ー1 +しごとば 仕事場 堺市学園町1ー1 しさ 示唆 視差 しさい 仔細 子細 しさく 試作 @@ -4802,7 +4790,7 @@ たいきゅうりょく 耐久力 たいきょく 対極 たいきん 大金 -たいく 体躯 +たいく 体躯 体育 たいくつ 退屈 たいけい 体系 体型 たいけん 体験 @@ -4949,8 +4937,6 @@ たんらく 短絡 だ 騨 駄 陀 楕 舵 柁 打 惰 妥 堕 唾 田 朶 だい 大 第 内 代 台 題 醍 -たいく 体育 -たいいく 体育 だいいち 第一 だいがく 大学 だいがくいん 大学院 @@ -5673,7 +5659,7 @@ どんき 鈍器 どんてん 曇天 どんよく 貪欲 -な 名 七 那 奈 魚 菜 字 肴 納 無 儺 己 汝 何 +な 名 七 那 奈 魚 菜 字 肴 納 無 儺 己 汝 何 ない 内 無 ないがい 内外 ないき 内記 内規 @@ -6816,7 +6802,7 @@ ぼうし 防止 ぼうそう 暴走 ぼうだい 膨大 -ぼうちょう 膨張 防潮 傍聴 +ぼうちょう 膨張 防潮 傍聴 ぼうとう 冒頭 ぼうどう 暴動 ぼうねんかい 忘年会 @@ -7314,7 +7300,6 @@ よこう 予行 予稿 よこうち 横内 よこく 予告 -ようたつ よこて 横手 よこはま 横浜 よこもじ 横文字 @@ -7426,7 +7411,7 @@ りゅうど 粒度 りゆう 理由 りょ 虜 旅 慮 侶 -りょう 量 寮 両 領 陵 遼 諒 良 糧 稜 瞭 療 猟 涼 梁 料 凌 僚 亮 了 漁 糧 粮 霊 +りょう 量 寮 両 領 陵 遼 諒 良 糧 稜 瞭 療 猟 涼 梁 料 凌 僚 亮 了 漁 粮 霊 りょうあし 両足 りょういき 領域 りょうか 量化 diff --git a/map/skk2ktrans b/map/skk2ktrans index 083da0f..2a9403d 100755 --- a/map/skk2ktrans +++ b/map/skk2ktrans @@ -1,2 +1,143 @@ -#!/bin/rc -tcs -sf ujis | awk '$1 !~ /;;/ {gsub("(^\/|\/$)", "", $2); gsub(" ", " "); gsub("\/", " ", $2);} {print}' +#!/usr/bin/env python3 +"""Convert an EUC-JP SKK dictionary to strans dictionary format.""" + +import argparse +import sys +from collections import OrderedDict +from pathlib import Path + + +class InputError(Exception): + pass + + +def candidates(field, source, lineno): + """Return literal candidates, dropping SKK annotations and expressions.""" + if not field.startswith("/"): + raise InputError(f"{source}:{lineno}: candidate list does not start with /") + + result = [] + buf = [] + annotation = False + escaped = False + closed = False + brackets = 0 + parens = 0 + quoted = False + for ch in field[1:]: + if escaped: + if not annotation: + buf.append(ch) + escaped = False + closed = False + continue + if ch == "\\": + escaped = True + closed = False + continue + if ch == "/" and (annotation or (brackets == 0 and parens == 0)): + candidate = "".join(buf) + if "\0" in candidate: + raise InputError(f"{source}:{lineno}: NUL in candidate") + if (candidate and not candidate.startswith(("(", "[", "#")) + and not any(c in " \t\r\n" for c in candidate)): + result.append(candidate) + buf = [] + annotation = False + closed = True + continue + if ch == ";" and not annotation and brackets == 0 and parens == 0: + annotation = True + closed = False + continue + if not annotation: + if ch == '"' and parens: + quoted = not quoted + elif not quoted: + if ch == "[": + brackets += 1 + elif ch == "]" and brackets: + brackets -= 1 + elif ch == "(": + parens += 1 + elif ch == ")" and parens: + parens -= 1 + buf.append(ch) + closed = False + + if escaped: + raise InputError(f"{source}:{lineno}: trailing escape") + if not closed: + raise InputError(f"{source}:{lineno}: unterminated candidate list") + return result + + +def convert(inputs): + rows = OrderedDict() + seen = {} + for source, data in inputs: + try: + text = data.decode("euc_jp") + except UnicodeDecodeError as error: + raise InputError(f"{source}: invalid EUC-JP input: {error}") from error + for lineno, line in enumerate(text.splitlines(), 1): + if not line or line.startswith(";;"): + continue + fields = line.split(None, 1) + if len(fields) != 2: + raise InputError(f"{source}:{lineno}: missing candidate list") + key, field = fields + if any(c in " \t\r\n" for c in key): + raise InputError(f"{source}:{lineno}: whitespace in key") + if "\0" in key: + raise InputError(f"{source}:{lineno}: NUL in key") + if len(key) > 64: + raise InputError(f"{source}:{lineno}: key exceeds 64 runes") + if key not in rows: + rows[key] = [] + seen[key] = set() + for candidate in candidates(field, source, lineno): + if candidate not in seen[key]: + rows[key].append(candidate) + seen[key].add(candidate) + return rows + + +def read_inputs(names): + if not names: + return [("", sys.stdin.buffer.read())] + inputs = [] + stdin_used = False + for name in names: + if name == "-": + if stdin_used: + raise InputError("standard input may be specified only once") + stdin_used = True + inputs.append(("", sys.stdin.buffer.read())) + else: + path = Path(name) + try: + inputs.append((name, path.read_bytes())) + except OSError as error: + raise InputError(f"{name}: {error.strerror}") from error + return inputs + + +def main(): + parser = argparse.ArgumentParser( + description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV") + parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)") + args = parser.parse_args() + try: + rows = convert(read_inputs(args.dictionary)) + except InputError as error: + print(f"skk2ktrans: {error}", file=sys.stderr) + return 1 + + lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values] + sys.stdout.buffer.write("".join(lines).encode("utf-8")) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/map/verifymap.py b/map/verifymap.py new file mode 100755 index 0000000..9b55d13 --- /dev/null +++ b/map/verifymap.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Verify the text maps and dictionaries consumed by strans.""" + +import argparse +import sys +from pathlib import Path + + +def verify(path): + errors = [] + try: + data = path.read_bytes() + except OSError as error: + return [f"{path}: {error.strerror}"] + try: + text = data.decode("utf-8") + except UnicodeDecodeError as error: + return [f"{path}: invalid UTF-8: {error}"] + + keys = {} + for lineno, line in enumerate(text.split("\n"), 1): + where = f"{path}:{lineno}" + if "\r" in line: + errors.append(f"{where}: carriage return is not canonical") + line = line.replace("\r", "") + if not line or line.startswith(";"): + continue + if "\0" in line: + errors.append(f"{where}: embedded NUL") + if line.count("\t") != 1: + errors.append(f"{where}: expected exactly one tab") + continue + key, value = line.split("\t") + if not key: + errors.append(f"{where}: empty key") + if len(key) > 64: + errors.append(f"{where}: key has {len(key)} runes; maximum is 64") + if key in keys: + errors.append(f"{where}: duplicate key; first defined on line {keys[key]}") + else: + keys[key] = lineno + if not value: + errors.append(f"{where}: empty value") + continue + if value != " ".join(value.split(" ")): + errors.append(f"{where}: noncanonical candidate spacing") + values = value.split(" ") + if len(values) != len(set(values)): + errors.append(f"{where}: duplicate candidate") + return errors + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("file", nargs="+", type=Path) + args = parser.parse_args() + errors = [] + for path in args.file: + errors.extend(verify(path)) + for error in errors: + print(error, file=sys.stderr) + return bool(errors) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/data/skk/expected.tsv b/tests/data/skk/expected.tsv new file mode 100644 index 0000000..4ac5403 --- /dev/null +++ b/tests/data/skk/expected.tsv @@ -0,0 +1,3 @@ +かんじ 漢字 幹事 感じ +えがお 笑顔 +きごう 斜線/入り 記号;付き 普通 diff --git a/tests/data/skk/source.skk.utf8 b/tests/data/skk/source.skk.utf8 new file mode 100644 index 0000000..9d28bc2 --- /dev/null +++ b/tests/data/skk/source.skk.utf8 @@ -0,0 +1,6 @@ +;; converter fixture; the test transcodes this file to EUC-JP +かんじ /漢字;common/幹事/ +えがお /笑顔;face/ +かんじ /感じ/漢字;duplicate/ +きごう /斜線\/入り;escaped slash/記号\;付き;escaped semicolon/普通/ +むこう /候補 with space/(concat "式/" "候補")/[無効/候補/]/#0/ diff --git a/tests/skk2ktrans_test.py b/tests/skk2ktrans_test.py new file mode 100755 index 0000000..3d2eab3 --- /dev/null +++ b/tests/skk2ktrans_test.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +CONVERTER = ROOT / "map" / "skk2ktrans" +FIXTURES = Path(__file__).parent / "data" / "skk" + + +class Skk2KtransTest(unittest.TestCase): + def test_euc_jp_conversion_is_stable_and_literal(self): + source = (FIXTURES / "source.skk.utf8").read_text(encoding="utf-8") + expected = (FIXTURES / "expected.tsv").read_bytes() + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "source.skk" + input_path.write_bytes(source.encode("euc_jp")) + result = subprocess.run( + [CONVERTER, input_path], capture_output=True, check=False) + self.assertEqual(result.returncode, 0, result.stderr.decode()) + self.assertEqual(result.stdout, expected) + + def test_invalid_euc_jp_is_rejected(self): + result = subprocess.run( + [CONVERTER], input=b"\xff\xff\n", capture_output=True, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn(b"invalid EUC-JP", result.stderr) + + +if __name__ == "__main__": + unittest.main()