data: validate and repair Japanese dictionaries

This commit is contained in:
2026-08-12 15:34:00 +09:00
parent f0ba7fc5f6
commit d2fa51ceba
9 changed files with 324 additions and 39 deletions

View File

@@ -0,0 +1,3 @@
かんじ 漢字 幹事 感じ
えがお 笑顔
きごう 斜線/入り 記号;付き 普通
1 かんじ 漢字 幹事 感じ
2 えがお 笑顔
3 きごう 斜線/入り 記号;付き 普通

View File

@@ -0,0 +1,6 @@
;; converter fixture; the test transcodes this file to EUC-JP
かんじ /漢字;common/幹事/
えがお /笑顔;face/
かんじ /感じ/漢字;duplicate/
きごう /斜線\/入り;escaped slash/記号\;付き;escaped semicolon/普通/
むこう /候補 with space/(concat "式/" "候補")/[無効/候補/]/#0/

34
tests/skk2ktrans_test.py Executable file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/env python3
import subprocess
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
CONVERTER = ROOT / "map" / "skk2ktrans"
FIXTURES = Path(__file__).parent / "data" / "skk"
class Skk2KtransTest(unittest.TestCase):
def test_euc_jp_conversion_is_stable_and_literal(self):
source = (FIXTURES / "source.skk.utf8").read_text(encoding="utf-8")
expected = (FIXTURES / "expected.tsv").read_bytes()
with tempfile.TemporaryDirectory() as directory:
input_path = Path(directory) / "source.skk"
input_path.write_bytes(source.encode("euc_jp"))
result = subprocess.run(
[CONVERTER, input_path], capture_output=True, check=False)
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertEqual(result.stdout, expected)
def test_invalid_euc_jp_is_rejected(self):
result = subprocess.run(
[CONVERTER], input=b"\xff\xff\n", capture_output=True, check=False)
self.assertNotEqual(result.returncode, 0)
self.assertIn(b"invalid EUC-JP", result.stderr)
if __name__ == "__main__":
unittest.main()