data(hanja): words, not only syllables

The import kept only readings of one syllable, so the Hanja search could
convert 한 but never 한자, 학교, or 대한민국 — the conversion every other
Korean input method offers.  libhangul's table has 187k readings; both
scripts now keep them all, and the search finds a word as readily as a
syllable.  The daemon pays for it: 24 MB instead of 12, and 170 ms to
start instead of 30.
This commit is contained in:
2026-08-17 12:50:05 +09:00
parent 3527bcd489
commit 2e53627b7d
7 changed files with 424873 additions and 45 deletions

View File

@@ -42,25 +42,27 @@ class MkhanjaTest(unittest.TestCase):
self.addCleanup(tmp.cleanup)
return path
def test_imports_single_bmp_hanja_rows(self):
def test_imports_bmp_hanja_rows(self):
result = run(
IMPORT,
self.source(
"# Copyright holder\n"
"# BSD license\n"
"한:漢:first\n"
"가:㐀:extension A\n"
"김:金:compatibility\n"
"방학:放:word reading\n"
"학:學校:word value\n"
"한:𠀀:astral\n"
"ㄱ:加:jamo\n"
"\ud55c:\u6f22:first\n"
"\uac00:\u3400:extension A\n"
"\uae40:\u91d1:compatibility\n"
"\ud55c\uc790:\u6f22\u5b57:word\n"
"\ud55c:\U00020000:astral\n"
"\u3131:\u52a0:jamo\n"
"\ud55c\uae00:\u97d3glyph:mixed\n"
),
)
self.assertEqual(result.returncode, 0, result.stderr)
lines = result.stdout.splitlines()
self.assertEqual(lines[:2], ["# Copyright holder", "# BSD license"])
self.assertEqual(lines[2:], ["", "\t", "\t", "\t"])
self.assertEqual(lines[2:], ["", "\u6f22\t\ud55c", "\u3400\t\uac00",
"\u91d1\t\uae40",
"\u6f22\u5b57\t\ud55c\uc790"])
def test_import_rejects_malformed_and_duplicate_rows(self):
bad = [
@@ -91,6 +93,7 @@ class MkhanjaTest(unittest.TestCase):
"\t\n"
"\t\n"
"\t\n"
"漢字\t한자\n"
),
)
self.assertEqual(result.returncode, 0, result.stderr)
@@ -103,13 +106,14 @@ class MkhanjaTest(unittest.TestCase):
"\t漢 韓",
"\t㐀 家",
"\t",
"한자\t漢字",
],
)
def test_generator_rejects_bad_rows(self):
bad = [
"\t\n",
"\t\n",
"a\t\n",
"\ta\n",
"𠀀\t\n",
"\t\n\t\n",
"漢 한\n",