Hojun-Cho
2e53627b7d
The import kept only readings of one syllable, so the Hanja search could convert 한 but never 한자, 학교, or 대한민국 — the conversion every other Korean input method offers. libhangul's table has 187k readings; both scripts now keep them all, and the search finds a word as readily as a syllable. The daemon pays for it: 24 MB instead of 12, and 170 ms to start instead of 30.
145 lines
4.7 KiB
Python
145 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
IMPORT = ROOT / "map" / "libhangul2hanja"
|
|
GENERATE = ROOT / "map" / "mkhanja"
|
|
TIMEOUT = 10
|
|
|
|
|
|
def run(program, source=None, stdin=None, hostile=False):
|
|
command = [sys.executable, "-B", str(program)]
|
|
if source is not None:
|
|
command.append(str(source))
|
|
env = None
|
|
if hostile:
|
|
env = os.environ.copy()
|
|
env.update(LC_ALL="C", LANG="C", PYTHONCOERCECLOCALE="0", PYTHONUTF8="0")
|
|
env.pop("PYTHONIOENCODING", None)
|
|
return subprocess.run(
|
|
command,
|
|
input=stdin,
|
|
capture_output=True,
|
|
encoding="utf-8",
|
|
env=env,
|
|
check=False,
|
|
timeout=TIMEOUT,
|
|
)
|
|
|
|
|
|
class MkhanjaTest(unittest.TestCase):
|
|
def source(self, text):
|
|
tmp = tempfile.TemporaryDirectory()
|
|
path = Path(tmp.name) / "hanja.txt"
|
|
path.write_text(text, encoding="utf-8")
|
|
self.addCleanup(tmp.cleanup)
|
|
return path
|
|
|
|
def test_imports_bmp_hanja_rows(self):
|
|
result = run(
|
|
IMPORT,
|
|
self.source(
|
|
"# Copyright holder\n"
|
|
"# BSD license\n"
|
|
"\ud55c:\u6f22:first\n"
|
|
"\uac00:\u3400:extension A\n"
|
|
"\uae40:\u91d1:compatibility\n"
|
|
"\ud55c\uc790:\u6f22\u5b57:word\n"
|
|
"\ud55c:\U00020000:astral\n"
|
|
"\u3131:\u52a0:jamo\n"
|
|
"\ud55c\uae00:\u97d3glyph:mixed\n"
|
|
),
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
lines = result.stdout.splitlines()
|
|
self.assertEqual(lines[:2], ["# Copyright holder", "# BSD license"])
|
|
self.assertEqual(lines[2:], ["", "\u6f22\t\ud55c", "\u3400\t\uac00",
|
|
"\u91d1\t\uae40",
|
|
"\u6f22\u5b57\t\ud55c\uc790"])
|
|
|
|
def test_import_rejects_malformed_and_duplicate_rows(self):
|
|
bad = [
|
|
"한 漢\n",
|
|
"한:漢:first\n한:漢:again\n",
|
|
]
|
|
for text in bad:
|
|
with self.subTest(text=repr(text)):
|
|
self.assertEqual(run(IMPORT, self.source(text)).returncode, 1)
|
|
|
|
def test_import_uses_utf8_without_locale(self):
|
|
text = "# 저작권\n한:漢:first\n"
|
|
expected = "# 저작권\n\n漢\t한\n"
|
|
for source, stdin in ((self.source(text), None), (None, text)):
|
|
with self.subTest(source=source):
|
|
result = run(IMPORT, source, stdin, hostile=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(result.stdout, expected)
|
|
|
|
def test_groups_readings_and_preserves_order(self):
|
|
result = run(
|
|
GENERATE,
|
|
self.source(
|
|
"# Copyright holder\n"
|
|
"# BSD license\n"
|
|
"漢\t한\n"
|
|
"韓\t한\n"
|
|
"㐀\t가\n"
|
|
"家\t가\n"
|
|
"金\t금\n"
|
|
"漢字\t한자\n"
|
|
),
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(
|
|
result.stdout.splitlines(),
|
|
[
|
|
";; Copyright holder",
|
|
";; BSD license",
|
|
"",
|
|
"한\t漢 韓",
|
|
"가\t㐀 家",
|
|
"금\t金",
|
|
"한자\t漢字",
|
|
],
|
|
)
|
|
|
|
def test_generator_rejects_bad_rows(self):
|
|
bad = [
|
|
"漢a\t한\n",
|
|
"漢\t한a\n",
|
|
"𠀀\t한\n",
|
|
"漢\t한\n漢\t한\n",
|
|
"漢 한\n",
|
|
]
|
|
for text in bad:
|
|
with self.subTest(text=repr(text)):
|
|
self.assertEqual(run(GENERATE, self.source(text)).returncode, 1)
|
|
|
|
def test_generator_uses_utf8_without_locale(self):
|
|
result = run(GENERATE, self.source("# 저작권\n漢\t한\n"), hostile=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(result.stdout, ";; 저작권\n\n한\t漢\n")
|
|
|
|
result = run(GENERATE, hostile=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn("한\t", result.stdout)
|
|
|
|
def test_generator_keeps_runtime_candidate_limit(self):
|
|
rows = "".join(f"{chr(0x4E00 + n)}\t한\n" for n in range(129))
|
|
result = run(GENERATE, self.source(rows))
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
candidates = result.stdout.rstrip().split("\t", 1)[1].split()
|
|
self.assertEqual(len(candidates), 128)
|
|
self.assertEqual(candidates[-1], chr(0x4E00 + 127))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|