Hojun-Cho
62ddc2177b
Every Korean keyboard's 한자 key answers a lone consonant with the KS X 1001 symbol palette, and has since 한글 워드프로세서: ㅁ for ※ ○ △ ㈜, ㄴ for the brackets, ㄹ for the units, ㅇ for the circled numbers. strans sends that key to the same search as a syllable -- Khanja is Ctrl+H at strans.c:830, and startsearch seeds the query with whatever ko.c left pending -- but every one of hanja.dict's 187286 readings is a syllable, so the popup came up with a query in it and nothing to pick: ㅁ: 0 candidates ㄴ: 0 candidates ㄹ: 0 candidates 한: 99 candidates 韓 漢 寒 限 閑 恨 旱 汗 翰 邯 罕 悍 澣 閒 瀚 libhangul ships that palette beside the Hanja table already imported here: data/hanja/mssymbol.txt, same commit, same author, same BSD-3 terms, same key:value:comment format -- and keyed by the compatibility jamo ko.c already holds, U+3141 for ㅁ. So the engine does not change at all; the same dictlookup on the same trie now finds something: ㅁ: 75 candidates ＃ ＆ ＊ ＠ § ※ ☆ ★ ○ ● ◎ ◇ ◆ □ ■ △ ▲ ▽ ▼ ㄴ: 23 candidates ＂ （ ） ［ ］ ｛ ｝ ‘ ’ “ ” 〔 〕 〈 〉 《 》 「 」 ㄹ: 94 candidates ＄ ％ ￦ Ｆ ′ ″ ℃ Å ￠ ￡ ￥ ¤ ℉ ‰ € ㎕ ㎖ ㎗ ℓ 한: 99 candidates 韓 漢 寒 限 閑 恨 旱 汗 翰 邯 罕 悍 澣 閒 瀚 Both scripts widen by one rule -- a syllable reading gives Hanja, a jamo reading gives a symbol -- and hanja.src regenerates byte for byte as it was, because upstream's own non-syllable readings are words like ㄱ자집 whose values were never Hanja and still fall out. 985 of mssymbol.txt's 987 rows survive: its ideographic space and its soft hyphen do not, since a candidate the popup cannot draw is not a candidate, and the row format separates candidates with a space besides. The two keyspaces cannot collide -- one is syllables, one is single jamo -- so the 187286 existing rows are unchanged, byte for byte, and 18 rows join them. mkhanja takes a source list as mkemoji already does, and keeps each upstream header, which is why the licence text now appears twice. 89 unit, check-live, check-stress and valgrind all clean. The five new assertions were checked by breaking the change five ways: dropping mssymbol.src from SOURCES, letting issymbol keep a formatting character, letting a jamo reading keep Hanja, widening isjamo to the vowels, and making mkhanja reject jamo readings. Each fails only the tests that exist for it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
191 lines
6.4 KiB
Python
191 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
IMPORT = ROOT / "map" / "libhangul2hanja"
|
|
GENERATE = ROOT / "map" / "mkhanja"
|
|
TIMEOUT = 10
|
|
|
|
|
|
def run(program, source=None, stdin=None, hostile=False):
|
|
command = [sys.executable, "-B", str(program)]
|
|
if source is not None:
|
|
command.append(str(source))
|
|
env = None
|
|
if hostile:
|
|
env = os.environ.copy()
|
|
env.update(LC_ALL="C", LANG="C", PYTHONCOERCECLOCALE="0", PYTHONUTF8="0")
|
|
env.pop("PYTHONIOENCODING", None)
|
|
return subprocess.run(
|
|
command,
|
|
input=stdin,
|
|
capture_output=True,
|
|
encoding="utf-8",
|
|
env=env,
|
|
check=False,
|
|
timeout=TIMEOUT,
|
|
)
|
|
|
|
|
|
class MkhanjaTest(unittest.TestCase):
|
|
def source(self, text):
|
|
tmp = tempfile.TemporaryDirectory()
|
|
path = Path(tmp.name) / "hanja.txt"
|
|
path.write_text(text, encoding="utf-8")
|
|
self.addCleanup(tmp.cleanup)
|
|
return path
|
|
|
|
def test_imports_bmp_hanja_rows(self):
|
|
result = run(
|
|
IMPORT,
|
|
self.source(
|
|
"# Copyright holder\n"
|
|
"# BSD license\n"
|
|
"\ud55c:\u6f22:first\n"
|
|
"\uac00:\u3400:extension A\n"
|
|
"\uae40:\u91d1:compatibility\n"
|
|
"\ud55c\uc790:\u6f22\u5b57:word\n"
|
|
"\ud55c:\U00020000:astral\n"
|
|
"\u3131:\u52a0:jamo\n"
|
|
"\ud55c\uae00:\u97d3glyph:mixed\n"
|
|
),
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
lines = result.stdout.splitlines()
|
|
self.assertEqual(lines[:2], ["# Copyright holder", "# BSD license"])
|
|
self.assertEqual(lines[2:], ["", "\u6f22\t\ud55c", "\u3400\t\uac00",
|
|
"\u91d1\t\uae40",
|
|
"\u6f22\u5b57\t\ud55c\uc790"])
|
|
|
|
def test_imports_symbol_rows_under_a_jamo(self):
|
|
result = run(
|
|
IMPORT,
|
|
self.source(
|
|
"\u3141:\u203b:reference mark\n"
|
|
"\u3134:\u300c:bracket\n"
|
|
"\u3131:\u3000:ideographic space\n"
|
|
"\u3131:\u00ad:soft hyphen\n"
|
|
"\u3131:\u52a0:Hanja under a jamo\n"
|
|
"\u3141:\u203b\u203b:two runes\n"
|
|
"\u3141\u3134:\u203b:two jamo\n"
|
|
"\u314f:\u203b:a vowel\n"
|
|
),
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(result.stdout.splitlines(),
|
|
["\u203b\t\u3141", "\u300c\t\u3134"])
|
|
|
|
def test_import_rejects_malformed_and_duplicate_rows(self):
|
|
bad = [
|
|
"한 漢\n",
|
|
"한:漢:first\n한:漢:again\n",
|
|
]
|
|
for text in bad:
|
|
with self.subTest(text=repr(text)):
|
|
self.assertEqual(run(IMPORT, self.source(text)).returncode, 1)
|
|
|
|
def test_import_uses_utf8_without_locale(self):
|
|
text = "# 저작권\n한:漢:first\n"
|
|
expected = "# 저작권\n\n漢\t한\n"
|
|
for source, stdin in ((self.source(text), None), (None, text)):
|
|
with self.subTest(source=source):
|
|
result = run(IMPORT, source, stdin, hostile=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(result.stdout, expected)
|
|
|
|
def test_groups_readings_and_preserves_order(self):
|
|
result = run(
|
|
GENERATE,
|
|
self.source(
|
|
"# Copyright holder\n"
|
|
"# BSD license\n"
|
|
"漢\t한\n"
|
|
"韓\t한\n"
|
|
"㐀\t가\n"
|
|
"家\t가\n"
|
|
"金\t금\n"
|
|
"漢字\t한자\n"
|
|
),
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(
|
|
result.stdout.splitlines(),
|
|
[
|
|
";; Copyright holder",
|
|
";; BSD license",
|
|
"",
|
|
"한\t漢 韓",
|
|
"가\t㐀 家",
|
|
"금\t金",
|
|
"한자\t漢字",
|
|
],
|
|
)
|
|
|
|
def test_groups_symbols_under_their_jamo(self):
|
|
result = run(
|
|
GENERATE,
|
|
self.source(
|
|
"※\tㅁ\n"
|
|
"○\tㅁ\n"
|
|
"「\tㄴ\n"
|
|
),
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(result.stdout.splitlines(),
|
|
["ㅁ\t※ ○", "ㄴ\t「"])
|
|
|
|
def test_generator_rejects_bad_rows(self):
|
|
bad = [
|
|
"漢a\t한\n",
|
|
"漢\t한a\n",
|
|
"𠀀\t한\n",
|
|
"漢\t한\n漢\t한\n",
|
|
"漢 한\n",
|
|
"加\tㅁ\n",
|
|
"※\t한\n",
|
|
"※\tㅁㄴ\n",
|
|
"※※\tㅁ\n",
|
|
"\u3000\tㄱ\n",
|
|
"\u00ad\tㄱ\n",
|
|
]
|
|
for text in bad:
|
|
with self.subTest(text=repr(text)):
|
|
self.assertEqual(run(GENERATE, self.source(text)).returncode, 1)
|
|
|
|
def test_generator_uses_utf8_without_locale(self):
|
|
result = run(GENERATE, self.source("# 저작권\n漢\t한\n"), hostile=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(result.stdout, ";; 저작권\n\n한\t漢\n")
|
|
|
|
result = run(GENERATE, hostile=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn("한\t", result.stdout)
|
|
|
|
def test_generator_reads_both_sources(self):
|
|
result = run(GENERATE)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
lines = result.stdout.splitlines()
|
|
self.assertEqual(lines.count(";; All rights reserved."), 2)
|
|
rows = dict(line.split("\t") for line in lines if "\t" in line)
|
|
self.assertIn("漢", rows["한"].split())
|
|
self.assertIn("※", rows["ㅁ"].split())
|
|
|
|
def test_generator_keeps_runtime_candidate_limit(self):
|
|
rows = "".join(f"{chr(0x4E00 + n)}\t한\n" for n in range(129))
|
|
result = run(GENERATE, self.source(rows))
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
candidates = result.stdout.rstrip().split("\t", 1)[1].split()
|
|
self.assertEqual(len(candidates), 128)
|
|
self.assertEqual(candidates[-1], chr(0x4E00 + 127))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|