data(hanja): words, not only syllables

The import kept only readings of one syllable, so the Hanja search could
convert 한 but never 한자, 학교, or 대한민국 — the conversion every other
Korean input method offers.  libhangul's table has 187k readings; both
scripts now keep them all, and the search finds a word as readily as a
syllable.  The daemon pays for it: 24 MB instead of 12, and 170 ms to
start instead of 30.

README.md

@@ -58,8 +58,11 @@ typed keys, and what they spell in the current language, against a prefix
 of every emoji's CLDR name and keywords in English, Korean, and Japanese,
 and against ASCII symbol aliases such as `->` and `<=`. `Ctrl+H` takes the
 syllable being composed as its query and composes on from it; `Esc` gives
-the syllable back; the Hanja dictionary lists one modern Hangul syllable at
+the syllable back. The Hanja dictionary converts a word as well as a
-a time. Leaving the field or clicking elsewhere commits what is pending.
+syllable, so `Ctrl+H` and then 한자 gives 漢字, and 대한민국 gives 大韓民國;
+a syllable already committed is the application's text, not the engine's,
+and cannot be converted. Leaving the field or
+clicking elsewhere commits what is pending.
 
 ## Preedit and candidates
 

map/README

@@ -3,18 +3,19 @@
 ## Korean Hanja data
 
 `hanja.src` is the tracked, reviewable source for Korean Hanja conversion.
-Every data row is exactly one Hanja character, a tab, and one modern Hangul
+Every data row is Hanja, a tab, and its modern Hangul reading, a character
-syllable:
+or a word:
 
 ```
 漢	한
+漢字	한자
 ```
 
-It contains no word rows such as `견출지` or `방학`.  `mkhanja` validates this
+`mkhanja` validates that contract and groups rows by their reading to
-contract and groups rows by their Hangul reading to produce the existing
+produce the runtime dictionary.  Candidate order follows source order.  The
-runtime dictionary format.  Candidate order follows source order.  The source
+source keeps all retained pairs for review; the generated dictionary stores
-keeps all retained pairs for review; the generated dictionary stores the first
+the first 128 candidates per reading because that is the engine's lookup
-128 candidates per reading because that is the engine's lookup limit.
+limit.
 
 The source is derived from libhangul release tag `libhangul-0.2.0`.  The
 annotated tag object is `20afc38922e3595ee3ed5b186f2ea05afe663763`, its
@@ -30,11 +31,10 @@ map/libhangul2hanja upstream >map/hanja.src
 map/mkhanja >map/hanja.dict
 ```
 
-`libhangul2hanja` retains only rows whose reading is one modern Hangul
+`libhangul2hanja` retains only rows whose reading is modern Hangul syllables
-syllable and whose value is one Hanja character supported by the current
+throughout and whose value is Hanja the popup can draw: U+3400–U+4DBF,
-popup: U+3400–U+4DBF, U+4E00–U+9FFF, or U+F900–U+FAFF.  Thus word readings,
+U+4E00–U+9FFF, or U+F900–U+FAFF.  Thus jamo readings, mixed values, and
-multi-character values, jamo readings, and supplementary-plane ideographs
+supplementary-plane ideographs are omitted.  The import preserves the upstream license header and row order.
-are omitted.  The import preserves the upstream license header and row order.
 The retained data is BSD 3-Clause licensed by Choe Hwanjin; the complete text
 is in `LICENSES/BSD-3-Clause-libhangul-hanja.txt`.
 

map/libhangul2hanja

@@ -1,21 +1,18 @@
 #!/usr/bin/env python3
-"""Extract single-character Hanja readings from libhangul data."""
+"""Extract Hanja readings from libhangul data."""
 
 import sys
 from pathlib import Path
 
 
 def ishangul(s):
-    return len(s) == 1 and 0xAC00 <= ord(s) <= 0xD7A3
+    return s != "" and all(0xAC00 <= ord(c) <= 0xD7A3 for c in s)
 
 
 def ishanja(s):
-    if len(s) != 1:
+    return s != "" and all(0x3400 <= ord(c) <= 0x4DBF
-        return False
+                           or 0x4E00 <= ord(c) <= 0x9FFF
-    c = ord(s)
+                           or 0xF900 <= ord(c) <= 0xFAFF for c in s)
-    return (0x3400 <= c <= 0x4DBF
-            or 0x4E00 <= c <= 0x9FFF
-            or 0xF900 <= c <= 0xFAFF)
 
 
 def extract(src, name):
@@ -44,7 +41,7 @@ def extract(src, name):
         seen.add(pair)
         entries.append(pair)
     if not entries:
-        raise ValueError(f"{name}: no single-character Hanja readings")
+        raise ValueError(f"{name}: no Hanja readings")
     return comments, entries
 
 

map/mkhanja

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Write hanja.dict from a one-Hanja-per-row UTF-8 source."""
+"""Write hanja.dict from a Hanja-per-row UTF-8 source."""
 
 import sys
 from pathlib import Path
@@ -10,16 +10,13 @@ MAXCANDIDATES = 128
 
 
 def ishangul(s):
-    return len(s) == 1 and 0xAC00 <= ord(s) <= 0xD7A3
+    return s != "" and all(0xAC00 <= ord(c) <= 0xD7A3 for c in s)
 
 
 def ishanja(s):
-    if len(s) != 1:
+    return s != "" and all(0x3400 <= ord(c) <= 0x4DBF
-        return False
+                           or 0x4E00 <= ord(c) <= 0x9FFF
-    c = ord(s)
+                           or 0xF900 <= ord(c) <= 0xFAFF for c in s)
-    return (0x3400 <= c <= 0x4DBF
-            or 0x4E00 <= c <= 0x9FFF
-            or 0xF900 <= c <= 0xFAFF)
 
 
 def read(path):
@@ -42,9 +39,9 @@ def read(path):
                 raise ValueError(f"{path}:{lineno}: need Hanja<TAB>reading")
             hanja, reading = fields
             if not ishanja(hanja):
-                raise ValueError(f"{path}:{lineno}: need one BMP Hanja character")
+                raise ValueError(f"{path}:{lineno}: need BMP Hanja")
             if not ishangul(reading):
-                raise ValueError(f"{path}:{lineno}: need one Hangul syllable")
+                raise ValueError(f"{path}:{lineno}: need Hangul syllables")
             pair = (hanja, reading)
             if pair in seen:
                 raise ValueError(f"{path}:{lineno}: duplicate Hanja reading")

tests/mkhanja_test.py

@@ -42,25 +42,27 @@ class MkhanjaTest(unittest.TestCase):
         self.addCleanup(tmp.cleanup)
         return path
 
-    def test_imports_single_bmp_hanja_rows(self):
+    def test_imports_bmp_hanja_rows(self):
         result = run(
             IMPORT,
             self.source(
                 "# Copyright holder\n"
                 "# BSD license\n"
-                "한:漢:first\n"
+                "\ud55c:\u6f22:first\n"
-                "가:㐀:extension A\n"
+                "\uac00:\u3400:extension A\n"
-                "김:金:compatibility\n"
+                "\uae40:\u91d1:compatibility\n"
-                "방학:放:word reading\n"
+                "\ud55c\uc790:\u6f22\u5b57:word\n"
-                "학:學校:word value\n"
+                "\ud55c:\U00020000:astral\n"
-                "한:𠀀:astral\n"
+                "\u3131:\u52a0:jamo\n"
-                "ㄱ:加:jamo\n"
+                "\ud55c\uae00:\u97d3glyph:mixed\n"
             ),
         )
         self.assertEqual(result.returncode, 0, result.stderr)
         lines = result.stdout.splitlines()
         self.assertEqual(lines[:2], ["# Copyright holder", "# BSD license"])
-        self.assertEqual(lines[2:], ["", "漢\t한", "㐀\t가", "金\t김"])
+        self.assertEqual(lines[2:], ["", "\u6f22\t\ud55c", "\u3400\t\uac00",
+                                     "\u91d1\t\uae40",
+                                     "\u6f22\u5b57\t\ud55c\uc790"])
 
     def test_import_rejects_malformed_and_duplicate_rows(self):
         bad = [
@@ -91,6 +93,7 @@ class MkhanjaTest(unittest.TestCase):
                 "㐀\t가\n"
                 "家\t가\n"
                 "金\t금\n"
+                "漢字\t한자\n"
             ),
         )
         self.assertEqual(result.returncode, 0, result.stderr)
@@ -103,13 +106,14 @@ class MkhanjaTest(unittest.TestCase):
                 "한\t漢 韓",
                 "가\t㐀 家",
                 "금\t金",
+                "한자\t漢字",
             ],
         )
 
     def test_generator_rejects_bad_rows(self):
         bad = [
-            "漢韓\t한\n",
+            "漢a\t한\n",
-            "漢\t한자\n",
+            "漢\t한a\n",
             "𠀀\t한\n",
             "漢\t한\n漢\t한\n",
             "漢 한\n",