From abff5ed12242ab9d4ea972b27670f4a60c667e91 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Mon, 17 Aug 2026 01:07:10 +0900 Subject: [PATCH] emoji: search the dictionary by prefix in the engine, not in the data mkemoji wrote a row for every prefix of every alias, so that a query matched as it was typed; the trie is a prefix index already, and with a real emoji list those rows would be four times the aliases themselves. Now emoji.dict has one row per alias, trienode() names a key's node, and dictlookup walks the entries at and below it, the key's own first, up to Maxkouho. Trie children are appended rather than pushed, so the walk keeps the file's order and a bare digit still picks the superscript or subscript it always did. The hand-written symbol rows move to symbol.src; emoji.src is left to the emoji. mkemoji reads both by default, or the files it is given. --- dict.c | 51 ++++++-- fn.h | 1 + map/emoji.dict | 290 ++++++++++-------------------------------- map/emoji.src | 104 --------------- map/mkemoji | 36 +++--- map/symbol.src | 103 +++++++++++++++ tests/engine_test.c | 7 +- tests/mkemoji_test.py | 31 +++-- trie.c | 38 ++++-- 9 files changed, 275 insertions(+), 386 deletions(-) create mode 100644 map/symbol.src diff --git a/dict.c b/dict.c index a97e8bf..ddeb61e 100644 --- a/dict.c +++ b/dict.c @@ -2,22 +2,20 @@ #include "fn.h" /* - * Fills out[] with up to max candidates for key: the space-separated words - * of the entry, minus the key itself except in the emoji dictionary, where - * a query may name its own answer. + * Appends the space-separated words of a node's entry to out[], up to + * max: its candidates, minus the key itself, except in the emoji + * dictionary, where a query may name its own answer. */ -int -dictlookup(Lang *l, Str *key, Str *out, int max) +static int +words(Lang *l, Str *key, Tnode *nd, Str *out, int n, int max) { char *p, *e, *sp; Str tmp; - int n, vlen; - if(l == nil || key->n == 0 || - trielookup(l->dict, key, &p, &vlen) != TrieExact) - return 0; - n = 0; - e = p + vlen; + if(nd->val == nil) + return n; + p = nd->val; + e = p + nd->vlen; while(n < max && p < e){ while(p < e && *p == ' ') p++; @@ -31,6 +29,37 @@ dictlookup(Lang *l, Str *key, Str *out, int max) return n; } +/* The entries at and below a node, in the dictionary's order. */ +static int +below(Lang *l, Str *key, int ni, Str *out, int n, int max) +{ + Tnode *nd; + int ci; + + nd = &l->dict->nodes[ni]; + n = words(l, key, nd, out, n, max); + for(ci = nd->child; ci >= 0 && n < max; ci = l->dict->nodes[ci].sibling) + n = below(l, key, ci, out, n, max); + return n; +} + +/* + * Fills out[] with up to max candidates for key. The emoji dictionary is + * searched by prefix, the key's own entry first; the others exactly. + */ +int +dictlookup(Lang *l, Str *key, Str *out, int max) +{ + int ni; + + ni = key->n == 0 ? -1 : trienode(l->dict, key); + if(ni < 0) + return 0; + if(l->lang == LangEMOJI) + return below(l, key, ni, out, 0, max); + return words(l, key, &l->dict->nodes[ni], out, 0, max); +} + static Trie* langopen(char *dir, char *name, char *ext) { diff --git a/fn.h b/fn.h index 2f690fa..0605aa3 100644 --- a/fn.h +++ b/fn.h @@ -15,6 +15,7 @@ Trie* trienew(void); void trieput(Trie*, char*, int, char*, int); Trie* trieopen(char*); void trieclose(Trie*); +int trienode(Trie*, Str*); int trielookup(Trie*, Str*, char**, int*); Lang* getlang(int); diff --git a/map/emoji.dict b/map/emoji.dict index 2e9d606..415261b 100644 --- a/map/emoji.dict +++ b/map/emoji.dict @@ -1,35 +1,20 @@ -! ⚠ ≠ !! ⚠ -!= ≠ -* ★ ** ★ -+ ± +- ± -- → -> → -. · … ÷ -.. · … +.. · ... … ./ ÷ -: ☹ ☺ :( ☹ :) ☺ -< ← ≤ ♥ ≠ <- ← -<3 ♥ <= ≤ +<3 ♥ <> ≠ -= ≡ ⇒ +!= ≠ == ≡ => ⇒ -> ≥ >= ≥ -^ ¹ ² ³ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁽ ⁾ ⁺ ⁻ ⁼ ⁰ ⁱ ⁿ -^( ⁽ -^) ⁾ -^+ ⁺ -^- ⁻ -^0 ⁰ ^1 ¹ ^2 ² ^3 ³ @@ -39,15 +24,6 @@ ^7 ⁷ ^8 ⁸ ^9 ⁹ -^= ⁼ -^i ⁱ -^n ⁿ -_ ₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₍ ₎ ₊ ₋ ₌ ₀ ₐ ₑ ₒ ₓ -_( ₍ -_) ₎ -_+ ₊ -_- ₋ -_0 ₀ _1 ₁ _2 ₂ _3 ₃ @@ -57,236 +33,108 @@ _6 ₆ _7 ₇ _8 ₈ _9 ₉ +^( ⁽ +^) ⁾ +^+ ⁺ +^- ⁻ +^= ⁼ +_( ₍ +_) ₎ +_+ ₊ +_- ₋ _= ₌ +~= ≈ +^0 ⁰ +_0 ₀ _a ₐ -_e ₑ -_o ₒ -_x ₓ -a α -al α -alp α -alph α alpha α -b β 😊 -be β -bet β beta β -bl 😊 -blu 😊 -blus 😊 -blush 😊 -c χ ☕️ -ch χ chi χ -co ☕️ -cof ☕️ -coff ☕️ -coffe ☕️ -coffee ☕️ -d ° δ Δ ↓ -de Δ ° δ deg ° -del δ -delt δ delta δ +de Δ dn ↓ -e ε η -ep ε +_e ₑ eps ε -et η eta η -f 凸 -fu 凸 -fuc 凸 fuck 凸 -g γ Γ 😀 -ga Γ γ -gam γ -gamm γ gamma γ -gr 😀 -gri 😀 -grin 😀 -h 😄 😊 ❤️ -ha 😄 😊 -hap 😄 😊 -happ 😄 😊 -happy 😄 😊 -he ❤️ -hea ❤️ -hear ❤️ -heart ❤️ -i ∫ ∞ ι +ga Γ +^i ⁱ ii ∫ -in ∞ inf ∞ -io ι -iot ι iota ι -j 😂 -jo 😂 -joy 😂 -k κ -ka κ -kap κ -kapp κ kappa κ -l λ Λ 😂 ❤️ 👍 -la Λ λ -lam λ -lamb λ -lambd λ lambda λ -li 👍 -lik 👍 -like 👍 -lo 😂 ❤️ -lol 😂 -lov ❤️ -love ❤️ -m × μ -mu μ × +la Λ mul × -n ν +mu μ +^n ⁿ nu ν -o ω Ω ● -om Ω ω -ome ω -omeg ω omega ω +om Ω oo ● -p φ Φ π Π ∏ ψ Ψ -ph Φ φ +_o ₒ phi φ +ph Φ pi π Π pp ∏ -ps Ψ ψ psi ψ -r ρ -rh ρ +ps Ψ rho ρ -s σ Σ √ ∑ 😀 😄 😊 -si Σ σ -sig σ -sigm σ sigma σ -sm 😀 😄 😊 -smi 😀 😄 😊 -smil 😀 😄 😊 -smile 😀 😄 😊 +si Σ sq √ ss ∑ -t τ θ Θ 👍 -ta τ tau τ -th Θ θ 👍 -the θ -thet θ theta θ -thu 👍 -thum 👍 -thumb 👍 -thumbs 👍 -thumbsu 👍 -thumbsup 👍 -u ↑ υ -up ↑ υ +th Θ +up ↑ ups υ -v ✓ vv ✓ -x ξ Ξ ✗ xi ξ Ξ xx ✗ -z ζ -ze ζ -zet ζ +_x ₓ zeta ζ -~ ≈ -~= ≈ -あ ❤️ -あい ❤️ -い 👍 -いい 👍 -いいね 👍 -え 😀 😄 😊 -えが 😀 😄 😊 -えがお 😀 😄 😊 -こ ☕️ -こー ☕️ -こーひ ☕️ -こーひー ☕️ -さ 👍 -さん 👍 -さんせ 👍 -さんせい 👍 -ば 😂 -ばく 😂 -ばくし 😂 -ばくしょ 😂 -ばくしょう 😂 -ほ 😄 😊 -ほほ 😄 😊 -ほほえ 😄 😊 -ほほえみ 😄 😊 -わ 😀 😂 -わら 😀 😂 -わらい 😀 😂 -ア ❤️ -アイ ❤️ -イ 👍 -イイ 👍 -イイネ 👍 -エ 😀 😄 😊 -エガ 😀 😄 😊 -エガオ 😀 😄 😊 -コ ☕️ -コー ☕️ -コーヒ ☕️ -コーヒー ☕️ -サ 👍 -サン 👍 -サンセ 👍 -サンセイ 👍 -ハ ❤️ -ハー ❤️ -ハート ❤️ -バ 😂 -バク 😂 -バクシ 😂 -バクショ 😂 -バクショウ 😂 -ホ 😄 😊 -ホホ 😄 😊 -ホホエ 😄 😊 -ホホエミ 😄 😊 -ワ 😀 😂 -ワラ 😀 😂 -ワライ 😀 😂 -따 👍 -따봉 👍 -미 😄 😊 -미소 😄 😊 -방 😊 -방긋 😊 -사 ❤️ -사랑 ❤️ -스 😀 😄 -스마 😀 😄 -스마일 😀 😄 -엄 👍 -엄지 👍 -웃 😀 😄 😂 -웃겨 😂 -웃다 😀 웃음 😀 😄 😂 -좋 👍 -좋아 👍 -좋아요 👍 -찬 👍 -찬성 👍 -커 ☕️ -커피 ☕️ -폭 😂 +웃다 😀 +스마일 😀 😄 +えがお 😀 😄 😊 +エガオ 😀 😄 😊 +わらい 😀 😂 +ワライ 😀 😂 +smile 😀 😄 😊 +grin 😀 +미소 😄 😊 +ほほえみ 😄 😊 +ホホエミ 😄 😊 +happy 😄 😊 폭소 😂 -하 ❤️ +웃겨 😂 +ばくしょう 😂 +バクショウ 😂 +joy 😂 +lol 😂 +방긋 😊 +blush 😊 +사랑 ❤️ 하트 ❤️ +あい ❤️ +アイ ❤️ +ハート ❤️ +heart ❤️ +love ❤️ +좋아요 👍 +찬성 👍 +따봉 👍 +엄지 👍 +いいね 👍 +イイネ 👍 +さんせい 👍 +サンセイ 👍 +thumbsup 👍 +like 👍 +커피 ☕️ +こーひー ☕️ +コーヒー ☕️ +coffee ☕️ diff --git a/map/emoji.src b/map/emoji.src index cccab72..76f82b5 100644 --- a/map/emoji.src +++ b/map/emoji.src @@ -1,107 +1,3 @@ -# Result first, followed by one or more TAB-separated aliases. -⚠ !! -★ ** -± +- -→ -> -· .. -… ... -÷ ./ -☹ :( -☺ :) - -# Bare 1-9 choose prefix candidates; keep digit aliases in matching slots. -← <- -≤ <= -♥ <3 -≠ <> != -≡ == -⇒ => -≥ >= -¹ ^1 -² ^2 -³ ^3 -⁴ ^4 -⁵ ^5 -⁶ ^6 -⁷ ^7 -⁸ ^8 -⁹ ^9 -₁ _1 -₂ _2 -₃ _3 -₄ _4 -₅ _5 -₆ _6 -₇ _7 -₈ _8 -₉ _9 - -⁽ ^( -⁾ ^) -⁺ ^+ -⁻ ^- -⁼ ^= -₍ _( -₎ _) -₊ _+ -₋ _- -₌ _= -≈ ~= -⁰ ^0 -₀ _0 -ₐ _a -α alpha -β beta -χ chi -° deg -δ delta -Δ De -↓ dn -ₑ _e -ε eps -η eta -凸 fuck -γ gamma -Γ Ga -ⁱ ^i -∫ II -∞ inf -ι iota -κ kappa -λ lambda -Λ La -× mul -μ mu -ⁿ ^n -ν nu -ω omega -Ω Om -● oo -ₒ _o -φ phi -Φ Ph -π pi -Π Pi -∏ PP -ψ psi -Ψ Ps -ρ rho -σ sigma -Σ Si -√ sq -∑ SS -τ tau -θ theta -Θ Th -↑ up -υ ups -✓ vv -ξ xi -Ξ Xi -✗ xx -ₓ _x -ζ zeta - # Small multilingual emoji seed. Shared aliases intentionally produce choices. 😀 웃음 웃다 스마일 えがお エガオ わらい ワライ smile grin 😄 웃음 미소 스마일 えがお エガオ ほほえみ ホホエミ smile happy diff --git a/map/mkemoji b/map/mkemoji index 661997a..065dbaf 100755 --- a/map/mkemoji +++ b/map/mkemoji @@ -1,13 +1,15 @@ #!/usr/bin/env python3 -"""Write emoji.dict to stdout from a result-first UTF-8 TSV file.""" +"""Write emoji.dict to stdout from result-first UTF-8 TSV files.""" import sys import unicodedata from pathlib import Path -SOURCE = Path(__file__).with_name("emoji.src") +SOURCES = [Path(__file__).with_name(name) + for name in ("symbol.src", "emoji.src")] MAXRUNES = 64 +MAXCANDIDATES = 128 def fold(s): @@ -44,31 +46,23 @@ def read(path): return entries -def add(table, key, result): - values = table.setdefault(key, []) - if result not in values: - values.append(result) - - def build(entries): - exact = {} - prefix = {} + """One row per alias, in order of first appearance; the engine searches + the dictionary by prefix, so no prefix rows are needed.""" + table = {} for result, alias in entries: - for n in range(1, len(alias) + 1): - add(exact if n == len(alias) else prefix, alias[:n], result) - for key in sorted(exact.keys() | prefix.keys()): - values = exact.get(key, []) + prefix.get(key, []) - values = list(dict.fromkeys(values)) - yield f"{key}\t{' '.join(values)}" + values = table.setdefault(alias, []) + if result not in values: + values.append(result) + for alias, values in table.items(): + yield f"{alias}\t{' '.join(values[:MAXCANDIDATES])}" def main(): - if len(sys.argv) > 2: - print(f"usage: {sys.argv[0]} [emoji.src]", file=sys.stderr) - return 2 - path = Path(sys.argv[1]) if len(sys.argv) == 2 else SOURCE + paths = [Path(arg) for arg in sys.argv[1:]] or SOURCES try: - for line in build(read(path)): + entries = [entry for path in paths for entry in read(path)] + for line in build(entries): print(line) except (OSError, UnicodeError, ValueError) as error: print(error, file=sys.stderr) diff --git a/map/symbol.src b/map/symbol.src new file mode 100644 index 0000000..7b21483 --- /dev/null +++ b/map/symbol.src @@ -0,0 +1,103 @@ +# Result first, followed by one or more TAB-separated aliases. +⚠ !! +★ ** +± +- +→ -> +· .. +… ... +÷ ./ +☹ :( +☺ :) + +# Bare 1-9 choose from a prefix search; keep digit aliases in matching slots. +← <- +≤ <= +♥ <3 +≠ <> != +≡ == +⇒ => +≥ >= +¹ ^1 +² ^2 +³ ^3 +⁴ ^4 +⁵ ^5 +⁶ ^6 +⁷ ^7 +⁸ ^8 +⁹ ^9 +₁ _1 +₂ _2 +₃ _3 +₄ _4 +₅ _5 +₆ _6 +₇ _7 +₈ _8 +₉ _9 + +⁽ ^( +⁾ ^) +⁺ ^+ +⁻ ^- +⁼ ^= +₍ _( +₎ _) +₊ _+ +₋ _- +₌ _= +≈ ~= +⁰ ^0 +₀ _0 +ₐ _a +α alpha +β beta +χ chi +° deg +δ delta +Δ De +↓ dn +ₑ _e +ε eps +η eta +凸 fuck +γ gamma +Γ Ga +ⁱ ^i +∫ II +∞ inf +ι iota +κ kappa +λ lambda +Λ La +× mul +μ mu +ⁿ ^n +ν nu +ω omega +Ω Om +● oo +ₒ _o +φ phi +Φ Ph +π pi +Π Pi +∏ PP +ψ psi +Ψ Ps +ρ rho +σ sigma +Σ Si +√ sq +∑ SS +τ tau +θ theta +Θ Th +↑ up +υ ups +✓ vv +ξ xi +Ξ Xi +✗ xx +ₓ _x +ζ zeta diff --git a/tests/engine_test.c b/tests/engine_test.c index eb949b0..5acb091 100644 --- a/tests/engine_test.c +++ b/tests/engine_test.c @@ -1331,13 +1331,14 @@ engine_emoji_queries(struct ct *t) CT_CHECK(t, keystroke('e', Mctrl, &com)); CT_CHECK(t, keystroke('a', 0, &com)); checkstr(t, "raw query wins", "a", &search.text); - CT_EQ_INT(t, 3, im.nkouho); + CT_EQ_INT(t, 4, im.nkouho); checkstr(t, "raw candidate", "A", &im.kouho[0]); checkstr(t, "deduplicated candidate", "B", &im.kouho[1]); - checkstr(t, "local candidate", "C", &im.kouho[2]); + checkstr(t, "raw prefix candidate", "α", &im.kouho[2]); + checkstr(t, "local candidate", "C", &im.kouho[3]); CT_CHECK(t, draindraw(&dc) > 0); checkstr(t, "prefix popup query", "a", &dc.pre); - CT_EQ_INT(t, 3, dc.nkouho); + CT_EQ_INT(t, 4, dc.nkouho); CT_CHECK(t, keystroke(Kesc, 0, &com)); CT_CHECK(t, keystroke('e', Mctrl, &com)); diff --git a/tests/mkemoji_test.py b/tests/mkemoji_test.py index de79d83..54cb8b8 100644 --- a/tests/mkemoji_test.py +++ b/tests/mkemoji_test.py @@ -12,10 +12,8 @@ MKEMOJI = ROOT / "map" / "mkemoji" TIMEOUT = 10 -def generate(source=None): - args = [sys.executable, "-B", str(MKEMOJI)] - if source is not None: - args.append(str(source)) +def generate(*sources): + args = [sys.executable, "-B", str(MKEMOJI)] + [str(s) for s in sources] return subprocess.run( args, capture_output=True, text=True, check=False, timeout=TIMEOUT ) @@ -37,27 +35,32 @@ class MkemojiTest(unittest.TestCase): result = generate() self.assertEqual(result.returncode, 0, result.stderr) data = table(result.stdout) - self.assertEqual(data["^"].split()[:9], list("¹²³⁴⁵⁶⁷⁸⁹")) - self.assertEqual(data["_"].split()[:9], list("₁₂₃₄₅₆₇₈₉")) - self.assertEqual(data["<"].split()[2], "♥") + keys = list(data) + self.assertEqual([k for k in keys if k[:1] == "^"][:9], + [f"^{d}" for d in range(1, 10)]) + self.assertEqual([k for k in keys if k[:1] == "_"][:9], + [f"_{d}" for d in range(1, 10)]) + self.assertEqual([k for k in keys if k[:1] == "<"][2], "<3") self.assertEqual(data["^1"], "¹") self.assertEqual(data["_2"], "₂") - self.assertEqual(data["<3"], "♥") + self.assertEqual(data["<3"].split()[0], "♥") + self.assertIn("😀", data["smile"].split()) + self.assertIn("😀", data["웃음"].split()) - def test_fold_normalize_and_exact_first(self): + def test_fold_normalize_and_source_order(self): source = self.source( - "β\tALPHABET\n" + "β\tALPHABET\talpha\n" "α\talpha\n" - "e\u0301\tE\u0301\n" + "é\tÉ\n" "#\thash\n" ) first = generate(source) - second = generate(source) + second = generate(source, source) self.assertEqual(first.returncode, 0, first.stderr) self.assertEqual(first.stdout, second.stdout) data = table(first.stdout) - self.assertEqual(data["alpha"].split(), ["α", "β"]) - self.assertEqual(data["al"].split(), ["β", "α"]) + self.assertEqual(list(data), ["alphabet", "alpha", "é", "hash"]) + self.assertEqual(data["alpha"].split(), ["β", "α"]) self.assertEqual(data["é"], "é") self.assertEqual(data["hash"], "#") diff --git a/trie.c b/trie.c index 35d5f88..ce94259 100644 --- a/trie.c +++ b/trie.c @@ -35,15 +35,21 @@ find(Trie *t, int ni, char c) return -1; } +/* Appends, so that a walk of the children keeps the file's order. */ static int add(Trie *t, int ni, char c) { - int pi; + int last, pi; + last = -1; + for(pi = t->nodes[ni].child; pi >= 0; pi = t->nodes[pi].sibling) + last = pi; pi = newnode(t); t->nodes[pi].c = c; - t->nodes[pi].sibling = t->nodes[ni].child; - t->nodes[ni].child = pi; + if(last < 0) + t->nodes[ni].child = pi; + else + t->nodes[last].sibling = pi; return pi; } @@ -153,24 +159,32 @@ trieclose(Trie *t) free(t); } -/* A nil trie is an unloaded map: every key misses. */ +/* The node key leads to, or -1. A nil trie is an unloaded map: every key misses. */ int -trielookup(Trie *t, Str *key, char **val, int *vlen) +trienode(Trie *t, Str *key) { char buf[Maxutf]; int i, klen, ni; - *val = nil; - *vlen = 0; if(t == nil) - return TrieMiss; + return -1; klen = stoutf(key, buf, sizeof buf); ni = 0; - for(i = 0; i < klen; i++){ + for(i = 0; i < klen && ni >= 0; i++) ni = find(t, ni, buf[i]); - if(ni < 0) - return TrieMiss; - } + return ni; +} + +int +trielookup(Trie *t, Str *key, char **val, int *vlen) +{ + int ni; + + *val = nil; + *vlen = 0; + ni = trienode(t, key); + if(ni < 0) + return TrieMiss; if(t->nodes[ni].val == nil) return TriePrefix; *val = t->nodes[ni].val;