emoji: search the dictionary by prefix in the engine, not in the data

mkemoji wrote a row for every prefix of every alias, so that a query
matched as it was typed; the trie is a prefix index already, and with a
real emoji list those rows would be four times the aliases themselves.
Now emoji.dict has one row per alias, trienode() names a key's node, and
dictlookup walks the entries at and below it, the key's own first, up to
Maxkouho.  Trie children are appended rather than pushed, so the walk
keeps the file's order and a bare digit still picks the superscript or
subscript it always did.

The hand-written symbol rows move to symbol.src; emoji.src is left to
the emoji.  mkemoji reads both by default, or the files it is given.

dict.c

@@ -2,22 +2,20 @@
 #include "fn.h"
 
 /*
- * Fills out[] with up to max candidates for key: the space-separated words
- * of the entry, minus the key itself except in the emoji dictionary, where
- * a query may name its own answer.
+ * Appends the space-separated words of a node's entry to out[], up to
+ * max: its candidates, minus the key itself, except in the emoji
+ * dictionary, where a query may name its own answer.
  */
-int
-dictlookup(Lang *l, Str *key, Str *out, int max)
+static int
+words(Lang *l, Str *key, Tnode *nd, Str *out, int n, int max)
 {
 	char *p, *e, *sp;
 	Str tmp;
-	int n, vlen;
 
-	if(l == nil || key->n == 0 ||
-	   trielookup(l->dict, key, &p, &vlen) != TrieExact)
-		return 0;
-	n = 0;
-	e = p + vlen;
+	if(nd->val == nil)
+		return n;
+	p = nd->val;
+	e = p + nd->vlen;
 	while(n < max && p < e){
 		while(p < e && *p == ' ')
 			p++;
@@ -31,6 +29,37 @@ dictlookup(Lang *l, Str *key, Str *out, int max)
 	return n;
 }
 
+/* The entries at and below a node, in the dictionary's order. */
+static int
+below(Lang *l, Str *key, int ni, Str *out, int n, int max)
+{
+	Tnode *nd;
+	int ci;
+
+	nd = &l->dict->nodes[ni];
+	n = words(l, key, nd, out, n, max);
+	for(ci = nd->child; ci >= 0 && n < max; ci = l->dict->nodes[ci].sibling)
+		n = below(l, key, ci, out, n, max);
+	return n;
+}
+
+/*
+ * Fills out[] with up to max candidates for key.  The emoji dictionary is
+ * searched by prefix, the key's own entry first; the others exactly.
+ */
+int
+dictlookup(Lang *l, Str *key, Str *out, int max)
+{
+	int ni;
+
+	ni = key->n == 0 ? -1 : trienode(l->dict, key);
+	if(ni < 0)
+		return 0;
+	if(l->lang == LangEMOJI)
+		return below(l, key, ni, out, 0, max);
+	return words(l, key, &l->dict->nodes[ni], out, 0, max);
+}
+
 static Trie*
 langopen(char *dir, char *name, char *ext)
 {

fn.h

@@ -15,6 +15,7 @@ Trie*	trienew(void);
 void	trieput(Trie*, char*, int, char*, int);
 Trie*	trieopen(char*);
 void	trieclose(Trie*);
+int	trienode(Trie*, Str*);
 int	trielookup(Trie*, Str*, char**, int*);
 
 Lang*	getlang(int);

map/emoji.dict

@@ -1,35 +1,20 @@
-!	⚠ ≠
 !!	⚠
-!=	≠
-*	★
 **	★
-+	±
 +-	±
--	→
 ->	→
-.	· … ÷
-..	· …
+..	·
 ...	…
 ./	÷
-:	☹ ☺
 :(	☹
 :)	☺
-<	← ≤ ♥ ≠
 <-	←
-<3	♥
 <=	≤
+<3	♥
 <>	≠
-=	≡ ⇒
+!=	≠
 ==	≡
 =>	⇒
->	≥
 >=	≥
-^	¹ ² ³ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁽ ⁾ ⁺ ⁻ ⁼ ⁰ ⁱ ⁿ
-^(	⁽
-^)	⁾
-^+	⁺
-^-	⁻
-^0	⁰
 ^1	¹
 ^2	²
 ^3	³
@@ -39,15 +24,6 @@
 ^7	⁷
 ^8	⁸
 ^9	⁹
-^=	⁼
-^i	ⁱ
-^n	ⁿ
-_	₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₍ ₎ ₊ ₋ ₌ ₀ ₐ ₑ ₒ ₓ
-_(	₍
-_)	₎
-_+	₊
-_-	₋
-_0	₀
 _1	₁
 _2	₂
 _3	₃
@@ -57,236 +33,108 @@ _6	₆
 _7	₇
 _8	₈
 _9	₉
+^(	⁽
+^)	⁾
+^+	⁺
+^-	⁻
+^=	⁼
+_(	₍
+_)	₎
+_+	₊
+_-	₋
 _=	₌
+~=	≈
+^0	⁰
+_0	₀
 _a	ₐ
-_e	ₑ
-_o	ₒ
-_x	ₓ
-a	α
-al	α
-alp	α
-alph	α
 alpha	α
-b	β 😊
-be	β
-bet	β
 beta	β
-bl	😊
-blu	😊
-blus	😊
-blush	😊
-c	χ ☕️
-ch	χ
 chi	χ
-co	☕️
-cof	☕️
-coff	☕️
-coffe	☕️
-coffee	☕️
-d	° δ Δ ↓
-de	Δ ° δ
 deg	°
-del	δ
-delt	δ
 delta	δ
+de	Δ
 dn	↓
-e	ε η
-ep	ε
+_e	ₑ
 eps	ε
-et	η
 eta	η
-f	凸
-fu	凸
-fuc	凸
 fuck	凸
-g	γ Γ 😀
-ga	Γ γ
-gam	γ
-gamm	γ
 gamma	γ
-gr	😀
-gri	😀
-grin	😀
-h	😄 😊 ❤️
-ha	😄 😊
-hap	😄 😊
-happ	😄 😊
-happy	😄 😊
-he	❤️
-hea	❤️
-hear	❤️
-heart	❤️
-i	∫ ∞ ι
+ga	Γ
+^i	ⁱ
 ii	∫
-in	∞
 inf	∞
-io	ι
-iot	ι
 iota	ι
-j	😂
-jo	😂
-joy	😂
-k	κ
-ka	κ
-kap	κ
-kapp	κ
 kappa	κ
-l	λ Λ 😂 ❤️ 👍
-la	Λ λ
-lam	λ
-lamb	λ
-lambd	λ
 lambda	λ
-li	👍
-lik	👍
-like	👍
-lo	😂 ❤️
-lol	😂
-lov	❤️
-love	❤️
-m	× μ
-mu	μ ×
+la	Λ
 mul	×
-n	ν
+mu	μ
+^n	ⁿ
 nu	ν
-o	ω Ω ●
-om	Ω ω
-ome	ω
-omeg	ω
 omega	ω
+om	Ω
 oo	●
-p	φ Φ π Π ∏ ψ Ψ
-ph	Φ φ
+_o	ₒ
 phi	φ
+ph	Φ
 pi	π Π
 pp	∏
-ps	Ψ ψ
 psi	ψ
-r	ρ
-rh	ρ
+ps	Ψ
 rho	ρ
-s	σ Σ √ ∑ 😀 😄 😊
-si	Σ σ
-sig	σ
-sigm	σ
 sigma	σ
-sm	😀 😄 😊
-smi	😀 😄 😊
-smil	😀 😄 😊
-smile	😀 😄 😊
+si	Σ
 sq	√
 ss	∑
-t	τ θ Θ 👍
-ta	τ
 tau	τ
-th	Θ θ 👍
-the	θ
-thet	θ
 theta	θ
-thu	👍
-thum	👍
-thumb	👍
-thumbs	👍
-thumbsu	👍
-thumbsup	👍
-u	↑ υ
-up	↑ υ
+th	Θ
+up	↑
 ups	υ
-v	✓
 vv	✓
-x	ξ Ξ ✗
 xi	ξ Ξ
 xx	✗
-z	ζ
-ze	ζ
-zet	ζ
+_x	ₓ
 zeta	ζ
-~	≈
-~=	≈
-あ	❤️
-あい	❤️
-い	👍
-いい	👍
-いいね	👍
-え	😀 😄 😊
-えが	😀 😄 😊
-えがお	😀 😄 😊
-こ	☕️
-こー	☕️
-こーひ	☕️
-こーひー	☕️
-さ	👍
-さん	👍
-さんせ	👍
-さんせい	👍
-ば	😂
-ばく	😂
-ばくし	😂
-ばくしょ	😂
-ばくしょう	😂
-ほ	😄 😊
-ほほ	😄 😊
-ほほえ	😄 😊
-ほほえみ	😄 😊
-わ	😀 😂
-わら	😀 😂
-わらい	😀 😂
-ア	❤️
-アイ	❤️
-イ	👍
-イイ	👍
-イイネ	👍
-エ	😀 😄 😊
-エガ	😀 😄 😊
-エガオ	😀 😄 😊
-コ	☕️
-コー	☕️
-コーヒ	☕️
-コーヒー	☕️
-サ	👍
-サン	👍
-サンセ	👍
-サンセイ	👍
-ハ	❤️
-ハー	❤️
-ハート	❤️
-バ	😂
-バク	😂
-バクシ	😂
-バクショ	😂
-バクショウ	😂
-ホ	😄 😊
-ホホ	😄 😊
-ホホエ	😄 😊
-ホホエミ	😄 😊
-ワ	😀 😂
-ワラ	😀 😂
-ワライ	😀 😂
-따	👍
-따봉	👍
-미	😄 😊
-미소	😄 😊
-방	😊
-방긋	😊
-사	❤️
-사랑	❤️
-스	😀 😄
-스마	😀 😄
-스마일	😀 😄
-엄	👍
-엄지	👍
-웃	😀 😄 😂
-웃겨	😂
-웃다	😀
 웃음	😀 😄 😂
-좋	👍
-좋아	👍
-좋아요	👍
-찬	👍
-찬성	👍
-커	☕️
-커피	☕️
-폭	😂
+웃다	😀
+스마일	😀 😄
+えがお	😀 😄 😊
+エガオ	😀 😄 😊
+わらい	😀 😂
+ワライ	😀 😂
+smile	😀 😄 😊
+grin	😀
+미소	😄 😊
+ほほえみ	😄 😊
+ホホエミ	😄 😊
+happy	😄 😊
 폭소	😂
-하	❤️
+웃겨	😂
+ばくしょう	😂
+バクショウ	😂
+joy	😂
+lol	😂
+방긋	😊
+blush	😊
+사랑	❤️
 하트	❤️
+あい	❤️
+アイ	❤️
+ハート	❤️
+heart	❤️
+love	❤️
+좋아요	👍
+찬성	👍
+따봉	👍
+엄지	👍
+いいね	👍
+イイネ	👍
+さんせい	👍
+サンセイ	👍
+thumbsup	👍
+like	👍
+커피	☕️
+こーひー	☕️
+コーヒー	☕️
+coffee	☕️

map/emoji.src

@@ -1,107 +1,3 @@
-# Result first, followed by one or more TAB-separated aliases.
-⚠	!!
-★	**
-±	+-
-→	->
-·	..
-…	...
-÷	./
-☹	:(
-☺	:)
-
-# Bare 1-9 choose prefix candidates; keep digit aliases in matching slots.
-←	<-
-≤	<=
-♥	<3
-≠	<>	!=
-≡	==
-⇒	=>
-≥	>=
-¹	^1
-²	^2
-³	^3
-⁴	^4
-⁵	^5
-⁶	^6
-⁷	^7
-⁸	^8
-⁹	^9
-₁	_1
-₂	_2
-₃	_3
-₄	_4
-₅	_5
-₆	_6
-₇	_7
-₈	_8
-₉	_9
-
-⁽	^(
-⁾	^)
-⁺	^+
-⁻	^-
-⁼	^=
-₍	_(
-₎	_)
-₊	_+
-₋	_-
-₌	_=
-≈	~=
-⁰	^0
-₀	_0
-ₐ	_a
-α	alpha
-β	beta
-χ	chi
-°	deg
-δ	delta
-Δ	De
-↓	dn
-ₑ	_e
-ε	eps
-η	eta
-凸	fuck
-γ	gamma
-Γ	Ga
-ⁱ	^i
-∫	II
-∞	inf
-ι	iota
-κ	kappa
-λ	lambda
-Λ	La
-×	mul
-μ	mu
-ⁿ	^n
-ν	nu
-ω	omega
-Ω	Om
-●	oo
-ₒ	_o
-φ	phi
-Φ	Ph
-π	pi
-Π	Pi
-∏	PP
-ψ	psi
-Ψ	Ps
-ρ	rho
-σ	sigma
-Σ	Si
-√	sq
-∑	SS
-τ	tau
-θ	theta
-Θ	Th
-↑	up
-υ	ups
-✓	vv
-ξ	xi
-Ξ	Xi
-✗	xx
-ₓ	_x
-ζ	zeta
-
 # Small multilingual emoji seed. Shared aliases intentionally produce choices.
 😀	웃음	웃다	스마일	えがお	エガオ	わらい	ワライ	smile	grin
 😄	웃음	미소	스마일	えがお	エガオ	ほほえみ	ホホエミ	smile	happy

map/mkemoji

@@ -1,13 +1,15 @@
 #!/usr/bin/env python3
-"""Write emoji.dict to stdout from a result-first UTF-8 TSV file."""
+"""Write emoji.dict to stdout from result-first UTF-8 TSV files."""
 
 import sys
 import unicodedata
 from pathlib import Path
 
 
-SOURCE = Path(__file__).with_name("emoji.src")
+SOURCES = [Path(__file__).with_name(name)
+           for name in ("symbol.src", "emoji.src")]
 MAXRUNES = 64
+MAXCANDIDATES = 128
 
 
 def fold(s):
@@ -44,31 +46,23 @@ def read(path):
     return entries
 
 
-def add(table, key, result):
-    values = table.setdefault(key, [])
-    if result not in values:
-        values.append(result)
-
-
 def build(entries):
-    exact = {}
-    prefix = {}
+    """One row per alias, in order of first appearance; the engine searches
+    the dictionary by prefix, so no prefix rows are needed."""
+    table = {}
     for result, alias in entries:
-        for n in range(1, len(alias) + 1):
-            add(exact if n == len(alias) else prefix, alias[:n], result)
-    for key in sorted(exact.keys() | prefix.keys()):
-        values = exact.get(key, []) + prefix.get(key, [])
-        values = list(dict.fromkeys(values))
-        yield f"{key}\t{' '.join(values)}"
+        values = table.setdefault(alias, [])
+        if result not in values:
+            values.append(result)
+    for alias, values in table.items():
+        yield f"{alias}\t{' '.join(values[:MAXCANDIDATES])}"
 
 
 def main():
-    if len(sys.argv) > 2:
-        print(f"usage: {sys.argv[0]} [emoji.src]", file=sys.stderr)
-        return 2
-    path = Path(sys.argv[1]) if len(sys.argv) == 2 else SOURCE
+    paths = [Path(arg) for arg in sys.argv[1:]] or SOURCES
     try:
-        for line in build(read(path)):
+        entries = [entry for path in paths for entry in read(path)]
+        for line in build(entries):
             print(line)
     except (OSError, UnicodeError, ValueError) as error:
         print(error, file=sys.stderr)

map/symbol.src

@@ -0,0 +1,103 @@
+# Result first, followed by one or more TAB-separated aliases.
+⚠	!!
+★	**
+±	+-
+→	->
+·	..
+…	...
+÷	./
+☹	:(
+☺	:)
+
+# Bare 1-9 choose from a prefix search; keep digit aliases in matching slots.
+←	<-
+≤	<=
+♥	<3
+≠	<>	!=
+≡	==
+⇒	=>
+≥	>=
+¹	^1
+²	^2
+³	^3
+⁴	^4
+⁵	^5
+⁶	^6
+⁷	^7
+⁸	^8
+⁹	^9
+₁	_1
+₂	_2
+₃	_3
+₄	_4
+₅	_5
+₆	_6
+₇	_7
+₈	_8
+₉	_9
+
+⁽	^(
+⁾	^)
+⁺	^+
+⁻	^-
+⁼	^=
+₍	_(
+₎	_)
+₊	_+
+₋	_-
+₌	_=
+≈	~=
+⁰	^0
+₀	_0
+ₐ	_a
+α	alpha
+β	beta
+χ	chi
+°	deg
+δ	delta
+Δ	De
+↓	dn
+ₑ	_e
+ε	eps
+η	eta
+凸	fuck
+γ	gamma
+Γ	Ga
+ⁱ	^i
+∫	II
+∞	inf
+ι	iota
+κ	kappa
+λ	lambda
+Λ	La
+×	mul
+μ	mu
+ⁿ	^n
+ν	nu
+ω	omega
+Ω	Om
+●	oo
+ₒ	_o
+φ	phi
+Φ	Ph
+π	pi
+Π	Pi
+∏	PP
+ψ	psi
+Ψ	Ps
+ρ	rho
+σ	sigma
+Σ	Si
+√	sq
+∑	SS
+τ	tau
+θ	theta
+Θ	Th
+↑	up
+υ	ups
+✓	vv
+ξ	xi
+Ξ	Xi
+✗	xx
+ₓ	_x
+ζ	zeta

tests/engine_test.c

@@ -1331,13 +1331,14 @@ engine_emoji_queries(struct ct *t)
 	CT_CHECK(t, keystroke('e', Mctrl, &com));
 	CT_CHECK(t, keystroke('a', 0, &com));
 	checkstr(t, "raw query wins", "a", &search.text);
-	CT_EQ_INT(t, 3, im.nkouho);
+	CT_EQ_INT(t, 4, im.nkouho);
 	checkstr(t, "raw candidate", "A", &im.kouho[0]);
 	checkstr(t, "deduplicated candidate", "B", &im.kouho[1]);
-	checkstr(t, "local candidate", "C", &im.kouho[2]);
+	checkstr(t, "raw prefix candidate", "α", &im.kouho[2]);
+	checkstr(t, "local candidate", "C", &im.kouho[3]);
 	CT_CHECK(t, draindraw(&dc) > 0);
 	checkstr(t, "prefix popup query", "a", &dc.pre);
-	CT_EQ_INT(t, 3, dc.nkouho);
+	CT_EQ_INT(t, 4, dc.nkouho);
 	CT_CHECK(t, keystroke(Kesc, 0, &com));
 
 	CT_CHECK(t, keystroke('e', Mctrl, &com));

tests/mkemoji_test.py

@@ -12,10 +12,8 @@ MKEMOJI = ROOT / "map" / "mkemoji"
 TIMEOUT = 10
 
 
-def generate(source=None):
-    args = [sys.executable, "-B", str(MKEMOJI)]
-    if source is not None:
-        args.append(str(source))
+def generate(*sources):
+    args = [sys.executable, "-B", str(MKEMOJI)] + [str(s) for s in sources]
     return subprocess.run(
         args, capture_output=True, text=True, check=False, timeout=TIMEOUT
     )
@@ -37,27 +35,32 @@ class MkemojiTest(unittest.TestCase):
         result = generate()
         self.assertEqual(result.returncode, 0, result.stderr)
         data = table(result.stdout)
-        self.assertEqual(data["^"].split()[:9], list("¹²³⁴⁵⁶⁷⁸⁹"))
-        self.assertEqual(data["_"].split()[:9], list("₁₂₃₄₅₆₇₈₉"))
-        self.assertEqual(data["<"].split()[2], "♥")
+        keys = list(data)
+        self.assertEqual([k for k in keys if k[:1] == "^"][:9],
+                         [f"^{d}" for d in range(1, 10)])
+        self.assertEqual([k for k in keys if k[:1] == "_"][:9],
+                         [f"_{d}" for d in range(1, 10)])
+        self.assertEqual([k for k in keys if k[:1] == "<"][2], "<3")
         self.assertEqual(data["^1"], "¹")
         self.assertEqual(data["_2"], "₂")
-        self.assertEqual(data["<3"], "♥")
+        self.assertEqual(data["<3"].split()[0], "♥")
+        self.assertIn("😀", data["smile"].split())
+        self.assertIn("😀", data["웃음"].split())
 
-    def test_fold_normalize_and_exact_first(self):
+    def test_fold_normalize_and_source_order(self):
         source = self.source(
-            "β\tALPHABET\n"
+            "β\tALPHABET\talpha\n"
             "α\talpha\n"
-            "e\u0301\tE\u0301\n"
+            "é\tÉ\n"
             "#\thash\n"
         )
         first = generate(source)
-        second = generate(source)
+        second = generate(source, source)
         self.assertEqual(first.returncode, 0, first.stderr)
         self.assertEqual(first.stdout, second.stdout)
         data = table(first.stdout)
-        self.assertEqual(data["alpha"].split(), ["α", "β"])
-        self.assertEqual(data["al"].split(), ["β", "α"])
+        self.assertEqual(list(data), ["alphabet", "alpha", "é", "hash"])
+        self.assertEqual(data["alpha"].split(), ["β", "α"])
         self.assertEqual(data["é"], "é")
         self.assertEqual(data["hash"], "#")
 

trie.c

@@ -35,15 +35,21 @@ find(Trie *t, int ni, char c)
 	return -1;
 }
 
+/* Appends, so that a walk of the children keeps the file's order. */
 static int
 add(Trie *t, int ni, char c)
 {
-	int pi;
+	int last, pi;
 
+	last = -1;
+	for(pi = t->nodes[ni].child; pi >= 0; pi = t->nodes[pi].sibling)
+		last = pi;
 	pi = newnode(t);
 	t->nodes[pi].c = c;
-	t->nodes[pi].sibling = t->nodes[ni].child;
-	t->nodes[ni].child = pi;
+	if(last < 0)
+		t->nodes[ni].child = pi;
+	else
+		t->nodes[last].sibling = pi;
 	return pi;
 }
 
@@ -153,24 +159,32 @@ trieclose(Trie *t)
 	free(t);
 }
 
-/* A nil trie is an unloaded map: every key misses. */
+/* The node key leads to, or -1.  A nil trie is an unloaded map: every key misses. */
 int
-trielookup(Trie *t, Str *key, char **val, int *vlen)
+trienode(Trie *t, Str *key)
 {
 	char buf[Maxutf];
 	int i, klen, ni;
 
-	*val = nil;
-	*vlen = 0;
 	if(t == nil)
-		return TrieMiss;
+		return -1;
 	klen = stoutf(key, buf, sizeof buf);
 	ni = 0;
-	for(i = 0; i < klen; i++){
+	for(i = 0; i < klen && ni >= 0; i++)
 		ni = find(t, ni, buf[i]);
-		if(ni < 0)
-			return TrieMiss;
-	}
+	return ni;
+}
+
+int
+trielookup(Trie *t, Str *key, char **val, int *vlen)
+{
+	int ni;
+
+	*val = nil;
+	*vlen = 0;
+	ni = trienode(t, key);
+	if(ni < 0)
+		return TrieMiss;
 	if(t->nodes[ni].val == nil)
 		return TriePrefix;
 	*val = t->nodes[ni].val;