emoji: search the dictionary by prefix in the engine, not in the data
mkemoji wrote a row for every prefix of every alias, so that a query matched as it was typed; the trie is a prefix index already, and with a real emoji list those rows would be four times the aliases themselves. Now emoji.dict has one row per alias, trienode() names a key's node, and dictlookup walks the entries at and below it, the key's own first, up to Maxkouho. Trie children are appended rather than pushed, so the walk keeps the file's order and a bare digit still picks the superscript or subscript it always did. The hand-written symbol rows move to symbol.src; emoji.src is left to the emoji. mkemoji reads both by default, or the files it is given.
This commit is contained in:
36
map/mkemoji
36
map/mkemoji
@@ -1,13 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write emoji.dict to stdout from a result-first UTF-8 TSV file."""
|
||||
"""Write emoji.dict to stdout from result-first UTF-8 TSV files."""
|
||||
|
||||
import sys
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SOURCE = Path(__file__).with_name("emoji.src")
|
||||
SOURCES = [Path(__file__).with_name(name)
|
||||
for name in ("symbol.src", "emoji.src")]
|
||||
MAXRUNES = 64
|
||||
MAXCANDIDATES = 128
|
||||
|
||||
|
||||
def fold(s):
|
||||
@@ -44,31 +46,23 @@ def read(path):
|
||||
return entries
|
||||
|
||||
|
||||
def add(table, key, result):
|
||||
values = table.setdefault(key, [])
|
||||
if result not in values:
|
||||
values.append(result)
|
||||
|
||||
|
||||
def build(entries):
|
||||
exact = {}
|
||||
prefix = {}
|
||||
"""One row per alias, in order of first appearance; the engine searches
|
||||
the dictionary by prefix, so no prefix rows are needed."""
|
||||
table = {}
|
||||
for result, alias in entries:
|
||||
for n in range(1, len(alias) + 1):
|
||||
add(exact if n == len(alias) else prefix, alias[:n], result)
|
||||
for key in sorted(exact.keys() | prefix.keys()):
|
||||
values = exact.get(key, []) + prefix.get(key, [])
|
||||
values = list(dict.fromkeys(values))
|
||||
yield f"{key}\t{' '.join(values)}"
|
||||
values = table.setdefault(alias, [])
|
||||
if result not in values:
|
||||
values.append(result)
|
||||
for alias, values in table.items():
|
||||
yield f"{alias}\t{' '.join(values[:MAXCANDIDATES])}"
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 2:
|
||||
print(f"usage: {sys.argv[0]} [emoji.src]", file=sys.stderr)
|
||||
return 2
|
||||
path = Path(sys.argv[1]) if len(sys.argv) == 2 else SOURCE
|
||||
paths = [Path(arg) for arg in sys.argv[1:]] or SOURCES
|
||||
try:
|
||||
for line in build(read(path)):
|
||||
entries = [entry for path in paths for entry in read(path)]
|
||||
for line in build(entries):
|
||||
print(line)
|
||||
except (OSError, UnicodeError, ValueError) as error:
|
||||
print(error, file=sys.stderr)
|
||||
|
||||
Reference in New Issue
Block a user