Hojun-Cho
abff5ed122
mkemoji wrote a row for every prefix of every alias, so that a query matched as it was typed; the trie is a prefix index already, and with a real emoji list those rows would be four times the aliases themselves. Now emoji.dict has one row per alias, trienode() names a key's node, and dictlookup walks the entries at and below it, the key's own first, up to Maxkouho. Trie children are appended rather than pushed, so the walk keeps the file's order and a bare digit still picks the superscript or subscript it always did. The hand-written symbol rows move to symbol.src; emoji.src is left to the emoji. mkemoji reads both by default, or the files it is given.
75 lines
2.3 KiB
Python
Executable File
75 lines
2.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Write emoji.dict to stdout from result-first UTF-8 TSV files."""
|
|
|
|
import sys
|
|
import unicodedata
|
|
from pathlib import Path
|
|
|
|
|
|
SOURCES = [Path(__file__).with_name(name)
|
|
for name in ("symbol.src", "emoji.src")]
|
|
MAXRUNES = 64
|
|
MAXCANDIDATES = 128
|
|
|
|
|
|
def fold(s):
|
|
s = "".join(chr(ord(c) + 32) if "A" <= c <= "Z" else c for c in s)
|
|
return unicodedata.normalize("NFC", s)
|
|
|
|
|
|
def hascontrol(s):
|
|
return any(unicodedata.category(c) == "Cc" for c in s)
|
|
|
|
|
|
def read(path):
|
|
entries = []
|
|
with path.open(encoding="utf-8") as src:
|
|
for lineno, raw in enumerate(src, 1):
|
|
line = raw.rstrip("\r\n")
|
|
if not line.strip():
|
|
continue
|
|
if "\t" not in line and line.lstrip().startswith("#"):
|
|
continue
|
|
fields = line.split("\t")
|
|
if len(fields) < 2:
|
|
raise ValueError(f"{path}:{lineno}: need a result and an alias")
|
|
result = unicodedata.normalize("NFC", fields[0])
|
|
if (not result or len(result) > MAXRUNES or hascontrol(result)
|
|
or any(c.isspace() for c in result)):
|
|
raise ValueError(f"{path}:{lineno}: bad result")
|
|
for field in fields[1:]:
|
|
alias = fold(field)
|
|
if (not alias or len(alias) > MAXRUNES or hascontrol(alias)
|
|
or alias != alias.strip() or alias.startswith(";")):
|
|
raise ValueError(f"{path}:{lineno}: bad alias")
|
|
entries.append((result, alias))
|
|
return entries
|
|
|
|
|
|
def build(entries):
|
|
"""One row per alias, in order of first appearance; the engine searches
|
|
the dictionary by prefix, so no prefix rows are needed."""
|
|
table = {}
|
|
for result, alias in entries:
|
|
values = table.setdefault(alias, [])
|
|
if result not in values:
|
|
values.append(result)
|
|
for alias, values in table.items():
|
|
yield f"{alias}\t{' '.join(values[:MAXCANDIDATES])}"
|
|
|
|
|
|
def main():
|
|
paths = [Path(arg) for arg in sys.argv[1:]] or SOURCES
|
|
try:
|
|
entries = [entry for path in paths for entry in read(path)]
|
|
for line in build(entries):
|
|
print(line)
|
|
except (OSError, UnicodeError, ValueError) as error:
|
|
print(error, file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|