Hojun-Cho
0ba7dc7f50
Δ Γ Λ Ω Φ Ψ Σ Θ answered only to De, Ga, La, Om, Ph, Ps, Si, Th, while their small letters answered to delta, gamma and the rest; they answer to the names too now. mkemoji and cldr2emoji write UTF-8 whatever the locale, as the other generators already did; verify-map calls python3 the one way; the tests' include path drops a directory nothing includes through; and bench.sh says which binary is missing instead of blaming the daemon ten seconds later.
87 lines
2.7 KiB
Python
Executable File
87 lines
2.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Write emoji.dict to stdout from result-first UTF-8 TSV files."""
|
|
|
|
import sys
|
|
import unicodedata
|
|
from pathlib import Path
|
|
|
|
|
|
SOURCES = [Path(__file__).with_name(name)
|
|
for name in ("symbol.src", "emoji.src")]
|
|
MAXRUNES = 64
|
|
MAXCANDIDATES = 128
|
|
|
|
|
|
HIRA = {c: c + 0x60 for c in range(0x3041, 0x3097)}
|
|
KATA = {c: c - 0x60 for c in range(0x30A1, 0x30F7)}
|
|
|
|
|
|
def fold(s):
|
|
s = "".join(chr(ord(c) + 32) if "A" <= c <= "Z" else c for c in s)
|
|
return unicodedata.normalize("NFC", s)
|
|
|
|
|
|
def kana(alias):
|
|
"""The alias, and in the other kana where it has any: a query typed
|
|
in either Japanese mode finds it."""
|
|
return {alias, alias.translate(HIRA), alias.translate(KATA)}
|
|
|
|
|
|
def hascontrol(s):
|
|
return any(unicodedata.category(c) == "Cc" for c in s)
|
|
|
|
|
|
def read(path):
|
|
entries = []
|
|
with path.open(encoding="utf-8") as src:
|
|
for lineno, raw in enumerate(src, 1):
|
|
line = raw.rstrip("\r\n")
|
|
if not line.strip():
|
|
continue
|
|
if "\t" not in line and line.lstrip().startswith("#"):
|
|
continue
|
|
fields = line.split("\t")
|
|
if len(fields) < 2:
|
|
raise ValueError(f"{path}:{lineno}: need a result and an alias")
|
|
result = unicodedata.normalize("NFC", fields[0])
|
|
if (not result or len(result) > MAXRUNES or hascontrol(result)
|
|
or any(c.isspace() for c in result)):
|
|
raise ValueError(f"{path}:{lineno}: bad result")
|
|
for field in fields[1:]:
|
|
alias = fold(field)
|
|
if (not alias or len(alias) > MAXRUNES or hascontrol(alias)
|
|
or alias != alias.strip() or alias.startswith(";")):
|
|
raise ValueError(f"{path}:{lineno}: bad alias")
|
|
entries.extend((result, a) for a in sorted(kana(alias)))
|
|
return entries
|
|
|
|
|
|
def build(entries):
|
|
"""One row per alias, in order of first appearance; the engine searches
|
|
the dictionary by prefix, so no prefix rows are needed."""
|
|
table = {}
|
|
for result, alias in entries:
|
|
values = table.setdefault(alias, [])
|
|
if result not in values:
|
|
values.append(result)
|
|
for alias, values in table.items():
|
|
yield f"{alias}\t{' '.join(values[:MAXCANDIDATES])}"
|
|
|
|
|
|
def main():
|
|
sys.stdout.reconfigure(encoding="utf-8")
|
|
sys.stderr.reconfigure(encoding="utf-8")
|
|
paths = [Path(arg) for arg in sys.argv[1:]] or SOURCES
|
|
try:
|
|
entries = [entry for path in paths for entry in read(path)]
|
|
for line in build(entries):
|
|
print(line)
|
|
except (OSError, UnicodeError, ValueError) as error:
|
|
print(error, file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|