144 lines
3.7 KiB
Python
Executable File
144 lines
3.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Convert an EUC-JP SKK dictionary to strans dictionary format."""
|
|
|
|
import argparse
|
|
import sys
|
|
from collections import OrderedDict
|
|
from pathlib import Path
|
|
|
|
|
|
class InputError(Exception):
|
|
pass
|
|
|
|
|
|
def candidates(field, source, lineno):
|
|
"""Return literal candidates, dropping SKK annotations and expressions."""
|
|
if not field.startswith("/"):
|
|
raise InputError(f"{source}:{lineno}: candidate list does not start with /")
|
|
|
|
result = []
|
|
buf = []
|
|
annotation = False
|
|
escaped = False
|
|
closed = False
|
|
brackets = 0
|
|
parens = 0
|
|
quoted = False
|
|
for ch in field[1:]:
|
|
if escaped:
|
|
if not annotation:
|
|
buf.append(ch)
|
|
escaped = False
|
|
closed = False
|
|
continue
|
|
if ch == "\\":
|
|
escaped = True
|
|
closed = False
|
|
continue
|
|
if ch == "/" and (annotation or (brackets == 0 and parens == 0)):
|
|
candidate = "".join(buf)
|
|
if "\0" in candidate:
|
|
raise InputError(f"{source}:{lineno}: NUL in candidate")
|
|
if (candidate and not candidate.startswith(("(", "[", "#"))
|
|
and not any(c in " \t\r\n" for c in candidate)):
|
|
result.append(candidate)
|
|
buf = []
|
|
annotation = False
|
|
closed = True
|
|
continue
|
|
if ch == ";" and not annotation and brackets == 0 and parens == 0:
|
|
annotation = True
|
|
closed = False
|
|
continue
|
|
if not annotation:
|
|
if ch == '"' and parens:
|
|
quoted = not quoted
|
|
elif not quoted:
|
|
if ch == "[":
|
|
brackets += 1
|
|
elif ch == "]" and brackets:
|
|
brackets -= 1
|
|
elif ch == "(":
|
|
parens += 1
|
|
elif ch == ")" and parens:
|
|
parens -= 1
|
|
buf.append(ch)
|
|
closed = False
|
|
|
|
if escaped:
|
|
raise InputError(f"{source}:{lineno}: trailing escape")
|
|
if not closed:
|
|
raise InputError(f"{source}:{lineno}: unterminated candidate list")
|
|
return result
|
|
|
|
|
|
def convert(inputs):
|
|
rows = OrderedDict()
|
|
seen = {}
|
|
for source, data in inputs:
|
|
try:
|
|
text = data.decode("euc_jp")
|
|
except UnicodeDecodeError as error:
|
|
raise InputError(f"{source}: invalid EUC-JP input: {error}") from error
|
|
for lineno, line in enumerate(text.splitlines(), 1):
|
|
if not line or line.startswith(";;"):
|
|
continue
|
|
fields = line.split(None, 1)
|
|
if len(fields) != 2:
|
|
raise InputError(f"{source}:{lineno}: missing candidate list")
|
|
key, field = fields
|
|
if any(c in " \t\r\n" for c in key):
|
|
raise InputError(f"{source}:{lineno}: whitespace in key")
|
|
if "\0" in key:
|
|
raise InputError(f"{source}:{lineno}: NUL in key")
|
|
if len(key) > 64:
|
|
raise InputError(f"{source}:{lineno}: key exceeds 64 runes")
|
|
if key not in rows:
|
|
rows[key] = []
|
|
seen[key] = set()
|
|
for candidate in candidates(field, source, lineno):
|
|
if candidate not in seen[key]:
|
|
rows[key].append(candidate)
|
|
seen[key].add(candidate)
|
|
return rows
|
|
|
|
|
|
def read_inputs(names):
|
|
if not names:
|
|
return [("<stdin>", sys.stdin.buffer.read())]
|
|
inputs = []
|
|
stdin_used = False
|
|
for name in names:
|
|
if name == "-":
|
|
if stdin_used:
|
|
raise InputError("standard input may be specified only once")
|
|
stdin_used = True
|
|
inputs.append(("<stdin>", sys.stdin.buffer.read()))
|
|
else:
|
|
path = Path(name)
|
|
try:
|
|
inputs.append((name, path.read_bytes()))
|
|
except OSError as error:
|
|
raise InputError(f"{name}: {error.strerror}") from error
|
|
return inputs
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV")
|
|
parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)")
|
|
args = parser.parse_args()
|
|
try:
|
|
rows = convert(read_inputs(args.dictionary))
|
|
except InputError as error:
|
|
print(f"skk2ktrans: {error}", file=sys.stderr)
|
|
return 1
|
|
|
|
lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values]
|
|
sys.stdout.buffer.write("".join(lines).encode("utf-8"))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|