data: simplify and validate Japanese imports
This commit is contained in:
186
map/skk2ktrans
186
map/skk2ktrans
@@ -1,143 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert an EUC-JP SKK dictionary to strans dictionary format."""
|
||||
#!/usr/bin/env bash
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from pathlib import Path
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
class InputError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def candidates(field, source, lineno):
|
||||
"""Return literal candidates, dropping SKK annotations and expressions."""
|
||||
if not field.startswith("/"):
|
||||
raise InputError(f"{source}:{lineno}: candidate list does not start with /")
|
||||
|
||||
result = []
|
||||
buf = []
|
||||
annotation = False
|
||||
escaped = False
|
||||
closed = False
|
||||
brackets = 0
|
||||
parens = 0
|
||||
quoted = False
|
||||
for ch in field[1:]:
|
||||
if escaped:
|
||||
if not annotation:
|
||||
buf.append(ch)
|
||||
escaped = False
|
||||
closed = False
|
||||
continue
|
||||
if ch == "\\":
|
||||
escaped = True
|
||||
closed = False
|
||||
continue
|
||||
if ch == "/" and (annotation or (brackets == 0 and parens == 0)):
|
||||
candidate = "".join(buf)
|
||||
if "\0" in candidate:
|
||||
raise InputError(f"{source}:{lineno}: NUL in candidate")
|
||||
if (candidate and not candidate.startswith(("(", "[", "#"))
|
||||
and not any(c in " \t\r\n" for c in candidate)):
|
||||
result.append(candidate)
|
||||
buf = []
|
||||
annotation = False
|
||||
closed = True
|
||||
continue
|
||||
if ch == ";" and not annotation and brackets == 0 and parens == 0:
|
||||
annotation = True
|
||||
closed = False
|
||||
continue
|
||||
if not annotation:
|
||||
if ch == '"' and parens:
|
||||
quoted = not quoted
|
||||
elif not quoted:
|
||||
if ch == "[":
|
||||
brackets += 1
|
||||
elif ch == "]" and brackets:
|
||||
brackets -= 1
|
||||
elif ch == "(":
|
||||
parens += 1
|
||||
elif ch == ")" and parens:
|
||||
parens -= 1
|
||||
buf.append(ch)
|
||||
closed = False
|
||||
|
||||
if escaped:
|
||||
raise InputError(f"{source}:{lineno}: trailing escape")
|
||||
if not closed:
|
||||
raise InputError(f"{source}:{lineno}: unterminated candidate list")
|
||||
return result
|
||||
|
||||
|
||||
def convert(inputs):
|
||||
rows = OrderedDict()
|
||||
seen = {}
|
||||
for source, data in inputs:
|
||||
try:
|
||||
text = data.decode("euc_jp")
|
||||
except UnicodeDecodeError as error:
|
||||
raise InputError(f"{source}: invalid EUC-JP input: {error}") from error
|
||||
for lineno, line in enumerate(text.splitlines(), 1):
|
||||
if not line or line.startswith(";;"):
|
||||
continue
|
||||
fields = line.split(None, 1)
|
||||
if len(fields) != 2:
|
||||
raise InputError(f"{source}:{lineno}: missing candidate list")
|
||||
key, field = fields
|
||||
if any(c in " \t\r\n" for c in key):
|
||||
raise InputError(f"{source}:{lineno}: whitespace in key")
|
||||
if "\0" in key:
|
||||
raise InputError(f"{source}:{lineno}: NUL in key")
|
||||
if len(key) > 64:
|
||||
raise InputError(f"{source}:{lineno}: key exceeds 64 runes")
|
||||
if key not in rows:
|
||||
rows[key] = []
|
||||
seen[key] = set()
|
||||
for candidate in candidates(field, source, lineno):
|
||||
if candidate not in seen[key]:
|
||||
rows[key].append(candidate)
|
||||
seen[key].add(candidate)
|
||||
return rows
|
||||
|
||||
|
||||
def read_inputs(names):
|
||||
if not names:
|
||||
return [("<stdin>", sys.stdin.buffer.read())]
|
||||
inputs = []
|
||||
stdin_used = False
|
||||
for name in names:
|
||||
if name == "-":
|
||||
if stdin_used:
|
||||
raise InputError("standard input may be specified only once")
|
||||
stdin_used = True
|
||||
inputs.append(("<stdin>", sys.stdin.buffer.read()))
|
||||
else:
|
||||
path = Path(name)
|
||||
try:
|
||||
inputs.append((name, path.read_bytes()))
|
||||
except OSError as error:
|
||||
raise InputError(f"{name}: {error.strerror}") from error
|
||||
return inputs
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV")
|
||||
parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
rows = convert(read_inputs(args.dictionary))
|
||||
except InputError as error:
|
||||
print(f"skk2ktrans: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values]
|
||||
sys.stdout.buffer.write("".join(lines).encode("utf-8"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
iconv -f EUC-JP -t UTF-8 "$@" | awk '
|
||||
function fail(s) {
|
||||
print "skk2ktrans: " FILENAME ":" FNR ": " s >"/dev/stderr"
|
||||
bad = 1
|
||||
exit 1
|
||||
}
|
||||
function add(k, v, id) {
|
||||
sub(/;.*/, "", v)
|
||||
if(v == "" || v ~ /^[([#]/ || v ~ /[[:space:]]/)
|
||||
return
|
||||
id = k SUBSEP v
|
||||
if(seen[id])
|
||||
return
|
||||
seen[id] = 1
|
||||
if(!(k in row))
|
||||
order[++nkey] = k
|
||||
else
|
||||
row[k] = row[k] " "
|
||||
row[k] = row[k] v
|
||||
}
|
||||
/^;;/ || /^[[:space:]]*$/ {
|
||||
next
|
||||
}
|
||||
{
|
||||
if(!match($0, /[[:space:]]+/))
|
||||
fail("missing candidate list")
|
||||
key = substr($0, 1, RSTART-1)
|
||||
field = substr($0, RSTART+RLENGTH)
|
||||
if(key == "" || key ~ /^;/ || field !~ /^\/.*\/$/)
|
||||
fail("invalid row")
|
||||
if(field ~ /\\/)
|
||||
fail("escaped candidates are unsupported")
|
||||
n = split(substr(field, 2, length(field)-2), value, "/")
|
||||
for(i = 1; i <= n; i++)
|
||||
add(key, value[i])
|
||||
}
|
||||
END {
|
||||
if(bad)
|
||||
exit 1
|
||||
for(i = 1; i <= nkey; i++)
|
||||
print order[i] "\t" row[order[i]]
|
||||
}
|
||||
'
|
||||
|
||||
Reference in New Issue
Block a user