diff --git a/dict.c b/dict.c index 55ab5f8..cb4739c 100644 --- a/dict.c +++ b/dict.c @@ -63,7 +63,7 @@ dictopen(char *path) Hmap *h; Biobuf *b; Str key; - char *line, *tab; + char *line, *tab, *p, *e; int len, lineno; b = Bopen(path, OREAD); @@ -81,13 +81,21 @@ dictopen(char *path) continue; } tab = strchr(line, '\t'); - if(tab == nil || tab >= line + len - 1){ - free(line); - continue; - } + if(tab == nil || tab == line || tab >= line + len - 1 || + strchr(tab+1, '\t') != nil) + die("malformed dictionary: %s:%d", path, lineno); *tab = '\0'; if(utflen(line) > Maxrunes) die("dictionary key too long: %s:%d", path, lineno); + for(p = tab+1; p < line+len; p = e+1){ + e = memchr(p, ' ', line+len-p); + if(e == nil) + e = line+len; + if(utfnlen(p, e-p) > Maxrunes) + die("dictionary candidate too long: %s:%d", path, lineno); + if(e == line+len) + break; + } sinit(&key, line, tab - line); hmapset(&h, &key, tab+1, len - (tab - line) - 1); free(line); diff --git a/map/README b/map/README index 828a41e..073cb20 100644 --- a/map/README +++ b/map/README @@ -20,23 +20,22 @@ in source order, duplicate candidates were removed, and the empty From the repository root, fetch a known upstream revision and convert it with: ``` -map/grabskkdicts map/skkdicts REVISION +git clone https://github.com/skk-dev/dict.git map/skkdicts +git -C map/skkdicts checkout --detach REVISION map/skk2ktrans map/skkdicts/SKK-JISYO.M >map/kanji.dict.new python3 map/verifymap.py map/kanji.dict.new ``` Use a full skk-dev/dict commit ID for `REVISION` and record it when replacing -the bundled data. Omitting `REVISION` intentionally fetches the upstream -default branch and is not reproducible. `grabskkdicts` refuses to overwrite an -existing destination. +the bundled data. Git is needed only to fetch upstream data; it is not part of +the normal build image. `skk2ktrans` accepts one or more EUC-JP SKK files (or standard input), writes UTF-8 tab-separated rows, and merges input in command-line and source order. -It strips SKK annotations, deduplicates candidates, and omits Lisp expressions, -numeric conversion entries, bracket forms, and candidates containing -ASCII whitespace because those forms cannot be consumed as literal candidates -by `dict.c`. +It strips annotations, deduplicates candidates, and omits expressions and +candidates containing whitespace. Escaped candidate delimiters are rejected; +rewrite or omit those entries before import. -`verifymap.py` checks UTF-8, row structure, unique keys, the 64-rune key limit, -canonical candidate spacing, and duplicate dictionary candidates. Pass it the -exact `.map` and `.dict` files installed by the build. +`verifymap.py` checks UTF-8, row structure, unique keys, 64-rune keys and +values, canonical candidate spacing, and duplicate dictionary candidates. +Pass it the exact `.map` and `.dict` files installed by the build. diff --git a/map/grabskkdicts b/map/grabskkdicts deleted file mode 100755 index 6447127..0000000 --- a/map/grabskkdicts +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -set -eu - -repo=https://github.com/skk-dev/dict.git -dest=${1:-skkdicts} -revision=${2:-} - -if [ "$#" -gt 2 ] || [ -z "$dest" ]; then - echo "usage: $0 [destination [revision]]" >&2 - exit 2 -fi -if [ -e "$dest" ]; then - echo "$0: destination already exists: $dest" >&2 - exit 1 -fi - -git clone "$repo" "$dest" -if [ -n "$revision" ]; then - git -C "$dest" checkout --detach "$revision" -fi diff --git a/map/skk2ktrans b/map/skk2ktrans index 2a9403d..a8e077a 100755 --- a/map/skk2ktrans +++ b/map/skk2ktrans @@ -1,143 +1,47 @@ -#!/usr/bin/env python3 -"""Convert an EUC-JP SKK dictionary to strans dictionary format.""" +#!/usr/bin/env bash -import argparse -import sys -from collections import OrderedDict -from pathlib import Path +set -euo pipefail - -class InputError(Exception): - pass - - -def candidates(field, source, lineno): - """Return literal candidates, dropping SKK annotations and expressions.""" - if not field.startswith("/"): - raise InputError(f"{source}:{lineno}: candidate list does not start with /") - - result = [] - buf = [] - annotation = False - escaped = False - closed = False - brackets = 0 - parens = 0 - quoted = False - for ch in field[1:]: - if escaped: - if not annotation: - buf.append(ch) - escaped = False - closed = False - continue - if ch == "\\": - escaped = True - closed = False - continue - if ch == "/" and (annotation or (brackets == 0 and parens == 0)): - candidate = "".join(buf) - if "\0" in candidate: - raise InputError(f"{source}:{lineno}: NUL in candidate") - if (candidate and not candidate.startswith(("(", "[", "#")) - and not any(c in " \t\r\n" for c in candidate)): - result.append(candidate) - buf = [] - annotation = False - closed = True - continue - if ch == ";" and not annotation and brackets == 0 and parens == 0: - annotation = True - closed = False - continue - if not annotation: - if ch == '"' and parens: - quoted = not quoted - elif not quoted: - if ch == "[": - brackets += 1 - elif ch == "]" and brackets: - brackets -= 1 - elif ch == "(": - parens += 1 - elif ch == ")" and parens: - parens -= 1 - buf.append(ch) - closed = False - - if escaped: - raise InputError(f"{source}:{lineno}: trailing escape") - if not closed: - raise InputError(f"{source}:{lineno}: unterminated candidate list") - return result - - -def convert(inputs): - rows = OrderedDict() - seen = {} - for source, data in inputs: - try: - text = data.decode("euc_jp") - except UnicodeDecodeError as error: - raise InputError(f"{source}: invalid EUC-JP input: {error}") from error - for lineno, line in enumerate(text.splitlines(), 1): - if not line or line.startswith(";;"): - continue - fields = line.split(None, 1) - if len(fields) != 2: - raise InputError(f"{source}:{lineno}: missing candidate list") - key, field = fields - if any(c in " \t\r\n" for c in key): - raise InputError(f"{source}:{lineno}: whitespace in key") - if "\0" in key: - raise InputError(f"{source}:{lineno}: NUL in key") - if len(key) > 64: - raise InputError(f"{source}:{lineno}: key exceeds 64 runes") - if key not in rows: - rows[key] = [] - seen[key] = set() - for candidate in candidates(field, source, lineno): - if candidate not in seen[key]: - rows[key].append(candidate) - seen[key].add(candidate) - return rows - - -def read_inputs(names): - if not names: - return [("", sys.stdin.buffer.read())] - inputs = [] - stdin_used = False - for name in names: - if name == "-": - if stdin_used: - raise InputError("standard input may be specified only once") - stdin_used = True - inputs.append(("", sys.stdin.buffer.read())) - else: - path = Path(name) - try: - inputs.append((name, path.read_bytes())) - except OSError as error: - raise InputError(f"{name}: {error.strerror}") from error - return inputs - - -def main(): - parser = argparse.ArgumentParser( - description="convert EUC-JP SKK dictionaries to UTF-8 strans TSV") - parser.add_argument("dictionary", nargs="*", help="SKK file (default: stdin)") - args = parser.parse_args() - try: - rows = convert(read_inputs(args.dictionary)) - except InputError as error: - print(f"skk2ktrans: {error}", file=sys.stderr) - return 1 - - lines = [f"{key}\t{' '.join(values)}\n" for key, values in rows.items() if values] - sys.stdout.buffer.write("".join(lines).encode("utf-8")) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) +iconv -f EUC-JP -t UTF-8 "$@" | awk ' +function fail(s) { + print "skk2ktrans: " FILENAME ":" FNR ": " s >"/dev/stderr" + bad = 1 + exit 1 +} +function add(k, v, id) { + sub(/;.*/, "", v) + if(v == "" || v ~ /^[([#]/ || v ~ /[[:space:]]/) + return + id = k SUBSEP v + if(seen[id]) + return + seen[id] = 1 + if(!(k in row)) + order[++nkey] = k + else + row[k] = row[k] " " + row[k] = row[k] v +} +/^;;/ || /^[[:space:]]*$/ { + next +} +{ + if(!match($0, /[[:space:]]+/)) + fail("missing candidate list") + key = substr($0, 1, RSTART-1) + field = substr($0, RSTART+RLENGTH) + if(key == "" || key ~ /^;/ || field !~ /^\/.*\/$/) + fail("invalid row") + if(field ~ /\\/) + fail("escaped candidates are unsupported") + n = split(substr(field, 2, length(field)-2), value, "/") + for(i = 1; i <= n; i++) + add(key, value[i]) +} +END { + if(bad) + exit 1 + for(i = 1; i <= nkey; i++) + print order[i] "\t" row[order[i]] +} +' diff --git a/map/verifymap.py b/map/verifymap.py index 9b55d13..d35611a 100755 --- a/map/verifymap.py +++ b/map/verifymap.py @@ -45,6 +45,13 @@ def verify(path): if value != " ".join(value.split(" ")): errors.append(f"{where}: noncanonical candidate spacing") values = value.split(" ") + if path.name.endswith(".map"): + if len(value) > 64: + errors.append(f"{where}: value exceeds 64 runes") + else: + for candidate in values: + if len(candidate) > 64: + errors.append(f"{where}: candidate exceeds 64 runes") if len(values) != len(set(values)): errors.append(f"{where}: duplicate candidate") return errors diff --git a/tests/data/skk/expected.tsv b/tests/data/skk/expected.tsv index 4ac5403..9f2a6ae 100644 --- a/tests/data/skk/expected.tsv +++ b/tests/data/skk/expected.tsv @@ -1,3 +1,3 @@ かんじ 漢字 幹事 感じ えがお 笑顔 -きごう 斜線/入り 記号;付き 普通 +きごう 記号 普通 diff --git a/tests/data/skk/source.skk.utf8 b/tests/data/skk/source.skk.utf8 index 9d28bc2..08ddf47 100644 --- a/tests/data/skk/source.skk.utf8 +++ b/tests/data/skk/source.skk.utf8 @@ -2,5 +2,5 @@ かんじ /漢字;common/幹事/ えがお /笑顔;face/ かんじ /感じ/漢字;duplicate/ -きごう /斜線\/入り;escaped slash/記号\;付き;escaped semicolon/普通/ -むこう /候補 with space/(concat "式/" "候補")/[無効/候補/]/#0/ +きごう /記号;symbol/普通/ +むこう /候補 with space/(concat "式" "候補")/[無効]/#0/ diff --git a/tests/data/trie.map b/tests/data/trie.map index 1275077..17ae282 100644 --- a/tests/data/trie.map +++ b/tests/data/trie.map @@ -3,4 +3,3 @@ ab beta 한 값 duplicate first duplicate second -tabs one two diff --git a/tests/skk2ktrans_test.py b/tests/skk2ktrans_test.py index 3d2eab3..dc2463c 100755 --- a/tests/skk2ktrans_test.py +++ b/tests/skk2ktrans_test.py @@ -27,7 +27,14 @@ class Skk2KtransTest(unittest.TestCase): result = subprocess.run( [CONVERTER], input=b"\xff\xff\n", capture_output=True, check=False) self.assertNotEqual(result.returncode, 0) - self.assertIn(b"invalid EUC-JP", result.stderr) + self.assertTrue(result.stderr) + + def test_escaped_candidate_is_rejected(self): + source = b"key /escaped\\/slash/\n" + result = subprocess.run( + [CONVERTER], input=source, capture_output=True, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn(b"escaped candidates are unsupported", result.stderr) if __name__ == "__main__": diff --git a/tests/trie_test.c b/tests/trie_test.c index d6230b7..2d517cd 100644 --- a/tests/trie_test.c +++ b/tests/trie_test.c @@ -24,11 +24,6 @@ trie_exact_prefix_and_duplicate(struct ct *t) goto cleanup; CT_EQ_INT(t, 6, n); CT_EQ_MEM(t, "second", v, n); - v = trieget(trie, "tabs", 4, &n); - if(!CT_CHECK(t, v != nil)) - goto cleanup; - CT_EQ_INT(t, 7, n); - CT_EQ_MEM(t, "one\ttwo", v, n); CT_EQ_PTR(t, nil, trieget(trie, "missing", 7, &n)); cleanup: trieclose(trie); diff --git a/trie.c b/trie.c index af92b47..e3f1847 100644 --- a/trie.c +++ b/trie.c @@ -80,18 +80,23 @@ trieopen(char *path) vlen = strlen(line); if(vlen > 0 && line[vlen-1] == '\r') line[--vlen] = '\0'; - if(line[0] == '\0'){ + if(line[0] == '\0' || line[0] == ';'){ free(line); continue; } tab = strchr(line, '\t'); - if(tab == nil || tab[1] == '\0') + if(tab == nil || tab == line || tab[1] == '\0' || + strchr(tab+1, '\t') != nil) die("malformed map: %s", path); *tab = '\0'; key = line; klen = tab - line; + if(utflen(key) > Maxrunes) + die("map key too long: %s", path); val = tab + 1; vlen = strlen(val); + if(utflen(val) > Maxrunes) + die("map value too long: %s", path); insert(t, key, klen, val, vlen); free(line); }