diff --git a/Makefile b/Makefile index 1d1e80a..0c6527b 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ DOCKER_IMAGE = strans-build DOCKER_RUN = docker run --rm --user "$$(id -u):$$(id -g)" \ -v "$(CURDIR):/src" $(DOCKER_IMAGE) -SRCS = dict.c font.c hash.c ibus.c ipc.c ko.c main.c popup_layout.c \ +SRCS = dict.c font.c ibus.c ipc.c ko.c main.c popup_layout.c \ srv.c str.c strans.c trie.c vi.c win.c XIMSRCS = xim/xim.c xim/keymap.c xim/ximtext.c XIMOBJS = $(XIMSRCS:.c=.o) diff --git a/dat.h b/dat.h index 9135766..ef5aa1f 100644 --- a/dat.h +++ b/dat.h @@ -57,31 +57,19 @@ struct Emit Str dict; }; -typedef struct Hnode Hnode; -struct Hnode -{ - int filled; - int next; - char *key; - int klen; - char *val; - int vlen; -}; - typedef struct Tnode Tnode; struct Tnode { + char *val; + int vlen; int child; int sibling; char c; - char *val; - int vlen; }; typedef struct Trie Trie; struct Trie { - int root; Tnode *nodes; int n; int cap; @@ -94,16 +82,6 @@ enum TrieExact, }; -typedef struct Hmap Hmap; -struct Hmap -{ - int nbs; - int nsz; - int len; - int cap; - uchar *nodes; -}; - typedef struct Lang Lang; typedef struct Im Im; struct Lang @@ -115,7 +93,7 @@ struct Lang void (*back)(Im*); void (*dictq)(Im*); Trie *map; - Hmap *dict; + Trie *dict; }; struct Im diff --git a/dict.c b/dict.c index ddb520d..e509439 100644 --- a/dict.c +++ b/dict.c @@ -1,59 +1,34 @@ -#include #include "dat.h" #include "fn.h" -static char* -readline(Biobuf *b, char *path) -{ - char *line; - - errno = 0; - line = Brdstr(b, '\n', 1); - if(errno != 0) - die("can't read %s: %s", path, strerror(errno)); - return line; -} - +/* Candidates are the space-separated words of the value; the reading itself + * is a candidate only for the emoji dictionary. */ void dictlookup(Dictreq *req, Dictres *res) { Lang *l; - Hmap *dict; - Hnode *n; char *p, *e, *sp; Str tmp; + int vlen; res->key = req->pre; res->nkouho = 0; res->lang = req->lang; res->seq = req->seq; - if(req->key.n == 0) - return; l = getlang(req->lang); - dict = l ? l->dict : nil; - if(dict == nil) + if(req->key.n == 0 || l == nil || + trielookup(l->dict, &req->key, &p, &vlen) != TrieExact) return; - n = hmapget(dict, &req->key); - if(n == nil || n->vlen == 0) - return; - p = n->val; - e = p + n->vlen; + e = p + vlen; while(res->nkouho < Maxkouho && p < e){ while(p < e && *p == ' ') p++; - if(p == e) - break; sp = p; while(p < e && *p != ' ') p++; - if(!sinit(&tmp, sp, p - sp)){ - res->nkouho = 0; - return; - } - if(req->lang == LangEMOJI || scmp(&tmp, &req->key) != 0) + if(sinit(&tmp, sp, p - sp) && tmp.n > 0 && + (req->lang == LangEMOJI || scmp(&tmp, &req->key) != 0)) res->kouho[res->nkouho++] = tmp; - if(p < e) - p++; } } @@ -74,70 +49,29 @@ dictthread(void*) } } -static Hmap* -dictopen(char *path) +static Trie* +langopen(char *dir, char *name, char *ext) { - Hmap *h; - Biobuf *b; - Str key, tmp; - char *line, *tab, *p, *e; - int len, lineno; + char *path; + Trie *t; - b = Bopen(path, OREAD); - if(b == nil) - die("can't open %s: %s", path, strerror(errno)); - h = hmapalloc(4096); - lineno = 0; - while((line = readline(b, path)) != nil){ - lineno++; - len = Blinelen(b); - if(memchr(line, '\0', len) != nil) - die("NUL in dictionary: %s:%d", path, lineno); - if(len > 0 && line[len-1] == '\r') - line[--len] = '\0'; - if(len == 0 || line[0] == ';'){ - free(line); - continue; - } - tab = memchr(line, '\t', len); - if(tab == nil || tab == line || tab >= line + len - 1 || - memchr(tab+1, '\t', line+len-(tab+1)) != nil) - die("malformed dictionary: %s:%d", path, lineno); - *tab = '\0'; - if(!sinit(&key, line, tab-line)) - die("invalid or oversized dictionary key: %s:%d", - path, lineno); - for(p = tab+1; p < line+len; p = e+1){ - e = memchr(p, ' ', line+len-p); - if(e == nil) - e = line+len; - if(!sinit(&tmp, p, e-p)) - die("invalid or oversized dictionary candidate: %s:%d", - path, lineno); - if(e == line+len) - break; - } - hmapset(&h, &key, tab+1, len - (tab - line) - 1); - free(line); - } - if(Bterm(b) < 0) - die("can't close %s: %s", path, strerror(errno)); - return h; + path = smprint("%s/%s.%s", dir, name, ext); + if(path == nil) + die("out of memory"); + t = trieopen(path); + free(path); + return t; } void -dictinit(char *dir) +langinit(char *dir) { - char *path; - int i; + Lang *l; - for(i = 0; i < nlang; i++){ - if(langs[i].dictname == nil) - continue; - path = smprint("%s/%s.dict", dir, langs[i].dictname); - if(path == nil) - die("out of memory"); - langs[i].dict = dictopen(path); - free(path); + for(l = langs; l < langs + nlang; l++){ + if(l->mapname != nil) + l->map = langopen(dir, l->mapname, "map"); + if(l->dictname != nil) + l->dict = langopen(dir, l->dictname, "dict"); } } diff --git a/fn.h b/fn.h index 62d88c0..99fc549 100644 --- a/fn.h +++ b/fn.h @@ -9,19 +9,16 @@ int scmp(Str*, Str*); int stoutf(Str*, char*, int); Rune slastr(Str*); -Hmap* hmapalloc(int); -void hmapfree(Hmap*); -void hmapset(Hmap**, Str*, const char*, int); -Hnode* hmapget(Hmap*, Str*); int mapget(Trie*, Str*, Str*); +Trie* trienew(void); +void trieput(Trie*, char*, int, char*, int); Trie* trieopen(char*); void trieclose(Trie*); -int trielookup(Trie*, char*, int, char**, int*); +int trielookup(Trie*, Str*, char**, int*); Lang* getlang(int); -void mapinit(char*); -void dictinit(char*); +void langinit(char*); void dictthread(void*); void dictlookup(Dictreq*, Dictres*); diff --git a/hash.c b/hash.c deleted file mode 100644 index a02983c..0000000 --- a/hash.c +++ /dev/null @@ -1,175 +0,0 @@ -#include -#include "dat.h" -#include "fn.h" - -enum { - Tagsize = sizeof(Hnode), -}; - -static uvlong -hash(Str *s) -{ - uvlong h; - int i; - - h = 7; - for(i = 0; i < s->n; i++) - h = h*31 + s->r[i]; - return h; -} - -Hmap* -hmapalloc(int nbuckets) -{ - void *store; - Hmap *h; - int nsz; - - if(nbuckets < 1) - return nil; - nsz = Tagsize; - if((ulong)nbuckets > (ULONG_MAX-sizeof(*h))/(ulong)nsz) - return nil; - store = emalloc(sizeof(*h) + (ulong)nbuckets * nsz); - h = store; - h->nbs = nbuckets; - h->nsz = nsz; - h->len = h->cap = nbuckets; - h->nodes = (uchar*)store + sizeof(*h); - return h; -} - -static int -keycmp(Hnode *n, Str *key) -{ - char buf[Maxutf]; - int len; - - len = stoutf(key, buf, sizeof(buf)); - if(n->klen != len) - return 1; - return memcmp(n->key, buf, len); -} - -Hnode* -hmapget(Hmap *h, Str *key) -{ - Hnode *n; - uchar *v; - - if(h == nil || key == nil || key->n < 0 || key->n > Maxrunes) - return nil; - v = h->nodes + (hash(key) % h->nbs) * (ulong)h->nsz; - for(;;){ - n = (Hnode*)v; - if(n->filled && keycmp(n, key) == 0) - return n; - if(n->next == 0) - break; - v = h->nodes + (ulong)n->next * h->nsz; - } - return nil; -} - -static char* -sdup(Str *s, int *len) -{ - char buf[Maxutf]; - char *p; - int n; - - n = stoutf(s, buf, sizeof(buf)); - p = emalloc((ulong)n + 1); - memmove(p, buf, n); - p[n] = '\0'; - *len = n; - return p; -} - -static char* -memdup(const char *src, int n) -{ - char *p; - - if(n == 0) - return nil; - p = emalloc((ulong)n + 1); - memmove(p, src, n); - p[n] = '\0'; - return p; -} - -void -hmapfree(Hmap *h) -{ - Hnode *n; - int i; - - if(h == nil) - return; - for(i = 0; i < h->len; i++){ - n = (Hnode*)(h->nodes + (ulong)i * h->nsz); - if(!n->filled) - continue; - free(n->key); - free(n->val); - } - free(h); -} - -void -hmapset(Hmap **store, Str *key, const char *val, int vlen) -{ - char *newval; - Hnode *n; - uchar *v; - Hmap *h; - int next; - ulong diff; - - if(store == nil || *store == nil || key == nil || - key->n < 0 || key->n > Maxrunes || vlen < 0 || - (vlen > 0 && val == nil)) - return; - newval = memdup(val, vlen); - h = *store; - v = h->nodes + (hash(key) % h->nbs) * (ulong)h->nsz; - for(;;){ - n = (Hnode*)v; - next = n->next; - if(n->filled == 0) - goto replace; - if(keycmp(n, key) == 0) - goto replace; - if(next == 0) - break; - v = h->nodes + (ulong)next * h->nsz; - } - if(h->cap == h->len){ - diff = v - h->nodes; - if(h->cap > INT_MAX/2 || - (ulong)h->cap > (ULONG_MAX-sizeof(*h))/(2*(ulong)h->nsz)) - die("hash table is too large"); - h->cap *= 2; - *store = erealloc(*store, - sizeof(*h) + (ulong)h->cap * h->nsz); - h = *store; - h->nodes = (uchar*)*store + sizeof(*h); - v = h->nodes + diff; - n = (Hnode*)v; - } - n->next = h->len; - memset(h->nodes + (ulong)h->len * h->nsz, 0, h->nsz); - h->len++; - v = h->nodes + (ulong)n->next * h->nsz; - n = (Hnode*)v; -replace: - if(n->filled == 0){ - n->key = sdup(key, &n->klen); - n->filled = 1; - } - n->next = next; - free(n->val); - n->val = newval; - n->vlen = vlen; -} diff --git a/main.c b/main.c index 169ac4e..d35eacf 100644 --- a/main.c +++ b/main.c @@ -58,8 +58,7 @@ threadmain(int argc, char **argv) keyc = chancreate(sizeof(Keyreq), 0); dictreqc = chancreate(sizeof(Dictreq), 4); dictresc = chancreate(sizeof(Dictres), 0); - mapinit(argv[1]); - dictinit(argv[1]); + langinit(argv[1]); srvinit(); proccreate(drawthread, nil, 16384); proccreate(srvthread, nil, 16384); diff --git a/strans.c b/strans.c index 872c3e9..40ac3d0 100644 --- a/strans.c +++ b/strans.c @@ -405,18 +405,16 @@ getlang(int lang) static int maplookup(Trie *t, Str *key, Str *out) { - char buf[Maxutf], *v; - int klen, match, vlen; + char *v; + int match, vlen; - if(out != nil) - sclear(out); - if(t == nil || key == nil || key->n == 0) + sclear(out); + if(key->n == 0) return 0; - klen = stoutf(key, buf, sizeof(buf)); - match = trielookup(t, buf, klen, &v, &vlen); + match = trielookup(t, key, &v, &vlen); if(match == TrieMiss) return 0; - if(match == TrieExact && out != nil && !sinit(out, v, vlen)) + if(match == TrieExact && !sinit(out, v, vlen)) return 0; return 1; } @@ -1052,33 +1050,11 @@ imthread(void*) int mapget(Trie *t, Str *key, Str *out) { - char buf[Maxutf], *v; - int klen, match, vlen; + char *v; + int vlen; - if(out != nil) - sclear(out); - if(key == nil || out == nil || key->n == 0) - return 0; - klen = stoutf(key, buf, sizeof(buf)); - match = trielookup(t, buf, klen, &v, &vlen); - if(match != TrieExact) + sclear(out); + if(key->n == 0 || trielookup(t, key, &v, &vlen) != TrieExact) return 0; return sinit(out, v, vlen); } - -void -mapinit(char *dir) -{ - char *path; - int i; - - for(i = 0; i < nelem(langs); i++){ - if(langs[i].mapname == nil) - continue; - path = smprint("%s/%s.map", dir, langs[i].mapname); - if(path == nil) - die("out of memory"); - langs[i].map = trieopen(path); - free(path); - } -} diff --git a/tests/Makefile b/tests/Makefile index 98ae1b2..e7544c7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -28,11 +28,11 @@ SMOKE = ibus_live_test ibus_client_smoke gtk_live_test xim_live_test \ ipc_live_test FAULT = daemon_collision_test daemon_failure_test daemon_restart_test LIVE = $(SMOKE) $(FAULT) -TESTSRC = test_util.c str_test.c hash_test.c trie_test.c \ +TESTSRC = test_util.c str_test.c trie_test.c \ ko_test.c vi_test.c engine_test.c dict_test.c ipc_test.c \ popup_test.c font_test.c ibus_test.c server_test.c xim_adapter_test.c TESTOBJ = $(TESTSRC:.c=.o) -PARENTSRC = str.c hash.c trie.c dict.c ko.c vi.c ipc.c popup_layout.c \ +PARENTSRC = str.c trie.c dict.c ko.c vi.c ipc.c popup_layout.c \ font.c PARENTOBJ = $(PARENTSRC:%.c=unit_%.o) XIMOBJ = unit_xim_keymap.o unit_ximtext.o diff --git a/tests/dict_test.c b/tests/dict_test.c index f413107..e54b77e 100644 --- a/tests/dict_test.c +++ b/tests/dict_test.c @@ -3,21 +3,20 @@ void dictionary_candidates(struct ct *t) { - static char malformed[] = { 'o', 'k', ' ', (char)0x80 }; char many[512], item[8]; char *p; Dictreq req; Dictres res; - Hmap *saved; + Trie *saved; Lang *lang; Str key; int i; lang = getlang(LangJP); saved = lang->dict; - lang->dict = hmapalloc(1); + lang->dict = trienew(); key = mkstr("かな"); - hmapset(&lang->dict, &key, " 候補1 かな 候補2 ", + trieput(lang->dict, "かな", strlen("かな"), " 候補1 かな 候補2 ", strlen(" 候補1 かな 候補2 ")); memset(&req, 0, sizeof req); req.key = key; @@ -42,7 +41,7 @@ dictionary_candidates(struct ct *t) p += strlen(item); } *p = '\0'; - hmapset(&lang->dict, &key, many, strlen(many)); + trieput(lang->dict, "key", 3, many, strlen(many)); req.key = key; req.pre = mkstr("preedit-two"); req.seq = 29; @@ -54,13 +53,8 @@ dictionary_candidates(struct ct *t) CT_EQ_INT(t, 0, scmp(&req.pre, &res.key)); checkstr(t, "first capped candidate", "c00", &res.kouho[0]); checkstr(t, "last capped candidate", "c31", &res.kouho[31]); - key = mkstr("malformed"); - hmapset(&lang->dict, &key, malformed, sizeof malformed); - req.key = key; - dictlookup(&req, &res); - CT_EQ_INT(t, 0, res.nkouho); cleanup: - hmapfree(lang->dict); + trieclose(lang->dict); lang->dict = saved; } @@ -77,13 +71,13 @@ dictionary_misses_clear_result(struct ct *t) }; Dictreq req; Dictres res; - Hmap *saved; + Trie *saved; Lang *lang; int i; lang = getlang(LangJP); saved = lang->dict; - lang->dict = hmapalloc(1); + lang->dict = trienew(); for(i = 0; i < nelem(cases); i++){ memset(&res, 0xa5, sizeof res); req.key = mkstr(cases[i].key); @@ -98,7 +92,7 @@ dictionary_misses_clear_result(struct ct *t) CT_EQ_UINT(t, 0xf00d0000U + i, res.seq); checkstr(t, cases[i].name, cases[i].pre, &res.key); } - hmapfree(lang->dict); + trieclose(lang->dict); lang->dict = saved; } @@ -107,15 +101,15 @@ dictionary_emoji_identity(struct ct *t) { Dictreq req; Dictres res; - Hmap *saved; + Trie *saved; Lang *lang; Str key; lang = getlang(LangEMOJI); saved = lang->dict; - lang->dict = hmapalloc(2); + lang->dict = trienew(); key = mkstr("é"); - hmapset(&lang->dict, &key, "é", strlen("é")); + trieput(lang->dict, "é", strlen("é"), "é", strlen("é")); memset(&req, 0, sizeof req); req.key = key; req.pre = key; @@ -124,6 +118,6 @@ dictionary_emoji_identity(struct ct *t) dictlookup(&req, &res); if(CT_EQ_INT(t, 1, res.nkouho)) checkstr(t, "emoji identity candidate", "é", &res.kouho[0]); - hmapfree(lang->dict); + trieclose(lang->dict); lang->dict = saved; } diff --git a/tests/engine_test.c b/tests/engine_test.c index df3b501..be615ea 100644 --- a/tests/engine_test.c +++ b/tests/engine_test.c @@ -311,14 +311,14 @@ engine_rejects_stale_dictionary_results(struct ct *t) { Dictreq a, b; Dictres res; - Hmap *saved; + Trie *saved; Lang *jp; char one, two; int n; jp = getlang(LangJP); saved = jp->dict; - jp->dict = hmapalloc(1); + jp->dict = trienew(); init(); im.l = jp; while(channbrecv(dictreqc, &a) > 0) @@ -362,7 +362,7 @@ engine_rejects_stale_dictionary_results(struct ct *t) cleanup: while(channbrecv(dictreqc, &a) > 0) ; - hmapfree(jp->dict); + trieclose(jp->dict); jp->dict = saved; } @@ -720,7 +720,7 @@ struct Searchfix Im im; Search search; Lang *dictlang; - Hmap *dict; + Trie *dict; int activecap; int visible; int candidatechosen; @@ -939,12 +939,9 @@ engine_candidate_completion(struct ct *t) } static void -setdict(Hmap **dict, char *key, char *val) +setdict(Trie *dict, char *key, char *val) { - Str s; - - s = mkstr(key); - hmapset(dict, &s, val, strlen(val)); + trieput(dict, key, strlen(key), val, strlen(val)); } static void @@ -964,22 +961,22 @@ static void emojibegin(Searchfix *f, int showpre) { searchsave(f, LangEMOJI); - f->dictlang->dict = hmapalloc(32); - setdict(&f->dictlang->dict, "a", "A B"); - setdict(&f->dictlang->dict, "ㅁ", "B C"); - setdict(&f->dictlang->dict, "alpha", "α"); - setdict(&f->dictlang->dict, "smil", "😀 😄"); - setdict(&f->dictlang->dict, "smile", "😀 😄"); - setdict(&f->dictlang->dict, "웃음", "😀 😄 😂"); - setdict(&f->dictlang->dict, "えがお", "😀 😄 😊"); - setdict(&f->dictlang->dict, "エガオ", "😀 😄 😊"); - setdict(&f->dictlang->dict, "heart", "❤️"); - setdict(&f->dictlang->dict, "coffee", "☕️"); - setdict(&f->dictlang->dict, "^", "¹ ² ³ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁽"); - setdict(&f->dictlang->dict, "^0", "⁰"); - setdict(&f->dictlang->dict, "_", "₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₍"); - setdict(&f->dictlang->dict, "<", "← ≤ ♥ ≠"); - setdict(&f->dictlang->dict, "many", + f->dictlang->dict = trienew(); + setdict(f->dictlang->dict, "a", "A B"); + setdict(f->dictlang->dict, "ㅁ", "B C"); + setdict(f->dictlang->dict, "alpha", "α"); + setdict(f->dictlang->dict, "smil", "😀 😄"); + setdict(f->dictlang->dict, "smile", "😀 😄"); + setdict(f->dictlang->dict, "웃음", "😀 😄 😂"); + setdict(f->dictlang->dict, "えがお", "😀 😄 😊"); + setdict(f->dictlang->dict, "エガオ", "😀 😄 😊"); + setdict(f->dictlang->dict, "heart", "❤️"); + setdict(f->dictlang->dict, "coffee", "☕️"); + setdict(f->dictlang->dict, "^", "¹ ² ³ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁽"); + setdict(f->dictlang->dict, "^0", "⁰"); + setdict(f->dictlang->dict, "_", "₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₍"); + setdict(f->dictlang->dict, "<", "← ≤ ♥ ≠"); + setdict(f->dictlang->dict, "many", "c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12"); init(); im.l = getlang(LangKO); @@ -989,11 +986,11 @@ emojibegin(Searchfix *f, int showpre) static void searchend(Searchfix *f) { - Hmap *dict; + Trie *dict; dict = f->dictlang->dict; f->dictlang->dict = f->dict; - hmapfree(dict); + trieclose(dict); draindraw(nil); im = f->im; search = f->search; @@ -1079,12 +1076,12 @@ static void hanjabegin(Searchfix *f, int lang) { searchsave(f, LangHANJA); - f->dictlang->dict = hmapalloc(8); - setdict(&f->dictlang->dict, "한", "漢 韓"); - setdict(&f->dictlang->dict, "가", "家"); - setdict(&f->dictlang->dict, "ㄱ", "假"); - setdict(&f->dictlang->dict, "한글", "文"); - setdict(&f->dictlang->dict, "か", "仮"); + f->dictlang->dict = trienew(); + setdict(f->dictlang->dict, "한", "漢 韓"); + setdict(f->dictlang->dict, "가", "家"); + setdict(f->dictlang->dict, "ㄱ", "假"); + setdict(f->dictlang->dict, "한글", "文"); + setdict(f->dictlang->dict, "か", "仮"); init(); im.l = getlang(lang); } @@ -1156,15 +1153,14 @@ engine_japanese_candidates(struct ct *t) { Dictreq req; Dictres res; - Hmap *saved; + Trie *saved; Lang *jp; - Str com, key, shown; + Str com, shown; jp = getlang(LangJP); saved = jp->dict; - jp->dict = hmapalloc(8); - key = mkstr("かんじ"); - hmapset(&jp->dict, &key, "漢字 幹事", strlen("漢字 幹事")); + jp->dict = trienew(); + setdict(jp->dict, "かんじ", "漢字 幹事"); init(); im.l = jp; sclear(&com); @@ -1189,7 +1185,7 @@ engine_japanese_candidates(struct ct *t) checkstr(t, "selected Kanji", "幹事", &com); cleanup: newestrequest(&req); - hmapfree(jp->dict); + trieclose(jp->dict); jp->dict = saved; } diff --git a/tests/hash_test.c b/tests/hash_test.c deleted file mode 100644 index c20478a..0000000 --- a/tests/hash_test.c +++ /dev/null @@ -1,112 +0,0 @@ -#include "test.h" - -void -hmap_set_replace_and_grow(struct ct *t) -{ - char keybuf[16], valbuf[16], source[] = "copied"; - Hmap *h; - Hnode *n; - Str key; - int i; - - h = hmapalloc(1); - for(i = 0; i < 4; i++){ - snprint(keybuf, sizeof keybuf, "key%d", i); - snprint(valbuf, sizeof valbuf, "value%d", i); - key = mkstr(keybuf); - hmapset(&h, &key, valbuf, strlen(valbuf)); - } - for(i = 0; i < 4; i++){ - snprint(keybuf, sizeof keybuf, "key%d", i); - snprint(valbuf, sizeof valbuf, "value%d", i); - key = mkstr(keybuf); - n = hmapget(h, &key); - if(n == nil || strcmp(n->val, valbuf) != 0) - CT_ERRORF(t, "%s: collision chain lost value", keybuf); - } - key = mkstr("key1"); - hmapset(&h, &key, source, strlen(source)); - source[0] = 'X'; - n = hmapget(h, &key); - if(!CT_CHECK(t, n != nil)) - goto cleanup; - CT_EQ_STR(t, "copied", n->val); - key = mkstr("key2"); - n = hmapget(h, &key); - if(!CT_CHECK(t, n != nil)) - goto cleanup; - CT_EQ_STR(t, "value2", n->val); - key = mkstr("key1"); - hmapset(&h, &key, nil, 0); - n = hmapget(h, &key); - if(!CT_CHECK(t, n != nil)) - goto cleanup; - CT_EQ_INT(t, 0, n->vlen); - CT_EQ_PTR(t, nil, n->val); - key = mkstr("missing"); - CT_EQ_PTR(t, nil, hmapget(h, &key)); -cleanup: - hmapfree(h); -} - -void -hmap_long_utf8_keys(struct ct *t) -{ - Hmap *h; - Hnode *n; - Str a, b; - int i; - - for(i = 0; i < Maxrunes-1; i++) - a.r[i] = b.r[i] = 0x1f600; - a.r[Maxrunes-1] = 0x1f601; - b.r[Maxrunes-1] = 0x1f602; - a.n = b.n = Maxrunes; - h = hmapalloc(1); - hmapset(&h, &a, "first", 5); - hmapset(&h, &b, "second", 6); - n = hmapget(h, &a); - if(!CT_CHECK(t, n != nil)) - goto cleanup; - CT_EQ_STR(t, "first", n != nil ? n->val : nil); - n = hmapget(h, &b); - if(!CT_CHECK(t, n != nil)) - goto cleanup; - CT_EQ_STR(t, "second", n != nil ? n->val : nil); -cleanup: - hmapfree(h); -} - -void -hmap_binary_keys_and_invalid_lengths(struct ct *t) -{ - Hmap *h; - Hnode *n; - Str key, other; - - h = hmapalloc(1); - CT_CHECK(t, h != nil); - key.n = 3; - key.r[0] = 'a'; - key.r[1] = 0; - key.r[2] = 'b'; - other = key; - other.r[2] = 'c'; - hmapset(&h, &key, "one", 3); - hmapset(&h, &other, "two", 3); - n = hmapget(h, &key); - if(CT_CHECK(t, n != nil)){ - CT_EQ_INT(t, 3, n->klen); - CT_EQ_MEM(t, "one", n->val, n->vlen); - } - n = hmapget(h, &other); - if(CT_CHECK(t, n != nil)) - CT_EQ_MEM(t, "two", n->val, n->vlen); - hmapset(&h, &key, "bad", -1); - hmapset(&h, &key, nil, 1); - n = hmapget(h, &key); - if(CT_CHECK(t, n != nil)) - CT_EQ_MEM(t, "one", n->val, n->vlen); - CT_EQ_PTR(t, nil, hmapalloc(0)); - hmapfree(h); -} diff --git a/tests/test.h b/tests/test.h index c947085..c4e54ad 100644 --- a/tests/test.h +++ b/tests/test.h @@ -12,11 +12,8 @@ void str_init_utf8(struct ct*); void str_edit_and_alias(struct ct*); void str_utf8_capacity(struct ct*); void str_invalid_and_full_appends(struct ct*); -void hmap_set_replace_and_grow(struct ct*); -void hmap_long_utf8_keys(struct ct*); -void hmap_binary_keys_and_invalid_lengths(struct ct*); void trie_exact_prefix_and_duplicate(struct ct*); -void trie_optional_outputs_and_invalid_lengths(struct ct*); +void trie_put_and_unloaded(struct ct*); void popup_layout(struct ct*); void font_render(struct ct*); void production_maps_load(struct ct*); diff --git a/tests/trie_test.c b/tests/trie_test.c index 18bb6d7..44de96f 100644 --- a/tests/trie_test.c +++ b/tests/trie_test.c @@ -5,26 +5,26 @@ trie_exact_prefix_and_duplicate(struct ct *t) { static const struct { char *key; - int klen; int match; char *want; } cases[] = { - { "a", 1, TrieExact, "alpha" }, - { "dupli", 5, TriePrefix, nil }, - { "duplicate", 9, TrieExact, "second" }, - { "missing", 7, TrieMiss, nil }, - { nil, 1, TrieMiss, nil }, - { "a", -1, TrieMiss, nil }, + { "a", TrieExact, "alpha" }, + { "ab", TrieExact, "beta" }, + { "한", TrieExact, "값" }, + { "dupli", TriePrefix, nil }, + { "duplicate", TrieExact, "second" }, + { "missing", TrieMiss, nil }, + { "", TriePrefix, nil }, }; char *v; Trie *trie; + Str key; int i, match, n; trie = trieopen("data/trie.map"); for(i = 0; i < nelem(cases); i++){ - v = "unchanged"; - n = 77; - match = trielookup(trie, cases[i].key, cases[i].klen, &v, &n); + key = mkstr(cases[i].key); + match = trielookup(trie, &key, &v, &n); if(match != cases[i].match){ CT_ERRORF(t, "case %d: want match %d, got %d", i, cases[i].match, match); @@ -42,22 +42,28 @@ trie_exact_prefix_and_duplicate(struct ct *t) } void -trie_optional_outputs_and_invalid_lengths(struct ct *t) +trie_put_and_unloaded(struct ct *t) { char *v; Trie *trie; + Str key; int n; - trie = trieopen("data/trie.map"); - CT_EQ_INT(t, TrieExact, trielookup(trie, "a", 1, &v, nil)); - CT_CHECK(t, v != nil); - CT_EQ_INT(t, TrieExact, trielookup(trie, "a", 1, nil, &n)); - CT_EQ_INT(t, 5, n); - CT_EQ_INT(t, TriePrefix, - trielookup(trie, "dupli", 5, nil, nil)); - CT_EQ_INT(t, TriePrefix, trielookup(trie, nil, 0, &v, &n)); - CT_EQ_PTR(t, nil, v); - CT_EQ_INT(t, 0, n); + key = mkstr("k"); + CT_EQ_INT(t, TrieMiss, trielookup(nil, &key, &v, &n)); + trie = trienew(); + CT_EQ_INT(t, TrieMiss, trielookup(trie, &key, &v, &n)); + trieput(trie, "k", 1, "one two", 7); + trieput(trie, "ka", 2, "", 0); + if(CT_EQ_INT(t, TrieExact, trielookup(trie, &key, &v, &n))) + CT_EQ_MEM(t, "one two", v, 7); + key = mkstr("ka"); + if(CT_EQ_INT(t, TrieExact, trielookup(trie, &key, &v, &n))) + CT_EQ_INT(t, 0, n); + trieput(trie, "k", 1, "three", 5); + key = mkstr("k"); + if(CT_EQ_INT(t, TrieExact, trielookup(trie, &key, &v, &n))) + CT_EQ_MEM(t, "three", v, 5); trieclose(trie); } diff --git a/tests/unit_test.c b/tests/unit_test.c index cea5747..501e41c 100644 --- a/tests/unit_test.c +++ b/tests/unit_test.c @@ -67,11 +67,8 @@ static const struct ct_test tests[] = { { "str/edit-and-alias", str_edit_and_alias }, { "str/utf8-capacity", str_utf8_capacity }, { "str/invalid-full-appends", str_invalid_and_full_appends }, - { "hmap/set-replace-grow", hmap_set_replace_and_grow }, - { "hmap/long-utf8-keys", hmap_long_utf8_keys }, - { "hmap/binary-invalid-lengths", hmap_binary_keys_and_invalid_lengths }, { "trie/exact-prefix-duplicate", trie_exact_prefix_and_duplicate }, - { "trie/optional-invalid-lengths", trie_optional_outputs_and_invalid_lengths }, + { "trie/put-and-unloaded", trie_put_and_unloaded }, { "popup/layout", popup_layout }, { "font/render", font_render }, { "map/production-lifecycle", production_maps_load }, diff --git a/trie.c b/trie.c index 4fcb017..35d5f88 100644 --- a/trie.c +++ b/trie.c @@ -1,33 +1,21 @@ #include -#include #include "dat.h" #include "fn.h" -static char* -readline(Biobuf *b, char *path) -{ - char *line; - - errno = 0; - line = Brdstr(b, '\n', 1); - if(errno != 0) - die("can't read %s: %s", path, strerror(errno)); - return line; -} +/* + * A byte trie over UTF-8 keys. Maps need prefix matches while composing; + * dictionaries need exact matches; both are small enough for one structure. + * Node 0 is the root. + */ static int newnode(Trie *t) { - int cap; int i; - if(t->n >= t->cap){ - if(t->cap > INT_MAX/2 || - (ulong)t->cap > ULONG_MAX/(2*sizeof(Tnode))) - die("map is too large"); - cap = t->cap * 2; - t->nodes = erealloc(t->nodes, (ulong)cap * sizeof(Tnode)); - t->cap = cap; + if(t->n == t->cap){ + t->cap *= 2; + t->nodes = erealloc(t->nodes, t->cap * sizeof(Tnode)); } i = t->n++; memset(&t->nodes[i], 0, sizeof(Tnode)); @@ -59,13 +47,25 @@ add(Trie *t, int ni, char c) return pi; } -static void -insert(Trie *t, char *key, int klen, char *val, int vlen) +Trie* +trienew(void) { - int ni, ci; - int i; + Trie *t; - ni = t->root; + t = emalloc(sizeof(*t)); + t->cap = 1024; + t->nodes = emalloc(t->cap * sizeof(Tnode)); + t->n = 0; + newnode(t); + return t; +} + +void +trieput(Trie *t, char *key, int klen, char *val, int vlen) +{ + int ni, ci, i; + + ni = 0; for(i = 0; i < klen; i++){ ci = find(t, ni, key[i]); if(ci < 0) @@ -73,57 +73,70 @@ insert(Trie *t, char *key, int klen, char *val, int vlen) ni = ci; } free(t->nodes[ni].val); - t->nodes[ni].val = emalloc((ulong)vlen + 1); + t->nodes[ni].val = emalloc(vlen + 1); memmove(t->nodes[ni].val, val, vlen); t->nodes[ni].val[vlen] = '\0'; t->nodes[ni].vlen = vlen; } +static char* +readline(Biobuf *b, char *path) +{ + char *line; + + errno = 0; + line = Brdstr(b, '\n', 1); + if(errno != 0) + die("can't read %s: %r", path); + return line; +} + +/* + * A file holds "keyvalue" lines; ';' starts a comment. Keys and each + * space-separated word of a value must fit a Str, which is what lookups + * hand back. + */ Trie* trieopen(char *path) { Trie *t; Biobuf *b; - char *line, *tab, *key, *val; Str s; - int klen, vlen; + char *e, *line, *p, *tab; + int len, lineno; b = Bopen(path, OREAD); if(b == nil) - die("can't open %s: %s", path, strerror(errno)); - t = emalloc(sizeof(*t)); - t->cap = 1024; - t->nodes = emalloc((ulong)t->cap * sizeof(Tnode)); - t->n = 0; - t->root = newnode(t); - while((line = readline(b, path)) != nil){ - vlen = Blinelen(b); - if(memchr(line, '\0', vlen) != nil) - die("NUL in map: %s", path); - if(vlen > 0 && line[vlen-1] == '\r') - line[--vlen] = '\0'; - if(line[0] == '\0' || line[0] == ';'){ + die("can't open %s: %r", path); + t = trienew(); + for(lineno = 1; (line = readline(b, path)) != nil; lineno++){ + len = Blinelen(b); + if(memchr(line, '\0', len) != nil) + die("%s:%d: NUL byte", path, lineno); + if(len > 0 && line[len-1] == '\r') + line[--len] = '\0'; + if(len == 0 || line[0] == ';'){ free(line); continue; } - tab = memchr(line, '\t', vlen); - if(tab == nil || tab == line || tab == line+vlen-1 || - memchr(tab+1, '\t', line+vlen-(tab+1)) != nil) - die("malformed map: %s", path); - *tab = '\0'; - key = line; - klen = tab - line; - if(!sinit(&s, key, klen)) - die("invalid or oversized map key: %s", path); - val = tab + 1; - vlen = line + vlen - val; - if(!sinit(&s, val, vlen)) - die("invalid or oversized map value: %s", path); - insert(t, key, klen, val, vlen); + tab = memchr(line, '\t', len); + if(tab == nil || tab == line || tab == line+len-1 || + memchr(tab+1, '\t', line+len-(tab+1)) != nil) + die("%s:%d: expected key, tab, value", path, lineno); + if(!sinit(&s, line, tab-line)) + die("%s:%d: invalid or oversized key", path, lineno); + for(p = tab+1; p < line+len; p = e+1){ + e = memchr(p, ' ', line+len-p); + if(e == nil) + e = line+len; + if(!sinit(&s, p, e-p)) + die("%s:%d: invalid or oversized value", path, lineno); + } + trieput(t, line, tab-line, tab+1, line+len-(tab+1)); free(line); } if(Bterm(b) < 0) - die("can't close %s: %s", path, strerror(errno)); + die("can't close %s: %r", path); return t; } @@ -140,30 +153,27 @@ trieclose(Trie *t) free(t); } +/* A nil trie is an unloaded map: every key misses. */ int -trielookup(Trie *t, char *key, int klen, char **val, int *vlen) +trielookup(Trie *t, Str *key, char **val, int *vlen) { - int ni; - int i; + char buf[Maxutf]; + int i, klen, ni; - if(val != nil) - *val = nil; - if(vlen != nil) - *vlen = 0; - if(t == nil || klen < 0 || (klen > 0 && key == nil)) + *val = nil; + *vlen = 0; + if(t == nil) return TrieMiss; - ni = t->root; + klen = stoutf(key, buf, sizeof buf); + ni = 0; for(i = 0; i < klen; i++){ - ni = find(t, ni, key[i]); + ni = find(t, ni, buf[i]); if(ni < 0) return TrieMiss; } - if(t->nodes[ni].val != nil){ - if(val != nil) - *val = t->nodes[ni].val; - if(vlen != nil) - *vlen = t->nodes[ni].vlen; - return TrieExact; - } - return TriePrefix; + if(t->nodes[ni].val == nil) + return TriePrefix; + *val = t->nodes[ni].val; + *vlen = t->nodes[ni].vlen; + return TrieExact; }