Hojun-Cho/strans
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

data: one trie for maps and dictionaries, one loader

Browse Source 
The hash map served a single exact-match lookup that the trie already
answers, at the price of a second container, a second file loader with
its own drift, and a Str-to-UTF-8 conversion on every chain probe. The
files are small (kanji.dict is 7.5k lines), so the trie holds both.
trieopen validates keys and each space-separated value word against Str
and reports path:line; trielookup takes the Str every caller holds and
treats a nil trie as an unloaded map; trienew/trieput exist for tests.
The overflow guards on growth, Trie.root (always 0) and the per-language
init loop written twice are gone.
This commit is contained in:
Hojun-Cho
2026-08-16 15:56:32 +09:00
parent eb0f88f764
commit c7718ece52
15 changed files with 206 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@@ -19,7 +19,7 @@ DOCKER_IMAGE = strans-build
DOCKER_RUN = docker run --rm --user "$$(id -u):$$(id -g)" \ DOCKER_RUN = docker run --rm --user "$$(id -u):$$(id -g)" \
-v "$(CURDIR):/src" $(DOCKER_IMAGE) -v "$(CURDIR):/src" $(DOCKER_IMAGE)
SRCS = dict.c font.c hash.c ibus.c ipc.c ko.c main.c popup_layout.c \ SRCS = dict.c font.c ibus.c ipc.c ko.c main.c popup_layout.c \
srv.c str.c strans.c trie.c vi.c win.c srv.c str.c strans.c trie.c vi.c win.c
XIMSRCS = xim/xim.c xim/keymap.c xim/ximtext.c XIMSRCS = xim/xim.c xim/keymap.c xim/ximtext.c
XIMOBJS = $(XIMSRCS:.c=.o) XIMOBJS = $(XIMSRCS:.c=.o)

28
dat.h
View File

@@ -57,31 +57,19 @@ struct Emit
Str dict; Str dict;
}; };
typedef struct Hnode Hnode;
struct Hnode
{
int filled;
int next;
char *key;
int klen;
char *val;
int vlen;
};
typedef struct Tnode Tnode; typedef struct Tnode Tnode;
struct Tnode struct Tnode
{ {
char *val;
int vlen;
int child; int child;
int sibling; int sibling;
char c; char c;
char *val;
int vlen;
}; };
typedef struct Trie Trie; typedef struct Trie Trie;
struct Trie struct Trie
{ {
int root;
Tnode *nodes; Tnode *nodes;
int n; int n;
int cap; int cap;
@@ -94,16 +82,6 @@ enum
TrieExact, TrieExact,
}; };
typedef struct Hmap Hmap;
struct Hmap
{
int nbs;
int nsz;
int len;
int cap;
uchar *nodes;
};
typedef struct Lang Lang; typedef struct Lang Lang;
typedef struct Im Im; typedef struct Im Im;
struct Lang struct Lang
@@ -115,7 +93,7 @@ struct Lang
void (*back)(Im*); void (*back)(Im*);
void (*dictq)(Im*); void (*dictq)(Im*);
Trie *map; Trie *map;
Hmap *dict; Trie *dict;
}; };
struct Im struct Im

116
dict.c
View File

@@ -1,59 +1,34 @@
#include <errno.h>
#include "dat.h" #include "dat.h"
#include "fn.h" #include "fn.h"
static char* /* Candidates are the space-separated words of the value; the reading itself
readline(Biobuf *b, char *path) * is a candidate only for the emoji dictionary. */
{
char *line;
errno = 0;
line = Brdstr(b, '\n', 1);
if(errno != 0)
die("can't read %s: %s", path, strerror(errno));
return line;
}
void void
dictlookup(Dictreq *req, Dictres *res) dictlookup(Dictreq *req, Dictres *res)
{ {
Lang *l; Lang *l;
Hmap *dict;
Hnode *n;
char *p, *e, *sp; char *p, *e, *sp;
Str tmp; Str tmp;
int vlen;
res->key = req->pre; res->key = req->pre;
res->nkouho = 0; res->nkouho = 0;
res->lang = req->lang; res->lang = req->lang;
res->seq = req->seq; res->seq = req->seq;
if(req->key.n == 0)
return;
l = getlang(req->lang); l = getlang(req->lang);
dict = l ? l->dict : nil; if(req->key.n == 0 || l == nil ||
if(dict == nil) trielookup(l->dict, &req->key, &p, &vlen) != TrieExact)
return; return;
n = hmapget(dict, &req->key); e = p + vlen;
if(n == nil || n->vlen == 0)
return;
p = n->val;
e = p + n->vlen;
while(res->nkouho < Maxkouho && p < e){ while(res->nkouho < Maxkouho && p < e){
while(p < e && *p == ' ') while(p < e && *p == ' ')
p++; p++;
if(p == e)
break;
sp = p; sp = p;
while(p < e && *p != ' ') while(p < e && *p != ' ')
p++; p++;
if(!sinit(&tmp, sp, p - sp)){ if(sinit(&tmp, sp, p - sp) && tmp.n > 0 &&
res->nkouho = 0; (req->lang == LangEMOJI || scmp(&tmp, &req->key) != 0))
return;
}
if(req->lang == LangEMOJI || scmp(&tmp, &req->key) != 0)
res->kouho[res->nkouho++] = tmp; res->kouho[res->nkouho++] = tmp;
if(p < e)
p++;
} }
} }
@@ -74,70 +49,29 @@ dictthread(void*)
} }
} }
static Hmap* static Trie*
dictopen(char *path) langopen(char *dir, char *name, char *ext)
{ {
Hmap *h; char *path;
Biobuf *b; Trie *t;
Str key, tmp;
char *line, *tab, *p, *e;
int len, lineno;
b = Bopen(path, OREAD); path = smprint("%s/%s.%s", dir, name, ext);
if(b == nil) if(path == nil)
die("can't open %s: %s", path, strerror(errno)); die("out of memory");
h = hmapalloc(4096); t = trieopen(path);
lineno = 0; free(path);
while((line = readline(b, path)) != nil){ return t;
lineno++;
len = Blinelen(b);
if(memchr(line, '\0', len) != nil)
die("NUL in dictionary: %s:%d", path, lineno);
if(len > 0 && line[len-1] == '\r')
line[--len] = '\0';
if(len == 0 || line[0] == ';'){
free(line);
continue;
}
tab = memchr(line, '\t', len);
if(tab == nil || tab == line || tab >= line + len - 1 ||
memchr(tab+1, '\t', line+len-(tab+1)) != nil)
die("malformed dictionary: %s:%d", path, lineno);
*tab = '\0';
if(!sinit(&key, line, tab-line))
die("invalid or oversized dictionary key: %s:%d",
path, lineno);
for(p = tab+1; p < line+len; p = e+1){
e = memchr(p, ' ', line+len-p);
if(e == nil)
e = line+len;
if(!sinit(&tmp, p, e-p))
die("invalid or oversized dictionary candidate: %s:%d",
path, lineno);
if(e == line+len)
break;
}
hmapset(&h, &key, tab+1, len - (tab - line) - 1);
free(line);
}
if(Bterm(b) < 0)
die("can't close %s: %s", path, strerror(errno));
return h;
} }
void void
dictinit(char *dir) langinit(char *dir)
{ {
char *path; Lang *l;
int i;
for(i = 0; i < nlang; i++){ for(l = langs; l < langs + nlang; l++){
if(langs[i].dictname == nil) if(l->mapname != nil)
continue; l->map = langopen(dir, l->mapname, "map");
path = smprint("%s/%s.dict", dir, langs[i].dictname); if(l->dictname != nil)
if(path == nil) l->dict = langopen(dir, l->dictname, "dict");
die("out of memory");
langs[i].dict = dictopen(path);
free(path);
} }
} }

11
fn.h
View File

@@ -9,19 +9,16 @@ int scmp(Str*, Str*);
int stoutf(Str*, char*, int); int stoutf(Str*, char*, int);
Rune slastr(Str*); Rune slastr(Str*);
Hmap* hmapalloc(int);
void hmapfree(Hmap*);
void hmapset(Hmap**, Str*, const char*, int);
Hnode* hmapget(Hmap*, Str*);
int mapget(Trie*, Str*, Str*); int mapget(Trie*, Str*, Str*);
Trie* trienew(void);
void trieput(Trie*, char*, int, char*, int);
Trie* trieopen(char*); Trie* trieopen(char*);
void trieclose(Trie*); void trieclose(Trie*);
int trielookup(Trie*, char*, int, char**, int*); int trielookup(Trie*, Str*, char**, int*);
Lang* getlang(int); Lang* getlang(int);
void mapinit(char*); void langinit(char*);
void dictinit(char*);
void dictthread(void*); void dictthread(void*);
void dictlookup(Dictreq*, Dictres*); void dictlookup(Dictreq*, Dictres*);

175
hash.c
View File

@@ -1,175 +0,0 @@
#include <limits.h>
#include "dat.h"
#include "fn.h"
enum {
Tagsize = sizeof(Hnode),
};
static uvlong
hash(Str *s)
{
uvlong h;
int i;
h = 7;
for(i = 0; i < s->n; i++)
h = h*31 + s->r[i];
return h;
}
Hmap*
hmapalloc(int nbuckets)
{
void *store;
Hmap *h;
int nsz;
if(nbuckets < 1)
return nil;
nsz = Tagsize;
if((ulong)nbuckets > (ULONG_MAX-sizeof(*h))/(ulong)nsz)
return nil;
store = emalloc(sizeof(*h) + (ulong)nbuckets * nsz);
h = store;
h->nbs = nbuckets;
h->nsz = nsz;
h->len = h->cap = nbuckets;
h->nodes = (uchar*)store + sizeof(*h);
return h;
}
static int
keycmp(Hnode *n, Str *key)
{
char buf[Maxutf];
int len;
len = stoutf(key, buf, sizeof(buf));
if(n->klen != len)
return 1;
return memcmp(n->key, buf, len);
}
Hnode*
hmapget(Hmap *h, Str *key)
{
Hnode *n;
uchar *v;
if(h == nil || key == nil || key->n < 0 || key->n > Maxrunes)
return nil;
v = h->nodes + (hash(key) % h->nbs) * (ulong)h->nsz;
for(;;){
n = (Hnode*)v;
if(n->filled && keycmp(n, key) == 0)
return n;
if(n->next == 0)
break;
v = h->nodes + (ulong)n->next * h->nsz;
}
return nil;
}
static char*
sdup(Str *s, int *len)
{
char buf[Maxutf];
char *p;
int n;
n = stoutf(s, buf, sizeof(buf));
p = emalloc((ulong)n + 1);
memmove(p, buf, n);
p[n] = '\0';
*len = n;
return p;
}
static char*
memdup(const char *src, int n)
{
char *p;
if(n == 0)
return nil;
p = emalloc((ulong)n + 1);
memmove(p, src, n);
p[n] = '\0';
return p;
}
void
hmapfree(Hmap *h)
{
Hnode *n;
int i;
if(h == nil)
return;
for(i = 0; i < h->len; i++){
n = (Hnode*)(h->nodes + (ulong)i * h->nsz);
if(!n->filled)
continue;
free(n->key);
free(n->val);
}
free(h);
}
void
hmapset(Hmap **store, Str *key, const char *val, int vlen)
{
char *newval;
Hnode *n;
uchar *v;
Hmap *h;
int next;
ulong diff;
if(store == nil || *store == nil || key == nil ||
key->n < 0 || key->n > Maxrunes || vlen < 0 ||
(vlen > 0 && val == nil))
return;
newval = memdup(val, vlen);
h = *store;
v = h->nodes + (hash(key) % h->nbs) * (ulong)h->nsz;
for(;;){
n = (Hnode*)v;
next = n->next;
if(n->filled == 0)
goto replace;
if(keycmp(n, key) == 0)
goto replace;
if(next == 0)
break;
v = h->nodes + (ulong)next * h->nsz;
}
if(h->cap == h->len){
diff = v - h->nodes;
if(h->cap > INT_MAX/2 ||
(ulong)h->cap > (ULONG_MAX-sizeof(*h))/(2*(ulong)h->nsz))
die("hash table is too large");
h->cap *= 2;
*store = erealloc(*store,
sizeof(*h) + (ulong)h->cap * h->nsz);
h = *store;
h->nodes = (uchar*)*store + sizeof(*h);
v = h->nodes + diff;
n = (Hnode*)v;
}
n->next = h->len;
memset(h->nodes + (ulong)h->len * h->nsz, 0, h->nsz);
h->len++;
v = h->nodes + (ulong)n->next * h->nsz;
n = (Hnode*)v;
replace:
if(n->filled == 0){
n->key = sdup(key, &n->klen);
n->filled = 1;
}
n->next = next;
free(n->val);
n->val = newval;
n->vlen = vlen;
}

3
main.c
View File

@@ -58,8 +58,7 @@ threadmain(int argc, char **argv)
keyc = chancreate(sizeof(Keyreq), 0); keyc = chancreate(sizeof(Keyreq), 0);
dictreqc = chancreate(sizeof(Dictreq), 4); dictreqc = chancreate(sizeof(Dictreq), 4);
dictresc = chancreate(sizeof(Dictres), 0); dictresc = chancreate(sizeof(Dictres), 0);
mapinit(argv[1]); langinit(argv[1]);
dictinit(argv[1]);
srvinit(); srvinit();
proccreate(drawthread, nil, 16384); proccreate(drawthread, nil, 16384);
proccreate(srvthread, nil, 16384); proccreate(srvthread, nil, 16384);

44
strans.c
View File

@@ -405,18 +405,16 @@ getlang(int lang)
static int static int
maplookup(Trie *t, Str *key, Str *out) maplookup(Trie *t, Str *key, Str *out)
{ {
char buf[Maxutf], *v; char *v;
int klen, match, vlen; int match, vlen;
if(out != nil) sclear(out);
sclear(out); if(key->n == 0)
if(t == nil || key == nil || key->n == 0)
return 0; return 0;
klen = stoutf(key, buf, sizeof(buf)); match = trielookup(t, key, &v, &vlen);
match = trielookup(t, buf, klen, &v, &vlen);
if(match == TrieMiss) if(match == TrieMiss)
return 0; return 0;
if(match == TrieExact && out != nil && !sinit(out, v, vlen)) if(match == TrieExact && !sinit(out, v, vlen))
return 0; return 0;
return 1; return 1;
} }
@@ -1052,33 +1050,11 @@ imthread(void*)
int int
mapget(Trie *t, Str *key, Str *out) mapget(Trie *t, Str *key, Str *out)
{ {
char buf[Maxutf], *v; char *v;
int klen, match, vlen; int vlen;
if(out != nil) sclear(out);
sclear(out); if(key->n == 0 || trielookup(t, key, &v, &vlen) != TrieExact)
if(key == nil || out == nil || key->n == 0)
return 0;
klen = stoutf(key, buf, sizeof(buf));
match = trielookup(t, buf, klen, &v, &vlen);
if(match != TrieExact)
return 0; return 0;
return sinit(out, v, vlen); return sinit(out, v, vlen);
} }
void
mapinit(char *dir)
{
char *path;
int i;
for(i = 0; i < nelem(langs); i++){
if(langs[i].mapname == nil)
continue;
path = smprint("%s/%s.map", dir, langs[i].mapname);
if(path == nil)
die("out of memory");
langs[i].map = trieopen(path);
free(path);
}
}

4
tests/Makefile
View File

@@ -28,11 +28,11 @@ SMOKE = ibus_live_test ibus_client_smoke gtk_live_test xim_live_test \
ipc_live_test ipc_live_test
FAULT = daemon_collision_test daemon_failure_test daemon_restart_test FAULT = daemon_collision_test daemon_failure_test daemon_restart_test
LIVE = $(SMOKE) $(FAULT) LIVE = $(SMOKE) $(FAULT)
TESTSRC = test_util.c str_test.c hash_test.c trie_test.c \ TESTSRC = test_util.c str_test.c trie_test.c \
ko_test.c vi_test.c engine_test.c dict_test.c ipc_test.c \ ko_test.c vi_test.c engine_test.c dict_test.c ipc_test.c \
popup_test.c font_test.c ibus_test.c server_test.c xim_adapter_test.c popup_test.c font_test.c ibus_test.c server_test.c xim_adapter_test.c
TESTOBJ = $(TESTSRC:.c=.o) TESTOBJ = $(TESTSRC:.c=.o)
PARENTSRC = str.c hash.c trie.c dict.c ko.c vi.c ipc.c popup_layout.c \ PARENTSRC = str.c trie.c dict.c ko.c vi.c ipc.c popup_layout.c \
font.c font.c
PARENTOBJ = $(PARENTSRC:%.c=unit_%.o) PARENTOBJ = $(PARENTSRC:%.c=unit_%.o)
XIMOBJ = unit_xim_keymap.o unit_ximtext.o XIMOBJ = unit_xim_keymap.o unit_ximtext.o

30
tests/dict_test.c
View File

@@ -3,21 +3,20 @@
void void
dictionary_candidates(struct ct *t) dictionary_candidates(struct ct *t)
{ {
static char malformed[] = { 'o', 'k', ' ', (char)0x80 };
char many[512], item[8]; char many[512], item[8];
char *p; char *p;
Dictreq req; Dictreq req;
Dictres res; Dictres res;
Hmap *saved; Trie *saved;
Lang *lang; Lang *lang;
Str key; Str key;
int i; int i;
lang = getlang(LangJP); lang = getlang(LangJP);
saved = lang->dict; saved = lang->dict;
lang->dict = hmapalloc(1); lang->dict = trienew();
key = mkstr("かな"); key = mkstr("かな");
hmapset(&lang->dict, &key, " 候補1 かな 候補2 ", trieput(lang->dict, "かな", strlen("かな"), " 候補1 かな 候補2 ",
strlen(" 候補1 かな 候補2 ")); strlen(" 候補1 かな 候補2 "));
memset(&req, 0, sizeof req); memset(&req, 0, sizeof req);
req.key = key; req.key = key;
@@ -42,7 +41,7 @@ dictionary_candidates(struct ct *t)
p += strlen(item); p += strlen(item);
} }
*p = '\0'; *p = '\0';
hmapset(&lang->dict, &key, many, strlen(many)); trieput(lang->dict, "key", 3, many, strlen(many));
req.key = key; req.key = key;
req.pre = mkstr("preedit-two"); req.pre = mkstr("preedit-two");
req.seq = 29; req.seq = 29;
@@ -54,13 +53,8 @@ dictionary_candidates(struct ct *t)
CT_EQ_INT(t, 0, scmp(&req.pre, &res.key)); CT_EQ_INT(t, 0, scmp(&req.pre, &res.key));
checkstr(t, "first capped candidate", "c00", &res.kouho[0]); checkstr(t, "first capped candidate", "c00", &res.kouho[0]);
checkstr(t, "last capped candidate", "c31", &res.kouho[31]); checkstr(t, "last capped candidate", "c31", &res.kouho[31]);
key = mkstr("malformed");
hmapset(&lang->dict, &key, malformed, sizeof malformed);
req.key = key;
dictlookup(&req, &res);
CT_EQ_INT(t, 0, res.nkouho);
cleanup: cleanup:
hmapfree(lang->dict); trieclose(lang->dict);
lang->dict = saved; lang->dict = saved;
} }
@@ -77,13 +71,13 @@ dictionary_misses_clear_result(struct ct *t)
}; };
Dictreq req; Dictreq req;
Dictres res; Dictres res;
Hmap *saved; Trie *saved;
Lang *lang; Lang *lang;
int i; int i;
lang = getlang(LangJP); lang = getlang(LangJP);
saved = lang->dict; saved = lang->dict;
lang->dict = hmapalloc(1); lang->dict = trienew();
for(i = 0; i < nelem(cases); i++){ for(i = 0; i < nelem(cases); i++){
memset(&res, 0xa5, sizeof res); memset(&res, 0xa5, sizeof res);
req.key = mkstr(cases[i].key); req.key = mkstr(cases[i].key);
@@ -98,7 +92,7 @@ dictionary_misses_clear_result(struct ct *t)
CT_EQ_UINT(t, 0xf00d0000U + i, res.seq); CT_EQ_UINT(t, 0xf00d0000U + i, res.seq);
checkstr(t, cases[i].name, cases[i].pre, &res.key); checkstr(t, cases[i].name, cases[i].pre, &res.key);
} }
hmapfree(lang->dict); trieclose(lang->dict);
lang->dict = saved; lang->dict = saved;
} }
@@ -107,15 +101,15 @@ dictionary_emoji_identity(struct ct *t)
{ {
Dictreq req; Dictreq req;
Dictres res; Dictres res;
Hmap *saved; Trie *saved;
Lang *lang; Lang *lang;
Str key; Str key;
lang = getlang(LangEMOJI); lang = getlang(LangEMOJI);
saved = lang->dict; saved = lang->dict;
lang->dict = hmapalloc(2); lang->dict = trienew();
key = mkstr("é"); key = mkstr("é");
hmapset(&lang->dict, &key, "é", strlen("é")); trieput(lang->dict, "é", strlen("é"), "é", strlen("é"));
memset(&req, 0, sizeof req); memset(&req, 0, sizeof req);
req.key = key; req.key = key;
req.pre = key; req.pre = key;
@@ -124,6 +118,6 @@ dictionary_emoji_identity(struct ct *t)
dictlookup(&req, &res); dictlookup(&req, &res);
if(CT_EQ_INT(t, 1, res.nkouho)) if(CT_EQ_INT(t, 1, res.nkouho))
checkstr(t, "emoji identity candidate", "é", &res.kouho[0]); checkstr(t, "emoji identity candidate", "é", &res.kouho[0]);
hmapfree(lang->dict); trieclose(lang->dict);
lang->dict = saved; lang->dict = saved;
} }

74
tests/engine_test.c
View File

@@ -311,14 +311,14 @@ engine_rejects_stale_dictionary_results(struct ct *t)
{ {
Dictreq a, b; Dictreq a, b;
Dictres res; Dictres res;
Hmap *saved; Trie *saved;
Lang *jp; Lang *jp;
char one, two; char one, two;
int n; int n;
jp = getlang(LangJP); jp = getlang(LangJP);
saved = jp->dict; saved = jp->dict;
jp->dict = hmapalloc(1); jp->dict = trienew();
init(); init();
im.l = jp; im.l = jp;
while(channbrecv(dictreqc, &a) > 0) while(channbrecv(dictreqc, &a) > 0)
@@ -362,7 +362,7 @@ engine_rejects_stale_dictionary_results(struct ct *t)
cleanup: cleanup:
while(channbrecv(dictreqc, &a) > 0) while(channbrecv(dictreqc, &a) > 0)
; ;
hmapfree(jp->dict); trieclose(jp->dict);
jp->dict = saved; jp->dict = saved;
} }
@@ -720,7 +720,7 @@ struct Searchfix
Im im; Im im;
Search search; Search search;
Lang *dictlang; Lang *dictlang;
Hmap *dict; Trie *dict;
int activecap; int activecap;
int visible; int visible;
int candidatechosen; int candidatechosen;
@@ -939,12 +939,9 @@ engine_candidate_completion(struct ct *t)
} }
static void static void
setdict(Hmap **dict, char *key, char *val) setdict(Trie *dict, char *key, char *val)
{ {
Str s; trieput(dict, key, strlen(key), val, strlen(val));
s = mkstr(key);
hmapset(dict, &s, val, strlen(val));
} }
static void static void
@@ -964,22 +961,22 @@ static void
emojibegin(Searchfix *f, int showpre) emojibegin(Searchfix *f, int showpre)
{ {
searchsave(f, LangEMOJI); searchsave(f, LangEMOJI);
f->dictlang->dict = hmapalloc(32); f->dictlang->dict = trienew();
setdict(&f->dictlang->dict, "a", "A B"); setdict(f->dictlang->dict, "a", "A B");
setdict(&f->dictlang->dict, "ㅁ", "B C"); setdict(f->dictlang->dict, "ㅁ", "B C");
setdict(&f->dictlang->dict, "alpha", "α"); setdict(f->dictlang->dict, "alpha", "α");
setdict(&f->dictlang->dict, "smil", "😀 😄"); setdict(f->dictlang->dict, "smil", "😀 😄");
setdict(&f->dictlang->dict, "smile", "😀 😄"); setdict(f->dictlang->dict, "smile", "😀 😄");
setdict(&f->dictlang->dict, "웃음", "😀 😄 😂"); setdict(f->dictlang->dict, "웃음", "😀 😄 😂");
setdict(&f->dictlang->dict, "えがお", "😀 😄 😊"); setdict(f->dictlang->dict, "えがお", "😀 😄 😊");
setdict(&f->dictlang->dict, "エガオ", "😀 😄 😊"); setdict(f->dictlang->dict, "エガオ", "😀 😄 😊");
setdict(&f->dictlang->dict, "heart", "❤️"); setdict(f->dictlang->dict, "heart", "❤️");
setdict(&f->dictlang->dict, "coffee", "☕️"); setdict(f->dictlang->dict, "coffee", "☕️");
setdict(&f->dictlang->dict, "^", "¹ ² ³ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁽"); setdict(f->dictlang->dict, "^", "¹ ² ³ ⁴ ⁵ ⁶ ⁷ ⁸ ⁹ ⁽");
setdict(&f->dictlang->dict, "^0", "⁰"); setdict(f->dictlang->dict, "^0", "⁰");
setdict(&f->dictlang->dict, "_", "₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₍"); setdict(f->dictlang->dict, "_", "₁ ₂ ₃ ₄ ₅ ₆ ₇ ₈ ₉ ₍");
setdict(&f->dictlang->dict, "<", "← ≤ ♥ ≠"); setdict(f->dictlang->dict, "<", "← ≤ ♥ ≠");
setdict(&f->dictlang->dict, "many", setdict(f->dictlang->dict, "many",
"c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12"); "c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12");
init(); init();
im.l = getlang(LangKO); im.l = getlang(LangKO);
@@ -989,11 +986,11 @@ emojibegin(Searchfix *f, int showpre)
static void static void
searchend(Searchfix *f) searchend(Searchfix *f)
{ {
Hmap *dict; Trie *dict;
dict = f->dictlang->dict; dict = f->dictlang->dict;
f->dictlang->dict = f->dict; f->dictlang->dict = f->dict;
hmapfree(dict); trieclose(dict);
draindraw(nil); draindraw(nil);
im = f->im; im = f->im;
search = f->search; search = f->search;
@@ -1079,12 +1076,12 @@ static void
hanjabegin(Searchfix *f, int lang) hanjabegin(Searchfix *f, int lang)
{ {
searchsave(f, LangHANJA); searchsave(f, LangHANJA);
f->dictlang->dict = hmapalloc(8); f->dictlang->dict = trienew();
setdict(&f->dictlang->dict, "한", "漢 韓"); setdict(f->dictlang->dict, "한", "漢 韓");
setdict(&f->dictlang->dict, "가", "家"); setdict(f->dictlang->dict, "가", "家");
setdict(&f->dictlang->dict, "ㄱ", "假"); setdict(f->dictlang->dict, "ㄱ", "假");
setdict(&f->dictlang->dict, "한글", "文"); setdict(f->dictlang->dict, "한글", "文");
setdict(&f->dictlang->dict, "か", "仮"); setdict(f->dictlang->dict, "か", "仮");
init(); init();
im.l = getlang(lang); im.l = getlang(lang);
} }
@@ -1156,15 +1153,14 @@ engine_japanese_candidates(struct ct *t)
{ {
Dictreq req; Dictreq req;
Dictres res; Dictres res;
Hmap *saved; Trie *saved;
Lang *jp; Lang *jp;
Str com, key, shown; Str com, shown;
jp = getlang(LangJP); jp = getlang(LangJP);
saved = jp->dict; saved = jp->dict;
jp->dict = hmapalloc(8); jp->dict = trienew();
key = mkstr("かんじ"); setdict(jp->dict, "かんじ", "漢字 幹事");
hmapset(&jp->dict, &key, "漢字 幹事", strlen("漢字 幹事"));
init(); init();
im.l = jp; im.l = jp;
sclear(&com); sclear(&com);
@@ -1189,7 +1185,7 @@ engine_japanese_candidates(struct ct *t)
checkstr(t, "selected Kanji", "幹事", &com); checkstr(t, "selected Kanji", "幹事", &com);
cleanup: cleanup:
newestrequest(&req); newestrequest(&req);
hmapfree(jp->dict); trieclose(jp->dict);
jp->dict = saved; jp->dict = saved;
} }

112
tests/hash_test.c
View File

@@ -1,112 +0,0 @@
#include "test.h"
void
hmap_set_replace_and_grow(struct ct *t)
{
char keybuf[16], valbuf[16], source[] = "copied";
Hmap *h;
Hnode *n;
Str key;
int i;
h = hmapalloc(1);
for(i = 0; i < 4; i++){
snprint(keybuf, sizeof keybuf, "key%d", i);
snprint(valbuf, sizeof valbuf, "value%d", i);
key = mkstr(keybuf);
hmapset(&h, &key, valbuf, strlen(valbuf));
}
for(i = 0; i < 4; i++){
snprint(keybuf, sizeof keybuf, "key%d", i);
snprint(valbuf, sizeof valbuf, "value%d", i);
key = mkstr(keybuf);
n = hmapget(h, &key);
if(n == nil || strcmp(n->val, valbuf) != 0)
CT_ERRORF(t, "%s: collision chain lost value", keybuf);
}
key = mkstr("key1");
hmapset(&h, &key, source, strlen(source));
source[0] = 'X';
n = hmapget(h, &key);
if(!CT_CHECK(t, n != nil))
goto cleanup;
CT_EQ_STR(t, "copied", n->val);
key = mkstr("key2");
n = hmapget(h, &key);
if(!CT_CHECK(t, n != nil))
goto cleanup;
CT_EQ_STR(t, "value2", n->val);
key = mkstr("key1");
hmapset(&h, &key, nil, 0);
n = hmapget(h, &key);
if(!CT_CHECK(t, n != nil))
goto cleanup;
CT_EQ_INT(t, 0, n->vlen);
CT_EQ_PTR(t, nil, n->val);
key = mkstr("missing");
CT_EQ_PTR(t, nil, hmapget(h, &key));
cleanup:
hmapfree(h);
}
void
hmap_long_utf8_keys(struct ct *t)
{
Hmap *h;
Hnode *n;
Str a, b;
int i;
for(i = 0; i < Maxrunes-1; i++)
a.r[i] = b.r[i] = 0x1f600;
a.r[Maxrunes-1] = 0x1f601;
b.r[Maxrunes-1] = 0x1f602;
a.n = b.n = Maxrunes;
h = hmapalloc(1);
hmapset(&h, &a, "first", 5);
hmapset(&h, &b, "second", 6);
n = hmapget(h, &a);
if(!CT_CHECK(t, n != nil))
goto cleanup;
CT_EQ_STR(t, "first", n != nil ? n->val : nil);
n = hmapget(h, &b);
if(!CT_CHECK(t, n != nil))
goto cleanup;
CT_EQ_STR(t, "second", n != nil ? n->val : nil);
cleanup:
hmapfree(h);
}
void
hmap_binary_keys_and_invalid_lengths(struct ct *t)
{
Hmap *h;
Hnode *n;
Str key, other;
h = hmapalloc(1);
CT_CHECK(t, h != nil);
key.n = 3;
key.r[0] = 'a';
key.r[1] = 0;
key.r[2] = 'b';
other = key;
other.r[2] = 'c';
hmapset(&h, &key, "one", 3);
hmapset(&h, &other, "two", 3);
n = hmapget(h, &key);
if(CT_CHECK(t, n != nil)){
CT_EQ_INT(t, 3, n->klen);
CT_EQ_MEM(t, "one", n->val, n->vlen);
}
n = hmapget(h, &other);
if(CT_CHECK(t, n != nil))
CT_EQ_MEM(t, "two", n->val, n->vlen);
hmapset(&h, &key, "bad", -1);
hmapset(&h, &key, nil, 1);
n = hmapget(h, &key);
if(CT_CHECK(t, n != nil))
CT_EQ_MEM(t, "one", n->val, n->vlen);
CT_EQ_PTR(t, nil, hmapalloc(0));
hmapfree(h);
}

5
tests/test.h
View File

@@ -12,11 +12,8 @@ void str_init_utf8(struct ct*);
void str_edit_and_alias(struct ct*); void str_edit_and_alias(struct ct*);
void str_utf8_capacity(struct ct*); void str_utf8_capacity(struct ct*);
void str_invalid_and_full_appends(struct ct*); void str_invalid_and_full_appends(struct ct*);
void hmap_set_replace_and_grow(struct ct*);
void hmap_long_utf8_keys(struct ct*);
void hmap_binary_keys_and_invalid_lengths(struct ct*);
void trie_exact_prefix_and_duplicate(struct ct*); void trie_exact_prefix_and_duplicate(struct ct*);
void trie_optional_outputs_and_invalid_lengths(struct ct*); void trie_put_and_unloaded(struct ct*);
void popup_layout(struct ct*); void popup_layout(struct ct*);
void font_render(struct ct*); void font_render(struct ct*);
void production_maps_load(struct ct*); void production_maps_load(struct ct*);

48
tests/trie_test.c
View File

@@ -5,26 +5,26 @@ trie_exact_prefix_and_duplicate(struct ct *t)
{ {
static const struct { static const struct {
char *key; char *key;
int klen;
int match; int match;
char *want; char *want;
} cases[] = { } cases[] = {
{ "a", 1, TrieExact, "alpha" }, { "a", TrieExact, "alpha" },
{ "dupli", 5, TriePrefix, nil }, { "ab", TrieExact, "beta" },
{ "duplicate", 9, TrieExact, "second" }, { "한", TrieExact, "값" },
{ "missing", 7, TrieMiss, nil }, { "dupli", TriePrefix, nil },
{ nil, 1, TrieMiss, nil }, { "duplicate", TrieExact, "second" },
{ "a", -1, TrieMiss, nil }, { "missing", TrieMiss, nil },
{ "", TriePrefix, nil },
}; };
char *v; char *v;
Trie *trie; Trie *trie;
Str key;
int i, match, n; int i, match, n;
trie = trieopen("data/trie.map"); trie = trieopen("data/trie.map");
for(i = 0; i < nelem(cases); i++){ for(i = 0; i < nelem(cases); i++){
v = "unchanged"; key = mkstr(cases[i].key);
n = 77; match = trielookup(trie, &key, &v, &n);
match = trielookup(trie, cases[i].key, cases[i].klen, &v, &n);
if(match != cases[i].match){ if(match != cases[i].match){
CT_ERRORF(t, "case %d: want match %d, got %d", CT_ERRORF(t, "case %d: want match %d, got %d",
i, cases[i].match, match); i, cases[i].match, match);
@@ -42,22 +42,28 @@ trie_exact_prefix_and_duplicate(struct ct *t)
} }
void void
trie_optional_outputs_and_invalid_lengths(struct ct *t) trie_put_and_unloaded(struct ct *t)
{ {
char *v; char *v;
Trie *trie; Trie *trie;
Str key;
int n; int n;
trie = trieopen("data/trie.map"); key = mkstr("k");
CT_EQ_INT(t, TrieExact, trielookup(trie, "a", 1, &v, nil)); CT_EQ_INT(t, TrieMiss, trielookup(nil, &key, &v, &n));
CT_CHECK(t, v != nil); trie = trienew();
CT_EQ_INT(t, TrieExact, trielookup(trie, "a", 1, nil, &n)); CT_EQ_INT(t, TrieMiss, trielookup(trie, &key, &v, &n));
CT_EQ_INT(t, 5, n); trieput(trie, "k", 1, "one two", 7);
CT_EQ_INT(t, TriePrefix, trieput(trie, "ka", 2, "", 0);
trielookup(trie, "dupli", 5, nil, nil)); if(CT_EQ_INT(t, TrieExact, trielookup(trie, &key, &v, &n)))
CT_EQ_INT(t, TriePrefix, trielookup(trie, nil, 0, &v, &n)); CT_EQ_MEM(t, "one two", v, 7);
CT_EQ_PTR(t, nil, v); key = mkstr("ka");
CT_EQ_INT(t, 0, n); if(CT_EQ_INT(t, TrieExact, trielookup(trie, &key, &v, &n)))
CT_EQ_INT(t, 0, n);
trieput(trie, "k", 1, "three", 5);
key = mkstr("k");
if(CT_EQ_INT(t, TrieExact, trielookup(trie, &key, &v, &n)))
CT_EQ_MEM(t, "three", v, 5);
trieclose(trie); trieclose(trie);
} }

5
tests/unit_test.c
View File

@@ -67,11 +67,8 @@ static const struct ct_test tests[] = {
{ "str/edit-and-alias", str_edit_and_alias }, { "str/edit-and-alias", str_edit_and_alias },
{ "str/utf8-capacity", str_utf8_capacity }, { "str/utf8-capacity", str_utf8_capacity },
{ "str/invalid-full-appends", str_invalid_and_full_appends }, { "str/invalid-full-appends", str_invalid_and_full_appends },
{ "hmap/set-replace-grow", hmap_set_replace_and_grow },
{ "hmap/long-utf8-keys", hmap_long_utf8_keys },
{ "hmap/binary-invalid-lengths", hmap_binary_keys_and_invalid_lengths },
{ "trie/exact-prefix-duplicate", trie_exact_prefix_and_duplicate }, { "trie/exact-prefix-duplicate", trie_exact_prefix_and_duplicate },
{ "trie/optional-invalid-lengths", trie_optional_outputs_and_invalid_lengths }, { "trie/put-and-unloaded", trie_put_and_unloaded },
{ "popup/layout", popup_layout }, { "popup/layout", popup_layout },
{ "font/render", font_render }, { "font/render", font_render },
{ "map/production-lifecycle", production_maps_load }, { "map/production-lifecycle", production_maps_load },

158
trie.c
View File

@@ -1,33 +1,21 @@
#include <errno.h> #include <errno.h>
#include <limits.h>
#include "dat.h" #include "dat.h"
#include "fn.h" #include "fn.h"
static char* /*
readline(Biobuf *b, char *path) * A byte trie over UTF-8 keys. Maps need prefix matches while composing;
{ * dictionaries need exact matches; both are small enough for one structure.
char *line; * Node 0 is the root.
*/
errno = 0;
line = Brdstr(b, '\n', 1);
if(errno != 0)
die("can't read %s: %s", path, strerror(errno));
return line;
}
static int static int
newnode(Trie *t) newnode(Trie *t)
{ {
int cap;
int i; int i;
if(t->n >= t->cap){ if(t->n == t->cap){
if(t->cap > INT_MAX/2 || t->cap *= 2;
(ulong)t->cap > ULONG_MAX/(2*sizeof(Tnode))) t->nodes = erealloc(t->nodes, t->cap * sizeof(Tnode));
die("map is too large");
cap = t->cap * 2;
t->nodes = erealloc(t->nodes, (ulong)cap * sizeof(Tnode));
t->cap = cap;
} }
i = t->n++; i = t->n++;
memset(&t->nodes[i], 0, sizeof(Tnode)); memset(&t->nodes[i], 0, sizeof(Tnode));
@@ -59,13 +47,25 @@ add(Trie *t, int ni, char c)
return pi; return pi;
} }
static void Trie*
insert(Trie *t, char *key, int klen, char *val, int vlen) trienew(void)
{ {
int ni, ci; Trie *t;
int i;
ni = t->root; t = emalloc(sizeof(*t));
t->cap = 1024;
t->nodes = emalloc(t->cap * sizeof(Tnode));
t->n = 0;
newnode(t);
return t;
}
void
trieput(Trie *t, char *key, int klen, char *val, int vlen)
{
int ni, ci, i;
ni = 0;
for(i = 0; i < klen; i++){ for(i = 0; i < klen; i++){
ci = find(t, ni, key[i]); ci = find(t, ni, key[i]);
if(ci < 0) if(ci < 0)
@@ -73,57 +73,70 @@ insert(Trie *t, char *key, int klen, char *val, int vlen)
ni = ci; ni = ci;
} }
free(t->nodes[ni].val); free(t->nodes[ni].val);
t->nodes[ni].val = emalloc((ulong)vlen + 1); t->nodes[ni].val = emalloc(vlen + 1);
memmove(t->nodes[ni].val, val, vlen); memmove(t->nodes[ni].val, val, vlen);
t->nodes[ni].val[vlen] = '\0'; t->nodes[ni].val[vlen] = '\0';
t->nodes[ni].vlen = vlen; t->nodes[ni].vlen = vlen;
} }
static char*
readline(Biobuf *b, char *path)
{
char *line;
errno = 0;
line = Brdstr(b, '\n', 1);
if(errno != 0)
die("can't read %s: %r", path);
return line;
}
/*
* A file holds "key<tab>value" lines; ';' starts a comment. Keys and each
* space-separated word of a value must fit a Str, which is what lookups
* hand back.
*/
Trie* Trie*
trieopen(char *path) trieopen(char *path)
{ {
Trie *t; Trie *t;
Biobuf *b; Biobuf *b;
char *line, *tab, *key, *val;
Str s; Str s;
int klen, vlen; char *e, *line, *p, *tab;
int len, lineno;
b = Bopen(path, OREAD); b = Bopen(path, OREAD);
if(b == nil) if(b == nil)
die("can't open %s: %s", path, strerror(errno)); die("can't open %s: %r", path);
t = emalloc(sizeof(*t)); t = trienew();
t->cap = 1024; for(lineno = 1; (line = readline(b, path)) != nil; lineno++){
t->nodes = emalloc((ulong)t->cap * sizeof(Tnode)); len = Blinelen(b);
t->n = 0; if(memchr(line, '\0', len) != nil)
t->root = newnode(t); die("%s:%d: NUL byte", path, lineno);
while((line = readline(b, path)) != nil){ if(len > 0 && line[len-1] == '\r')
vlen = Blinelen(b); line[--len] = '\0';
if(memchr(line, '\0', vlen) != nil) if(len == 0 || line[0] == ';'){
die("NUL in map: %s", path);
if(vlen > 0 && line[vlen-1] == '\r')
line[--vlen] = '\0';
if(line[0] == '\0' || line[0] == ';'){
free(line); free(line);
continue; continue;
} }
tab = memchr(line, '\t', vlen); tab = memchr(line, '\t', len);
if(tab == nil || tab == line || tab == line+vlen-1 || if(tab == nil || tab == line || tab == line+len-1 ||
memchr(tab+1, '\t', line+vlen-(tab+1)) != nil) memchr(tab+1, '\t', line+len-(tab+1)) != nil)
die("malformed map: %s", path); die("%s:%d: expected key, tab, value", path, lineno);
*tab = '\0'; if(!sinit(&s, line, tab-line))
key = line; die("%s:%d: invalid or oversized key", path, lineno);
klen = tab - line; for(p = tab+1; p < line+len; p = e+1){
if(!sinit(&s, key, klen)) e = memchr(p, ' ', line+len-p);
die("invalid or oversized map key: %s", path); if(e == nil)
val = tab + 1; e = line+len;
vlen = line + vlen - val; if(!sinit(&s, p, e-p))
if(!sinit(&s, val, vlen)) die("%s:%d: invalid or oversized value", path, lineno);
die("invalid or oversized map value: %s", path); }
insert(t, key, klen, val, vlen); trieput(t, line, tab-line, tab+1, line+len-(tab+1));
free(line); free(line);
} }
if(Bterm(b) < 0) if(Bterm(b) < 0)
die("can't close %s: %s", path, strerror(errno)); die("can't close %s: %r", path);
return t; return t;
} }
@@ -140,30 +153,27 @@ trieclose(Trie *t)
free(t); free(t);
} }
/* A nil trie is an unloaded map: every key misses. */
int int
trielookup(Trie *t, char *key, int klen, char **val, int *vlen) trielookup(Trie *t, Str *key, char **val, int *vlen)
{ {
int ni; char buf[Maxutf];
int i; int i, klen, ni;
if(val != nil) *val = nil;
*val = nil; *vlen = 0;
if(vlen != nil) if(t == nil)
*vlen = 0;
if(t == nil || klen < 0 || (klen > 0 && key == nil))
return TrieMiss; return TrieMiss;
ni = t->root; klen = stoutf(key, buf, sizeof buf);
ni = 0;
for(i = 0; i < klen; i++){ for(i = 0; i < klen; i++){
ni = find(t, ni, key[i]); ni = find(t, ni, buf[i]);
if(ni < 0) if(ni < 0)
return TrieMiss; return TrieMiss;
} }
if(t->nodes[ni].val != nil){ if(t->nodes[ni].val == nil)
if(val != nil) return TriePrefix;
*val = t->nodes[ni].val; *val = t->nodes[ni].val;
if(vlen != nil) *vlen = t->nodes[ni].vlen;
*vlen = t->nodes[ni].vlen; return TrieExact;
return TrieExact;
}
return TriePrefix;
} }