data: one trie for maps and dictionaries, one loader
The hash map served a single exact-match lookup that the trie already answers, at the price of a second container, a second file loader with its own drift, and a Str-to-UTF-8 conversion on every chain probe. The files are small (kanji.dict is 7.5k lines), so the trie holds both. trieopen validates keys and each space-separated value word against Str and reports path:line; trielookup takes the Str every caller holds and treats a nil trie as an unloaded map; trienew/trieput exist for tests. The overflow guards on growth, Trie.root (always 0) and the per-language init loop written twice are gone.
This commit is contained in:
116
dict.c
116
dict.c
@@ -1,59 +1,34 @@
|
||||
#include <errno.h>
|
||||
#include "dat.h"
|
||||
#include "fn.h"
|
||||
|
||||
static char*
|
||||
readline(Biobuf *b, char *path)
|
||||
{
|
||||
char *line;
|
||||
|
||||
errno = 0;
|
||||
line = Brdstr(b, '\n', 1);
|
||||
if(errno != 0)
|
||||
die("can't read %s: %s", path, strerror(errno));
|
||||
return line;
|
||||
}
|
||||
|
||||
/* Candidates are the space-separated words of the value; the reading itself
|
||||
* is a candidate only for the emoji dictionary. */
|
||||
void
|
||||
dictlookup(Dictreq *req, Dictres *res)
|
||||
{
|
||||
Lang *l;
|
||||
Hmap *dict;
|
||||
Hnode *n;
|
||||
char *p, *e, *sp;
|
||||
Str tmp;
|
||||
int vlen;
|
||||
|
||||
res->key = req->pre;
|
||||
res->nkouho = 0;
|
||||
res->lang = req->lang;
|
||||
res->seq = req->seq;
|
||||
if(req->key.n == 0)
|
||||
return;
|
||||
l = getlang(req->lang);
|
||||
dict = l ? l->dict : nil;
|
||||
if(dict == nil)
|
||||
if(req->key.n == 0 || l == nil ||
|
||||
trielookup(l->dict, &req->key, &p, &vlen) != TrieExact)
|
||||
return;
|
||||
n = hmapget(dict, &req->key);
|
||||
if(n == nil || n->vlen == 0)
|
||||
return;
|
||||
p = n->val;
|
||||
e = p + n->vlen;
|
||||
e = p + vlen;
|
||||
while(res->nkouho < Maxkouho && p < e){
|
||||
while(p < e && *p == ' ')
|
||||
p++;
|
||||
if(p == e)
|
||||
break;
|
||||
sp = p;
|
||||
while(p < e && *p != ' ')
|
||||
p++;
|
||||
if(!sinit(&tmp, sp, p - sp)){
|
||||
res->nkouho = 0;
|
||||
return;
|
||||
}
|
||||
if(req->lang == LangEMOJI || scmp(&tmp, &req->key) != 0)
|
||||
if(sinit(&tmp, sp, p - sp) && tmp.n > 0 &&
|
||||
(req->lang == LangEMOJI || scmp(&tmp, &req->key) != 0))
|
||||
res->kouho[res->nkouho++] = tmp;
|
||||
if(p < e)
|
||||
p++;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,70 +49,29 @@ dictthread(void*)
|
||||
}
|
||||
}
|
||||
|
||||
static Hmap*
|
||||
dictopen(char *path)
|
||||
static Trie*
|
||||
langopen(char *dir, char *name, char *ext)
|
||||
{
|
||||
Hmap *h;
|
||||
Biobuf *b;
|
||||
Str key, tmp;
|
||||
char *line, *tab, *p, *e;
|
||||
int len, lineno;
|
||||
char *path;
|
||||
Trie *t;
|
||||
|
||||
b = Bopen(path, OREAD);
|
||||
if(b == nil)
|
||||
die("can't open %s: %s", path, strerror(errno));
|
||||
h = hmapalloc(4096);
|
||||
lineno = 0;
|
||||
while((line = readline(b, path)) != nil){
|
||||
lineno++;
|
||||
len = Blinelen(b);
|
||||
if(memchr(line, '\0', len) != nil)
|
||||
die("NUL in dictionary: %s:%d", path, lineno);
|
||||
if(len > 0 && line[len-1] == '\r')
|
||||
line[--len] = '\0';
|
||||
if(len == 0 || line[0] == ';'){
|
||||
free(line);
|
||||
continue;
|
||||
}
|
||||
tab = memchr(line, '\t', len);
|
||||
if(tab == nil || tab == line || tab >= line + len - 1 ||
|
||||
memchr(tab+1, '\t', line+len-(tab+1)) != nil)
|
||||
die("malformed dictionary: %s:%d", path, lineno);
|
||||
*tab = '\0';
|
||||
if(!sinit(&key, line, tab-line))
|
||||
die("invalid or oversized dictionary key: %s:%d",
|
||||
path, lineno);
|
||||
for(p = tab+1; p < line+len; p = e+1){
|
||||
e = memchr(p, ' ', line+len-p);
|
||||
if(e == nil)
|
||||
e = line+len;
|
||||
if(!sinit(&tmp, p, e-p))
|
||||
die("invalid or oversized dictionary candidate: %s:%d",
|
||||
path, lineno);
|
||||
if(e == line+len)
|
||||
break;
|
||||
}
|
||||
hmapset(&h, &key, tab+1, len - (tab - line) - 1);
|
||||
free(line);
|
||||
}
|
||||
if(Bterm(b) < 0)
|
||||
die("can't close %s: %s", path, strerror(errno));
|
||||
return h;
|
||||
path = smprint("%s/%s.%s", dir, name, ext);
|
||||
if(path == nil)
|
||||
die("out of memory");
|
||||
t = trieopen(path);
|
||||
free(path);
|
||||
return t;
|
||||
}
|
||||
|
||||
void
|
||||
dictinit(char *dir)
|
||||
langinit(char *dir)
|
||||
{
|
||||
char *path;
|
||||
int i;
|
||||
Lang *l;
|
||||
|
||||
for(i = 0; i < nlang; i++){
|
||||
if(langs[i].dictname == nil)
|
||||
continue;
|
||||
path = smprint("%s/%s.dict", dir, langs[i].dictname);
|
||||
if(path == nil)
|
||||
die("out of memory");
|
||||
langs[i].dict = dictopen(path);
|
||||
free(path);
|
||||
for(l = langs; l < langs + nlang; l++){
|
||||
if(l->mapname != nil)
|
||||
l->map = langopen(dir, l->mapname, "map");
|
||||
if(l->dictname != nil)
|
||||
l->dict = langopen(dir, l->dictname, "dict");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user