Hojun-Cho/strans
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

data: one trie for maps and dictionaries, one loader

Browse Source 
The hash map served a single exact-match lookup that the trie already
answers, at the price of a second container, a second file loader with
its own drift, and a Str-to-UTF-8 conversion on every chain probe. The
files are small (kanji.dict is 7.5k lines), so the trie holds both.
trieopen validates keys and each space-separated value word against Str
and reports path:line; trielookup takes the Str every caller holds and
treats a nil trie as an unloaded map; trienew/trieput exist for tests.
The overflow guards on growth, Trie.root (always 0) and the per-language
init loop written twice are gone.
This commit is contained in:
Hojun-Cho
2026-08-16 15:56:32 +09:00
parent eb0f88f764
commit c7718ece52
15 changed files with 206 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
trie.c
View File

@@ -1,33 +1,21 @@
#include <errno.h>
#include <limits.h>
#include "dat.h"
#include "fn.h"
static char*
readline(Biobuf *b, char *path)
{
char *line;
errno = 0;
line = Brdstr(b, '\n', 1);
if(errno != 0)
die("can't read %s: %s", path, strerror(errno));
return line;
}
/*
* A byte trie over UTF-8 keys. Maps need prefix matches while composing;
* dictionaries need exact matches; both are small enough for one structure.
* Node 0 is the root.
*/
static int
newnode(Trie *t)
{
int cap;
int i;
if(t->n >= t->cap){
if(t->cap > INT_MAX/2 ||
(ulong)t->cap > ULONG_MAX/(2*sizeof(Tnode)))
die("map is too large");
cap = t->cap * 2;
t->nodes = erealloc(t->nodes, (ulong)cap * sizeof(Tnode));
t->cap = cap;
if(t->n == t->cap){
t->cap *= 2;
t->nodes = erealloc(t->nodes, t->cap * sizeof(Tnode));
}
i = t->n++;
memset(&t->nodes[i], 0, sizeof(Tnode));
@@ -59,13 +47,25 @@ add(Trie *t, int ni, char c)
return pi;
}
static void
insert(Trie *t, char *key, int klen, char *val, int vlen)
Trie*
trienew(void)
{
int ni, ci;
int i;
Trie *t;
ni = t->root;
t = emalloc(sizeof(*t));
t->cap = 1024;
t->nodes = emalloc(t->cap * sizeof(Tnode));
t->n = 0;
newnode(t);
return t;
}
void
trieput(Trie *t, char *key, int klen, char *val, int vlen)
{
int ni, ci, i;
ni = 0;
for(i = 0; i < klen; i++){
ci = find(t, ni, key[i]);
if(ci < 0)
@@ -73,57 +73,70 @@ insert(Trie *t, char *key, int klen, char *val, int vlen)
ni = ci;
}
free(t->nodes[ni].val);
t->nodes[ni].val = emalloc((ulong)vlen + 1);
t->nodes[ni].val = emalloc(vlen + 1);
memmove(t->nodes[ni].val, val, vlen);
t->nodes[ni].val[vlen] = '\0';
t->nodes[ni].vlen = vlen;
}
static char*
readline(Biobuf *b, char *path)
{
char *line;
errno = 0;
line = Brdstr(b, '\n', 1);
if(errno != 0)
die("can't read %s: %r", path);
return line;
}
/*
* A file holds "key<tab>value" lines; ';' starts a comment. Keys and each
* space-separated word of a value must fit a Str, which is what lookups
* hand back.
*/
Trie*
trieopen(char *path)
{
Trie *t;
Biobuf *b;
char *line, *tab, *key, *val;
Str s;
int klen, vlen;
char *e, *line, *p, *tab;
int len, lineno;
b = Bopen(path, OREAD);
if(b == nil)
die("can't open %s: %s", path, strerror(errno));
t = emalloc(sizeof(*t));
t->cap = 1024;
t->nodes = emalloc((ulong)t->cap * sizeof(Tnode));
t->n = 0;
t->root = newnode(t);
while((line = readline(b, path)) != nil){
vlen = Blinelen(b);
if(memchr(line, '\0', vlen) != nil)
die("NUL in map: %s", path);
if(vlen > 0 && line[vlen-1] == '\r')
line[--vlen] = '\0';
if(line[0] == '\0' || line[0] == ';'){
die("can't open %s: %r", path);
t = trienew();
for(lineno = 1; (line = readline(b, path)) != nil; lineno++){
len = Blinelen(b);
if(memchr(line, '\0', len) != nil)
die("%s:%d: NUL byte", path, lineno);
if(len > 0 && line[len-1] == '\r')
line[--len] = '\0';
if(len == 0 || line[0] == ';'){
free(line);
continue;
}
tab = memchr(line, '\t', vlen);
if(tab == nil || tab == line || tab == line+vlen-1 ||
memchr(tab+1, '\t', line+vlen-(tab+1)) != nil)
die("malformed map: %s", path);
*tab = '\0';
key = line;
klen = tab - line;
if(!sinit(&s, key, klen))
die("invalid or oversized map key: %s", path);
val = tab + 1;
vlen = line + vlen - val;
if(!sinit(&s, val, vlen))
die("invalid or oversized map value: %s", path);
insert(t, key, klen, val, vlen);
tab = memchr(line, '\t', len);
if(tab == nil || tab == line || tab == line+len-1 ||
memchr(tab+1, '\t', line+len-(tab+1)) != nil)
die("%s:%d: expected key, tab, value", path, lineno);
if(!sinit(&s, line, tab-line))
die("%s:%d: invalid or oversized key", path, lineno);
for(p = tab+1; p < line+len; p = e+1){
e = memchr(p, ' ', line+len-p);
if(e == nil)
e = line+len;
if(!sinit(&s, p, e-p))
die("%s:%d: invalid or oversized value", path, lineno);
}
trieput(t, line, tab-line, tab+1, line+len-(tab+1));
free(line);
}
if(Bterm(b) < 0)
die("can't close %s: %s", path, strerror(errno));
die("can't close %s: %r", path);
return t;
}
@@ -140,30 +153,27 @@ trieclose(Trie *t)
free(t);
}
/* A nil trie is an unloaded map: every key misses. */
int
trielookup(Trie *t, char *key, int klen, char **val, int *vlen)
trielookup(Trie *t, Str *key, char **val, int *vlen)
{
int ni;
int i;
char buf[Maxutf];
int i, klen, ni;
if(val != nil)
*val = nil;
if(vlen != nil)
*vlen = 0;
if(t == nil || klen < 0 || (klen > 0 && key == nil))
*val = nil;
*vlen = 0;
if(t == nil)
return TrieMiss;
ni = t->root;
klen = stoutf(key, buf, sizeof buf);
ni = 0;
for(i = 0; i < klen; i++){
ni = find(t, ni, key[i]);
ni = find(t, ni, buf[i]);
if(ni < 0)
return TrieMiss;
}
if(t->nodes[ni].val != nil){
if(val != nil)
*val = t->nodes[ni].val;
if(vlen != nil)
*vlen = t->nodes[ni].vlen;
return TrieExact;
}
return TriePrefix;
if(t->nodes[ni].val == nil)
return TriePrefix;
*val = t->nodes[ni].val;
*vlen = t->nodes[ni].vlen;
return TrieExact;
}