Hojun-Cho/strans
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(data): reject partial and malformed map data

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-14 23:09:55 +09:00
parent 38475318db
commit bfe83d0119
10 changed files with 220 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
dat.h
View File

@@ -87,6 +87,13 @@ struct Trie
int cap;
};
enum
{
TrieMiss,
TriePrefix,
TrieExact,
};
typedef struct Hmap Hmap;
struct Hmap
{

46
dict.c
View File

@@ -1,6 +1,19 @@
#include <errno.h>
#include "dat.h"
#include "fn.h"
static char*
readline(Biobuf *b, char *path)
{
char *line;
errno = 0;
line = Brdstr(b, '\n', 1);
if(errno != 0)
die("can't read %s: %s", path, strerror(errno));
return line;
}
void
dictlookup(Dictreq *req, Dictres *res)
{
@@ -33,7 +46,10 @@ dictlookup(Dictreq *req, Dictres *res)
sp = p;
while(p < e && *p != ' ')
p++;
sinit(&tmp, sp, p - sp);
if(!sinit(&tmp, sp, p - sp)){
res->nkouho = 0;
return;
}
if(req->lang == LangEMOJI || scmp(&tmp, &req->key) != 0)
res->kouho[res->nkouho++] = tmp;
if(p < e)
@@ -63,45 +79,49 @@ dictopen(char *path)
{
Hmap *h;
Biobuf *b;
Str key;
Str key, tmp;
char *line, *tab, *p, *e;
int len, lineno;
b = Bopen(path, OREAD);
if(b == nil)
die("can't open: %s", path);
die("can't open %s: %s", path, strerror(errno));
h = hmapalloc(4096);
lineno = 0;
while((line = Brdstr(b, '\n', 1)) != nil){
while((line = readline(b, path)) != nil){
lineno++;
len = strlen(line);
len = Blinelen(b);
if(memchr(line, '\0', len) != nil)
die("NUL in dictionary: %s:%d", path, lineno);
if(len > 0 && line[len-1] == '\r')
line[--len] = '\0';
if(len == 0 || line[0] == ';'){
free(line);
continue;
}
tab = strchr(line, '\t');
tab = memchr(line, '\t', len);
if(tab == nil || tab == line || tab >= line + len - 1 ||
strchr(tab+1, '\t') != nil)
memchr(tab+1, '\t', line+len-(tab+1)) != nil)
die("malformed dictionary: %s:%d", path, lineno);
*tab = '\0';
if(utflen(line) > Maxrunes)
die("dictionary key too long: %s:%d", path, lineno);
if(!sinit(&key, line, tab-line))
die("invalid or oversized dictionary key: %s:%d",
path, lineno);
for(p = tab+1; p < line+len; p = e+1){
e = memchr(p, ' ', line+len-p);
if(e == nil)
e = line+len;
if(utfnlen(p, e-p) > Maxrunes)
die("dictionary candidate too long: %s:%d", path, lineno);
if(!sinit(&tmp, p, e-p))
die("invalid or oversized dictionary candidate: %s:%d",
path, lineno);
if(e == line+len)
break;
}
sinit(&key, line, tab - line);
hmapset(&h, &key, tab+1, len - (tab - line) - 1);
free(line);
}
Bterm(b);
if(Bterm(b) < 0)
die("can't close %s: %s", path, strerror(errno));
return h;
}

3
fn.h
View File

@@ -1,6 +1,6 @@
void die(char*, ...);
void sinit(Str*, char*, int);
int sinit(Str*, char*, int);
void sclear(Str*);
void sputr(Str*, Rune);
void spopr(Str*);
@@ -17,7 +17,6 @@ int mapget(Trie*, Str*, Str*);
Trie* trieopen(char*);
void trieclose(Trie*);
char* trieget(Trie*, char*, int, int*);
int trielookup(Trie*, char*, int, char**, int*);
Lang* getlang(int);

31
hash.c
View File

@@ -1,3 +1,4 @@
#include <limits.h>
#include "dat.h"
#include "fn.h"
@@ -27,7 +28,9 @@ hmapalloc(int nbuckets)
if(nbuckets < 1)
return nil;
nsz = Tagsize;
store = emalloc(sizeof(*h) + nbuckets * nsz);
if((ulong)nbuckets > (ULONG_MAX-sizeof(*h))/(ulong)nsz)
return nil;
store = emalloc(sizeof(*h) + (ulong)nbuckets * nsz);
h = store;
h->nbs = nbuckets;
h->nsz = nsz;
@@ -56,14 +59,14 @@ hmapget(Hmap *h, Str *key)
if(h == nil || key == nil || key->n < 0 || key->n > Maxrunes)
return nil;
v = h->nodes + (hash(key) % h->nbs) * h->nsz;
v = h->nodes + (hash(key) % h->nbs) * (ulong)h->nsz;
for(;;){
n = (Hnode*)v;
if(n->filled && keycmp(n, key) == 0)
return n;
if(n->next == 0)
break;
v = h->nodes + n->next * h->nsz;
v = h->nodes + (ulong)n->next * h->nsz;
}
return nil;
}
@@ -76,7 +79,7 @@ sdup(Str *s, int *len)
int n;
n = stoutf(s, buf, sizeof(buf));
p = emalloc(n + 1);
p = emalloc((ulong)n + 1);
memmove(p, buf, n);
p[n] = '\0';
*len = n;
@@ -90,7 +93,7 @@ memdup(const char *src, int n)
if(n == 0)
return nil;
p = emalloc(n + 1);
p = emalloc((ulong)n + 1);
memmove(p, src, n);
p[n] = '\0';
return p;
@@ -105,7 +108,7 @@ hmapfree(Hmap *h)
if(h == nil)
return;
for(i = 0; i < h->len; i++){
n = (Hnode*)(h->nodes + i * h->nsz);
n = (Hnode*)(h->nodes + (ulong)i * h->nsz);
if(!n->filled)
continue;
free(n->key);
@@ -122,7 +125,7 @@ hmapset(Hmap **store, Str *key, const char *val, int vlen)
uchar *v;
Hmap *h;
int next;
vlong diff;
ulong diff;
if(store == nil || *store == nil || key == nil ||
key->n < 0 || key->n > Maxrunes || vlen < 0 ||
@@ -130,7 +133,7 @@ hmapset(Hmap **store, Str *key, const char *val, int vlen)
return;
newval = memdup(val, vlen);
h = *store;
v = h->nodes + (hash(key) % h->nbs) * h->nsz;
v = h->nodes + (hash(key) % h->nbs) * (ulong)h->nsz;
for(;;){
n = (Hnode*)v;
next = n->next;
@@ -140,21 +143,25 @@ hmapset(Hmap **store, Str *key, const char *val, int vlen)
goto replace;
if(next == 0)
break;
v = h->nodes + next * h->nsz;
v = h->nodes + (ulong)next * h->nsz;
}
if(h->cap == h->len){
diff = v - h->nodes;
if(h->cap > INT_MAX/2 ||
(ulong)h->cap > (ULONG_MAX-sizeof(*h))/(2*(ulong)h->nsz))
die("hash table is too large");
h->cap *= 2;
*store = erealloc(*store, sizeof(*h) + h->cap * h->nsz);
*store = erealloc(*store,
sizeof(*h) + (ulong)h->cap * h->nsz);
h = *store;
h->nodes = (uchar*)*store + sizeof(*h);
v = h->nodes + diff;
n = (Hnode*)v;
}
n->next = h->len;
memset(h->nodes + h->len * h->nsz, 0, h->nsz);
memset(h->nodes + (ulong)h->len * h->nsz, 0, h->nsz);
h->len++;
v = h->nodes + n->next * h->nsz;
v = h->nodes + (ulong)n->next * h->nsz;
n = (Hnode*)v;
replace:
if(n->filled == 0){

25
str.c
View File

@@ -1,24 +1,33 @@
#include "dat.h"
#include "fn.h"
void
int
sinit(Str *s, char *src, int n)
{
Str tmp = {0};
Rune r;
int len;
if(s == nil)
return 0;
s->n = 0;
if(n < 0 || (n > 0 && src == nil))
return;
while(n > 0 && s->n < Maxrunes){
return 0;
while(n > 0){
if(tmp.n >= Maxrunes)
return 0;
if(!fullrune(src, n))
break;
len = chartorune(&s->r[s->n], src);
if(len > n)
break;
s->n++;
return 0;
len = chartorune(&r, src);
if(len > n || (r == Runeerror && len == 1) ||
(r >= 0xd800 && r <= 0xdfff))
return 0;
tmp.r[tmp.n++] = r;
src += len;
n -= len;
}
*s = tmp;
return 1;
}
void

28
strans.c
View File

@@ -406,17 +406,18 @@ static int
maplookup(Trie *t, Str *key, Str *out)
{
char buf[Maxutf], *v;
int klen, vlen;
int klen, match, vlen;
if(key->n == 0)
if(out != nil)
sclear(out);
if(t == nil || key == nil || key->n == 0)
return 0;
v = nil;
vlen = 0;
klen = stoutf(key, buf, sizeof(buf));
if(!trielookup(t, buf, klen, &v, &vlen))
match = trielookup(t, buf, klen, &v, &vlen);
if(match == TrieMiss)
return 0;
if(match == TrieExact && out != nil && !sinit(out, v, vlen))
return 0;
if(out != nil && v != nil)
sinit(out, v, vlen);
return 1;
}
@@ -1043,16 +1044,17 @@ int
mapget(Trie *t, Str *key, Str *out)
{
char buf[Maxutf], *v;
int klen, vlen;
int klen, match, vlen;
if(key->n == 0)
if(out != nil)
sclear(out);
if(key == nil || out == nil || key->n == 0)
return 0;
klen = stoutf(key, buf, sizeof(buf));
v = trieget(t, buf, klen, &vlen);
if(v == nil)
match = trielookup(t, buf, klen, &v, &vlen);
if(match != TrieExact)
return 0;
sinit(out, v, vlen);
return 1;
return sinit(out, v, vlen);
}
void

6
tests/dict_test.c
View File

@@ -3,6 +3,7 @@
void
dictionary_candidates(struct ct *t)
{
static char malformed[] = { 'o', 'k', ' ', (char)0x80 };
char many[512], item[8];
char *p;
Dictreq req;
@@ -53,6 +54,11 @@ dictionary_candidates(struct ct *t)
CT_EQ_INT(t, 0, scmp(&req.pre, &res.key));
checkstr(t, "first capped candidate", "c00", &res.kouho[0]);
checkstr(t, "last capped candidate", "c31", &res.kouho[31]);
key = mkstr("malformed");
hmapset(&lang->dict, &key, malformed, sizeof malformed);
req.key = key;
dictlookup(&req, &res);
CT_EQ_INT(t, 0, res.nkouho);
cleanup:
hmapfree(lang->dict);
lang->dict = saved;

43
tests/str_test.c
View File

@@ -3,19 +3,48 @@
void
str_init_utf8(struct ct *t)
{
static char badtail[] = { 'a', (char)0x80 };
static char incomplete[] = { (char)0xea, (char)0xb0 };
static char overlong[] = { (char)0xc0, (char)0xaf };
static char outofrange[] = {
(char)0xf4, (char)0x90, (char)0x80, (char)0x80,
};
static char surrogate[] = {
(char)0xed, (char)0xa0, (char)0x80,
};
static char unexpected[] = { (char)0x80 };
static const struct {
char *name;
char *src;
int n;
int ok;
char *want;
} cases[] = {
{ "canonical Runeerror", "\xef\xbf\xbd", 3, 1, "\xef\xbf\xbd" },
{ "incomplete", incomplete, sizeof incomplete, 0, "" },
{ "unexpected continuation", unexpected, sizeof unexpected, 0, "" },
{ "overlong", overlong, sizeof overlong, 0, "" },
{ "surrogate", surrogate, sizeof surrogate, 0, "" },
{ "out of range", outofrange, sizeof outofrange, 0, "" },
{ "bad tail", badtail, sizeof badtail, 0, "" },
};
char full[Maxrunes+2];
char incomplete[] = { (char)0xea, (char)0xb0 };
Str s;
int i;
s = mkstr("A한😀");
CT_EQ_INT(t, 3, s.n);
checkstr(t, "round trip", "A한😀", &s);
sinit(&s, incomplete, sizeof incomplete);
CT_EQ_INT(t, 0, s.n);
for(i = 0; i < nelem(cases); i++){
memset(&s, 0xa5, sizeof s);
if(sinit(&s, cases[i].src, cases[i].n) != cases[i].ok)
CT_ERRORF(t, "%s: wrong validity", cases[i].name);
checkstr(t, cases[i].name, cases[i].want, &s);
}
memset(full, 'a', sizeof full);
full[sizeof full-1] = '\0';
sinit(&s, full, strlen(full));
CT_EQ_INT(t, Maxrunes, s.n);
CT_CHECK(t, !sinit(&s, full, strlen(full)));
CT_EQ_INT(t, 0, s.n);
}
void
@@ -89,9 +118,9 @@ str_invalid_and_full_appends(struct ct *t)
Str s;
int i;
sinit(&s, nil, 1);
CT_CHECK(t, !sinit(&s, nil, 1));
CT_EQ_INT(t, 0, s.n);
sinit(&s, "x", -1);
CT_CHECK(t, !sinit(&s, "x", -1));
CT_EQ_INT(t, 0, s.n);
for(i = 0; i < Maxrunes; i++)
s.r[i] = 'a';

69
tests/trie_test.c
View File

@@ -3,29 +3,41 @@
void
trie_exact_prefix_and_duplicate(struct ct *t)
{
char *v, *sentinel;
static const struct {
char *key;
int klen;
int match;
char *want;
} cases[] = {
{ "a", 1, TrieExact, "alpha" },
{ "dupli", 5, TriePrefix, nil },
{ "duplicate", 9, TrieExact, "second" },
{ "missing", 7, TrieMiss, nil },
{ nil, 1, TrieMiss, nil },
{ "a", -1, TrieMiss, nil },
};
char *v;
Trie *trie;
int n;
int i, match, n;
trie = trieopen("data/trie.map");
v = trieget(trie, "a", 1, &n);
if(!CT_CHECK(t, v != nil))
goto cleanup;
CT_EQ_INT(t, 5, n);
CT_EQ_MEM(t, "alpha", v, n);
sentinel = "unchanged";
v = sentinel;
n = 77;
CT_CHECK(t, trielookup(trie, "dupli", 5, &v, &n));
CT_EQ_PTR(t, sentinel, v);
CT_EQ_INT(t, 77, n);
v = trieget(trie, "duplicate", 9, &n);
if(!CT_CHECK(t, v != nil))
goto cleanup;
CT_EQ_INT(t, 6, n);
CT_EQ_MEM(t, "second", v, n);
CT_EQ_PTR(t, nil, trieget(trie, "missing", 7, &n));
cleanup:
for(i = 0; i < nelem(cases); i++){
v = "unchanged";
n = 77;
match = trielookup(trie, cases[i].key, cases[i].klen, &v, &n);
if(match != cases[i].match){
CT_ERRORF(t, "case %d: want match %d, got %d",
i, cases[i].match, match);
continue;
}
if(cases[i].want == nil){
CT_EQ_PTR(t, nil, v);
CT_EQ_INT(t, 0, n);
}else{
CT_EQ_INT(t, strlen(cases[i].want), n);
CT_EQ_MEM(t, cases[i].want, v, n);
}
}
trieclose(trie);
}
@@ -37,18 +49,15 @@ trie_optional_outputs_and_invalid_lengths(struct ct *t)
int n;
trie = trieopen("data/trie.map");
CT_CHECK(t, trieget(trie, "a", 1, nil) != nil);
v = nil;
CT_CHECK(t, trielookup(trie, "a", 1, &v, nil));
CT_EQ_INT(t, TrieExact, trielookup(trie, "a", 1, &v, nil));
CT_CHECK(t, v != nil);
n = -1;
CT_CHECK(t, trielookup(trie, "a", 1, nil, &n));
CT_EQ_INT(t, TrieExact, trielookup(trie, "a", 1, nil, &n));
CT_EQ_INT(t, 5, n);
CT_CHECK(t, trielookup(trie, "dupli", 5, nil, nil));
CT_EQ_PTR(t, nil, trieget(trie, nil, 1, &n));
CT_EQ_PTR(t, nil, trieget(trie, "a", -1, &n));
CT_CHECK(t, !trielookup(trie, nil, 1, &v, &n));
CT_CHECK(t, !trielookup(trie, "a", -1, &v, &n));
CT_EQ_INT(t, TriePrefix,
trielookup(trie, "dupli", 5, nil, nil));
CT_EQ_INT(t, TriePrefix, trielookup(trie, nil, 0, &v, &n));
CT_EQ_PTR(t, nil, v);
CT_EQ_INT(t, 0, n);
trieclose(trie);
}

87
trie.c
View File

@@ -1,14 +1,33 @@
#include <errno.h>
#include <limits.h>
#include "dat.h"
#include "fn.h"
static char*
readline(Biobuf *b, char *path)
{
char *line;
errno = 0;
line = Brdstr(b, '\n', 1);
if(errno != 0)
die("can't read %s: %s", path, strerror(errno));
return line;
}
static int
newnode(Trie *t)
{
int cap;
int i;
if(t->n >= t->cap){
t->cap *= 2;
t->nodes = erealloc(t->nodes, t->cap * sizeof(Tnode));
if(t->cap > INT_MAX/2 ||
(ulong)t->cap > ULONG_MAX/(2*sizeof(Tnode)))
die("map is too large");
cap = t->cap * 2;
t->nodes = erealloc(t->nodes, (ulong)cap * sizeof(Tnode));
t->cap = cap;
}
i = t->n++;
memset(&t->nodes[i], 0, sizeof(Tnode));
@@ -54,7 +73,7 @@ insert(Trie *t, char *key, int klen, char *val, int vlen)
ni = ci;
}
free(t->nodes[ni].val);
t->nodes[ni].val = emalloc(vlen + 1);
t->nodes[ni].val = emalloc((ulong)vlen + 1);
memmove(t->nodes[ni].val, val, vlen);
t->nodes[ni].val[vlen] = '\0';
t->nodes[ni].vlen = vlen;
@@ -66,41 +85,45 @@ trieopen(char *path)
Trie *t;
Biobuf *b;
char *line, *tab, *key, *val;
Str s;
int klen, vlen;
b = Bopen(path, OREAD);
if(b == nil)
die("can't open: %s", path);
die("can't open %s: %s", path, strerror(errno));
t = emalloc(sizeof(*t));
t->cap = 1024;
t->nodes = emalloc(t->cap * sizeof(Tnode));
t->nodes = emalloc((ulong)t->cap * sizeof(Tnode));
t->n = 0;
t->root = newnode(t);
while((line = Brdstr(b, '\n', 1)) != nil){
vlen = strlen(line);
while((line = readline(b, path)) != nil){
vlen = Blinelen(b);
if(memchr(line, '\0', vlen) != nil)
die("NUL in map: %s", path);
if(vlen > 0 && line[vlen-1] == '\r')
line[--vlen] = '\0';
if(line[0] == '\0' || line[0] == ';'){
free(line);
continue;
}
tab = strchr(line, '\t');
if(tab == nil || tab == line || tab[1] == '\0' ||
strchr(tab+1, '\t') != nil)
tab = memchr(line, '\t', vlen);
if(tab == nil || tab == line || tab == line+vlen-1 ||
memchr(tab+1, '\t', line+vlen-(tab+1)) != nil)
die("malformed map: %s", path);
*tab = '\0';
key = line;
klen = tab - line;
if(utflen(key) > Maxrunes)
die("map key too long: %s", path);
if(!sinit(&s, key, klen))
die("invalid or oversized map key: %s", path);
val = tab + 1;
vlen = strlen(val);
if(utflen(val) > Maxrunes)
die("map value too long: %s", path);
vlen = line + vlen - val;
if(!sinit(&s, val, vlen))
die("invalid or oversized map value: %s", path);
insert(t, key, klen, val, vlen);
free(line);
}
Bterm(b);
if(Bterm(b) < 0)
die("can't close %s: %s", path, strerror(errno));
return t;
}
@@ -117,46 +140,30 @@ trieclose(Trie *t)
free(t);
}
char*
trieget(Trie *t, char *key, int klen, int *vlen)
{
int ni;
int i;
if(t == nil || klen < 0 || (klen > 0 && key == nil))
return nil;
ni = t->root;
for(i = 0; i < klen; i++){
ni = find(t, ni, key[i]);
if(ni < 0)
return nil;
}
if(t->nodes[ni].val == nil)
return nil;
if(vlen != nil)
*vlen = t->nodes[ni].vlen;
return t->nodes[ni].val;
}
int
trielookup(Trie *t, char *key, int klen, char **val, int *vlen)
{
int ni;
int i;
if(val != nil)
*val = nil;
if(vlen != nil)
*vlen = 0;
if(t == nil || klen < 0 || (klen > 0 && key == nil))
return 0;
return TrieMiss;
ni = t->root;
for(i = 0; i < klen; i++){
ni = find(t, ni, key[i]);
if(ni < 0)
return 0;
return TrieMiss;
}
if(t->nodes[ni].val != nil){
if(val != nil)
*val = t->nodes[ni].val;
if(vlen != nil)
*vlen = t->nodes[ni].vlen;
return TrieExact;
}
return 1;
return TriePrefix;
}