From bfe83d0119f01a4e40327c6410ef1db30c759abe Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Fri, 14 Aug 2026 23:09:55 +0900 Subject: [PATCH] fix(data): reject partial and malformed map data --- dat.h | 7 ++++ dict.c | 46 ++++++++++++++++++------- fn.h | 3 +- hash.c | 31 ++++++++++------- str.c | 25 +++++++++----- strans.c | 28 ++++++++------- tests/dict_test.c | 6 ++++ tests/str_test.c | 43 +++++++++++++++++++---- tests/trie_test.c | 69 +++++++++++++++++++++---------------- trie.c | 87 +++++++++++++++++++++++++---------------------- 10 files changed, 220 insertions(+), 125 deletions(-) diff --git a/dat.h b/dat.h index 75bced6..9135766 100644 --- a/dat.h +++ b/dat.h @@ -87,6 +87,13 @@ struct Trie int cap; }; +enum +{ + TrieMiss, + TriePrefix, + TrieExact, +}; + typedef struct Hmap Hmap; struct Hmap { diff --git a/dict.c b/dict.c index b8ab4d0..ddb520d 100644 --- a/dict.c +++ b/dict.c @@ -1,6 +1,19 @@ +#include #include "dat.h" #include "fn.h" +static char* +readline(Biobuf *b, char *path) +{ + char *line; + + errno = 0; + line = Brdstr(b, '\n', 1); + if(errno != 0) + die("can't read %s: %s", path, strerror(errno)); + return line; +} + void dictlookup(Dictreq *req, Dictres *res) { @@ -33,7 +46,10 @@ dictlookup(Dictreq *req, Dictres *res) sp = p; while(p < e && *p != ' ') p++; - sinit(&tmp, sp, p - sp); + if(!sinit(&tmp, sp, p - sp)){ + res->nkouho = 0; + return; + } if(req->lang == LangEMOJI || scmp(&tmp, &req->key) != 0) res->kouho[res->nkouho++] = tmp; if(p < e) @@ -63,45 +79,49 @@ dictopen(char *path) { Hmap *h; Biobuf *b; - Str key; + Str key, tmp; char *line, *tab, *p, *e; int len, lineno; b = Bopen(path, OREAD); if(b == nil) - die("can't open: %s", path); + die("can't open %s: %s", path, strerror(errno)); h = hmapalloc(4096); lineno = 0; - while((line = Brdstr(b, '\n', 1)) != nil){ + while((line = readline(b, path)) != nil){ lineno++; - len = strlen(line); + len = Blinelen(b); + if(memchr(line, '\0', len) != nil) + die("NUL in dictionary: %s:%d", path, lineno); if(len > 0 && line[len-1] == '\r') line[--len] = '\0'; if(len == 0 || line[0] == ';'){ free(line); continue; } - tab = strchr(line, '\t'); + tab = memchr(line, '\t', len); if(tab == nil || tab == line || tab >= line + len - 1 || - strchr(tab+1, '\t') != nil) + memchr(tab+1, '\t', line+len-(tab+1)) != nil) die("malformed dictionary: %s:%d", path, lineno); *tab = '\0'; - if(utflen(line) > Maxrunes) - die("dictionary key too long: %s:%d", path, lineno); + if(!sinit(&key, line, tab-line)) + die("invalid or oversized dictionary key: %s:%d", + path, lineno); for(p = tab+1; p < line+len; p = e+1){ e = memchr(p, ' ', line+len-p); if(e == nil) e = line+len; - if(utfnlen(p, e-p) > Maxrunes) - die("dictionary candidate too long: %s:%d", path, lineno); + if(!sinit(&tmp, p, e-p)) + die("invalid or oversized dictionary candidate: %s:%d", + path, lineno); if(e == line+len) break; } - sinit(&key, line, tab - line); hmapset(&h, &key, tab+1, len - (tab - line) - 1); free(line); } - Bterm(b); + if(Bterm(b) < 0) + die("can't close %s: %s", path, strerror(errno)); return h; } diff --git a/fn.h b/fn.h index d8fc16f..62d88c0 100644 --- a/fn.h +++ b/fn.h @@ -1,6 +1,6 @@ void die(char*, ...); -void sinit(Str*, char*, int); +int sinit(Str*, char*, int); void sclear(Str*); void sputr(Str*, Rune); void spopr(Str*); @@ -17,7 +17,6 @@ int mapget(Trie*, Str*, Str*); Trie* trieopen(char*); void trieclose(Trie*); -char* trieget(Trie*, char*, int, int*); int trielookup(Trie*, char*, int, char**, int*); Lang* getlang(int); diff --git a/hash.c b/hash.c index 9522075..a02983c 100644 --- a/hash.c +++ b/hash.c @@ -1,3 +1,4 @@ +#include #include "dat.h" #include "fn.h" @@ -27,7 +28,9 @@ hmapalloc(int nbuckets) if(nbuckets < 1) return nil; nsz = Tagsize; - store = emalloc(sizeof(*h) + nbuckets * nsz); + if((ulong)nbuckets > (ULONG_MAX-sizeof(*h))/(ulong)nsz) + return nil; + store = emalloc(sizeof(*h) + (ulong)nbuckets * nsz); h = store; h->nbs = nbuckets; h->nsz = nsz; @@ -56,14 +59,14 @@ hmapget(Hmap *h, Str *key) if(h == nil || key == nil || key->n < 0 || key->n > Maxrunes) return nil; - v = h->nodes + (hash(key) % h->nbs) * h->nsz; + v = h->nodes + (hash(key) % h->nbs) * (ulong)h->nsz; for(;;){ n = (Hnode*)v; if(n->filled && keycmp(n, key) == 0) return n; if(n->next == 0) break; - v = h->nodes + n->next * h->nsz; + v = h->nodes + (ulong)n->next * h->nsz; } return nil; } @@ -76,7 +79,7 @@ sdup(Str *s, int *len) int n; n = stoutf(s, buf, sizeof(buf)); - p = emalloc(n + 1); + p = emalloc((ulong)n + 1); memmove(p, buf, n); p[n] = '\0'; *len = n; @@ -90,7 +93,7 @@ memdup(const char *src, int n) if(n == 0) return nil; - p = emalloc(n + 1); + p = emalloc((ulong)n + 1); memmove(p, src, n); p[n] = '\0'; return p; @@ -105,7 +108,7 @@ hmapfree(Hmap *h) if(h == nil) return; for(i = 0; i < h->len; i++){ - n = (Hnode*)(h->nodes + i * h->nsz); + n = (Hnode*)(h->nodes + (ulong)i * h->nsz); if(!n->filled) continue; free(n->key); @@ -122,7 +125,7 @@ hmapset(Hmap **store, Str *key, const char *val, int vlen) uchar *v; Hmap *h; int next; - vlong diff; + ulong diff; if(store == nil || *store == nil || key == nil || key->n < 0 || key->n > Maxrunes || vlen < 0 || @@ -130,7 +133,7 @@ hmapset(Hmap **store, Str *key, const char *val, int vlen) return; newval = memdup(val, vlen); h = *store; - v = h->nodes + (hash(key) % h->nbs) * h->nsz; + v = h->nodes + (hash(key) % h->nbs) * (ulong)h->nsz; for(;;){ n = (Hnode*)v; next = n->next; @@ -140,21 +143,25 @@ hmapset(Hmap **store, Str *key, const char *val, int vlen) goto replace; if(next == 0) break; - v = h->nodes + next * h->nsz; + v = h->nodes + (ulong)next * h->nsz; } if(h->cap == h->len){ diff = v - h->nodes; + if(h->cap > INT_MAX/2 || + (ulong)h->cap > (ULONG_MAX-sizeof(*h))/(2*(ulong)h->nsz)) + die("hash table is too large"); h->cap *= 2; - *store = erealloc(*store, sizeof(*h) + h->cap * h->nsz); + *store = erealloc(*store, + sizeof(*h) + (ulong)h->cap * h->nsz); h = *store; h->nodes = (uchar*)*store + sizeof(*h); v = h->nodes + diff; n = (Hnode*)v; } n->next = h->len; - memset(h->nodes + h->len * h->nsz, 0, h->nsz); + memset(h->nodes + (ulong)h->len * h->nsz, 0, h->nsz); h->len++; - v = h->nodes + n->next * h->nsz; + v = h->nodes + (ulong)n->next * h->nsz; n = (Hnode*)v; replace: if(n->filled == 0){ diff --git a/str.c b/str.c index 2d694b3..173bd4d 100644 --- a/str.c +++ b/str.c @@ -1,24 +1,33 @@ #include "dat.h" #include "fn.h" -void +int sinit(Str *s, char *src, int n) { + Str tmp = {0}; + Rune r; int len; + if(s == nil) + return 0; s->n = 0; if(n < 0 || (n > 0 && src == nil)) - return; - while(n > 0 && s->n < Maxrunes){ + return 0; + while(n > 0){ + if(tmp.n >= Maxrunes) + return 0; if(!fullrune(src, n)) - break; - len = chartorune(&s->r[s->n], src); - if(len > n) - break; - s->n++; + return 0; + len = chartorune(&r, src); + if(len > n || (r == Runeerror && len == 1) || + (r >= 0xd800 && r <= 0xdfff)) + return 0; + tmp.r[tmp.n++] = r; src += len; n -= len; } + *s = tmp; + return 1; } void diff --git a/strans.c b/strans.c index ca26959..222f8cd 100644 --- a/strans.c +++ b/strans.c @@ -406,17 +406,18 @@ static int maplookup(Trie *t, Str *key, Str *out) { char buf[Maxutf], *v; - int klen, vlen; + int klen, match, vlen; - if(key->n == 0) + if(out != nil) + sclear(out); + if(t == nil || key == nil || key->n == 0) return 0; - v = nil; - vlen = 0; klen = stoutf(key, buf, sizeof(buf)); - if(!trielookup(t, buf, klen, &v, &vlen)) + match = trielookup(t, buf, klen, &v, &vlen); + if(match == TrieMiss) + return 0; + if(match == TrieExact && out != nil && !sinit(out, v, vlen)) return 0; - if(out != nil && v != nil) - sinit(out, v, vlen); return 1; } @@ -1043,16 +1044,17 @@ int mapget(Trie *t, Str *key, Str *out) { char buf[Maxutf], *v; - int klen, vlen; + int klen, match, vlen; - if(key->n == 0) + if(out != nil) + sclear(out); + if(key == nil || out == nil || key->n == 0) return 0; klen = stoutf(key, buf, sizeof(buf)); - v = trieget(t, buf, klen, &vlen); - if(v == nil) + match = trielookup(t, buf, klen, &v, &vlen); + if(match != TrieExact) return 0; - sinit(out, v, vlen); - return 1; + return sinit(out, v, vlen); } void diff --git a/tests/dict_test.c b/tests/dict_test.c index 9868242..f413107 100644 --- a/tests/dict_test.c +++ b/tests/dict_test.c @@ -3,6 +3,7 @@ void dictionary_candidates(struct ct *t) { + static char malformed[] = { 'o', 'k', ' ', (char)0x80 }; char many[512], item[8]; char *p; Dictreq req; @@ -53,6 +54,11 @@ dictionary_candidates(struct ct *t) CT_EQ_INT(t, 0, scmp(&req.pre, &res.key)); checkstr(t, "first capped candidate", "c00", &res.kouho[0]); checkstr(t, "last capped candidate", "c31", &res.kouho[31]); + key = mkstr("malformed"); + hmapset(&lang->dict, &key, malformed, sizeof malformed); + req.key = key; + dictlookup(&req, &res); + CT_EQ_INT(t, 0, res.nkouho); cleanup: hmapfree(lang->dict); lang->dict = saved; diff --git a/tests/str_test.c b/tests/str_test.c index 0794365..4cc89d8 100644 --- a/tests/str_test.c +++ b/tests/str_test.c @@ -3,19 +3,48 @@ void str_init_utf8(struct ct *t) { + static char badtail[] = { 'a', (char)0x80 }; + static char incomplete[] = { (char)0xea, (char)0xb0 }; + static char overlong[] = { (char)0xc0, (char)0xaf }; + static char outofrange[] = { + (char)0xf4, (char)0x90, (char)0x80, (char)0x80, + }; + static char surrogate[] = { + (char)0xed, (char)0xa0, (char)0x80, + }; + static char unexpected[] = { (char)0x80 }; + static const struct { + char *name; + char *src; + int n; + int ok; + char *want; + } cases[] = { + { "canonical Runeerror", "\xef\xbf\xbd", 3, 1, "\xef\xbf\xbd" }, + { "incomplete", incomplete, sizeof incomplete, 0, "" }, + { "unexpected continuation", unexpected, sizeof unexpected, 0, "" }, + { "overlong", overlong, sizeof overlong, 0, "" }, + { "surrogate", surrogate, sizeof surrogate, 0, "" }, + { "out of range", outofrange, sizeof outofrange, 0, "" }, + { "bad tail", badtail, sizeof badtail, 0, "" }, + }; char full[Maxrunes+2]; - char incomplete[] = { (char)0xea, (char)0xb0 }; Str s; + int i; s = mkstr("A한😀"); CT_EQ_INT(t, 3, s.n); checkstr(t, "round trip", "A한😀", &s); - sinit(&s, incomplete, sizeof incomplete); - CT_EQ_INT(t, 0, s.n); + for(i = 0; i < nelem(cases); i++){ + memset(&s, 0xa5, sizeof s); + if(sinit(&s, cases[i].src, cases[i].n) != cases[i].ok) + CT_ERRORF(t, "%s: wrong validity", cases[i].name); + checkstr(t, cases[i].name, cases[i].want, &s); + } memset(full, 'a', sizeof full); full[sizeof full-1] = '\0'; - sinit(&s, full, strlen(full)); - CT_EQ_INT(t, Maxrunes, s.n); + CT_CHECK(t, !sinit(&s, full, strlen(full))); + CT_EQ_INT(t, 0, s.n); } void @@ -89,9 +118,9 @@ str_invalid_and_full_appends(struct ct *t) Str s; int i; - sinit(&s, nil, 1); + CT_CHECK(t, !sinit(&s, nil, 1)); CT_EQ_INT(t, 0, s.n); - sinit(&s, "x", -1); + CT_CHECK(t, !sinit(&s, "x", -1)); CT_EQ_INT(t, 0, s.n); for(i = 0; i < Maxrunes; i++) s.r[i] = 'a'; diff --git a/tests/trie_test.c b/tests/trie_test.c index 2d517cd..18bb6d7 100644 --- a/tests/trie_test.c +++ b/tests/trie_test.c @@ -3,29 +3,41 @@ void trie_exact_prefix_and_duplicate(struct ct *t) { - char *v, *sentinel; + static const struct { + char *key; + int klen; + int match; + char *want; + } cases[] = { + { "a", 1, TrieExact, "alpha" }, + { "dupli", 5, TriePrefix, nil }, + { "duplicate", 9, TrieExact, "second" }, + { "missing", 7, TrieMiss, nil }, + { nil, 1, TrieMiss, nil }, + { "a", -1, TrieMiss, nil }, + }; + char *v; Trie *trie; - int n; + int i, match, n; trie = trieopen("data/trie.map"); - v = trieget(trie, "a", 1, &n); - if(!CT_CHECK(t, v != nil)) - goto cleanup; - CT_EQ_INT(t, 5, n); - CT_EQ_MEM(t, "alpha", v, n); - sentinel = "unchanged"; - v = sentinel; - n = 77; - CT_CHECK(t, trielookup(trie, "dupli", 5, &v, &n)); - CT_EQ_PTR(t, sentinel, v); - CT_EQ_INT(t, 77, n); - v = trieget(trie, "duplicate", 9, &n); - if(!CT_CHECK(t, v != nil)) - goto cleanup; - CT_EQ_INT(t, 6, n); - CT_EQ_MEM(t, "second", v, n); - CT_EQ_PTR(t, nil, trieget(trie, "missing", 7, &n)); -cleanup: + for(i = 0; i < nelem(cases); i++){ + v = "unchanged"; + n = 77; + match = trielookup(trie, cases[i].key, cases[i].klen, &v, &n); + if(match != cases[i].match){ + CT_ERRORF(t, "case %d: want match %d, got %d", + i, cases[i].match, match); + continue; + } + if(cases[i].want == nil){ + CT_EQ_PTR(t, nil, v); + CT_EQ_INT(t, 0, n); + }else{ + CT_EQ_INT(t, strlen(cases[i].want), n); + CT_EQ_MEM(t, cases[i].want, v, n); + } + } trieclose(trie); } @@ -37,18 +49,15 @@ trie_optional_outputs_and_invalid_lengths(struct ct *t) int n; trie = trieopen("data/trie.map"); - CT_CHECK(t, trieget(trie, "a", 1, nil) != nil); - v = nil; - CT_CHECK(t, trielookup(trie, "a", 1, &v, nil)); + CT_EQ_INT(t, TrieExact, trielookup(trie, "a", 1, &v, nil)); CT_CHECK(t, v != nil); - n = -1; - CT_CHECK(t, trielookup(trie, "a", 1, nil, &n)); + CT_EQ_INT(t, TrieExact, trielookup(trie, "a", 1, nil, &n)); CT_EQ_INT(t, 5, n); - CT_CHECK(t, trielookup(trie, "dupli", 5, nil, nil)); - CT_EQ_PTR(t, nil, trieget(trie, nil, 1, &n)); - CT_EQ_PTR(t, nil, trieget(trie, "a", -1, &n)); - CT_CHECK(t, !trielookup(trie, nil, 1, &v, &n)); - CT_CHECK(t, !trielookup(trie, "a", -1, &v, &n)); + CT_EQ_INT(t, TriePrefix, + trielookup(trie, "dupli", 5, nil, nil)); + CT_EQ_INT(t, TriePrefix, trielookup(trie, nil, 0, &v, &n)); + CT_EQ_PTR(t, nil, v); + CT_EQ_INT(t, 0, n); trieclose(trie); } diff --git a/trie.c b/trie.c index e3f1847..4fcb017 100644 --- a/trie.c +++ b/trie.c @@ -1,14 +1,33 @@ +#include +#include #include "dat.h" #include "fn.h" +static char* +readline(Biobuf *b, char *path) +{ + char *line; + + errno = 0; + line = Brdstr(b, '\n', 1); + if(errno != 0) + die("can't read %s: %s", path, strerror(errno)); + return line; +} + static int newnode(Trie *t) { + int cap; int i; if(t->n >= t->cap){ - t->cap *= 2; - t->nodes = erealloc(t->nodes, t->cap * sizeof(Tnode)); + if(t->cap > INT_MAX/2 || + (ulong)t->cap > ULONG_MAX/(2*sizeof(Tnode))) + die("map is too large"); + cap = t->cap * 2; + t->nodes = erealloc(t->nodes, (ulong)cap * sizeof(Tnode)); + t->cap = cap; } i = t->n++; memset(&t->nodes[i], 0, sizeof(Tnode)); @@ -54,7 +73,7 @@ insert(Trie *t, char *key, int klen, char *val, int vlen) ni = ci; } free(t->nodes[ni].val); - t->nodes[ni].val = emalloc(vlen + 1); + t->nodes[ni].val = emalloc((ulong)vlen + 1); memmove(t->nodes[ni].val, val, vlen); t->nodes[ni].val[vlen] = '\0'; t->nodes[ni].vlen = vlen; @@ -66,41 +85,45 @@ trieopen(char *path) Trie *t; Biobuf *b; char *line, *tab, *key, *val; + Str s; int klen, vlen; b = Bopen(path, OREAD); if(b == nil) - die("can't open: %s", path); + die("can't open %s: %s", path, strerror(errno)); t = emalloc(sizeof(*t)); t->cap = 1024; - t->nodes = emalloc(t->cap * sizeof(Tnode)); + t->nodes = emalloc((ulong)t->cap * sizeof(Tnode)); t->n = 0; t->root = newnode(t); - while((line = Brdstr(b, '\n', 1)) != nil){ - vlen = strlen(line); + while((line = readline(b, path)) != nil){ + vlen = Blinelen(b); + if(memchr(line, '\0', vlen) != nil) + die("NUL in map: %s", path); if(vlen > 0 && line[vlen-1] == '\r') line[--vlen] = '\0'; if(line[0] == '\0' || line[0] == ';'){ free(line); continue; } - tab = strchr(line, '\t'); - if(tab == nil || tab == line || tab[1] == '\0' || - strchr(tab+1, '\t') != nil) + tab = memchr(line, '\t', vlen); + if(tab == nil || tab == line || tab == line+vlen-1 || + memchr(tab+1, '\t', line+vlen-(tab+1)) != nil) die("malformed map: %s", path); *tab = '\0'; key = line; klen = tab - line; - if(utflen(key) > Maxrunes) - die("map key too long: %s", path); + if(!sinit(&s, key, klen)) + die("invalid or oversized map key: %s", path); val = tab + 1; - vlen = strlen(val); - if(utflen(val) > Maxrunes) - die("map value too long: %s", path); + vlen = line + vlen - val; + if(!sinit(&s, val, vlen)) + die("invalid or oversized map value: %s", path); insert(t, key, klen, val, vlen); free(line); } - Bterm(b); + if(Bterm(b) < 0) + die("can't close %s: %s", path, strerror(errno)); return t; } @@ -117,46 +140,30 @@ trieclose(Trie *t) free(t); } -char* -trieget(Trie *t, char *key, int klen, int *vlen) -{ - int ni; - int i; - - if(t == nil || klen < 0 || (klen > 0 && key == nil)) - return nil; - ni = t->root; - for(i = 0; i < klen; i++){ - ni = find(t, ni, key[i]); - if(ni < 0) - return nil; - } - if(t->nodes[ni].val == nil) - return nil; - if(vlen != nil) - *vlen = t->nodes[ni].vlen; - return t->nodes[ni].val; -} - int trielookup(Trie *t, char *key, int klen, char **val, int *vlen) { int ni; int i; + if(val != nil) + *val = nil; + if(vlen != nil) + *vlen = 0; if(t == nil || klen < 0 || (klen > 0 && key == nil)) - return 0; + return TrieMiss; ni = t->root; for(i = 0; i < klen; i++){ ni = find(t, ni, key[i]); if(ni < 0) - return 0; + return TrieMiss; } if(t->nodes[ni].val != nil){ if(val != nil) *val = t->nodes[ni].val; if(vlen != nil) *vlen = t->nodes[ni].vlen; + return TrieExact; } - return 1; + return TriePrefix; }