fix: preserve complete Japanese readings

dat.h

@@ -110,7 +110,8 @@ struct Im
 {
 	Lang	*l;
 	Str	pre;
-	Str	raw;		/* physical keys for the current Telex preedit */
+	/* Telex history, or the short pending romaji in a Japanese mode. */
+	Str	raw;
 	Str	kouho[Maxkouho];
 	int	nkouho;
 	int	sel;

strans.c

@@ -5,6 +5,9 @@ static Im im;
 static int popup = 0;
 static int visible = 0;
 static void dictqmap(Im*);
+static void dictqjp(Im*);
+static void backjp(Im*);
+static int maplookup(Trie*, Str*, Str*);
 
 typedef struct Search Search;
 struct Search
@@ -23,8 +26,8 @@ backmap(Im *im)
 
 Lang langs[] = {
 	{LangEN,    "english", nil,      transmap, backmap, dictqmap, nil, nil},
-	{LangJP,    "hira",    "kanji",  transmap, backmap, dictqmap, nil, nil},
+	{LangJP,    "hira",    "kanji",  transmap, backjp,  dictqjp,  nil, nil},
-	{LangJPK,   "kata",    "kanji",  transmap, backmap, dictqmap, nil, nil},
+	{LangJPK,   "kata",    "kanji",  transmap, backjp,  dictqjp,  nil, nil},
 	{LangKO,    "hangul",  nil,      transko,  backko,  dictqmap, nil, nil},
 	{LangEMOJI, nil,       "emoji",  nil,      nil,     nil,      nil, nil},
 	{LangVI,    "telex",   nil,      transvi,  backvi,  dictqmap, nil, nil},
@@ -38,6 +41,48 @@ clearkouho(void)
 	im.sel = -1;
 }
 
+static int
+isjp(Im *p)
+{
+	return p->l->lang == LangJP || p->l->lang == LangJPK;
+}
+
+/*
+ * Japanese keeps completed kana in pre and an ambiguous romaji syllable in
+ * raw.  A mapped raw suffix is part of the reading even while it remains
+ * pending (notably n, which may still grow into nya).
+ */
+static int
+jpreading(Im *p, Str *s)
+{
+	Str mapped;
+
+	*s = p->pre;
+	if(p->raw.n == 0)
+		return 1;
+	if(mapget(p->l->map, &p->raw, &mapped)){
+		sappend(s, &mapped);
+		return 1;
+	}
+	sappend(s, &p->raw);
+	return 0;
+}
+
+static int
+haspre(Im *p)
+{
+	return p->pre.n != 0 || (isjp(p) && p->raw.n != 0);
+}
+
+static void
+impre(Str *s)
+{
+	if(isjp(&im))
+		jpreading(&im, s);
+	else
+		*s = im.pre;
+}
+
 static void
 setkouho(Dictres *res)
 {
@@ -55,14 +100,18 @@ show(void)
 {
 	static Drawcmd old;
 	Drawcmd dc;
-	Str *pre;
+	Str pre;
 	int i, first, n;
 
 	sclear(&dc.pre);
-	pre = search.on ? &search.text : &im.pre;
+	if(search.on)
+		pre = search.text;
+	else
+		impre(&pre);
 	if(popup){
-		if(search.on || im.l->map == nil || !mapget(im.l->map, pre, &dc.pre))
+		if(search.on || isjp(&im) || im.l->map == nil ||
-			dc.pre = *pre;
+		   !mapget(im.l->map, &pre, &dc.pre))
+			dc.pre = pre;
 	}
 	first = im.sel >= Maxdisp ? im.sel - Maxdisp + 1 : 0;
 	n = im.nkouho - first;
@@ -96,7 +145,10 @@ dictsend(Im *im, Str *key)
 
 	req.key = *key;
 	req.lang = im->l->lang;
-	req.pre = im->pre;
+	if(isjp(im))
+		jpreading(im, &req.pre);
+	else
+		req.pre = im->pre;
 	channbsend(dictreqc, &req);
 }
 
@@ -112,6 +164,18 @@ dictqmap(Im *im)
 	dictsend(im, &dict);
 }
 
+static void
+dictqjp(Im *im)
+{
+	Str reading;
+
+	clearkouho();
+	show();
+	if(im->l->dict == nil || !jpreading(im, &reading) || reading.n == 0)
+		return;
+	dictsend(im, &reading);
+}
+
 static int
 setlang(int c)
 {
@@ -129,7 +193,10 @@ commit(Str *com)
 {
 	Str val;
 
-	if(im.l->map != nil && mapget(im.l->map, &im.pre, &val))
+	if(isjp(&im)){
+		jpreading(&im, &val);
+		sappend(com, &val);
+	}else if(im.l->map != nil && mapget(im.l->map, &im.pre, &val))
 		sappend(com, &val);
 	else
 		sappend(com, &im.pre);
@@ -137,6 +204,51 @@ commit(Str *com)
 	sclear(&im.raw);
 }
 
+static void
+backjp(Im *im)
+{
+	if(im->raw.n != 0)
+		spopr(&im->raw);
+	else
+		spopr(&im->pre);
+}
+
+static int
+dotransjp(Rune c, Str *com)
+{
+	Str key, mapped;
+
+	if(c == '\'' && im.raw.n == 1 && im.raw.r[0] == 'n'){
+		if(mapget(im.l->map, &im.raw, &mapped))
+			sappend(&im.pre, &mapped);
+		sclear(&im.raw);
+		dictqjp(&im);
+		return 1;
+	}
+	key = im.raw;
+	if(key.n < Maxrunes)
+		sputr(&key, c);
+	if(maplookup(im.l->map, &key, &mapped)){
+		im.raw = key;
+		dictqjp(&im);
+		return 1;
+	}
+	if(im.raw.n != 0 && mapget(im.l->map, &im.raw, &mapped)){
+		sappend(&im.pre, &mapped);
+		sclear(&im.raw);
+		sclear(&key);
+		sputr(&key, c);
+		if(maplookup(im.l->map, &key, &mapped)){
+			im.raw = key;
+			dictqjp(&im);
+			return 1;
+		}
+	}
+	commit(com);
+	clearkouho();
+	return 0;
+}
+
 static int
 dotrans(Rune c, Str *com)
 {
@@ -144,6 +256,8 @@ dotrans(Rune c, Str *com)
 	Dictreq req;
 	Str mapped;
 
+	if(isjp(&im))
+		return dotransjp(c, com);
 	e = im.l->trans(&im, c);
 	if(im.l->lang == LangVI){
 		if(e.s.n > 0 || !e.eat)
@@ -347,7 +461,7 @@ endsearch(void)
 static void
 startsearch(Str *com)
 {
-	if(im.pre.n > 0)
+	if(haspre(&im))
 		commit(com);
 	sclear(&im.pre);
 	sclear(&im.raw);
@@ -495,7 +609,7 @@ keystroke(u32int ks, u32int mod, Str *com)
 			reset();
 			return 1;
 		}
-		if(im.pre.n > 0){
+		if(haspre(&im)){
 			commit(com);
 			reset();
 			return 1;
@@ -503,10 +617,10 @@ keystroke(u32int ks, u32int mod, Str *com)
 		return 0;
 	}
 	if(ks == Kback){
-		if(im.pre.n == 0)
+		if(!haspre(&im))
 			return 0;
 		im.l->back(&im);
-		if(im.pre.n == 0){
+		if(!haspre(&im)){
 			reset();
 			return 1;
 		}
@@ -514,7 +628,7 @@ keystroke(u32int ks, u32int mod, Str *com)
 		return 1;
 	}
 	if(ks == Kesc){
-		if(im.pre.n == 0)
+		if(!haspre(&im))
 			return 0;
 		reset();
 		return 1;
@@ -535,9 +649,14 @@ keystroke(u32int ks, u32int mod, Str *com)
 			show();
 			return 1;
 		}
-		reset();
+		if(getlang(c) != nil){
-		if(setlang(c))
+			if(isjp(&im) && haspre(&im))
+				commit(com);
+			reset();
+			setlang(c);
 			return 1;
+		}
+		reset();
 		return 0;
 	}
 	if(ks == '0' && im.nkouho > 0){
@@ -546,7 +665,7 @@ keystroke(u32int ks, u32int mod, Str *com)
 		return 1;
 	}
 	if(ks > 0x7f || ks == ' '){
-		if(im.pre.n == 0)
+		if(!haspre(&im))
 			return 0;
 		commit(com);
 		sputr(com, ks);
@@ -575,16 +694,23 @@ imhandlekey(Keyreq *kr)
 	sclear(&res.commit);
 	sclear(&res.preedit);
 	res.eaten = keystroke(kr->ks, kr->mod, &res.commit);
-	if(kr->want)
+	if(kr->want){
-		res.preedit = search.on ? search.text : im.pre;
+		if(search.on)
+			res.preedit = search.text;
+		else
+			impre(&res.preedit);
+	}
 	chansend(kr->reply, &res);
 }
 
 static void
 dictresult(Dictres *res)
 {
+	Str pre;
+
+	impre(&pre);
 	if(search.on || res->lang != im.l->lang ||
-	   scmp(&res->key, &im.pre) != 0)
+	   scmp(&res->key, &pre) != 0)
 		return;
 	setkouho(res);
 	show();

tests/engine_test.c

@@ -2,6 +2,8 @@
 #include "../strans.c"
 #include "test.h"
 
+static int typekeys(struct ct*, char*, Str*);
+
 void
 vietnamese_state_lifetime(struct ct *t)
 {
@@ -73,7 +75,7 @@ engine_backspace_clears_candidates(struct ct *t)
 	im.sel = 0;
 	CT_CHECK(t, keystroke(Kback, 0, &com));
 	checkstr(t, "backspace commit", "", &com);
-	checkstr(t, "raw preedit after backspace", "n", &im.pre);
+	checkstr(t, "pending romaji after backspace", "n", &im.raw);
 	shown = shownpre(&im);
 	checkstr(t, "shown preedit after backspace", "ん", &shown);
 	CT_EQ_INT(t, 0, im.nkouho);
@@ -113,10 +115,11 @@ engine_commit_contract(struct ct *t)
 
 	init();
 	im.l = getlang(LangJP);
-	im.pre = mkstr("ka");
 	im.nkouho = 1;
 	im.sel = -1;
 	sclear(&com);
+	if(!typekeys(t, "ka", &com))
+		return;
 	CT_CHECK(t, keystroke(Kret, 0, &com));
 	checkstr(t, "return commit", "か", &com);
 	CT_EQ_INT(t, 0, im.pre.n);
@@ -126,12 +129,13 @@ engine_commit_contract(struct ct *t)
 
 	init();
 	im.l = getlang(LangJP);
-	im.pre = mkstr("ka");
+	sclear(&com);
+	if(!typekeys(t, "ka", &com))
+		return;
 	im.kouho[0] = mkstr("c1");
 	im.kouho[1] = mkstr("c2");
 	im.nkouho = 2;
 	im.sel = 1;
-	sclear(&com);
 	CT_CHECK(t, keystroke(Kret, 0, &com));
 	checkstr(t, "selected candidate", "c2", &com);
 	CT_EQ_INT(t, 0, im.pre.n);
@@ -140,10 +144,11 @@ engine_commit_contract(struct ct *t)
 
 	init();
 	im.l = getlang(LangJP);
-	im.pre = mkstr("ka");
+	sclear(&com);
+	if(!typekeys(t, "ka", &com))
+		return;
 	im.nkouho = 1;
 	im.sel = -1;
-	sclear(&com);
 	CT_CHECK(t, !keystroke('q', 0, &com));
 	checkstr(t, "uneaten-key commit", "か", &com);
 	CT_EQ_INT(t, 0, im.pre.n);
@@ -153,8 +158,9 @@ engine_commit_contract(struct ct *t)
 
 	init();
 	im.l = getlang(LangJP);
-	im.pre = mkstr("ka");
 	sclear(&com);
+	if(!typekeys(t, "ka", &com))
+		return;
 	CT_CHECK(t, keystroke(0xf008, 0, &com));
 	CT_EQ_INT(t, 2, com.n);
 	CT_EQ_UINT(t, 0x304b, com.r[0]);
@@ -346,6 +352,135 @@ typekeys(struct ct *t, char *keys, Str *com)
 	return 1;
 }
 
+void
+engine_japanese_readings(struct ct *t)
+{
+	static const struct {
+		char *keys;
+		char *want;
+	} cases[] = {
+		{ "nya", "にゃ" },
+		{ "n'ya", "んや" },
+		{ "nnya", "んにゃ" },
+		{ "kin'youbi", "きんようび" },
+	};
+	Str com, shown;
+	int i;
+
+	for(i = 0; i < nelem(cases); i++){
+		init();
+		im.l = getlang(LangJP);
+		sclear(&com);
+		if(!typekeys(t, cases[i].keys, &com))
+			continue;
+		checkstr(t, cases[i].keys, "", &com);
+		shown = shownpre(&im);
+		checkstr(t, cases[i].keys, cases[i].want, &shown);
+		CT_CHECK(t, keystroke(Kret, 0, &com));
+		checkstr(t, "committed reading", cases[i].want, &com);
+	}
+}
+
+static int
+newestrequest(Dictreq *req)
+{
+	int n;
+
+	n = 0;
+	while(channbrecv(dictreqc, req) > 0)
+		n++;
+	return n;
+}
+
+void
+engine_japanese_candidates(struct ct *t)
+{
+	Dictreq req;
+	Dictres res;
+	Hmap *saved;
+	Lang *jp;
+	Str com, key, shown;
+
+	jp = getlang(LangJP);
+	saved = jp->dict;
+	jp->dict = hmapalloc(8);
+	key = mkstr("かんじ");
+	hmapset(&jp->dict, &key, "漢字 幹事", strlen("漢字 幹事"));
+	init();
+	im.l = jp;
+	sclear(&com);
+	if(!typekeys(t, "kanji", &com))
+		goto cleanup;
+	shown = shownpre(&im);
+	checkstr(t, "complete Japanese reading", "かんじ", &shown);
+	if(!CT_CHECK(t, newestrequest(&req) > 0))
+		goto cleanup;
+	checkstr(t, "dictionary lookup key", "かんじ", &req.key);
+	checkstr(t, "dictionary request identity", "かんじ", &req.pre);
+	dictlookup(&req, &res);
+	dictresult(&res);
+	if(!CT_EQ_INT(t, 2, im.nkouho))
+		goto cleanup;
+	checkstr(t, "candidate one", "漢字", &im.kouho[0]);
+	checkstr(t, "candidate two", "幹事", &im.kouho[1]);
+	CT_CHECK(t, keystroke(Kdown, 0, &com));
+	CT_EQ_INT(t, 0, im.sel);
+	CT_CHECK(t, keystroke(Kdown, 0, &com));
+	CT_EQ_INT(t, 1, im.sel);
+	CT_CHECK(t, keystroke(Kret, 0, &com));
+	checkstr(t, "selected Kanji", "幹事", &com);
+cleanup:
+	newestrequest(&req);
+	hmapfree(jp->dict);
+	jp->dict = saved;
+}
+
+void
+engine_japanese_backspace_and_boundaries(struct ct *t)
+{
+	Str com, shown;
+
+	init();
+	im.l = getlang(LangJP);
+	sclear(&com);
+	if(typekeys(t, "kanji", &com)){
+		CT_CHECK(t, keystroke(Kback, 0, &com));
+		shown = shownpre(&im);
+		checkstr(t, "backspace pending i", "かんj", &shown);
+		CT_CHECK(t, keystroke(Kback, 0, &com));
+		shown = shownpre(&im);
+		checkstr(t, "backspace pending j", "かん", &shown);
+		CT_CHECK(t, keystroke(Kback, 0, &com));
+		shown = shownpre(&im);
+		checkstr(t, "backspace accumulated kana", "か", &shown);
+	}
+
+	init();
+	im.l = getlang(LangJP);
+	sclear(&com);
+	if(typekeys(t, "kanji", &com)){
+		CT_CHECK(t, keystroke('k', Mctrl, &com));
+		checkstr(t, "Hiragana mode boundary", "かんじ", &com);
+		CT_EQ_INT(t, LangJPK, im.l->lang);
+		CT_EQ_INT(t, 0, im.pre.n);
+		CT_EQ_INT(t, 0, im.raw.n);
+		sclear(&com);
+		if(typekeys(t, "ka", &com)){
+			CT_CHECK(t, keystroke('n', Mctrl, &com));
+			checkstr(t, "Katakana mode boundary", "カ", &com);
+			CT_EQ_INT(t, LangJP, im.l->lang);
+		}
+	}
+
+	init();
+	im.l = getlang(LangJP);
+	sclear(&com);
+	if(typekeys(t, "kanji", &com)){
+		CT_CHECK(t, keystroke(' ', 0, &com));
+		checkstr(t, "real input boundary", "かんじ ", &com);
+	}
+}
+
 void
 engine_emoji_single_candidate(struct ct *t)
 {
@@ -556,7 +691,8 @@ engine_emoji_preedit_languages(struct ct *t)
 	sclear(&com);
 	if(typekeys(t, "ka", &com)){
 		CT_CHECK(t, !keystroke(Kmodfirst+2, 0, &com));
-		checkstr(t, "modifier preserves Japanese", "ka", &im.pre);
+		shown = shownpre(&im);
+		checkstr(t, "modifier preserves Japanese", "か", &shown);
 		CT_CHECK(t, keystroke('E', Mctrl|Mshift, &com));
 		checkstr(t, "committed Japanese preedit", "か", &com);
 		CT_CHECK(t, search.on);

tests/test.h

@@ -33,6 +33,9 @@ void	engine_selects_visible_candidate(struct ct*);
 void	engine_commit_contract(struct ct*);
 void	engine_language_switch_state(struct ct*);
 void	engine_telex_history_bound(struct ct*);
+void	engine_japanese_readings(struct ct*);
+void	engine_japanese_candidates(struct ct*);
+void	engine_japanese_backspace_and_boundaries(struct ct*);
 void	engine_emoji_single_candidate(struct ct*);
 void	engine_emoji_queries(struct ct*);
 void	engine_emoji_digit_aliases(struct ct*);

tests/test_util.c

@@ -24,8 +24,18 @@ checkstr(struct ct *t, char *where, char *want, Str *got)
 Str
 shownpre(Im *state)
 {
-	Str shown;
+	Str mapped, shown;
 
+	if(state->l->lang == LangJP || state->l->lang == LangJPK){
+		shown = state->pre;
+		if(state->raw.n == 0)
+			return shown;
+		if(mapget(state->l->map, &state->raw, &mapped))
+			sappend(&shown, &mapped);
+		else
+			sappend(&shown, &state->raw);
+		return shown;
+	}
 	if(state->l->map != nil && mapget(state->l->map, &state->pre, &shown))
 		return shown;
 	return state->pre;

tests/trie_test.c

@@ -76,10 +76,14 @@ transmap_states(struct ct *t)
 	};
 	Emit e;
 	Im state;
+	Trie *fixture, *saved;
 	int i;
 
+	fixture = trieopen("data/hira.map");
 	memset(&state, 0, sizeof state);
 	state.l = getlang(LangJP);
+	saved = state.l->map;
+	state.l->map = fixture;
 	for(i = 0; i < nelem(cases); i++){
 		state.pre = mkstr(cases[i].pre);
 		e = transmap(&state, cases[i].key);
@@ -90,4 +94,6 @@ transmap_states(struct ct *t)
 		checkstr(t, "next", cases[i].next, &e.next);
 		checkstr(t, "mapped", cases[i].mapped, &e.dict);
 	}
+	trieclose(fixture);
+	state.l->map = saved;
 }

tests/unit_test.c

@@ -46,12 +46,14 @@ erealloc(void *p, ulong n)
 static void
 testmapinit(void)
 {
-	Lang *jp;
+	Lang *jp, *kata;
 
 	jp = getlang(LangJP);
-	if(jp == nil)
+	kata = getlang(LangJPK);
+	if(jp == nil || kata == nil)
 		die("test language is not registered");
-	jp->map = trieopen("data/hira.map");
+	jp->map = trieopen("../map/hira.map");
+	kata->map = trieopen("../map/kata.map");
 	memset(&testvi, 0, sizeof testvi);
 	testvi.lang = LangVI;
 	testvi.mapname = "telex";
@@ -80,6 +82,9 @@ static const struct ct_test tests[] = {
 	{ "engine/commit-contract", engine_commit_contract },
 	{ "engine/language-switch-state", engine_language_switch_state },
 	{ "engine/telex-history-bound", engine_telex_history_bound },
+	{ "engine/japanese-readings", engine_japanese_readings },
+	{ "engine/japanese-candidates", engine_japanese_candidates },
+	{ "engine/japanese-backspace-boundaries", engine_japanese_backspace_and_boundaries },
 	{ "engine/emoji-single-candidate", engine_emoji_single_candidate },
 	{ "engine/emoji-queries", engine_emoji_queries },
 	{ "engine/emoji-digit-aliases", engine_emoji_digit_aliases },