engine: the chosen candidate is what the client shows

The preedit sent back to the owner was always impre(), the reading, so
かく stayed underlined in the application while the popup showed 書く and
Enter inserted 書く.  Nothing was hidden -- the popup had it -- but the
text did not stand where it was going to land, and mozc, fcitx5 and kime
all put the candidate inline.

Only the reply changes.  impre() still answers the reading everywhere it
is the reading that is wanted: commitim, the okurigana mark, the seed
Ctrl+H takes, and snapshot, so a popup that draws the preedit itself
keeps the reading above the list where it says something the highlighted
row does not.  Consulting im.sel here is safe where consulting it inside
impre() would not be: transstr memsets a local Im, whose sel is 0 and not
-1, and this reply only ever reads the engine's own.

Nothing in the suite pinned the old answer, so the test is new.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

README.md

@@ -33,16 +33,17 @@ Japanese composes a whole reading and then offers its Kanji with none
 chosen. `Space` and `Tab` step through the candidates and wrap (`Shift`
 reverses), the reading in Katakana last, so a word the dictionary lacks
 converts with one `Space`; `Up`/`Down` and `PageUp`/`PageDown` move without
-wrapping. `Enter` commits the chosen candidate, or the reading, as `0` and
-typing on always do. Once a candidate is chosen `1`-`9` take that row of
-the page shown; before that the digits type. `Backspace` deletes the last
-kana shown, and romaji that made no kana stays as typed until it is mended;
-`Esc` cancels the reading, and on a chosen candidate both go back to it.
-The dictionary keys verbs and adjectives by stem and okurigana, so `kaku`
-offers かく's readings and then 書く and its kin — typing the okurigana
-with `Shift`, as SKK does, `kaKu`, asks for that split first. Katakana
-mode does not convert, so there `Space` and `Tab` commit the reading and
-go on to the application.
+wrapping. A chosen candidate takes the reading's place in the preedit, so
+it stands where it will land, and `Enter` commits it, or the reading, as
+`0` and typing on always do. Once a candidate is chosen `1`-`9` take that
+row of the page shown; before that the digits type. `Backspace` deletes
+the last kana shown, and romaji that made no kana stays as typed until it
+is mended; `Esc` cancels the reading, and on a chosen candidate both go
+back to it. The dictionary keys verbs and adjectives by stem and
+okurigana, so `kaku` offers かく's readings and then 書く and its kin —
+typing the okurigana with `Shift`, as SKK does, `kaKu`, asks for that
+split first. Katakana mode does not convert, so there `Space` and `Tab`
+commit the reading and go on to the application.
 
 Korean composes one syllable at a time. `Enter`, `Tab`, `Esc` and any key
 that is not a jamo commit the syllable and go on to the application, so
@@ -60,12 +61,15 @@ a syllable — 한자 gives 漢字, 대한민국 gives 大韓民國 — and `Esc
 syllable back. A word is committed a syllable at a time, so the reading
 also reaches back into the text the application already holds, as far as
 the dictionary knows the whole of it: type 한자 and then `Ctrl+H` and the
-query is 한자, not 자, and picking 漢字 takes the 한 back. A reading is
-also the start of the longer words it begins, whose conversions follow
-its own, so 대한 answers 大寒 first and 大韓民國 further down. A lone
-consonant is a reading too, and answers with the symbol table a Korean
-keyboard's 한자 key has always offered: ㅁ gives ※ ○ △ ㈜, ㄴ the brackets
-「」『』, ㄹ the units ℃ ㎏ ℓ, ㅇ the circled numbers ①②③.
+query is 한자, not 자, and picking 漢字 takes the 한 back. `Backspace`
+gives that reach back before it deletes what was typed, so 상 written
+with 태 typed answers 狀態, and one `Backspace` answers 態 for the
+syllable alone. A reading is also the start of the longer words it
+begins, whose conversions follow its own, so 대한 answers 大寒 first and
+大韓民國 further down. A lone consonant is a reading too, and answers
+with the symbol table a Korean keyboard's 한자 key has always offered:
+ㅁ gives ※ ○ △ ㈜, ㄴ the brackets 「」『』, ㄹ the units ℃ ㎏ ℓ, ㅇ the
+circled numbers ①②③.
 
 Reaching back needs the application to hand over the text around its
 cursor and to take some of it away again. The Wayland input method, IBus

bench/main.c

@@ -99,7 +99,7 @@ iterations(char *s)
 		goto Bad;
 	errno = 0;
 	n = strtoumax(s, &end, 10);
-	if(errno != 0 || *end != '\0' || n == 0 || n > UINT64_MAX)
+	if(errno != 0 || *end != '\0' || n == 0)
 		goto Bad;
 	return n;
 Bad:
@@ -157,7 +157,7 @@ main(int argc, char **argv)
 
 	loadkeys(argv[1]);
 	niter = argc == 3 ? iterations(argv[2]) : 1000;
-	if(nkeys > UINT64_MAX || niter > UINT64_MAX / nkeys){
+	if(niter > UINT64_MAX / nkeys){
 		fprintf(stderr, "iteration count overflows key total\n");
 		exit(1);
 	}

str.c

@@ -1,7 +1,8 @@
 #include "dat.h"
 #include "fn.h"
 
-/* Fills s from n bytes of UTF-8; whole, valid, and at most Maxrunes. */
+/* Fills s from n bytes of UTF-8; whole, valid, and at most Maxrunes.
+ * Anything else leaves s empty. */
 int
 sinit(Str *s, char *src, int n)
 {
@@ -35,8 +36,7 @@ stail(Str *s, char *src, int n)
 	nr = utfnlen(src, n);
 	for(p = src; nr > Maxrunes; nr--)
 		p += chartorune(&r, p);
-	if(!sinit(s, p, n - (p - src)))
-		sclear(s);
+	sinit(s, p, n - (p - src));
 }
 
 void

strans.c

@@ -653,14 +653,16 @@ emojiquery(void)
 static void
 hanjaquery(void)
 {
-	Str tail;
+	Str kouho[Maxkouho], tail;
+	int i, n;
 
 	transstr(im.l, &search.seed, &search.raw, &tail);
 	search.text = search.back;
 	sappend(&search.text, &tail);
 	clearkouho();
-	im.nkouho = dictprefix(getlang(LangHANJA)->dict, &search.text,
-		im.kouho, Maxkouho);
+	n = dictprefix(getlang(LangHANJA)->dict, &search.text, kouho, Maxkouho);
+	for(i = 0; i < n; i++)
+		addkouho(&kouho[i]);
 	selectfirst();
 }
 
@@ -793,7 +795,9 @@ startsearch(int lang, Str *com)
 	sclear(&seed);
 	if(lang == LangHANJA && !search.lang && im.sel < 0){
 		impre(&im, &seed);
-		reachback(&seed, &back);
+		/* Nothing pending is no reading, and reaches into nothing. */
+		if(seed.n != 0)
+			reachback(&seed, &back);
 	}else
 		flush(com);
 	reset();
@@ -846,6 +850,8 @@ searchkey(u32int ks, u32int mod, Str *com)
 	if(ks == Kback){
 		if(search.raw.n != 0)
 			spopr(&search.raw);
+		else if(search.back.n != 0)
+			sclear(&search.back);	/* the reach undone, in its turn */
 		else if(search.seed.n != 0)
 			spopr(&search.seed);
 		else{
@@ -966,6 +972,7 @@ transition(u32int ks, u32int mod, Str *com)
 		else if(ks == Kesc)
 			reset();
 		else{
+			okuriat = -1;	/* the mark goes with the kana it marked */
 			im.l->back(&im);
 			if(haspre(&im))
 				dictqjp();
@@ -1090,6 +1097,8 @@ imhandlekey(Keyreq *kr)
 	if(kr->owner == activeowner){
 		if(search.lang)
 			searchpre(&res.preedit);
+		else if(im.sel >= 0)
+			res.preedit = im.kouho[im.sel];	/* what Enter would commit */
 		else
 			impre(&im, &res.preedit);
 	}

tests/Makefile

@@ -89,7 +89,8 @@ gtk_live_test: gtk_live_test.c $(LIVESRC) ../ipc.c ../ipc.h
 		$(LDFLAGS) -o $@ gtk_live_test.c live.c ../ipc.c $(GTK_LIBS) \
 		-pthread $(LDLIBS)
 
-../gtk/im-strans.so: ../gtk/main.c ../ipc.c ../ipc.h
+# The parent make knows what these are built from; always ask it.
+../gtk/im-strans.so:
 	$(MAKE) -C ../gtk
 
 ../strans:
@@ -145,4 +146,5 @@ unit_%.o: ../%.c ../dat.h ../fn.h ../ipc.h
 clean:
 	rm -f $(OBJS) $(PROG) $(STRESS) $(LIVE)
 
-.PHONY: all check check-live check-stress test clean
+.PHONY: all check check-live check-stress test clean \
+	../strans ../gtk/im-strans.so

tests/engine_test.c

@@ -411,6 +411,7 @@ engine_commit_contract(struct ct *t)
 		{ 'd', "ㅇ", "ㅇㅇㅇ" },
 		{ 'k', "ㅏ", "ㅏㅏㅏ" },
 	};
+	Drawcmd dc;
 	Keyres res;
 	Str all, com;
 	char owner;
@@ -495,6 +496,26 @@ engine_commit_contract(struct ct *t)
 	CT_CHECK(t, !keystroke(0xf008, 0, &com));
 	checkstr(t, "non-ASCII key commit", "か", &com);
 	CT_EQ_INT(t, 0, im.pre.n);
+
+	/* A chosen candidate is what the client shows and what Enter
+	 * commits; the popup keeps the reading above the list. */
+	init();
+	im.l = getlang(LangJP);
+	draindraw(nil);
+	ownerrequestcap(&owner, 1, Keypress, 'k', 0);
+	res = ownerrequestcap(&owner, 1, Keypress, 'a', 0);
+	checkstr(t, "the reading before a candidate", "か", &res.preedit);
+	im.kouho[0] = mkstr("蚊");
+	im.nkouho = 1;
+	res = ownerrequestcap(&owner, 1, Keypress, Kdown, 0);
+	checkstr(t, "the candidate the client shows", "蚊", &res.preedit);
+	clientpre = 0;
+	redraw();
+	if(CT_CHECK(t, draindraw(&dc) > 0))
+		checkstr(t, "the popup keeps the reading", "か", &dc.pre);
+	res = ownerrequestcap(&owner, 1, Keypress, Kret, 0);
+	checkstr(t, "Enter commits what was shown", "蚊", &res.commit);
+	CT_EQ_INT(t, 0, res.preedit.n);
 }
 
 void
@@ -1277,6 +1298,13 @@ engine_japanese_candidates(struct ct *t)
 		checkstr(t, "then the second", "描く", &im.kouho[1]);
 		checkstr(t, "then the reading's own", "確", &im.kouho[2]);
 	}
+	/* Backspace edits the reading, so the split it marked is gone. */
+	CT_CHECK(t, keystroke(Kback, 0, &com));
+	CT_CHECK(t, keystroke('k', 0, &com));
+	CT_CHECK(t, keystroke('u', 0, &com));
+	if(CT_EQ_INT(t, 3, im.nkouho))
+		checkstr(t, "the mark did not outlive its kana", "確",
+			&im.kouho[0]);
 	CT_CHECK(t, keystroke(Kesc, 0, &com));
 	setdict(jp->dict, "いt", "言 行");
 	setdict(jp->dict, "いっt", "言 行");
@@ -2005,6 +2033,37 @@ engine_hanja_reaches_back(struct ct *t)
 		CT_EQ_INT(t, 0, takeback);
 	}
 
+	/* Backspace gives the reach back before it eats what was typed. */
+	surround = mkstr("민");
+	im.pre = mkstr("국");
+	sclear(&com);
+	CT_CHECK(t, keystroke('h', Mctrl, &com));
+	checkstr(t, "the reach joined the reading", "민국", &search.text);
+	CT_CHECK(t, keystroke(Kback, 0, &com));
+	checkstr(t, "the client's syllable goes back first", "국",
+		&search.text);
+	CT_EQ_INT(t, 0, search.back.n);
+	sclear(&com);
+	CT_CHECK(t, keystroke(Kret, 0, &com));
+	checkstr(t, "only what was typed is committed", "국", &com);
+	CT_EQ_INT(t, 0, takeback);
+
+	/* Nothing pending is no reading, and reaches into nothing. */
+	surround = mkstr("민");
+	sclear(&im.pre);
+	sclear(&com);
+	CT_CHECK(t, keystroke('h', Mctrl, &com));
+	CT_EQ_INT(t, 0, search.back.n);
+	CT_EQ_INT(t, 0, im.nkouho);
+	if(typekeys(t, "als", &com)){
+		checkstr(t, "the reading typed is the whole query", "민",
+			&search.text);
+		CT_CHECK(t, keystroke(Kret, 0, &com));
+		checkstr(t, "the client's syllable is not converted", "民",
+			&com);
+		CT_EQ_INT(t, 0, takeback);
+	}
+
 	/* The frontend's own request carries the text and is told to take it. */
 	init();
 	im.l = getlang(LangKO);
@@ -2034,6 +2093,8 @@ engine_hanja_word_prefix(struct ct *t)
 	Str com;
 
 	hanjabegin(&f, LangKO);
+	/* Two readings under one prefix may share a word; it shows once. */
+	setdict(f.dictlang->dict, "민민", "民");
 	sclear(&com);
 	if(!CT_CHECK(t, keystroke('h', Mctrl, &com)) ||
 	   !typekeys(t, "als", &com))

tests/ibus_test.c

@@ -13,6 +13,8 @@ struct Ibusfix
 {
 	Channel	*oldreply;
 	Pump	pump;
+	/* A fake connection is nothing but an address to tell two apart. */
+	char	fake[2];
 	DBusConnection	*c1;
 	DBusConnection	*c2;
 };
@@ -81,8 +83,8 @@ cleanup:
 	CT_CHECK(t, rmdir(root) == 0);
 }
 
-static int
-ibusbegin(struct ct *t, Ibusfix *f)
+static void
+ibusbegin(Ibusfix *f)
 {
 	Drawcmd dc;
 
@@ -97,9 +99,8 @@ ibusbegin(struct ct *t, Ibusfix *f)
 	f->oldreply = replyc;
 	replyc = chancreate(sizeof(Keyres), 0);
 	pumpstart(&f->pump, Maxcontexts);
-	f->c1 = malloc(1);
-	f->c2 = malloc(1);
-	return CT_CHECK(t, f->c1 != nil && f->c2 != nil);
+	f->c1 = (DBusConnection*)&f->fake[0];
+	f->c2 = (DBusConnection*)&f->fake[1];
 }
 
 static void
@@ -119,8 +120,6 @@ ibusend(Ibusfix *f)
 	testengineinit(LangEN);
 	while(channbrecv(drawc, &dc) > 0)
 		;
-	free(f->c1);
-	free(f->c2);
 	chanfree(replyc);
 	replyc = f->oldreply;
 }
@@ -168,8 +167,7 @@ ibus_capability_policy(struct ct *t)
 	Ictx *a, *b;
 	Keyres res;
 
-	if(!ibusbegin(t, &f))
-		goto cleanup;
+	ibusbegin(&f);
 	a = newcontext(f.c1, "/context/cap-a");
 	b = newcontext(f.c2, "/context/cap-b");
 	if(!CT_CHECK(t, a != nil && b != nil))
@@ -223,8 +221,7 @@ ibus_private_input_policy(struct ct *t)
 	char text[Maxutf];
 	int i;
 
-	if(!ibusbegin(t, &f))
-		goto cleanup;
+	ibusbegin(&f);
 	ctx = newcontext(f.c1, "/context/private");
 	if(!CT_CHECK(t, ctx != nil))
 		goto cleanup;
@@ -274,8 +271,7 @@ ibus_context_lifecycle(struct ct *t)
 	Caret at;
 	char text[Maxutf];
 
-	if(!ibusbegin(t, &f))
-		goto cleanup;
+	ibusbegin(&f);
 	a = newcontext(f.c1, "/context/one");
 	b = newcontext(f.c2, "/context/one");
 	if(!CT_CHECK(t, a != nil && b != nil && a != b))
@@ -387,8 +383,7 @@ ibus_active_release_lifecycle(struct ct *t)
 	Ictx *ctx, *reused;
 	Keyres res;
 
-	if(!ibusbegin(t, &f))
-		goto cleanup;
+	ibusbegin(&f);
 	ctx = newcontext(f.c1, "/context/reset");
 	if(!CT_CHECK(t, ctx != nil))
 		goto cleanup;

tests/ipc_test.c

@@ -302,6 +302,7 @@ ipc_response_fragmented_and_truncated(struct ct *t)
 		}
 	if(i == n && CT_EQ_INT(t, 0, ipcreadresp(fd[1], 1, commit, preedit,
 	   &resp))){
+		CT_EQ_INT(t, 2, resp.del);
 		CT_EQ_STR(t, "abc", commit);
 		CT_EQ_STR(t, "xy", preedit);
 	}

tests/xim_adapter_test.c

@@ -398,8 +398,8 @@ struct Freejob
 	Channel	*done;
 };
 
-static int
-ximbegin(struct ct *t, Ximfix *f, int lang)
+static void
+ximbegin(Ximfix *f, int lang)
 {
 	Drawcmd dc;
 
@@ -410,8 +410,6 @@ ximbegin(struct ct *t, Ximfix *f, int lang)
 	f->oldreply = replyc;
 	replyc = chancreate(sizeof(Keyres), 0);
 	pumpstart(&f->pump, 16);
-	USED(t);
-	return 1;
 }
 
 static void
@@ -466,8 +464,7 @@ xim_adapter_key_contract(struct ct *t)
 	u32int allmod;
 
 	memset(&state, 0, sizeof state);
-	if(!ximbegin(t, &f, LangJP))
-		goto cleanup;
+	ximbegin(&f, LangJP);
 	allmod = ~0;
 	keypress(&state, 'x', allmod, &res);
 	req = nexttrace(t, &f, Keypress, &state);
@@ -487,7 +484,6 @@ xim_adapter_key_contract(struct ct *t)
 	CT_CHECK(t, !res.eaten);
 	checkstr(t, "preedit", "か", &res.commit);
 	checkstr(t, "preedit", "", &res.preedit);
-cleanup:
 	ximend(&f);
 }
 
@@ -502,8 +498,7 @@ xim_adapter_release_lifecycle(struct ct *t)
 	memset(&a, 0, sizeof a);
 	memset(&b, 0, sizeof b);
 	dead = nil;
-	if(!ximbegin(t, &f, LangJP))
-		goto cleanup;
+	ximbegin(&f, LangJP);
 	keypress(&a, 'k', 0, &res);
 	nexttrace(t, &f, Keypress, &a);
 	keypress(&a, 'a', 0, &res);
@@ -580,8 +575,7 @@ xim_adapter_free_waits_for_release(struct ct *t)
 	state = nil;
 	freed = nil;
 	workeractive = 0;
-	if(!ximbegin(t, &f, LangJP))
-		goto cleanup;
+	ximbegin(&f, LangJP);
 	state = calloc(1, sizeof *state);
 	freed = chancreate(sizeof(uchar), 0);
 	if(!CT_CHECK(t, state != nil && freed != nil))
@@ -736,8 +730,7 @@ xim_adapter_placement_updates(struct ct *t)
 	ic = (xcb_im_input_context_t*)(uintptr)71;
 	memset(&state, 0, sizeof state);
 	oldstate = kstate;
-	if(!ximbegin(t, &f, LangJP))
-		goto cleanup;
+	ximbegin(&f, LangJP);
 	kstate = testkeystate("us");
 	if(!CT_CHECK(t, kstate != nil))
 		goto cleanup;
@@ -891,8 +884,7 @@ xim_adapter_callback_cleanup(struct ct *t)
 
 	ic = (xcb_im_input_context_t*)(uintptr)29;
 	memset(&state, 0, sizeof state);
-	if(!ximbegin(t, &f, LangJP))
-		return;
+	ximbegin(&f, LangJP);
 	wireclear();
 	wirebind(0, ic, &state);
 	state.xic = ic;
@@ -980,8 +972,7 @@ xim_adapter_callback_transfer(struct ct *t)
 	aic = (xcb_im_input_context_t*)(uintptr)50;
 	memset(&a, 0, sizeof a);
 	memset(&b, 0, sizeof b);
-	if(!ximbegin(t, &f, LangJP))
-		goto cleanup;
+	ximbegin(&f, LangJP);
 	wireclear();
 	a.xic = aic;
 	a.clientpre = 1;
@@ -996,7 +987,6 @@ xim_adapter_callback_transfer(struct ct *t)
 	CT_EQ_PTR(t, nil, preowner);
 	release(&b);
 	nexttrace(t, &f, Keyrelease, &b);
-cleanup:
 	preowner = nil;
 	ximend(&f);
 }
@@ -1016,8 +1006,7 @@ xim_adapter_callback_owner_loss(struct ct *t)
 	ic = (xcb_im_input_context_t*)(uintptr)51;
 	memset(&state, 0, sizeof state);
 	foreign = 0;
-	if(!ximbegin(t, &f, LangJP))
-		goto cleanup;
+	ximbegin(&f, LangJP);
 	wireclear();
 	wirebind(0, ic, &state);
 	state.xic = ic;
@@ -1086,7 +1075,6 @@ xim_adapter_callback_owner_loss(struct ct *t)
 	CT_CHECK(t, channbrecv(f.pump.trace, &trace) > 0);
 	CT_EQ_INT(t, Keyrelease, trace.op);
 	CT_EQ_PTR(t, &foreign, trace.owner);
-cleanup:
 	preowner = nil;
 	ximend(&f);
 }

wl.c

@@ -127,14 +127,12 @@ sendrequest(int op, u32int ks, u32int mod, Keyres *res)
 static int
 backbytes(int n)
 {
-	char utf[Maxutf];
-	Str s;
-	int i;
+	int i, nb;
 
-	sclear(&s);
+	nb = 0;
 	for(i = max(surround.n - n, 0); i < surround.n; i++)
-		sputr(&s, surround.r[i]);
-	return stoutf(&s, utf, sizeof utf);
+		nb += runelen(surround.r[i]);
+	return nb;
 }
 
 /* The engine's modifier bits, read from the grab's own keyboard state. */