engine: Ctrl+H reaches back into the text the client already has

Korean commits a syllable as the next one begins, so by the time the
한자 key is pressed only the last syllable is still ours: typing 한자 and
then Ctrl+H asked about 자 alone and answered 子, leaving 한子 in the
document -- the mixed Hangul and Hanja that was refused as dictionary
data, made by the interaction instead.  To get 漢字 the key had to be
pressed before typing, which no other Korean input method asks for.

A request now carries the client's own text just before the cursor, and
the reading reaches back through it as far as the dictionary still knows
the whole of it: 한 joins 자 and 대한민 joins 국.  Nothing but the
dictionary says where to stop, because a reading is syllables, so a key
holding a space or an already converted Hanja leads nowhere and the
reach ends there.  A pick answers with the count of runes to take back
first; Escape gives back only what was pending, and a query with no
candidate types only the part the client lacks.

A frontend that sends no surrounding text reaches back by nothing and
behaves exactly as before, which is what XIM will keep doing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

dat.h

@@ -207,6 +207,7 @@ enum
 struct Keyres
 {
 	int	eaten;	/* Keycap: the caller is the owner and clientpre took. */
+	int	del;	/* runes of the client's own text to take back first */
 	Str	commit;
 	Str	preedit;
 };
@@ -219,6 +220,7 @@ struct Keyreq
 	u32int	ks;
 	u32int	mod;
 	Caret	caret;
+	Str	surround;	/* the client's own text just before the cursor */
 	Channel	*reply;
 };
 

strans.c

@@ -7,17 +7,23 @@ static void *lostowner;	/* the context the engine was taken from */
 static Str lost;
 static int clientpre;	/* the owner draws its own preedit */
 static Caret caret;
+static Str surround;	/* the owner's own text just before the cursor */
+static int takeback;	/* runes of it a pick takes back before committing */
 static Rune modemark;	/* the mode the last key switched to */
 static int okuriat = -1;	/* where Shift marked the okurigana, or -1 */
 static Drawcmd lastdraw;
 static Emit transjp(Im*, Rune);
 static void backjp(Im*);
 
-/* seed is what Ctrl+H found pending; raw the keys typed since; text both. */
+/*
+ * back is the client's own text the reading reached into; seed what
+ * Ctrl+H found pending; raw the keys typed since; text all three.
+ */
 typedef struct Search Search;
 struct Search
 {
 	int	lang;
+	Str	back;
 	Str	seed;
 	Str	raw;
 	Str	text;
@@ -185,6 +191,17 @@ impre(Im *p, Str *s)
 		*s = p->pre;
 }
 
+/* The query as it is still ours to show: the client has the rest. */
+static void
+searchpre(Str *s)
+{
+	int i;
+
+	sclear(s);
+	for(i = search.back.n; i < search.text.n; i++)
+		sputr(s, search.text.r[i]);
+}
+
 static void
 snapshot(Drawcmd *dc)
 {
@@ -193,7 +210,7 @@ snapshot(Drawcmd *dc)
 
 	memset(dc, 0, sizeof *dc);
 	if(search.lang)
-		pre = search.text;
+		searchpre(&pre);
 	else
 		impre(&im, &pre);
 	if(!clientpre)
@@ -636,7 +653,11 @@ emojiquery(void)
 static void
 hanjaquery(void)
 {
-	transstr(im.l, &search.seed, &search.raw, &search.text);
+	Str tail;
+
+	transstr(im.l, &search.seed, &search.raw, &tail);
+	search.text = search.back;
+	sappend(&search.text, &tail);
 	clearkouho();
 	im.nkouho = dictprefix(getlang(LangHANJA)->dict, &search.text,
 		im.kouho, Maxkouho);
@@ -693,9 +714,12 @@ reset(void)
 static void
 flush(Str *com)
 {
-	if(search.lang)
+	Str pre;
-		sappend(com, &search.text);
+
-	else if(im.sel >= 0)
+	if(search.lang){
+		searchpre(&pre);
+		sappend(com, &pre);
+	}else if(im.sel >= 0)
 		sappend(com, &im.kouho[im.sel]);
 	else
 		commitim(&im, com);
@@ -716,6 +740,46 @@ takelost(void *owner, Str *com)
 	lostowner = nil;
 }
 
+/* The syllables from i to the end of the client's text, then seed. */
+static void
+reachkey(int i, Str *seed, Str *key)
+{
+	sclear(key);
+	for(; i < surround.n; i++)
+		sputr(key, surround.r[i]);
+	sappend(key, seed);
+}
+
+/*
+ * Ctrl+H converts a word, and a word is committed one syllable at a
+ * time: only its last syllable is still pending.  So the reading reaches
+ * back into the text the client already has, as far as the dictionary
+ * still knows the whole of it -- 한자 converts as a word where the 한 is
+ * the client's and only the 자 is ours.  Nothing but the dictionary says
+ * where to stop: a reading is syllables, so a key holding a space or a
+ * Hanja leads nowhere and the reach ends there.
+ */
+static void
+reachback(Str *seed, Str *back)
+{
+	Str key;
+	Trie *t;
+	int i, best;
+
+	t = getlang(LangHANJA)->dict;
+	best = surround.n;
+	for(i = surround.n - 1; i >= 0; i--){
+		if(surround.n - i + seed->n > Maxrunes)
+			break;
+		reachkey(i, seed, &key);
+		if(trienode(t, &key) >= 0)
+			best = i;
+	}
+	sclear(back);
+	for(i = best; i < surround.n; i++)
+		sputr(back, surround.r[i]);
+}
+
 /*
  * A search takes over the keys.  Hanja converts the pending syllable, so
  * it becomes the query; anything else pending is committed.
@@ -723,23 +787,28 @@ takelost(void *owner, Str *com)
 static void
 startsearch(int lang, Str *com)
 {
-	Str seed;
+	Str back, seed;
 
+	sclear(&back);
 	sclear(&seed);
-	if(lang == LangHANJA && !search.lang && im.sel < 0)
+	if(lang == LangHANJA && !search.lang && im.sel < 0){
 		impre(&im, &seed);
-	else
+		reachback(&seed, &back);
+	}else
 		flush(com);
 	reset();
 	search.lang = lang;
+	search.back = back;
 	search.seed = seed;
 	if(lang == LangHANJA)
 		searchquery();
 }
 
+/* The client keeps what the reading reached back into; we take it back. */
 static void
 picksearch(int n, Str *com)
 {
+	takeback = search.back.n;
 	sappend(com, &im.kouho[n]);
 	endsearch();
 }
@@ -954,6 +1023,8 @@ init(void)
 	activeowner = nil;
 	lostowner = nil;
 	clientpre = 0;
+	sclear(&surround);
+	takeback = 0;
 	memset(&lastdraw, 0, sizeof lastdraw);
 }
 
@@ -972,6 +1043,7 @@ imhandlekey(Keyreq *kr)
 	sclear(&res.commit);
 	sclear(&res.preedit);
 	res.eaten = 1;
+	takeback = 0;
 	switch(kr->op){
 	case Keyrelease:
 		takelost(kr->owner, &res.commit);
@@ -1009,6 +1081,7 @@ imhandlekey(Keyreq *kr)
 				clientpre = kr->clientpre;
 			}
 			caret = kr->caret;
+			surround = kr->surround;
 			res.eaten = transition(kr->ks, kr->mod, &res.commit);
 		}
 		break;
@@ -1016,10 +1089,11 @@ imhandlekey(Keyreq *kr)
 	redraw();
 	if(kr->owner == activeowner){
 		if(search.lang)
-			res.preedit = search.text;
+			searchpre(&res.preedit);
 		else
 			impre(&im, &res.preedit);
 	}
+	res.del = takeback;
 	chansend(kr->reply, &res);
 }
 

tests/engine_test.c

@@ -10,6 +10,7 @@ keystroke(u32int ks, u32int mod, Str *com)
 {
 	int eaten;
 
+	takeback = 0;
 	eaten = keymeaningful(ks) && transition(ks, mod, com);
 	redraw();
 	return eaten;
@@ -175,7 +176,7 @@ engine_candidate_shortcut_modifiers(struct ct *t)
 
 static Keyres
 ownerrequestatcap(void *owner, int cap, int op, Rune key, u32int mod,
-	Caret *caret)
+	Caret *caret, char *before)
 {
 	Channel *reply;
 	Keyreq req;
@@ -190,6 +191,8 @@ ownerrequestatcap(void *owner, int cap, int op, Rune key, u32int mod,
 	req.mod = mod;
 	if(caret != nil)
 		req.caret = *caret;
+	if(before != nil)
+		req.surround = mkstr(before);
 	req.reply = reply;
 	imhandlekey(&req);
 	chanrecv(reply, &res);
@@ -200,7 +203,7 @@ ownerrequestatcap(void *owner, int cap, int op, Rune key, u32int mod,
 static Keyres
 ownerrequestat(void *owner, int op, Rune key, u32int mod, Caret *caret)
 {
-	return ownerrequestatcap(owner, 0, op, key, mod, caret);
+	return ownerrequestatcap(owner, 0, op, key, mod, caret, nil);
 }
 
 static Keyres
@@ -212,7 +215,7 @@ ownerrequest(void *owner, int op, Rune key, u32int mod)
 static Keyres
 ownerrequestcap(void *owner, int cap, int op, Rune key, u32int mod)
 {
-	return ownerrequestatcap(owner, cap, op, key, mod, nil);
+	return ownerrequestatcap(owner, cap, op, key, mod, nil, nil);
 }
 
 void
@@ -1945,6 +1948,81 @@ cleanup:
 	searchend(&f);
 }
 
+/*
+ * A word is committed one syllable at a time, so Ctrl+H reaches back
+ * into the text the client already has, and a pick takes it back.
+ */
+void
+engine_hanja_reaches_back(struct ct *t)
+{
+	Searchfix f;
+	Keyres res;
+	Str com;
+	char owner;
+
+	hanjabegin(&f, LangKO);
+	surround = mkstr("민");
+	im.pre = mkstr("국");
+	sclear(&com);
+	if(!CT_CHECK(t, keystroke('h', Mctrl, &com)))
+		goto cleanup;
+	checkstr(t, "the client's syllable joined the reading", "민국",
+		&search.text);
+	checkstr(t, "what the client already has", "민", &search.back);
+	CT_EQ_INT(t, 1, im.nkouho);
+	CT_CHECK(t, keystroke(Kret, 0, &com));
+	checkstr(t, "the word converted", "民國", &com);
+	CT_EQ_INT(t, 1, takeback);
+
+	/* Escape gives back what was pending, and nothing of the client's. */
+	surround = mkstr("민");
+	im.pre = mkstr("국");
+	sclear(&com);
+	CT_CHECK(t, keystroke('h', Mctrl, &com));
+	CT_CHECK(t, keystroke(Kesc, 0, &com));
+	checkstr(t, "only the pending syllable comes back", "국", &im.pre);
+	CT_EQ_INT(t, 0, takeback);
+
+	/* A reading nothing is keyed by stops the reach where it stands. */
+	surround = mkstr("가");
+	im.pre = mkstr("국");
+	sclear(&com);
+	CT_CHECK(t, keystroke('h', Mctrl, &com));
+	checkstr(t, "no reach without a reading", "국", &search.text);
+	CT_EQ_INT(t, 0, search.back.n);
+	CT_CHECK(t, keystroke(Kesc, 0, &com));
+
+	/* A query without candidates types only what the client lacks. */
+	surround = mkstr("민");
+	im.pre = mkstr("국");
+	sclear(&com);
+	CT_CHECK(t, keystroke('h', Mctrl, &com));
+	if(typekeys(t, "r", &com)){
+		CT_EQ_INT(t, 0, im.nkouho);
+		CT_CHECK(t, keystroke(Kret, 0, &com));
+		checkstr(t, "the client's syllable is not typed again", "국ㄱ",
+			&com);
+		CT_EQ_INT(t, 0, takeback);
+	}
+
+	/* The frontend's own request carries the text and is told to take it. */
+	init();
+	im.l = getlang(LangKO);
+	ownerrequestatcap(&owner, 1, Keypress, 'r', 0, nil, "민");
+	ownerrequestatcap(&owner, 1, Keypress, 'n', 0, nil, "민");
+	res = ownerrequestatcap(&owner, 1, Keypress, 'r', 0, nil, "민");
+	checkstr(t, "the syllable the request composed", "국", &res.preedit);
+	CT_EQ_INT(t, 0, res.del);
+	ownerrequestatcap(&owner, 1, Keypress, 'h', Mctrl, nil, "민");
+	checkstr(t, "the request's own text joined the reading", "민국",
+		&search.text);
+	res = ownerrequestatcap(&owner, 1, Keypress, Kret, 0, nil, "민");
+	checkstr(t, "the word the request converted", "民國", &res.commit);
+	CT_EQ_INT(t, 1, res.del);
+cleanup:
+	searchend(&f);
+}
+
 /*
  * A reading is the start of the longer words it begins: its own
  * conversions come first, and past them theirs.

tests/test.h

@@ -86,6 +86,7 @@ void	engine_emoji_preedit_languages(struct ct*);
 void	engine_emoji_start_and_unknown(struct ct*);
 void	engine_hanja_search(struct ct*);
 void	engine_hanja_unknown_and_cancel(struct ct*);
+void	engine_hanja_reaches_back(struct ct*);
 void	engine_hanja_word_prefix(struct ct*);
 void	engine_hanja_korean_keys(struct ct*);
 void	engine_hanja_backspace(struct ct*);

tests/unit_test.c

@@ -107,6 +107,7 @@ static const struct ct_test tests[] = {
 	{ "engine/emoji-start-and-unknown", engine_emoji_start_and_unknown },
 	{ "engine/hanja-search", engine_hanja_search },
 	{ "engine/hanja-unknown-cancel", engine_hanja_unknown_and_cancel },
+	{ "engine/hanja-reaches-back", engine_hanja_reaches_back },
 	{ "engine/hanja-word-prefix", engine_hanja_word_prefix },
 	{ "engine/hanja-korean-keys", engine_hanja_korean_keys },
 	{ "engine/hanja-backspace", engine_hanja_backspace },