hanja: force UTF-8 filter streams

map/libhangul2hanja

@@ -49,6 +49,9 @@ def extract(src, name):
 
 
 def main():
+    sys.stdin.reconfigure(encoding="utf-8")
+    sys.stdout.reconfigure(encoding="utf-8")
+    sys.stderr.reconfigure(encoding="utf-8")
     if len(sys.argv) > 2:
         print(f"usage: {sys.argv[0]} [libhangul-hanja.txt]", file=sys.stderr)
         return 2

map/mkhanja

@@ -56,6 +56,8 @@ def read(path):
 
 
 def main():
+    sys.stdout.reconfigure(encoding="utf-8")
+    sys.stderr.reconfigure(encoding="utf-8")
     if len(sys.argv) > 2:
         print(f"usage: {sys.argv[0]} [hanja.src]", file=sys.stderr)
         return 2

tests/mkhanja_test.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import os
 import subprocess
 import sys
 import tempfile
@@ -12,11 +13,21 @@ IMPORT = ROOT / "map" / "libhangul2hanja"
 GENERATE = ROOT / "map" / "mkhanja"
 
 
-def run(program, source):
+def run(program, source=None, stdin=None, hostile=False):
+    command = [sys.executable, "-B", str(program)]
+    if source is not None:
+        command.append(str(source))
+    env = None
+    if hostile:
+        env = os.environ.copy()
+        env.update(LC_ALL="C", LANG="C", PYTHONCOERCECLOCALE="0", PYTHONUTF8="0")
+        env.pop("PYTHONIOENCODING", None)
     return subprocess.run(
-        [sys.executable, "-B", str(program), str(source)],
+        command,
+        input=stdin,
         capture_output=True,
-        text=True,
+        encoding="utf-8",
+        env=env,
         check=False,
     )
 
@@ -58,6 +69,15 @@ class MkhanjaTest(unittest.TestCase):
             with self.subTest(text=repr(text)):
                 self.assertEqual(run(IMPORT, self.source(text)).returncode, 1)
 
+    def test_import_uses_utf8_without_locale(self):
+        text = "# 저작권\n한:漢:first\n"
+        expected = "# 저작권\n\n漢\t한\n"
+        for source, stdin in ((self.source(text), None), (None, text)):
+            with self.subTest(source=source):
+                result = run(IMPORT, source, stdin, hostile=True)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertEqual(result.stdout, expected)
+
     def test_groups_readings_and_preserves_order(self):
         result = run(
             GENERATE,
@@ -96,6 +116,15 @@ class MkhanjaTest(unittest.TestCase):
             with self.subTest(text=repr(text)):
                 self.assertEqual(run(GENERATE, self.source(text)).returncode, 1)
 
+    def test_generator_uses_utf8_without_locale(self):
+        result = run(GENERATE, self.source("# 저작권\n漢\t한\n"), hostile=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(result.stdout, ";; 저작권\n\n한\t漢\n")
+
+        result = run(GENERATE, hostile=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("한\t", result.stdout)
+
     def test_generator_keeps_runtime_candidate_limit(self):
         rows = "".join(f"{chr(0x4E00 + n)}\t한\n" for n in range(33))
         result = run(GENERATE, self.source(rows))