From 011a8b8435fdf578e03d0f1d6c866b0123b1b3e7 Mon Sep 17 00:00:00 2001 From: Oscar Madera <80536682+oscarbol09@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:44:56 -0500 Subject: [PATCH] fix(upstream_triage): force UTF-8 stdout and decode git output as UTF-8 (#544) (#547) * fix(upstream_triage): force UTF-8 stdout and decode git output as UTF-8 (#544) * test(tools-utf8): restore assertion in salary converter test and isolate upstream_triage test --- CHANGELOG.md | 9 +++++++++ tests/test_tools_utf8_output.py | 22 ++++++++++++++++++++++ tools/upstream_triage.py | 20 +++++++++++++++++--- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e341a0ed..ea4995d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,14 @@ per-file diff commands. ### Fixed +- **`upstream_triage` forces UTF-8 stdout and decodes git output as UTF-8** (#544, + `tools/upstream_triage.py`, `tests/test_tools_utf8_output.py`) - like the other Python tools, + the triage report now calls `_force_utf8_output()` on startup and decodes git subprocess + streams with `encoding="utf-8", errors="replace"`. Under Windows default ANSI code pages (such as + `cp1252`), upstream commits with Cyrillic, CJK, or emoji characters in their subjects raised + `UnicodeEncodeError` when writing stdout or `UnicodeDecodeError` in subprocess pipes, terminating + triage runs before the report could be emitted. + - **`jobnet-search` routes external ads to employer `jobAdUrl` rather than broken `find-job` links** (#538, `.agents/skills/jobnet-search/cli/src/commands/search.ts`, `.agents/skills/jobnet-search/cli/README.md`, `.agents/skills/jobnet-search/cli/tests/search-normalization.test.ts`) - external job postings on @@ -139,6 +147,7 @@ per-file diff commands. says and asserts the shipped file's version still parses out unchanged; it fails on master. Found while re-checking #519's class of drift. + - **`/reset profile` now targets the headings the shipped skill files actually have** (`.claude/commands/reset.md` Step 3, `tests/test_reset_command.py`) - two of its Step 3 targets had drifted. The `05-cv-templates.md` step anchored on a section beginning diff --git a/tests/test_tools_utf8_output.py b/tests/test_tools_utf8_output.py index ea8de69b..ecf99cb3 100644 --- a/tests/test_tools_utf8_output.py +++ b/tests/test_tools_utf8_output.py @@ -149,6 +149,28 @@ class ToolsWriteUtf8(unittest.TestCase): ]) self.assert_clean_utf8(proc, 0, CYRILLIC) + def test_upstream_triage_handles_non_latin_commit_subjects(self): + repo = self.tmp / "repo" + repo.mkdir() + tools_dir = repo / "tools" + tools_dir.mkdir() + shutil.copy(TOOLS / "upstream_triage.py", tools_dir / "upstream_triage.py") + subprocess.run(["git", "init", "-b", "master"], cwd=repo, check=True, capture_output=True) + subprocess.run(["git", "config", "user.name", "Test"], cwd=repo, check=True, capture_output=True) + subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=repo, check=True, capture_output=True) + subprocess.run(["git", "remote", "add", "upstream", "https://github.com/MadsLorentzen/ai-job-search.git"], cwd=repo, check=True, capture_output=True) + (repo / "file.txt").write_text("hello\n", encoding="utf-8") + subprocess.run(["git", "add", "file.txt"], cwd=repo, check=True, capture_output=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, capture_output=True) + (repo / "file.txt").write_text("change\n", encoding="utf-8") + subprocess.run(["git", "add", "file.txt"], cwd=repo, check=True, capture_output=True) + msg = f"feat: add support for {CYRILLIC}" + subprocess.run(["git", "commit", "-m", msg], cwd=repo, check=True, capture_output=True) + subprocess.run(["git", "update-ref", "refs/remotes/upstream/master", "HEAD"], cwd=repo, check=True, capture_output=True) + subprocess.run(["git", "reset", "--hard", "HEAD~1"], cwd=repo, check=True, capture_output=True) + proc = run_legacy_stdout([repo / "tools" / "upstream_triage.py"], cwd=repo) + self.assert_clean_utf8(proc, 0, CYRILLIC) + if __name__ == "__main__": unittest.main() diff --git a/tools/upstream_triage.py b/tools/upstream_triage.py index 74a2fb87..bd73e7ce 100755 --- a/tools/upstream_triage.py +++ b/tools/upstream_triage.py @@ -36,9 +36,22 @@ import subprocess import sys +def _force_utf8_output() -> None: + """Write UTF-8 whatever the host's default encoding is. + + A piped stdout on Windows defaults to the ANSI code page (cp1252 on most + Western installs), so printing a commit subject or author outside it + raised UnicodeEncodeError before the report could be emitted. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure: + reconfigure(encoding="utf-8") + + def git(*args: str) -> str: return subprocess.run( - ["git", *args], capture_output=True, text=True, check=True + ["git", *args], capture_output=True, text=True, encoding="utf-8", errors="replace", check=True ).stdout @@ -50,10 +63,10 @@ def rev_list(range_spec: str) -> list[str]: def patch_id(sha: str) -> str | None: """Stable patch-id for a commit, or None if it has no diff.""" show = subprocess.run( - ["git", "show", sha], capture_output=True, text=True, check=True + ["git", "show", sha], capture_output=True, text=True, encoding="utf-8", errors="replace", check=True ).stdout r = subprocess.run( - ["git", "patch-id", "--stable"], input=show, capture_output=True, text=True + ["git", "patch-id", "--stable"], input=show, capture_output=True, text=True, encoding="utf-8", errors="replace" ) line = r.stdout.strip() return line.split()[0] if line else None @@ -111,6 +124,7 @@ def commit_cell(short: str, sha: str, slug: str | None) -> str: def main() -> int: + _force_utf8_output() ap = argparse.ArgumentParser() ap.add_argument("--remote", default="upstream") ap.add_argument("--branch", default="master")