diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e66df58..b7abdcc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -428,6 +428,8 @@ per-file diff commands. - **Upstream triage distinguishes subject-only matches from verified patch matches** (#569) - matching subjects with different patches now name the fork commit and request verification instead of claiming the change was already cherry-picked. +- **Upstream version checks decode Git output as UTF-8** (#587) - future non-Latin framework lines and non-Latin branch names no longer depend on the host code page, guarding a latent case absent from the shipped framework files. + ### Security - **CI no longer leaves the job token in `.git/config` or expands a PR-controlled name into a diff --git a/tests/test_check_upstream_updates.py b/tests/test_check_upstream_updates.py index d38cc120..640e4473 100644 --- a/tests/test_check_upstream_updates.py +++ b/tests/test_check_upstream_updates.py @@ -1,3 +1,4 @@ +import os import shutil import subprocess import sys @@ -185,6 +186,45 @@ class UpstreamRefMissingFileTests(UpstreamCheckerRepoFixture): self.assertIn("[WARNING]", result.stdout) +class LegacyEncodingTests(UpstreamCheckerRepoFixture): + def setUp(self): + super().setUp() + # Use the actual shipped framework files, whose UTF-8 prose git show + # returns, rather than a constructed string passed to run_git(). + for rel in FRAMEWORK_FILES: + shutil.copy(REPO_ROOT / rel, self.root / rel) + subprocess.run(["git", "add", "-A"], cwd=self.root, check=True, capture_output=True) + subprocess.run(["git", "commit", "-qm", "copy actual framework files"], cwd=self.root, check=True, capture_output=True) + self.add_remote("upstream", TEMPLATE_URL) + self.materialize_remote_ref("upstream") + + def run_legacy_checker(self, *args, ascii_git_codec=True): + env = dict(os.environ, LC_ALL="C", LANG="C", PYTHONUTF8="0", + PYTHONCOERCECLOCALE="0", PYTHONIOENCODING="cp1252") + if not ascii_git_codec: + # Keep valid Unicode argv while independently forcing legacy stdout. + # Linux's ASCII filesystem decoder would instead create surrogate + # escapes, testing argument decoding rather than console output. + env["PYTHONUTF8"] = "1" + return subprocess.run( + [sys.executable, str(self.root / "tools/check_upstream_updates.py"), "--no-fetch", *args], + cwd=self.root, env=env, capture_output=True, + ) + + def test_actual_framework_prose_is_decoded_as_utf8(self): + result = self.run_legacy_checker() + self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace")) + self.assertIn("[OK]", result.stdout.decode("utf-8")) + + def test_non_latin_branch_is_printed_as_utf8(self): + branch = "release-版本" + subprocess.run(["git", "update-ref", f"refs/remotes/upstream/{branch}", "HEAD"], + cwd=self.root, check=True, capture_output=True) + result = self.run_legacy_checker("--branch", branch, ascii_git_codec=False) + self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", "replace")) + self.assertIn(f"upstream/{branch}", result.stdout.decode("utf-8")) + + class IncompleteComparisonTests(UpstreamCheckerRepoFixture): def setUp(self): super().setUp() diff --git a/tools/check_upstream_updates.py b/tools/check_upstream_updates.py index 9dcb040a..fa315eba 100755 --- a/tools/check_upstream_updates.py +++ b/tools/check_upstream_updates.py @@ -38,7 +38,10 @@ FRAMEWORK_FILES = [ UPSTREAM_REPO_SLUG = "MadsLorentzen/ai-job-search" def run_git(args: list[str]) -> tuple[int, str, str]: - res = subprocess.run(["git"] + args, cwd=str(ROOT), capture_output=True, text=True) + res = subprocess.run( + ["git"] + args, cwd=str(ROOT), capture_output=True, text=True, + encoding="utf-8", errors="replace", + ) return res.returncode, res.stdout, res.stderr def get_remote_url(remote_name: str) -> str: @@ -65,7 +68,16 @@ def parse_semver(version_str: str) -> tuple[int, int, int]: return (0, 0, 0) return tuple(int(x) for x in match.groups()) +def _force_utf8_output() -> None: + """Use UTF-8 for piped output even on legacy-console hosts.""" + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure: + reconfigure(encoding="utf-8") + + def main() -> int: + _force_utf8_output() parser = argparse.ArgumentParser(description="Check for framework updates upstream.") parser.add_argument("--remote", default="upstream", help="Name of the git remote for upstream (default: upstream)") parser.add_argument("--branch", default="master", help="Branch name of the upstream repo (default: master)")