From 822d19a2ba91b3ecbaab6ac8e7b0b253325b4b28 Mon Sep 17 00:00:00 2001
From: jonathanwxh-cell <jonathanwxh@gmail.com>
Date: Sun, 18 Oct 2026 02:06:36 +0800
Subject: [PATCH] fix(salary): accept BOM-prefixed UTF-8 data files (#602)

* fix(salary): accept BOM-prefixed UTF-8 data files

Fixes #594. Thanks to @rahul05ranjan for reporting the failure and identifying the decoder change.

* docs(changelog): entry for #602

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Mads Lorentzen <madslorentzen_17@hotmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
---
 CHANGELOG.md                |  7 ++++++
 salary_lookup.py            |  2 +-
 tests/test_salary_lookup.py | 49 +++++++++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 012c20a0..54768a8a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -68,6 +68,13 @@ per-file diff commands.
 
 ### Fixed
 
+- **`salary_lookup.py` accepts a BOM-prefixed `salary_data.json`** (#594, #602, `salary_lookup.py`,
+  `tests/test_salary_lookup.py`) - the data file is opened with `utf-8-sig`, so a file saved by
+  Windows PowerShell `Out-File`/`Set-Content` or by an editor that prepends a byte-order mark now
+  validates and loads instead of exiting 1 with "Unexpected UTF-8 BOM". Plain UTF-8 files read
+  identically, and malformed JSON after a BOM still reports its line and column. Reported by
+  rahul05ranjan; same class as #485 and #509.
+
 - **Upstream triage emits cherry-pick suggestions in dependency order**
   (`tools/upstream_triage.py`) - ready-to-run commands now list ancestors before
   descendants, so a follow-up patch is not suggested before the change it edits.
diff --git a/salary_lookup.py b/salary_lookup.py
index 6c921627..50403700 100644
--- a/salary_lookup.py
+++ b/salary_lookup.py
@@ -149,7 +149,7 @@ def read_raw_data():
         print("step will be skipped during /apply.", file=sys.stderr)
         sys.exit(1)
     try:
-        with open(DATA_FILE, "r", encoding="utf-8") as f:
+        with open(DATA_FILE, "r", encoding="utf-8-sig") as f:
             data = json.load(f)
     except json.JSONDecodeError as exc:
         fail_data_error(f"invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}")
diff --git a/tests/test_salary_lookup.py b/tests/test_salary_lookup.py
index 5c77af98..974148ef 100644
--- a/tests/test_salary_lookup.py
+++ b/tests/test_salary_lookup.py
@@ -1,6 +1,10 @@
 """Tests for salary_lookup.py — format_entry, match_score, and search_company."""
 
 import io
+import json
+import shutil
+import subprocess
+import sys
 import tempfile
 import unittest
 from contextlib import redirect_stderr, redirect_stdout
@@ -600,5 +604,50 @@ class TestSearchCompanyScoreThreshold(unittest.TestCase):
         self.assertEqual(results[0]["company"], "Novo Nordisk")
 
 
+class TestSalaryFileEncoding(unittest.TestCase):
+    """Exercise user-created UTF-8 files through the documented CLI."""
+
+    def setUp(self):
+        directory = tempfile.TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        self.root = Path(directory.name)
+        self.script = self.root / "salary_lookup.py"
+        shutil.copyfile(salary_lookup.__file__, self.script)
+        self.data_file = self.root / "salary_data.json"
+        self.entry = {
+            "company": "Novo Nordisk A/S",
+            "city": "Bagsværd",
+            "categories": {"engineering": {"count": 120, "index": 112.3}},
+        }
+
+    def run_cli(self, *args):
+        return subprocess.run(
+            [sys.executable, str(self.script), *args],
+            cwd=self.root, capture_output=True, text=True, encoding="utf-8",
+            timeout=30,
+        )
+
+    def test_utf8_with_and_without_bom(self):
+        payload = {"metadata": {"index_baseline": 100}, "companies": [self.entry]}
+        for encoding in ("utf-8", "utf-8-sig"):
+            with self.subTest(encoding=encoding):
+                self.data_file.write_text(
+                    json.dumps(payload, ensure_ascii=False), encoding=encoding,
+                )
+                validated = self.run_cli("--validate")
+                self.assertEqual(validated.returncode, 0, validated.stderr)
+                self.assertIn("OK - no issues found.", validated.stdout)
+                lookup = self.run_cli("Novo Nordisk", "--json")
+                self.assertEqual(lookup.returncode, 0, lookup.stderr)
+                self.assertEqual(json.loads(lookup.stdout), [self.entry])
+
+    def test_bom_does_not_hide_invalid_json(self):
+        self.data_file.write_text('{"companies": [}', encoding="utf-8-sig")
+        result = self.run_cli("--validate")
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("invalid JSON at line 1, column 16", result.stderr)
+        self.assertNotIn("Unexpected UTF-8 BOM", result.stderr)
+
+
 if __name__ == "__main__":
     unittest.main()