fix(convert_salary_excel): pair count/index columns by category name, not adjacency (#219)

The sequential scan assumed count/index pairs are always adjacent.
Interleaved columns like Count_A, Count_B, Index_A, Index_B produced
wrong pairings (Count_B ↔ Index_A), silently corrupting data.

Now columns are grouped by type, then matched by the category name
derived from stripping type words. Unmatched columns fall back to
standalone value columns using the original header name.
This commit is contained in:
Oscar Madera
2026-07-22 20:34:48 +02:00
committed by GitHub
parent a68028bc54
commit 3609f584b5
2 changed files with 67 additions and 40 deletions
+25
View File
@@ -157,5 +157,30 @@ class DetectColumnTypeTests(unittest.TestCase):
self.assertEqual(companies[0]["categories"]["salary_index"], {"index": 105.5}) self.assertEqual(companies[0]["categories"]["salary_index"], {"index": 105.5})
def test_parse_sheet_pairs_interleaved_count_index_columns_by_name(self):
ws = FakeWorksheet([
("Company", "Antal kvinder", "Antal mænd", "Kvinder indeks", "Mænd indeks"),
("Example Corp", 15, 20, 95.0, 108.0),
])
companies = parse_sheet(ws)
categories = companies[0]["categories"]
self.assertEqual(categories["kvinder"], {"count": 15, "index": 95.0})
self.assertEqual(categories["mænd"], {"count": 20, "index": 108.0})
def test_parse_sheet_non_adjacent_columns_no_cross_match(self):
ws = FakeWorksheet([
("Company", "Count_A", "Count_B", "Index_A", "Index_B"),
("Example Corp", 10, 20, 100.0, 200.0),
])
companies = parse_sheet(ws)
categories = companies[0]["categories"]
self.assertEqual(categories["a"], {"count": 10, "index": 100.0})
self.assertEqual(categories["b"], {"count": 20, "index": 200.0})
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+41 -39
View File
@@ -132,53 +132,55 @@ def parse_sheet(ws, sheet_label=None):
continue continue
data_cols.append((i, h)) data_cols.append((i, h))
# Try to detect paired count/index columns per category # Group data columns by detected type and derive category names
# Heuristic: if columns come in pairs and alternate count/index, group them count_cols = []
categories = [] index_cols = []
i = 0 untyped_cols = []
while i < len(data_cols):
col_idx, col_header = data_cols[i] for col_idx, col_header in data_cols:
col_type = detect_column_type(col_header) col_type = detect_column_type(col_header)
if col_type == "count":
if i + 1 < len(data_cols):
next_col_idx, next_col_header = data_cols[i + 1]
next_col_type = detect_column_type(next_col_header)
# If we have a count/index pair, group them
if col_type == "count" and next_col_type == "index":
# Use the header minus the count/index suffix as category name
cat_name = strip_type_patterns(col_header, COUNT_PATTERNS) cat_name = strip_type_patterns(col_header, COUNT_PATTERNS)
if not cat_name: count_cols.append((col_idx, col_header, cat_name))
cat_name = f"category_{len(categories)+1}" elif col_type == "index":
else:
cat_name = cat_name.replace(" ", "_").replace("-", "_")
categories.append({
"name": cat_name,
"count_col": col_idx,
"index_col": next_col_idx,
})
i += 2
continue
elif col_type == "index" and next_col_type == "count":
cat_name = strip_type_patterns(col_header, INDEX_PATTERNS) cat_name = strip_type_patterns(col_header, INDEX_PATTERNS)
if not cat_name: index_cols.append((col_idx, col_header, cat_name))
cat_name = f"category_{len(categories)+1}"
else: else:
cat_name = cat_name.replace(" ", "_").replace("-", "_") untyped_cols.append((col_idx, col_header))
# Pair count/index columns by matching category name
categories = []
used_counts = set()
used_indexes = set()
for ci, (c_idx, c_header, c_cat) in enumerate(count_cols):
for ii, (i_idx, i_header, i_cat) in enumerate(index_cols):
if ii in used_indexes:
continue
if c_cat and i_cat and c_cat == i_cat:
cat_name = c_cat.replace(" ", "_").replace("-", "_")
categories.append({ categories.append({
"name": cat_name, "name": cat_name,
"index_col": col_idx, "count_col": c_idx,
"count_col": next_col_idx, "index_col": i_idx,
}) })
i += 2 used_counts.add(ci)
continue used_indexes.add(ii)
break
# Single column - treat as a standalone value # Remaining unmatched count columns become standalone (use original header)
categories.append({ for ci, (c_idx, c_header, _) in enumerate(count_cols):
"name": col_header.lower().replace(" ", "_"), if ci not in used_counts:
"value_col": col_idx, categories.append({"name": c_header.lower().replace(" ", "_"), "value_col": c_idx})
})
i += 1 # Remaining unmatched index columns become standalone (use original header)
for ii, (i_idx, i_header, _) in enumerate(index_cols):
if ii not in used_indexes:
categories.append({"name": i_header.lower().replace(" ", "_"), "value_col": i_idx})
# Untyped columns become standalone
for col_idx, col_header in untyped_cols:
categories.append({"name": col_header.lower().replace(" ", "_"), "value_col": col_idx})
# Parse data rows # Parse data rows
companies = [] companies = []