diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 81fc8ee..aeef1a0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -30,7 +30,7 @@ There is no frontend build system, external queue, cloud database, or required A `ingestion.py` owns file decoding and structural validation. -It accepts CSV and XLSX input, normalizes the data into `CsvTable`, and rejects malformed files before a run is created. Upload reads are capped at the configured size limit plus one byte so oversized requests can be rejected early. +It accepts CSV and XLSX input, normalizes the data into `InputTable`, and rejects malformed files before a run is created. Upload reads are capped at the configured size limit plus one byte so oversized requests can be rejected early. Staged upload filenames are generated by the application rather than taken from user-provided paths. Staging copies are removed after successful run creation, and abandoned stage directories expire automatically. diff --git a/src/rowbridge/candidates.py b/src/rowbridge/candidates.py index 0440c89..1e79999 100644 --- a/src/rowbridge/candidates.py +++ b/src/rowbridge/candidates.py @@ -8,7 +8,7 @@ from rapidfuzz.fuzz import WRatio from rowbridge.matching_utils import normalize_text, parse_amount, parse_date -from rowbridge.models import CandidateGeneration, CsvTable, FieldMapping, MatchSettings +from rowbridge.models import CandidateGeneration, FieldMapping, InputTable, MatchSettings def _primary_block_keys(value: str) -> tuple[str, ...]: @@ -55,8 +55,8 @@ def _add_to_index(index: dict[str, set[int]], keys: Iterable[str], row_index: in def generate_candidates( - table_a: CsvTable, - table_b: CsvTable, + table_a: InputTable, + table_b: InputTable, mapping: FieldMapping, settings: MatchSettings, ) -> CandidateGeneration: diff --git a/src/rowbridge/ingestion.py b/src/rowbridge/ingestion.py index 75ee9e9..729a5db 100644 --- a/src/rowbridge/ingestion.py +++ b/src/rowbridge/ingestion.py @@ -11,7 +11,7 @@ from openpyxl import load_workbook -from rowbridge.models import CsvTable +from rowbridge.models import InputTable MAX_UPLOAD_BYTES = 5 * 1024 * 1024 MAX_XLSX_UNCOMPRESSED_BYTES = 50 * 1024 * 1024 @@ -26,10 +26,6 @@ class InputFileError(ValueError): pass -# Backward-compatible name for older imports while the input layer becomes format-neutral. -CsvInputError = InputFileError - - def _validate_size(content: bytes, filename: str) -> None: if not content: raise InputFileError(f"{filename} is empty") @@ -85,7 +81,7 @@ def _normalize_headers(values: list[str], filename: str) -> tuple[str, ...]: return headers -def parse_csv_bytes(content: bytes, filename: str) -> CsvTable: +def parse_csv_bytes(content: bytes, filename: str) -> InputTable: _validate_size(content, filename) text, encoding = _decode_csv(content, filename) delimiter = _detect_delimiter(text) @@ -110,7 +106,7 @@ def parse_csv_bytes(content: bytes, filename: str) -> CsvTable: if not rows: raise InputFileError(f"{filename} has no data rows") - return CsvTable( + return InputTable( filename=filename, headers=headers, rows=tuple(rows), @@ -150,7 +146,7 @@ def _validate_xlsx_archive(content: bytes, filename: str) -> None: raise InputFileError(f"{filename} is not a valid XLSX workbook") from exc -def parse_xlsx_bytes(content: bytes, filename: str) -> CsvTable: +def parse_xlsx_bytes(content: bytes, filename: str) -> InputTable: _validate_size(content, filename) _validate_xlsx_archive(content, filename) @@ -197,7 +193,7 @@ def parse_xlsx_bytes(content: bytes, filename: str) -> CsvTable: if not rows: raise InputFileError(f"{filename} has no data rows") - return CsvTable( + return InputTable( filename=filename, headers=headers, rows=tuple(rows), @@ -208,7 +204,7 @@ def parse_xlsx_bytes(content: bytes, filename: str) -> CsvTable: workbook.close() -def parse_input_bytes(content: bytes, filename: str) -> CsvTable: +def parse_input_bytes(content: bytes, filename: str) -> InputTable: suffix = Path(filename).suffix.lower() if suffix not in SUPPORTED_SUFFIXES: raise InputFileError(f"{filename} is not supported. Use a .csv or .xlsx file.") @@ -231,7 +227,7 @@ def write_staged_input(path: Path, content: bytes) -> None: path.write_bytes(content) -def read_staged_input(path: Path, original_filename: str) -> CsvTable: +def read_staged_input(path: Path, original_filename: str) -> InputTable: if not path.is_file(): raise InputFileError("Staged upload was not found. Upload the files again.") return parse_input_bytes(path.read_bytes(), original_filename) @@ -272,5 +268,5 @@ def cleanup_staged_uploads( return removed -def preview_rows(table: CsvTable) -> tuple[dict[str, str], ...]: +def preview_rows(table: InputTable) -> tuple[dict[str, str], ...]: return table.rows[:PREVIEW_ROWS] diff --git a/src/rowbridge/matching.py b/src/rowbridge/matching.py index 1343c1c..0a9a60c 100644 --- a/src/rowbridge/matching.py +++ b/src/rowbridge/matching.py @@ -11,9 +11,9 @@ CandidateGeneration, CandidateScore, ComparisonRule, - CsvTable, Evidence, FieldMapping, + InputTable, MatchDecision, MatchSettings, MatchStatus, @@ -175,8 +175,8 @@ def score_pair( def _score_candidates( - table_a: CsvTable, - table_b: CsvTable, + table_a: InputTable, + table_b: InputTable, mapping: FieldMapping, settings: MatchSettings, generation: CandidateGeneration, @@ -236,8 +236,8 @@ def _is_ambiguous( def reconcile_with_diagnostics( - table_a: CsvTable, - table_b: CsvTable, + table_a: InputTable, + table_b: InputTable, mapping: FieldMapping, settings: MatchSettings, ) -> tuple[tuple[MatchDecision, ...], CandidateGeneration]: @@ -313,8 +313,8 @@ def reconcile_with_diagnostics( def reconcile( - table_a: CsvTable, - table_b: CsvTable, + table_a: InputTable, + table_b: InputTable, mapping: FieldMapping, settings: MatchSettings, ) -> tuple[MatchDecision, ...]: diff --git a/src/rowbridge/models.py b/src/rowbridge/models.py index 68232f5..f2f5ca0 100644 --- a/src/rowbridge/models.py +++ b/src/rowbridge/models.py @@ -27,7 +27,7 @@ class RuleKind(StrEnum): @dataclass(frozen=True, slots=True) -class CsvTable: +class InputTable: filename: str headers: tuple[str, ...] rows: tuple[RowPayload, ...] @@ -37,6 +37,9 @@ class CsvTable: sheet_name: str | None = None +CsvTable = InputTable + + @dataclass(frozen=True, slots=True) class FieldMapping: primary_a: str diff --git a/src/rowbridge/service.py b/src/rowbridge/service.py index 848005f..c06c453 100644 --- a/src/rowbridge/service.py +++ b/src/rowbridge/service.py @@ -6,7 +6,7 @@ from rowbridge.matching import reconcile from rowbridge.matching_utils import parse_amount, parse_date -from rowbridge.models import CsvTable, FieldMapping, MatchSettings +from rowbridge.models import FieldMapping, InputTable, MatchSettings from rowbridge.storage import Repository @@ -30,7 +30,7 @@ def _validate_distinct_roles(mapping: FieldMapping) -> None: def _validate_parseable_column( - table: CsvTable, + table: InputTable, column: str, parser: Callable[[str], object | None], role: str, @@ -48,7 +48,7 @@ def _validate_parseable_column( ) -def validate_mapping(table_a: CsvTable, table_b: CsvTable, mapping: FieldMapping) -> None: +def validate_mapping(table_a: InputTable, table_b: InputTable, mapping: FieldMapping) -> None: selections = ( ("primary_a", mapping.primary_a, table_a.headers), ("primary_b", mapping.primary_b, table_b.headers), @@ -82,8 +82,8 @@ def validate_mapping(table_a: CsvTable, table_b: CsvTable, mapping: FieldMapping def create_reconciliation_run( repository: Repository, - table_a: CsvTable, - table_b: CsvTable, + table_a: InputTable, + table_b: InputTable, mapping: FieldMapping, settings: MatchSettings, ) -> str: diff --git a/src/rowbridge/web.py b/src/rowbridge/web.py index 2d0b2bd..1cffe9a 100644 --- a/src/rowbridge/web.py +++ b/src/rowbridge/web.py @@ -30,7 +30,7 @@ staged_filename, write_staged_input, ) -from rowbridge.models import CsvTable, FieldMapping, MatchSettings, MatchStatus, RowPayload +from rowbridge.models import FieldMapping, InputTable, MatchSettings, MatchStatus, RowPayload from rowbridge.service import create_reconciliation_run from rowbridge.storage import Repository, StoredMatch @@ -61,7 +61,7 @@ def _display_value(match: StoredMatch, side: str, column: str) -> str: return _payload_value(payload, column) -def _table_details(table: CsvTable) -> str: +def _table_details(table: InputTable) -> str: if table.source_format == "xlsx": return f"Excel workbook ยท sheet {table.sheet_name or 'first data sheet'}" delimiter_names = {",": "comma", ";": "semicolon", "\t": "tab", "|": "pipe"} diff --git a/tests/test_matching.py b/tests/test_matching.py index eada21b..31c75e3 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -1,4 +1,5 @@ from decimal import Decimal +from unittest.mock import patch from rowbridge.candidates import generate_candidates from rowbridge.matching import ( @@ -8,7 +9,15 @@ score_pair, ) from rowbridge.matching_utils import normalize_text, parse_amount -from rowbridge.models import CsvTable, FieldMapping, MatchSettings, MatchStatus, RuleKind +from rowbridge.models import ( + CandidateGeneration, + CandidateScore, + FieldMapping, + InputTable, + MatchSettings, + MatchStatus, + RuleKind, +) def test_normalize_text_removes_case_spacing_and_punctuation() -> None: @@ -27,7 +36,7 @@ def test_normalize_text_preserves_unicode_letters() -> None: def test_reconcile_matches_cyrillic_primary_values() -> None: - table_a = CsvTable( + table_a = InputTable( filename="a.csv", headers=("name",), rows=( @@ -39,7 +48,7 @@ def test_reconcile_matches_cyrillic_primary_values() -> None: }, ), ) - table_b = CsvTable( + table_b = InputTable( filename="b.csv", headers=("name",), rows=( @@ -203,12 +212,12 @@ def test_score_pair_uses_primary_secondary_amount_and_date_evidence() -> None: def test_candidate_generation_avoids_cartesian_product_for_exact_ids() -> None: row_count = 200 - table_a = CsvTable( + table_a = InputTable( filename="a.csv", headers=("id",), rows=tuple({"id": f"ITEM-{index:05d}"} for index in range(row_count)), ) - table_b = CsvTable( + table_b = InputTable( filename="b.csv", headers=("id",), rows=tuple({"id": f"ITEM{index:05d}"} for index in range(row_count)), @@ -227,12 +236,12 @@ def test_candidate_generation_avoids_cartesian_product_for_exact_ids() -> None: def test_support_fields_do_not_rescue_unrelated_primary_without_secondary_text() -> None: - table_a = CsvTable( + table_a = InputTable( filename="a.csv", headers=("ref", "amount", "date"), rows=({"ref": "INV-00126", "amount": "460.00", "date": "2026-09-30"},), ) - table_b = CsvTable( + table_b = InputTable( filename="b.csv", headers=("ref", "amount", "date"), rows=({"ref": "INV-00999", "amount": "460.00", "date": "2026-09-30"},), @@ -252,7 +261,7 @@ def test_support_fields_do_not_rescue_unrelated_primary_without_secondary_text() def test_secondary_text_can_rescue_mismatched_reference_for_review() -> None: - table_a = CsvTable( + table_a = InputTable( filename="a.csv", headers=("ref", "customer", "amount", "date"), rows=( @@ -264,7 +273,7 @@ def test_secondary_text_can_rescue_mismatched_reference_for_review() -> None: }, ), ) - table_b = CsvTable( + table_b = InputTable( filename="b.csv", headers=("ref", "payer", "amount", "date"), rows=( @@ -296,8 +305,8 @@ def test_secondary_text_can_rescue_mismatched_reference_for_review() -> None: def test_fallback_recovers_candidate_when_primary_blocks_do_not_overlap() -> None: - table_a = CsvTable(filename="a.csv", headers=("id",), rows=({"id": "XNV0012Z"},)) - table_b = CsvTable(filename="b.csv", headers=("id",), rows=({"id": "INV00123"},)) + table_a = InputTable(filename="a.csv", headers=("id",), rows=({"id": "XNV0012Z"},)) + table_b = InputTable(filename="b.csv", headers=("id",), rows=({"id": "INV00123"},)) settings = MatchSettings(review_threshold=0.5, minimum_primary_similarity=0.5) generation = generate_candidates( @@ -312,12 +321,12 @@ def test_fallback_recovers_candidate_when_primary_blocks_do_not_overlap() -> Non def test_reconcile_keeps_side_b_one_to_one_and_marks_competition_for_review() -> None: - table_a = CsvTable( + table_a = InputTable( filename="a.csv", headers=("name",), rows=({"name": "Alpha"}, {"name": "Alpha"}), ) - table_b = CsvTable(filename="b.csv", headers=("name",), rows=({"name": "Alpha"},)) + table_b = InputTable(filename="b.csv", headers=("name",), rows=({"name": "Alpha"},)) decisions = reconcile( table_a, table_b, @@ -336,9 +345,53 @@ def test_reconcile_keeps_side_b_one_to_one_and_marks_competition_for_review() -> assert len(unmatched_a) == 1 +def test_score_ordered_resolver_is_greedy_not_global_assignment() -> None: + table_a = InputTable( + filename="a.csv", + headers=("id",), + rows=({"id": "A0"}, {"id": "A1"}), + ) + table_b = InputTable( + filename="b.csv", + headers=("id",), + rows=({"id": "B0"}, {"id": "B1"}), + ) + generation = CandidateGeneration( + by_a=((0, 1), (0,)), + possible_pairs=4, + generated_pairs=3, + fallback_rows=0, + ) + scored = ( + CandidateScore(a_index=0, b_index=0, score=0.95, evidence=()), + CandidateScore(a_index=0, b_index=1, score=0.94, evidence=()), + CandidateScore(a_index=1, b_index=0, score=0.93, evidence=()), + ) + + with ( + patch("rowbridge.matching.generate_candidates", return_value=generation), + patch("rowbridge.matching._score_candidates", return_value=scored), + ): + decisions, _ = reconcile_with_diagnostics( + table_a, + table_b, + FieldMapping(primary_a="id", primary_b="id"), + MatchSettings(), + ) + + paired = [ + (decision.a_index, decision.b_index) + for decision in decisions + if decision.a_index is not None and decision.b_index is not None + ] + + assert paired == [(0, 0)] + assert scored[1].score + scored[2].score > scored[0].score + + def test_primary_only_mapping_can_auto_match_exact_values() -> None: - table_a = CsvTable(filename="a.csv", headers=("name",), rows=({"name": "Acme Ltd"},)) - table_b = CsvTable(filename="b.csv", headers=("name",), rows=({"name": "ACME LTD"},)) + table_a = InputTable(filename="a.csv", headers=("name",), rows=({"name": "Acme Ltd"},)) + table_b = InputTable(filename="b.csv", headers=("name",), rows=({"name": "ACME LTD"},)) decisions = reconcile( table_a, table_b, @@ -351,8 +404,8 @@ def test_primary_only_mapping_can_auto_match_exact_values() -> None: def test_large_inputs_skip_unbounded_global_fuzzy_fallback() -> None: - table_a = CsvTable(filename="a.csv", headers=("id",), rows=({"id": "NO-MATCH-HERE"},)) - table_b = CsvTable( + table_a = InputTable(filename="a.csv", headers=("id",), rows=({"id": "NO-MATCH-HERE"},)) + table_b = InputTable( filename="b.csv", headers=("id",), rows=tuple({"id": f"B-{index:05d}"} for index in range(5_001)), @@ -372,12 +425,12 @@ def test_large_inputs_skip_unbounded_global_fuzzy_fallback() -> None: def test_reconcile_scales_to_thousands_of_exact_one_to_one_candidates() -> None: row_count = 2_000 - table_a = CsvTable( + table_a = InputTable( filename="a.csv", headers=("id",), rows=tuple({"id": f"ITEM-{index:05d}"} for index in range(row_count)), ) - table_b = CsvTable( + table_b = InputTable( filename="b.csv", headers=("id",), rows=tuple({"id": f"ITEM{index:05d}"} for index in range(row_count)),