"""CSV and XLSX parsing against the real sample corpus (ADR-0004, ADR-0018).""" import pytest from src.application.ingestion import ContentType, DocumentParseError, parse_csv, parse_xlsx from tests.support.documents import BRANCHES_XLSX, QA_XLSX, load_document pytestmark = pytest.mark.unit def _field(text: str, label: str) -> str | None: for line in text.split("\n"): name, _, value = line.partition(":") if name == label: return value.strip() return None # ── the two real sheet shapes ───────────────────────────────────── def test_parse_xlsx_qa_sheet_renders_question_and_answer_labels() -> None: """A two-column q/a sheet needs no special case; the generic renderer serves it.""" parsed = parse_xlsx(load_document(QA_XLSX)) assert parsed.units[0].content_type is ContentType.TABLE_ROW assert _field(parsed.units[0].text, "q") assert _field(parsed.units[0].text, "a") def test_parse_xlsx_branch_sheet_renders_every_column_with_its_header() -> None: """The same renderer, a completely different schema, no shape detection.""" parsed = parse_xlsx(load_document(BRANCHES_XLSX)) first = parsed.units[0].text assert len(first.split("\n")) > 5 assert all(":" in line for line in first.split("\n")) def test_parse_xlsx_dead_second_sheet_is_skipped() -> None: """The q/a workbook carries an empty Sheet2 seen throughout the corpus.""" parsed = parse_xlsx(load_document(QA_XLSX)) assert parsed.block_count == 150 # ── merged cells ────────────────────────────────────────────────── def test_parse_xlsx_merged_title_row_is_not_treated_as_header() -> None: """A merged banner spans the sheet; the real header sits below it.""" parsed = parse_xlsx(load_document(BRANCHES_XLSX)) assert _field(parsed.units[0].text, "استان") is not None def test_parse_xlsx_two_row_tall_merged_header_is_not_emitted_as_data() -> None: """Regression: forward fill duplicates the header into the row beneath it. Without skipping that duplicate the first chunk was the header labeling itself (`استان: استان`). """ parsed = parse_xlsx(load_document(BRANCHES_XLSX)) assert _field(parsed.units[0].text, "استان") != "استان" def test_parse_xlsx_vertically_merged_column_fills_every_row_in_its_range() -> None: """openpyxl stores a merged range's value only in its top-left cell. The province is merged across each of its branches, so without forward filling every branch but the first loses the field entirely. """ parsed = parse_xlsx(load_document(BRANCHES_XLSX)) provinces = [_field(unit.text, "استان") for unit in parsed.units] assert all(province for province in provinces) assert len(set(provinces)) == 31 def test_parse_xlsx_branch_rows_are_self_contained() -> None: """Each branch is its own chunk carrying its province, not just its name.""" parsed = parse_xlsx(load_document(BRANCHES_XLSX)) second = parsed.units[1].text assert _field(second, "استان") == _field(parsed.units[0].text, "استان") assert _field(second, "شعبه") != _field(parsed.units[0].text, "شعبه") # ── csv ─────────────────────────────────────────────────────────── def test_parse_csv_renders_rows_with_headers() -> None: data = b"branch,city,phone\nvali asr,tehran,021\nazadi,karaj,026\n" parsed = parse_csv(data) assert parsed.block_count == 2 assert _field(parsed.units[0].text, "branch") == "vali asr" assert _field(parsed.units[1].text, "city") == "karaj" def test_parse_csv_sniffs_semicolon_delimiter() -> None: data = b"branch;city;phone\nvali asr;tehran;021\nazadi;karaj;026\n" parsed = parse_csv(data) assert _field(parsed.units[0].text, "city") == "tehran" def test_parse_csv_unprovable_header_falls_back_to_joined_cells() -> None: """Short text over short text is genuinely ambiguous, so nothing is labeled. The cost of this rule is unlabeled rows; the cost of guessing is labeling every row from a data row, which is how a headerless compensation table became `80: 70`. Losing a label is recoverable, mislabeling is not. """ parsed = parse_csv(b"branch,city\nvali asr,tehran\nazadi,karaj\n") assert parsed.units[0].text == "branch | city" assert parsed.block_count == 3 def test_parse_csv_empty_cells_are_omitted() -> None: data = b"branch,city,phone\nvali asr,,021\n" parsed = parse_csv(data) assert _field(parsed.units[0].text, "city") is None assert _field(parsed.units[0].text, "phone") == "021" def test_parse_csv_blank_rows_are_skipped() -> None: data = b"branch,city,phone\nvali asr,tehran,021\n\n,,\nazadi,karaj,026\n" parsed = parse_csv(data) assert parsed.block_count == 2 @pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig", "cp1256"]) def test_parse_csv_decodes_farsi_in_each_supported_encoding(encoding: str) -> None: """cp1256 is common for Farsi exported by older Excel.""" tehran = "تهران" parsed = parse_csv(f"branch,city,phone\nvali asr,{tehran},021\n".encode(encoding)) assert _field(parsed.units[0].text, "city") == tehran def test_parse_csv_normalizes_persian_letterforms() -> None: arabic_yeh, persian_yeh = chr(0x064A), chr(0x06CC) parsed = parse_csv(f"branch,city,phone\nvali asr,{arabic_yeh},021\n".encode()) assert _field(parsed.units[0].text, "city") == persian_yeh # ── failure modes ───────────────────────────────────────────────── def test_parse_csv_cp1256_fallback_always_decodes_rather_than_raising() -> None: """cp1256 is single-byte, so it maps every byte and never fails. That makes the "could not decode" branch unreachable in practice: bytes that are not valid UTF-8 come back as mojibake instead of an error. The guard stays as defence, but this documents the real behaviour so nobody relies on a decode failure to reject a bad upload. """ parsed = parse_csv(b"branch,city,phone\nvali asr,\xff\xfe,021\n") assert parsed.block_count == 1 def test_parse_csv_empty_file_raises_parse_error() -> None: with pytest.raises(DocumentParseError, match="no rows"): parse_csv(b"") def test_parse_xlsx_unopenable_bytes_raise_parse_error() -> None: with pytest.raises(DocumentParseError, match="Could not open XLSX"): parse_xlsx(b"not a workbook")