feat(ingestion): add DOCX/CSV/XLSX parsing and fixed-size chunking (ADR-0018)

Adds src/application/ingestion/ -- Persian normalization, DOCX body
walk with structural data/layout table classification, CSV/XLSX row
rendering, and fixed-size token chunking (cl100k_base, 400/60/512) --
as pure functions per ADR-0015, tested against real production
documents (asia_data_sample, kept out of the repo). ADR-0018 records
where this diverges from ADR-0004 (fixed-size default, no invented
headings/tree, structural table classification, header-provable
labeling only). Plan 001's scope line is corrected from CSV-only to
DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is
updated to match current implementation state.

tests/unit/application/test_spreadsheet_parser.py

@@ -0,0 +1,183 @@
+"""CSV and XLSX parsing against the real sample corpus (ADR-0004, ADR-0018)."""
+
+import pytest
+
+from src.application.ingestion import ContentType, DocumentParseError, parse_csv, parse_xlsx
+from tests.support.documents import BRANCHES_XLSX, QA_XLSX, load_document
+
+pytestmark = pytest.mark.unit
+
+
+def _field(text: str, label: str) -> str | None:
+    for line in text.split("\n"):
+        name, _, value = line.partition(":")
+        if name == label:
+            return value.strip()
+    return None
+
+
+# ── the two real sheet shapes ─────────────────────────────────────
+
+
+def test_parse_xlsx_qa_sheet_renders_question_and_answer_labels() -> None:
+    """A two-column q/a sheet needs no special case; the generic renderer serves it."""
+    parsed = parse_xlsx(load_document(QA_XLSX))
+
+    assert parsed.units[0].content_type is ContentType.TABLE_ROW
+    assert _field(parsed.units[0].text, "q")
+    assert _field(parsed.units[0].text, "a")
+
+
+def test_parse_xlsx_branch_sheet_renders_every_column_with_its_header() -> None:
+    """The same renderer, a completely different schema, no shape detection."""
+    parsed = parse_xlsx(load_document(BRANCHES_XLSX))
+
+    first = parsed.units[0].text
+    assert len(first.split("\n")) > 5
+    assert all(":" in line for line in first.split("\n"))
+
+
+def test_parse_xlsx_dead_second_sheet_is_skipped() -> None:
+    """The q/a workbook carries an empty Sheet2 seen throughout the corpus."""
+    parsed = parse_xlsx(load_document(QA_XLSX))
+
+    assert parsed.block_count == 150
+
+
+# ── merged cells ──────────────────────────────────────────────────
+
+
+def test_parse_xlsx_merged_title_row_is_not_treated_as_header() -> None:
+    """A merged banner spans the sheet; the real header sits below it."""
+    parsed = parse_xlsx(load_document(BRANCHES_XLSX))
+
+    assert _field(parsed.units[0].text, "استان") is not None
+
+
+def test_parse_xlsx_two_row_tall_merged_header_is_not_emitted_as_data() -> None:
+    """Regression: forward fill duplicates the header into the row beneath it.
+
+    Without skipping that duplicate the first chunk was the header labeling
+    itself (`استان: استان`).
+    """
+    parsed = parse_xlsx(load_document(BRANCHES_XLSX))
+
+    assert _field(parsed.units[0].text, "استان") != "استان"
+
+
+def test_parse_xlsx_vertically_merged_column_fills_every_row_in_its_range() -> None:
+    """openpyxl stores a merged range's value only in its top-left cell.
+
+    The province is merged across each of its branches, so without forward
+    filling every branch but the first loses the field entirely.
+    """
+    parsed = parse_xlsx(load_document(BRANCHES_XLSX))
+
+    provinces = [_field(unit.text, "استان") for unit in parsed.units]
+
+    assert all(province for province in provinces)
+    assert len(set(provinces)) == 31
+
+
+def test_parse_xlsx_branch_rows_are_self_contained() -> None:
+    """Each branch is its own chunk carrying its province, not just its name."""
+    parsed = parse_xlsx(load_document(BRANCHES_XLSX))
+
+    second = parsed.units[1].text
+
+    assert _field(second, "استان") == _field(parsed.units[0].text, "استان")
+    assert _field(second, "شعبه") != _field(parsed.units[0].text, "شعبه")
+
+
+# ── csv ───────────────────────────────────────────────────────────
+
+
+def test_parse_csv_renders_rows_with_headers() -> None:
+    data = b"branch,city,phone\nvali asr,tehran,021\nazadi,karaj,026\n"
+
+    parsed = parse_csv(data)
+
+    assert parsed.block_count == 2
+    assert _field(parsed.units[0].text, "branch") == "vali asr"
+    assert _field(parsed.units[1].text, "city") == "karaj"
+
+
+def test_parse_csv_sniffs_semicolon_delimiter() -> None:
+    data = b"branch;city;phone\nvali asr;tehran;021\nazadi;karaj;026\n"
+
+    parsed = parse_csv(data)
+
+    assert _field(parsed.units[0].text, "city") == "tehran"
+
+
+def test_parse_csv_unprovable_header_falls_back_to_joined_cells() -> None:
+    """Short text over short text is genuinely ambiguous, so nothing is labeled.
+
+    The cost of this rule is unlabeled rows; the cost of guessing is labeling
+    every row from a data row, which is how a headerless compensation table
+    became `80: 70`. Losing a label is recoverable, mislabeling is not.
+    """
+    parsed = parse_csv(b"branch,city\nvali asr,tehran\nazadi,karaj\n")
+
+    assert parsed.units[0].text == "branch | city"
+    assert parsed.block_count == 3
+
+
+def test_parse_csv_empty_cells_are_omitted() -> None:
+    data = b"branch,city,phone\nvali asr,,021\n"
+
+    parsed = parse_csv(data)
+
+    assert _field(parsed.units[0].text, "city") is None
+    assert _field(parsed.units[0].text, "phone") == "021"
+
+
+def test_parse_csv_blank_rows_are_skipped() -> None:
+    data = b"branch,city,phone\nvali asr,tehran,021\n\n,,\nazadi,karaj,026\n"
+
+    parsed = parse_csv(data)
+
+    assert parsed.block_count == 2
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig", "cp1256"])
+def test_parse_csv_decodes_farsi_in_each_supported_encoding(encoding: str) -> None:
+    """cp1256 is common for Farsi exported by older Excel."""
+    tehran = "تهران"
+    parsed = parse_csv(f"branch,city,phone\nvali asr,{tehran},021\n".encode(encoding))
+
+    assert _field(parsed.units[0].text, "city") == tehran
+
+
+def test_parse_csv_normalizes_persian_letterforms() -> None:
+    arabic_yeh, persian_yeh = chr(0x064A), chr(0x06CC)
+
+    parsed = parse_csv(f"branch,city,phone\nvali asr,{arabic_yeh},021\n".encode())
+
+    assert _field(parsed.units[0].text, "city") == persian_yeh
+
+
+# ── failure modes ─────────────────────────────────────────────────
+
+
+def test_parse_csv_cp1256_fallback_always_decodes_rather_than_raising() -> None:
+    """cp1256 is single-byte, so it maps every byte and never fails.
+
+    That makes the "could not decode" branch unreachable in practice: bytes
+    that are not valid UTF-8 come back as mojibake instead of an error. The
+    guard stays as defence, but this documents the real behaviour so nobody
+    relies on a decode failure to reject a bad upload.
+    """
+    parsed = parse_csv(b"branch,city,phone\nvali asr,\xff\xfe,021\n")
+
+    assert parsed.block_count == 1
+
+
+def test_parse_csv_empty_file_raises_parse_error() -> None:
+    with pytest.raises(DocumentParseError, match="no rows"):
+        parse_csv(b"")
+
+
+def test_parse_xlsx_unopenable_bytes_raise_parse_error() -> None:
+    with pytest.raises(DocumentParseError, match="Could not open XLSX"):
+        parse_xlsx(b"not a workbook")