feat(ingestion): add DOCX/CSV/XLSX parsing and fixed-size chunking (ADR-0018)

Adds src/application/ingestion/ -- Persian normalization, DOCX body
walk with structural data/layout table classification, CSV/XLSX row
rendering, and fixed-size token chunking (cl100k_base, 400/60/512) --
as pure functions per ADR-0015, tested against real production
documents (asia_data_sample, kept out of the repo). ADR-0018 records
where this diverges from ADR-0004 (fixed-size default, no invented
headings/tree, structural table classification, header-provable
labeling only). Plan 001's scope line is corrected from CSV-only to
DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is
updated to match current implementation state.

tests/unit/application/test_docx_parser.py

@@ -0,0 +1,225 @@
+"""DOCX parsing against the real sample corpus (ADR-0004, ADR-0018).
+
+Every assertion here was written after reading the document it covers, so the
+numbers are observations rather than predictions.
+"""
+
+import pytest
+
+from src.application.ingestion import ContentType, DocumentParseError, parse_docx
+from src.application.ingestion.docx_parser import heading_level_from_style
+from src.config import ChunkingSettings
+from tests.support.documents import (
+    LAYOUT_DOCX,
+    LIST_DOCX,
+    MIXED_DOCX,
+    PROSE_DOCX,
+    TABLE_DOCX,
+    load_document,
+)
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture
+def settings() -> ChunkingSettings:
+    return ChunkingSettings()
+
+
+def _units(filename: str, settings: ChunkingSettings):
+    return parse_docx(load_document(filename), settings).units
+
+
+def _types(units) -> set[ContentType]:
+    return {unit.content_type for unit in units}
+
+
+# ── heading styles ────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    ("style", "expected"),
+    [
+        ("Heading1", 1),  # the styleId Word actually stores
+        ("Heading 1", 1),  # the friendly name python-docx reports
+        ("heading 2", 2),
+        ("HEADING3", 3),
+        ("Normal", None),
+        ("List Paragraph", None),
+        ("Heading", None),
+        ("Headingfoo", None),
+    ],
+)
+def test_heading_level_from_style_maps_word_styles_only(style: str, expected: int | None) -> None:
+    assert heading_level_from_style(style) == expected
+
+
+def test_parse_docx_invents_no_headings_when_document_declares_none(
+    settings: ChunkingSettings,
+) -> None:
+    """Every document in the corpus lacks Heading styles, so this is the norm.
+
+    The evaluation repo upgraded paragraphs to headings by text pattern; a
+    wrongly-detected heading silently reshapes the tree, so ADR-0018 drops it.
+    """
+    parsed = parse_docx(load_document(PROSE_DOCX), settings)
+
+    assert "#" not in (parsed.markdown or "")
+
+
+# ── prose documents ───────────────────────────────────────────────
+
+
+def test_parse_docx_prose_document_becomes_one_flowing_block(
+    settings: ChunkingSettings,
+) -> None:
+    """Consecutive paragraphs accumulate into one prose unit (ADR-0004).
+
+    One unit per paragraph would hand the splitter 36 fragments instead of
+    flowing text.
+    """
+    units = _units(PROSE_DOCX, settings)
+
+    assert len(units) == 1
+    assert units[0].content_type is ContentType.PARAGRAPH
+    assert units[0].text.count("\n\n") == 35
+
+
+def test_parse_docx_list_styled_paragraphs_stay_prose(settings: ChunkingSettings) -> None:
+    units = _units(LIST_DOCX, settings)
+
+    assert _types(units) == {ContentType.PARAGRAPH}
+
+
+def test_parse_docx_normalizes_persian_letterforms(settings: ChunkingSettings) -> None:
+    parsed = parse_docx(load_document(PROSE_DOCX), settings)
+
+    # No Arabic kaf or yeh survives normalization.
+    assert chr(0x0643) not in (parsed.markdown or "")
+    assert chr(0x064A) not in (parsed.markdown or "")
+
+
+def test_parse_docx_preserves_paragraph_breaks_in_markdown(
+    settings: ChunkingSettings,
+) -> None:
+    """Normalization collapses whitespace, so it must run per block.
+
+    Applied to the assembled document it would flatten every paragraph onto a
+    single line -- the easiest way to get this wrong.
+    """
+    parsed = parse_docx(load_document(PROSE_DOCX), settings)
+
+    assert "\n\n" in (parsed.markdown or "")
+
+
+# ── data tables ───────────────────────────────────────────────────
+
+
+def test_parse_docx_data_table_becomes_one_unit_per_row(settings: ChunkingSettings) -> None:
+    """A 35-row contact table: one header row plus 34 data rows."""
+    units = _units(TABLE_DOCX, settings)
+
+    assert len(units) == 34
+    assert _types(units) == {ContentType.TABLE_ROW}
+
+
+def test_parse_docx_data_table_rows_are_labeled_with_headers(
+    settings: ChunkingSettings,
+) -> None:
+    units = _units(TABLE_DOCX, settings)
+
+    assert "\n" in units[0].text
+    assert all(":" in line for line in units[0].text.split("\n"))
+
+
+def test_parse_docx_vertically_merged_cell_repeats_on_every_row(
+    settings: ChunkingSettings,
+) -> None:
+    """The group column is merged down several rows; each row must carry it."""
+    units = _units(TABLE_DOCX, settings)
+
+    labels = [line.split(":")[0] for unit in units for line in unit.text.split("\n")]
+    group_label = units[1].text.split(":")[0]
+
+    assert labels.count(group_label) > 1
+
+
+def test_parse_docx_empty_cell_is_omitted_not_rendered_as_bare_label(
+    settings: ChunkingSettings,
+) -> None:
+    """The first data row has no group; it should have one fewer field."""
+    units = _units(TABLE_DOCX, settings)
+
+    assert len(units[0].text.split("\n")) < len(units[1].text.split("\n"))
+
+
+# ── tables without a header ───────────────────────────────────────
+
+
+def test_parse_docx_headerless_table_keeps_every_row(settings: ChunkingSettings) -> None:
+    """A 30-row compensation list plus an 11-row table, all rows preserved.
+
+    Regression: scanning forward for a header row discarded the rows above the
+    match and labeled the rest from a data row (`80: 70`).
+    """
+    units = _units(MIXED_DOCX, settings)
+    rows = [unit for unit in units if unit.content_type is ContentType.TABLE_ROW]
+
+    assert len(rows) == 41
+
+
+def test_parse_docx_headerless_table_rows_are_unlabeled(settings: ChunkingSettings) -> None:
+    """No header is provable, so cells are joined rather than mislabeled."""
+    units = _units(MIXED_DOCX, settings)
+    first_row = next(unit for unit in units if unit.content_type is ContentType.TABLE_ROW)
+
+    assert " | " in first_row.text
+    assert first_row.text.startswith("1 | ")
+
+
+def test_parse_docx_mixed_document_interleaves_prose_and_rows(
+    settings: ChunkingSettings,
+) -> None:
+    units = _units(MIXED_DOCX, settings)
+
+    assert _types(units) == {ContentType.PARAGRAPH, ContentType.TABLE_ROW}
+
+
+# ── layout tables ─────────────────────────────────────────────────
+
+
+def test_parse_docx_extracts_document_held_inside_a_table_cell(
+    settings: ChunkingSettings,
+) -> None:
+    """One merged cell holds 77k characters across 738 paragraphs.
+
+    Regression: a merged cell reported once per spanned grid position was
+    deduplicated by `id()`, which lxml reuses across element proxies, so the
+    whole sub-document vanished and the file yielded 70 tokens.
+    """
+    units = _units(LAYOUT_DOCX, settings)
+
+    assert sum(len(unit.text) for unit in units) > 70_000
+
+
+def test_parse_docx_layout_table_cells_become_prose_not_rows(
+    settings: ChunkingSettings,
+) -> None:
+    """A cell that alone exceeds chunk_size holds a document, not a field."""
+    units = _units(LAYOUT_DOCX, settings)
+
+    assert ContentType.PARAGRAPH in _types(units)
+
+
+# ── failure modes ─────────────────────────────────────────────────
+
+
+def test_parse_docx_unopenable_bytes_raises_parse_error(settings: ChunkingSettings) -> None:
+    with pytest.raises(DocumentParseError, match="Could not open DOCX"):
+        parse_docx(b"not a docx at all", settings)
+
+
+def test_parse_docx_legacy_doc_binary_raises_parse_error(settings: ChunkingSettings) -> None:
+    """`.doc` is rejected rather than converted (ADR-0018); python-docx cannot open it."""
+    with pytest.raises(DocumentParseError):
+        parse_docx(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512, settings)