"""DOCX parsing against the real sample corpus (ADR-0004, ADR-0018). Every assertion here was written after reading the document it covers, so the numbers are observations rather than predictions. """ import pytest from src.application.ingestion import ContentType, DocumentParseError, parse_docx from src.application.ingestion.docx_parser import heading_level_from_style from src.config import ChunkingSettings from tests.support.documents import ( LAYOUT_DOCX, LIST_DOCX, MIXED_DOCX, PROSE_DOCX, TABLE_DOCX, load_document, ) pytestmark = pytest.mark.unit @pytest.fixture def settings() -> ChunkingSettings: return ChunkingSettings() def _units(filename: str, settings: ChunkingSettings): return parse_docx(load_document(filename), settings).units def _types(units) -> set[ContentType]: return {unit.content_type for unit in units} # ── heading styles ──────────────────────────────────────────────── @pytest.mark.parametrize( ("style", "expected"), [ ("Heading1", 1), # the styleId Word actually stores ("Heading 1", 1), # the friendly name python-docx reports ("heading 2", 2), ("HEADING3", 3), ("Normal", None), ("List Paragraph", None), ("Heading", None), ("Headingfoo", None), ], ) def test_heading_level_from_style_maps_word_styles_only(style: str, expected: int | None) -> None: assert heading_level_from_style(style) == expected def test_parse_docx_invents_no_headings_when_document_declares_none( settings: ChunkingSettings, ) -> None: """Every document in the corpus lacks Heading styles, so this is the norm. The evaluation repo upgraded paragraphs to headings by text pattern; a wrongly-detected heading silently reshapes the tree, so ADR-0018 drops it. """ parsed = parse_docx(load_document(PROSE_DOCX), settings) assert "#" not in (parsed.markdown or "") # ── prose documents ─────────────────────────────────────────────── def test_parse_docx_prose_document_becomes_one_flowing_block( settings: ChunkingSettings, ) -> None: """Consecutive paragraphs accumulate into one prose unit (ADR-0004). One unit per paragraph would hand the splitter 36 fragments instead of flowing text. """ units = _units(PROSE_DOCX, settings) assert len(units) == 1 assert units[0].content_type is ContentType.PARAGRAPH assert units[0].text.count("\n\n") == 35 def test_parse_docx_list_styled_paragraphs_stay_prose(settings: ChunkingSettings) -> None: units = _units(LIST_DOCX, settings) assert _types(units) == {ContentType.PARAGRAPH} def test_parse_docx_normalizes_persian_letterforms(settings: ChunkingSettings) -> None: parsed = parse_docx(load_document(PROSE_DOCX), settings) # No Arabic kaf or yeh survives normalization. assert chr(0x0643) not in (parsed.markdown or "") assert chr(0x064A) not in (parsed.markdown or "") def test_parse_docx_preserves_paragraph_breaks_in_markdown( settings: ChunkingSettings, ) -> None: """Normalization collapses whitespace, so it must run per block. Applied to the assembled document it would flatten every paragraph onto a single line -- the easiest way to get this wrong. """ parsed = parse_docx(load_document(PROSE_DOCX), settings) assert "\n\n" in (parsed.markdown or "") # ── data tables ─────────────────────────────────────────────────── def test_parse_docx_data_table_becomes_one_unit_per_row(settings: ChunkingSettings) -> None: """A 35-row contact table: one header row plus 34 data rows.""" units = _units(TABLE_DOCX, settings) assert len(units) == 34 assert _types(units) == {ContentType.TABLE_ROW} def test_parse_docx_data_table_rows_are_labeled_with_headers( settings: ChunkingSettings, ) -> None: units = _units(TABLE_DOCX, settings) assert "\n" in units[0].text assert all(":" in line for line in units[0].text.split("\n")) def test_parse_docx_vertically_merged_cell_repeats_on_every_row( settings: ChunkingSettings, ) -> None: """The group column is merged down several rows; each row must carry it.""" units = _units(TABLE_DOCX, settings) labels = [line.split(":")[0] for unit in units for line in unit.text.split("\n")] group_label = units[1].text.split(":")[0] assert labels.count(group_label) > 1 def test_parse_docx_empty_cell_is_omitted_not_rendered_as_bare_label( settings: ChunkingSettings, ) -> None: """The first data row has no group; it should have one fewer field.""" units = _units(TABLE_DOCX, settings) assert len(units[0].text.split("\n")) < len(units[1].text.split("\n")) # ── tables without a header ─────────────────────────────────────── def test_parse_docx_headerless_table_keeps_every_row(settings: ChunkingSettings) -> None: """A 30-row compensation list plus an 11-row table, all rows preserved. Regression: scanning forward for a header row discarded the rows above the match and labeled the rest from a data row (`80: 70`). """ units = _units(MIXED_DOCX, settings) rows = [unit for unit in units if unit.content_type is ContentType.TABLE_ROW] assert len(rows) == 41 def test_parse_docx_headerless_table_rows_are_unlabeled(settings: ChunkingSettings) -> None: """No header is provable, so cells are joined rather than mislabeled.""" units = _units(MIXED_DOCX, settings) first_row = next(unit for unit in units if unit.content_type is ContentType.TABLE_ROW) assert " | " in first_row.text assert first_row.text.startswith("1 | ") def test_parse_docx_mixed_document_interleaves_prose_and_rows( settings: ChunkingSettings, ) -> None: units = _units(MIXED_DOCX, settings) assert _types(units) == {ContentType.PARAGRAPH, ContentType.TABLE_ROW} # ── layout tables ───────────────────────────────────────────────── def test_parse_docx_extracts_document_held_inside_a_table_cell( settings: ChunkingSettings, ) -> None: """One merged cell holds 77k characters across 738 paragraphs. Regression: a merged cell reported once per spanned grid position was deduplicated by `id()`, which lxml reuses across element proxies, so the whole sub-document vanished and the file yielded 70 tokens. """ units = _units(LAYOUT_DOCX, settings) assert sum(len(unit.text) for unit in units) > 70_000 def test_parse_docx_layout_table_cells_become_prose_not_rows( settings: ChunkingSettings, ) -> None: """A cell that alone exceeds chunk_size holds a document, not a field.""" units = _units(LAYOUT_DOCX, settings) assert ContentType.PARAGRAPH in _types(units) # ── failure modes ───────────────────────────────────────────────── def test_parse_docx_unopenable_bytes_raises_parse_error(settings: ChunkingSettings) -> None: with pytest.raises(DocumentParseError, match="Could not open DOCX"): parse_docx(b"not a docx at all", settings) def test_parse_docx_legacy_doc_binary_raises_parse_error(settings: ChunkingSettings) -> None: """`.doc` is rejected rather than converted (ADR-0018); python-docx cannot open it.""" with pytest.raises(DocumentParseError): parse_docx(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512, settings)