chatbot_v3/tests/unit/application/test_docx_parser.py

"""DOCX parsing against the real sample corpus (ADR-0004, ADR-0018).

Every assertion here was written after reading the document it covers, so the
numbers are observations rather than predictions.
"""

import pytest

from src.application.ingestion import ContentType, DocumentParseError, parse_docx
from src.application.ingestion.docx_parser import heading_level_from_style
from src.config import ChunkingSettings
from tests.support.documents import (
    LAYOUT_DOCX,
    LIST_DOCX,
    MIXED_DOCX,
    PROSE_DOCX,
    TABLE_DOCX,
    load_document,
)

pytestmark = pytest.mark.unit


@pytest.fixture
def settings() -> ChunkingSettings:
    return ChunkingSettings()


def _units(filename: str, settings: ChunkingSettings):
    return parse_docx(load_document(filename), settings).units


def _types(units) -> set[ContentType]:
    return {unit.content_type for unit in units}


# ── heading styles ────────────────────────────────────────────────


@pytest.mark.parametrize(
    ("style", "expected"),
    [
        ("Heading1", 1),  # the styleId Word actually stores
        ("Heading 1", 1),  # the friendly name python-docx reports
        ("heading 2", 2),
        ("HEADING3", 3),
        ("Normal", None),
        ("List Paragraph", None),
        ("Heading", None),
        ("Headingfoo", None),
    ],
)
def test_heading_level_from_style_maps_word_styles_only(style: str, expected: int | None) -> None:
    assert heading_level_from_style(style) == expected


def test_parse_docx_invents_no_headings_when_document_declares_none(
    settings: ChunkingSettings,
) -> None:
    """Every document in the corpus lacks Heading styles, so this is the norm.

    The evaluation repo upgraded paragraphs to headings by text pattern; a
    wrongly-detected heading silently reshapes the tree, so ADR-0018 drops it.
    """
    parsed = parse_docx(load_document(PROSE_DOCX), settings)

    assert "#" not in (parsed.markdown or "")


# ── prose documents ───────────────────────────────────────────────


def test_parse_docx_prose_document_becomes_one_flowing_block(
    settings: ChunkingSettings,
) -> None:
    """Consecutive paragraphs accumulate into one prose unit (ADR-0004).

    One unit per paragraph would hand the splitter 36 fragments instead of
    flowing text.
    """
    units = _units(PROSE_DOCX, settings)

    assert len(units) == 1
    assert units[0].content_type is ContentType.PARAGRAPH
    assert units[0].text.count("\n\n") == 35


def test_parse_docx_list_styled_paragraphs_stay_prose(settings: ChunkingSettings) -> None:
    units = _units(LIST_DOCX, settings)

    assert _types(units) == {ContentType.PARAGRAPH}


def test_parse_docx_normalizes_persian_letterforms(settings: ChunkingSettings) -> None:
    parsed = parse_docx(load_document(PROSE_DOCX), settings)

    # No Arabic kaf or yeh survives normalization.
    assert chr(0x0643) not in (parsed.markdown or "")
    assert chr(0x064A) not in (parsed.markdown or "")


def test_parse_docx_preserves_paragraph_breaks_in_markdown(
    settings: ChunkingSettings,
) -> None:
    """Normalization collapses whitespace, so it must run per block.

    Applied to the assembled document it would flatten every paragraph onto a
    single line -- the easiest way to get this wrong.
    """
    parsed = parse_docx(load_document(PROSE_DOCX), settings)

    assert "\n\n" in (parsed.markdown or "")


# ── data tables ───────────────────────────────────────────────────


def test_parse_docx_data_table_becomes_one_unit_per_row(settings: ChunkingSettings) -> None:
    """A 35-row contact table: one header row plus 34 data rows."""
    units = _units(TABLE_DOCX, settings)

    assert len(units) == 34
    assert _types(units) == {ContentType.TABLE_ROW}


def test_parse_docx_data_table_rows_are_labeled_with_headers(
    settings: ChunkingSettings,
) -> None:
    units = _units(TABLE_DOCX, settings)

    assert "\n" in units[0].text
    assert all(":" in line for line in units[0].text.split("\n"))


def test_parse_docx_vertically_merged_cell_repeats_on_every_row(
    settings: ChunkingSettings,
) -> None:
    """The group column is merged down several rows; each row must carry it."""
    units = _units(TABLE_DOCX, settings)

    labels = [line.split(":")[0] for unit in units for line in unit.text.split("\n")]
    group_label = units[1].text.split(":")[0]

    assert labels.count(group_label) > 1


def test_parse_docx_empty_cell_is_omitted_not_rendered_as_bare_label(
    settings: ChunkingSettings,
) -> None:
    """The first data row has no group; it should have one fewer field."""
    units = _units(TABLE_DOCX, settings)

    assert len(units[0].text.split("\n")) < len(units[1].text.split("\n"))


# ── tables without a header ───────────────────────────────────────


def test_parse_docx_headerless_table_keeps_every_row(settings: ChunkingSettings) -> None:
    """A 30-row compensation list plus an 11-row table, all rows preserved.

    Regression: scanning forward for a header row discarded the rows above the
    match and labeled the rest from a data row (`80: 70`).
    """
    units = _units(MIXED_DOCX, settings)
    rows = [unit for unit in units if unit.content_type is ContentType.TABLE_ROW]

    assert len(rows) == 41


def test_parse_docx_headerless_table_rows_are_unlabeled(settings: ChunkingSettings) -> None:
    """No header is provable, so cells are joined rather than mislabeled."""
    units = _units(MIXED_DOCX, settings)
    first_row = next(unit for unit in units if unit.content_type is ContentType.TABLE_ROW)

    assert " | " in first_row.text
    assert first_row.text.startswith("1 | ")


def test_parse_docx_mixed_document_interleaves_prose_and_rows(
    settings: ChunkingSettings,
) -> None:
    units = _units(MIXED_DOCX, settings)

    assert _types(units) == {ContentType.PARAGRAPH, ContentType.TABLE_ROW}


# ── layout tables ─────────────────────────────────────────────────


def test_parse_docx_extracts_document_held_inside_a_table_cell(
    settings: ChunkingSettings,
) -> None:
    """One merged cell holds 77k characters across 738 paragraphs.

    Regression: a merged cell reported once per spanned grid position was
    deduplicated by `id()`, which lxml reuses across element proxies, so the
    whole sub-document vanished and the file yielded 70 tokens.
    """
    units = _units(LAYOUT_DOCX, settings)

    assert sum(len(unit.text) for unit in units) > 70_000


def test_parse_docx_layout_table_cells_become_prose_not_rows(
    settings: ChunkingSettings,
) -> None:
    """A cell that alone exceeds chunk_size holds a document, not a field."""
    units = _units(LAYOUT_DOCX, settings)

    assert ContentType.PARAGRAPH in _types(units)


# ── failure modes ─────────────────────────────────────────────────


def test_parse_docx_unopenable_bytes_raises_parse_error(settings: ChunkingSettings) -> None:
    with pytest.raises(DocumentParseError, match="Could not open DOCX"):
        parse_docx(b"not a docx at all", settings)


def test_parse_docx_legacy_doc_binary_raises_parse_error(settings: ChunkingSettings) -> None:
    """`.doc` is rejected rather than converted (ADR-0018); python-docx cannot open it."""
    with pytest.raises(DocumentParseError):
        parse_docx(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512, settings)