feat(ingestion): add DOCX/CSV/XLSX parsing and fixed-size chunking (ADR-0018)
Adds src/application/ingestion/ -- Persian normalization, DOCX body walk with structural data/layout table classification, CSV/XLSX row rendering, and fixed-size token chunking (cl100k_base, 400/60/512) -- as pure functions per ADR-0015, tested against real production documents (asia_data_sample, kept out of the repo). ADR-0018 records where this diverges from ADR-0004 (fixed-size default, no invented headings/tree, structural table classification, header-provable labeling only). Plan 001's scope line is corrected from CSV-only to DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is updated to match current implementation state.
This commit is contained in:
95
tests/unit/application/test_normalization.py
Normal file
95
tests/unit/application/test_normalization.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""Persian normalization (ADR-0018).
|
||||
|
||||
Characters are written as escapes rather than literals: Arabic letterforms are
|
||||
visually indistinguishable in a monospace editor, and literals would render
|
||||
right-to-left and visually reorder each assertion.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from src.application.ingestion import normalize_persian_text
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
ARABIC_KAF = chr(0x0643)
|
||||
PERSIAN_KEHEH = chr(0x06A9)
|
||||
ARABIC_YEH = chr(0x064A)
|
||||
ALEF_MAKSURA = chr(0x0649)
|
||||
PERSIAN_YEH = chr(0x06CC)
|
||||
ALEF_HAMZA_ABOVE = chr(0x0623)
|
||||
ALEF_HAMZA_BELOW = chr(0x0625)
|
||||
ALEF = chr(0x0627)
|
||||
ALEF_FINAL_FORM = chr(0xFE8E)
|
||||
|
||||
FATHA = chr(0x064E)
|
||||
SHADDA = chr(0x0651)
|
||||
TATWEEL = chr(0x0640)
|
||||
SUPERSCRIPT_ALEF = chr(0x0670)
|
||||
|
||||
PERSIAN_DIGITS = "".join(chr(0x06F1 + offset) for offset in range(3))
|
||||
ARABIC_SEMICOLON = chr(0x061B)
|
||||
ARABIC_THOUSANDS_SEPARATOR = chr(0x066C)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source", "expected"),
|
||||
[
|
||||
(ARABIC_KAF, PERSIAN_KEHEH),
|
||||
(ARABIC_YEH, PERSIAN_YEH),
|
||||
(ALEF_MAKSURA, PERSIAN_YEH),
|
||||
(ALEF_HAMZA_ABOVE, ALEF),
|
||||
(ALEF_HAMZA_BELOW, ALEF),
|
||||
],
|
||||
)
|
||||
def test_normalize_arabic_letterform_folds_to_persian(source: str, expected: str) -> None:
|
||||
assert normalize_persian_text(source) == expected
|
||||
|
||||
|
||||
def test_normalize_mixed_keyboard_spellings_converge_to_one_form() -> None:
|
||||
"""The defect this module exists for: one word, two spellings, two embeddings."""
|
||||
arabic_spelling = f"{ARABIC_KAF}{ARABIC_YEH}"
|
||||
persian_spelling = f"{PERSIAN_KEHEH}{PERSIAN_YEH}"
|
||||
|
||||
assert normalize_persian_text(arabic_spelling) == normalize_persian_text(persian_spelling)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mark", [FATHA, SHADDA, TATWEEL, SUPERSCRIPT_ALEF])
|
||||
def test_normalize_diacritic_is_removed(mark: str) -> None:
|
||||
assert normalize_persian_text(f"{ALEF}{mark}{ALEF}") == f"{ALEF}{ALEF}"
|
||||
|
||||
|
||||
def test_normalize_not_sign_becomes_space() -> None:
|
||||
assert normalize_persian_text(f"{ALEF}¬{ALEF}") == f"{ALEF} {ALEF}"
|
||||
|
||||
|
||||
def test_normalize_applies_nfkc_compatibility_forms() -> None:
|
||||
# U+FE8E is the final presentation form of alef; NFKC maps it to the base
|
||||
# letter, so a document pasted from a PDF renderer matches a typed one.
|
||||
assert normalize_persian_text(ALEF_FINAL_FORM) == ALEF
|
||||
|
||||
|
||||
def test_normalize_collapses_whitespace_runs_and_strips() -> None:
|
||||
assert normalize_persian_text(" a \t\n b ") == "a b"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"preserved",
|
||||
[PERSIAN_DIGITS, ARABIC_SEMICOLON, ARABIC_THOUSANDS_SEPARATOR],
|
||||
)
|
||||
def test_normalize_digits_and_punctuation_survive_unchanged(preserved: str) -> None:
|
||||
"""ADR-0018 deliberately folds letters only.
|
||||
|
||||
Rewriting Persian digits to Western ones would render citations wrong to a
|
||||
Persian reader, so this guards against a later 'helpful' addition.
|
||||
"""
|
||||
assert normalize_persian_text(preserved) == preserved
|
||||
|
||||
|
||||
def test_normalize_already_normalized_text_is_unchanged() -> None:
|
||||
normalized = normalize_persian_text(f"{ARABIC_KAF}{FATHA} {ALEF_HAMZA_ABOVE}")
|
||||
|
||||
assert normalize_persian_text(normalized) == normalized
|
||||
|
||||
|
||||
def test_normalize_empty_string_returns_empty() -> None:
|
||||
assert normalize_persian_text("") == ""
|
||||
Reference in New Issue
Block a user