Ali Zarinkolah
5cdfb70085
Adds src/application/ingestion/ -- Persian normalization, DOCX body walk with structural data/layout table classification, CSV/XLSX row rendering, and fixed-size token chunking (cl100k_base, 400/60/512) -- as pure functions per ADR-0015, tested against real production documents (asia_data_sample, kept out of the repo). ADR-0018 records where this diverges from ADR-0004 (fixed-size default, no invented headings/tree, structural table classification, header-provable labeling only). Plan 001's scope line is corrected from CSV-only to DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is updated to match current implementation state.
96 lines
3.1 KiB
Python
96 lines
3.1 KiB
Python
"""Persian normalization (ADR-0018).
|
|
|
|
Characters are written as escapes rather than literals: Arabic letterforms are
|
|
visually indistinguishable in a monospace editor, and literals would render
|
|
right-to-left and visually reorder each assertion.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from src.application.ingestion import normalize_persian_text
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
ARABIC_KAF = chr(0x0643)
|
|
PERSIAN_KEHEH = chr(0x06A9)
|
|
ARABIC_YEH = chr(0x064A)
|
|
ALEF_MAKSURA = chr(0x0649)
|
|
PERSIAN_YEH = chr(0x06CC)
|
|
ALEF_HAMZA_ABOVE = chr(0x0623)
|
|
ALEF_HAMZA_BELOW = chr(0x0625)
|
|
ALEF = chr(0x0627)
|
|
ALEF_FINAL_FORM = chr(0xFE8E)
|
|
|
|
FATHA = chr(0x064E)
|
|
SHADDA = chr(0x0651)
|
|
TATWEEL = chr(0x0640)
|
|
SUPERSCRIPT_ALEF = chr(0x0670)
|
|
|
|
PERSIAN_DIGITS = "".join(chr(0x06F1 + offset) for offset in range(3))
|
|
ARABIC_SEMICOLON = chr(0x061B)
|
|
ARABIC_THOUSANDS_SEPARATOR = chr(0x066C)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("source", "expected"),
|
|
[
|
|
(ARABIC_KAF, PERSIAN_KEHEH),
|
|
(ARABIC_YEH, PERSIAN_YEH),
|
|
(ALEF_MAKSURA, PERSIAN_YEH),
|
|
(ALEF_HAMZA_ABOVE, ALEF),
|
|
(ALEF_HAMZA_BELOW, ALEF),
|
|
],
|
|
)
|
|
def test_normalize_arabic_letterform_folds_to_persian(source: str, expected: str) -> None:
|
|
assert normalize_persian_text(source) == expected
|
|
|
|
|
|
def test_normalize_mixed_keyboard_spellings_converge_to_one_form() -> None:
|
|
"""The defect this module exists for: one word, two spellings, two embeddings."""
|
|
arabic_spelling = f"{ARABIC_KAF}{ARABIC_YEH}"
|
|
persian_spelling = f"{PERSIAN_KEHEH}{PERSIAN_YEH}"
|
|
|
|
assert normalize_persian_text(arabic_spelling) == normalize_persian_text(persian_spelling)
|
|
|
|
|
|
@pytest.mark.parametrize("mark", [FATHA, SHADDA, TATWEEL, SUPERSCRIPT_ALEF])
|
|
def test_normalize_diacritic_is_removed(mark: str) -> None:
|
|
assert normalize_persian_text(f"{ALEF}{mark}{ALEF}") == f"{ALEF}{ALEF}"
|
|
|
|
|
|
def test_normalize_not_sign_becomes_space() -> None:
|
|
assert normalize_persian_text(f"{ALEF}¬{ALEF}") == f"{ALEF} {ALEF}"
|
|
|
|
|
|
def test_normalize_applies_nfkc_compatibility_forms() -> None:
|
|
# U+FE8E is the final presentation form of alef; NFKC maps it to the base
|
|
# letter, so a document pasted from a PDF renderer matches a typed one.
|
|
assert normalize_persian_text(ALEF_FINAL_FORM) == ALEF
|
|
|
|
|
|
def test_normalize_collapses_whitespace_runs_and_strips() -> None:
|
|
assert normalize_persian_text(" a \t\n b ") == "a b"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"preserved",
|
|
[PERSIAN_DIGITS, ARABIC_SEMICOLON, ARABIC_THOUSANDS_SEPARATOR],
|
|
)
|
|
def test_normalize_digits_and_punctuation_survive_unchanged(preserved: str) -> None:
|
|
"""ADR-0018 deliberately folds letters only.
|
|
|
|
Rewriting Persian digits to Western ones would render citations wrong to a
|
|
Persian reader, so this guards against a later 'helpful' addition.
|
|
"""
|
|
assert normalize_persian_text(preserved) == preserved
|
|
|
|
|
|
def test_normalize_already_normalized_text_is_unchanged() -> None:
|
|
normalized = normalize_persian_text(f"{ARABIC_KAF}{FATHA} {ALEF_HAMZA_ABOVE}")
|
|
|
|
assert normalize_persian_text(normalized) == normalized
|
|
|
|
|
|
def test_normalize_empty_string_returns_empty() -> None:
|
|
assert normalize_persian_text("") == ""
|