"""Persian normalization (ADR-0018). Characters are written as escapes rather than literals: Arabic letterforms are visually indistinguishable in a monospace editor, and literals would render right-to-left and visually reorder each assertion. """ import pytest from src.application.ingestion import normalize_persian_text pytestmark = pytest.mark.unit ARABIC_KAF = chr(0x0643) PERSIAN_KEHEH = chr(0x06A9) ARABIC_YEH = chr(0x064A) ALEF_MAKSURA = chr(0x0649) PERSIAN_YEH = chr(0x06CC) ALEF_HAMZA_ABOVE = chr(0x0623) ALEF_HAMZA_BELOW = chr(0x0625) ALEF = chr(0x0627) ALEF_FINAL_FORM = chr(0xFE8E) FATHA = chr(0x064E) SHADDA = chr(0x0651) TATWEEL = chr(0x0640) SUPERSCRIPT_ALEF = chr(0x0670) PERSIAN_DIGITS = "".join(chr(0x06F1 + offset) for offset in range(3)) ARABIC_SEMICOLON = chr(0x061B) ARABIC_THOUSANDS_SEPARATOR = chr(0x066C) @pytest.mark.parametrize( ("source", "expected"), [ (ARABIC_KAF, PERSIAN_KEHEH), (ARABIC_YEH, PERSIAN_YEH), (ALEF_MAKSURA, PERSIAN_YEH), (ALEF_HAMZA_ABOVE, ALEF), (ALEF_HAMZA_BELOW, ALEF), ], ) def test_normalize_arabic_letterform_folds_to_persian(source: str, expected: str) -> None: assert normalize_persian_text(source) == expected def test_normalize_mixed_keyboard_spellings_converge_to_one_form() -> None: """The defect this module exists for: one word, two spellings, two embeddings.""" arabic_spelling = f"{ARABIC_KAF}{ARABIC_YEH}" persian_spelling = f"{PERSIAN_KEHEH}{PERSIAN_YEH}" assert normalize_persian_text(arabic_spelling) == normalize_persian_text(persian_spelling) @pytest.mark.parametrize("mark", [FATHA, SHADDA, TATWEEL, SUPERSCRIPT_ALEF]) def test_normalize_diacritic_is_removed(mark: str) -> None: assert normalize_persian_text(f"{ALEF}{mark}{ALEF}") == f"{ALEF}{ALEF}" def test_normalize_not_sign_becomes_space() -> None: assert normalize_persian_text(f"{ALEF}¬{ALEF}") == f"{ALEF} {ALEF}" def test_normalize_applies_nfkc_compatibility_forms() -> None: # U+FE8E is the final presentation form of alef; NFKC maps it to the base # letter, so a document pasted from a PDF renderer matches a typed one. assert normalize_persian_text(ALEF_FINAL_FORM) == ALEF def test_normalize_collapses_whitespace_runs_and_strips() -> None: assert normalize_persian_text(" a \t\n b ") == "a b" @pytest.mark.parametrize( "preserved", [PERSIAN_DIGITS, ARABIC_SEMICOLON, ARABIC_THOUSANDS_SEPARATOR], ) def test_normalize_digits_and_punctuation_survive_unchanged(preserved: str) -> None: """ADR-0018 deliberately folds letters only. Rewriting Persian digits to Western ones would render citations wrong to a Persian reader, so this guards against a later 'helpful' addition. """ assert normalize_persian_text(preserved) == preserved def test_normalize_already_normalized_text_is_unchanged() -> None: normalized = normalize_persian_text(f"{ARABIC_KAF}{FATHA} {ALEF_HAMZA_ABOVE}") assert normalize_persian_text(normalized) == normalized def test_normalize_empty_string_returns_empty() -> None: assert normalize_persian_text("") == ""