"""Persian text normalization (ADR-0018).

Applied to every extracted text block before chunking, for all source formats.

The problem this solves is silent: Persian authored on mixed Arabic/Persian
keyboards contains both U+06A9 and U+0643 for "k", both U+06CC and U+064A for
"y". Those are distinct codepoints and therefore distinct tokens to every
embedding model, so the same word embeds two different ways depending on which
key the author pressed.

Letter folding only -- digits and punctuation are left as authored, because
chunk content is what citations render back to the reader and Western digits
inside Persian prose read as wrong.
"""

import re
import unicodedata

# Both tables are written as codepoints rather than character literals. Arabic
# letterforms are visually indistinguishable from one another (and alef from a
# Latin "l") in a monospace editor -- which is the very confusion this module
# exists to resolve -- and literals would render right-to-left, visually
# reordering the source line.
#
# `str.translate` accepts an ordinal->ordinal mapping directly, and an ordinal
# mapped to None is deleted.

_ARABIC_KAF = 0x0643
_ARABIC_YEH = 0x064A
_ALEF_MAKSURA = 0x0649
_ALEF_HAMZA_ABOVE = 0x0623
_ALEF_HAMZA_BELOW = 0x0625
_NOT_SIGN = 0x00AC

_PERSIAN_KEHEH = 0x06A9
_PERSIAN_YEH = 0x06CC
_ALEF = 0x0627
_SPACE = 0x0020

_LETTER_FOLDING: dict[int, int] = {
    _ARABIC_KAF: _PERSIAN_KEHEH,
    _ARABIC_YEH: _PERSIAN_YEH,
    _ALEF_MAKSURA: _PERSIAN_YEH,
    _ALEF_HAMZA_ABOVE: _ALEF,
    _ALEF_HAMZA_BELOW: _ALEF,
    # A soft-hyphen artifact from documents exported by older Word versions.
    _NOT_SIGN: _SPACE,
}

_TATWEEL = 0x0640
_SUPERSCRIPT_ALEF = 0x0670
_HARAKAT = range(0x064B, 0x0660)

# Applied after NFKC, which can itself decompose presentation forms into a
# base letter plus a combining mark.
_MARK_REMOVAL: dict[int, int | None] = dict.fromkeys(_HARAKAT)
_MARK_REMOVAL[_TATWEEL] = None
_MARK_REMOVAL[_SUPERSCRIPT_ALEF] = None

_WHITESPACE = re.compile(r"\s+")


def normalize_persian_text(text: str) -> str:
    """Fold Arabic letterforms to Persian and collapse whitespace.

    Call this per text block, **before** blocks are assembled into a document.
    The whitespace collapse maps `\\n` to a space, so running it over assembled
    markdown would flatten every heading and paragraph onto one line.
    """
    text = text.translate(_LETTER_FOLDING)
    text = unicodedata.normalize("NFKC", text)
    text = text.translate(_MARK_REMOVAL)
    return _WHITESPACE.sub(" ", text).strip()