"""The `fa_norm_stop` BM25 analyzer (ADR-0001, ADR-0005).

Ported from the `emet` evaluation lab
(`src/chatbot_gh/adapters/sparse/analyzers.py`), which benchmarked four Farsi
analyzer variants on the real corpus and found `fa_norm_stop` the best
performer. This is a *measured* artifact: changing the normalization,
tokenization, or stopword list invalidates that result, so improvements belong
in a new emet benchmark run rather than in an edit here.

Deliberately independent of `src/application/ingestion/normalization.py`.
Those solve different problems: `normalize_persian_text` shapes chunk content
that gets cited back to the reader, so ADR-0018 has it preserve digits and
punctuation as authored. This module shapes index terms nobody ever sees, so
it folds digits and diacritics freely. Sharing one function between them would
let a display-motivated tweak silently perturb the benchmarked sparse index.
"""

import re
import unicodedata

# Several Arabic letterforms are visually indistinguishable from Latin ones in
# a monospace editor (alef from "l", heh from "o"), and literals render
# right-to-left, visually reordering the source line. `normalization.py` writes
# them as codepoints for that reason; this module follows the same convention.
_ZWNJ = 0x200C
_ARABIC_YEH = 0x064A
_ARABIC_KAF = 0x0643
_TEH_MARBUTA = 0x0629
_HAMZA_ON_WAW = 0x0624
_ALEF_HAMZA_BELOW = 0x0625
_ALEF_HAMZA_ABOVE = 0x0623

_PERSIAN_YEH = 0x06CC
_PERSIAN_KEHEH = 0x06A9
_HEH = 0x0647
_WAW = 0x0648
_ALEF = 0x0627
_SPACE = 0x0020

# Persian (U+06F0-U+06F9) and Arabic-Indic (U+0660-U+0669) digits both fold to
# ASCII, so the same number matches however it was authored.
_EASTERN_DIGITS = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789")

# ZWNJ becomes a space (splitting compounds into separate terms) and the
# Arabic letterforms fold to their Persian equivalents. Every entry is a
# single codepoint mapping to a single codepoint over disjoint sources, so
# applying them in one pass is equivalent to emet's chained `str.replace`
# calls -- provided NFC runs first, since NFC is what composes the hamza
# forms this table then folds.
_FOLDING: dict[int, int] = {
    _ZWNJ: _SPACE,
    _ARABIC_YEH: _PERSIAN_YEH,
    _ARABIC_KAF: _PERSIAN_KEHEH,
    _TEH_MARBUTA: _HEH,
    _HAMZA_ON_WAW: _WAW,
    _ALEF_HAMZA_BELOW: _ALEF,
    _ALEF_HAMZA_ABOVE: _ALEF,
}
_FOLDING.update(_EASTERN_DIGITS)

# Common Persian/Arabic stopwords (function words + FAQ noise), plus the
# English function words that appear in a mixed-script corpus. Kept small and
# explicit -- deliberately not a full hazm list. `_HEH_ALEF` is the plural
# suffix "ha"; written as a codepoint pair because both of its letters are
# Latin-confusable, which is exactly the case Ruff's RUF001 flags.
_HEH_ALEF = chr(_HEH) + chr(_ALEF)

_PERSIAN_STOPWORDS: frozenset[str] = frozenset(
    {
        "و",
        "در",
        "به",
        "از",
        "که",
        "این",
        "را",
        "با",
        "برای",
        "آن",
        "یک",
        "است",
        "شد",
        "شده",
        "می",
        "های",
        _HEH_ALEF,
        "یا",
        "تا",
        "بر",
        "اگر",
        "هم",
        "نیز",
        "ولی",
        "اما",
        "چه",
        "چون",
        "روی",
        "پس",
        "پیش",
        "هر",
        "هیچ",
        "بود",
        "باشد",
        "هست",
        "نیست",
        "کند",
        "کرد",
        "کردن",
        "شود",
        "the",
        "a",
        "an",
        "of",
        "to",
        "and",
        "in",
        "on",
        "for",
        "is",
        "are",
    }
)

# Word characters minus underscore. Note this KEEPS digits: an insurance
# corpus is full of policy numbers, dates, and amounts, and those are exactly
# the tokens a lexical index should be able to match on.
_TOKEN_RE = re.compile(r"[^\W_]+", re.UNICODE)

FA_NORM_STOP = "fa_norm_stop"


def _normalize_fa(text: str) -> str:
    return unicodedata.normalize("NFC", text).translate(_FOLDING)


def _tokenize_raw(text: str) -> list[str]:
    return [m.group(0).lower() for m in _TOKEN_RE.finditer(text)]


def analyze(text: str, analyzer: str = FA_NORM_STOP) -> list[str]:
    """Tokenize `text` into sparse-index terms.

    Only `fa_norm_stop` is implemented -- emet's other three variants
    (`raw`, `fa_norm`, `fa_norm_stem`) lost the benchmark and exist there as
    experiment arms, not as configurations this service should run.
    """
    if analyzer != FA_NORM_STOP:
        raise ValueError(f"Unknown analyzer '{analyzer}'")
    tokens = _tokenize_raw(_normalize_fa(text))
    return [token for token in tokens if token not in _PERSIAN_STOPWORDS]