Ali Zarinkolah
5c0a5938f8
Why: - Plan 001 Phase 4 needs batched, concurrency-bounded embedding wired into the inline upload path, with process-wide capacity/timeout/chunk-limit guards (ADR-0017). - The BM25 analyzer and dense-model config are ported from the `emet` evaluation lab, which benchmarked them against the real Farsi corpus (bm25-fa-norm-stop; nomic-embed-text-v2-moe at 768-dim; text-embedding-3-large at native 3072-dim), closing open items in ADR-0001/ADR-0005. Changes: - New: embedding ports, orchestration (embed_chunks), request-bounds helpers, and dense/sparse adapters (analyzers.py, bm25.py, openai_compatible.py). - upload.py now parses/chunks/embeds inline behind INGESTION_MAX_CONCURRENCY (503), INGESTION_TIMEOUT_SECONDS (504), and the chunk-count ceiling (413); every failure path still writes a terminal job row. - Lifespan builds and warms both dense embedders at startup (fail-soft) and creates the sparse embedder and concurrency semaphore. - httpx moves from dev to main dependencies (adapters use it directly). Impact: - Qdrant point upserts are still Phase 5 -- chunks_indexed stays 0. - New EMBEDDING_* env vars documented in .env.example; safe defaults. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
151 lines
4.7 KiB
Python
151 lines
4.7 KiB
Python
"""The `fa_norm_stop` BM25 analyzer (ADR-0001, ADR-0005).
|
|
|
|
Ported from the `emet` evaluation lab
|
|
(`src/chatbot_gh/adapters/sparse/analyzers.py`), which benchmarked four Farsi
|
|
analyzer variants on the real corpus and found `fa_norm_stop` the best
|
|
performer. This is a *measured* artifact: changing the normalization,
|
|
tokenization, or stopword list invalidates that result, so improvements belong
|
|
in a new emet benchmark run rather than in an edit here.
|
|
|
|
Deliberately independent of `src/application/ingestion/normalization.py`.
|
|
Those solve different problems: `normalize_persian_text` shapes chunk content
|
|
that gets cited back to the reader, so ADR-0018 has it preserve digits and
|
|
punctuation as authored. This module shapes index terms nobody ever sees, so
|
|
it folds digits and diacritics freely. Sharing one function between them would
|
|
let a display-motivated tweak silently perturb the benchmarked sparse index.
|
|
"""
|
|
|
|
import re
|
|
import unicodedata
|
|
|
|
# Several Arabic letterforms are visually indistinguishable from Latin ones in
|
|
# a monospace editor (alef from "l", heh from "o"), and literals render
|
|
# right-to-left, visually reordering the source line. `normalization.py` writes
|
|
# them as codepoints for that reason; this module follows the same convention.
|
|
_ZWNJ = 0x200C
|
|
_ARABIC_YEH = 0x064A
|
|
_ARABIC_KAF = 0x0643
|
|
_TEH_MARBUTA = 0x0629
|
|
_HAMZA_ON_WAW = 0x0624
|
|
_ALEF_HAMZA_BELOW = 0x0625
|
|
_ALEF_HAMZA_ABOVE = 0x0623
|
|
|
|
_PERSIAN_YEH = 0x06CC
|
|
_PERSIAN_KEHEH = 0x06A9
|
|
_HEH = 0x0647
|
|
_WAW = 0x0648
|
|
_ALEF = 0x0627
|
|
_SPACE = 0x0020
|
|
|
|
# Persian (U+06F0-U+06F9) and Arabic-Indic (U+0660-U+0669) digits both fold to
|
|
# ASCII, so the same number matches however it was authored.
|
|
_EASTERN_DIGITS = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789")
|
|
|
|
# ZWNJ becomes a space (splitting compounds into separate terms) and the
|
|
# Arabic letterforms fold to their Persian equivalents. Every entry is a
|
|
# single codepoint mapping to a single codepoint over disjoint sources, so
|
|
# applying them in one pass is equivalent to emet's chained `str.replace`
|
|
# calls -- provided NFC runs first, since NFC is what composes the hamza
|
|
# forms this table then folds.
|
|
_FOLDING: dict[int, int] = {
|
|
_ZWNJ: _SPACE,
|
|
_ARABIC_YEH: _PERSIAN_YEH,
|
|
_ARABIC_KAF: _PERSIAN_KEHEH,
|
|
_TEH_MARBUTA: _HEH,
|
|
_HAMZA_ON_WAW: _WAW,
|
|
_ALEF_HAMZA_BELOW: _ALEF,
|
|
_ALEF_HAMZA_ABOVE: _ALEF,
|
|
}
|
|
_FOLDING.update(_EASTERN_DIGITS)
|
|
|
|
# Common Persian/Arabic stopwords (function words + FAQ noise), plus the
|
|
# English function words that appear in a mixed-script corpus. Kept small and
|
|
# explicit -- deliberately not a full hazm list. `_HEH_ALEF` is the plural
|
|
# suffix "ha"; written as a codepoint pair because both of its letters are
|
|
# Latin-confusable, which is exactly the case Ruff's RUF001 flags.
|
|
_HEH_ALEF = chr(_HEH) + chr(_ALEF)
|
|
|
|
_PERSIAN_STOPWORDS: frozenset[str] = frozenset(
|
|
{
|
|
"و",
|
|
"در",
|
|
"به",
|
|
"از",
|
|
"که",
|
|
"این",
|
|
"را",
|
|
"با",
|
|
"برای",
|
|
"آن",
|
|
"یک",
|
|
"است",
|
|
"شد",
|
|
"شده",
|
|
"می",
|
|
"های",
|
|
_HEH_ALEF,
|
|
"یا",
|
|
"تا",
|
|
"بر",
|
|
"اگر",
|
|
"هم",
|
|
"نیز",
|
|
"ولی",
|
|
"اما",
|
|
"چه",
|
|
"چون",
|
|
"روی",
|
|
"پس",
|
|
"پیش",
|
|
"هر",
|
|
"هیچ",
|
|
"بود",
|
|
"باشد",
|
|
"هست",
|
|
"نیست",
|
|
"کند",
|
|
"کرد",
|
|
"کردن",
|
|
"شود",
|
|
"the",
|
|
"a",
|
|
"an",
|
|
"of",
|
|
"to",
|
|
"and",
|
|
"in",
|
|
"on",
|
|
"for",
|
|
"is",
|
|
"are",
|
|
}
|
|
)
|
|
|
|
# Word characters minus underscore. Note this KEEPS digits: an insurance
|
|
# corpus is full of policy numbers, dates, and amounts, and those are exactly
|
|
# the tokens a lexical index should be able to match on.
|
|
_TOKEN_RE = re.compile(r"[^\W_]+", re.UNICODE)
|
|
|
|
FA_NORM_STOP = "fa_norm_stop"
|
|
|
|
|
|
def _normalize_fa(text: str) -> str:
|
|
return unicodedata.normalize("NFC", text).translate(_FOLDING)
|
|
|
|
|
|
def _tokenize_raw(text: str) -> list[str]:
|
|
return [m.group(0).lower() for m in _TOKEN_RE.finditer(text)]
|
|
|
|
|
|
def analyze(text: str, analyzer: str = FA_NORM_STOP) -> list[str]:
|
|
"""Tokenize `text` into sparse-index terms.
|
|
|
|
Only `fa_norm_stop` is implemented -- emet's other three variants
|
|
(`raw`, `fa_norm`, `fa_norm_stem`) lost the benchmark and exist there as
|
|
experiment arms, not as configurations this service should run.
|
|
"""
|
|
if analyzer != FA_NORM_STOP:
|
|
raise ValueError(f"Unknown analyzer '{analyzer}'")
|
|
tokens = _tokenize_raw(_normalize_fa(text))
|
|
return [token for token in tokens if token not in _PERSIAN_STOPWORDS]
|