Ali Zarinkolah
5cdfb70085
Adds src/application/ingestion/ -- Persian normalization, DOCX body walk with structural data/layout table classification, CSV/XLSX row rendering, and fixed-size token chunking (cl100k_base, 400/60/512) -- as pure functions per ADR-0015, tested against real production documents (asia_data_sample, kept out of the repo). ADR-0018 records where this diverges from ADR-0004 (fixed-size default, no invented headings/tree, structural table classification, header-provable labeling only). Plan 001's scope line is corrected from CSV-only to DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is updated to match current implementation state.
74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
"""Persian text normalization (ADR-0018).
|
|
|
|
Applied to every extracted text block before chunking, for all source formats.
|
|
|
|
The problem this solves is silent: Persian authored on mixed Arabic/Persian
|
|
keyboards contains both U+06A9 and U+0643 for "k", both U+06CC and U+064A for
|
|
"y". Those are distinct codepoints and therefore distinct tokens to every
|
|
embedding model, so the same word embeds two different ways depending on which
|
|
key the author pressed.
|
|
|
|
Letter folding only -- digits and punctuation are left as authored, because
|
|
chunk content is what citations render back to the reader and Western digits
|
|
inside Persian prose read as wrong.
|
|
"""
|
|
|
|
import re
|
|
import unicodedata
|
|
|
|
# Both tables are written as codepoints rather than character literals. Arabic
|
|
# letterforms are visually indistinguishable from one another (and alef from a
|
|
# Latin "l") in a monospace editor -- which is the very confusion this module
|
|
# exists to resolve -- and literals would render right-to-left, visually
|
|
# reordering the source line.
|
|
#
|
|
# `str.translate` accepts an ordinal->ordinal mapping directly, and an ordinal
|
|
# mapped to None is deleted.
|
|
|
|
_ARABIC_KAF = 0x0643
|
|
_ARABIC_YEH = 0x064A
|
|
_ALEF_MAKSURA = 0x0649
|
|
_ALEF_HAMZA_ABOVE = 0x0623
|
|
_ALEF_HAMZA_BELOW = 0x0625
|
|
_NOT_SIGN = 0x00AC
|
|
|
|
_PERSIAN_KEHEH = 0x06A9
|
|
_PERSIAN_YEH = 0x06CC
|
|
_ALEF = 0x0627
|
|
_SPACE = 0x0020
|
|
|
|
_LETTER_FOLDING: dict[int, int] = {
|
|
_ARABIC_KAF: _PERSIAN_KEHEH,
|
|
_ARABIC_YEH: _PERSIAN_YEH,
|
|
_ALEF_MAKSURA: _PERSIAN_YEH,
|
|
_ALEF_HAMZA_ABOVE: _ALEF,
|
|
_ALEF_HAMZA_BELOW: _ALEF,
|
|
# A soft-hyphen artifact from documents exported by older Word versions.
|
|
_NOT_SIGN: _SPACE,
|
|
}
|
|
|
|
_TATWEEL = 0x0640
|
|
_SUPERSCRIPT_ALEF = 0x0670
|
|
_HARAKAT = range(0x064B, 0x0660)
|
|
|
|
# Applied after NFKC, which can itself decompose presentation forms into a
|
|
# base letter plus a combining mark.
|
|
_MARK_REMOVAL: dict[int, int | None] = dict.fromkeys(_HARAKAT)
|
|
_MARK_REMOVAL[_TATWEEL] = None
|
|
_MARK_REMOVAL[_SUPERSCRIPT_ALEF] = None
|
|
|
|
_WHITESPACE = re.compile(r"\s+")
|
|
|
|
|
|
def normalize_persian_text(text: str) -> str:
|
|
"""Fold Arabic letterforms to Persian and collapse whitespace.
|
|
|
|
Call this per text block, **before** blocks are assembled into a document.
|
|
The whitespace collapse maps `\\n` to a space, so running it over assembled
|
|
markdown would flatten every heading and paragraph onto one line.
|
|
"""
|
|
text = text.translate(_LETTER_FOLDING)
|
|
text = unicodedata.normalize("NFKC", text)
|
|
text = text.translate(_MARK_REMOVAL)
|
|
return _WHITESPACE.sub(" ", text).strip()
|