feat(ingestion): add DOCX/CSV/XLSX parsing and fixed-size chunking (ADR-0018)

Adds src/application/ingestion/ -- Persian normalization, DOCX body
walk with structural data/layout table classification, CSV/XLSX row
rendering, and fixed-size token chunking (cl100k_base, 400/60/512) --
as pure functions per ADR-0015, tested against real production
documents (asia_data_sample, kept out of the repo). ADR-0018 records
where this diverges from ADR-0004 (fixed-size default, no invented
headings/tree, structural table classification, header-provable
labeling only). Plan 001's scope line is corrected from CSV-only to
DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is
updated to match current implementation state.

src/application/ingestion/__init__.py

@@ -0,0 +1,47 @@
+"""Document parsing and fixed-size chunking (ADR-0004, ADR-0018).
+
+Everything here is pure and synchronous: no I/O, no ports, no SDK clients
+(ADR-0015 reserves ports for external side effects). Parsing and chunking are
+blocking CPU work, so callers run them through `anyio.to_thread.run_sync` with
+the ingestion `CapacityLimiter` rather than on the event loop (ADR-0017).
+"""
+
+from src.application.ingestion.chunking import chunk_document, chunk_id_for, split_by_tokens
+from src.application.ingestion.docx_parser import parse_docx
+from src.application.ingestion.errors import (
+    ChunkLimitExceededError,
+    ChunkTooLargeError,
+    DocumentParseError,
+    IngestionError,
+    UnsupportedSourceTypeError,
+)
+from src.application.ingestion.models import (
+    Chunk,
+    ContentType,
+    ParsedDocument,
+    StructuralUnit,
+)
+from src.application.ingestion.normalization import normalize_persian_text
+from src.application.ingestion.spreadsheet_parser import parse_csv, parse_xlsx
+from src.application.ingestion.tokenizer import count_tokens, get_encoder
+
+__all__ = [
+    "Chunk",
+    "ChunkLimitExceededError",
+    "ChunkTooLargeError",
+    "ContentType",
+    "DocumentParseError",
+    "IngestionError",
+    "ParsedDocument",
+    "StructuralUnit",
+    "UnsupportedSourceTypeError",
+    "chunk_document",
+    "chunk_id_for",
+    "count_tokens",
+    "get_encoder",
+    "normalize_persian_text",
+    "parse_csv",
+    "parse_docx",
+    "parse_xlsx",
+    "split_by_tokens",
+]

src/application/ingestion/chunking.py

@@ -0,0 +1,129 @@
+"""Fixed-size chunking with overlap (ADR-0018).
+
+DOCX markdown is split on token windows; spreadsheet rows are already atomic
+and bypass the splitter, falling through it only when a single row exceeds the
+embedding model's sequence length.
+"""
+
+import uuid
+
+from src.application.ingestion.errors import ChunkTooLargeError, DocumentParseError
+from src.application.ingestion.models import Chunk, ContentType, ParsedDocument
+from src.application.ingestion.tokenizer import count_tokens, get_encoder
+from src.config import ChunkingSettings
+
+# Fixed namespace so chunk ids stay stable across processes and releases.
+CHUNK_ID_NAMESPACE = uuid.UUID("6f9619ff-8b86-d011-b42d-00c04fc964ff")
+
+
+def chunk_id_for(file_id: uuid.UUID, chunk_index: int) -> uuid.UUID:
+    """Return the deterministic point id for a chunk (ADR-0001).
+
+    Derived from `file_id` and the immutable ingestion ordinal, so re-ingesting
+    a file upserts its points instead of duplicating them.
+    """
+    return uuid.uuid5(CHUNK_ID_NAMESPACE, f"{file_id}:{chunk_index}")
+
+
+def split_by_tokens(text: str, *, chunk_size: int, overlap: int, encoding_name: str) -> list[str]:
+    """Split text into overlapping windows of at most `chunk_size` tokens."""
+    if overlap >= chunk_size:
+        raise ValueError(f"overlap ({overlap}) must be smaller than chunk_size ({chunk_size})")
+
+    encoder = get_encoder(encoding_name)
+    tokens = encoder.encode(text)
+    if len(tokens) <= chunk_size:
+        return [text]
+
+    windows: list[str] = []
+    start = 0
+    while start < len(tokens):
+        end = min(start + chunk_size, len(tokens))
+        windows.append(encoder.decode(tokens[start:end]))
+        if end >= len(tokens):
+            break
+        start = end - overlap
+
+    return windows
+
+
+def _split_oversized(text: str, settings: ChunkingSettings) -> list[str]:
+    """Split a row only if it exceeds the cap; otherwise keep it atomic."""
+    if count_tokens(text, settings.encoding_name) <= settings.max_chunk_tokens:
+        return [text]
+    return split_by_tokens(
+        text,
+        chunk_size=settings.chunk_size,
+        overlap=settings.chunk_overlap,
+        encoding_name=settings.encoding_name,
+    )
+
+
+def _content_units(
+    parsed: ParsedDocument, settings: ChunkingSettings
+) -> list[tuple[str, ContentType]]:
+    """Reduce a parsed document's structural units to ordered text pieces.
+
+    Prose is cut into token windows; a table row is atomic and survives whole
+    unless it alone exceeds the model's sequence length (ADR-0004).
+    """
+    if not parsed.units:
+        raise DocumentParseError("Parsed document has no structural units")
+
+    pieces: list[tuple[str, ContentType]] = []
+    for unit in parsed.units:
+        if unit.content_type is ContentType.PARAGRAPH:
+            windows = split_by_tokens(
+                unit.text,
+                chunk_size=settings.chunk_size,
+                overlap=settings.chunk_overlap,
+                encoding_name=settings.encoding_name,
+            )
+        else:
+            windows = _split_oversized(unit.text, settings)
+        pieces.extend((window, unit.content_type) for window in windows)
+
+    return pieces
+
+
+def chunk_document(
+    parsed: ParsedDocument,
+    *,
+    file_id: uuid.UUID,
+    settings: ChunkingSettings,
+) -> list[Chunk]:
+    """Turn a parsed document into ordered, neighbor-linked chunks."""
+    units = [
+        (text.strip(), content_type) for text, content_type in _content_units(parsed, settings)
+    ]
+    # Drop blanks *before* assigning indices: an index gap would break the
+    # previous/next chain that retrieval-time expansion walks.
+    units = [(text, content_type) for text, content_type in units if text]
+
+    chunks: list[Chunk] = []
+    for index, (text, content_type) in enumerate(units):
+        token_count = count_tokens(text, settings.encoding_name)
+        if token_count > settings.max_chunk_tokens:
+            raise ChunkTooLargeError(
+                f"chunk {index} is {token_count} tokens, over the "
+                f"{settings.max_chunk_tokens}-token cap"
+            )
+        chunks.append(
+            Chunk(
+                chunk_id=chunk_id_for(file_id, index),
+                chunk_index=index,
+                order_id=float(index + 1),
+                content=text,
+                content_type=content_type,
+                token_count=token_count,
+                character_count=len(text),
+            )
+        )
+
+    for position, chunk in enumerate(chunks):
+        if position > 0:
+            chunk.previous_chunk_id = chunks[position - 1].chunk_id
+        if position < len(chunks) - 1:
+            chunk.next_chunk_id = chunks[position + 1].chunk_id
+
+    return chunks

src/application/ingestion/docx_parser.py

@@ -0,0 +1,165 @@
+"""DOCX parsing into ordered structural units (ADR-0004, ADR-0018).
+
+The body is walked in document order and decomposed into structural units
+before any chunking runs: runs of flowing prose become `PARAGRAPH` units the
+splitter cuts into token windows, and data-table rows become atomic
+`TABLE_ROW` units.
+
+The one classification this makes is between a *data* table and a table used
+as page layout, and it is made structurally rather than by inspecting content:
+a data cell fits inside a chunk by definition, so a table holding a cell that
+alone exceeds `chunk_size`, or a cell containing nested tables, is a layout
+container whose cells are prose.
+"""
+
+import io
+
+from docx import Document
+from docx.document import Document as DocxDocument
+from docx.oxml.table import CT_Tbl
+from docx.oxml.text.paragraph import CT_P
+from docx.table import Table, _Cell
+from docx.text.paragraph import Paragraph
+
+from src.application.ingestion.errors import DocumentParseError
+from src.application.ingestion.models import (
+    ContentType,
+    ParsedDocument,
+    StructuralUnit,
+)
+from src.application.ingestion.normalization import normalize_persian_text
+from src.application.ingestion.tabular import clean_rows, render_rows
+from src.application.ingestion.tokenizer import count_tokens
+from src.config import ChunkingSettings
+
+_NORMAL_STYLE = "Normal"
+
+
+def heading_level_from_style(style_name: str) -> int | None:
+    """Return the heading level of a paragraph style, or None for body text.
+
+    Word stores the styleId (`Heading1`), not the friendly name (`Heading 1`),
+    so both spellings must resolve. Only real Word styles count -- no heading
+    is ever inferred from the text itself (ADR-0018).
+    """
+    normalized = style_name.strip().lower().replace(" ", "")
+    if not normalized.startswith("heading"):
+        return None
+    suffix = normalized.removeprefix("heading")
+    return int(suffix) if suffix.isdigit() else None
+
+
+def _paragraph_style(paragraph: Paragraph) -> str:
+    style = paragraph.style.name if paragraph.style is not None else None
+    return style or _NORMAL_STYLE
+
+
+def _is_layout_table(table: Table, settings: ChunkingSettings) -> bool:
+    """Whether a table is page layout rather than data.
+
+    Structural, not a content heuristic: a data cell is small enough to be a
+    chunk, so a cell that alone overflows `chunk_size` -- or that nests another
+    table -- holds a document, not a field. In the sample corpus this separates
+    by two orders of magnitude (32-171 tokens for data tables against 54,007
+    for a cell containing a whole sub-document).
+    """
+    for row in table.rows:
+        for cell in row.cells:
+            if cell.tables:
+                return True
+            if count_tokens(cell.text, settings.encoding_name) > settings.chunk_size:
+                return True
+    return False
+
+
+def _cell_prose(cell: _Cell, settings: ChunkingSettings) -> list[StructuralUnit]:
+    """Extract a layout cell's contents as units, recursing into nested tables."""
+    units: list[StructuralUnit] = []
+    for block in _iter_block_items(cell, settings):
+        units.append(block)
+    return units
+
+
+def _table_units(table: Table, settings: ChunkingSettings) -> list[StructuralUnit]:
+    """Convert a table to structural units."""
+    if _is_layout_table(table, settings):
+        units: list[StructuralUnit] = []
+        # Walk the physical `w:tc` elements rather than `row.cells`, which
+        # repeats a merged cell once per grid position it spans. Identity
+        # tracking is not an option here: lxml builds element proxies on
+        # demand, so `id()` is neither stable nor unique across them.
+        for row in table.rows:
+            for tc in row._tr.tc_lst:
+                units.extend(_cell_prose(_Cell(tc, table), settings))
+        return units
+
+    rows = clean_rows([[cell.text for cell in row.cells] for row in table.rows])
+    return [
+        StructuralUnit(text=text, content_type=ContentType.TABLE_ROW) for text in render_rows(rows)
+    ]
+
+
+def _iter_block_items(
+    container: DocxDocument | _Cell, settings: ChunkingSettings
+) -> list[StructuralUnit]:
+    """Walk a body or cell in document order, emitting structural units.
+
+    Consecutive paragraphs accumulate into one prose unit rather than becoming
+    one unit each: ADR-0004 treats "the whole remaining run of paragraphs" as a
+    single prose block, so the splitter sees flowing text instead of a series
+    of one-sentence fragments.
+    """
+    element = container.element.body if isinstance(container, DocxDocument) else container._tc
+
+    units: list[StructuralUnit] = []
+    prose: list[str] = []
+
+    def flush() -> None:
+        if prose:
+            units.append(
+                StructuralUnit(text="\n\n".join(prose), content_type=ContentType.PARAGRAPH)
+            )
+            prose.clear()
+
+    for child in element:
+        if isinstance(child, CT_P):
+            paragraph = Paragraph(child, container)
+            # `Paragraph.text` includes hyperlink text, which a raw `w:r` walk
+            # silently drops.
+            text = normalize_persian_text(paragraph.text)
+            if not text:
+                continue
+            level = heading_level_from_style(_paragraph_style(paragraph))
+            prose.append(f"{'#' * level} {text}" if level else text)
+        elif isinstance(child, CT_Tbl):
+            table_units = _table_units(Table(child, container), settings)
+            # A layout table is prose; keep it in the surrounding prose block
+            # instead of fragmenting the document around it.
+            if table_units and all(
+                unit.content_type is ContentType.PARAGRAPH for unit in table_units
+            ):
+                prose.extend(unit.text for unit in table_units)
+            else:
+                flush()
+                units.extend(table_units)
+
+    flush()
+    return units
+
+
+def parse_docx(data: bytes, settings: ChunkingSettings) -> ParsedDocument:
+    """Parse DOCX bytes into ordered structural units."""
+    try:
+        doc = Document(io.BytesIO(data))
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open DOCX: {exc}") from exc
+
+    units = _iter_block_items(doc, settings)
+    if not units:
+        raise DocumentParseError("Document contains no text content")
+
+    return ParsedDocument(
+        units=units,
+        markdown="\n\n".join(unit.text for unit in units),
+        block_count=len(units),
+    )

src/application/ingestion/errors.py

@@ -0,0 +1,41 @@
+"""Errors raised by the parsing and chunking pipeline (ADR-0018).
+
+These carry no HTTP knowledge — the API layer maps them to status codes
+(ADR-0015: `application/` contains no FastAPI request objects).
+"""
+
+
+class IngestionError(Exception):
+    """Base class for ingestion failures."""
+
+
+class DocumentParseError(IngestionError):
+    """A source file could not be decoded, opened, or yielded no text.
+
+    Maps to `400` per ADR-0017 ("unparseable file → 400").
+    """
+
+
+class UnsupportedSourceTypeError(IngestionError):
+    """A source file's type is not ingestible in this version.
+
+    Maps to `415`. `.doc` lands here until an out-of-process conversion
+    service exists (ADR-0018).
+    """
+
+
+class ChunkLimitExceededError(IngestionError):
+    """A document produced more chunks than `INGESTION_MAX_CHUNKS_PER_FILE`.
+
+    Maps to `413` per ADR-0017.
+    """
+
+
+class ChunkTooLargeError(IngestionError):
+    """A chunk exceeded the embedding model's sequence length.
+
+    This is an internal invariant violation, not a user error: the splitter is
+    supposed to make it impossible. It exists because the failure it guards
+    against is silent — `nomic-embed-text-v2-moe` truncates over-long input
+    without raising (ADR-0004).
+    """

src/application/ingestion/models.py

@@ -0,0 +1,66 @@
+"""Domain models for parsing and chunking (ADR-0001, ADR-0004, ADR-0018)."""
+
+import uuid
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+class ContentType(StrEnum):
+    """ADR-0004's finalized `content_type` value set.
+
+    v1 emits `PARAGRAPH` and `TABLE_ROW` only; `QA_PAIR` and `IMAGE_CAPTION`
+    are defined but not produced yet (ADR-0018).
+    """
+
+    PARAGRAPH = "paragraph"
+    TABLE_ROW = "table_row"
+    QA_PAIR = "qa_pair"
+    IMAGE_CAPTION = "image_caption"
+
+
+class StructuralUnit(BaseModel):
+    """One structural unit of a document, in reading order (ADR-0004).
+
+    A document is decomposed into these *before* any chunking runs, because
+    the two kinds are chunked differently:
+
+    - `PARAGRAPH` is a run of flowing prose; the fixed-size splitter cuts it
+      into token windows.
+    - `TABLE_ROW` is already atomic; it becomes one chunk, and is split only
+      when a single row is too large for the embedding model.
+    """
+
+    text: str
+    content_type: ContentType
+
+
+class ParsedDocument(BaseModel):
+    """The output of a parser, before chunking.
+
+    `units` is the content, in document order. `markdown` is the same content
+    rendered as one string, for eyeballing a parse; nothing chunks from it.
+    """
+
+    units: list[StructuralUnit] = Field(default_factory=list)
+    markdown: str | None = None
+    block_count: int = 0
+
+
+class Chunk(BaseModel):
+    """One indexable unit of a document.
+
+    `chunk_id` is a deterministic UUIDv5 of `file_id` and `chunk_index`
+    (ADR-0001), so re-ingesting a file upserts its points rather than
+    duplicating them.
+    """
+
+    chunk_id: uuid.UUID
+    chunk_index: int
+    order_id: float
+    content: str
+    content_type: ContentType
+    previous_chunk_id: uuid.UUID | None = None
+    next_chunk_id: uuid.UUID | None = None
+    token_count: int
+    character_count: int

src/application/ingestion/normalization.py

@@ -0,0 +1,73 @@
+"""Persian text normalization (ADR-0018).
+
+Applied to every extracted text block before chunking, for all source formats.
+
+The problem this solves is silent: Persian authored on mixed Arabic/Persian
+keyboards contains both U+06A9 and U+0643 for "k", both U+06CC and U+064A for
+"y". Those are distinct codepoints and therefore distinct tokens to every
+embedding model, so the same word embeds two different ways depending on which
+key the author pressed.
+
+Letter folding only -- digits and punctuation are left as authored, because
+chunk content is what citations render back to the reader and Western digits
+inside Persian prose read as wrong.
+"""
+
+import re
+import unicodedata
+
+# Both tables are written as codepoints rather than character literals. Arabic
+# letterforms are visually indistinguishable from one another (and alef from a
+# Latin "l") in a monospace editor -- which is the very confusion this module
+# exists to resolve -- and literals would render right-to-left, visually
+# reordering the source line.
+#
+# `str.translate` accepts an ordinal->ordinal mapping directly, and an ordinal
+# mapped to None is deleted.
+
+_ARABIC_KAF = 0x0643
+_ARABIC_YEH = 0x064A
+_ALEF_MAKSURA = 0x0649
+_ALEF_HAMZA_ABOVE = 0x0623
+_ALEF_HAMZA_BELOW = 0x0625
+_NOT_SIGN = 0x00AC
+
+_PERSIAN_KEHEH = 0x06A9
+_PERSIAN_YEH = 0x06CC
+_ALEF = 0x0627
+_SPACE = 0x0020
+
+_LETTER_FOLDING: dict[int, int] = {
+    _ARABIC_KAF: _PERSIAN_KEHEH,
+    _ARABIC_YEH: _PERSIAN_YEH,
+    _ALEF_MAKSURA: _PERSIAN_YEH,
+    _ALEF_HAMZA_ABOVE: _ALEF,
+    _ALEF_HAMZA_BELOW: _ALEF,
+    # A soft-hyphen artifact from documents exported by older Word versions.
+    _NOT_SIGN: _SPACE,
+}
+
+_TATWEEL = 0x0640
+_SUPERSCRIPT_ALEF = 0x0670
+_HARAKAT = range(0x064B, 0x0660)
+
+# Applied after NFKC, which can itself decompose presentation forms into a
+# base letter plus a combining mark.
+_MARK_REMOVAL: dict[int, int | None] = dict.fromkeys(_HARAKAT)
+_MARK_REMOVAL[_TATWEEL] = None
+_MARK_REMOVAL[_SUPERSCRIPT_ALEF] = None
+
+_WHITESPACE = re.compile(r"\s+")
+
+
+def normalize_persian_text(text: str) -> str:
+    """Fold Arabic letterforms to Persian and collapse whitespace.
+
+    Call this per text block, **before** blocks are assembled into a document.
+    The whitespace collapse maps `\\n` to a space, so running it over assembled
+    markdown would flatten every heading and paragraph onto one line.
+    """
+    text = text.translate(_LETTER_FOLDING)
+    text = unicodedata.normalize("NFKC", text)
+    text = text.translate(_MARK_REMOVAL)
+    return _WHITESPACE.sub(" ", text).strip()

src/application/ingestion/spreadsheet_parser.py

@@ -0,0 +1,121 @@
+"""CSV and XLSX parsing: row = chunk (ADR-0004, ADR-0018).
+
+Both formats reduce to rows and hand them to the shared renderer in
+`tabular`, so a Q&A sheet and a branch directory go through one code path with
+no shape detection.
+"""
+
+import csv
+import io
+
+import openpyxl
+from openpyxl.worksheet.worksheet import Worksheet
+
+from src.application.ingestion.errors import DocumentParseError
+from src.application.ingestion.models import ContentType, ParsedDocument, StructuralUnit
+from src.application.ingestion.tabular import Row, clean_cell, clean_rows, render_rows
+
+# Farsi exports from older Excel are frequently cp1256 (Windows Arabic).
+_ENCODINGS = ("utf-8-sig", "utf-8", "cp1256")
+
+_SNIFF_BYTES = 8192
+
+
+def _decode(data: bytes) -> str:
+    for encoding in _ENCODINGS:
+        try:
+            return data.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+    raise DocumentParseError(f"Could not decode file as any of: {', '.join(_ENCODINGS)}")
+
+
+def _to_units(rendered: list[str]) -> list[StructuralUnit]:
+    return [StructuralUnit(text=text, content_type=ContentType.TABLE_ROW) for text in rendered]
+
+
+def parse_csv(data: bytes) -> ParsedDocument:
+    """Parse CSV bytes into one structural unit per row."""
+    text = _decode(data)
+
+    try:
+        dialect = csv.Sniffer().sniff(text[:_SNIFF_BYTES])
+        reader = csv.reader(io.StringIO(text), dialect)
+    except csv.Error:
+        # A single-column file has no delimiter to find; that is not an error.
+        reader = csv.reader(io.StringIO(text))
+
+    rows = clean_rows(reader)
+    if not rows:
+        raise DocumentParseError("File contains no rows")
+
+    units = _to_units(render_rows(rows))
+    if not units:
+        raise DocumentParseError("File contains no data rows")
+
+    return ParsedDocument(units=units, block_count=len(units))
+
+
+def _forward_fill_merges(worksheet: Worksheet) -> dict[tuple[int, int], str]:
+    """Map every cell of a merged range to the range's value.
+
+    openpyxl stores a merged range's value only in its top-left cell; the rest
+    read as None. Without this a branch row inherits nothing from the province
+    cell merged above it and silently loses that field (ADR-0004).
+
+    Iterate the range collection itself and read corners via `bounds`: `.ranges`
+    is a set subclass and `.min_row` and friends are descriptors, neither of
+    which resolves to an int for a type checker.
+    """
+    filled: dict[tuple[int, int], str] = {}
+
+    for merged in worksheet.merged_cells:
+        min_col, min_row, max_col, max_row = merged.bounds
+        value = clean_cell(worksheet.cell(row=min_row, column=min_col).value)
+        if not value:
+            continue
+        for row in range(min_row, max_row + 1):
+            for column in range(min_col, max_col + 1):
+                filled[(row, column)] = value
+
+    return filled
+
+
+def _sheet_rows(worksheet: Worksheet) -> list[Row]:
+    """Read a worksheet into normalized text rows, merges resolved."""
+    merged = _forward_fill_merges(worksheet)
+    rows: list[Row] = []
+
+    for row_index, row in enumerate(worksheet.iter_rows(), start=1):
+        values = [
+            merged.get((row_index, column_index), clean_cell(cell.value))
+            for column_index, cell in enumerate(row, start=1)
+        ]
+        if any(values):
+            rows.append(values)
+
+    return rows
+
+
+def parse_xlsx(data: bytes) -> ParsedDocument:
+    """Parse XLSX bytes into one structural unit per row, across all sheets."""
+    try:
+        workbook = openpyxl.load_workbook(io.BytesIO(data), data_only=True)
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open XLSX: {exc}") from exc
+
+    units: list[StructuralUnit] = []
+    try:
+        for worksheet in workbook.worksheets:
+            rows = _sheet_rows(worksheet)
+            # The dead second sheet seen throughout the sample corpus.
+            if not rows:
+                continue
+            units.extend(_to_units(render_rows(rows)))
+    finally:
+        workbook.close()
+
+    if not units:
+        raise DocumentParseError("Workbook contains no data rows")
+
+    return ParsedDocument(units=units, block_count=len(units))

src/application/ingestion/tabular.py

@@ -0,0 +1,167 @@
+"""Shared row-to-chunk rendering for every tabular source (ADR-0004, ADR-0018).
+
+A table row is an atomic structural unit regardless of the container it
+arrived in -- a docx table, an xlsx sheet, or a csv file -- so one renderer
+serves all three and a Q&A sheet needs no special case against a branch
+directory:
+
+    q: ...                 ردیف: 1
+    a: ...                 استان: اردبیل
+                           شعبه: پارس آباد
+
+The header rules exist because real tables are not uniform. Of the tables in
+the sample corpus, some carry a header row, one is a bare list of values with
+no header at all, and one is page decoration. The guiding constraint is
+therefore: **never invent structure that is not provably there, and never
+discard a row.** A wrongly-detected header turns every chunk into nonsense
+(`80: 70`), which is worse than an unlabeled row.
+"""
+
+from collections.abc import Iterable, Sequence
+
+from src.application.ingestion.normalization import normalize_persian_text
+
+Row = Sequence[str]
+
+# A header cell is a label, not a sentence.
+MAX_HEADER_CELL_LENGTH = 80
+
+# How much shorter a header cell must be than the column beneath it before
+# length alone is taken as evidence of a header (the `q`/`a` case, where
+# one-character labels sit above paragraph-long answers).
+_HEADER_LENGTH_RATIO = 3.0
+
+
+def clean_cell(value: object) -> str:
+    """Normalize a cell to text; None and blank cells become the empty string.
+
+    `object` rather than a union: a spreadsheet cell holds whatever the
+    workbook stored -- str, int, float, bool, datetime, a formula error -- and
+    every one of them is handled the same way, by rendering it.
+    """
+    if value is None:
+        return ""
+    return normalize_persian_text(str(value))
+
+
+def _is_numeric(text: str) -> bool:
+    return bool(text) and text.replace(",", "").replace(".", "").replace("-", "").isdigit()
+
+
+def _looks_like_title_row(row: Row) -> bool:
+    """A merged title spanning the sheet resolves to one value, or repeats it."""
+    populated = [cell for cell in row if cell]
+    return len(populated) < 2 or len(set(populated)) == 1
+
+
+def has_header(rows: Sequence[Row]) -> bool:
+    """Whether the first row labels the columns beneath it.
+
+    Decided by comparing row 0 against the column below it, not by how row 0
+    looks on its own: a label row is *inconsistent* with its data (text above
+    numbers, or a short label above long prose), while a data row is
+    consistent with the rows that follow. This is the test `csv.Sniffer`
+    uses, and it is a property of the table rather than a pattern borrowed
+    from one document.
+    """
+    if len(rows) < 2:
+        return False
+
+    candidate, data = rows[0], rows[1:]
+    if not all(cell for cell in candidate[: len(data[0])] if cell) and _looks_like_title_row(
+        candidate
+    ):
+        return False
+    if any(len(cell) > MAX_HEADER_CELL_LENGTH for cell in candidate):
+        return False
+
+    for index, label in enumerate(candidate):
+        if not label:
+            continue
+        column = [row[index] for row in data if index < len(row) and row[index]]
+        if not column:
+            continue
+
+        # A text label above a numeric column.
+        if not _is_numeric(label) and all(_is_numeric(value) for value in column):
+            return True
+
+        # A short label above a column of much longer values.
+        mean_length = sum(len(value) for value in column) / len(column)
+        if mean_length > len(label) * _HEADER_LENGTH_RATIO:
+            return True
+
+    return False
+
+
+def strip_title_rows(rows: Sequence[Row]) -> Sequence[Row]:
+    """Drop leading merged-title and blank rows.
+
+    Structural, not a content guess: these rows are the artifact of a merged
+    range spanning the sheet width, so they carry one value across many cells.
+    Only *leading* rows are dropped, so no data row is ever lost.
+    """
+    start = 0
+    while start < len(rows) and _looks_like_title_row(rows[start]):
+        start += 1
+    return rows[start:]
+
+
+def _column_name(header: Row, index: int) -> str:
+    """Return a header label, falling back positionally past the header width."""
+    if index < len(header) and header[index]:
+        return header[index]
+    return f"column_{index + 1}"
+
+
+def _dedupe_horizontal_merge(row: Row) -> list[str]:
+    """Collapse the repeats a horizontally merged cell produces.
+
+    Both python-docx and openpyxl report a merged cell once per grid column it
+    spans, so an unlabeled row would otherwise repeat the same value.
+    """
+    collapsed: list[str] = []
+    for cell in row:
+        if cell and (not collapsed or collapsed[-1] != cell):
+            collapsed.append(cell)
+    return collapsed
+
+
+def render_rows(rows: Sequence[Row]) -> list[str]:
+    """Render table rows as text, one string per row.
+
+    With a provable header each row becomes `"{header}: {value}"` lines, which
+    makes it self-describing. Without one, cells are joined with `" | "` --
+    unlabeled, but never mislabeled.
+    """
+    rows = strip_title_rows(rows)
+    if not rows:
+        return []
+
+    if not has_header(rows):
+        return [text for row in rows if (text := " | ".join(_dedupe_horizontal_merge(row)))]
+
+    header, data = rows[0], rows[1:]
+    # A header merged vertically across two rows resolves to the same text in
+    # the row below it; that duplicate is the header, not data.
+    while data and list(data[0]) == list(header):
+        data = data[1:]
+
+    rendered: list[str] = []
+    for row in data:
+        lines = [
+            f"{_column_name(header, index)}: {value}" for index, value in enumerate(row) if value
+        ]
+        if lines:
+            rendered.append("\n".join(lines))
+    return rendered
+
+
+def clean_rows(rows: Iterable[Iterable[object]]) -> list[Row]:
+    """Normalize every cell and drop rows that are entirely empty."""
+    cleaned: list[Row] = []
+    for row in rows:
+        values = [clean_cell(cell) for cell in row]
+        if any(values):
+            cleaned.append(values)
+    return cleaned

src/application/ingestion/tokenizer.py

@@ -0,0 +1,33 @@
+"""Token counting for chunk sizing (ADR-0018).
+
+`cl100k_base` is a deliberate proxy for the embedding models' own tokenizers.
+`text-embedding-3-large` has an 8191-token window and never binds;
+`nomic-embed-text-v2-moe`'s 512-token sequence length is the only real
+constraint. cl100k tokenizes Persian inefficiently while nomic's multilingual
+tokenizer does not, so a cl100k count reliably over-estimates the nomic count --
+safe in the conservative direction, without shipping a second tokenizer and its
+model download into the ingestion path.
+"""
+
+from functools import lru_cache
+
+import tiktoken
+
+
+@lru_cache(maxsize=4)
+def get_encoder(encoding_name: str) -> tiktoken.Encoding:
+    """Return a cached tiktoken encoder.
+
+    Deliberately not a module-level constant: `tiktoken` fetches the BPE
+    vocabulary over the network the first time an encoding is used, and
+    ADR-0012 forbids external resource setup as an import-time side effect.
+    The lifespan warms this at startup so a process fails fast at boot rather
+    than inside the first ingestion request. Set `TIKTOKEN_CACHE_DIR` to a
+    pre-populated directory for offline deployments.
+    """
+    return tiktoken.get_encoding(encoding_name)
+
+
+def count_tokens(text: str, encoding_name: str) -> int:
+    """Return the number of tokens `text` encodes to."""
+    return len(get_encoder(encoding_name).encode(text))

src/bootstrap/lifespan.py

@@ -2,8 +2,10 @@ from collections.abc import AsyncIterator, Callable
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
 
 import structlog
+from anyio import to_thread
 from fastapi import FastAPI
 
+from src.application.ingestion import get_encoder
 from src.bootstrap.dependencies import AppResources
 from src.config import Settings
 from src.infrastructure.minio.client import create_client as create_minio_client
@@ -22,6 +24,15 @@ def create_lifespan(
         resolved_settings = settings or Settings()
         configure_logging(resolved_settings.logging)
 
+        # tiktoken fetches its vocabulary over the network on first use, so warm
+        # it here: a missing vocabulary should fail the process at boot, not the
+        # first upload. Blocking, hence the thread.
+        await to_thread.run_sync(get_encoder, resolved_settings.chunking.encoding_name)
+        logger.info(
+            "lifespan.tokenizer.loaded",
+            encoding=resolved_settings.chunking.encoding_name,
+        )
+
         db_engine = create_engine(resolved_settings.postgres)
         db_sessionmaker = create_sessionmaker(db_engine)
         logger.info("lifespan.postgres.engine.created")

src/config.py

@@ -1,4 +1,4 @@
-from pydantic import Field
+from pydantic import Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -43,6 +43,38 @@ class IngestionSettings(BaseSettings):
     embed_concurrency: int = 4
 
 
+class ChunkingSettings(BaseSettings):
+    """Parsing and chunking parameters (ADR-0018).
+
+    `max_chunk_tokens` is `nomic-embed-text-v2-moe`'s sequence length. Text past
+    it is silently truncated by the model rather than rejected, so the cap is
+    enforced here instead. `chunk_size` sits well under it to leave room for the
+    `search_document: ` task prefix and any heading text carried into a chunk.
+    """
+
+    model_config = SettingsConfigDict(env_prefix="CHUNKING_", extra="ignore")
+
+    strategy: str = "fixed_size"
+    chunk_size: int = 400
+    chunk_overlap: int = 60
+    max_chunk_tokens: int = 512
+    encoding_name: str = "cl100k_base"
+
+    @model_validator(mode="after")
+    def _validate_sizes(self) -> "ChunkingSettings":
+        if self.chunk_overlap >= self.chunk_size:
+            raise ValueError(
+                f"chunk_overlap ({self.chunk_overlap}) must be smaller than "
+                f"chunk_size ({self.chunk_size}); otherwise splitting never advances"
+            )
+        if self.chunk_size > self.max_chunk_tokens:
+            raise ValueError(
+                f"chunk_size ({self.chunk_size}) must not exceed "
+                f"max_chunk_tokens ({self.max_chunk_tokens})"
+            )
+        return self
+
+
 class QdrantSettings(BaseSettings):
     model_config = SettingsConfigDict(env_prefix="QDRANT_", extra="ignore")
 
@@ -71,6 +103,7 @@ class Settings(BaseSettings):
     postgres: PostgresSettings = Field(default_factory=PostgresSettings)
     minio: MinioSettings = Field(default_factory=MinioSettings)
     ingestion: IngestionSettings = Field(default_factory=IngestionSettings)
+    chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
     qdrant: QdrantSettings = Field(default_factory=QdrantSettings)
     app: AppLimitSettings = Field(default_factory=AppLimitSettings)
     logging: LoggingSettings = Field(default_factory=LoggingSettings)