feat(ingestion): add DOCX/CSV/XLSX parsing and fixed-size chunking (ADR-0018)

Adds src/application/ingestion/ -- Persian normalization, DOCX body
walk with structural data/layout table classification, CSV/XLSX row
rendering, and fixed-size token chunking (cl100k_base, 400/60/512) --
as pure functions per ADR-0015, tested against real production
documents (asia_data_sample, kept out of the repo). ADR-0018 records
where this diverges from ADR-0004 (fixed-size default, no invented
headings/tree, structural table classification, header-provable
labeling only). Plan 001's scope line is corrected from CSV-only to
DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is
updated to match current implementation state.

src/application/ingestion/chunking.py

@@ -0,0 +1,129 @@
+"""Fixed-size chunking with overlap (ADR-0018).
+
+DOCX markdown is split on token windows; spreadsheet rows are already atomic
+and bypass the splitter, falling through it only when a single row exceeds the
+embedding model's sequence length.
+"""
+
+import uuid
+
+from src.application.ingestion.errors import ChunkTooLargeError, DocumentParseError
+from src.application.ingestion.models import Chunk, ContentType, ParsedDocument
+from src.application.ingestion.tokenizer import count_tokens, get_encoder
+from src.config import ChunkingSettings
+
+# Fixed namespace so chunk ids stay stable across processes and releases.
+CHUNK_ID_NAMESPACE = uuid.UUID("6f9619ff-8b86-d011-b42d-00c04fc964ff")
+
+
+def chunk_id_for(file_id: uuid.UUID, chunk_index: int) -> uuid.UUID:
+    """Return the deterministic point id for a chunk (ADR-0001).
+
+    Derived from `file_id` and the immutable ingestion ordinal, so re-ingesting
+    a file upserts its points instead of duplicating them.
+    """
+    return uuid.uuid5(CHUNK_ID_NAMESPACE, f"{file_id}:{chunk_index}")
+
+
+def split_by_tokens(text: str, *, chunk_size: int, overlap: int, encoding_name: str) -> list[str]:
+    """Split text into overlapping windows of at most `chunk_size` tokens."""
+    if overlap >= chunk_size:
+        raise ValueError(f"overlap ({overlap}) must be smaller than chunk_size ({chunk_size})")
+
+    encoder = get_encoder(encoding_name)
+    tokens = encoder.encode(text)
+    if len(tokens) <= chunk_size:
+        return [text]
+
+    windows: list[str] = []
+    start = 0
+    while start < len(tokens):
+        end = min(start + chunk_size, len(tokens))
+        windows.append(encoder.decode(tokens[start:end]))
+        if end >= len(tokens):
+            break
+        start = end - overlap
+
+    return windows
+
+
+def _split_oversized(text: str, settings: ChunkingSettings) -> list[str]:
+    """Split a row only if it exceeds the cap; otherwise keep it atomic."""
+    if count_tokens(text, settings.encoding_name) <= settings.max_chunk_tokens:
+        return [text]
+    return split_by_tokens(
+        text,
+        chunk_size=settings.chunk_size,
+        overlap=settings.chunk_overlap,
+        encoding_name=settings.encoding_name,
+    )
+
+
+def _content_units(
+    parsed: ParsedDocument, settings: ChunkingSettings
+) -> list[tuple[str, ContentType]]:
+    """Reduce a parsed document's structural units to ordered text pieces.
+
+    Prose is cut into token windows; a table row is atomic and survives whole
+    unless it alone exceeds the model's sequence length (ADR-0004).
+    """
+    if not parsed.units:
+        raise DocumentParseError("Parsed document has no structural units")
+
+    pieces: list[tuple[str, ContentType]] = []
+    for unit in parsed.units:
+        if unit.content_type is ContentType.PARAGRAPH:
+            windows = split_by_tokens(
+                unit.text,
+                chunk_size=settings.chunk_size,
+                overlap=settings.chunk_overlap,
+                encoding_name=settings.encoding_name,
+            )
+        else:
+            windows = _split_oversized(unit.text, settings)
+        pieces.extend((window, unit.content_type) for window in windows)
+
+    return pieces
+
+
+def chunk_document(
+    parsed: ParsedDocument,
+    *,
+    file_id: uuid.UUID,
+    settings: ChunkingSettings,
+) -> list[Chunk]:
+    """Turn a parsed document into ordered, neighbor-linked chunks."""
+    units = [
+        (text.strip(), content_type) for text, content_type in _content_units(parsed, settings)
+    ]
+    # Drop blanks *before* assigning indices: an index gap would break the
+    # previous/next chain that retrieval-time expansion walks.
+    units = [(text, content_type) for text, content_type in units if text]
+
+    chunks: list[Chunk] = []
+    for index, (text, content_type) in enumerate(units):
+        token_count = count_tokens(text, settings.encoding_name)
+        if token_count > settings.max_chunk_tokens:
+            raise ChunkTooLargeError(
+                f"chunk {index} is {token_count} tokens, over the "
+                f"{settings.max_chunk_tokens}-token cap"
+            )
+        chunks.append(
+            Chunk(
+                chunk_id=chunk_id_for(file_id, index),
+                chunk_index=index,
+                order_id=float(index + 1),
+                content=text,
+                content_type=content_type,
+                token_count=token_count,
+                character_count=len(text),
+            )
+        )
+
+    for position, chunk in enumerate(chunks):
+        if position > 0:
+            chunk.previous_chunk_id = chunks[position - 1].chunk_id
+        if position < len(chunks) - 1:
+            chunk.next_chunk_id = chunks[position + 1].chunk_id
+
+    return chunks