docs(claude): record plan 002 phase 3 in the project status

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.claude/agents/Explore.md

@@ -0,0 +1,8 @@
+---
+name: Explore
+description: Fast, read-only codebase search
+model: sonnet
+effort: low
+tools: Read, Grep, Glob, Bash, WebFetch, WebSearch
+maxTurns: 20
+---

.claude/hooks/python_quality.py

@@ -0,0 +1,380 @@
+#!/usr/bin/env python3
+"""Run Python quality tools for Claude Code Write/Edit hooks.
+
+The hook is intentionally non-blocking: it formats/fixes what Ruff can fix safely,
+then reports remaining Ruff/ty diagnostics back to Claude as additional context.
+
+It also audits each edit for *suppression*: inline ignore comments, typing weakened
+to `Any`, dynamic-attribute escapes, and config-level rule downgrades. The point is
+that silencing a diagnostic can never be quieter than fixing it. Diagnostics are
+remembered between runs, so a diagnostic that disappears in the same edit that
+introduced a suppression marker is reported loudly rather than passing as "clean".
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+PYTHON_SUFFIXES = {".py", ".pyi"}
+CONFIG_NAMES = {"pyproject.toml", "ty.toml", ".ty.toml", "ruff.toml", ".ruff.toml", "setup.cfg"}
+SKIP_PARTS = {".git", ".venv", "__pycache__"}
+STATE_RELATIVE = Path(".claude") / "hooks" / ".quality_state.json"
+
+# Markers that make a diagnostic go away without necessarily fixing what it found.
+SUPPRESSION_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
+    ("type-checker suppression", re.compile(r"#\s*(?:ty|type|pyright|mypy)\s*:\s*ignore")),
+    ("lint suppression", re.compile(r"#\s*noqa\b")),
+    ("coverage suppression", re.compile(r"#\s*pragma:\s*no\s*cover")),
+    ("typing weakened to Any", re.compile(r"(?:->|:)\s*(?:typing\.)?Any\b|\bcast\s*\(")),
+    ("dynamic attribute escape", re.compile(r"\b(?:get|set|has)attr\s*\(")),
+    ("broad exception swallow", re.compile(r"\bexcept\s+(?:BaseException|Exception)\b")),
+    ("narrowing assert", re.compile(r"\bassert\s+.+\bis\s+not\s+None\b")),
+)
+
+# Config edits that disable checks repo-wide. These files are not Python, so the
+# normal per-file check never runs on them and the effect is otherwise invisible.
+CONFIG_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
+    ("rule downgraded", re.compile(r'^\s*[\w-]+\s*=\s*"(?:ignore|warn)"')),
+    ("rule/override block", re.compile(r"^\s*\[\[?tool\.(?:ty|ruff)[\w.]*\]\]?")),
+    ("lint ignore list", re.compile(r"^\s*(?:ignore|extend-ignore|per-file-ignores)\s*=")),
+    ("paths excluded from checks", re.compile(r"^\s*(?:exclude|extend-exclude)\s*=")),
+)
+
+HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
+TY_DIAGNOSTIC_RE = re.compile(r"^(?:error|warning)\[([a-z0-9-]+)\]")
+RUFF_DIAGNOSTIC_RE = re.compile(r"^.+?:\d+:\d+:\s+([A-Z]+\d+)\b")
+
+
+def main() -> int:
+    payload = _read_payload()
+    file_path = _extract_file_path(payload)
+    if file_path is None:
+        return 0
+
+    repo_root = _repo_root()
+    path = _resolve_path(file_path, repo_root)
+
+    if not path.is_file() or not SKIP_PARTS.isdisjoint(path.parts):
+        return 0
+
+    is_python = path.suffix in PYTHON_SUFFIXES
+    is_config = not is_python and path.name in CONFIG_NAMES
+    if not (is_python or is_config):
+        return 0
+
+    relative_path = _display_path(path, repo_root)
+
+    if is_config:
+        findings = _scan_added_lines(_added_lines(path, repo_root), CONFIG_PATTERNS)
+        message = _build_config_message(relative_path=relative_path, findings=findings)
+        _emit(message)
+        return 0
+
+    before = _sha256(path)
+
+    fix = _run(["uv", "run", "ruff", "check", "--fix", str(path)], repo_root)
+    fmt = _run(["uv", "run", "ruff", "format", str(path)], repo_root)
+    lint = _run(["uv", "run", "ruff", "check", str(path)], repo_root)
+
+    # `--error-on-warning` makes ty warnings visible to the hook while this
+    # script still exits 0, so warnings are reported but do not stop Claude.
+    types = _run(["uv", "run", "ty", "check", "--error-on-warning", str(path)], repo_root)
+
+    after = _sha256(path)
+    changed = before != after
+
+    current_codes = _diagnostic_codes(lint, types)
+    previous_codes = _read_previous_codes(repo_root, relative_path)
+    _write_current_codes(repo_root, relative_path, current_codes)
+
+    findings = _scan_added_lines(_added_lines(path, repo_root), SUPPRESSION_PATTERNS)
+
+    message = _build_message(
+        relative_path=relative_path,
+        changed=changed,
+        results=(fix, fmt, lint, types),
+        findings=findings,
+        resolved_codes=sorted(previous_codes - current_codes),
+    )
+    _emit(message)
+
+    return 0
+
+
+def _emit(message: str) -> None:
+    if not message:
+        return
+    print(
+        json.dumps(
+            {
+                "suppressOutput": True,
+                "hookSpecificOutput": {
+                    "hookEventName": "PostToolUse",
+                    "additionalContext": message,
+                },
+            }
+        )
+    )
+
+
+def _read_payload() -> dict[str, object]:
+    try:
+        raw = sys.stdin.read()
+        return json.loads(raw) if raw.strip() else {}
+    except json.JSONDecodeError:
+        return {}
+
+
+def _extract_file_path(payload: dict[str, object]) -> str | None:
+    response = payload.get("tool_response")
+    if isinstance(response, dict):
+        for key in ("filePath", "file_path"):
+            value = response.get(key)
+            if isinstance(value, str) and value:
+                return value
+
+    tool_input = payload.get("tool_input")
+    if isinstance(tool_input, dict):
+        value = tool_input.get("file_path")
+        if isinstance(value, str) and value:
+            return value
+
+    return None
+
+
+def _repo_root() -> Path:
+    result = subprocess.run(
+        ["git", "rev-parse", "--show-toplevel"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode == 0:
+        return Path(result.stdout.strip()).resolve()
+    return Path.cwd().resolve()
+
+
+def _resolve_path(file_path: str, repo_root: Path) -> Path:
+    path = Path(file_path).expanduser()
+    if not path.is_absolute():
+        path = repo_root / path
+    return path.resolve()
+
+
+def _display_path(path: Path, repo_root: Path) -> str:
+    try:
+        return str(path.relative_to(repo_root))
+    except ValueError:
+        return str(path)
+
+
+def _sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _run(command: list[str], cwd: Path) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        command,
+        cwd=cwd,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+def _combined_output(result: subprocess.CompletedProcess[str]) -> str:
+    return "\n".join(part.strip() for part in (result.stdout, result.stderr) if part.strip())
+
+
+def _command_label(result: subprocess.CompletedProcess[str]) -> str:
+    return " ".join(result.args) if isinstance(result.args, list) else str(result.args)
+
+
+def _diagnostic_codes(*results: subprocess.CompletedProcess[str]) -> set[str]:
+    """Extract stable rule identifiers (ty rule names, Ruff codes) from tool output."""
+    codes: set[str] = set()
+    for result in results:
+        for line in _combined_output(result).splitlines():
+            stripped = line.strip()
+            ty_match = TY_DIAGNOSTIC_RE.match(stripped)
+            if ty_match:
+                codes.add(ty_match.group(1))
+                continue
+            ruff_match = RUFF_DIAGNOSTIC_RE.match(stripped)
+            if ruff_match:
+                codes.add(ruff_match.group(1))
+    return codes
+
+
+def _state_path(repo_root: Path) -> Path:
+    return repo_root / STATE_RELATIVE
+
+
+def _read_previous_codes(repo_root: Path, relative_path: str) -> set[str]:
+    try:
+        state = json.loads(_state_path(repo_root).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return set()
+    entry = state.get(relative_path) if isinstance(state, dict) else None
+    if not isinstance(entry, list):
+        return set()
+    return {code for code in entry if isinstance(code, str)}
+
+
+def _write_current_codes(repo_root: Path, relative_path: str, codes: set[str]) -> None:
+    state_path = _state_path(repo_root)
+    try:
+        state = json.loads(state_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        state = {}
+    if not isinstance(state, dict):
+        state = {}
+
+    if codes:
+        state[relative_path] = sorted(codes)
+    else:
+        state.pop(relative_path, None)
+
+    try:
+        state_path.parent.mkdir(parents=True, exist_ok=True)
+        state_path.write_text(json.dumps(state, indent=2, sort_keys=True), encoding="utf-8")
+    except OSError:
+        pass
+
+
+def _added_lines(path: Path, repo_root: Path) -> list[tuple[int, str]]:
+    """Return (line number, text) for lines this working tree adds over HEAD."""
+    relative_path = _display_path(path, repo_root)
+
+    tracked = _run(["git", "ls-files", "--error-unmatch", "--", relative_path], repo_root)
+    if tracked.returncode != 0:
+        try:
+            text = path.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            return []
+        return list(enumerate(text.splitlines(), start=1))
+
+    diff = _run(["git", "diff", "-U0", "HEAD", "--", relative_path], repo_root)
+    if diff.returncode != 0:
+        return []
+    return _parse_added_lines(diff.stdout)
+
+
+def _parse_added_lines(diff_text: str) -> list[tuple[int, str]]:
+    added: list[tuple[int, str]] = []
+    lineno = 0
+    for line in diff_text.splitlines():
+        hunk = HUNK_RE.match(line)
+        if hunk:
+            lineno = int(hunk.group(1))
+            continue
+        if line.startswith(("+++", "---")):
+            continue
+        if line.startswith("+"):
+            added.append((lineno, line[1:]))
+            lineno += 1
+        elif not line.startswith("-"):
+            lineno += 1
+    return added
+
+
+def _scan_added_lines(
+    added: list[tuple[int, str]],
+    patterns: tuple[tuple[str, re.Pattern[str]], ...],
+) -> list[str]:
+    findings: list[str] = []
+    seen: set[tuple[int, str]] = set()
+    for lineno, text in added:
+        for category, pattern in patterns:
+            if pattern.search(text) and (lineno, category) not in seen:
+                seen.add((lineno, category))
+                findings.append(f"L{lineno} [{category}]: {text.strip()[:160]}")
+    return findings
+
+
+def _build_message(
+    *,
+    relative_path: str,
+    changed: bool,
+    results: tuple[subprocess.CompletedProcess[str], ...],
+    findings: list[str],
+    resolved_codes: list[str],
+) -> str:
+    lines: list[str] = []
+
+    if changed:
+        lines.append(
+            f"Python quality hook updated `{relative_path}` with Ruff safe fixes/formatting."
+        )
+        lines.append("Read the file before making another manual edit to avoid stale text.")
+
+    for result in results:
+        if result.returncode == 0:
+            continue
+
+        output = _combined_output(result) or f"Command exited with status {result.returncode}."
+        lines.extend(
+            [
+                "",
+                f"Remaining diagnostics from `{_command_label(result)}`:",
+                "```text",
+                output[-6000:],
+                "```",
+            ]
+        )
+
+    if findings:
+        lines.extend(
+            [
+                "",
+                f"Suppression audit — `{relative_path}` adds these lines over HEAD:",
+                "```text",
+                "\n".join(findings[-60:]),
+                "```",
+                "A silenced diagnostic is not a fixed diagnostic. For each entry: either the",
+                "code as written requires it, or it exists to make a checker pass — and the",
+                "second case hides a bug rather than resolving it. If a checker reported a real",
+                "problem, fix the problem. If you believe the checker is wrong, say so to the",
+                "user instead of suppressing it yourself.",
+            ]
+        )
+
+        if resolved_codes:
+            lines.extend(
+                [
+                    "",
+                    "!! These diagnostics vanished in the same edit that added the markers",
+                    f"   above: {', '.join(resolved_codes)}",
+                    "If any pointed at a genuine logic error, that error is still present and",
+                    "is now merely unreported. Do NOT call this file clean. Tell the user what",
+                    "you changed and why each diagnostic no longer applies.",
+                ]
+            )
+
+    return "\n".join(lines).strip()
+
+
+def _build_config_message(*, relative_path: str, findings: list[str]) -> str:
+    if not findings:
+        return ""
+    return "\n".join(
+        [
+            f"Quality-config audit — `{relative_path}` adds these lines over HEAD:",
+            "```text",
+            "\n".join(findings[-60:]),
+            "```",
+            "These lines can disable or downgrade checks across the whole repository, and no",
+            "per-file check runs on this file. Loosening checker configuration is the user's",
+            "decision, not yours: if you changed it to make diagnostics go away, revert it and",
+            "report the diagnostics instead.",
+        ]
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.claude/settings.json

@@ -0,0 +1,20 @@
+{
+  "enabledPlugins": {
+    "astral@astral-sh": true
+  },
+  "hooks": {
+    "PostToolUse": [
+      {
+        "matcher": "Write|Edit",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "repo=$(git rev-parse --show-toplevel 2>/dev/null || pwd); python3 \"$repo/.claude/hooks/python_quality.py\"",
+            "timeout": 120,
+            "statusMessage": "Running Ruff and ty on Python edit"
+          }
+        ]
+      }
+    ]
+  }
+}

.env.example

@@ -0,0 +1,110 @@
+# Application local development environment.
+#
+# Usage:
+#   cp .env.example .env
+#   docker compose -f docker-compose.yml up -d
+#
+# All values below are non-secret local-development defaults matching
+# docker-compose.yml. Do not commit .env.
+
+# Application
+APP_ENV=local
+APP_READINESS_CHECK_TIMEOUT_SECONDS=2.0
+# Set by CI/CD at build/deploy time; never computed at runtime.
+APP_SERVICE_VERSION=dev
+
+# Logging
+LOG_LEVEL=INFO
+LOG_JSON_FORMAT=false
+# Optional second sink, always JSON regardless of LOG_JSON_FORMAT. Local dev
+# only -- leave unset in production, where stdout/stderr collection is
+# preferred over an in-container log file.
+# LOG_FILE_PATH=logs/app.log
+LOG_FILE_MAX_BYTES=10485760
+LOG_FILE_BACKUP_COUNT=5
+
+# Postgres (application database, separate from Langfuse's Postgres)
+# Use 127.0.0.1 rather than localhost: some environments resolve localhost to
+# the IPv6 loopback first, and Docker only publishes these ports on IPv4.
+POSTGRES_HOST=127.0.0.1
+POSTGRES_PORT=5433
+POSTGRES_USER=chatbot
+POSTGRES_PASSWORD=chatbot
+POSTGRES_DB=chatbot
+
+# MinIO (application bucket, separate from Langfuse's MinIO)
+MINIO_ENDPOINT=127.0.0.1:9100
+MINIO_ACCESS_KEY=chatbot
+MINIO_SECRET_KEY=chatbot-secret
+MINIO_SECURE=false
+MINIO_BUCKET=chatbot-source-files
+
+# Inline ingestion bounds (ADR-0017; no broker, no queue).
+# INGESTION_TIMEOUT_SECONDS must stay below any proxy/client read timeout.
+INGESTION_MAX_CONCURRENCY=4
+INGESTION_THREAD_POOL_SIZE=8
+INGESTION_TIMEOUT_SECONDS=120.0
+INGESTION_MAX_UPLOAD_SIZE_MB=25
+INGESTION_MAX_CHUNKS_PER_FILE=5000
+INGESTION_EMBED_BATCH_SIZE=128
+INGESTION_EMBED_CONCURRENCY=4
+
+# Qdrant
+QDRANT_URL=http://127.0.0.1:6343
+QDRANT_API_KEY=
+QDRANT_COLLECTION=chunks
+QDRANT_UPSERT_BATCH_SIZE=128
+QDRANT_UPSERT_CONCURRENCY=4
+
+# Dense embedders (ADR-0001). Both speak an OpenAI-compatible /embeddings
+# endpoint, so one adapter serves both. Models and endpoints are the ones the
+# `emet` evaluation lab benchmarked as winners on the Farsi corpus.
+#
+# dense_nomic runs behind Ollama's OpenAI-compat shim, which accepts any
+# non-empty API key. KEEP_ALIVE holds the model resident: a cold load of
+# nomic-embed-text-v2-moe takes >150s, well past INGESTION_TIMEOUT_SECONDS,
+# so an idle-then-upload would otherwise 504.
+EMBEDDING_NOMIC_BASE_URL=http://192.168.10.10:11435/v1
+EMBEDDING_NOMIC_MODEL=nomic-embed-text-v2-moe
+EMBEDDING_NOMIC_API_KEY=sk-not-set
+EMBEDDING_NOMIC_KEEP_ALIVE=30m
+EMBEDDING_NOMIC_TIMEOUT_SECONDS=30.0
+# Empty = emet parity. The model card specifies `search_document: ` (ADR-0004),
+# but the benchmark ran without it and the prefix shifts the vector a lot
+# (cosine 0.57 on identical text) — so if you set this, the query side must
+# send `search_query: ` to match, or retrieval gets worse rather than better.
+EMBEDDING_NOMIC_DOCUMENT_PREFIX=
+
+# Leave DIMENSIONS empty for text-embedding-3-large's native 3072, which is
+# what was benchmarked. Setting it truncates via Matryoshka and is a
+# re-embedding migration, not a config tweak.
+EMBEDDING_OPENAI_BASE_URL=https://api.openai.com/v1
+EMBEDDING_OPENAI_MODEL=text-embedding-3-large
+EMBEDDING_OPENAI_API_KEY=
+EMBEDDING_OPENAI_DIMENSIONS=
+EMBEDDING_OPENAI_DOCUMENT_PREFIX=
+EMBEDDING_OPENAI_TIMEOUT_SECONDS=30.0
+
+# Sparse BM25 (ADR-0001, ADR-0005): the benchmarked `bm25-fa-norm-stop`.
+# k/b saturation is applied client-side; IDF comes from Qdrant's
+# modifier="idf" on the sparse vector field. AVG_LEN is the average document
+# length in analyzer tokens — emet's placeholder, worth recalibrating from
+# real corpus statistics.
+EMBEDDING_SPARSE_ANALYZER=fa_norm_stop
+EMBEDDING_SPARSE_K=1.2
+EMBEDDING_SPARSE_B=0.75
+EMBEDDING_SPARSE_AVG_LEN=256.0
+
+# Parsing and chunking (ADR-0018).
+# max_chunk_tokens is nomic-embed-text-v2-moe's sequence length; text past it
+# is silently truncated by the model, so the cap is enforced before embedding.
+# chunk_size sits under it to leave room for the `search_document: ` prefix.
+CHUNKING_STRATEGY=fixed_size
+CHUNKING_CHUNK_SIZE=400
+CHUNKING_CHUNK_OVERLAP=60
+CHUNKING_MAX_CHUNK_TOKENS=512
+CHUNKING_ENCODING_NAME=cl100k_base
+
+# tiktoken downloads its vocabulary on first use; point this at a
+# pre-populated directory for offline/air-gapped deployments.
+# TIKTOKEN_CACHE_DIR=

.env.langfuse.example

@@ -0,0 +1,77 @@
+# Langfuse local development environment.
+#
+# Usage:
+#   cp .env.langfuse.example .env.langfuse
+#   # replace all CHANGE_ME values before starting
+#   docker compose --env-file .env.langfuse -f docker-compose.langfuse.yml up -d
+#
+# Generate secrets with commands such as:
+#   openssl rand -base64 32
+#   openssl rand -hex 16
+#   openssl rand -hex 32
+#
+# Do not commit .env.langfuse.
+
+# Public URL used by Langfuse itself. For local browser access this should match
+# http://localhost:3000. For a VM, set this to your HTTPS domain.
+LANGFUSE_NEXTAUTH_URL=http://localhost:3000
+
+# Host ports. Internal container ports stay unchanged and are used for
+# container-to-container communication.
+LANGFUSE_WEB_PORT=3000
+LANGFUSE_WORKER_PORT=3030
+LANGFUSE_POSTGRES_HOST_PORT=15432
+LANGFUSE_REDIS_HOST_PORT=16379
+LANGFUSE_CLICKHOUSE_HTTP_PORT=18123
+LANGFUSE_CLICKHOUSE_NATIVE_PORT=19000
+LANGFUSE_MINIO_API_PORT=9090
+LANGFUSE_MINIO_CONSOLE_PORT=9091
+
+# Core Langfuse secrets. Replace these values.
+LANGFUSE_NEXTAUTH_SECRET=CHANGE_ME_generate_with_openssl_rand_base64_32
+LANGFUSE_SALT=CHANGE_ME_generate_with_openssl_rand_hex_16
+# Must be exactly 64 hex characters. Generate with: openssl rand -hex 32
+LANGFUSE_ENCRYPTION_KEY=CHANGE_ME_generate_with_openssl_rand_hex_32
+
+# Langfuse internal Postgres. This is not the chatbot application's Postgres.
+LANGFUSE_POSTGRES_VERSION=17
+LANGFUSE_POSTGRES_USER=postgres
+LANGFUSE_POSTGRES_PASSWORD=CHANGE_ME_langfuse_postgres_password
+LANGFUSE_POSTGRES_DB=postgres
+
+# Langfuse internal ClickHouse.
+LANGFUSE_CLICKHOUSE_USER=clickhouse
+LANGFUSE_CLICKHOUSE_PASSWORD=CHANGE_ME_langfuse_clickhouse_password
+LANGFUSE_CLICKHOUSE_CLUSTER_ENABLED=false
+
+# Langfuse internal Redis.
+LANGFUSE_REDIS_PORT=6379
+LANGFUSE_REDIS_AUTH=CHANGE_ME_langfuse_redis_password
+
+# Langfuse local MinIO/object storage.
+LANGFUSE_MINIO_ROOT_USER=minio
+LANGFUSE_MINIO_ROOT_PASSWORD=CHANGE_ME_langfuse_minio_password
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=CHANGE_ME_langfuse_minio_password
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=CHANGE_ME_langfuse_minio_password
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT_PUBLIC=http://localhost:9090
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=CHANGE_ME_langfuse_minio_password
+LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT=http://localhost:9090
+
+# Development defaults.
+LANGFUSE_TELEMETRY_ENABLED=false
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=false
+
+# Optional headless initialization. Leave blank to create the organization,
+# project, user, and API keys in the UI after startup.
+LANGFUSE_INIT_ORG_ID=
+LANGFUSE_INIT_ORG_NAME=
+LANGFUSE_INIT_PROJECT_ID=
+LANGFUSE_INIT_PROJECT_NAME=
+LANGFUSE_INIT_PROJECT_PUBLIC_KEY=
+LANGFUSE_INIT_PROJECT_SECRET_KEY=
+LANGFUSE_INIT_USER_EMAIL=
+LANGFUSE_INIT_USER_NAME=
+LANGFUSE_INIT_USER_PASSWORD=

.gitignore

@@ -9,5 +9,17 @@ wheels/
 # Virtual environments
 .venv
 
-# Enviroment variables 
+# Test and coverage artifacts
+.pytest_cache/
+.coverage
+coverage.xml
+htmlcov/
+
+# Environment variables
 .env
+.env.langfuse
+infra/langfuse/.env
+
+# Claude Code local overrides
+.claude/settings.local.json
+.claude/hooks/.quality_state.json

CLAUDE.md

@@ -0,0 +1,404 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project status
+
+This repo is ADR-driven and early in implementation. Working today: the FastAPI
+app factory and lifespan wiring (`src/bootstrap/`), `/healthz` and `/readyz`,
+structlog config, Postgres/MinIO/Qdrant clients (`src/infrastructure/`), five
+SQLAlchemy models with one Alembic migration, document parsing plus fixed-size
+chunking (`src/application/ingestion/`), API-key auth, `POST`/`GET /v1/files`
+with durable two-phase job creation (`src/application/files/`), and bounded
+inline embedding: `dense_nomic`/`dense_openai` adapters over an
+OpenAI-compatible HTTP client and a `bm25-fa-norm-stop` sparse adapter
+(`src/infrastructure/embedding/`), wired into the upload path behind
+`INGESTION_MAX_CONCURRENCY` (`503`), `INGESTION_TIMEOUT_SECONDS` (`504`), and
+the chunk-count ceiling (`413`). The embedding configuration is **ported from
+the `emet` evaluation lab** (`~/code/talie/emet`), which benchmarked these
+models and analyzers on the real Farsi corpus — the analyzer and BM25 weights
+are verified token-for-token against it, so treat them as a measured artifact
+and re-benchmark rather than tune them in place (ADR-0005). Also working: the
+`chunks` collection bootstrap (`src/infrastructure/qdrant/collection.py`, run as
+a deployment step via `uv run python -m src.cli.qdrant_bootstrap` — never at
+startup) and tenant-scoped point upserts (`src/application/points/` behind the
+`PointStorage` port), so an upload is searchable by the time `201` returns.
+Also working: `tenant_domains` plus `/v1/domains` (`src/application/domains/`),
+a strict per-tenant allowlist — `POST /v1/files` rejects an unregistered or
+disabled `domain` with `400` before anything is written, and domain management
+sits behind its own `domains:read`/`domains:write` scopes, never `files:write`.
+Also working: the operator runbook (`docs/runbook.md`), tenant/API-key/domain
+provisioning (`uv run python -m src.cli.provision_tenant` — the third deployment
+step, since nothing over HTTP can create the first tenant), a Testcontainers
+e2e suite in the default pytest run (`tests/e2e/test_ingestion_slice.py`:
+duplicate upload, retry after failure, tenant isolation, capacity, timeout,
+parse and Qdrant failure), and the one Compose-based test — `scripts/smoke.sh`
+driving `tests/e2e/test_compose_smoke.py` against a real uvicorn process, which
+skips itself unless `SMOKE_BASE_URL` is set. That maps to plan 001 Phases 1-6
+done.
+
+Plan 002 (`/v1/points` CRUD and keyword search) is **Phases 1-3 done**. Phase 1
+landed the `PointRepository` port (`src/application/ports/point_repository.py`)
+with its `Point` read model (`src/application/points/point.py`), the Qdrant adapter
+(`src/infrastructure/qdrant/point_repository.py`), request/response schemas
+(`src/api/schemas/points.py`), and lifespan wiring. This port is **separate from
+`PointStorage`**, which stays exactly the two bulk operations ingestion
+performs — reads, single-point edits, and keyword search have a different caller
+and a different tenant-filter obligation, so do not accrete them onto the
+ingestion port. `tenant_id` is a required keyword argument on every
+`PointRepository` method by design; keep it that way, because it is what turns a
+forgotten tenant filter into a type error. The `chunks` collection also gained
+full-text `content`, `is_active`, and `chunk_index` payload indexes, so a
+deployed environment needs `qdrant_bootstrap` re-run (indexes are additive — no
+rebuild, no re-embedding).
+
+Two adapter mechanics there are load-bearing and easy to "simplify" into bugs:
+reads go through `scroll` with a `HasIdCondition` rather than `retrieve` (which
+takes no filter, and would move the tenant check to *after* Qdrant answered),
+and ordered listing paginates by `order_id` value rather than offset (Qdrant
+returns no page offset under `order_by`, and an offset cursor skips or repeats
+rows under a concurrent insert).
+
+Phase 2 added the **read routes**: `GET /v1/points/{point_id}`,
+`GET /v1/points?file_id=...`, `GET /v1/points/count`, `GET /v1/points/search`,
+and `GET /v1/files/{file_id}/points`, over `src/application/points/queries.py`
+(`src/api/routers/points.py`). All are gated on `points:read`, which — with
+`points:write` — is now in `DEFAULT_SCOPES`; `GET /v1/files/{file_id}/points`
+uses `points:read` rather than `files:write`, so the scope follows the data
+rather than the URL prefix. `PointNotFoundError` maps to `404` in
+`src/api/errors.py`, never `403`. Three route-level rules are load-bearing:
+`/count` and `/search` are declared **before** `/{point_id}` (FastAPI matches in
+declaration order, so reordering them makes `/v1/points/count` a `422`),
+`file_id` is **required** on the listing (the cursor is an `order_id` value and
+`order_id` is unique only within one file), and `search_points` folds the query
+with `normalize_persian_text` before matching, because ingestion letter-folds
+content and an unfolded Arabic-keyboard query would return an empty result set
+silently rather than erroring (ADR-0002).
+
+Phase 3 added **soft delete**: `DELETE /v1/points/{point_id}` and
+`DELETE /v1/files/{file_id}`, over `src/application/points/deletion.py` (with
+the pure relinking primitive in `src/application/points/relinking.py`) and
+`src/application/files/deletion.py`. Both are gated on `points:write` — the
+file route included, since the data it destroys is points. Nothing is ever
+removed from Qdrant.
+
+Four rules there are load-bearing, and three of them look like complications
+until the concurrency is taken seriously:
+
+- `patches_for_removal` computes **what is still missing between the state just
+  read and the desired end state**, not "the patches a delete implies". That is
+  what makes a normal delete, a second delete of an already-inactive point (a
+  no-op success, never `404`), and recovery from a half-applied batch one code
+  path. Rewriting it as a straight-line "deactivate, patch prev, patch next"
+  breaks all three.
+- Qdrant has no multi-point transaction and reports success for a filtered
+  `set_payload` that matched nothing, so a batch whose second operation loses a
+  version race applies its first anyway. `soft_delete_point` therefore re-plans
+  and re-applies up to three times, verifying by read-back, and only then raises
+  `PointVersionConflictError` (`409`). A single-shot delete would be able to
+  leave a stale pointer, which ADR-0002 calls a defect.
+- A soft-deleted point **keeps its own** `previous_chunk_id`/`next_chunk_id`;
+  only the surviving neighbours are rewritten. Those pointers are unreachable
+  rather than stale, they are the only record of where the point sat, and the
+  retry re-plans from them. The whole-file sweep follows from the same rule:
+  every point leaves at once, so no survivor can dangle and no pointer is
+  touched at all.
+- `DELETE /v1/files/{file_id}` marks the `source_files` row `soft_deleted`
+  **after** the point sweep, in its own short transaction (no session is held
+  across the Qdrant work). Order matters: a half-finished sweep leaves the row
+  `active` and a retried `DELETE` finishes it, and retiring the row is what
+  makes a later re-upload of the same bytes re-ingest instead of matching
+  `find_active_by_content_hash` and returning a file whose points are gone.
+
+Audit rows are still Phase 4/6 work; Phase 3 emits log events only
+(`points.soft_deleted`, `files.soft_deleted`, `points.relink.neighbour_missing`,
+and the two `*.conflict` warnings). The completion and conflict events carry
+ADR-0011's `duration_ms` plus `rounds`, and the pair is what makes them
+diagnostic: relinking itself is O(1) (that is what the adjacency pointers buy),
+so a single-point delete costs a fixed ~5 Qdrant round trips and a `rounds`
+above 1 means contention, not a slow store. The whole-file sweep is the one
+whose cost scales — two round trips per 100-point page.
+
+Also worth knowing before touching the points tests: `tests/support/point_contract.py`
+holds **one** scenario suite run against both `FakePointRepository` (unit) and
+`QdrantPointRepository` (integration), so new repository behaviour belongs there
+rather than in one of the two runners — that is what keeps the fake from drifting
+more permissive than the real store.
+
+Not built yet: plan 002 Phases 4-6 — create/replace/patch, reorder and batch,
+the `api_request_logs`/`point_audit_events` tables, and the runbook section on
+inspecting and repairing a file's pointer chain — and `src/agent/`.
+
+Architecture decisions live in `docs/adr/` (18 ADRs plus the 0000 template;
+0001–0004 are `Accepted` — 0004 amended by 0018; 0014 is `Superseded by 0017`;
+the rest — 0005–0013 and 0015–0018 — are `Proposed`). Implementation plans live
+in `docs/plans/`:
+`001-ingestion-vertical-slice.md` and
+`002-point-crud-and-keyword-search.md`. **Read the relevant
+ADR(s) before implementing anything** — the ADRs are the source of truth for
+package layout, boundaries, and invariants; code should not silently diverge
+from them. If an ADR needs to change to fit new work, update the ADR rather
+than quietly contradicting it.
+
+## Commands
+
+Package manager is `uv`; Python 3.13.
+
+```bash
+uv sync                                   # install deps
+uv run ruff check --fix <path>            # lint (autofix)
+uv run ruff format <path>                 # format
+uv run ty check <path>                    # type check
+uv run pytest                             # run tests
+uv run pytest -m unit                     # fast, no external services (default dev loop)
+uv run pytest -m "integration and postgres"   # a single integration boundary
+uv run pytest tests/path/to/test_file.py::test_name   # single test
+uv run alembic upgrade head               # apply migrations (never create_all() at runtime)
+uv run fastapi dev src/main.py            # run the API locally (once main.py exists)
+```
+
+A PostToolUse hook (`.claude/hooks/python_quality.py`) already runs
+`ruff check --fix`, `ruff format`, `ruff check`, and `ty check` on every
+Python file you Write/Edit and reports remaining diagnostics back to you —
+you don't need to run these manually after every edit, but do run the full
+suite before considering a task done.
+
+### Never silence a diagnostic
+
+A `ty`/Ruff diagnostic is frequently a real bug — most often the checker is
+right and the code is wrong. **Do not make a diagnostic go away by any means
+other than correcting the code it points at.** Specifically, never do the
+following to get a clean check:
+
+- add `# ty: ignore`, `# type: ignore`, `# noqa`, or `# pragma: no cover`
+- widen a type to `Any`, or wrap a value in `cast(...)`
+- replace `x.attr` with `getattr(x, "attr")`, wrap the call in
+  `except Exception`, or otherwise restructure code so the checker can no
+  longer see the problem
+- edit `[tool.ty.rules]`, `[[tool.ty.overrides]]`, `[tool.ruff.lint]`
+  `ignore`/`per-file-ignores`, or any `exclude` in `pyproject.toml`/`ty.toml`/
+  `ruff.toml` — checker configuration is the user's decision, not yours
+- delete or skip a failing test, or narrow its assertions
+
+If a dependency is missing, add it (`uv add`). If a third-party package genuinely
+ships no types, say so and let the user decide on a scoped override. **If you
+believe a diagnostic is a false positive, stop and tell the user — do not
+suppress it yourself.** That claim is exactly the case where silencing is most
+expensive, so it is theirs to confirm.
+
+The hook audits every edit for these patterns and reports them, including on
+config files. If it flags a line you added, either justify it explicitly in your
+reply or revert it — never leave it unmentioned, and never describe a file as
+clean when its diagnostics were suppressed rather than fixed.
+
+Local Langfuse (observability) stack: `docker compose --env-file .env.langfuse -f docker-compose.langfuse.yml up -d`, UI at `http://localhost:3000` (see README.md).
+
+## Architecture
+
+This is a **modular monolith** (ADR-0015) backing a tenant-scoped RAG chatbot.
+Today it is a **single process with no background work**: ingestion runs inline
+in the `POST /v1/files` request and returns a terminal result (ADR-0017). There
+is no message broker, no transactional outbox, no queue, and no worker process —
+ADR-0014 describes that design and is deferred, not deleted.
+
+### Package layout (target shape, created as needed)
+
+```
+src/
+├── bootstrap/        # per-process composition: lifespan, dependency wiring
+├── api/               # FastAPI HTTP adapter only (routers/, dependencies/, schemas/, router.py)
+├── application/       # use-case services: files/, ingestion/, points/, retrieval/, threads/, ports/
+├── agent/             # LangGraph graph.py, state.py, nodes/, prompts/, tools/, persistence.py
+└── infrastructure/    # concrete adapters: postgres/, qdrant/, minio/, embedding/, langgraph/, observability/
+```
+
+`messaging/`, `workers/`, and `infrastructure/rabbitmq/` belong to the ADR-0014
+target shape and are **not created** while ADR-0017 stands. The FastAPI route is
+the only entry adapter; it calls `application/ingestion/` directly.
+
+Dependency direction is one-way and enforced by convention, not tooling:
+
+```
+FastAPI routes / LangGraph nodes
+  -> application services -> application ports -> infrastructure adapters
+```
+
+Infrastructure never imports routers/graph nodes; LangGraph nodes never call route functions or construct SDK
+clients directly; application services never import concrete
+MinIO/Qdrant/SQLAlchemy client-construction code. Use ports only for
+external side effects/persistence — not around pure local functions.
+(ADR-0015)
+
+### Prefer deep modules over shallow ones
+
+When a package exposes several small pure functions that a caller must
+compose correctly every time (right dispatch, right order, right
+thread/async offload), give it one entry point that owns that composition,
+and keep the small functions internal — exported only where their own unit
+tests need them. A shallow interface (one whose surface is nearly as complex
+as its implementation) pushes a correctness obligation onto every call site;
+a deep one absorbs it once. Apply the deletion test when unsure: if deleting
+the wrapper would concentrate the composition logic back into every caller
+rather than just relocate it, the wrapper is worth having.
+
+Worked example: `src/application/ingestion/` exposes `parse_and_chunk_document`
+as its only caller-facing entry point. It dispatches on source type and owns
+the `anyio.to_thread.run_sync` + `CapacityLimiter` offload ADR-0017 requires;
+`parse_docx`/`parse_csv`/`parse_xlsx`/`chunk_document` stay in the package,
+exported mainly for their own tests, not for outside callers to reach for
+directly. `src/application/points/` follows the same shape: `index_chunks` is
+the only caller-facing entry point, owning payload construction, batching,
+the `upsert_concurrency` semaphore, and the ordering rule that the soft-delete
+sweep runs only after every upsert succeeds; `build_chunk_payload` stays
+internal. Follow this pattern in `application/` as new packages are added
+there — `retrieval/`, `threads/` — rather than exposing their internals as the
+primary surface.
+
+### Resource lifetime rules (ADR-0012)
+
+- Application-lifetime objects (SQLAlchemy engine/sessionmaker, Qdrant client,
+  LangGraph checkpointer/store, compiled graph, shared HTTP/model/embedding
+  clients, the ingestion `CapacityLimiter`) are created once in the FastAPI
+  lifespan and closed there. Never construct mutable network/DB clients at
+  import time.
+- One `AsyncSession` per request/job unit of work — never shared globally.
+- Routes/application services own transaction boundaries (explicit
+  `commit()`); repositories don't commit/rollback/close sessions they didn't
+  create.
+- Lower layers receive dependencies as explicit parameters, not via imported
+  singletons — this is what makes FastAPI dependency overrides and test
+  fixtures work.
+
+### Ingestion flow (the first vertical slice, ADR-0017 + plan 001)
+
+Ingestion is **inline in the request**, in three phases — and the phase
+boundaries are the point:
+
+```
+txn A (short):  auth+tenant -> validate -> insert source_files,
+                ingestion_jobs(status=running) -> COMMIT, release connection
+no txn:         store bytes in MinIO -> parse+chunk (threads) ->
+                embed dense+sparse (async, batched, semaphore-bounded) ->
+                upsert Qdrant points (deterministic ids)
+txn B (short):  ingestion_jobs -> succeeded/failed + counters,
+                append ingestion_job_events -> COMMIT
+201 Created { file_id, ingestion_job_id, status, chunks_indexed }
+```
+
+**Never hold a Postgres session/transaction open across the work phase** — it
+pins a pool connection for the whole upload. `ingestion_jobs` is a record of an
+attempt, not a queue.
+
+Work placement is not optional:
+- **Async + batched + `asyncio.Semaphore`**: dense embedders (network I/O; batch
+  before parallelizing, never an unbounded `gather`), Qdrant upserts.
+- **`anyio.to_thread.run_sync` + `CapacityLimiter`**: `python-docx`/`csv`
+  parsing, chunking, hashing, the BM25 sparse pipeline, the sync `minio` SDK.
+  Calling these from `async def` directly is a defect — one big parse stalls
+  every concurrent request.
+- **Not computed at ingest**: `late_interaction` (jina-colbert-v2, GPU).
+  Populating it is the trigger to move ingestion back off the request.
+
+Bounds are enforced server-side and all have status codes: size/chunk ceiling
+`413`, `INGESTION_MAX_CONCURRENCY` `503`, `INGESTION_TIMEOUT_SECONDS` `504`,
+embedder failure `502`. A timeout must still write a terminal job status.
+Retried uploads must be safe: deterministic point IDs, `(tenant_id, domain,
+content_sha256)` idempotency, no terminal job returning to `running`.
+
+### Retrieval / agent (ADR-0001, 0003, 0005, 0006, 0007)
+
+- Single Qdrant collection `chunks`, shared across tenants, with named vectors
+  `dense_nomic` (nomic-embed-text-v2-moe), `dense_openai`, `sparse` (BM25,
+  Farsi-tuned), and `late_interaction` (jina-colbert-v2, rerank-only, on-disk).
+  Multitenancy via Qdrant's `is_tenant` payload index on `tenant_id`, `m: 0` +
+  `payload_m: 16` HNSW config — every query/prefetch carries a server-derived
+  `tenant_id`/`domain` filter, never client-supplied.
+- Retrieval = 3 parallel prefetches (dense_nomic, dense_openai, sparse) → RRF
+  fusion → late-interaction rerank over the fused top-N only (bounded cost) →
+  context-window expansion via `previous_chunk_id`/`next_chunk_id` pointers.
+- Conversational agent is an explicit LangGraph `StateGraph` (not a prebuilt
+  tool-calling agent): `load_memory -> triage -> {chitchat|out_of_scope|
+  handoff_request|account|knowledge->contextualize->retrieve->grade->
+  {clarify|generate->verify}} -> write_memory`. Escalation and retry-budget
+  policy are graph edges, not model discretion; `generate` answers only from
+  retrieved chunks with per-claim citations; `verify` checks groundedness.
+- LangGraph persistence: single `AsyncPostgresSaver` backed by one
+  `psycopg_pool.AsyncConnectionPool`, graph compiled once at startup.
+  `.setup()` (DDL) runs as a deployment step, never at app startup. The
+  LangGraph `thread_id` **is** the conversation identifier this service
+  knows — no separate thread/session table; this service stores no
+  conversation metadata beyond the checkpoint sequence.
+
+### Postgres conventions (ADR-0009)
+
+`domain` is never free-form: it must match an `active` `tenant_domains` row for
+the authenticated tenant (ADR-0009). Domain sets are per-tenant and vary in
+size. The key itself is immutable — it is denormalized into every Qdrant point
+payload and into `source_files`, so renaming it is a migration, not an edit.
+
+UUID primary keys (app-generated), `timestamptz` for all timestamps,
+`Numeric(18, 8)` for money (never floats), `JSONB` for flexible metadata but
+typed/indexed columns for query-critical fields, string status columns with
+`CHECK` constraints (not native Postgres enums — they're painful to migrate).
+`metadata` column maps to a `metadata_` attribute (reserved name on
+Declarative models). All DDL goes through Alembic; FastAPI never creates or
+alters tables at startup.
+
+### Observability (ADR-0010, 0011)
+
+Langfuse is the observability/evaluation plane (traces, prompt versions,
+datasets/experiments, user feedback) — it is not the transactional database.
+Postgres remains system of record for tenants, API keys, audit, jobs,
+`graph_runs`, `llm_calls`/`llm_pricing`. Correlate the two via `request_id`,
+`tenant_id`, `thread_id`, `run_id`. Use `structlog` with stable event names
+and structured fields (`logger.info("graph.run.completed", ...)`), not
+interpolated prose; JSON logs by default in production, plus an optional
+local-only JSON file sink independent of the console renderer (`LOG_FILE_PATH`).
+
+**Add logging in the same change that adds the code, not as a follow-up.**
+When you add a new service-level entry point (an `application/` function a
+route calls directly, an ingestion phase, a mutation) or a new failure branch
+inside one, add its `logger.*` event in that same diff, using ADR-0011's
+level/event-naming table. Deferring it means re-deriving the failure modes and
+field names later from code that no longer has them in working memory — as
+happened with `src/application/files/upload.py`, where four failure branches
+(`parse_failed`, `chunk_limit_exceeded`, `embedding_failed`, `index_failed`)
+shipped with no log event and had to be retrofitted.
+
+This does not mean logging every function. Pure functions, models, schemas,
+and repositories (`infrastructure/postgres/repositories/`) stay silent by
+convention — the caller that turns their result into a business-meaningful
+outcome (job succeeded, upload rejected, domain disabled) is where the event
+belongs, not the row-level function underneath it.
+
+## Testing (ADR-0016)
+
+- pytest, `pytest-asyncio` strict mode, `httpx.AsyncClient` + `ASGITransport`
+  + `LifespanManager` for API tests (not `TestClient`) so lifespan/ADR-0012
+  wiring is actually exercised.
+- One primary marker per test: `unit`, `integration`, or `e2e`; integration
+  tests also carry `postgres`/`minio`/`qdrant`; `live_provider` for
+  opt-in credential-gated external calls (never in routine runs); `slow` only
+  when materially over the normal integration budget.
+- Test naming: `test_<unit>_<scenario>_<outcome>`, Arrange–Act–Assert.
+- Layout mirrors architecture: `tests/unit/{application,agent}`,
+  `tests/integration/{postgres,minio,qdrant}`, `tests/e2e/`.
+- Integration **and e2e** tests use **Testcontainers** (never a developer's
+  local services or Langfuse-owned storage/credentials) — this is the standard
+  automated mechanism, not Docker Compose. Compose is reserved for exactly one
+  thing: the serialized operational smoke test of the *running web process*
+  (`scripts/smoke.sh`), which is gated out of `uv run pytest`. Shared container
+  fixtures live in `tests/support/containers.py`, registered from the root
+  `tests/conftest.py` via `pytest_plugins` (a non-root conftest cannot declare
+  it). Isolate data per test via unique
+  keys/queue/collection names; parallel integration execution is disabled
+  until fixture isolation is proven safe.
+- Pytest never calls a live/paid model provider in routine runs — that's
+  Langfuse's job (dataset experiments), not pytest's. Migrations are always
+  tested through real Alembic upgrade, never `Base.metadata.create_all()`.
+- Invariants worth testing explicitly wherever they apply: tenant identity
+  only from server-side auth context (never request-suppliable); cross-tenant
+  access returns 404, not 403; the file + job row commit atomically before the
+  work phase; no session is held across parse/embed/upsert; a timeout or failure
+  always writes a terminal job status; retrying an upload must not duplicate
+  chunks or regress terminal job state.

README.md

@@ -0,0 +1,94 @@
+# Talie chatbot service
+
+Architecture decisions live in [`docs/adr`](docs/adr). The first implementation
+milestone is documented in the [ingestion vertical-slice plan](docs/plans/001-ingestion-vertical-slice.md).
+Day-to-day operation — tuning the ingestion bounds, the proxy timeout
+requirement, and how to investigate or retry a failed upload — is the
+[operator runbook](docs/runbook.md).
+
+## Provisioning the datastores
+
+Both schema steps run as explicit deployment steps. The application performs no
+DDL at startup — not for Postgres (ADR-0009) and not for Qdrant (ADR-0001,
+"Collection provisioning").
+
+```bash
+docker compose up -d                          # Postgres, MinIO, Qdrant
+uv run alembic upgrade head                   # Postgres schema
+uv run python -m src.cli.qdrant_bootstrap     # the `chunks` collection
+uv run fastapi dev src/main.py
+```
+
+Nothing over HTTP can create the first tenant — every `/v1` route needs an API
+key, and a key cannot exist before its tenant. One command issues both, plus any
+domains, printing the key once (only its hash is stored):
+
+```bash
+uv run python -m src.cli.provision_tenant --slug acme --domain fire
+```
+
+Before a tenant can upload, its domains must be registered — `POST /v1/files`
+rejects an unregistered or disabled `domain` with `400`. The calling backend
+manages them over `/v1/domains` using a key with the `domains:write` scope:
+
+```bash
+curl -X POST http://localhost:8000/v1/domains \
+  -H "Authorization: Bearer $API_KEY" \
+  -H 'Content-Type: application/json' \
+  -d '{"domain": "fire", "display_name": "Fire insurance"}'
+```
+
+Both bootstrap commands are idempotent and safe to re-run. `qdrant_bootstrap` verifies an
+existing collection against the pinned schema and exits non-zero on a mismatch,
+rather than leaving a silently degraded sparse index in place.
+
+`./scripts/smoke.sh` verifies the whole path — Compose up, both deployment
+steps, provisioning, an upload through the running web process to indexed Qdrant
+points. See the [runbook](docs/runbook.md#12-verifying-a-deployment).
+
+## Local Langfuse
+
+This repo includes a root-level development Compose file for Langfuse:
+
+- [`docker-compose.langfuse.yml`](docker-compose.langfuse.yml)
+- [`.env.langfuse.example`](.env.langfuse.example)
+
+Start Langfuse locally:
+
+```bash
+cp .env.langfuse.example .env.langfuse
+# edit .env.langfuse and replace CHANGE_ME values
+
+docker compose --env-file .env.langfuse -f docker-compose.langfuse.yml up -d
+```
+
+Open:
+
+```text
+http://localhost:3000
+```
+
+If the chatbot app runs on your host machine, configure it with:
+
+```env
+LANGFUSE_HOST=http://localhost:3000
+```
+
+If the chatbot app later runs inside the same Compose project/network as
+Langfuse, configure it with:
+
+```env
+LANGFUSE_HOST=http://langfuse-web:3000
+```
+
+A future app stack can be launched together with Langfuse using multiple Compose
+files:
+
+```bash
+docker compose \
+  -f docker-compose.yml \
+  -f docker-compose.langfuse.yml \
+  --env-file .env \
+  --env-file .env.langfuse \
+  up -d
+```

alembic.ini

@@ -0,0 +1,149 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts.
+# this is typically a path given in POSIX (e.g. forward slashes)
+# format, relative to the token %(here)s which refers to the location of this
+# ini file
+script_location = %(here)s/alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+# Or organize into date-based subdirectories (requires recursive_version_locations = true)
+# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.  for multiple paths, the path separator
+# is defined by "path_separator" below.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the tzdata library which can be installed by adding
+# `alembic[tz]` to the pip requirements.
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to <script_location>/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "path_separator"
+# below.
+# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions
+
+# path_separator; This indicates what character is used to split lists of file
+# paths, including version_locations and prepend_sys_path within configparser
+# files such as alembic.ini.
+# The default rendered in new alembic.ini files is "os", which uses os.pathsep
+# to provide os-dependent path splitting.
+#
+# Note that in order to support legacy alembic.ini files, this default does NOT
+# take place if path_separator is not present in alembic.ini.  If this
+# option is omitted entirely, fallback logic is as follows:
+#
+# 1. Parsing of the version_locations option falls back to using the legacy
+#    "version_path_separator" key, which if absent then falls back to the legacy
+#    behavior of splitting on spaces and/or commas.
+# 2. Parsing of the prepend_sys_path option falls back to the legacy
+#    behavior of splitting on spaces, commas, or colons.
+#
+# Valid values for path_separator are:
+#
+# path_separator = :
+# path_separator = ;
+# path_separator = space
+# path_separator = newline
+#
+# Use os.pathsep. Default configuration used for new projects.
+path_separator = os
+
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# database URL.  This is consumed by the user-maintained env.py script only.
+# Left unset here: env.py falls back to Settings().postgres.dsn (ADR-0009),
+# and test fixtures may override it programmatically before invoking Alembic.
+# sqlalchemy.url =
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module
+# hooks = ruff
+# ruff.type = module
+# ruff.module = ruff
+# ruff.options = check --fix REVISION_SCRIPT_FILENAME
+
+# Alternatively, use the exec runner to execute a binary found on your PATH
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = ruff
+# ruff.options = check --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration.  This is also consumed by the user-maintained
+# env.py script only.
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration with an async dbapi.

alembic/env.py

@@ -0,0 +1,89 @@
+import asyncio
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from alembic import context
+from src.config import Settings
+from src.infrastructure.postgres.models import Base
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# Application models' MetaData, used for 'autogenerate' support. The database
+# URL is likewise sourced from application settings by default, so both
+# migrations and the app read the same env-derived configuration (ADR-0009) —
+# unless a caller (e.g. a test fixture pointing at a Testcontainers database)
+# has already set sqlalchemy.url on this Config before invoking Alembic.
+target_metadata = Base.metadata
+if not config.get_main_option("sqlalchemy.url"):
+    config.set_main_option("sqlalchemy.url", Settings().postgres.dsn)
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    """In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    connectable = async_engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+
+    asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,28 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    ${downgrades if downgrades else "pass"}

alembic/versions/41335d162de8_create_tenant_domains.py

@@ -0,0 +1,48 @@
+"""create tenant_domains
+
+Revision ID: 41335d162de8
+Revises: bfc6c81c2542
+Create Date: 2026-08-20 17:48:29.443293
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '41335d162de8'
+down_revision: Union[str, Sequence[str], None] = 'bfc6c81c2542'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('tenant_domains',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('tenant_id', sa.Uuid(), nullable=False),
+    sa.Column('domain', sa.String(length=80), nullable=False),
+    sa.Column('display_name', sa.String(length=200), nullable=False),
+    sa.Column('status', sa.String(length=20), server_default='active', nullable=False),
+    sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default='{}', nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('disabled_at', sa.DateTime(timezone=True), nullable=True),
+    sa.CheckConstraint("status IN ('active', 'disabled')", name='ck_tenant_domains_status'),
+    sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('tenant_id', 'domain', name='uq_tenant_domains_tenant_id_domain')
+    )
+    op.create_index(op.f('ix_tenant_domains_tenant_id'), 'tenant_domains', ['tenant_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_tenant_domains_tenant_id'), table_name='tenant_domains')
+    op.drop_table('tenant_domains')
+    # ### end Alembic commands ###

alembic/versions/bfc6c81c2542_create_tenants_api_keys_source_files_.py

@@ -0,0 +1,156 @@
+"""create tenants, api_keys, source_files, ingestion_jobs, ingestion_job_events
+
+Revision ID: bfc6c81c2542
+Revises: 
+Create Date: 2026-08-16 11:39:02.649094
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = 'bfc6c81c2542'
+down_revision: Union[str, Sequence[str], None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('tenants',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('slug', sa.String(length=80), nullable=False),
+    sa.Column('name', sa.String(length=200), nullable=False),
+    sa.Column('status', sa.String(length=20), server_default='active', nullable=False),
+    sa.Column('settings', postgresql.JSONB(astext_type=sa.Text()), server_default='{}', nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
+    sa.CheckConstraint("status IN ('active', 'suspended', 'deleted')", name='ck_tenants_status'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_tenants_slug'), 'tenants', ['slug'], unique=True)
+    op.create_table('api_keys',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('tenant_id', sa.Uuid(), nullable=False),
+    sa.Column('name', sa.String(length=200), nullable=False),
+    sa.Column('key_prefix', sa.String(length=32), nullable=False),
+    sa.Column('key_hash', sa.String(length=255), nullable=False),
+    sa.Column('scopes', postgresql.JSONB(astext_type=sa.Text()), server_default='[]', nullable=False),
+    sa.Column('actor_type', sa.String(length=20), server_default='backend', nullable=False),
+    sa.Column('status', sa.String(length=20), server_default='active', nullable=False),
+    sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('last_used_at', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('created_by', sa.String(length=200), nullable=True),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.CheckConstraint("actor_type IN ('backend', 'admin', 'worker')", name='ck_api_keys_actor_type'),
+    sa.CheckConstraint("status IN ('active', 'revoked', 'expired')", name='ck_api_keys_status'),
+    sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_api_keys_key_prefix'), 'api_keys', ['key_prefix'], unique=True)
+    op.create_index(op.f('ix_api_keys_tenant_id'), 'api_keys', ['tenant_id'], unique=False)
+    op.create_index('ix_api_keys_tenant_id_status', 'api_keys', ['tenant_id', 'status'], unique=False)
+    op.create_table('source_files',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('tenant_id', sa.Uuid(), nullable=False),
+    sa.Column('domain', sa.String(length=80), nullable=False),
+    sa.Column('source_filename', sa.String(length=500), nullable=False),
+    sa.Column('source_type', sa.String(length=10), nullable=False),
+    sa.Column('content_sha256', sa.String(length=64), nullable=False),
+    sa.Column('byte_size', sa.BigInteger(), nullable=False),
+    sa.Column('storage_uri', sa.String(length=1000), nullable=True),
+    sa.Column('status', sa.String(length=20), server_default='active', nullable=False),
+    sa.Column('created_by_api_key_id', sa.Uuid(), nullable=True),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True),
+    sa.CheckConstraint("source_type IN ('csv', 'xlsx', 'docx', 'doc')", name='ck_source_files_source_type'),
+    sa.CheckConstraint("status IN ('active', 'superseded', 'soft_deleted', 'purged')", name='ck_source_files_status'),
+    sa.ForeignKeyConstraint(['created_by_api_key_id'], ['api_keys.id'], ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_source_files_tenant_id'), 'source_files', ['tenant_id'], unique=False)
+    op.create_index('ix_source_files_tenant_id_content_sha256', 'source_files', ['tenant_id', 'content_sha256'], unique=False)
+    op.create_index('ix_source_files_tenant_id_domain_created_at', 'source_files', ['tenant_id', 'domain', 'created_at'], unique=False)
+    op.create_table('ingestion_jobs',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('tenant_id', sa.Uuid(), nullable=False),
+    sa.Column('source_file_id', sa.Uuid(), nullable=False),
+    sa.Column('requested_by_api_key_id', sa.Uuid(), nullable=True),
+    sa.Column('status', sa.String(length=20), server_default='queued', nullable=False),
+    sa.Column('chunking_strategy', sa.String(length=20), nullable=True),
+    sa.Column('embedding_model_versions', postgresql.JSONB(astext_type=sa.Text()), server_default='{}', nullable=False),
+    sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('points_created', sa.Integer(), server_default='0', nullable=False),
+    sa.Column('points_updated', sa.Integer(), server_default='0', nullable=False),
+    sa.Column('points_soft_deleted', sa.Integer(), server_default='0', nullable=False),
+    sa.Column('points_skipped', sa.Integer(), server_default='0', nullable=False),
+    sa.Column('error_code', sa.String(length=100), nullable=True),
+    sa.Column('error_message', sa.Text(), nullable=True),
+    sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default='{}', nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.CheckConstraint("chunking_strategy IN ('semantic', 'fixed_size')", name='ck_ingestion_jobs_chunking_strategy'),
+    sa.CheckConstraint("status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled')", name='ck_ingestion_jobs_status'),
+    sa.ForeignKeyConstraint(['requested_by_api_key_id'], ['api_keys.id'], ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['source_file_id'], ['source_files.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_ingestion_jobs_source_file_id'), 'ingestion_jobs', ['source_file_id'], unique=False)
+    op.create_index('ix_ingestion_jobs_source_file_id_created_at', 'ingestion_jobs', ['source_file_id', 'created_at'], unique=False)
+    op.create_index(op.f('ix_ingestion_jobs_tenant_id'), 'ingestion_jobs', ['tenant_id'], unique=False)
+    op.create_index('ix_ingestion_jobs_tenant_id_status_created_at', 'ingestion_jobs', ['tenant_id', 'status', 'created_at'], unique=False)
+    op.create_table('ingestion_job_events',
+    sa.Column('id', sa.Uuid(), nullable=False),
+    sa.Column('tenant_id', sa.Uuid(), nullable=False),
+    sa.Column('ingestion_job_id', sa.Uuid(), nullable=False),
+    sa.Column('level', sa.String(length=10), nullable=False),
+    sa.Column('stage', sa.String(length=20), nullable=False),
+    sa.Column('message', sa.String(length=1000), nullable=False),
+    sa.Column('details', postgresql.JSONB(astext_type=sa.Text()), server_default='{}', nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.CheckConstraint("level IN ('info', 'warning', 'error')", name='ck_ingestion_job_events_level'),
+    sa.CheckConstraint("stage IN ('received', 'parsed', 'chunked', 'embedded', 'upserted', 'completed')", name='ck_ingestion_job_events_stage'),
+    sa.ForeignKeyConstraint(['ingestion_job_id'], ['ingestion_jobs.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_ingestion_job_events_ingestion_job_id'), 'ingestion_job_events', ['ingestion_job_id'], unique=False)
+    op.create_index('ix_ingestion_job_events_ingestion_job_id_created_at', 'ingestion_job_events', ['ingestion_job_id', 'created_at'], unique=False)
+    op.create_index(op.f('ix_ingestion_job_events_tenant_id'), 'ingestion_job_events', ['tenant_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_ingestion_job_events_tenant_id'), table_name='ingestion_job_events')
+    op.drop_index('ix_ingestion_job_events_ingestion_job_id_created_at', table_name='ingestion_job_events')
+    op.drop_index(op.f('ix_ingestion_job_events_ingestion_job_id'), table_name='ingestion_job_events')
+    op.drop_table('ingestion_job_events')
+    op.drop_index('ix_ingestion_jobs_tenant_id_status_created_at', table_name='ingestion_jobs')
+    op.drop_index(op.f('ix_ingestion_jobs_tenant_id'), table_name='ingestion_jobs')
+    op.drop_index('ix_ingestion_jobs_source_file_id_created_at', table_name='ingestion_jobs')
+    op.drop_index(op.f('ix_ingestion_jobs_source_file_id'), table_name='ingestion_jobs')
+    op.drop_table('ingestion_jobs')
+    op.drop_index('ix_source_files_tenant_id_domain_created_at', table_name='source_files')
+    op.drop_index('ix_source_files_tenant_id_content_sha256', table_name='source_files')
+    op.drop_index(op.f('ix_source_files_tenant_id'), table_name='source_files')
+    op.drop_table('source_files')
+    op.drop_index('ix_api_keys_tenant_id_status', table_name='api_keys')
+    op.drop_index(op.f('ix_api_keys_tenant_id'), table_name='api_keys')
+    op.drop_index(op.f('ix_api_keys_key_prefix'), table_name='api_keys')
+    op.drop_table('api_keys')
+    op.drop_index(op.f('ix_tenants_slug'), table_name='tenants')
+    op.drop_table('tenants')
+    # ### end Alembic commands ###

docker-compose.langfuse.yml

@@ -0,0 +1,198 @@
+# Development/staging Langfuse v4 Docker Compose stack.
+#
+# Based on the official Langfuse Docker Compose deployment:
+# https://langfuse.com/self-hosting/deployment/docker-compose
+#
+# This file is intentionally separate from the future application compose file.
+# Run it alone for Langfuse-only development, or combine it with the app stack:
+#
+#   docker compose \
+#     -f docker-compose.yml \
+#     -f docker-compose.langfuse.yml \
+#     --env-file .env \
+#     --env-file .env.langfuse \
+#     up -d
+#
+# If the chatbot app runs in the same Compose project/network, configure it with:
+#   LANGFUSE_HOST=http://langfuse-web:3000
+# If the chatbot app runs directly on the host machine, configure it with:
+#   LANGFUSE_HOST=http://localhost:3000
+#
+# Do not use this compose stack as-is for high-availability production.
+services:
+  langfuse-worker:
+    image: docker.io/langfuse/langfuse-worker:4
+    restart: always
+    depends_on: &langfuse-depends-on
+      langfuse-postgres:
+        condition: service_healthy
+      langfuse-minio:
+        condition: service_healthy
+      langfuse-redis:
+        condition: service_healthy
+      langfuse-clickhouse:
+        condition: service_healthy
+    ports:
+      - 127.0.0.1:${LANGFUSE_WORKER_PORT:-3030}:3030
+    environment: &langfuse-worker-env
+      NEXTAUTH_URL: ${LANGFUSE_NEXTAUTH_URL:-http://localhost:3000}
+      DATABASE_URL: postgresql://${LANGFUSE_POSTGRES_USER:-postgres}:${LANGFUSE_POSTGRES_PASSWORD:-postgres}@langfuse-postgres:5432/${LANGFUSE_POSTGRES_DB:-postgres}
+      SALT: ${LANGFUSE_SALT:-mysalt} # CHANGEME
+      ENCRYPTION_KEY: ${LANGFUSE_ENCRYPTION_KEY:-0000000000000000000000000000000000000000000000000000000000000000} # CHANGEME: generate via `openssl rand -hex 32`
+      TELEMETRY_ENABLED: ${LANGFUSE_TELEMETRY_ENABLED:-false}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      CLICKHOUSE_MIGRATION_URL: clickhouse://langfuse-clickhouse:9000
+      CLICKHOUSE_URL: http://langfuse-clickhouse:8123
+      CLICKHOUSE_USER: ${LANGFUSE_CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${LANGFUSE_CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
+      CLICKHOUSE_CLUSTER_ENABLED: ${LANGFUSE_CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false}
+      LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false}
+      LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: http://langfuse-minio:9000
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: http://langfuse-minio:9000
+      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED: ${LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED:-false}
+      LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES: ${LANGFUSE_OBSERVATION_FIELD_SIZE_LIMIT_BYTES:-2097152}
+      LANGFUSE_S3_BATCH_EXPORT_ENABLED: ${LANGFUSE_S3_BATCH_EXPORT_ENABLED:-false}
+      LANGFUSE_S3_BATCH_EXPORT_BUCKET: ${LANGFUSE_S3_BATCH_EXPORT_BUCKET:-langfuse}
+      LANGFUSE_S3_BATCH_EXPORT_PREFIX: ${LANGFUSE_S3_BATCH_EXPORT_PREFIX:-exports/}
+      LANGFUSE_S3_BATCH_EXPORT_REGION: ${LANGFUSE_S3_BATCH_EXPORT_REGION:-auto}
+      LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: http://langfuse-minio:9000
+      LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: ${LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT:-http://localhost:9090}
+      LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: ${LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY: ${LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY:-miniosecret} # CHANGEME
+      LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: ${LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE:-true}
+      LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
+      LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
+      REDIS_HOST: langfuse-redis
+      REDIS_PORT: ${LANGFUSE_REDIS_PORT:-6379}
+      REDIS_AUTH: ${LANGFUSE_REDIS_AUTH:-myredissecret} # CHANGEME
+      LANGFUSE_BULLMQ_SKIP_REDIS_VERSION_CHECK: ${LANGFUSE_BULLMQ_SKIP_REDIS_VERSION_CHECK:-false}
+      REDIS_TLS_ENABLED: ${LANGFUSE_REDIS_TLS_ENABLED:-false}
+      REDIS_TLS_CA: ${LANGFUSE_REDIS_TLS_CA:-/certs/ca.crt}
+      REDIS_TLS_CERT: ${LANGFUSE_REDIS_TLS_CERT:-/certs/redis.crt}
+      REDIS_TLS_KEY: ${LANGFUSE_REDIS_TLS_KEY:-/certs/redis.key}
+      EMAIL_FROM_ADDRESS: ${LANGFUSE_EMAIL_FROM_ADDRESS:-}
+      SMTP_CONNECTION_URL: ${LANGFUSE_SMTP_CONNECTION_URL:-}
+
+  langfuse-web:
+    image: docker.io/langfuse/langfuse:4
+    restart: always
+    depends_on: *langfuse-depends-on
+    ports:
+      - ${LANGFUSE_WEB_PORT:-3000}:3000
+    environment:
+      <<: *langfuse-worker-env
+      NEXTAUTH_SECRET: ${LANGFUSE_NEXTAUTH_SECRET:-mysecret} # CHANGEME
+      # This value is used by browser-facing media upload flows and must be
+      # reachable from the browser, not just from inside the Docker network.
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT_PUBLIC:-http://localhost:9090}
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+
+  langfuse-clickhouse:
+    image: docker.io/clickhouse/clickhouse-server:25.12
+    restart: always
+    user: "101:101"
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: ${LANGFUSE_CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${LANGFUSE_CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - 127.0.0.1:${LANGFUSE_CLICKHOUSE_HTTP_PORT:-18123}:8123
+      - 127.0.0.1:${LANGFUSE_CLICKHOUSE_NATIVE_PORT:-19000}:9000
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  langfuse-minio:
+    image: cgr.dev/chainguard/minio
+    restart: always
+    entrypoint: sh
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: ${LANGFUSE_MINIO_ROOT_USER:-minio}
+      MINIO_ROOT_PASSWORD: ${LANGFUSE_MINIO_ROOT_PASSWORD:-miniosecret} # CHANGEME
+    ports:
+      - ${LANGFUSE_MINIO_API_PORT:-9090}:9000
+      - 127.0.0.1:${LANGFUSE_MINIO_CONSOLE_PORT:-9091}:9001
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  langfuse-redis:
+    image: docker.io/redis:7
+    restart: always
+    command: >
+      --requirepass ${LANGFUSE_REDIS_AUTH:-myredissecret}
+      --maxmemory-policy noeviction
+    ports:
+      - 127.0.0.1:${LANGFUSE_REDIS_HOST_PORT:-16379}:6379
+    volumes:
+      - langfuse_redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
+
+  langfuse-postgres:
+    image: docker.io/postgres:${LANGFUSE_POSTGRES_VERSION:-17}
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${LANGFUSE_POSTGRES_USER:-postgres}"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+    environment:
+      POSTGRES_USER: ${LANGFUSE_POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${LANGFUSE_POSTGRES_PASSWORD:-postgres} # CHANGEME
+      POSTGRES_DB: ${LANGFUSE_POSTGRES_DB:-postgres}
+      TZ: UTC
+      PGTZ: UTC
+    ports:
+      - 127.0.0.1:${LANGFUSE_POSTGRES_HOST_PORT:-15432}:5432
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
+    driver: local
+  langfuse_redis_data:
+    driver: local

docker-compose.yml

@@ -0,0 +1,74 @@
+# Application-owned local infrastructure: Postgres, MinIO, Qdrant.
+#
+# Intentionally separate from docker-compose.langfuse.yml (different service
+# names, volumes, and ports). Run alone for app development, or combine with
+# Langfuse:
+#
+#   docker compose \
+#     -f docker-compose.yml \
+#     -f docker-compose.langfuse.yml \
+#     --env-file .env \
+#     --env-file .env.langfuse \
+#     up -d
+#
+# Do not use this compose stack as-is for production.
+services:
+  app-postgres:
+    image: docker.io/postgres:17
+    restart: always
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER:-chatbot}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-chatbot}
+      POSTGRES_DB: ${POSTGRES_DB:-chatbot}
+    ports:
+      - 127.0.0.1:${POSTGRES_PORT:-5433}:5432
+    volumes:
+      - app_postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-chatbot}"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+
+  app-minio:
+    image: cgr.dev/chainguard/minio
+    restart: always
+    entrypoint: sh
+    command: -c 'mkdir -p /data/${MINIO_BUCKET:-chatbot-source-files} && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:-chatbot}
+      MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:-chatbot-secret}
+    ports:
+      - 127.0.0.1:9100:9000
+      - 127.0.0.1:9101:9001
+    volumes:
+      - app_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  app-qdrant:
+    image: docker.io/qdrant/qdrant
+    restart: always
+    ports:
+      - 127.0.0.1:6343:6333
+      - 127.0.0.1:6344:6334
+    volumes:
+      - app_qdrant_data:/qdrant/storage
+    healthcheck:
+      test: ["CMD-SHELL", "bash -c '</dev/tcp/localhost/6333' 2>/dev/null || exit 1"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+      start_period: 3s
+
+volumes:
+  app_postgres_data:
+    driver: local
+  app_minio_data:
+    driver: local
+  app_qdrant_data:
+    driver: local

docs/adr/0001-ingestion-pipeline-and-collection-schema.md

@@ -49,7 +49,7 @@ One collection, e.g. `chunks`, shared by all tenants and domains.
 | Name | Type | Purpose | Notes |
 |---|---|---|---|
 | `dense_nomic` | dense vector | primary semantic similarity (multilingual, incl. Persian) | `nomic-embed-text-v2-moe`, 768-dim ([0004](0004-docx-csv-chunking-strategy.md)) |
-| `dense_openai` | dense vector | second semantic signal | OpenAI large embedding model (e.g. `text-embedding-3-large`), dimension per OpenAI's `dimensions` param (TBD — full 3072 vs. a truncated size) |
+| `dense_openai` | dense vector | second semantic signal | `text-embedding-3-large` at its **native 3072 dimensions** — the `dimensions` param is deliberately left unset (see below) |
 | `sparse` | sparse vector | lexical/keyword-sensitive retrieval | `bm25-fa-norm-stop` — Qdrant FastEmbed's BM25 sparse encoder configured for Persian (stopword removal + normalization), not a separately trained model |
 | `late_interaction` | multivector | reserved for late-interaction rerank ([0003](0003-agent-hybrid-retrieval.md)) | `jina-colbert-v2` ([0005](0005-reranking-model-and-sparse-analyzer-selection.md)), `comparator: max_sim`, `hnsw_config: m=0` (rerank-only, never independently ANN-searched), stored **on disk** |
 
@@ -61,6 +61,35 @@ dense/sparse query latency. Two dense vectors are provisioned deliberately —
 `dense_nomic` and `dense_openai` are two independent semantic signals, both
 prefetched and fused at query time (ADR-0003), not a primary/fallback pair.
 
+#### Dense model endpoints and dimensions (resolved by the `emet` benchmark)
+
+Both dense models are reached over the **same OpenAI-compatible
+`/embeddings` API**, so one adapter
+(`src/infrastructure/embedding/openai_compatible.py`) serves both named
+vectors with different configuration:
+
+| Named vector | Model | Endpoint | Dimensions |
+|---|---|---|---|
+| `dense_nomic` | `nomic-embed-text-v2-moe` | self-hosted Ollama OpenAI-compat shim | **768** (verified against the live endpoint) |
+| `dense_openai` | `text-embedding-3-large` | OpenAI hosted API | **3072** (native; `dimensions` unset) |
+
+`dense_openai`'s dimension was previously listed as an open dependency. It is
+now pinned to the native 3072, because that is the configuration the `emet`
+lab benchmarked — it never passed a `dimensions` argument. Setting it later
+would truncate via Matryoshka and is a **re-embedding migration, not a config
+tweak**, exactly as the negative consequence below warns.
+
+Two operational notes about the self-hosted embedder, both learned by
+measurement rather than assumption:
+
+- **Cold load exceeds 150s**, far beyond `INGESTION_TIMEOUT_SECONDS`, so an
+  idle-then-upload would return `504`. Mitigated on both ends: Ollama's
+  `keep_alive` keeps the model resident, and the FastAPI lifespan warms each
+  dense embedder at startup (fail-soft — a down embedder must not block boot).
+- **Once warm it is fast**: ~0.30s for one input and ~0.34s for a batch of 16.
+  Batching is therefore nearly free, which is what keeps ADR-0017's inline
+  ingestion viable.
+
 ### Multitenancy / indexing config
 
 - HNSW: `m: 0` (disable the global index) + `payload_m: 16`, per Qdrant's
@@ -79,6 +108,38 @@ prefetched and fused at query time (ADR-0003), not a primary/fallback pair.
 - Payload index on `previous_chunk_id` / `next_chunk_id`: keyword index,
   used for O(1) adjacency retrieval (see below).
 
+### Collection provisioning
+
+The collection is created by an explicit **deployment step**, not by application
+startup and not lazily on first write:
+
+    uv run python -m src.cli.qdrant_bootstrap
+
+Creating a collection is DDL, and this project already keeps DDL out of the boot
+and request paths: [0009](0009-postgres-sqlalchemy-alembic-schema.md) requires
+Alembic for Postgres schema and forbids `create_all()` at startup, and
+[0012](0012-application-resource-lifetime-and-dependency-ownership.md) makes
+LangGraph's `.setup()` a deployment step for the same reason. Neither ADR named
+Qdrant explicitly; this section closes that gap rather than letting the placement
+be decided by whichever code happened to need it first.
+
+Doing it in the FastAPI lifespan was rejected: it couples process boot to Qdrant
+being reachable (which is `/readyz`'s job, not boot's), races across replicas,
+and turns a misconfigured collection into a silent skip. Doing it lazily on first
+upsert was rejected for putting DDL on a user request and hiding the
+misconfiguration until traffic arrives.
+
+`ensure_chunks_collection` is idempotent and **verifying**: against an existing
+collection it compares the dense dimensions and the sparse `modifier` to the
+pinned values and fails loudly on divergence. That check is the point of making
+the step explicit — both properties degrade silently in production if wrong (a
+missing `modifier="idf"` produces no error, just unweighted lexical retrieval).
+
+Payload indexes are (re)created on every run, since unlike vector configuration
+they can be added to a live collection. The full-text index on `content` is
+therefore deferred to the keyword-search work in
+[0002](0002-chunk-crud-and-search-api.md), not created here.
+
 ### Payload schema
 
 This schema is now decided for the fields below. Additional document-context
@@ -95,7 +156,7 @@ involves format-specific tradeoffs not yet made.
 | `chunk_id` | keyword | stable identifier for a single chunk |
 | `content_type` | keyword | classification of the chunk's content; exact value set (e.g. `paragraph`, `table_row`, `heading`) to be finalized alongside the chunking-strategy ADR |
 | `source_filename` | keyword | original uploaded filename |
-| `source_type` | keyword (`docx` \| `csv`) | which parser produced this chunk |
+| `source_type` | keyword (`docx` \| `xlsx` \| `csv`) | which parser produced this chunk — `xlsx` added by [0018](0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md) |
 | `order_id` | float (see below) | chunk's *display* position within the file; mutable so the backend can reorder/insert chunks |
 | `chunk_index` | integer | chunk's *original ingestion* ordinal — immutable, used to derive the deterministic point ID below (kept separate from `order_id` precisely because `order_id` can change) |
 | `previous_chunk_id` | keyword, nullable | `chunk_id` of the preceding chunk in display order (`null` for the first chunk in a file) — O(1) adjacency pointer for context-window expansion in ADR-0003 |
@@ -107,7 +168,7 @@ involves format-specific tradeoffs not yet made.
 | `updated_at` | datetime | last modification timestamp |
 | `created_by` | keyword | user/service that created the chunk |
 | `updated_by` | keyword | user/service that last modified the chunk |
-| `version` | integer | optimistic-concurrency counter, used in ADR-0002 |
+| `version` | integer | optimistic-concurrency counter, used in ADR-0002. Ingestion currently writes `1` unconditionally: the read-check-write that makes the guard meaningful costs one read per point and belongs with the `/v1/points` write paths, so plan 002 owns it. Safe while ingestion is the only writer of a file's points; it would clobber a concurrent manual edit's counter once `/v1/points` ships. |
 | `content_hash` | keyword | hash of the chunk's raw text; lets re-ingestion detect unchanged content and skip re-embedding it |
 | `embedding_model_version` | keyword | identifies which embedding model(s) produced this chunk's vectors; needed to know which chunks require re-embedding after a future model swap |
 
@@ -212,9 +273,14 @@ them — see ADR-0002 for how reorder/insert/delete operations keep
   ingestion time and both are queried at retrieval time — roughly double
   the dense embedding cost/latency of a single-dense-vector design, plus an
   external network dependency on OpenAI's API in the ingestion path.
-- `dense_openai`'s exact output dimension is still an open dependency that
+- ~~`dense_openai`'s exact output dimension is still an open dependency~~ —
-  should be pinned before ingestion is implemented — changing it later is a
+  **resolved**: pinned to the native 3072 (see "Dense model endpoints and
-  re-embedding migration, not a config tweak.
+  dimensions" above). The warning still stands for any future change:
+  re-dimensioning is a re-embedding migration, not a config tweak.
+- The `sparse` vector must be created with `modifier="idf"`. The client
+  computes only BM25's term-frequency saturation; without that modifier
+  Qdrant applies no IDF at all and lexical retrieval silently degrades
+  (ADR-0005).
 - `jina-colbert-v2` ([0005](0005-reranking-model-and-sparse-analyzer-selection.md))
   adds a hard GPU dependency to ingestion (not just query time, since the
   document-side multivector is computed here) and its commercial license is

docs/adr/0002-chunk-crud-and-search-api.md

@@ -75,6 +75,51 @@ retrieval used by the AI agent in ADR-0003; the two "search" concepts serve
 different callers (a human/admin managing chunks vs. an agent retrieving
 context) and should not be conflated in the API or in future discussion.
 
+Two properties follow from the index being a *filter*: results carry no
+relevance score, and their order is unspecified. The API therefore returns
+neither a score field nor a ranked list, and callers must not read the array
+order as relevance. A caller that wants ranking wants ADR-0003's path.
+
+#### The query is normalized the way ingested content was
+
+`normalize_persian_text` (ADR-0018) folds Arabic letterforms to their Persian
+equivalents — U+064A to U+06CC, U+0643 to U+06A9 — on every text block before
+chunking, so stored `content` is uniformly Persian-formed. A query string is
+not chunk content and never passes through that path, so a term typed on an
+Arabic keyboard reaches the index as a different codepoint sequence than the
+document it should match.
+
+The service therefore applies the same folding to the query before matching.
+Without it the endpoint fails in the worst available way: an exact-looking
+query returns an empty result set, with no error, no warning, and nothing in
+the logs to distinguish "no such term" from "the term is spelled with the
+other yeh". Note this is a *query-side* transformation only — it changes what
+is compared, never what is stored.
+
+This does not extend to stemming or synonyms. Qdrant's full-text index offers
+neither, and adding a Farsi analyzer here would duplicate the benchmarked BM25
+sparse pipeline (ADR-0005) in a code path that is not benchmarked against
+anything.
+
+#### Listing is scoped to one file, and paginates by `order_id`
+
+`GET /points?file_id=...` requires `file_id` rather than treating it as one
+optional filter among several, and its pagination cursor is an `order_id`
+value rather than an offset. Both follow from `order_id` being per-file:
+
+- A cursor is only meaningful against a totally ordered key. `order_id` orders
+  points within one file and says nothing across files, so an unscoped listing
+  has no stable sort to paginate along.
+- An offset cursor is wrong even within one file. Insert, reorder, and delete
+  all shift positions, so a page-two request issued after a concurrent insert
+  ahead of the cursor would repeat a row already returned — silently. Ranging
+  on `order_id > cursor` is unaffected: the reader has passed that value, and a
+  point inserted behind it was already served.
+
+The second point depends on `order_id` being unique within a file, which the
+gap-exhaustion rule below preserves by rejecting a reorder whose computed gap
+would collapse onto a neighbour value.
+
 ### Delete is soft by default
 
 `DELETE /points/{point_id}` and `DELETE /points?file_id=...` set
@@ -105,6 +150,97 @@ Qdrant's `update_filter`, giving an optimistic-concurrency-style guard
 against races between a concurrent ingestion re-run (ADR-0001) and a manual
 edit through this API.
 
+### Re-ingestion versus manual edits
+
+A file can be re-uploaded after someone has hand-edited one of its points
+through this API. **The newly ingested file wins.** Ingestion is authoritative
+for the content of the file it ingested; a manual edit is a correction that
+survives only until the source document is replaced.
+
+Concretely:
+
+- A point that still exists in the new version (same `file_id` +
+  `chunk_index`, hence the same deterministic point ID) is **overwritten in
+  place**. Ingestion performs a read-check-write so `version` is incremented
+  from whatever the manual edit left it at, rather than reset to `1`.
+- A point from the previous ingestion that is **absent** from the new version
+  is flagged `is_active: false` with `deleted_at` set. It is never removed
+  from Qdrant — the soft-delete rule above applies to re-ingestion exactly as
+  it applies to `DELETE`.
+- A manually created point (`POST /points`) is assigned a `chunk_index` past
+  the ingested range, so the same sweep deactivates it on the next upload of
+  its file. This is the intended consequence of "the new file wins", not an
+  accident of the sweep's bounds.
+
+Because the point ID is derived from the immutable `chunk_index`, an
+overwritten point cannot hold both the manual edit and the new file's content.
+The clobbered content is therefore recorded in `point_audit_events`
+(ADR-0009) as a `reingest_overwrite` operation carrying `before_version`, so
+the edit is recoverable from the audit trail even though it is no longer a
+live point.
+
+Rejected alternative: preserving manual edits by having ingestion skip points
+with `version > 1`. It breaks the guarantee that a successful upload leaves
+Qdrant matching the uploaded document, and it needs a second, separate rule
+for edited points that no longer exist in the new version — two divergent
+notions of authority over one file.
+
+### Re-embedding on content edit
+
+`PUT /points/{point_id}` can change `content`, which leaves the stored
+vectors stale unless they're recomputed. When `content` changes, the point
+is **re-embedded inline**, reusing the same async embedding ports and
+batch/semaphore bounds ingestion uses ([0017](0017-synchronous-ingestion-in-the-request-path.md)),
+for parity between the two write paths. When `content` is unchanged, the
+edit applies only the supplied vector/payload fields and skips re-embedding
+entirely. Failure modes on this path reuse ingestion's status codes: `502`
+on embedder failure, `504` if the edit's embedding step exceeds the same
+timeout budget class as ingestion. The `version` guard (`update_filter`)
+still applies to the write — re-embedding happens before the guarded write,
+not instead of it, so a stale-version edit still fails with `409` rather
+than re-embedding for nothing.
+
+Rejected alternatives: requiring the caller to supply vectors when content
+changes (pushes model knowledge onto the client, and is easy to get subtly
+wrong); marking the point stale for background re-embedding later (needs
+background work, which ADR-0017 currently rules out for this slice).
+
+### `order_id` gap exhaustion
+
+Repeatedly inserting into the same gap between two neighbors eventually
+exhausts float precision (ADR-0001's known limitation). This slice does
+**not** ship a renormalize endpoint. Instead, any operation that assigns a
+new fractional `order_id` between two neighbors (insert, reorder) computes
+the resulting gap and:
+
+- logs a structured warning (`points.order_id.gap_low`) with `file_id` and
+  the two neighbor point IDs once the gap falls under a defined safety
+  threshold, so the condition is observable before it becomes uninsertable;
+- **rejects** the write with `409` and a distinct error code if the
+  computed gap is no longer representable (would collapse to one of the two
+  neighbor values), instead of silently applying an imprecise value.
+
+Recovering from an exhausted gap is a manual data-fix operation covered by
+the operator runbook, not an endpoint this slice builds — deferring the
+renormalize primitive is acceptable, silently producing an unrepresentable
+gap is not.
+
+### `POST /points/batch` semantics
+
+Batch requests are **all-or-nothing**, capped at **100 operations per
+request**. The service layer validates every operation's `version`
+precondition before applying any of them; if any operation's precondition
+fails, the whole request is rejected with `409` and nothing is applied — no
+partially-applied batch ever reaches Qdrant. This follows directly from the
+`version`-guard rule above applied at the batch level, and from the
+pointer-relinking rule (a reorder/insert/delete's neighbor updates must land
+in the same `points/batch` call, and a partial relink is a defect): partial
+application of a batch is exactly the failure mode that would produce a
+stale pointer chain. The 100-operation cap is independent of ADR-0001's
+64–256-point bulk-ingestion batch sizing — that number is about upload
+throughput; this one bounds an admin/manual edit request to something that
+comfortably finishes inside a normal request timeout.
+
 ## Consequences
 
 ### Positive
@@ -122,6 +258,15 @@ edit through this API.
 - Optimistic concurrency via `version` requires every writer (ingestion and
   this API) to consistently read-check-write; a writer that skips this can
   silently clobber concurrent edits.
+- Inline re-embedding puts embedder latency and `502`/`504` failure modes on
+  an admin content edit, not just on ingestion — an edit that only intended
+  to fix a typo pays the same embedding cost as a fresh chunk.
+- Deferring the `order_id` renormalize endpoint means a file whose gaps are
+  genuinely exhausted has no automated recovery in this slice; an operator
+  must intervene by hand until that endpoint exists.
+- All-or-nothing batch semantics mean one stale operation in a 100-operation
+  batch fails the entire request, even when the other 99 operations are
+  independent and would have succeeded on their own.
 
 ## Alternatives Considered
 
@@ -132,3 +277,16 @@ edit through this API.
 - **Client-supplied `tenant_id` in request body**: rejected — trusting
   client input for the isolation boundary is a direct multitenancy security
   risk; it must come from server-side auth context.
+- **Caller-supplied vectors on content edit**: rejected — pushes embedding
+  model knowledge onto the client and makes it easy to silently desync
+  vectors from content.
+- **Mark-stale-and-re-embed-later on content edit**: rejected for this
+  slice — needs background work, which ADR-0017 currently rules out.
+- **Renormalize `order_id` automatically within this slice**: rejected —
+  nothing in current scope has hit gap exhaustion; building the primitive
+  now is speculative. Revisit if the logged warning starts firing in
+  practice.
+- **Partial-success batch semantics (per-operation status)**: rejected —
+  a partially-applied batch is exactly the failure mode that leaves the
+  pointer chain (`previous_chunk_id`/`next_chunk_id`) inconsistent, which
+  this ADR treats as a defect, not a degraded-but-acceptable outcome.

docs/adr/0004-docx-csv-chunking-strategy.md

@@ -4,6 +4,23 @@
 
 Accepted
 
+> Amended by
+> [ADR-0018](0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md):
+> v1 ships **fixed-size** chunking rather than the semantic-aware default
+> below, and defers `qa_pair` detection and image captioning. Docx tables do
+> become `table_row` units as this ADR specifies, but only when they are data:
+> a table holding a cell larger than one chunk, or a cell containing nested
+> tables, is treated as page layout and its cells are chunked as prose.
+> Row labels are applied only when row 0 is provably a header, and cells are
+> joined unlabeled otherwise. No document tree is built, and no heading is
+> inferred from text. `.doc` is rejected with `415` pending an out-of-process
+> conversion service rather than shelling out to LibreOffice. ADR-0018 also
+> adds a Persian normalization step at parse time and fixes the chunk-size
+> numbers this ADR left open. The spreadsheet row-to-chunk rules, the
+> embedding model, the task-prefix invariant, and the `content_type` value set
+> below all apply unchanged — except that `.csv` is read with the standard
+> library `csv` module rather than `pandas`.
+
 ## Context
 
 ADR-0001 deferred two things to "when we start the docx/csv chunking work":
@@ -45,6 +62,23 @@ from its model card: 768-dim output, Matryoshka-truncatable down to 256;
 every embedded string — `search_document: ` at ingestion time, `search_query: `
 on the agent's query side (ADR-0003).
 
+> **Amendment — the task prefix is currently not applied.** The `emet`
+> benchmark that selected this model ran *without* any prefix: its Ollama
+> deployment's template is a bare `{{ .Prompt }}` passthrough that injects
+> nothing, which was verified directly against the running endpoint. The
+> prefix is not cosmetic — embedding the same Persian text with and without
+> `search_document: ` yields a cosine of only **0.5741** — so applying it at
+> ingest while the query side omits `search_query: ` would make retrieval
+> *worse* than using neither.
+>
+> Implementation therefore defaults `EMBEDDING_NOMIC_DOCUMENT_PREFIX` to
+> empty, matching the measured configuration, and exposes it as config so the
+> prefixed variant is a one-line experiment rather than a code change. The
+> model card remains the reason to expect prefixing to help; what is missing
+> is evidence on *this* corpus. Turning it on is a paired change — ingest and
+> query must move together — and should be settled by an emet run that
+> measures the pair, not by an unmeasured edit here.
+
 ## Decision
 
 ### Parsing order: structural extraction before chunking

docs/adr/0005-reranking-model-and-sparse-analyzer-selection.md

@@ -2,9 +2,10 @@
 
 ## Status
 
-Proposed — the fusion/rerank *shape* and reranker model are decided; the
+Proposed — the fusion/rerank *shape*, the reranker model, and (as of the
-final BM25 analyzer and the commercial license status of the reranker are
+`emet` benchmark, see "Benchmark outcome" below) the **BM25 analyzer** are
-still open per the follow-up items below.
+decided. The commercial license status of the reranker remains open per the
+follow-up items below.
 
 ## Context
 
@@ -100,6 +101,50 @@ entirely to the analyzer stage, not the ranking formula:
 consistent with Farsi's high density of function words (ezafe particles,
 prepositions, common verbs) adding TF/IDF noise if left in.
 
+### 3a. Benchmark outcome: `bm25-fa-norm-stop` confirmed, and where the BM25 math runs
+
+The `emet` evaluation lab (`~/code/talie/emet`) ran the four-variant
+comparison above against the real Farsi corpus and confirmed
+**`bm25-fa-norm-stop`** as the winner. It is the only sparse variant promoted
+into emet's hybrid matrix (`emet/hybrid.yaml`). This closes follow-up item 4
+below.
+
+The winning analyzer is a specific, reproducible artifact, ported into
+`src/infrastructure/embedding/analyzers.py` and verified token-for-token
+against emet's implementation. Its details are load-bearing:
+
+- Unicode **NFC** (not NFKC), then ZWNJ → space, then Persian/Arabic-Indic
+  digits → ASCII, then `ي→ی ك→ک ة→ه ؤ→و إ→ا أ→ا`.
+- Tokenizer `[^\W_]+`, which **keeps digits**. This matters for an insurance
+  corpus: policy numbers, dates, and amounts are exactly the terms lexical
+  retrieval should match, and the digit folding above means a query in ASCII
+  digits matches a document authored in Persian ones.
+- A 51-entry stopword set (40 Persian/Arabic + 11 English, the corpus being
+  mixed-script). Deliberately not a full `hazm` list.
+- No stemming — `fa_norm_stem` was the losing arm.
+
+**The BM25 formula is split across two systems, deliberately.** The client
+applies term-frequency saturation, including the `k`/`b` document-length
+normalization; **IDF is supplied by Qdrant** via `modifier="idf"` on the
+sparse vector field, computed from collection-wide statistics rather than
+from a fixed client-side corpus.
+
+That split is a correctness trap worth stating plainly: a `chunks` collection
+created *without* `modifier="idf"` will score these vectors as saturated term
+frequencies with no IDF weighting at all — no error, no warning, just
+materially worse lexical retrieval. The collection bootstrap must set it.
+
+Document and query encoding are asymmetric in exactly one term: documents
+carry the `b` length normalization, queries do not (standard BM25 practice).
+Both sides must therefore encode through the same implementation, which is
+why the sparse port carries a `query` flag rather than leaving retrieval to
+grow a second, silently divergent encoder.
+
+Term → sparse-index mapping is `blake2b(token, digest_size=8) % (2**31 - 1)`,
+a pure hash with no vocabulary table, so it needs no shared state and stays
+identical across processes and between ingest and query time. Changing the
+hash orphans every stored sparse vector: that is a re-ingestion, not a deploy.
+
 ### 4. BM25 parameters: keep `k=1.2`, `b=0.75`; tune analyzer, not formula
 
 These are standard, well-validated defaults (Trotman, Puurula & Burgess,
@@ -159,8 +204,21 @@ comparison and `b` sweep in the follow-ups below.
 3. Run an ablation: single dense model + sparse + rerank vs. the current
    dual-dense-model + sparse + rerank setup, on real Farsi queries, to
    justify (or drop) the second dense vector (`dense_openai`).
-4. Compare `bm25-fa-norm-stop` vs. `bm25-fa-norm-stem` in isolation to
+4. ~~Compare `bm25-fa-norm-stop` vs. `bm25-fa-norm-stem` in isolation~~ —
-   determine whether gains come from stopword removal, stemming, or both.
+   **done**, see "Benchmark outcome" above. `fa_norm_stop` won; stemming was
+   not adopted.
 5. Sweep BM25 `b` (e.g. 0.5–0.9) for the winning analyzer, since document
    length varies significantly across the corpus (short chat messages vs.
    long articles) and `0.75` is a generic default, not corpus-tuned.
+6. **Recalibrate `avg_len`.** The client-side `b` term needs an average
+   document length in *analyzer tokens*. The ported value (256.0) is emet's
+   own placeholder, and emet measured it over short Q&A records rather than
+   this service's ~400-token chunks, so it is very likely miscalibrated here.
+   Exposed as `EMBEDDING_SPARSE_AVG_LEN` so it can be corrected from real
+   corpus statistics without a code change.
+7. **Re-benchmark the analyzer with diacritic stripping.** `fa_norm_stop`
+   does not remove harakat or tatweel, so `ســلام` and `سلام` are distinct
+   terms. `src/application/ingestion/normalization.py` already strips both
+   for chunk *content*; extending that to the analyzer is plausibly an
+   improvement but would deviate from the measured configuration, so it
+   belongs in an emet run rather than an unmeasured edit.

docs/adr/0008-rest-api-and-fastapi-boundary.md

@@ -164,16 +164,20 @@ metadata such as `domain`. File validation is server-side:
 - Derive `tenant_id`, `created_by`, and `updated_by` from `AuthContext`, not
   from form fields.
 
-Ingestion may be slow because it parses, chunks, embeds, and writes many
+Ingestion parses, chunks, embeds, and writes many Qdrant points.
-Qdrant points. The REST contract is job-shaped even if the first
+[ADR-0017](0017-synchronous-ingestion-in-the-request-path.md) supersedes the
-implementation runs inline:
+job-shaped contract originally specified here: ingestion runs inline and the
+response is terminal.
 
 ```text
-202 Accepted -> { file_id, ingestion_job_id, status: "queued" | "running" }
+201 Created -> { file_id, ingestion_job_id, status: "succeeded", chunks_indexed }
 ```
 
-A durable worker/job queue can be added later without changing the API
+`ingestion_job_id` is retained so the attempt stays inspectable via
-contract.
+`GET /v1/files/{file_id}`, and so a future move back to a queued `202 Accepted`
+contract (ADR-0014) is additive for clients that already read it. Ingestion
+failures are HTTP failures: `400` unparseable, `413` too large, `502` embedder
+failure, `503` at capacity, `504` past the ingestion timeout.
 
 ### Point endpoints replace the older `/chunks` sketches
 
@@ -228,7 +232,7 @@ Important status codes:
 
 | Status | Use |
 |---|---|
-| `202 Accepted` | Ingestion accepted as a job. |
+| `201 Created` | Ingestion completed inline (ADR-0017). |
 | `400 Bad Request` | Invalid domain/filter combinations or unsupported file type. |
 | `401 Unauthorized` | Missing/invalid API key. |
 | `403 Forbidden` | Valid key without required scope. |
@@ -252,8 +256,8 @@ and Qdrant operations can be correlated.
   records — while preserving the chunk payload schema underneath.
 - `/threads/{thread_id}/runs` remains compatible with the LangGraph thread/run
   model already chosen in ADR-0007.
-- Job-shaped file ingestion lets the first implementation be simple while
+- Inline file ingestion (ADR-0017) gives callers a terminal result in one
-  keeping room for a durable worker without breaking clients.
+  request, with failures surfaced as ordinary HTTP errors.
 - Router-level dependencies and typed FastAPI dependencies keep auth, tenant
   resolution, sessions, and scopes reusable instead of repeated per endpoint.
 
@@ -266,8 +270,9 @@ and Qdrant operations can be correlated.
   public product API.
 - API-key auth in Postgres adds a database lookup to every request unless
   short-lived caching is introduced. Caching must preserve revocation semantics.
-- A job-shaped ingestion contract needs a job status store even if the initial
+- Inline ingestion ties the upload's duration to proxy/client timeouts, and
-  implementation processes inline.
+  moving back to a queued `202` contract later is a breaking change for clients
+  (see ADR-0017's trigger list). The `ingestion_jobs` store is kept either way.
 - The REST layer now depends on the tenant/API-key, ingestion-job, audit, and
   usage tables defined in [ADR-0009](0009-postgres-sqlalchemy-alembic-schema.md).
 
@@ -288,10 +293,12 @@ and Qdrant operations can be correlated.
 - **Expose one generic `/v1/qdrant/*` proxy**: rejected. It would leak Qdrant's
   full API surface, bypass tenant/scoping rules too easily, and couple clients
   to storage operations the service should hide.
-- **Synchronous file ingestion only**: rejected as the contract. It is simpler
+- **Synchronous file ingestion only**: originally rejected here on the grounds
-  to implement, but embedding and late-interaction vector generation can be
+  that embedding and late-interaction vector generation can exceed HTTP timeouts;
-  slow enough to exceed HTTP timeouts. The job-shaped response gives the
+  **adopted** by ADR-0017 for the first slice. Dense embedding is async network
-  implementation room to evolve.
+  I/O that batches and runs concurrently, and late-interaction vectors are not
+  populated at ingest yet — which is what made the original objection decisive
+  and is now the named trigger for reverting to a job-shaped contract.
 - **Create threads with `POST /v1/threads`**: rejected for now. The main
   backend owns conversation/session records, and LangGraph can create a
   checkpoint sequence on the first run for a `thread_id`. A create endpoint

docs/adr/0009-postgres-sqlalchemy-alembic-schema.md

@@ -47,6 +47,7 @@ Use SQLAlchemy 2.x ORM models with typed mappings:
 class Base(DeclarativeBase):
     pass
 
+
 class Tenant(Base):
     __tablename__ = "tenants"
 
@@ -58,7 +59,11 @@ Conventions:
 
 - Use SQLAlchemy async sessions in FastAPI (`AsyncSession`) and one session per
   request/job unit of work.
-- Use Alembic for all DDL. FastAPI startup opens connections and checks
+- Use Alembic for all DDL. Bootstrap the migration environment once with
+  `uv run alembic init -t async alembic`, then commit the generated
+  `alembic.ini`, `alembic/env.py`, and `alembic/versions/` directory. Retain the
+  async Alembic template and configure it to load the application's database URL
+  and SQLAlchemy model metadata. FastAPI startup opens connections and checks
   readiness; it does not create or alter tables.
 - Prefer UUID primary keys generated by the application. Avoid integer IDs that
   leak tenant size and make distributed workers harder to compose.
@@ -93,13 +98,16 @@ One row per customer/tenant.
 | `slug` | Stable short name, unique, human-readable. |
 | `name` | Display name. |
 | `status` | `active` \| `suspended` \| `deleted`. Suspended tenants authenticate to a clear error but cannot run work. |
-| `settings` | JSONB for tenant-level feature flags/limits (max upload size, enabled file types, allowed domains, etc.). |
+| `settings` | JSONB for tenant-level feature flags/limits (max upload size, enabled file types, etc.). Allowed domains were previously listed here as well; they live in `tenant_domains` instead, per this ADR's own rule that query-critical fields get typed columns — `domain` is validated on every upload and filtered on every query. |
 | `created_at`, `updated_at`, `deleted_at` | Audit/soft-delete timestamps. |
 
 #### `tenant_domains`
 
-Optional but recommended. Validates the `domain` values used throughout Qdrant
+**Required.** (Previously "optional but recommended"; implemented and made
-payloads (`car`, `fire`, etc.) per tenant.
+mandatory alongside `/v1/domains`.) Validates the `domain` values used
+throughout Qdrant payloads (`car`, `fire`, etc.) per tenant. Domain sets are
+per-tenant and differ in size — one tenant may run 14 insurance lines and
+another 6 — so this is data, not an enum.
 
 | Column | Notes |
 |---|---|
@@ -111,7 +119,37 @@ payloads (`car`, `fire`, etc.) per tenant.
 | `metadata` | JSONB for domain-specific ingestion/retrieval settings. |
 
 This prevents arbitrary caller-supplied domains from silently creating new
-partitions in Qdrant.
+partitions in Qdrant. The failure it guards against is quiet: a typo such as
+`fier` for `fire` produces no error anywhere — the file is stored, parsed,
+embedded, and indexed into a partition retrieval never queries, so it is
+invisible rather than failed.
+
+##### Enforcement and management
+
+- **Strict allowlist.** `POST /v1/files` rejects a domain with no `active` row
+  for the tenant (`400`, error code `unknown_domain`). There is no auto-create
+  on first use: that would record the typo rather than prevent it. The check
+  runs inside the upload's first transaction, before any MinIO object, job row,
+  or Qdrant point is written.
+- **Managed over the API, not by an operator.** `/v1/domains` (list, create,
+  update, disable, enable) is the surface the calling backend uses. Domains are
+  created by an explicit, scoped call rather than as a side effect of an upload
+  — that distinction, not who makes the call, is what "strict" means here.
+- **Its own scope.** `domains:read`/`domains:write`, deliberately separate from
+  `files:write`. Folding domain creation into the upload scope would let an
+  upload key create partitions again, which is the exact hole this closes.
+  `api_keys.scopes` is already a free JSONB list, so this needs no schema change.
+- **`tenant_id` stays derived from the API key.** One key per tenant; nothing
+  request-suppliable. A platform key acting across tenants would need a real
+  actor model and is not adopted.
+- **`domain` is immutable; `display_name` is not.** The key is denormalized into
+  every Qdrant point payload and into `source_files`, so renaming it means
+  rewriting all of them — a migration, not a `PATCH`. The update schema
+  therefore has no `domain` field.
+- **Disable is not delete.** `status='disabled'` blocks new uploads and hides
+  the domain from listings, leaving already-indexed points intact and
+  retrievable. Actual removal needs the retention/erasure workflow this ADR and
+  plan 001 defer.
 
 #### `api_keys`
 
@@ -204,9 +242,12 @@ on the `content_hash` policy.
 
 #### `ingestion_jobs`
 
-One attempt to parse/chunk/embed/upsert a source file. This table is required
+One attempt to parse/chunk/embed/upsert a source file. Under
-even if the first implementation processes inline, because ADR-0008's file
+[ADR-0017](0017-synchronous-ingestion-in-the-request-path.md) that attempt runs
-upload contract is job-shaped.
+inline in the upload request, so a row is written `running` before the work and
+updated to a terminal status after it — the table is a durable record of the
+attempt, not a queue. It is what makes failures inspectable, re-ingestion
+idempotent, and a later move back to queued dispatch (ADR-0014) additive.
 
 | Column | Notes |
 |---|---|

docs/adr/0011-python-structured-logging-with-structlog.md

@@ -70,17 +70,30 @@ logger.info(
 Do not build log messages by interpolating operational metadata into prose.
 Prefer fields over long strings because fields are queryable.
 
-### Emit JSON logs by default in production
+### Emit JSON logs by default in production; console and file are independent sinks locally
 
 Production logs are JSON on stdout so process managers, container runtimes, and
-log collectors can ingest them directly. Local development may use a colored
+log collectors can ingest them directly. This does not change.
-console renderer controlled by configuration.
 
-File logging is optional and mainly for local development. If enabled, it must
+Locally, stdout and an optional file are two **independent, simultaneous**
-use explicit rotation settings such as `maxBytes` and `backupCount`. Do not rely
+handlers on the same logger, not a single renderer chosen by a flag — the same
-on a default `RotatingFileHandler` with no rotation parameters. In containerized
+structlog event fans out to both:
-production, stdout/stderr collection is preferred over writing `logs/app.log`
+
-inside the application container.
+- **Console handler**: always on, `structlog.dev.ConsoleRenderer(colors=True)`.
+  This is what a developer reads while the process runs, so it stays
+  human-readable regardless of whether file logging is also enabled.
+- **File handler**: off by default, enabled by setting `LOG_FILE_PATH`. Always
+  renders JSON (`structlog.processors.JSONRenderer()`), independent of the
+  console handler's renderer, so a saved log is machine-parseable even though
+  the terminal output next to it is not. Must use explicit rotation
+  (`RotatingFileHandler` with `maxBytes`/`backupCount` — never an unrotated
+  handler).
+
+In containerized production, stdout/stderr collection remains preferred over
+writing `logs/app.log` inside the application container, so `LOG_FILE_PATH` is
+expected to be unset there; the file handler exists for local development,
+where reading a colored terminal *and* keeping a JSON trail to grep/parse later
+are both useful at once.
 
 ### Configure stdlib and structlog together
 
@@ -188,6 +201,36 @@ Notes:
 - `structlog.contextvars.merge_contextvars` ensures request-bound fields appear
   on both structlog and stdlib logs processed through the formatter.
 
+### Bind process-level environment context once at startup
+
+Deployment identity — which build is running, in which environment, on which
+instance — answers a different question than request correlation: "is this
+issue specific to one deployment / one region / one instance?" rather than "is
+this issue specific to one request?" It does not vary per request, so it must
+not go through `structlog.contextvars`, which `RequestIdMiddleware` clears on
+every request; a value bound there before the first request would be wiped the
+moment that middleware runs.
+
+Instead, add a static structlog **processor** — a plain closure over values read
+once at `configure_logging()` time — so it runs on every event regardless of
+request context:
+
+```python
+def _bind_environment(settings: AppLimitSettings):
+    def processor(logger, method_name, event_dict):
+        event_dict["env"] = settings.env
+        event_dict["service_version"] = settings.service_version
+        return event_dict
+
+    return processor
+```
+
+`service_version` should be the deployed commit SHA or release tag (e.g. from a
+`GIT_SHA`/`APP_VERSION` build-time env var — not computed at runtime by
+shelling out to `git`). This makes "is this only happening on the new
+deployment?" answerable directly from logs, without cross-referencing a
+separate deployment record.
+
 ### Bind request context with contextvars
 
 At FastAPI ingress, clear stale context, bind request identifiers, and return the

docs/adr/0012-application-resource-lifetime-and-dependency-ownership.md

@@ -0,0 +1,270 @@
+# 0012. Application resource lifetime and dependency ownership
+
+## Status
+
+Proposed
+
+## Context
+
+ADR-0008 defines the FastAPI REST boundary and says the app lifespan owns
+long-lived resources. ADR-0009 chooses SQLAlchemy 2.x, Alembic migrations, and
+async sessions for the Postgres layer. We need a project-wide rule for how those
+resources, and future resources with similar lifetimes, are created, shared, and
+closed.
+
+The service will need objects with different lifetimes:
+
+- application-lifetime objects such as SQLAlchemy engines, session factories,
+  Qdrant clients, LangGraph checkpointers/stores, compiled graphs, HTTP clients,
+  observability clients, model clients, Redis clients, or other connection pools;
+- request/job-lifetime objects such as SQLAlchemy sessions, transactions,
+  authentication context, tenant context, request ids, and unit-of-work state;
+- operation-lifetime objects such as temporary files, one-off streams, locks, or
+  short-lived connections checked out for a single block.
+
+Using module-level mutable resource instances for everything would make startup,
+shutdown, testing, concurrency, and transaction boundaries hard to reason about.
+The most dangerous example is a global SQLAlchemy `Session`: sessions track
+identity state and transaction state, and are not safe to share across concurrent
+requests.
+
+At the same time, creating expensive clients or connection pools for every
+request would waste resources and lose pooling benefits. We need explicit
+ownership: the component that creates a resource is responsible for closing it.
+
+## Decision
+
+### Use FastAPI lifespan for application-lifetime resources
+
+Create long-lived shared resources in the FastAPI lifespan function and close
+them after the lifespan `yield`.
+
+Application-lifetime resources include:
+
+- SQLAlchemy async `Engine`;
+- SQLAlchemy `async_sessionmaker` bound to that engine;
+- Qdrant client;
+- LangGraph checkpointer/store and compiled graph;
+- shared `httpx.AsyncClient` instances;
+- Redis or cache clients;
+- Langfuse/observability clients;
+- embedding, reranking, LLM, or other expensive model clients.
+
+The lifespan function is the owner of these resources. It is responsible for
+calling cleanup methods such as `dispose()`, `aclose()`, `close()`, or equivalent
+shutdown hooks.
+
+Indicative shape:
+
+```python
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+import httpx
+from fastapi import FastAPI
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncIterator[None]:
+    app.state.database = Database(settings.database_url)
+    app.state.http_client = httpx.AsyncClient()
+
+    try:
+        yield
+    finally:
+        await app.state.http_client.aclose()
+        await app.state.database.close()
+
+
+app = FastAPI(lifespan=lifespan)
+```
+
+Avoid opening external resource clients as import-time side effects. Module-level
+constants, settings objects, pure functions, type aliases, and stateless helpers
+are acceptable. Mutable network/database clients should be constructed by the
+application owner, not by importing a module.
+
+### Treat SQLAlchemy engine and session as different lifetimes
+
+A SQLAlchemy `Engine` is an application-lifetime pool manager, not a single
+request's transaction. It may keep database connections open in a pool and reuse
+them across requests.
+
+A SQLAlchemy `AsyncSession` is a request/job-lifetime unit of work. It tracks ORM
+identity state and transaction state and may check out a database connection from
+the engine when needed.
+
+Therefore:
+
+- create one engine/session factory per FastAPI process during lifespan;
+- create one `AsyncSession` per request or background-job unit of work;
+- never share one `AsyncSession` globally across concurrent requests;
+- never run DDL such as `create_all()` at FastAPI startup; use Alembic as decided
+  in ADR-0009.
+
+### Use dependencies for request-lifetime resources
+
+Use FastAPI dependencies, usually `yield` dependencies, for objects that should
+exist for one request and then be cleaned up.
+
+Indicative database session dependency:
+
+```python
+from collections.abc import AsyncIterator
+
+from fastapi import Request
+from sqlalchemy.ext.asyncio import AsyncSession
+
+
+async def get_db_session(request: Request) -> AsyncIterator[AsyncSession]:
+    database: Database = request.app.state.database
+
+    async with database.session_factory() as session:
+        try:
+            yield session
+        except Exception:
+            await session.rollback()
+            raise
+```
+
+The dependency owns the session lifetime. Route handlers, services, and
+repositories must not close a session they did not create.
+
+Authentication and tenant context follow the same rule: resolve them through
+request dependencies, then pass the trusted context into code that needs it.
+
+### Keep transaction boundaries explicit
+
+Routes or application-service functions own transaction boundaries. Repository
+and CRUD functions receive a session and perform database work, but they should
+not secretly commit, rollback, or close the session unless their contract
+explicitly says they own a complete unit of work.
+
+Preferred shape:
+
+```python
+async def create_user(session: AsyncSession, data: CreateUserRequest) -> User:
+    user = User(email=data.email)
+    session.add(user)
+    return user
+
+
+@router.post("/users")
+async def create_user_endpoint(
+    body: CreateUserRequest,
+    session: SessionDep,
+) -> UserResponse:
+    user = await create_user(session, body)
+    await session.commit()
+    return UserResponse.model_validate(user)
+```
+
+Avoid hidden global access:
+
+```python
+# Do not do this.
+session = SessionLocal()
+
+
+async def create_user(data: CreateUserRequest) -> User:
+    session.add(User(email=data.email))
+    await session.commit()
+```
+
+This keeps unit-of-work ownership visible, makes multi-step transactions easier,
+and lets tests pass in their own sessions.
+
+### Use context managers for operation-lifetime resources
+
+Objects needed only inside one function or block should use `with` or
+`async with` rather than app state or module globals.
+
+Examples include file handles, temporary streams, one-off SQLAlchemy connections,
+locks, and response streams.
+
+```python
+async with http_client.stream("GET", url) as response:
+    ...
+```
+
+or:
+
+```python
+async with engine.connect() as connection:
+    ...
+```
+
+The context manager owns and releases the resource at block exit.
+
+### Pass dependencies explicitly through service boundaries
+
+Routers adapt HTTP requests into typed application calls. Services and
+repositories should receive the resources they use as explicit parameters:
+
+```python
+async def get_point(
+    session: AsyncSession,
+    qdrant: QdrantClient,
+    auth: AuthContext,
+    point_id: str,
+) -> PointResponse: ...
+```
+
+Do not have lower layers import mutable resource singletons. Explicit parameters
+make ownership clear and allow FastAPI dependency overrides, test fixtures, and
+background workers to supply equivalent resources.
+
+### Account for worker process multiplicity
+
+FastAPI lifespan runs once per worker process. If the service runs with multiple
+Uvicorn/Gunicorn workers, each worker has its own app instance, engine, and
+connection pool. Database pool sizes and external client limits must be chosen
+with worker count in mind.
+
+## Consequences
+
+### Positive
+- Resource ownership and cleanup become explicit: lifespan closes app resources,
+  dependencies close request resources, and context managers close block-scoped
+  resources.
+- SQLAlchemy sessions are not shared across concurrent requests, avoiding unsafe
+  identity-map and transaction-state reuse.
+- Connection pooling is preserved because expensive clients and engines are
+  created once per process instead of once per request.
+- Transaction boundaries are easier to reason about because routes or
+  application services decide when to commit or rollback.
+- Services and repositories are easier to test because sessions, clients, and
+  auth contexts can be injected directly.
+- The same policy can be reused for future Redis, Qdrant, Langfuse, HTTP, model,
+  and worker resources.
+
+### Negative
+- This adds boilerplate: lifespan setup, app state typing/conventions,
+  dependencies, and explicit parameters must be maintained.
+- Developers must distinguish engine/session, application/request, and
+  owner/borrower lifetimes instead of importing a convenient global.
+- App state access needs discipline and typing helpers so resources do not become
+  an unstructured service locator.
+- Multi-worker deployments require explicit pool sizing because each process owns
+  its own pools and clients.
+
+## Alternatives Considered
+
+- **Module-level resource singletons**: rejected as the default for mutable
+  external resources. Import-time construction makes startup order, tests,
+  reloads, and shutdown cleanup harder to control. Module-level constants and
+  stateless helpers remain acceptable.
+- **One global SQLAlchemy session**: rejected. A session is a unit of work with
+  identity and transaction state, not an application-wide pool. Sharing it across
+  requests is unsafe and makes rollback/error handling ambiguous.
+- **Create every client/session inside each CRUD function**: rejected. It hides
+  ownership and transaction boundaries, prevents multi-step units of work, and
+  wastes pooling benefits for expensive clients.
+- **Automatically commit every successful request in the session dependency**:
+  deferred as a default. It reduces route boilerplate, but it can make write
+  boundaries too implicit and does not fit every read/write or multi-transaction
+  flow. Routes or application services should own commits unless a future ADR
+  adopts a unit-of-work abstraction.
+- **Use a generic global service locator**: rejected for now. It can reduce
+  parameter lists, but it obscures dependencies and ownership. FastAPI
+  dependencies plus explicit function parameters are clearer at this stage.

docs/adr/0013-s3-compatible-object-storage-with-minio.md

@@ -0,0 +1,182 @@
+# 0013. S3-compatible object storage with MinIO
+
+## Status
+
+Proposed
+
+## Context
+
+ADR-0008 defines source-file upload endpoints and returns a job-shaped ingestion
+response. ADR-0009 models `source_files` and leaves room for a `storage_uri`
+where original files are retained. The service now needs a clear place to store
+uploaded file bytes and any derived ingestion artifacts without putting large
+binary/text payloads in Postgres or broker messages.
+
+The storage decision has to satisfy several constraints:
+
+- Uploaded files may be larger than is appropriate for Postgres rows.
+- Ingestion workers need to fetch the exact file bytes after the HTTP request has
+  completed.
+- Postgres should remain the source of truth for tenant metadata, file metadata,
+  ingestion job state, audit records, and retention decisions.
+- Broker messages should carry durable identifiers, not raw file content.
+- Local/self-hosted deployment should stay practical and avoid coupling this app
+  to Langfuse's internal object-storage services.
+
+## Decision
+
+Use **MinIO** as the application's S3-compatible object store for uploaded source
+files and derived ingestion blobs.
+
+MinIO stores bytes. Postgres stores metadata and authoritative state.
+
+### Store source-file bytes in MinIO
+
+When `POST /v1/files` receives an upload, the service stores the raw file in an
+application-owned MinIO bucket before or within the same application flow that
+creates the `source_files` and `ingestion_jobs` rows.
+
+`source_files` keeps the durable pointer and metadata, including fields already
+sketched in ADR-0009:
+
+- `tenant_id`;
+- `domain`;
+- `source_filename`;
+- `source_type`;
+- `content_sha256`;
+- `byte_size`;
+- `storage_uri` or equivalent bucket/key fields;
+- lifecycle status and audit timestamps.
+
+Object keys are internal identifiers, not trusted user filenames. Prefer a shape
+like:
+
+```text
+tenants/{tenant_id}/source-files/{file_id}/original
+```
+
+or, when content-addressing is useful:
+
+```text
+tenants/{tenant_id}/source-files/{file_id}/{content_sha256}
+```
+
+The original filename is stored as metadata in Postgres, not used as the object
+key authority.
+
+### Keep MinIO separate from authoritative state
+
+MinIO is not the source of truth for:
+
+- tenant identity or authorization;
+- file ownership;
+- ingestion job status;
+- ingestion progress events;
+- audit logs;
+- LangGraph checkpoints or Store memory;
+- Qdrant point state;
+- broker delivery state.
+
+Workers always load `source_files` and `ingestion_jobs` from Postgres by id, then
+use the stored object pointer to fetch bytes from MinIO.
+
+### Use MinIO for retained blobs, not queue payloads
+
+Broker messages reference ids such as `file_id` and `ingestion_job_id`; they do
+not contain raw file bytes, extracted text, chunks, embeddings, or large parser
+outputs.
+
+MinIO may also store derived blobs when retention policy allows it, for example:
+
+- converted `.doc` -> `.docx` outputs from ADR-0004;
+- extracted text snapshots used for debugging failed ingestion;
+- parser diagnostics or quarantined uploads;
+- exported artifacts generated by maintenance jobs.
+
+Those derived objects still need Postgres metadata if they are user-visible,
+auditable, or subject to retention/erasure policy.
+
+### Use application-owned buckets and credentials
+
+The app must not casually reuse Langfuse's internal MinIO bucket, credentials, or
+lifecycle. Langfuse object storage belongs to the Langfuse stack from ADR-0010;
+this service needs its own application bucket(s) and credentials.
+
+A local deployment may run MinIO in the same Docker environment, but with a
+separate bucket such as:
+
+```text
+chatbot-source-files
+```
+
+Production may use MinIO or another S3-compatible service behind the same object
+storage interface, provided it preserves tenant isolation, encryption, backup,
+and retention requirements.
+
+### Manage the client as an application-lifetime resource
+
+The object-storage client is an application-lifetime external client under
+ADR-0012. It is created by the FastAPI lifespan or worker process startup and
+closed on shutdown when the client library requires explicit cleanup.
+
+Lower-level ingestion services receive an object-storage client or interface
+explicitly. They do not import a module-level mutable MinIO singleton.
+
+For async code, object-storage operations must not block the event loop. Use an
+async-capable S3 client, or isolate synchronous SDK calls in the appropriate
+threadpool boundary.
+
+### Enforce privacy, retention, and integrity rules
+
+- Compute and store `content_sha256` for uploaded bytes.
+- Enforce allowed content types/extensions and maximum file size before retaining
+  files, as required by ADR-0008.
+- Prefer private buckets; generate short-lived presigned URLs only for explicit
+  internal workflows that need them.
+- Apply server-side encryption and backups according to deployment policy.
+- Deletion and tenant-erasure workflows must delete both Postgres metadata and
+  MinIO objects, while preserving audit records according to tenant/legal
+  retention requirements.
+
+## Consequences
+
+### Positive
+- Large uploaded files and derived blobs stay out of Postgres rows and broker
+  payloads.
+- Ingestion can run in separate workers after the HTTP request finishes because
+  workers can fetch the retained object by durable id.
+- The service keeps S3-compatible portability while using MinIO for local and
+  self-hosted deployments.
+- `source_files.storage_uri` from ADR-0009 now has a concrete storage backend.
+- Separating app MinIO from Langfuse MinIO avoids accidental coupling to
+  Langfuse's internal lifecycle and credentials.
+
+### Negative
+- Adds another external dependency to configure, secure, back up, and monitor.
+- Retention and tenant erasure now have to coordinate Postgres rows, Qdrant
+  points, and MinIO objects.
+- Upload flows need cleanup handling for partial failures, such as object upload
+  succeeding but the Postgres transaction failing.
+- Async FastAPI handlers need care if the chosen S3/MinIO client is synchronous.
+- Object keys and bucket policies become part of the security boundary; mistakes
+  can expose cross-tenant files.
+
+## Alternatives Considered
+
+- **Store uploaded files directly in Postgres**: rejected. It simplifies
+  transactional consistency, but bloats the relational database, complicates
+  backups, and mixes large binary/blob storage with metadata and audit queries.
+- **Store only extracted chunks in Qdrant and discard originals immediately**:
+  rejected as the default. It makes failed-ingestion debugging, reprocessing,
+  parser upgrades, and retention review harder. Tenants may still configure
+  shorter retention later.
+- **Use the broker message as file transport**: rejected. Brokers should carry
+  small work identifiers and metadata; raw files, extracted text, chunks, and
+  embeddings belong in object storage or the appropriate durable store.
+- **Reuse Langfuse's internal MinIO service/bucket**: rejected. Langfuse object
+  storage is part of the observability stack, not this application's file store.
+  Sharing it would couple credentials, retention, backups, and operational
+  lifecycle across unrelated systems.
+- **Bind directly to a cloud-only S3 provider**: rejected for now. The project is
+  self-hosted/local-development oriented, and MinIO gives an S3-compatible API
+  while keeping deployment portable.

docs/adr/0014-durable-job-dispatch-with-rabbitmq.md

@@ -0,0 +1,339 @@
+# 0014. Durable job dispatch with RabbitMQ
+
+## Status
+
+Superseded by
+[0017](0017-synchronous-ingestion-in-the-request-path.md)
+
+ADR-0017 defers this decision rather than rejecting it: ingestion currently runs
+inline in the `POST /v1/files` request, with no broker, no outbox, and no queue.
+This ADR remains the intended design for when ingestion becomes slow enough to
+need one — ADR-0017 lists the triggers, and names an in-process Postgres-claimed
+runner as the likely intermediate step before adopting a broker.
+
+## Context
+
+ADR-0008 makes file ingestion job-shaped: `POST /v1/files` returns `202 Accepted`
+with `file_id`, `ingestion_job_id`, and a queued/running status. ADR-0009 makes
+Postgres the durable source of truth for `source_files`, `ingestion_jobs`, and
+`ingestion_job_events`. ADR-0013 stores uploaded file bytes in MinIO so ingestion
+workers can fetch them after the request completes.
+
+What is still missing is the delivery mechanism that tells workers which durable
+jobs are ready to process.
+
+The broker decision has to preserve existing boundaries:
+
+- Postgres remains authoritative for job status, progress, audit, tenancy, and
+  retention-sensitive state.
+- MinIO stores file bytes and derived blobs, not queue state.
+- Broker messages carry ids and correlation metadata, not raw files or chunks.
+- Chat runs are not queued by this service: ADR-0007/0008 allow one in-flight run
+  per `thread_id` and return `409 Conflict` for concurrent runs.
+- Application-lifetime clients are created and closed by lifespan/startup owners
+  under ADR-0012, not import-time globals.
+
+## Decision
+
+Use **RabbitMQ** (via the **aio-pika** async SDK) as the durable pub/sub and
+job-dispatch broker for asynchronous ingestion and maintenance work.
+
+RabbitMQ's work-queue model gives ingestion dispatch what it needs directly:
+durable exchanges/queues, manual acknowledgement, redelivery on nack/crash, and
+per-queue dead-lettering for poison messages, all while Postgres remains the
+source of truth for application state.
+
+### Start with ingestion jobs
+
+The first broker-backed workflow is source-file ingestion:
+
+```text
+POST /v1/files
+  -> authenticate and resolve tenant
+  -> store source bytes in MinIO (ADR-0013)
+  -> create/update source_files row
+  -> create ingestion_jobs row with status='queued'
+  -> create an unpublished outbox_events row
+  -> commit one transaction
+  -> return 202 Accepted
+
+outbox publisher
+  -> publish the RabbitMQ message containing durable ids
+  -> mark the outbox event published
+```
+
+A dedicated ingestion worker process consumes the message:
+
+```text
+RabbitMQ message
+  -> load ingestion_jobs/source_files from Postgres
+  -> bind logging context from durable ids
+  -> fetch source object from MinIO
+  -> parse, chunk, embed, and mutate Qdrant points
+  -> append ingestion_job_events
+  -> mark ingestion_jobs succeeded/failed/cancelled
+  -> ack or nack/reject the message
+```
+
+### Worker-owned ingestion chunk CRUD
+
+For file ingestion and re-ingestion, generated chunk/point CRUD is performed by
+the ingestion worker, not by the FastAPI publisher/request path.
+
+The HTTP request path only owns the lightweight durable handoff:
+
+- authenticate the caller and derive tenant context;
+- store uploaded bytes in MinIO;
+- create or update `source_files`;
+- create `ingestion_jobs(status='queued')`;
+- create the corresponding unpublished `outbox_events` row;
+- commit the Postgres transaction and return `202 Accepted`.
+
+The separate outbox-publisher process owns publication of the RabbitMQ message
+containing durable ids and records its outcome on the outbox event.
+
+The ingestion worker owns the expensive and retryable side effects:
+
+- parsing source files;
+- chunking;
+- embedding and reranking-vector generation when applicable;
+- creating, updating, reordering, and soft-deleting generated Qdrant points;
+- appending `ingestion_job_events`;
+- updating ingestion job status, counters, and failure summaries.
+
+Shared chunk/point mutation logic should live in a service/repository layer that
+can be reused by both direct API routes and ingestion workers. Direct
+`/v1/points` CRUD remains a synchronous API responsibility unless the operation
+is explicitly bulk or job-shaped. Ingestion-generated mutations are asynchronous
+worker responsibility because they must be safe to retry and resume by
+`ingestion_job_id`.
+
+### Keep messages small and versioned
+
+Broker payloads carry identifiers and correlation metadata only. A typical
+message is:
+
+```json
+{
+  "type": "ingestion.job.created",
+  "version": 1,
+  "tenant_id": "...",
+  "file_id": "...",
+  "ingestion_job_id": "...",
+  "request_id": "...",
+  "api_key_id": "..."
+}
+```
+
+Do not put raw file bytes, extracted text, chunks, embeddings, raw prompts, model
+outputs, or secrets in RabbitMQ messages.
+
+Workers must treat message metadata as routing/correlation input, not as the only
+authority. Before doing tenant-scoped work, workers reload the durable job and
+file rows from Postgres and verify the ids are consistent.
+
+### Use explicit routing keys, a topic exchange, and durable queues
+
+Use stable, dot-separated routing keys. Initial routing keys:
+
+```text
+ingestion.job.created
+ingestion.job.retry_requested
+maintenance.retention.requested
+maintenance.erasure.requested
+```
+
+Create an application-owned durable topic exchange for dispatching work
+messages, and durable (quorum-type) queues bound to it by pattern, for example:
+
+```text
+exchange: chatbot.jobs (topic, durable)
+
+queue: ingestion.jobs   <- binding pattern ingestion.job.*
+queue: maintenance.jobs <- binding pattern maintenance.*
+```
+
+Multiple worker processes share a queue as competing consumers, so work is
+distributed without each process independently receiving every message. Set
+consumer prefetch (QoS) explicitly rather than relying on the client default, so
+one slow consumer cannot starve the others or accumulate unbounded unacked
+messages.
+
+Ack messages only after the worker has persisted the resulting job state and
+progress events to Postgres. Redelivery must be safe: ingestion processors should
+be idempotent by `ingestion_job_id` and Qdrant point ids/upsert semantics.
+
+### Run subscribers as worker processes, not FastAPI side effects
+
+The outbox-publisher process owns the RabbitMQ publisher connection/channel and
+publishes committed outbox events. FastAPI routes create durable outbox intent
+only; they do not publish ingestion notifications directly.
+
+Subscribers should run in separate worker process entrypoints. Do not start a
+consumer inside every FastAPI web worker by default: with multiple
+Uvicorn/Gunicorn workers, each process has its own lifespan and could create
+surprising duplicate consumers or resource pressure.
+
+Worker processes create their own application-lifetime resources at startup:
+
+- SQLAlchemy engine/sessionmaker;
+- MinIO/S3 object-storage client;
+- Qdrant client;
+- RabbitMQ connection/channel (aio-pika, via `aio_pika.connect_robust` for
+  automatic reconnection);
+- observability/logging clients;
+- ingestion model clients.
+
+Each message gets its own request/job-lifetime SQLAlchemy `AsyncSession` and
+explicit transaction boundaries, following ADR-0012.
+
+### Use a transactional outbox for job dispatch
+
+Use a transactional outbox to atomically record the ingestion job and the intent
+to dispatch it. The HTTP publisher does not publish directly to RabbitMQ after
+committing the ingestion-job transaction.
+
+Instead, the same Postgres transaction creates or updates the durable job state
+and inserts an unpublished outbox event:
+
+```text
+same DB transaction:
+  insert/update source_files
+  insert ingestion_jobs(status='queued')
+  insert outbox_events(type='ingestion.job.created', payload, published_at=NULL)
+  commit
+```
+
+A separate outbox publisher process reads unpublished events, publishes the
+versioned message to RabbitMQ, and records the publication outcome:
+
+```text
+outbox publisher:
+  claim unpublished outbox event
+  publish to RabbitMQ (persistent message, publisher confirms enabled)
+  mark outbox event published once the broker confirms receipt
+```
+
+This removes the failure window where an `ingestion_jobs` row commits but the API
+process crashes before recording that it must be dispatched. If the transaction
+commits, the job and its durable dispatch intent both exist; the outbox publisher
+can resume publication after a restart.
+
+An outbox event must have a stable UUID event id. Include that id in the
+RabbitMQ message so consumers and operators can correlate it back to the outbox
+record. RabbitMQ has no broker-native publish-deduplication mechanism comparable
+to some other brokers: publisher confirms guarantee the broker accepted the
+message, but the publisher can still crash after the broker confirms and before
+`published_at` is recorded, causing a duplicate publish on retry. There is no
+broker-side safety net for that case here — **workers must be idempotent by
+`ingestion_job_id`**, and Qdrant mutations must use deterministic point
+ids/upsert semantics, as the sole guarantee against duplicate processing.
+
+Outbox events are delivery records, not a replacement for `ingestion_jobs` or
+`ingestion_job_events`. Postgres continues to own application job state and
+progress. A stale-job scanner remains a repair/operational check for jobs or
+outbox events that have not advanced as expected, not the primary dispatch path.
+
+Add the `outbox_events` schema through an Alembic migration. Do not create or
+alter the table at FastAPI startup, as required by ADR-0009.
+
+### Configure retries, dead letters, and observability
+
+Declare queues with an explicit dead-letter exchange (`x-dead-letter-exchange`)
+pointing at a durable `chatbot.jobs.dlx` fanout exchange bound to a durable
+dead-letter queue. A message lands there when a worker rejects/nacks it with
+`requeue=False`. Workers should nack with `requeue=False` after a bounded local
+retry count for a given delivery, rather than looping redelivery indefinitely.
+Poison messages or repeatedly failing jobs should end as durable Postgres
+failures with an `ingestion_job_events` error entry, and the dead-letter queue
+gives operators a place to inspect the raw message for messages that failed
+before any Postgres state could be written.
+
+At worker ingress, bind structured logging context from durable identifiers:
+
+- `tenant_id`;
+- `request_id`;
+- `api_key_id`;
+- `file_id`;
+- `ingestion_job_id`;
+- RabbitMQ exchange/routing key/delivery tag when available.
+
+Use stable event names such as:
+
+- `ingestion.job.dispatched`;
+- `ingestion.job.received`;
+- `ingestion.job.completed`;
+- `ingestion.job.failed`;
+- `broker.publish.failed`;
+- `broker.message.redelivered`.
+
+### Do not queue chat runs through RabbitMQ yet
+
+ADR-0007 and ADR-0008 intentionally reject queueing concurrent chat runs per
+thread: one run may be in flight, and concurrent runs return `409 Conflict`.
+This ADR does not change that decision.
+
+A future ADR may introduce broker-backed work for chat-adjacent tasks such as
+summarization, retention, erasure, or offline evaluation, but synchronous chat run
+execution remains owned by the FastAPI/LangGraph boundary unless superseded.
+
+## Consequences
+
+### Positive
+- File ingestion can move out of HTTP request handling into scalable worker
+  processes without changing the REST contract from ADR-0008.
+- RabbitMQ gives durable delivery, manual ack/redelivery, per-queue
+  dead-lettering, and competing-consumer work distribution without adopting
+  Kafka-level operational complexity.
+- Postgres remains the auditable source of truth for jobs, progress, tenant
+  ownership, failure state, and the durable intent to dispatch each job.
+- The transactional outbox prevents a committed job from losing its initial
+  dispatch intent when the API process fails before a direct broker publish.
+- MinIO, Postgres, Qdrant, and the broker each have distinct responsibilities:
+  bytes, metadata/state, vector search, and work delivery.
+- Separate worker processes avoid surprising subscriptions in every FastAPI web
+  worker and make GPU/model-heavy ingestion resources easier to size.
+
+### Negative
+- Adds another external service to run, secure, monitor, and back up where
+  queue/exchange durability is required.
+- Developers must reason about redelivery and idempotency; handlers may run more
+  than once for the same `ingestion_job_id`.
+- Adds an `outbox_events` table, Alembic migration, outbox publisher process, and
+  monitoring for unpublished or stuck events.
+- RabbitMQ has no broker-native publish deduplication. Unlike brokers that offer
+  one, duplicate publication after a publisher crash between broker confirmation
+  and outbox-event marking is not mitigated by the broker at all; idempotent
+  consumers are the only protection.
+- RabbitMQ exchange, queue, binding, and dead-letter-exchange configuration must
+  be managed explicitly per environment.
+- The app now needs web, ingestion-worker, and outbox-publisher deployment/runbook
+  conventions.
+
+## Alternatives Considered
+
+- **NATS JetStream**: rejected. JetStream keeps the broker footprint lighter and
+  offers built-in publish deduplication, but the project wants RabbitMQ's more
+  mature work-queue/routing model (topic exchanges, per-queue DLX, prefetch-based
+  fair dispatch) and the aio-pika SDK for asyncio integration.
+- **Redis Streams**: rejected for this decision. Redis Streams would work for an
+  MVP and has simple local deployment, but RabbitMQ is a cleaner dedicated broker
+  for durable pub/sub and worker coordination without also mixing cache
+  responsibilities into the same service.
+- **Kafka**: rejected. Kafka is an excellent durable event log at high scale, but
+  its operational overhead is unnecessary for this service's initial ingestion
+  and maintenance jobs.
+- **FastAPI `BackgroundTasks` or in-process asyncio tasks**: rejected as the
+  durable design. They are simple, but work can be lost on process restart and
+  they do not coordinate multiple workers cleanly.
+- **Post-commit direct RabbitMQ publish plus a repair scanner**: rejected as the
+  primary dispatch design. It is simpler, but a crash after the job transaction
+  commits and before the publish is recorded can strand a queued job until the
+  scanner finds it. The transactional outbox persists the dispatch intent in the
+  same transaction as the job.
+- **Database polling only**: rejected as the main dispatch mechanism. Polling
+  Postgres for queued jobs is a useful repair path, but relying on polling alone
+  adds latency and unnecessary database load once a broker is available.
+- **Use the broker as the source of truth for job state**: rejected. Broker state
+  is delivery state. Application job state, progress, audit, and tenant ownership
+  remain in Postgres.

docs/adr/0015-modular-monolith-package-architecture.md

@@ -0,0 +1,292 @@
+# 0015. Modular monolith package architecture
+
+## Status
+
+Proposed
+
+> Amended by [ADR-0017](0017-synchronous-ingestion-in-the-request-path.md):
+> there is no broker, outbox publisher, queue, or separate worker process for
+> now. `messaging/`, `workers/`, and `infrastructure/rabbitmq/` are part of the
+> target shape but are **not created yet**; ingestion runs inline in the request,
+> so the FastAPI route is the only entry adapter and it calls
+> `application/ingestion/` directly. Every layering rule below applies unchanged.
+
+## Context
+
+The repository currently contains only a small FastAPI-oriented scaffold under
+`src/`, while the ADRs define several substantial capabilities and external
+boundaries: FastAPI HTTP endpoints, Postgres and Alembic, MinIO object storage,
+RabbitMQ with a transactional outbox, Qdrant point/retrieval operations,
+LangGraph conversational execution, and separate worker processes.
+
+Without an explicit package structure, implementation can drift toward route
+handlers that call SDK clients directly, workers that duplicate HTTP logic, and
+generic catch-all directories such as `utils`, `services`, or `clients`. That
+would make tenant isolation, transactions, retries, resource ownership, and tests
+harder to apply consistently.
+
+ADR-0012 requires application-lifetime resource ownership and explicit dependency
+passing. ADR-0014 requires workers to own ingestion-generated Chunk/Point CRUD,
+while direct point routes remain synchronous API work. The repository structure
+must support both entrypoints reusing the same application behavior without
+coupling worker code to FastAPI routes or LangGraph nodes to SDK details.
+
+## Decision
+
+Use a **modular monolith with explicit infrastructure adapters**. The application
+uses one deployable codebase with separate web, worker, and outbox-publisher
+process entrypoints. Business/application behavior is grouped by capability;
+external systems are isolated behind infrastructure adapters.
+
+Use `src/` as the explicit Python package root.
+
+### Package layout
+
+Create packages as they become necessary, following this structure:
+
+```text
+src/
+├── __init__.py
+├── main.py
+├── config.py
+├── bootstrap/
+│   ├── lifespan.py
+│   └── dependencies.py
+├── api/
+│   ├── dependencies/
+│   ├── routers/
+│   ├── schemas/
+│   └── router.py
+├── application/
+│   ├── files/
+│   ├── ingestion/
+│   ├── points/
+│   ├── retrieval/
+│   ├── threads/
+│   └── ports/
+├── agent/
+│   ├── graph.py
+│   ├── state.py
+│   ├── nodes/
+│   ├── prompts/
+│   ├── tools/
+│   └── persistence.py
+├── infrastructure/
+│   ├── postgres/
+│   │   ├── models/
+│   │   ├── repositories/
+│   │   ├── database.py
+│   │   └── outbox.py         # ADR-0017: not yet; no outbox today
+│   ├── qdrant/
+│   ├── minio/
+│   ├── embedding/            # async embedding adapters (batched, bounded)
+│   ├── rabbitmq/             # ADR-0017: not yet; ADR-0014 target only
+│   ├── langgraph/
+│   └── observability/
+├── messaging/                # ADR-0017: not yet; ADR-0014 target only
+│   ├── events.py
+│   ├── subjects.py
+│   └── outbox_publisher.py
+└── workers/                  # ADR-0017: not yet; ADR-0014 target only
+    ├── ingestion.py
+    └── maintenance.py
+```
+
+Keep Alembic configuration and migration revisions at the repository root:
+
+```text
+alembic.ini
+alembic/
+├── env.py
+└── versions/
+```
+
+Keep tests outside the application package and organize them by testing boundary:
+
+```text
+tests/
+├── unit/
+│   ├── application/
+│   └── agent/
+├── integration/
+│   ├── postgres/
+│   ├── minio/
+│   └── qdrant/
+└── e2e/
+```
+
+### Dependency direction
+
+Entry adapters call application services; application services depend on typed
+ports/contracts; infrastructure packages implement those ports.
+
+```text
+FastAPI routes / RabbitMQ workers / LangGraph nodes
+  -> application services
+  -> application ports
+  -> infrastructure adapters
+```
+
+The reverse direction is prohibited:
+
+- infrastructure adapters do not import FastAPI routers, workers, or graph nodes;
+- workers do not call FastAPI route functions;
+- LangGraph nodes do not call API route functions or construct mutable SDK clients;
+- application services do not import concrete MinIO, RabbitMQ, Qdrant, or
+  SQLAlchemy client construction code;
+- routes do not call raw Qdrant, MinIO, or RabbitMQ SDK methods directly.
+
+Use ports selectively for external side effects and persistence boundaries; do not
+add interfaces around pure local functions merely to satisfy a pattern.
+
+### API package
+
+`api/` is the HTTP adapter only:
+
+- `routers/` map HTTP operations to application-service calls;
+- `dependencies/` resolve request-lifetime objects such as `AuthContext` and
+  `AsyncSession`;
+- `schemas/` contains public Pydantic request/response/error models;
+- `router.py` composes versioned route groups.
+
+API schemas are separate from SQLAlchemy ORM models and RabbitMQ message schemas.
+Routes perform HTTP validation and response mapping, but not parsing, embedding,
+Qdrant mutations, or transaction-independent business workflows.
+
+### Application package
+
+`application/` contains reusable use-case behavior. It has no FastAPI request
+objects, RabbitMQ consumer loops, or SDK client construction.
+
+- `files/` creates source-file records, validates lifecycle actions, and returns
+  file/job status.
+- `ingestion/` performs parse/chunk/embed/index orchestration after the worker
+  receives a job.
+- `points/` applies tenant-aware direct Point CRUD rules and shared generated-point
+  mutation behavior.
+- `retrieval/` owns retrieval use cases used by the graph; it does not expose raw
+  Qdrant SDK details.
+- `threads/` coordinates run-level application behavior without taking ownership
+  of conversation/session records reserved for the main backend and LangGraph.
+- `ports/` defines narrow contracts for external side effects, including object
+  storage, message publishing, point storage, and repositories where useful.
+
+Both FastAPI routes and worker consumers call these services. This prevents a
+second, inconsistent ingestion implementation from growing inside `workers/`.
+
+### Agent package
+
+All LangGraph-specific application graph code lives in `agent/`:
+
+- `graph.py` builds and compiles the graph from explicitly passed dependencies;
+- `state.py` defines graph state and graph-facing result types;
+- `nodes/` contains focused graph-node behavior such as triage, retrieval,
+  generation, verification, and memory extraction;
+- `prompts/` holds prompt identifiers/templates or prompt access helpers;
+- `tools/` contains graph tool definitions;
+- `persistence.py` contains graph-facing persistence configuration/types.
+
+Concrete `AsyncPostgresSaver` and `AsyncPostgresStore` setup belongs in
+`infrastructure/langgraph/`, then is passed into the graph factory during
+bootstrap. Graph nodes call application services, particularly
+`application/retrieval/`, instead of embedding Qdrant query logic.
+
+### Infrastructure package
+
+`infrastructure/` contains concrete integrations and resource setup.
+
+- `postgres/` owns SQLAlchemy engine/sessionmaker setup, ORM models, repository
+  implementations, and transactional outbox persistence.
+- `qdrant/` owns Qdrant client lifecycle, collection/bootstrap helpers, low-level
+  point operations, and hybrid retrieval adapter mechanics.
+- `minio/` implements object-storage operations against MinIO/S3-compatible APIs.
+- `embedding/` implements the async dense/sparse embedding adapters, including
+  provider batching and the concurrency semaphore from ADR-0017.
+- `rabbitmq/` owns the RabbitMQ connection/channel lifecycle plus low-level
+  publish and consumer adapters (aio-pika). Not created while ADR-0017 stands.
+- `langgraph/` configures the concrete Postgres-backed LangGraph persistence
+  adapters.
+- `observability/` configures structlog and Langfuse integrations.
+
+Infrastructure code receives configuration and is created by a process owner; it
+must not create mutable external clients at import time.
+
+### Messaging and workers
+
+Under ADR-0017 neither package exists yet: ingestion runs inline in the request,
+so the FastAPI route plays the entry-adapter role described here and obeys the
+same rule — it binds logging context and invokes an application service, and
+holds no parsing/chunking/Qdrant business logic itself. The rest of this section
+describes the shape both packages take when ADR-0014 is adopted.
+
+`messaging/` contains versioned event schemas, stable routing-key names, and the
+outbox-publisher orchestration. The outbox publisher coordinates Postgres outbox
+records with the RabbitMQ adapter; it does not become a second source of job
+state.
+
+`workers/` contains thin process entrypoints and consumer loops. A worker creates
+application-lifetime dependencies, consumes a durable RabbitMQ message, binds
+job logging context, and invokes the corresponding application service. It does
+not hold parsing/chunking/Qdrant business logic itself.
+
+For example:
+
+```text
+workers/ingestion.py
+  -> application/ingestion/processor.py
+  -> application/points/service.py
+  -> infrastructure/qdrant/points.py
+```
+
+### Bootstrap and resource ownership
+
+`bootstrap/` composes configuration and concrete infrastructure adapters for each
+process entrypoint. FastAPI lifespan owns web-process resources; worker and outbox
+publisher startup own their corresponding resources. This implements ADR-0012
+without turning `app.state` or module globals into an untyped service locator.
+
+## Consequences
+
+### Positive
+
+- The repository has clear homes for LangGraph, Qdrant, MinIO, RabbitMQ, Postgres,
+  API, worker, and outbox-publisher code before implementation grows.
+- HTTP routes, worker consumers, and LangGraph nodes reuse application services
+  while remaining separate transport/execution adapters.
+- SDK-specific details are isolated, making integration tests and test doubles
+  practical without hiding all code behind unnecessary abstractions.
+- The layout directly supports ADR-0012's explicit resource ownership and
+  ADR-0014's worker-owned ingestion Chunk/Point CRUD.
+- A single codebase remains simple to deploy while allowing web, worker, and
+  outbox-publisher processes to scale independently.
+
+### Negative
+
+- The initial directory structure is more elaborate than a route-plus-models
+  FastAPI starter application.
+- Developers must maintain dependency direction rather than importing a concrete
+  client wherever it is convenient.
+- Some capabilities span several packages by design, for example an upload route,
+  application service, MinIO adapter, outbox repository, and publisher process.
+- Ports/contracts should remain narrow; excessive abstraction would add ceremony
+  without improving testability or substitutability.
+
+## Alternatives Considered
+
+- **Technology-first packages only**: rejected. Directories such as `db`,
+  `qdrant`, `rabbitmq`, and `langgraph` are immediately discoverable, but feature
+  workflows become scattered across every integration package and encourage
+  transport adapters to own business behavior.
+- **Feature-first packages only**: rejected. Keeping all file, point, and thread
+  code together is attractive, but it obscures ownership of shared external
+  clients and risks duplicating infrastructure integration logic across features.
+- **Full clean architecture with interfaces for every class/function**: rejected.
+  The application needs clear external boundaries, not abstraction around pure
+  helper functions. Ports are reserved for persistence and external side effects.
+- **Microservices for ingestion, retrieval, and chat from the start**: rejected.
+  The project needs independent web/worker processes, but a single modular
+  codebase avoids premature network boundaries, deployment complexity, and
+  distributed transaction concerns.
+- **Put LangGraph under `api/` or workers under `ingestion/` only**: rejected.
+  LangGraph and RabbitMQ workers are independent execution adapters; putting one
+  under another would invert dependencies and make reuse/testing harder.

docs/adr/0016-testing-strategy-and-quality-gates.md

@@ -0,0 +1,228 @@
+# 0016. Testing strategy and quality gates
+
+## Status
+
+Proposed
+
+> Amended by [ADR-0017](0017-synchronous-ingestion-in-the-request-path.md):
+> there is no broker, outbox, or queue, so the `rabbitmq` marker and
+> `tests/integration/rabbitmq/` are not carried until ADR-0014 is adopted.
+> Ingestion is exercised through the upload request itself, which now returns a
+> terminal result. Every reliability invariant below still applies — retrying an
+> upload stands in for redelivery.
+
+## Context
+
+The project has ADRs for tenant-scoped ingestion, explicit resource ownership,
+MinIO object storage, inline request-path ingestion,
+Qdrant indexing, and a modular monolith. It has no test runner, test fixtures,
+or executable test suite yet.
+
+The first CSV ingestion slice has correctness properties that cannot be left to
+manual testing: Alembic is the only schema-management path; tenant identity is
+trusted server-side context; the source file and its job row commit
+atomically; job execution is safe under at-least-once semantics; and generated Qdrant points are
+idempotent and tenant-filtered. ADR-0015 already reserves a test layout by
+boundary, while ADR-0012 requires explicit dependencies and resource lifetimes
+that should make tests practical without import-time client patching.
+
+Tests need to give fast feedback during implementation without replacing
+integration coverage with mocks or making routine development depend on Docker,
+provider credentials, live models, or Langfuse availability.
+
+ADR-0017 controls ingestion: `POST /v1/files` parses, chunks, embeds, and
+indexes inline, then returns a terminal `201`. `ingestion_jobs` records the
+attempt. There is no outbox, queue, or publisher to test — but the request's
+bounds (size, timeout, capacity) and its two-transaction shape are testable
+contracts.
+
+## Decision
+
+### Use pytest with explicit async support
+
+Use pytest as the test runner. Add `pytest`, `pytest-asyncio`, `httpx`,
+`asgi-lifespan`, `testcontainers`, `pytest-timeout`, and `pytest-cov` as
+development dependencies.
+
+Configure `pytest-asyncio` in strict mode. Async tests and fixtures must be
+explicit. FastAPI tests use `httpx.AsyncClient`, `ASGITransport`, and
+`LifespanManager` so they exercise application startup and shutdown according to
+ADR-0012. `FastAPI.TestClient` is not the default project test client.
+
+Register these markers:
+
+- one primary boundary marker per test: `unit`, `integration`, or `e2e`;
+- `postgres`, `minio`, or `qdrant` for the real service used by an integration
+  test (a `rabbitmq` marker returns with ADR-0014);
+- `slow` only where a test materially exceeds the normal integration feedback
+  target;
+- `live_provider` for an opt-in, credential-gated external-provider smoke test.
+
+Name tests `test_<unit>_<scenario>_<outcome>` and use Arrange–Act–Assert.
+
+### Test by architectural boundary
+
+Use the test layout reserved by ADR-0015, with shared support for fixtures and
+assertions:
+
+```text
+tests/
+├── conftest.py
+├── fakes.py
+├── support/
+│   ├── factories.py
+│   └── assertions.py
+├── unit/
+│   ├── application/
+│   └── agent/
+├── integration/
+│   ├── postgres/
+│   ├── minio/
+│   └── qdrant/
+└── e2e/
+```
+
+- **Unit tests** are the default development feedback loop. They cover pure
+  application policy, validation, lifecycle transitions, tenant propagation,
+  deterministic IDs, CSV chunking, event construction, and LangGraph
+  control-flow policy.
+- **Integration tests** validate a production infrastructure adapter against its
+  real backing service and its deployment-relevant behavior.
+- **End-to-end tests** prove a small vertical-slice acceptance contract through
+  the real application composition. They do not replace faster unit or adapter
+  tests.
+
+Hand-written fakes and spies implement narrow application-owned ports, not
+MinIO, Qdrant, or model SDK-shaped interfaces. Scripted model, embedder,
+retrieval, clock, and UUID fakes make normal test runs deterministic.
+
+### Apply pragmatic TDD
+
+For application behavior, HTTP contracts, database migrations, reliability
+rules, and defects, first write a focused failing test that describes the
+observable requirement. Make the smallest change that passes it, then refactor
+while the relevant suite is green.
+
+TDD applies to behavior and regressions, not as an artificial ritual for pure
+refactors or configuration-only changes with no observable behavior change.
+Those changes must preserve and extend existing relevant coverage as needed.
+
+Before introducing a concrete adapter, write tests for the consuming
+application port. Before each Alembic migration, write the empty-database
+migration test or extend the existing migration test. Add the corresponding
+real-adapter integration test before declaring that boundary complete.
+
+### Use disposable real infrastructure in integration tests
+
+Use Testcontainers as the standard automated integration-test resource mechanism
+for Postgres, MinIO, and Qdrant.
+
+- Tests never connect to a developer's local services or Langfuse-owned storage
+  and credentials.
+- Start containers at suite or session scope, then isolate data per test with
+  unique data, object-key prefixes, exchange/queue/binding names, and collection
+  names.
+- Disable parallel integration execution until fixture isolation and cleanup are
+  proven worker-safe.
+- Fixtures expose typed settings or connection values. Tests create production
+  adapters through their normal constructors.
+
+Docker Compose remains the mechanism for manual local validation and a later,
+serialized operational smoke test of the running web process, which performs
+ingestion inline under ADR-0017. It is not the default pytest fixture mechanism.
+
+### Treat invariants as reusable contracts
+
+Test the following requirements at the applicable application, adapter, and E2E
+boundaries:
+
+- Tenant identity comes from server-side authenticated context. Request payloads,
+  query parameters, object metadata, and job payloads cannot override it.
+- Cross-tenant access does not disclose tenant-owned data. Public routes normally
+  return `404` for inaccessible resources.
+- Alembic creates the schema from an empty database. Tests never use
+  `Base.metadata.create_all()`, and FastAPI startup performs readiness checks only,
+  never DDL.
+- The first upload transaction records `source_files` and a `running`
+  `ingestion_jobs` row atomically, and commits before any parse/embed work; no
+  session or transaction is held open across that work.
+- Every terminating path — success, parse failure, embedder failure, timeout —
+  writes a terminal job status and its `ingestion_job_events` row. A job is
+  never left in `running` by a handled failure.
+- Each bound maps to its status code: oversized upload `413`, capacity `503`,
+  timeout `504`, embedder failure `502`.
+- Retrying an upload does not regress terminal jobs, inflate counters, or create
+  duplicate logical chunks; identical content is recognized rather than
+  re-ingested.
+- Embedding is batched and concurrency-bounded rather than serial per chunk, and
+  blocking work is executed off the event loop under an explicit limiter.
+- MinIO keys are server-derived internal paths. Qdrant reads and mutations use a
+  server-derived tenant filter, deterministic point IDs, and upsert semantics.
+
+### Keep correctness tests separate from model-quality evaluation
+
+Pytest verifies deterministic application behavior: graph policy, schemas,
+redaction, correlation metadata, retry budgets, and graceful observability
+failure handling. Normal pytest runs never call a paid or live model provider.
+
+Langfuse datasets and experiments evaluate prompts, model behavior, retrieval,
+citations, and response quality. They are promotion evidence, not a replacement
+for application correctness tests. Live-provider smoke tests, if introduced,
+are opt-in, credential-gated, rate-limited, and excluded from ordinary local and
+pull-request runs.
+
+### Establish phased quality gates
+
+Initially require Ruff format checking, Ruff linting, Ty type checking, and unit
+tests. Require Docker-capable integration tests as their adapters are
+implemented. Run the separate-process Compose E2E smoke as a serialized
+pre-release or scheduled gate until it is reliable enough for every pull request.
+
+Collect coverage reports but do not set a percentage threshold before the first
+vertical slice has meaningful implementation. Later introduce a scoped,
+ratcheting threshold rather than encouraging low-value coverage.
+
+## Consequences
+
+### Positive
+
+- Unit tests provide fast, deterministic TDD feedback for core application
+  behavior.
+- Real-service tests cover the behaviors least safe to simulate: Alembic
+  migrations, object storage, transaction boundaries under real sessions, and
+  Qdrant filtering/upserts.
+- Explicit fakes reinforce the dependency direction and resource ownership rules
+  from ADR-0012 and ADR-0015.
+- The ingestion path has concrete tenant-isolation and reliability contracts,
+  rather than only a happy-path demonstration.
+- Live model quality can improve through Langfuse experiments without making
+  application tests nondeterministic or expensive.
+
+### Negative
+
+- Docker is required for integration and E2E suites.
+- Testcontainers add setup time and require careful fixture cleanup.
+- Maintaining real-service coverage and test data isolation adds engineering
+  effort.
+- E2E tests do not prove semantic quality of LLM responses; that remains an
+  evaluation responsibility.
+
+## Alternatives Considered
+
+- **Mock all external SDKs**: rejected. Mocks cannot prove migrations, real
+  transaction/connection behavior, MinIO semantics, or Qdrant tenant
+  filtering.
+- **Use full-stack Compose tests only**: rejected. They are slow and opaque for
+  the default development loop and make failures difficult to localize.
+- **Run all integration containers on every pytest invocation**: rejected. Test
+  boundaries should be selected deliberately for fast local feedback.
+- **Use live model providers in routine tests**: rejected because they are
+  nondeterministic, costly, slow, credential-dependent, and hard to assert.
+- **Use Langfuse as the primary regression-test runner**: rejected. Langfuse is
+  the quality/evaluation plane; pytest remains the deterministic application test
+  framework.
+- **Create schemas with `Base.metadata.create_all()` in fixtures**: rejected. It
+  bypasses the production Alembic migration path.
+- **Require strict test-first work for every non-behavioral refactor**: rejected.
+  TDD should protect observable behavior and regressions, not add ceremony where
+  no behavior changes.

docs/adr/0017-synchronous-ingestion-in-the-request-path.md

@@ -0,0 +1,269 @@
+# 0017. Synchronous ingestion in the request path
+
+## Status
+
+Proposed
+
+## Context
+
+ADR-0014 chose RabbitMQ plus a transactional outbox for durable job dispatch,
+with three process entrypoints (web, outbox publisher, ingestion worker).
+ADR-0008 made `POST /v1/files` job-shaped — `202 Accepted` with an
+`ingestion_job_id` and a non-terminal status — explicitly so that a durable
+queue could be added later without breaking clients.
+
+That contract was chosen on the assumption that ingestion is slow enough to
+exceed HTTP timeouts. For the first slice, that assumption does not hold, and
+paying for it costs a broker, an outbox table, a publisher process, a worker
+process, and a client that must poll for completion.
+
+What ingestion actually does per file, and how expensive each stage is:
+
+| Stage | Kind of work | Cost |
+|---|---|---|
+| Parse (`python-docx`, `csv`) | Blocking CPU, sync SDK | Modest; seconds at worst for a large document |
+| Chunk (fixed-size, ADR-0004) | Blocking CPU, pure Python | Negligible |
+| `dense_nomic`, `dense_openai` | **Async network I/O** | Dominant by wall-clock, but batchable and concurrent |
+| `sparse` (own BM25 pipeline, ADR-0005) | Blocking CPU | Modest, GIL-bound |
+| Qdrant upsert | Async network I/O | Modest, batchable |
+
+The dense embedding stage is the largest term, but it is I/O, not computation
+this process performs: both embedders take arrays of inputs, and independent
+batches can be in flight at once. A few hundred chunks is a small number of
+batched requests, issued concurrently — not a serial per-chunk round trip.
+
+`late_interaction` (jina-colbert-v2) is the one genuinely heavy ingest-time
+stage: a document-side multivector per chunk from a local GPU model requiring
+`flash_attn`. ADR-0001 defines the vector now to avoid collection recreation but
+does not require it to be populated, and plan 001 puts the reranker and GPU
+deployment explicitly out of scope for the first slice. It is therefore **not
+computed during synchronous ingestion**, and enabling it is a trigger to revisit
+this decision rather than a cost this decision has to carry.
+
+## Decision
+
+**`POST /v1/files` performs ingestion inline and returns a terminal result.** No
+broker, no outbox, no queue, no worker process, and no polling by the client.
+
+```text
+POST /v1/files
+  -> authenticate, resolve tenant, validate the upload
+  -> 201 Created { file_id, ingestion_job_id, status: "succeeded", chunks_indexed }
+```
+
+ADR-0008's job-shaped `202 Accepted` contract is superseded by this ADR;
+ADR-0014 remains `Superseded by 0017` and is the design to adopt if the triggers
+below are hit. `ingestion_jobs` is kept exactly as ADR-0009 defines it — it is
+now a record of an ingestion *attempt* rather than a queue entry, and it is what
+makes failures inspectable and re-ingestion idempotent.
+
+### Request shape
+
+The request runs in three phases, and the phase boundaries matter:
+
+```text
+1. txn A (short):  insert source_files
+                   insert ingestion_jobs(status='running', started_at=now())
+                   commit  -- release the connection
+2. no transaction: store bytes in MinIO
+                   parse + chunk            (threads)
+                   embed dense + sparse     (async, batched, bounded)
+                   upsert Qdrant points     (deterministic ids)
+3. txn B (short):  update ingestion_jobs -> succeeded/failed + counters
+                   append ingestion_job_events
+                   commit
+```
+
+**Never hold a Postgres session or transaction open across phase 2.** A slow
+upload would otherwise pin a pool connection — and an idle-in-transaction row
+lock — for the entire ingestion. Take a session, commit, release, and take a
+fresh one for phase 3. This is ADR-0012's "one session per unit of work" applied
+to a request that contains two units.
+
+Committing the job row *before* the work means a request that dies mid-ingestion
+still leaves durable evidence: a `running` job that never reached a terminal
+state, discoverable by age.
+
+### Embedding is concurrent and batched, not serial
+
+Chunks are embedded through the async embedding ports, batched per provider
+limits, with independent batches in flight concurrently and bounded by a
+semaphore:
+
+```python
+limiter = asyncio.Semaphore(settings.ingestion.embed_concurrency)
+
+async def embed_batch(batch: Sequence[str]) -> list[Vector]:
+    async with limiter:
+        return await embedder.embed(batch)
+
+batches = chunk_into_batches(texts, size=settings.ingestion.embed_batch_size)
+vectors = flatten(await asyncio.gather(*(embed_batch(b) for b in batches)))
+```
+
+Rules:
+
+- **Batch before you parallelize.** Both dense embedders accept arrays; sending
+  one request per chunk wastes far more time than concurrency can recover.
+- **Bound concurrency with a semaphore**, never an unbounded `gather` over every
+  batch. The limit exists for the provider's rate limits and for the self-hosted
+  nomic server's capacity — past saturation, extra concurrency just moves the
+  queue somewhere you cannot see it.
+- Retry `429`/transient failures with backoff *inside* the request's overall
+  timeout budget, not beyond it.
+- The two dense embedders are themselves independent and run concurrently with
+  each other.
+
+### Blocking work runs on threads
+
+Parsing (`python-docx`, `csv`), chunking, hashing, the sparse BM25 pipeline, and
+the synchronous `minio` SDK are blocking. They run via
+`anyio.to_thread.run_sync`, with an explicit `anyio.CapacityLimiter` so ingestion
+threads cannot exhaust the pool Starlette uses for sync route handlers and
+dependencies:
+
+```python
+chunks = await anyio.to_thread.run_sync(parse_and_chunk, raw_bytes, limiter=ingestion_limiter)
+```
+
+Calling any of them directly from an `async def` service is a defect: one large
+`python-docx` parse would block every concurrent request in the process.
+
+### The request is bounded, and says so when it cannot finish
+
+Synchronous ingestion means the client's timeout is now the system's deadline.
+Three bounds, all configured, all enforced server-side:
+
+- `INGESTION_MAX_UPLOAD_SIZE_MB` (and a max chunk count) reject work that is
+  obviously too large *before* any of it starts — a `413`, not a timeout.
+- `INGESTION_TIMEOUT_SECONDS` bounds the whole of phase 2. On expiry the job is
+  marked `failed` with a timeout error code in phase 3, and the response is
+  `504`. A timeout must never leave the job stuck in `running`.
+- `INGESTION_MAX_CONCURRENCY` bounds how many ingestions run in the process at
+  once. Over the limit, the request is rejected with `503` and `Retry-After`
+  rather than queued behind an unbounded wait — a queue that the client is
+  blocked on is the worst of both designs.
+
+Document the deployment consequence: proxy, load balancer, and client read
+timeouts must all exceed `INGESTION_TIMEOUT_SECONDS`, or the client will give up
+on work that is still succeeding.
+
+### Re-running an ingestion stays safe
+
+The idempotency requirements survive, because a client that times out will
+retry, and phase 2 has no transaction protecting it:
+
+- Point ids are deterministic from `file_id` + `chunk_index` (ADR-0001), so a
+  retried upload upserts rather than duplicates.
+- Identical uploads are recognized by `(tenant_id, domain, content_sha256)` and
+  return the existing file/job rather than re-ingesting (plan 001).
+- `tenant_id` comes from `AuthContext`, never from the request body.
+- A terminal job is never transitioned back to `running`.
+- A failed attempt never *removes* content from a working index. The
+  soft-delete sweep that retires a shortened file's leftover points runs only
+  after every upsert in the attempt has succeeded.
+
+  This is deliberately weaker than "replacement happens only after a successful
+  attempt", which an earlier revision of this ADR claimed. That guarantee is not
+  achievable alongside ADR-0001's deterministic point ids: those ids are exactly
+  what makes a retry idempotent, and they also mean a re-ingestion overwrites
+  points **in place**, so a crash partway through leaves a prefix updated and the
+  remainder still on the old content. Buying literal atomicity would mean
+  generation-suffixed ids and an activation flip, which contradicts ADR-0001 and
+  ADR-0002's stable point ids. Staging the new points as `is_active=false` and
+  flipping them on success is strictly worse — the in-place overwrite would
+  deactivate the previously live points, silently emptying a working index if the
+  attempt were interrupted.
+
+  What holds instead: the index is never emptied, never partially deleted, and a
+  retry converges — deterministic ids rewrite every point and the sweep re-runs,
+  reaching the exact correct state.
+
+### Failures are HTTP failures
+
+There is no dead-letter queue and no retry loop. A failed ingestion marks the
+job `failed` with an error code and a terminating `ingestion_job_events` row,
+and returns the corresponding status code (`400` for an unparseable file, `413`
+too large, `503` at capacity, `504` on timeout, `502` for an embedder failure).
+The client decides whether to retry, which is the correct owner of that decision
+when the client is synchronous.
+
+### `GET /v1/files/{file_id}` stays
+
+It reports the stored job status. It is no longer a polling mechanism for the
+upload, but it remains how an operator inspects a past ingestion, and how a
+client that lost its connection mid-upload discovers what happened.
+
+### When to revisit
+
+Return to a queue — ADR-0014's design, or an in-process Postgres-claimed runner
+as an intermediate step — when any of these becomes true:
+
+- `late_interaction` document vectors are populated at ingest (GPU, per-chunk
+  model inference: this alone is likely sufficient);
+- typical ingestion approaches `INGESTION_TIMEOUT_SECONDS`, or `504`/`503` rates
+  stop being negligible;
+- file types arrive that are large or slow enough to be unbounded (bulk XLSX,
+  scanned PDFs with OCR);
+- ingestion load starts degrading chat/retrieval latency in the same process;
+- ingestion needs to be retried automatically rather than by the caller.
+
+Track p95 ingestion duration, `503`/`504` counts, and the count of `running`
+jobs older than the timeout. Those are the trigger metrics.
+
+## Consequences
+
+### Positive
+- The client gets its answer in one call. No polling, no job-status endpoint in
+  the happy path, no "queued" state to explain to a frontend.
+- The first slice needs Postgres, MinIO, and Qdrant only — no broker, no outbox
+  table, no publisher or worker process, no exchange/queue/DLX provisioning.
+- Failures reach the caller directly, with a real status code and message,
+  instead of being discovered later by polling a job row.
+- One process to run, deploy, and reason about; `docker compose up` is the whole
+  application.
+- Batched, bounded-concurrent embedding is a property worth having regardless of
+  where ingestion runs — it carries over unchanged into a queued design.
+
+### Negative
+- The request's duration is now a product constraint. Proxy/client timeouts
+  become deployment configuration that can silently break uploads.
+- Ingestion competes with chat/retrieval for CPU, threads, and connections in
+  the same process, and a burst of uploads degrades API latency for everyone.
+- No automatic retry: a transient embedder failure surfaces as a `502` and
+  depends on the caller to retry.
+- A client disconnect does not cancel the work, and leaves a job that can sit in
+  `running` until observed by age.
+- Capacity rejection (`503`) is a worse experience than queueing for callers who
+  would rather wait — accepted deliberately, because a blocked client waiting on
+  a hidden queue is worse still.
+- Reversing this decision changes the HTTP contract (`201` with a terminal status
+  becomes `202` with a pending one), so clients would have to change. This is the
+  real cost of the decision, and the reason the trigger list above is explicit.
+
+## Alternatives Considered
+
+- **Job-shaped `202 Accepted` with a queue (ADR-0008 as written, ADR-0014)**:
+  rejected for now. It is the right design once ingestion is genuinely slow or
+  needs automatic retry, and the trigger list says when to adopt it. Building it
+  first would add a broker, an outbox, and two processes to make a
+  seconds-long operation asynchronous.
+- **In-process Postgres-claimed job runner with `202`**: rejected as the current
+  step, but it is the natural intermediate — it keeps the single process and the
+  single deployment while decoupling the request from the work. It is the first
+  thing to reach for when the triggers fire, before adopting a broker.
+- **Serial per-chunk embedding**: rejected. It is the version of synchronous
+  ingestion that genuinely would exceed HTTP timeouts, and it is avoidable with
+  batching plus bounded concurrency.
+- **Unbounded `asyncio.gather` over all batches**: rejected. It converts a large
+  upload into a rate-limit burst against the provider and an unbounded memory
+  spike locally; the semaphore is what makes the concurrency safe.
+- **Run the blocking stages on the event loop directly**: rejected. A single
+  large `python-docx` parse would stall every concurrent request in the process.
+- **`BackgroundTasks` after returning `201`**: rejected as the worst of both —
+  the client is told the work succeeded before it has, with no durable record
+  and no retry if the process restarts.
+- **Populate `late_interaction` during synchronous ingestion**: rejected for this
+  slice, consistent with plan 001's scope. Per-chunk GPU model inference is the
+  stage that makes ingestion unbounded; adding it is a trigger to revisit this
+  ADR, not something to absorb into a request.

docs/adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md

@@ -0,0 +1,306 @@
+# 0018. DOCX and spreadsheet parsing with fixed-size chunking
+
+## Status
+
+Proposed
+
+## Context
+
+ADR-0004 specified the full parsing and chunking design: structural extraction
+before chunking (table rows and Q&A pairs kept atomic, prose chunked
+separately), semantic-aware chunking as the default, image captioning via a
+vision API, and legacy `.doc` conversion through headless LibreOffice. Plan
+001's first vertical slice needs a working parser now, and that full design is
+substantially more work than the slice can absorb. This ADR records what v1
+actually ships and why it differs, so the code does not silently contradict an
+Accepted ADR.
+
+Four forces shaped the decision:
+
+**A working extractor already exists.** The `chunking_strategies_evaluation`
+repository — the harness the user built to compare chunking strategies against
+this same Farsi corpus — contains a DOCX extractor that walks the document body
+in reading order and handles the "prose lives inside table cells" pattern
+common in Farsi documents exported from older Word versions. Roughly 200 lines
+of it are production-quality; the rest is evaluation scaffolding (five
+competing strategies, an LLM-as-judge benchmark, a dashboard, an HTML report
+generator). Porting it is cheaper and better-tested against real documents than
+writing a parser from scratch.
+
+**Semantic chunking does not fit the inline request.** ADR-0004 chose
+semantic-aware chunking as the default on the strength of the user's own
+offline accuracy comparison, in which fixed-size was the "viable, simpler
+runner-up". That comparison measured retrieval accuracy, not ingestion cost.
+Semantic boundary detection requires embedding every sentence *before* chunk
+boundaries can be decided — a second network round-trip pass inside the request
+budget ADR-0017 bounds with `INGESTION_TIMEOUT_SECONDS`. ADR-0017's own cost
+table already assumes the cheaper strategy, listing "Chunk (fixed-size,
+ADR-0004) | Blocking CPU, pure Python | Negligible". ADR-0004 and ADR-0017 are
+therefore already in tension, and this ADR resolves it toward ADR-0017 for v1.
+Fixed-size is acceptable specifically because ADR-0001 gives every chunk
+`previous_chunk_id`/`next_chunk_id` pointers: a chunk boundary that cuts a
+thought in half is recoverable by expanding to neighbors at retrieval time
+(ADR-0003).
+
+**Nothing normalizes the text the dense embedders see.** ADR-0005 resolved the
+sparse analyzer as `bm25-fa-norm-stop` — "normalization + stopword removal",
+computed in our own BM25 pipeline outside Qdrant. That covers the sparse vector
+only. Persian text authored on mixed Arabic/Persian keyboards contains both
+`ک` (U+06A9) and `ك` (U+0643), both `ی` (U+06CC) and `ي` (U+064A); these are
+distinct codepoints and therefore distinct tokens to `nomic-embed-text-v2-moe`
+and `text-embedding-3-large` alike, so the same Persian word can embed two
+different ways depending on which key the author pressed. Word documents in
+this corpus reliably contain both forms.
+
+**The corpus is spreadsheets more than it is CSVs.** ADR-0004 inspected the
+real sample and found the tabular files are entirely `.xlsx`; no `.csv` exists
+in practice. Plan 001 and the `source_files.source_type` CHECK constraint both
+name `csv`. The row-to-chunk mapping is identical either way — only the reader
+differs — so v1 reads both rather than forcing a manual export step that would
+silently drop the merged-cell values ADR-0004 warns about.
+
+## Decision
+
+### 1. Formats
+
+v1 ingests `.docx`, `.csv`, and `.xlsx`.
+
+`.doc` is rejected with `415 Unsupported Media Type`. ADR-0004 specified
+conversion via `soffice --headless --convert-to docx`; that is a subprocess
+with a multi-second startup cost running inside the inline request ADR-0017
+defines, and it adds a system binary to the container image. Conversion is
+deferred to an out-of-process HTTP conversion service, tracked in the backlog.
+`source_files.source_type` continues to allow `doc` so the row can be recorded
+once conversion lands.
+
+### 2. Structural units before chunking
+
+Every source file is decomposed into ordered **structural units** before any
+chunking runs, as ADR-0004 requires. There are two kinds:
+
+- `PARAGRAPH` — a run of flowing prose. Consecutive paragraphs accumulate into
+  one unit rather than one unit each, so the splitter sees flowing text instead
+  of a series of one-sentence fragments ("the whole remaining run of
+  paragraphs", per ADR-0004).
+- `TABLE_ROW` — one row of a data table. Atomic; split only when a single row
+  exceeds the model's sequence length.
+
+Walk `doc.element.body` children in document order — not `doc.paragraphs` and
+`doc.tables` separately — so tables interleaved with paragraphs keep their
+position. This is ADR-0004's rule, unchanged.
+
+**No headings are invented.** A real `Heading N` Word style becomes a `#`
+prefix on its paragraph; where a document declares none, none appear. The
+evaluation repository upgrades paragraphs to headings by text pattern (`^بخش`,
+`^\d+[-.]\d+`, leading `*`); those patterns are tuned to a Farsi regulatory
+corpus, not an insurance one, and a wrongly-detected heading silently reshapes
+the document in a way that is hard to notice downstream. They are not adopted.
+
+There is also **no document tree**. An earlier draft built a `Document >
+Section > Article > Paragraph` hierarchy from heading styles. Not one document
+in the sample corpus carries a single `Heading` style, so that tree was flat in
+every real case, and nothing consumed it — chunking works from the unit list,
+and ADR-0001's payload has no tree field. It is not built.
+
+### 3. Data tables against layout tables
+
+A docx table is either data or page furniture, and the two need opposite
+treatment. The classification is **structural, never a reading of content**: a
+data cell is by definition small enough to be a chunk, so a table containing a
+cell that alone exceeds `chunk_size`, or a cell containing nested tables, is a
+layout container. In the sample corpus this separates by two orders of
+magnitude — 24 to 171 tokens for the largest cell of each data table, against
+54,007 tokens for a cell holding an entire sub-document across 738 paragraphs
+and 5 nested tables.
+
+- **Data table** → one `TABLE_ROW` unit per row, rendered by the same code that
+  renders spreadsheet rows.
+- **Layout table** → its cells are prose, recursed into and folded into the
+  surrounding prose block.
+
+### 4. Table rows are labeled only when a header is provable
+
+Applied to docx tables and spreadsheets alike:
+
+- A header is **row 0 or nothing**. Never scan further down for a
+  header-shaped row. Scanning discarded every row above the match and then
+  labeled the rest from a data row, turning a 30-row compensation table into
+  chunks reading `80: 70`.
+- Leading rows that are structurally a merged banner — fewer than two populated
+  cells, or one value repeated across the row — are skipped first. That is a
+  fact about the merge, not a guess about meaning.
+- Row 0 is accepted as a header only when it is **inconsistent with the column
+  beneath it**: a text label above a numeric column, or a short label above much
+  longer values. This is the test `csv.Sniffer.has_header` uses; it is a
+  property of the table rather than a pattern borrowed from one document.
+- When no header is provable, cells are joined with `" | "` — unlabeled, but
+  never mislabeled. Losing a label is recoverable at retrieval time; labeling
+  every row from a data row is not.
+- A header merged vertically across two rows resolves to the same text in the
+  row below it; that duplicate is skipped rather than emitted as data.
+- A cell merged across columns is reported once per grid position it spans;
+  those repeats are collapsed.
+
+
+ is a pipeline invariant
+
+### 5. Persian normalization is a pipeline invariant
+
+Every extracted text block is normalized before chunking, for all formats:
+
+- Arabic to Persian letter folding: `ك`→`ک`, `ي`→`ی`, `ى`→`ی`, `أ`/`إ`→`ا`
+- `unicodedata.normalize("NFKC")`
+- Removal of harakat (diacritics) and tatweel
+- `¬` → space, then collapse runs of whitespace
+
+Digits and punctuation are **not** rewritten. Persian digits (`۱۲۳`) and
+Persian punctuation (`؛`, `٬`) are left as authored, because chunk `content` is
+what citations render back to the user and Western digits inside Persian prose
+read as wrong.
+
+Normalization runs **per text block, before the markdown is assembled** — the
+whitespace-collapse step maps `\n` to a space, so applying it to an assembled
+document would flatten every heading and paragraph onto a single line.
+
+This complements rather than replaces ADR-0005's sparse-side normalization,
+which additionally removes stopwords and is specific to the BM25 vector. It
+also stabilizes the text that feeds `content_hash`.
+
+### 6. Spreadsheet handling
+
+ADR-0004's rules stand, under the header discipline of section 4: each cell is
+rendered as `"{column_header}: {cell_value}"`, merged cell ranges are
+forward-filled before rendering, and sheets with no non-empty data rows are
+skipped.
+
+Forward-filling merges is not cosmetic. openpyxl stores a merged range's value
+only in its top-left cell, so in the branch directory the province is present
+on the first branch of each province and absent from every other one. Filling
+the range makes each row-chunk self-contained — a branch carries its province
+even though the source cell is blank.
+
+One correction: `.csv` is read with the standard library's `csv` module, not
+`pandas` as ADR-0004 states. Adding pandas for delimiter handling and row
+iteration is not warranted. `.xlsx` uses `openpyxl`, as ADR-0004 assumed.
+
+There is no sheet-shape sniffing. A two-column Q&A sheet and a branch-directory
+sheet go through the same generic renderer; a Q&A row renders as
+`question: …\nanswer: …` and a branch row as `branch_name: …\ncity: …`, both
+self-describing without a schema heuristic that could misfire.
+
+### 7. Chunking
+
+The `fixed_size` strategy, over tokens counted with tiktoken `cl100k_base`:
+
+| Setting | Value |
+|---|---|
+| `chunk_size` | 400 tokens |
+| `chunk_overlap` | 60 tokens |
+| `max_chunk_tokens` | 512 (hard cap) |
+
+These numbers are recorded here because they exist in no ADR today — ADR-0001
+explicitly left "size/overlap are tunable config, not fixed by this ADR" open,
+and ADR-0004 gives only the 512 ceiling.
+
+`cl100k_base` is a deliberate proxy. `text-embedding-3-large` has an 8191-token
+window and never binds; `nomic-embed-text-v2-moe`'s 512-token sequence length
+is the only real constraint. cl100k tokenizes Persian inefficiently while
+nomic's multilingual tokenizer does not, so a cl100k count reliably
+*over-estimates* the nomic count — measuring with cl100k and capping at 512 is
+safe in the conservative direction, without shipping a second tokenizer and its
+model download into the ingestion path. The 400/512 gap leaves headroom for the
+mandatory `search_document: ` task prefix (ADR-0004) and any heading text
+carried into a chunk.
+
+Spreadsheet rows are atomic and bypass the splitter. A row that exceeds
+`max_chunk_tokens` falls through the fixed-size splitter in place, emitting
+several ordered chunks, rather than being silently truncated by the embedding
+model.
+
+### 8. `content_type` values emitted
+
+ADR-0004's four-value set is unchanged. v1 emits `paragraph` (DOCX prose and
+flattened tables) and `table_row` (spreadsheet rows). `qa_pair` and
+`image_caption` remain defined but are not produced.
+
+### 9. Deferred, not rejected
+
+Semantic-aware chunking; DOCX `table_row` and `qa_pair` structural detection;
+embedded-image captioning; `.doc` conversion; Matryoshka-256 truncation. Each
+remains ADR-0004's stated intent; this ADR only records that v1 does not ship
+them.
+
+## Consequences
+
+### Positive
+
+- Plan 001's ingestion slice is unblocked with a parser already proven against
+  this specific Farsi corpus, rather than one written speculatively.
+- Chunking stays pure, synchronous, and cheap — it fits ADR-0017's inline
+  request budget with no network round-trip, and runs safely under
+  `anyio.to_thread.run_sync`.
+- Persian normalization closes a real defect that would otherwise degrade both
+  dense vectors silently, with no error and no obvious symptom.
+- Concrete chunk-size numbers and their rationale are now recorded somewhere
+  other than a config default, so a later change is a visible decision.
+- Q&A sheets, branch directories, and docx contact tables are all served by one
+  renderer with no schema sniffing, so a new sheet shape needs no new code.
+- Refusing to label a table whose header is unprovable means the parser degrades
+  to unlabeled rows instead of producing confidently wrong ones, which is the
+  failure mode that is hard to notice downstream.
+
+### Negative
+
+- **v1 ships the strategy the user's own comparison ranked second.** Retrieval
+  accuracy is expected to be measurably lower than semantic chunking would
+  give. Neighbor expansion via ADR-0001's pointers is the mitigation, and it is
+  unproven at this scale.
+- A table whose header cannot be proven — short text over short text, which is
+  genuinely ambiguous — produces unlabeled `" | "` rows. A reader or model can
+  still see the values but not which column each belongs to.
+- The layout-table rule keys on `chunk_size`, so changing that setting silently
+  changes which tables are treated as data. The observed margin is two orders of
+  magnitude, so this is unlikely to flip in practice, but it is a coupling.
+- A DOCX that alternates literal `سوال:`/`پاسخ:` paragraphs loses question/answer
+  atomicity; a boundary can fall between a question and its answer, because
+  `qa_pair` detection is deferred.
+- cl100k is a proxy for nomic's tokenizer. The relationship is safe in the
+  conservative direction for Persian, but a document in another language could
+  in principle tokenize the other way; the 512 assertion is what catches it.
+- Normalizing stored `content` means the text served in citations is not
+  byte-identical to the source document. Letter folding was chosen over full
+  normalization specifically to keep this difference invisible to a reader.
+- `.doc` files are rejected outright rather than converted, so any legacy
+  document must be re-saved by hand until the conversion service lands.
+
+## Alternatives Considered
+
+- **Implement ADR-0004 in full now** (DOCX table-row detection with header
+  inference, Q&A pair heuristics, image captioning, semantic boundary
+  detection): rejected for v1 as roughly triple the work, none of which exists
+  in the evaluation repository to port, and which would block the first
+  ingestion slice on parser research.
+- **Semantic chunking inside the inline request**: rejected — it adds a
+  per-sentence embedding pass to a request already bounded by
+  `INGESTION_TIMEOUT_SECONDS`, and ADR-0017 chose inline ingestion on the
+  assumption that chunking is negligible. Revisit when ingestion moves back off
+  the request path.
+- **The `nomic-embed-text-v2-moe` tokenizer** for exact chunk sizing: rejected
+  — it requires `transformers`/`tokenizers` and a model file download in the
+  ingestion path to buy precision that the conservative cl100k over-estimate
+  already provides.
+- **Character-based splitting** (no tokenizer at all): rejected — Persian
+  characters-per-token varies enough that a character budget cannot guarantee
+  the 512-token ceiling that actually matters.
+- **Full Persian normalization** including digit unification (`۱۲۳`→`123`) and
+  punctuation mapping: rejected — it would improve lexical matching slightly
+  when a query uses the other digit form, at the cost of rendering Persian
+  citations with Western digits.
+- **Storing raw and normalized text as separate payload fields**: rejected —
+  ADR-0001 fixes the payload field list, and doubling the stored text per point
+  is not justified when letter folding alone is visually lossless.
+- **Row-chunking DOCX tables like spreadsheets**: rejected for v1 — the Farsi
+  `.doc` exports in this corpus use tables as page layout, with ordinary prose
+  inside cells, so treating every row as an atomic unit would shred paragraphs
+  mid-sentence. Revisit once real chunk output from the table-heavy documents
+  has been inspected.

docs/backlog.md

@@ -0,0 +1,109 @@
+# Backlog
+
+Ideas and open questions not yet ready to be an ADR decision or a plan phase.
+Each entry is short: what the idea is, which ADR/plan it would eventually
+touch, and what's still unresolved. When an entry is picked up, turn it into
+an ADR amendment (or a new ADR) and delete it from here — this file is not a
+permanent record, `docs/adr/` is.
+
+## Legacy `.doc` conversion
+
+Relates to: [ADR-0018](adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md).
+
+`.doc` is rejected with `415`. ADR-0004 specified `soffice --headless
+--convert-to docx`, which ADR-0018 rejected as a multi-second subprocess inside
+an inline request. Gotenberg is the obvious candidate since it is already in
+use elsewhere — **but verify before committing to it**: Gotenberg's LibreOffice
+route is built for converting *to PDF*, and `.doc` → `.docx` output may not be
+supported on that endpoint. If it is not, the options are a dedicated
+LibreOffice sidecar or asking uploaders to re-save.
+
+## Structural units ADR-0004 specifies but v1 does not emit
+
+Relates to: [ADR-0004](adr/0004-docx-csv-chunking-strategy.md),
+[ADR-0018](adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md).
+
+- **`qa_pair`**: one sample document alternates literal `سوال:`/`پاسخ:`
+  paragraphs. v1 chunks it as prose, so a chunk boundary can fall between a
+  question and its answer.
+- **`image_caption`**: two sample documents embed images with no alt text.
+  ADR-0004 routes these through a vision API at ingest; v1 drops them silently.
+
+Both need a decision on whether heuristic detection is worth the misfire risk —
+the header-detection work showed that guessing structure is expensive when wrong.
+
+## Tables whose header cannot be proven
+
+Relates to: [ADR-0018](adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md).
+
+A table of short text over short text (`branch,city` with no numeric or long
+column) is genuinely ambiguous, so v1 emits unlabeled `" | "` rows rather than
+risk labeling every row from a data row. No file in the current corpus hits
+this, but a future one will.
+
+The honest fix is not a better heuristic — it is to stop guessing: let the
+upload declare whether a sheet has a header, since the uploader knows. That is
+a `POST /v1/files` contract change, so it belongs with plan 001 Phase 3 rather
+than in the parser.
+
+## Recalibrate chunk size against nomic's tokenizer
+
+Relates to: [ADR-0018](adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md).
+**Revisit after retrieval quality is measurable** — deliberately deferred, not
+forgotten.
+
+ADR-0018 counts tokens with tiktoken `cl100k_base` and caps chunks at 512. But
+512 is `nomic-embed-text-v2-moe`'s limit, measured in *nomic's* tokenizer, not
+OpenAI's. Those are different units, and on Persian they differ by a lot.
+
+Measured against a real production document (`bimeh_havades.docx`, 5,911 chars
+of Farsi) via the Ollama server that already hosts the model:
+
+| Sample | cl100k tokens | nomic tokens | ratio |
+|---|---|---|---|
+| 300 chars | 217 | 80 | 2.71 |
+| 600 chars | 426 | 165 | 2.58 |
+| 1,200 chars | 846 | 303 | 2.79 |
+
+So **~2.7 cl100k tokens per nomic token** on Persian. The current
+`chunk_size=400` is therefore about **148 nomic tokens — roughly 29% of the
+512-token window**. Chunks land near 570 characters where ~1,500 would fit.
+
+Two things this measurement also established:
+
+- **Silent truncation is real, and now demonstrated.** Feeding 2,400 and 4,800
+  characters both returned `prompt_eval_count` of exactly 512, with no error
+  and no warning. This is what ADR-0004 meant by "silently truncated by the
+  model, not an error", confirmed on our own hardware.
+- **Measuring nomic tokens needs no new dependency.** Ollama's `/api/embed`
+  returns `prompt_eval_count`, so the real count is obtainable from the
+  embedding call we already have to make. Note the value saturates at 512, so
+  it cannot measure anything longer than the window — calibration samples must
+  stay under it.
+
+When picking this up, decide between: raising `chunk_size`/`max_chunk_tokens`
+in cl100k terms using a calibration ratio (cheap, drifts if the corpus language
+mix changes); counting with nomic's own tokenizer offline via HuggingFace
+`tokenizers` and its `tokenizer.json` (exact, and lighter than ADR-0018
+assumed — the tokenizer file only, not the 475M-param model weights); or
+keeping small chunks because neighbor expansion recovers the context anyway.
+
+Do not change this on the ratio alone. The reason to keep 400/60/512 for now is
+that smaller chunks are not automatically worse for retrieval — measure
+retrieval quality first, then decide.
+
+Also note `nomic-embed-text:latest` (v1.5) is on the same Ollama server with a
+2,048-token context, but it is the English-focused model; v2-moe is the
+multilingual one and the reason ADR-0004 chose it for Farsi. Do not switch to
+v1.5 just to get a bigger window.
+
+## Get LLM usage/price from the OpenAI API
+
+Relates to: [ADR-0009](adr/0009-postgres-sqlalchemy-alembic-schema.md)'s
+`llm_calls`/`llm_pricing` tables.
+
+Get token usage and price from the OpenAI API's response metadata, instead of
+computing/tracking them ourselves. Need to check whether OpenAI actually
+returns price, or only token counts — if only counts, we still need
+`llm_pricing` for price and this only changes how `llm_calls` gets its
+usage numbers.

docs/plans/001-ingestion-vertical-slice.md

@@ -0,0 +1,337 @@
+# 001. Ingestion vertical-slice implementation plan
+
+## Purpose
+
+This plan turns the accepted architectural direction in the ADRs into the first
+working product slice: a tenant-scoped DOCX/XLSX/CSV upload is stored in MinIO, represented
+by durable Postgres records, parsed/chunked/embedded inline in the request
+(ADR-0017), and indexed as Qdrant points before the response returns.
+
+This is an implementation plan, not an Architecture Decision Record. ADRs explain
+why major technologies and boundaries were chosen; this document defines the
+order, scope, and verification criteria for implementing them.
+
+## Architecture baseline
+
+The first vertical slice uses these responsibilities:
+
+| System | Responsibility |
+|---|---|
+| FastAPI | HTTP boundary, validation, authentication, tenant derivation, and job creation. |
+| Postgres | Tenant/auth data, source-file metadata, ingestion attempt records/progress, and audit. |
+| MinIO | Private source-file bytes and retained derived ingestion blobs. |
+| Ingestion service | Parsing, chunking, embedding, ingestion-generated Chunk/Point CRUD, and job status updates — inline in the request. |
+| Qdrant | Tenant-filtered generated chunks and their vectors/payloads. |
+
+The controlling ADRs are:
+
+- [ADR-0008](../adr/0008-rest-api-and-fastapi-boundary.md): FastAPI REST boundary
+  and job-shaped file ingestion contract.
+- [ADR-0009](../adr/0009-postgres-sqlalchemy-alembic-schema.md): Postgres source
+  files, jobs, audit, and migration conventions.
+- [ADR-0012](../adr/0012-application-resource-lifetime-and-dependency-ownership.md):
+  resource lifetime, dependency injection, and explicit transaction ownership.
+- [ADR-0013](../adr/0013-s3-compatible-object-storage-with-minio.md): MinIO object
+  storage boundary.
+- [ADR-0017](../adr/0017-synchronous-ingestion-in-the-request-path.md):
+  inline ingestion, batched/bounded-concurrent embedding,
+  `anyio.to_thread.run_sync` for blocking work, and request bounds. It supersedes
+  [ADR-0014](../adr/0014-durable-job-dispatch-with-rabbitmq.md), which remains
+  the design to adopt when a broker becomes necessary.
+
+The cited ADRs are currently proposed. Treat them as the implementation baseline
+only after the project owner accepts them; code should not silently diverge from
+them.
+
+## First release scope
+
+### In scope
+
+- `POST /v1/files` for authenticated tenant-scoped **DOCX, XLSX, and CSV** upload.
+  `.doc` is rejected with `415` pending an out-of-process conversion service
+  (ADR-0018). An earlier revision of this plan scoped the slice to CSV only and
+  placed DOCX out of scope; the real corpus is DOCX and XLSX, so ADR-0018
+  corrects that.
+- File validation, size limits, content hashing, and streaming upload to MinIO.
+- Alembic-managed Postgres schema for the minimal tenant/auth, source file,
+  ingestion job, and job event records needed by this slice.
+- Inline ingestion in `POST /v1/files`, with batched/bounded-concurrent
+  embedding, thread-offloaded parsing, and enforced size/timeout/capacity
+  bounds.
+- DOCX/XLSX/CSV parsing into structural units and deterministic chunk creation
+  (ADR-0018, implemented in `src/application/ingestion/`).
+- Tenant-filtered Qdrant point upserts using deterministic point identifiers.
+- Job status/progress persistence and `GET /v1/files/{file_id}` status lookup.
+- Structured correlation logging at HTTP and ingestion-stage boundaries.
+- Automated tests for the critical state transitions, redelivery, and tenant
+  boundaries.
+
+### Explicitly out of scope
+
+- Legacy `.doc` ingestion — rejected with `415` until an out-of-process
+  conversion service exists (ADR-0018). DOCX and XLSX are **in** scope; they were
+  listed here before ADR-0018 corrected the scope line.
+- `qa_pair` structural detection and embedded-image captioning (ADR-0004),
+  deferred by ADR-0018.
+- The conversational LangGraph API and SSE streaming.
+- Final reranker selection, GPU deployment, or unresolved model licensing from
+  ADR-0005.
+- Direct `/v1/points` CRUD endpoints beyond the reusable service layer required by
+  ingestion.
+- Full tenant erasure and hard-deletion workflow.
+- A public download API or presigned object URLs.
+- A message broker, transactional outbox, job queue, and separate
+  worker/publisher processes (ADR-0014, deferred by ADR-0017).
+- `late_interaction` (jina-colbert-v2) document vectors at ingest — populating
+  them is ADR-0017's primary trigger to move ingestion back off the request.
+- Exactly-once end-to-end processing. Job execution must instead be safe when a
+  job runs more than once.
+
+## Required invariants
+
+The implementation must preserve these rules from the ADRs:
+
+1. `tenant_id` is derived from trusted authentication context; it is never
+   accepted from the upload body, query parameters, MinIO metadata, or a
+   dispatch payload as authority.
+2. MinIO stores bytes; Postgres stores metadata, lifecycle state, job progress,
+   audit records, and the queue.
+3. Dispatch payloads contain stable IDs and correlation metadata only. They never
+   contain file bytes, extracted text, chunks, embeddings, secrets, raw prompts,
+   or raw model output.
+4. Ingestion works from the persisted job and source-file records, not from
+   request-supplied values.
+5. Parsing, chunking, embedding, and generated Qdrant Chunk/Point CRUD live in
+   the ingestion application service, not in the route handler. Blocking work
+   runs through `anyio.to_thread.run_sync`, never directly on the event loop.
+6. Qdrant reads and mutations are tenant-filtered. Ingestion-generated point IDs
+   are deterministic so retrying a job does not create duplicate logical chunks.
+7. No Postgres session or transaction is held open across parse/embed/upsert.
+   The job row is committed `running` before the work and updated to a terminal
+   status after it, in a second short transaction.
+8. Application clients are built at FastAPI startup and closed at shutdown. No
+   mutable clients are opened as import-time globals.
+9. Embedding is batched per provider limits and concurrency-bounded by a
+   semaphore; blocking work (parse, chunk, BM25, `minio`) runs through
+   `anyio.to_thread.run_sync` with an explicit `CapacityLimiter`.
+
+## Required decisions before implementing the affected phase
+
+The first implementation should use these defaults unless a later ADR changes
+them:
+
+### Source-file idempotency and replacement
+
+- Use `(tenant_id, domain, content_sha256)` to recognize identical uploads.
+- An identical active upload should return the existing source-file/job reference
+  rather than create a duplicate ingestion.
+- A changed upload creates a new ingestion job. A failed re-ingestion never
+  removes a working index: the soft-delete sweep for a shortened file runs only
+  after every upsert has succeeded. Because ADR-0001's point ids are
+  deterministic, upserts overwrite in place, so an interrupted attempt can leave
+  a prefix updated — it cannot empty or partially delete the index, and a retry
+  converges. See ADR-0017, "Re-running an ingestion stays safe".
+- Preserve the original filename in Postgres metadata. MinIO object keys remain
+  internal ID-based paths.
+
+This policy should be made explicit in ADR-0009 before implementing re-ingestion
+rather than becoming an accidental repository behavior.
+
+### File deletion
+
+For the first release, file deletion should be soft and job-shaped:
+
+1. mark the source file as deletion requested/soft deleted in Postgres;
+2. soft-delete the related Qdrant points inline, recording the attempt;
+4. retain the MinIO object until an explicit retention or hard-erasure workflow.
+
+Hard deletion requires a later retention/erasure implementation covering MinIO,
+Qdrant, and the relevant Postgres data.
+
+### Ingestion bounds and operations
+
+Before deploying an environment, define and document:
+
+- `INGESTION_MAX_CONCURRENCY` and the thread-pool capacity limiter, and their
+  relation to the process's CPU/memory budget;
+- `INGESTION_TIMEOUT_SECONDS`, and the proxy/load-balancer/client read timeouts
+  that must exceed it;
+- `INGESTION_EMBED_BATCH_SIZE` and `INGESTION_EMBED_CONCURRENCY`, sized to the
+  provider's rate limits and the self-hosted embedder's capacity;
+- alert thresholds for p95 ingestion duration, `503`/`504` rates, and jobs left
+  in `running` past the timeout;
+- how failed ingestions are inspected and retried.
+
+These are deployment/runbook settings, not new ADRs unless they change the
+reliability guarantee or system boundary.
+
+## Build order
+
+### Phase 1: Foundation and local dependencies
+
+1. Add typed configuration in `src/config.py` for Postgres, MinIO, Qdrant,
+   ingestion bounds, application limits, and logging.
+2. Populate `.env.example` with non-secret local-development configuration.
+3. Add application Docker Compose services for Postgres, MinIO, and Qdrant. Keep
+   application MinIO buckets/credentials separate from Langfuse infrastructure.
+4. Add direct Python dependencies and lock them with `uv`:
+   SQLAlchemy async/Postgres driver, MinIO/S3 client, Qdrant client, and
+   structured logging dependencies chosen by ADR-0011.
+5. Add the pytest foundation from ADR-0016: async test configuration, boundary
+   markers, and support for dependency-injected fakes. Add a lifespan smoke test
+   before creating external clients.
+6. Create FastAPI lifespan setup and typed dependency helpers without creating
+   schema at startup.
+
+**Exit criteria:** local infrastructure starts; readiness checks can report each
+required dependency; clients are opened/closed by process owners; fast unit tests
+run without Docker or live providers.
+
+### Phase 2: Database, migrations, and domain contracts
+
+1. Define SQLAlchemy models and Alembic migrations for the minimum required
+   tables: `tenants`, `api_keys`, `source_files`, `ingestion_jobs`, and
+   `ingestion_job_events`, per ADR-0009.
+2. Define Pydantic request/response schemas, including the terminal upload
+   response (`file_id`, `ingestion_job_id`, `status`, `chunks_indexed`).
+3. Implement explicit repositories/services with a request/job-lifetime
+   `AsyncSession`; routes/services own commit/rollback boundaries as specified by
+   ADR-0012.
+4. Write the empty-database migration test before each schema revision. Create
+   Testcontainers-based Postgres fixtures for tenants, hashed API keys, database
+   sessions, and migrations. Do not use `create_all()` in test fixtures.
+
+**Exit criteria:** migrations create the schema from an empty database; application
+startup performs no DDL; schema and repository tests verify tenant-scoped
+reads/writes and valid job transitions.
+
+### Phase 3: MinIO upload and durable job creation
+
+1. Implement API-key authentication and `AuthContext` tenant derivation.
+2. Implement `POST /v1/files` for DOCX, XLSX, and CSV, including streaming-size controls,
+   file-type validation, SHA-256 calculation, and a private MinIO upload using
+   an internal object key.
+3. In one short Postgres transaction, persist `source_files` and create
+   `ingestion_jobs(status='running')`, then commit and release the connection
+   before any parse/embed work.
+4. Return `201 Created` with `file_id`, `ingestion_job_id`, terminal status, and
+   `chunks_indexed` once ingestion completes.
+5. Implement `GET /v1/files/{file_id}` with tenant filtering and a public status
+   response that does not expose raw storage credentials or internal artifacts.
+6. Add cleanup/compensation handling for a MinIO upload that succeeds while the
+   database transaction fails.
+7. Add unit/API tests for trusted tenant derivation, upload validation, idempotency,
+   the terminal `201 Created` response, and tenant-scoped status. Add MinIO adapter integration tests
+   for server-derived private object paths and compensation behavior.
+
+**Exit criteria:** an authenticated upload creates a private object and a
+`running` job row committed before any ingestion work; a tenant cannot retrieve
+another tenant's file status.
+
+### Phase 4: Bounded execution primitives
+
+1. Add the async embedding ports and adapters in `src/infrastructure/embedding/`,
+   with per-provider batching and an `asyncio.Semaphore` bounding in-flight
+   batches.
+2. Route blocking work (parse, chunk, BM25, `minio`) through
+   `anyio.to_thread.run_sync` with an explicit `CapacityLimiter` created at
+   startup, so ingestion cannot exhaust Starlette's thread pool.
+3. Enforce the request bounds: `INGESTION_MAX_CONCURRENCY` (`503` + `Retry-After`
+   when exceeded), `INGESTION_TIMEOUT_SECONDS` around the whole work phase
+   (`504`), and the size/chunk-count ceiling (`413`) checked before work starts.
+4. Guarantee a bounded failure always writes a terminal job status — a timeout
+   must never leave a job stuck in `running`.
+5. Add unit tests for batching/concurrency limits, timeout-to-terminal-status,
+   and capacity rejection, using scripted embedder fakes.
+
+**Exit criteria:** embedding a few hundred chunks issues batched, concurrent
+requests rather than serial ones; exceeding any bound produces the right status
+code and a terminal job row.
+
+### Phase 5: Ingestion execution and Qdrant Chunk/Point CRUD
+
+> **Carried forward from Phase 4 — the `chunks` collection must create the
+> `sparse` vector with `modifier="idf"`.** The BM25 adapter computes only
+> term-frequency saturation client-side; IDF comes from Qdrant's
+> collection-wide statistics. Omit the modifier and there is no error and no
+> warning — sparse scoring silently loses its IDF term and lexical retrieval
+> degrades. See ADR-0005, "Benchmark outcome".
+>
+> Collection creation must also use the pinned dimensions from ADR-0001:
+> `dense_nomic` 768, `dense_openai` 3072.
+
+1. Implement the ingestion service called by the route, using the
+   application-lifetime database, MinIO, Qdrant, model, and logging clients.
+2. Validate the persisted records before fetching the MinIO object.
+3. Append progress events, parse the document, create deterministic chunks, embed them,
+   and upsert tenant-scoped Qdrant points — without holding a Postgres session
+   open across the work.
+4. In a second short transaction, mark the job `succeeded` with counters or
+   `failed` with a safe error summary, then return the terminal response.
+5. Make a retried upload safe: no duplicate logical chunks, no incorrect
+   counters, and no transition from a terminal state back to `running`.
+6. Add unit tests for deterministic chunks, point IDs, and terminal job
+   transitions. Add Testcontainers Qdrant and Postgres integration tests for
+   tenant-filtered upserts, terminal state persistence, retrying an upload, and
+   parser/Qdrant failure handling.
+
+The `chunks` collection itself is provisioned by a deployment step —
+`uv run python -m src.cli.qdrant_bootstrap` — not by FastAPI startup, for the
+same reason ADR-0009 keeps Alembic out of startup and ADR-0012 makes LangGraph's
+`.setup()` a deployment step. See ADR-0001, "Collection provisioning".
+
+**Exit criteria:** a successful upload returns `201` with a terminal status, and
+its points are retrievable only under the owning tenant's Qdrant filter. A forced
+failure mid-ingestion produces a `failed` job and the right HTTP status, and
+retrying the upload produces a correct final state without duplicate chunks.
+
+### Phase 6: Operations, integration tests, and documentation
+
+1. Add an operator runbook covering local startup, migrations, MinIO bucket
+   setup, the run command, ingestion-bound tuning, the proxy/client timeout
+   requirement, and how to retry a failed ingestion. — `docs/runbook.md`.
+2. Add a serialized Compose-based operational smoke test covering upload through
+   indexed points against the running web process. Testcontainers remains the
+   standard pytest mechanism for individual adapter integration tests. —
+   `scripts/smoke.sh` driving `tests/e2e/test_compose_smoke.py`, which skips
+   itself unless `SMOKE_BASE_URL` is set so `uv run pytest` never invokes
+   Compose.
+3. Add end-to-end tests for duplicate upload, retrying a failed upload, tenant
+   isolation, capacity/timeout rejection, and failed parser/Qdrant behavior. —
+   `tests/e2e/test_ingestion_slice.py`, on Testcontainers, in the default suite.
+4. Add health/readiness checks that distinguish process health from dependency
+   readiness. — `/healthz` and `/readyz`; `/readyz` additionally requires the
+   `chunks` collection to exist, since a reachable but unbootstrapped Qdrant
+   would `502` on the first upload.
+5. Update the README with local-start instructions and links to ADRs, this plan,
+   and the operations runbook.
+
+Provisioning a tenant and its first API key turned out to be a prerequisite for
+1 and 2 rather than a separate milestone: nothing over HTTP can create the first
+tenant, so `src/cli/provision_tenant.py` was added alongside the other two
+deployment-step commands.
+
+**Exit criteria:** a new developer can start the stack, apply migrations, upload a
+a document, observe the job through completion, and understand how to investigate or
+retry a failure.
+
+## Definition of done for the vertical slice
+
+The first slice is done when the following path works in local Compose and is
+covered by automated tests:
+
+```text
+POST /v1/files (authenticated DOCX/XLSX/CSV upload)
+  -> raw bytes stored privately in MinIO
+  -> source file and running job committed in Postgres, connection released
+  -> parse/chunk on threads, embed in bounded concurrent batches
+  -> deterministic Qdrant points upserted
+  -> Postgres records progress and terminal job status
+  -> 201 Created returns the terminal result in the same request
+  -> GET /v1/files/{file_id} reports that status within the owning tenant only
+```
+
+The next implementation work after this slice is direct Point CRUD, retrieval,
+and then the LangGraph conversational flow. Do not couple those later milestones
+to the initial ingestion path unless they are needed to preserve one of the
+invariants above.

docs/plans/002-point-crud-and-keyword-search.md

@@ -0,0 +1,270 @@
+# 002. Point CRUD and keyword-search implementation plan
+
+## Purpose
+
+This plan covers the milestone named at the end of
+[plan 001](001-ingestion-vertical-slice.md): direct, fine-grained management of
+individual Qdrant points through `/v1/points`, plus filter/keyword search over
+them. Ingestion (plan 001) writes points in bulk; this slice lets a human or
+admin frontend read, edit, reorder, and soft-delete them one at a time, under
+the same tenant isolation.
+
+This is an implementation plan, not an Architecture Decision Record. The ADRs
+explain why the collection schema, endpoints, and isolation rules are what they
+are; this document defines order, scope, and verification criteria.
+
+## Prerequisite
+
+Plan 001 is complete through Phase 6, so this prerequisite is satisfied. It
+required plan 001 through **Phase 5** before Phase 3 of this plan starts.
+Specifically this plan depends on: the `chunks` collection and its
+payload indexes actually existing, API-key authentication and `AuthContext`
+tenant derivation, the application-lifetime Qdrant client from the FastAPI
+lifespan, and the request-lifetime `AsyncSession` wiring. Phases 1–2 below
+(schemas and the read paths) can be built against the collection alone and do
+not need the full ingestion path.
+
+## Architecture baseline
+
+| System | Responsibility |
+|---|---|
+| FastAPI | HTTP boundary, auth, tenant derivation, request/response schemas, scope checks. |
+| Points application service | Point CRUD semantics: soft delete, ordering, neighbor relinking, optimistic concurrency. |
+| Qdrant | The only store for point content, vectors, and payload. |
+| Postgres | Audit of mutating operations. It does **not** hold a mirror of point state. |
+
+The controlling ADRs are:
+
+- [ADR-0001](../adr/0001-ingestion-pipeline-and-collection-schema.md): the
+  `chunks` collection, payload schema, payload indexes, deterministic point IDs,
+  `order_id` as a fractional float, and the
+  `previous_chunk_id`/`next_chunk_id` adjacency pointers.
+- [ADR-0002](../adr/0002-chunk-crud-and-search-api.md): the CRUD operation set,
+  soft-delete-by-default, reorder semantics, keyword search vs. semantic search,
+  and `version`-based optimistic concurrency.
+- [ADR-0008](../adr/0008-rest-api-and-fastapi-boundary.md): the `/v1/points`
+  REST surface, `GET /v1/files/{file_id}/points`,
+  `DELETE /v1/files/{file_id}`, and the `points:read`/`points:write` scopes.
+- [ADR-0012](../adr/0012-application-resource-lifetime-and-dependency-ownership.md):
+  the Qdrant client is application-lifetime and injected, never constructed per
+  request.
+- [ADR-0015](../adr/0015-modular-monolith-package-architecture.md): routes call
+  `application/points/`, which calls a port implemented in
+  `infrastructure/qdrant/`. Routers never build Qdrant filters or call the SDK.
+- [ADR-0016](../adr/0016-testing-strategy-and-quality-gates.md): unit tests
+  against a fake point port; Testcontainers Qdrant for the adapter.
+
+ADR-0001 through 0004 are Accepted; 0008, 0012, 0015, and 0016 are still
+Proposed. Treat the proposed ones as the implementation baseline only once the
+project owner accepts them, and update the ADR rather than diverging silently.
+
+## Scope
+
+### In scope
+
+- `POST /v1/points`, `GET /v1/points/{point_id}`, `PUT /v1/points/{point_id}`,
+  `PATCH /v1/points/{point_id}/payload`, `DELETE /v1/points/{point_id}`.
+- `GET /v1/points?file_id=...` (scroll, `order_by: order_id`, paginated),
+  `GET /v1/points/count`, `GET /v1/points/search?q=...`.
+- `PATCH /v1/points/{point_id}/order` with correct neighbor relinking.
+- `POST /v1/points/batch` for bulk multi-operation edits.
+- `GET /v1/files/{file_id}/points` and `DELETE /v1/files/{file_id}`
+  (bulk soft delete of a file's points), from ADR-0008.
+- Soft delete as the default for every delete path, with neighbor relinking.
+- Optimistic concurrency on every mutating path via the `version` payload field.
+- Audit rows in Postgres for mutating operations: both ADR-0009 tables,
+  `api_request_logs` (one row per API call, written from the request middleware)
+  and `point_audit_events` with the real `api_request_log_id` foreign key.
+- Automated tests for tenant isolation, pointer integrity, concurrency
+  conflicts, and pagination.
+
+### Explicitly out of scope
+
+- Hybrid dense+sparse retrieval, RRF fusion, and late-interaction rerank
+  (ADR-0003/0005) — that is the next plan, 003. `GET /v1/points/search` here is
+  **keyword/filter matching only**; do not let it grow a semantic mode.
+- Re-embedding a point on content edit (see the open decision below).
+- Hard deletion / compliance purge endpoints.
+- The LangGraph conversational API (ADR-0006/0007).
+- A frontend.
+
+## Required invariants
+
+1. `tenant_id` comes from the authenticated context and is injected into every
+   filter server-side — on reads *and* writes, on every code path. A
+   `tenant_id`, or any payload key that would override it, appearing in a
+   request body or query string is rejected, never honored.
+2. Cross-tenant access returns `404`, not `403` — the same rule plan 001 applies
+   to files. A caller must not be able to probe for the existence of another
+   tenant's point IDs.
+3. Soft delete is the default: `is_active=false` + `deleted_at` via
+   `set_payload`. Points are never removed from Qdrant by any endpoint in this
+   slice.
+4. Reads exclude inactive points unless the caller explicitly opts in.
+5. Any operation that changes a point's position or removes it from the sequence
+   — insert, reorder, delete — updates the affected neighbors'
+   `previous_chunk_id`/`next_chunk_id` in the **same** `points/batch` request.
+   A partial relink is a defect: ADR-0003's context-window expansion walks these
+   pointers.
+6. Point IDs stay derived from `file_id` + the immutable `chunk_index`.
+   Reordering changes `order_id` only, never the point ID.
+7. Every mutating operation is guarded by the `version` payload field via
+   Qdrant's `update_filter`, and increments it. A stale write returns `409`,
+   it does not silently clobber.
+8. `order_id` is a fractional float. Inserting or moving a point assigns a value
+   between its two new neighbors; it never renumbers siblings.
+9. Routers contain no Qdrant SDK calls and no filter construction. The Qdrant
+   client is injected from the lifespan (ADR-0012).
+
+## Decisions resolved before implementation
+
+An earlier revision of this plan listed three open decisions here. All are now
+settled, and one further question this plan deferred to a Phase 6 test has been
+settled too. They are recorded in the ADRs — these lines are a pointer, not a
+second source of truth.
+
+| Question | Resolution | Recorded in |
+|---|---|---|
+| Re-embedding on content edit | Re-embed inline, reusing ingestion's ports and bounds and its `502`/`504` codes. The re-embed happens *before* the version-guarded write, so a stale edit still `409`s rather than re-embedding for nothing. | ADR-0002, "Re-embedding on content edit" |
+| Fractional-key exhaustion | No renormalize endpoint in this slice. Log `points.order_id.gap_low` under a safety threshold; reject with `409` and a distinct error code if the gap would collapse onto a neighbor value. Recovery is a runbook operation. | ADR-0002, "`order_id` gap exhaustion" |
+| Batch semantics | All-or-nothing, capped at 100 operations. Every operation's `version` precondition is validated before any is applied; one failure rejects the whole request and nothing reaches Qdrant. | ADR-0002, "`POST /points/batch` semantics" |
+| Re-ingestion versus manual edits | The newly uploaded file wins. Surviving points are overwritten in place with an incremented `version`; points absent from the new version are flagged inactive, never removed; manually created points sit past the ingested `chunk_index` range and are swept by the same rule. Clobbered content is recorded in `point_audit_events` as `reingest_overwrite`. | ADR-0002, "Re-ingestion versus manual edits" |
+
+Phase 6's cross-slice end-to-end test therefore *verifies* the re-ingestion rule
+rather than forcing the decision.
+
+## Build order
+
+### Phase 1: Point contracts and the port
+
+1. Define the payload model in `src/application/points/` mirroring ADR-0001's
+   field list exactly, with the reserved/server-owned fields (`tenant_id`,
+   `version`, `chunk_index`, `deleted_at`) separated from caller-writable ones.
+2. Define the point port: get, upsert, set payload, scroll, count, keyword
+   search, batch. Tenant filter is a required parameter on every method — not an
+   optional argument a caller can forget.
+3. Implement the Qdrant adapter behind the port in
+   `src/infrastructure/qdrant/`, using the injected application-lifetime client.
+4. Add a fake port implementation for unit tests, with ordering and version
+   behavior faithful enough to test the service against.
+5. Define the Pydantic request/response schemas in `src/api/schemas/`. Vectors
+   are returned only when explicitly requested (ADR-0008).
+
+**Exit criteria:** the service layer can be unit-tested end to end against the
+fake; a Testcontainers Qdrant test confirms the adapter's filter construction
+and `order_by` scroll match the fake's semantics.
+
+### Phase 2: Read paths
+
+1. `GET /v1/points/{point_id}` — tenant-filtered retrieve; `404` when the point
+   belongs to another tenant or does not exist.
+2. `GET /v1/points?file_id=...` — scroll with `order_by: order_id`, stable
+   pagination, `is_active: true` implied.
+3. `GET /v1/files/{file_id}/points` — same listing, addressed by file.
+4. `GET /v1/points/count` — tenant/domain-filtered count.
+5. `GET /v1/points/search?q=...` — full-text payload match on `content` plus
+   structured filters. Name the response and docstring so it cannot be mistaken
+   for semantic retrieval.
+6. Enforce `points:read` scope on all of the above.
+7. Tests: tenant isolation returns `404`; inactive points are excluded by
+   default and included on explicit opt-in; pagination does not skip or repeat
+   under a concurrent insert.
+
+**Exit criteria:** a tenant can list and search only its own active points, in
+display order, with correct paging.
+
+### Phase 3: Soft delete and neighbor relinking
+
+1. Implement the relinking primitive in the service: given a point leaving the
+   sequence, compute the neighbor payload updates and emit them with the
+   deactivation in one `points/batch` call.
+2. `DELETE /v1/points/{point_id}` — soft delete plus relink.
+3. `DELETE /v1/files/{file_id}` — bulk soft delete of a file's active points.
+   The whole file leaves the sequence, so the boundary pointers must end
+   consistent (typically all null within that file).
+4. Deleting an already-inactive point is a no-op success, not a `404` and not a
+   second relink.
+5. Tests: after deleting a middle point, its old neighbors point at each other;
+   after deleting the first point, the new first has a null
+   `previous_chunk_id`; a full traversal of a file's pointer chain after a
+   series of deletes visits every active point exactly once and never enters an
+   inactive one.
+
+**Exit criteria:** no delete path can leave a stale or dangling pointer, and
+none removes a point from Qdrant.
+
+### Phase 4: Create, replace, and payload update
+
+1. `POST /v1/points` — create one point. Server assigns `tenant_id`, `version`,
+   and the derived point ID; the caller supplies content, position, and
+   metadata. Insert relinks neighbors like a reorder does.
+2. `PUT /v1/points/{point_id}` — upsert with `update_only`, guarded by
+   `version` via `update_filter`; `409` on a stale version. Apply the
+   re-embedding decision above.
+3. `PATCH /v1/points/{point_id}/payload` — payload-only update. Reject attempts
+   to write server-owned fields.
+4. Enforce `points:write` scope; write an audit row per mutation
+   (tenant, actor, point, operation, resulting version).
+5. Tests: concurrent writers at the same version — one succeeds, one gets `409`
+   and does not mutate; a `tenant_id` or `version` in the request body is
+   rejected; a create lands in the right sequence position.
+
+**Exit criteria:** manual edits and ingestion re-runs cannot silently clobber
+each other; server-owned fields are unwritable through any endpoint.
+
+### Phase 5: Reorder and batch
+
+1. `PATCH /v1/points/{point_id}/order` — assign a new fractional `order_id`
+   between the new neighbors and relink up to four points in one
+   `points/batch`, per ADR-0002.
+2. Handle the boundary moves (to first, to last) and the no-op move
+   (already in that position).
+3. Implement the fractional-gap decision from above.
+4. `POST /v1/points/batch` with the semantics decided above, including the
+   per-request operation cap.
+5. Tests: a randomized sequence of inserts, moves, and deletes leaves the
+   `order_id` ordering and the pointer chain agreeing with each other after
+   every step — this property test is the main defense for this phase.
+
+**Exit criteria:** display order derived from `order_id` and traversal order
+derived from the pointer chain are identical for every file, after any sequence
+of mutations.
+
+### Phase 6: Integration tests, operations, and documentation
+
+1. Testcontainers Qdrant integration tests for each endpoint's real filter and
+   ordering behavior, isolated per test by unique collection or tenant keys.
+2. An end-to-end test crossing plan 001 and this slice: ingest a CSV, list its
+   points, reorder one, soft-delete another, re-upload the same file, and assert
+   the manual edits interact with re-ingestion exactly as ADR-0002's
+   "Re-ingestion versus manual edits" specifies.
+3. Structured logging at the mutation boundary with stable event names
+   (`points.updated`, `points.reordered`, `points.soft_deleted`) carrying
+   `request_id`, `tenant_id`, `file_id`, and the resulting version.
+4. Extend the operator runbook: how to inspect a file's point sequence, how to
+   spot a broken pointer chain, and how to recover one.
+5. Update the README and ADR-0002 with anything this implementation settled.
+
+**Exit criteria:** every endpoint has an integration test against real Qdrant,
+and an operator can diagnose an ordering problem from the logs and the runbook.
+
+## Definition of done
+
+This slice is done when, in local Compose and under automated test:
+
+```text
+ingest a CSV (plan 001)
+  -> GET /v1/points?file_id=...      lists its active points in order_id order
+  -> PATCH .../order                 moves one point; order_id and the
+                                     previous/next chain still agree
+  -> PUT /v1/points/{id}             edits content under a version guard;
+                                     a stale write gets 409
+  -> DELETE /v1/points/{id}          soft-deletes and relinks neighbors
+  -> GET /v1/points/search?q=...     keyword-matches content within the tenant
+  -> every one of the above          returns 404, not 403, for another tenant
+```
+
+The next plan (003) is agent hybrid retrieval — three prefetches, RRF fusion,
+late-interaction rerank, and context-window expansion over the pointer chain
+this slice is responsible for keeping correct (ADR-0003, ADR-0005). Do not pull
+any of it into this plan.

docs/runbook.md

@@ -0,0 +1,303 @@
+# Operator runbook
+
+How to start this service, configure its ingestion bounds, and investigate or
+retry a failed upload. Architecture rationale lives in [`docs/adr/`](adr/); the
+implementation milestone is
+[plan 001](plans/001-ingestion-vertical-slice.md). This document covers
+operating what those describe.
+
+The service is a **single process with no background work**. `POST /v1/files`
+parses, chunks, embeds, and indexes inline and returns a terminal result
+(ADR-0017). There is no queue, no worker, and no automatic retry — the caller
+owns the retry decision, which makes the request's duration a deployment
+constraint. That fact drives most of this document.
+
+## 1. Prerequisites and local startup
+
+Docker, and [`uv`](https://docs.astral.sh/uv/) with Python 3.13.
+
+```bash
+cp .env.example .env      # non-secret local defaults; .env is gitignored
+uv sync
+docker compose up -d --wait
+```
+
+`docker-compose.yml` runs Postgres (`127.0.0.1:5433`), MinIO
+(`127.0.0.1:9100`, console `9101`), and Qdrant (`127.0.0.1:6343`). It is the
+local development stack and says so in its header — it is not a production
+deployment.
+
+## 2. Deployment steps
+
+Two schema steps run **before** the application, never at startup: FastAPI
+performs no DDL, for Postgres (ADR-0009) or for Qdrant (ADR-0001, "Collection
+provisioning"). Both commands and their reasoning are in the README's
+[Provisioning the datastores](../README.md#provisioning-the-datastores)
+section:
+
+```bash
+uv run alembic upgrade head                   # Postgres schema
+uv run python -m src.cli.qdrant_bootstrap     # the `chunks` collection
+```
+
+Both are idempotent. `qdrant_bootstrap` verifies an existing collection against
+the pinned schema and **exits non-zero on a mismatch** rather than leaving a
+silently degraded sparse index in place — the `sparse` vector's
+`modifier="idf"` and the pinned dense dimensions (768 / 3072) fail silently if
+wrong, which is why they are checked rather than assumed.
+
+Run both again after every deploy that ships a migration or a collection-schema
+change.
+
+## 3. MinIO bucket
+
+Under Compose the bucket already exists: the `app-minio` service's entrypoint
+runs `mkdir -p /data/${MINIO_BUCKET:-chatbot-source-files}` before starting the
+server, so first boot creates it. Nothing else needs to be done locally.
+
+Outside Compose, create the bucket named by `MINIO_BUCKET` before the first
+upload — the application never creates it. It must stay **private**; ADR-0013
+keeps source bytes non-public and this slice ships no download API or presigned
+URLs.
+
+## 4. Provisioning a tenant, an API key, and its domains
+
+Nothing over HTTP can bootstrap a tenant: every `/v1` route needs an API key,
+and a key cannot exist before its tenant. So the first key is issued by an
+operator command:
+
+```bash
+uv run python -m src.cli.provision_tenant \
+  --slug acme --domain fire --domain life --scopes files:write,domains:read
+```
+
+It prints `api_key=sk_...` **once**. Postgres stores only its SHA-256 hash
+(ADR-0009), so a lost key is reissued by re-running the command, never
+recovered. Structured logs carry only the non-secret `key_prefix` — a plaintext
+key must never reach a log sink (ADR-0011).
+
+Re-running with the same `--slug` reuses the tenant and any domains it already
+has, and issues an **additional** key. Both keys stay valid; this adds a key, it
+does not rotate one.
+
+Scopes are the security boundary between uploading, reading chunks, and
+managing the allowlist. Give an upload client `files:write` only. `domains:write`
+lets its holder create new domains, which is exactly what the allowlist exists to
+prevent an upload key from doing, and `points:read` lets its holder read the text
+of every chunk of every file — so an upload-only key gets neither.
+
+| Scope | Grants |
+|---|---|
+| `files:write` | Upload a document and read its ingestion status. |
+| `points:read` | Read, list, count, and keyword-search this tenant's points, including `GET /v1/files/{file_id}/points`. |
+| `points:write` | Create, edit, reorder, and soft-delete points (plan 002 Phases 3-5; no route uses it yet). |
+| `domains:read` / `domains:write` | Inspect and manage the domain allowlist. |
+| `admin` | Satisfies every scope check. |
+
+The command's `--scopes` default issues all of the above except `admin`, which
+suits a first operator key; narrow it explicitly for per-client keys.
+
+### Domains after the first one
+
+`POST /v1/files` rejects an unregistered or disabled `domain` with `400`
+(`unknown_domain`) before anything is written. Ongoing domain management is the
+`/v1/domains` API, under `domains:read` / `domains:write`:
+
+```bash
+curl -X POST http://localhost:8000/v1/domains \
+  -H "Authorization: Bearer $API_KEY" -H 'Content-Type: application/json' \
+  -d '{"domain": "fire", "display_name": "Fire insurance"}'
+```
+
+The `domain` key itself is immutable — it is denormalized into every Qdrant
+point payload and into `source_files`, so renaming it is a migration, not an
+edit (ADR-0009). Disabling a domain blocks new uploads; it does not delete
+existing points.
+
+## 5. Running the service
+
+```bash
+uv run fastapi dev src/main.py                                   # local, reload
+uv run uvicorn src.main:app --host 0.0.0.0 --port 8000           # deployed shape
+```
+
+Run more than one worker/replica only after reading §6: ingestion bounds are
+**per process**, so `INGESTION_MAX_CONCURRENCY` multiplies by the number of
+processes.
+
+## 6. Ingestion bounds and tuning
+
+Every bound is enforced server-side and maps to a status code. All are in
+`.env.example`. Ingestion is CPU- and network-bound in the request, so these are
+the numbers that decide whether the service degrades gracefully or falls over.
+
+| Setting | Bounds | On breach | Size it against |
+|---|---|---|---|
+| `INGESTION_MAX_CONCURRENCY` | Ingestions in flight **per process** | `503` + `Retry-After` | Memory per in-flight upload (whole file plus its chunks and vectors are resident) and the embedder's capacity. Rejecting is deliberate: ADR-0017 refuses rather than queues. |
+| `INGESTION_THREAD_POOL_SIZE` | Threads for blocking work (parse, chunk, hash, BM25, the sync `minio` SDK) | — (waits) | CPU cores. It exists to stop ingestion exhausting Starlette's own thread pool, so keep it below the total thread budget. |
+| `INGESTION_TIMEOUT_SECONDS` | The whole work phase | `504`, job marked `failed` | The slowest legitimate document, plus headroom. See §7 — this must stay under every read timeout in front of it. |
+| `INGESTION_MAX_UPLOAD_SIZE_MB` | Bytes accepted | `413` | Memory: the upload is read fully into the process before any work starts. |
+| `INGESTION_MAX_CHUNKS_PER_FILE` | Chunks per file, checked before embedding | `413` | Embedder cost/time per chunk × `INGESTION_TIMEOUT_SECONDS`. This is the real defence against one pathological file eating a slot. |
+| `INGESTION_EMBED_BATCH_SIZE` | Texts per embedder request | `502` on embedder failure | The provider's per-request limits. Batch before parallelizing. |
+| `INGESTION_EMBED_CONCURRENCY` | Concurrent embed batches | `502` | Provider rate limits and the self-hosted embedder's throughput. Never unbounded. |
+| `QDRANT_UPSERT_BATCH_SIZE` / `_CONCURRENCY` | Points per upsert and concurrent upserts | `502` (`index_error`) | Qdrant's ingest capacity; the batch size stays in ADR-0001's 64–256 band. |
+
+Two settings that look like tuning knobs but are not:
+
+- **`EMBEDDING_NOMIC_KEEP_ALIVE`** holds the self-hosted model resident. A cold
+  load of `nomic-embed-text-v2-moe` takes over 150 s — longer than any sane
+  `INGESTION_TIMEOUT_SECONDS` — so an idle period followed by an upload would
+  otherwise `504`. The lifespan also warms both dense embedders at startup for
+  the same reason.
+- **The BM25 analyzer and weights** (`EMBEDDING_SPARSE_*`) are a measured
+  artifact ported from the `emet` evaluation lab, verified token-for-token
+  against it (ADR-0005). Re-benchmark; do not tune them in place.
+
+## 7. The proxy and client read-timeout requirement
+
+**Every read timeout in front of this service must exceed
+`INGESTION_TIMEOUT_SECONDS`.** That includes the reverse proxy / ingress, any
+load balancer, and the calling backend's own HTTP client.
+
+If a proxy times out first, the client gets that proxy's error, the upload keeps
+running in the process, and the caller learns nothing about the outcome from the
+response. The job row still reaches a terminal status, so
+`GET /v1/files/{file_id}` remains the way to find out what happened — but the
+response contract is broken for that request. ADR-0017 names this the main cost
+of inline ingestion.
+
+A workable local ordering: client read timeout > proxy read timeout >
+`INGESTION_TIMEOUT_SECONDS`.
+
+## 8. Health and readiness
+
+| Endpoint | Question it answers | Use for |
+|---|---|---|
+| `GET /healthz` | Is the process alive? | Liveness probes / restart policy. Never depends on Postgres, MinIO, or Qdrant. |
+| `GET /readyz` | Can it actually serve? | Load-balancer admission and post-deploy gating. `200` with each dependency `true`, `503` if any is `false`. |
+
+`/readyz` checks Postgres, MinIO, and Qdrant reachability **and** that the
+`chunks` collection exists. A reachable-but-unbootstrapped Qdrant reports
+`{"qdrant": false}` on purpose: uploads to it would fail with `502`, so it is
+not ready, and this is how a skipped `qdrant_bootstrap` surfaces at deploy time
+instead of on a user's first upload.
+
+## 9. Investigating a failure
+
+Logs are structured (`structlog`, JSON in production) with stable event names —
+grep the event name, not prose (ADR-0011). Set `LOG_FILE_PATH` for a local
+JSON file sink alongside the console renderer; leave it unset in production,
+where stdout collection is preferred.
+
+**Correlate by `request_id`.** Every request has one, echoed in the
+`X-Request-Id` response header and included in every error envelope, and bound
+into every log line emitted while handling that request. A client reporting a
+failed upload should quote it. `tenant_id`, `file_id`, and `ingestion_job_id`
+are the other join keys.
+
+Events worth knowing:
+
+| Event | Level | Means |
+|---|---|---|
+| `ingestion.job.started` | info | Txn A committed; work phase beginning. Carries `tenant_id`, `ingestion_job_id`, `file_id`, `domain`, `source_type`. |
+| `ingestion.job.completed` | info | Terminal success, with `chunks_parsed`, `points_upserted`, `points_soft_deleted`. |
+| `ingestion.job.failed` | warning | Terminal failure. **`error_code` says which stage**: `storage_upload_failed`, `parse_failed`, `chunk_limit_exceeded`, `embedding_failed`, `index_failed`, `timeout`. |
+| `files.upload.duplicate` | info | Identical content already ingested; the existing file/job was returned and nothing was re-ingested. |
+| `domain.rejected` | warning | Upload refused before any row was written; `reason` is `unregistered` or `disabled`. |
+| `auth.failed` | warning | `reason` is `malformed_key`, `unknown_key`, `key_inactive`, `key_expired`, or `tenant_inactive`. Never contains key material. |
+| `auth.succeeded` | info | Carries `tenant_id`, `api_key_id`, `actor_type`. |
+| `lifespan.embedder.warm_failed` | warning | An embedder was unreachable at startup. Boot continues by design — `/readyz` and the first upload are where this bites. |
+| `qdrant.bootstrap.schema_mismatch` | error | The existing collection diverges from the pinned schema. The bootstrap exits non-zero; do not start the app against it. |
+| `api.unhandled_exception` | error | A bug: an exception with no mapping to the error envelope. Always worth a look. |
+
+A `503` (`ingestion_at_capacity`) is rejected before a job row exists, so it
+appears in the access log and metrics, not in `ingestion_jobs`.
+
+### The durable record
+
+Logs may roll; `ingestion_jobs` and `ingestion_job_events` do not. For one file:
+
+```sql
+SELECT id, status, error_code, error_message, points_created, points_soft_deleted,
+       created_at, updated_at
+FROM ingestion_jobs
+WHERE tenant_id = :tenant_id AND source_file_id = :file_id
+ORDER BY created_at DESC;
+
+SELECT stage, level, message, details, created_at
+FROM ingestion_job_events
+WHERE tenant_id = :tenant_id AND ingestion_job_id = :ingestion_job_id
+ORDER BY created_at;
+```
+
+Recent failures across a tenant:
+
+```sql
+SELECT error_code, count(*), max(created_at)
+FROM ingestion_jobs
+WHERE tenant_id = :tenant_id AND status = 'failed' AND created_at > now() - interval '1 day'
+GROUP BY error_code ORDER BY 2 DESC;
+```
+
+`GET /v1/files/{file_id}` reports the same terminal status over HTTP, scoped to
+the owning tenant — a file belonging to another tenant returns `404`, not `403`.
+
+## 10. Retrying a failed ingestion
+
+**Re-upload the same bytes.** There is no retry endpoint and no automatic retry;
+the client owns that decision (ADR-0017).
+
+What that guarantees:
+
+- Identical content with a **succeeded** job is recognized by
+  `(tenant_id, domain, content_sha256)` and returned as-is with `200` — no
+  re-ingestion, no duplicate points.
+- Identical content whose last job **failed** starts a fresh job against the
+  same `source_files` row. A terminal job is never moved back to `running`.
+- Point ids are deterministic from `file_id` + `chunk_index` (ADR-0001), so the
+  retry **overwrites in place** — it cannot duplicate chunks.
+- A failed attempt never empties or partially deletes a working index: the
+  soft-delete sweep that retires a shortened file's leftover points runs only
+  after every upsert has succeeded. An interrupted attempt can leave a prefix
+  updated; a retry converges (ADR-0017, "Re-running an ingestion stays safe").
+
+Fix the cause first — the `error_code` says where to look:
+
+| `error_code` | Usual cause |
+|---|---|
+| `parse_failed` | The file is corrupt or is not really the type its extension claims. Retrying identical bytes will fail identically. |
+| `chunk_limit_exceeded` | The file is genuinely too large for one inline ingestion. Split it, or raise `INGESTION_MAX_CHUNKS_PER_FILE` knowing what §6 says about the timeout. |
+| `embedding_failed` | The embedder is down, rate-limiting, or unauthenticated. Fix it, then retry — this one usually succeeds unchanged. |
+| `index_failed` | Qdrant is down, or the collection is missing (run `qdrant_bootstrap`). |
+| `timeout` | The work exceeded `INGESTION_TIMEOUT_SECONDS`. Check whether the embedder was cold (see `lifespan.embedder.warm_failed` and `KEEP_ALIVE`) before raising the bound. |
+| `storage_upload_failed` | MinIO is unreachable or the bucket is missing (§3). |
+
+## 11. What to alert on
+
+ADR-0017's own triggers for moving ingestion back off the request path. These
+are the numbers that say the inline design has stopped fitting:
+
+- **p95 ingestion duration** approaching `INGESTION_TIMEOUT_SECONDS`.
+- **`503` and `504` rates** ceasing to be negligible.
+- **Jobs stuck in `running` past the timeout** — every handled failure writes a
+  terminal status, so a non-zero count here means the process died mid-request:
+
+  ```sql
+  SELECT count(*) FROM ingestion_jobs
+  WHERE status = 'running' AND created_at < now() - interval '5 minutes';
+  ```
+
+Also worth alerting: any `qdrant.bootstrap.schema_mismatch`, a sustained
+`/readyz` `503`, and any `api.unhandled_exception`.
+
+## 12. Verifying a deployment
+
+```bash
+./scripts/smoke.sh
+```
+
+Brings up Compose, runs both deployment steps, provisions a throwaway tenant,
+starts the web process, and drives an upload through to indexed Qdrant points
+against the **running process** — including asserting the structured log output
+from §9. It is the only Compose-based test; everything else runs on
+Testcontainers under `uv run pytest` (ADR-0016). Run it before a release.

pyproject.toml

@@ -5,14 +5,52 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.13"
 dependencies = [
+    "alembic>=1.19.1",
+    "anyio>=4.11.0",
+    "asyncpg>=0.31.0",
     "fastapi[standard]==0.141.1",
-    "pydantic-settings>=2.14.2",
+    "httpx>=0.28.1",
+    "langgraph>=1.2.10",
+    "minio>=7.2.20",
+    "openpyxl>=3.1.5",
+    "pydantic-settings>=2.15.0",
+    "python-docx>=1.2.0",
+    "qdrant-client>=1.19.0",
+    "sqlalchemy>=2.0.51",
+    "structlog>=26.1.0",
+    "tiktoken>=0.13.0",
 ]
 
 [dependency-groups]
 dev = [
-    "ruff>=0.16.1",
+    "asgi-lifespan>=2.1.0",
-    "ty>=0.0.65",
+    "pytest>=8.3.5",
+    "pytest-asyncio>=0.25.3",
+    "pytest-cov>=6.0.0",
+    "pytest-timeout>=2.3.1",
+    "ruff>=0.16.2",
+    "testcontainers>=4.9.2",
+    "ty>=0.0.69",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "strict"
+timeout = 10
+# Bound the test function only, not fixture setup. Testcontainers' container
+# startup is charged to whichever test first pulls a session-scoped container
+# fixture; on a cold Docker cache that is ~25s and would trip the 10s budget
+# for every integration test, regardless of how fast the test itself is.
+timeout_func_only = true
+markers = [
+    "unit: fast tests with no external services",
+    "integration: tests against a real disposable service",
+    "e2e: end-to-end vertical-slice tests",
+    "postgres: integration test using Postgres",
+    "minio: integration test using MinIO",
+    "qdrant: integration test using Qdrant",
+    "slow: test exceeds normal integration feedback time",
+    "live_provider: opt-in test that calls an external provider",
 ]
 
 [tool.ruff]
@@ -45,3 +83,17 @@ indent-style = "space"
 
 [tool.ty.environment]
 python-version = "3.13"
+
+[tool.ty.rules]
+# High-value, currently ignore/warn by
+possibly-unresolved-reference = "error"
+missing-type-argument = "warn"
+possibly-missing-attribute = "warn"
+
+# Nice-to-have discipline:
+unused-ignore-comment = "warn"
+missing-override-decorator = "warn"
+
+# Turn OFF or downgrade if noisy on your codebase:
+possibly-missing-import = "ignore"
+division-by-zero = "ignore"

scripts/smoke.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# Serialized operational smoke test of the running web process (ADR-0016, plan
+# 001 Phase 6).
+#
+#   ./scripts/smoke.sh
+#
+# Brings up the Compose stack, runs both deployment steps for real, provisions
+# a tenant, starts uvicorn, and drives `tests/e2e/test_compose_smoke.py`
+# against it over a socket. This is the only Compose-based test: every other
+# test uses Testcontainers and an in-process ASGI transport, which is exactly
+# what makes this one worth having -- it is the only thing that exercises the
+# deployment steps, the real logging configuration, and a real HTTP server.
+#
+# Not part of `uv run pytest`: the smoke test skips itself unless SMOKE_BASE_URL
+# is set, so this script is the only way it runs. Run it before a release.
+#
+# Leaves the Compose stack running (it is the local dev stack); only the uvicorn
+# process and the temporary log file are cleaned up.
+set -euo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+PORT="${SMOKE_PORT:-8021}"
+SLUG="smoke-$(date +%s)"
+DOMAIN="smoke"
+LOG_FILE="$(mktemp -t smoke-app-log.XXXXXX.jsonl)"
+CONSOLE_LOG="$(mktemp -t smoke-app-console.XXXXXX.log)"
+APP_PID=""
+
+cleanup() {
+  if [[ -n "${APP_PID}" ]] && kill -0 "${APP_PID}" 2>/dev/null; then
+    kill "${APP_PID}" 2>/dev/null || true
+    wait "${APP_PID}" 2>/dev/null || true
+  fi
+  rm -f "${LOG_FILE}" "${CONSOLE_LOG}"
+}
+trap cleanup EXIT
+
+if [[ ! -f .env ]]; then
+  echo "no .env found; copy .env.example first (see docs/runbook.md)" >&2
+  exit 1
+fi
+
+echo "==> starting Postgres, MinIO, Qdrant"
+docker compose up -d --wait
+
+echo "==> applying deployment steps"
+uv run alembic upgrade head
+uv run python -m src.cli.qdrant_bootstrap
+
+echo "==> provisioning tenant '${SLUG}'"
+PROVISION_OUTPUT="$(uv run python -m src.cli.provision_tenant \
+  --slug "${SLUG}" --domain "${DOMAIN}" --scopes files:write 2>/dev/null)"
+API_KEY="$(printf '%s\n' "${PROVISION_OUTPUT}" | sed -n 's/^api_key=//p')"
+if [[ -z "${API_KEY}" ]]; then
+  echo "provisioning did not return an api_key" >&2
+  exit 1
+fi
+
+echo "==> starting the web process on port ${PORT}"
+# JSON to a file sink, because the smoke test asserts the real ADR-0011 log
+# output -- the one thing no in-process test can check.
+LOG_JSON_FORMAT=true LOG_FILE_PATH="${LOG_FILE}" \
+  uv run python -m uvicorn src.main:app --host 127.0.0.1 --port "${PORT}" \
+  >"${CONSOLE_LOG}" 2>&1 &
+APP_PID=$!
+
+echo "==> waiting for /readyz"
+for _ in $(seq 1 60); do
+  if curl -fsS "http://127.0.0.1:${PORT}/readyz" >/dev/null 2>&1; then
+    break
+  fi
+  if ! kill -0 "${APP_PID}" 2>/dev/null; then
+    echo "the web process exited before becoming ready:" >&2
+    tail -20 "${CONSOLE_LOG}" >&2
+    exit 1
+  fi
+  sleep 1
+done
+
+if ! curl -fsS "http://127.0.0.1:${PORT}/readyz" >/dev/null 2>&1; then
+  # Most often an unbootstrapped Qdrant or an unreachable embedder host; the
+  # runbook's health/readiness section covers reading this.
+  echo "the web process never became ready:" >&2
+  tail -20 "${CONSOLE_LOG}" >&2
+  exit 1
+fi
+
+echo "==> running the smoke test"
+SMOKE_BASE_URL="http://127.0.0.1:${PORT}" \
+SMOKE_API_KEY="${API_KEY}" \
+SMOKE_DOMAIN="${DOMAIN}" \
+SMOKE_LOG_PATH="${LOG_FILE}" \
+SMOKE_QDRANT_URL="${QDRANT_URL:-http://127.0.0.1:6343}" \
+SMOKE_QDRANT_COLLECTION="${QDRANT_COLLECTION:-chunks}" \
+  uv run python -m pytest tests/e2e/test_compose_smoke.py -q
+
+echo "==> smoke test passed"

src/api/dependencies/auth.py

@@ -0,0 +1,38 @@
+"""Auth dependencies (ADR-0008): resolve `AuthContext` from a bearer token,
+then gate routes on scope.
+"""
+
+from collections.abc import Awaitable, Callable
+from typing import Annotated
+
+from fastapi import Depends
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.application.auth.context import AuthContext
+from src.application.auth.errors import InvalidApiKeyError, MissingScopeError
+from src.application.auth.service import resolve_auth_context
+from src.bootstrap.dependencies import get_sessionmaker
+
+_bearer_scheme = HTTPBearer(auto_error=False)
+
+
+async def get_auth_context(
+    credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(_bearer_scheme)],
+    sessionmaker: Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)],
+) -> AuthContext:
+    if credentials is None:
+        raise InvalidApiKeyError("missing Authorization header")
+    return await resolve_auth_context(sessionmaker, credentials.credentials)
+
+
+AuthContextDep = Annotated[AuthContext, Depends(get_auth_context)]
+
+
+def require_scope(scope: str) -> Callable[[AuthContext], Awaitable[AuthContext]]:
+    async def _dependency(auth: AuthContextDep) -> AuthContext:
+        if not auth.has_scope(scope):
+            raise MissingScopeError(f"missing required scope '{scope}'")
+        return auth
+
+    return _dependency

src/api/errors.py

@@ -0,0 +1,138 @@
+"""Maps application exceptions to the ADR-0008 error envelope.
+
+This is the single place that knows the exception-type -> status-code
+mapping; application/infrastructure code never imports FastAPI or raises
+`HTTPException` (ADR-0015).
+"""
+
+import structlog
+from fastapi import FastAPI, Request, status
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from src.application.auth.errors import (
+    InvalidApiKeyError,
+    MissingScopeError,
+    TenantInactiveError,
+)
+from src.application.domains.errors import DomainAlreadyExistsError, UnknownDomainError
+from src.application.files.errors import (
+    FileTooLargeError,
+    InvalidUploadError,
+    SourceFileNotFoundError,
+)
+from src.application.ingestion.errors import (
+    ChunkLimitExceededError,
+    DocumentParseError,
+    EmbedderError,
+    IngestionAtCapacityError,
+    IngestionTimeoutError,
+    PointIndexingError,
+    UnsupportedSourceTypeError,
+)
+from src.application.points.errors import PointVersionConflictError
+from src.application.points.point import PointNotFoundError
+
+logger = structlog.get_logger(__name__)
+
+# A fixed backoff hint, not a computed retry budget: ADR-0017 rejects a
+# request outright at capacity rather than queueing it, so there is no
+# in-process estimate of when a slot will free up to report instead.
+_CAPACITY_RETRY_AFTER_SECONDS = 1
+
+# (exception type, status code, stable error code)
+_MAPPING: tuple[tuple[type[Exception], int, str], ...] = (
+    (InvalidApiKeyError, status.HTTP_401_UNAUTHORIZED, "invalid_api_key"),
+    (TenantInactiveError, status.HTTP_401_UNAUTHORIZED, "tenant_not_found"),
+    (MissingScopeError, status.HTTP_403_FORBIDDEN, "missing_scope"),
+    (InvalidUploadError, status.HTTP_400_BAD_REQUEST, "validation_error"),
+    # 404, never 403: a cross-tenant point id must be indistinguishable from a
+    # nonexistent one, or the API becomes an existence oracle (ADR-0016).
+    (PointNotFoundError, status.HTTP_404_NOT_FOUND, "not_found"),
+    (SourceFileNotFoundError, status.HTTP_404_NOT_FOUND, "not_found"),
+    # Not "the version guard fired once" — that is retried. This is the service
+    # giving up after repeated re-plans, i.e. a genuinely contended point.
+    (PointVersionConflictError, status.HTTP_409_CONFLICT, "conflict"),
+    (UnknownDomainError, status.HTTP_400_BAD_REQUEST, "unknown_domain"),
+    (DomainAlreadyExistsError, status.HTTP_409_CONFLICT, "conflict"),
+    (DocumentParseError, status.HTTP_400_BAD_REQUEST, "validation_error"),
+    (UnsupportedSourceTypeError, status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, "unsupported_media_type"),
+    (FileTooLargeError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
+    (ChunkLimitExceededError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
+    (EmbedderError, status.HTTP_502_BAD_GATEWAY, "embedder_error"),
+    (PointIndexingError, status.HTTP_502_BAD_GATEWAY, "index_error"),
+    (IngestionTimeoutError, status.HTTP_504_GATEWAY_TIMEOUT, "ingestion_timeout"),
+)
+
+
+def _request_id(request: Request) -> str | None:
+    return getattr(request.state, "request_id", None)
+
+
+def _envelope(
+    code: str, message: str, request_id: str | None, details: dict[str, object] | None = None
+) -> dict[str, object]:
+    return {
+        "error": {
+            "code": code,
+            "message": message,
+            "details": details or {},
+            "request_id": request_id,
+        }
+    }
+
+
+def register_exception_handlers(app: FastAPI) -> None:
+    for exc_type, status_code, error_code in _MAPPING:
+
+        def _handler(
+            request: Request,
+            exc: Exception,
+            status_code: int = status_code,
+            error_code: str = error_code,
+        ) -> JSONResponse:
+            return JSONResponse(
+                status_code=status_code,
+                content=_envelope(error_code, str(exc), _request_id(request)),
+            )
+
+        app.add_exception_handler(exc_type, _handler)
+
+    @app.exception_handler(IngestionAtCapacityError)
+    def _capacity_handler(request: Request, exc: IngestionAtCapacityError) -> JSONResponse:
+        return JSONResponse(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            content=_envelope("ingestion_at_capacity", str(exc), _request_id(request)),
+            headers={"Retry-After": str(_CAPACITY_RETRY_AFTER_SECONDS)},
+        )
+
+    @app.exception_handler(RequestValidationError)
+    def _validation_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
+        return JSONResponse(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+            content=_envelope(
+                "validation_error",
+                "request validation failed",
+                _request_id(request),
+                details={"errors": exc.errors()},
+            ),
+        )
+
+    @app.exception_handler(StarletteHTTPException)
+    def _http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
+        code = "not_found" if exc.status_code == status.HTTP_404_NOT_FOUND else "http_error"
+        return JSONResponse(
+            status_code=exc.status_code,
+            content=_envelope(code, str(exc.detail), _request_id(request)),
+        )
+
+    @app.exception_handler(Exception)
+    def _unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+        logger.exception("api.unhandled_exception", path=request.url.path)
+        return JSONResponse(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            content=_envelope(
+                "internal_error", "an unexpected error occurred", _request_id(request)
+            ),
+        )

src/api/middleware.py

@@ -0,0 +1,38 @@
+"""Per-request correlation id (ADR-0008, ADR-0011).
+
+Every request gets a `request_id`: reused from an incoming `X-Request-Id` if
+the caller supplied one, otherwise generated. It is bound into structlog's
+contextvars so every log line emitted while handling the request carries it,
+stored on `request.state` for exception handlers, and echoed back in the
+response header.
+"""
+
+import uuid
+from collections.abc import Awaitable, Callable
+from typing import override
+
+import structlog
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+_HEADER = "X-Request-Id"
+
+
+class RequestIdMiddleware(BaseHTTPMiddleware):
+    @override
+    async def dispatch(
+        self, request: Request, call_next: Callable[[Request], Awaitable[Response]]
+    ) -> Response:
+        request_id = request.headers.get(_HEADER) or str(uuid.uuid4())
+        request.state.request_id = request_id
+
+        structlog.contextvars.clear_contextvars()
+        structlog.contextvars.bind_contextvars(request_id=request_id)
+        try:
+            response = await call_next(request)
+        finally:
+            structlog.contextvars.clear_contextvars()
+
+        response.headers[_HEADER] = request_id
+        return response

src/api/router.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter
+
+from src.api.routers.domains import router as domains_router
+from src.api.routers.files import router as files_router
+from src.api.routers.points import router as points_router
+
+router = APIRouter()
+router.include_router(domains_router)
+router.include_router(files_router)
+router.include_router(points_router)

src/api/routers/domains.py

@@ -0,0 +1,110 @@
+"""`/v1/domains` (ADR-0008, ADR-0009).
+
+The management surface for a tenant's domain allowlist, used by the calling
+backend rather than by an operator with a psql prompt.
+
+Gated on `domains:read`/`domains:write`, deliberately **not** on `files:write`:
+if an upload key could create domains, the allowlist would no longer prevent a
+typo'd `domain` from creating a Qdrant partition, which is its only purpose.
+"""
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, status
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.api.dependencies.auth import require_scope
+from src.api.schemas.domains import (
+    CreateDomainRequest,
+    DomainListResponse,
+    DomainResponse,
+    UpdateDomainRequest,
+)
+from src.application.auth.context import AuthContext
+from src.application.domains import (
+    create_domain,
+    list_domains,
+    set_domain_status,
+    update_domain,
+)
+from src.bootstrap.dependencies import get_sessionmaker
+
+router = APIRouter(prefix="/domains", tags=["domains"])
+
+_RequireDomainsRead = Annotated[AuthContext, Depends(require_scope("domains:read"))]
+_RequireDomainsWrite = Annotated[AuthContext, Depends(require_scope("domains:write"))]
+_SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)]
+
+
+@router.get("")
+async def list_tenant_domains(
+    auth: _RequireDomainsRead,
+    sessionmaker: _SessionmakerDep,
+    include_disabled: bool = False,
+) -> DomainListResponse:
+    results = await list_domains(
+        sessionmaker, tenant_id=auth.tenant_id, include_disabled=include_disabled
+    )
+    return DomainListResponse(domains=[DomainResponse.from_result(item) for item in results])
+
+
+@router.post("", status_code=status.HTTP_201_CREATED)
+async def create_tenant_domain(
+    request: CreateDomainRequest,
+    auth: _RequireDomainsWrite,
+    sessionmaker: _SessionmakerDep,
+) -> DomainResponse:
+    result = await create_domain(
+        sessionmaker,
+        tenant_id=auth.tenant_id,
+        domain=request.domain,
+        display_name=request.display_name,
+        metadata=request.metadata,
+    )
+    return DomainResponse.from_result(result)
+
+
+@router.patch("/{domain}")
+async def update_tenant_domain(
+    domain: str,
+    request: UpdateDomainRequest,
+    auth: _RequireDomainsWrite,
+    sessionmaker: _SessionmakerDep,
+) -> DomainResponse:
+    result = await update_domain(
+        sessionmaker,
+        tenant_id=auth.tenant_id,
+        domain=domain,
+        display_name=request.display_name,
+    )
+    return DomainResponse.from_result(result)
+
+
+@router.delete("/{domain}")
+async def disable_tenant_domain(
+    domain: str,
+    auth: _RequireDomainsWrite,
+    sessionmaker: _SessionmakerDep,
+) -> DomainResponse:
+    """Disable, not delete.
+
+    Blocks new uploads and drops the domain from pickers while leaving the
+    points already indexed under it intact and retrievable. Actually removing
+    them needs the tenant-erasure workflow plan 001 defers.
+    """
+    result = await set_domain_status(
+        sessionmaker, tenant_id=auth.tenant_id, domain=domain, status="disabled"
+    )
+    return DomainResponse.from_result(result)
+
+
+@router.post("/{domain}/enable")
+async def enable_tenant_domain(
+    domain: str,
+    auth: _RequireDomainsWrite,
+    sessionmaker: _SessionmakerDep,
+) -> DomainResponse:
+    result = await set_domain_status(
+        sessionmaker, tenant_id=auth.tenant_id, domain=domain, status="active"
+    )
+    return DomainResponse.from_result(result)

src/api/routers/files.py

@@ -0,0 +1,157 @@
+"""`POST /v1/files`, `GET /v1/files/{file_id}`, `DELETE /v1/files/{file_id}` (ADR-0008).
+
+Routes adapt HTTP to `application/files` calls; they do not parse, hash,
+touch MinIO/Qdrant, or otherwise carry ingestion business logic (ADR-0015).
+"""
+
+import uuid
+from collections.abc import Sequence
+from typing import Annotated
+
+from anyio import CapacityLimiter, Semaphore
+from fastapi import APIRouter, Depends, Form, HTTPException, Response, UploadFile, status
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.api.dependencies.auth import require_scope
+from src.api.schemas.files import FileDeleteResponse, FileStatusResponse, FileUploadResponse
+from src.api.schemas.points import DEFAULT_PAGE_SIZE, LimitQuery, PointListResponse
+from src.application.auth.context import AuthContext
+from src.application.files.deletion import delete_source_file
+from src.application.files.status import get_file_status
+from src.application.files.upload import upload_source_file
+from src.application.points.queries import list_file_points
+from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
+from src.application.ports.object_storage import ObjectStorage
+from src.application.ports.point_repository import PointRepository
+from src.application.ports.point_storage import PointStorage
+from src.bootstrap.dependencies import (
+    get_dense_embedders,
+    get_ingestion_concurrency_limiter,
+    get_ingestion_limiter,
+    get_object_storage,
+    get_point_repository,
+    get_point_storage,
+    get_sessionmaker,
+    get_settings,
+    get_sparse_embedder,
+)
+from src.config import Settings
+
+router = APIRouter(prefix="/files", tags=["files"])
+
+_RequireFilesWrite = Annotated[AuthContext, Depends(require_scope("files:write"))]
+_RequirePointsRead = Annotated[AuthContext, Depends(require_scope("points:read"))]
+_RequirePointsWrite = Annotated[AuthContext, Depends(require_scope("points:write"))]
+_SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)]
+_ObjectStorageDep = Annotated[ObjectStorage, Depends(get_object_storage)]
+_PointStorageDep = Annotated[PointStorage, Depends(get_point_storage)]
+_PointRepositoryDep = Annotated[PointRepository, Depends(get_point_repository)]
+_SettingsDep = Annotated[Settings, Depends(get_settings)]
+_IngestionLimiterDep = Annotated[CapacityLimiter, Depends(get_ingestion_limiter)]
+_ConcurrencyLimiterDep = Annotated[Semaphore, Depends(get_ingestion_concurrency_limiter)]
+_DenseEmbeddersDep = Annotated[Sequence[DenseEmbedder], Depends(get_dense_embedders)]
+_SparseEmbedderDep = Annotated[SparseEmbedder, Depends(get_sparse_embedder)]
+
+
+@router.post("", status_code=status.HTTP_201_CREATED)
+async def upload_file(
+    response: Response,
+    file: UploadFile,
+    domain: Annotated[str, Form()],
+    auth: _RequireFilesWrite,
+    sessionmaker: _SessionmakerDep,
+    storage: _ObjectStorageDep,
+    point_storage: _PointStorageDep,
+    settings: _SettingsDep,
+    limiter: _IngestionLimiterDep,
+    concurrency_limiter: _ConcurrencyLimiterDep,
+    dense_embedders: _DenseEmbeddersDep,
+    sparse_embedder: _SparseEmbedderDep,
+) -> FileUploadResponse:
+    data = await file.read()
+    result = await upload_source_file(
+        sessionmaker=sessionmaker,
+        storage=storage,
+        point_storage=point_storage,
+        auth=auth,
+        domain=domain,
+        filename=file.filename or "",
+        data=data,
+        ingestion_settings=settings.ingestion,
+        chunking_settings=settings.chunking,
+        qdrant_settings=settings.qdrant,
+        thread_limiter=limiter,
+        concurrency_limiter=concurrency_limiter,
+        dense_embedders=dense_embedders,
+        sparse_embedder=sparse_embedder,
+    )
+    if not result.is_new_attempt:
+        response.status_code = status.HTTP_200_OK
+    return FileUploadResponse.from_result(result)
+
+
+@router.get("/{file_id}")
+async def get_file(
+    file_id: uuid.UUID,
+    auth: _RequireFilesWrite,
+    sessionmaker: _SessionmakerDep,
+) -> FileStatusResponse:
+    result = await get_file_status(sessionmaker, tenant_id=auth.tenant_id, source_file_id=file_id)
+    if result is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="file not found")
+    return FileStatusResponse.from_result(result)
+
+
+@router.delete("/{file_id}")
+async def delete_file(
+    file_id: uuid.UUID,
+    auth: _RequirePointsWrite,
+    sessionmaker: _SessionmakerDep,
+    repository: _PointRepositoryDep,
+) -> FileDeleteResponse:
+    """Soft-delete a file: every active point, then the `source_files` row.
+
+    Gated on `points:write` rather than `files:write` for the same reason as the
+    listing above — the data this destroys is points. Nothing is removed from
+    Qdrant (ADR-0002); the points are flagged inactive and the row is marked
+    `soft_deleted`, which is also what makes a later re-upload of the same bytes
+    ingest afresh instead of matching the duplicate path.
+
+    Deleting an already-deleted file is a success reporting `0` points.
+    """
+    points_soft_deleted = await delete_source_file(
+        sessionmaker,
+        repository,
+        tenant_id=auth.tenant_id,
+        source_file_id=file_id,
+        actor=f"api_key:{auth.api_key_id}",
+    )
+    return FileDeleteResponse(
+        file_id=file_id, status="soft_deleted", points_soft_deleted=points_soft_deleted
+    )
+
+
+@router.get("/{file_id}/points")
+async def list_points_for_file(
+    file_id: uuid.UUID,
+    auth: _RequirePointsRead,
+    repository: _PointRepositoryDep,
+    limit: LimitQuery = DEFAULT_PAGE_SIZE,
+    cursor: str | None = None,
+    include_inactive: bool = False,
+) -> PointListResponse:
+    """The same listing as `GET /v1/points?file_id=...`, addressed by file.
+
+    Gated on `points:read`, not `files:write`: the resource being read is the
+    file's chunks, so the scope follows the data rather than the URL prefix. An
+    upload-only key must not become a way to read every chunk of every file.
+    """
+    page = await list_file_points(
+        repository,
+        tenant_id=auth.tenant_id,
+        file_id=file_id,
+        limit=limit,
+        cursor=cursor,
+        include_inactive=include_inactive,
+    )
+    return PointListResponse.from_page(page)

src/api/routers/health.py

@@ -0,0 +1,38 @@
+import asyncio
+
+from fastapi import APIRouter, Request, Response, status
+
+from src.bootstrap.dependencies import AppResources
+from src.infrastructure.minio.client import ping as ping_minio
+from src.infrastructure.postgres.database import ping as ping_postgres
+from src.infrastructure.qdrant.client import ping as ping_qdrant
+
+router = APIRouter(tags=["health"])
+
+
+@router.get("/healthz")
+async def healthz() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@router.get("/readyz")
+async def readyz(request: Request, response: Response) -> dict[str, bool]:
+    resources: AppResources = request.app.state.resources
+    timeout = resources.settings.app.readiness_check_timeout_seconds
+
+    postgres_ready, minio_ready, qdrant_ready = await asyncio.gather(
+        ping_postgres(resources.db_engine, timeout),
+        ping_minio(resources.minio_client, timeout),
+        ping_qdrant(
+            resources.qdrant_client, timeout, collection=resources.settings.qdrant.collection
+        ),
+    )
+
+    result = {
+        "postgres": postgres_ready,
+        "minio": minio_ready,
+        "qdrant": qdrant_ready,
+    }
+    if not all(result.values()):
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+    return result

src/api/routers/points.py

@@ -0,0 +1,162 @@
+"""`/v1/points` read and soft-delete paths (ADR-0002, ADR-0008).
+
+Routes adapt HTTP to `application/points` calls. They build no Qdrant filters
+and hold no CRUD semantics (ADR-0015), and they never read a tenant from the
+request — `auth.tenant_id` is the only source, which is what makes ADR-0002's
+isolation rule structural rather than a habit.
+
+**Route order is load-bearing.** `/count` and `/search` are declared before
+`/{point_id}`. FastAPI matches in declaration order, so with `/{point_id}` first
+a request for `/v1/points/count` would try to parse `"count"` as a UUID and
+fail with `422` instead of counting anything. The failure is loud but confusing,
+and it comes back the moment someone reorders these for tidiness.
+
+Gated on `points:read`, separately from `files:write`: a key that can upload
+documents should not thereby be able to read every chunk of every file, and
+plan 002's mutating paths will want `points:write` distinct again.
+"""
+
+import uuid
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Query
+
+from src.api.dependencies.auth import require_scope
+from src.api.schemas.points import (
+    DEFAULT_PAGE_SIZE,
+    LimitQuery,
+    PointCountResponse,
+    PointListResponse,
+    PointResponse,
+    PointSearchResponse,
+)
+from src.application.auth.context import AuthContext
+from src.application.points.deletion import soft_delete_point
+from src.application.points.queries import (
+    count_points,
+    get_point,
+    list_file_points,
+    search_points,
+)
+from src.application.ports.point_repository import PointRepository
+from src.bootstrap.dependencies import get_point_repository
+
+router = APIRouter(prefix="/points", tags=["points"])
+
+_RequirePointsRead = Annotated[AuthContext, Depends(require_scope("points:read"))]
+_RequirePointsWrite = Annotated[AuthContext, Depends(require_scope("points:write"))]
+_PointRepositoryDep = Annotated[PointRepository, Depends(get_point_repository)]
+
+
+@router.get("/count")
+async def count_tenant_points(
+    auth: _RequirePointsRead,
+    repository: _PointRepositoryDep,
+    domain: str | None = None,
+    file_id: uuid.UUID | None = None,
+    include_inactive: bool = False,
+) -> PointCountResponse:
+    count = await count_points(
+        repository,
+        tenant_id=auth.tenant_id,
+        domain=domain,
+        file_id=file_id,
+        include_inactive=include_inactive,
+    )
+    return PointCountResponse(count=count)
+
+
+@router.get("/search")
+async def search_tenant_points(
+    auth: _RequirePointsRead,
+    repository: _PointRepositoryDep,
+    q: Annotated[str, Query(min_length=1)],
+    limit: LimitQuery = DEFAULT_PAGE_SIZE,
+    cursor: str | None = None,
+    domain: str | None = None,
+    file_id: uuid.UUID | None = None,
+    include_inactive: bool = False,
+) -> PointSearchResponse:
+    """Keyword search over point content — **not** semantic retrieval.
+
+    Matches Qdrant's full-text payload index on `content`, combined with the
+    structured filters below. Results are unranked: the index filters rather
+    than scores, so there is no relevance order and no score to return. Callers
+    wanting ranked answers want the agent retrieval path (plan 003), not this.
+    """
+    page = await search_points(
+        repository,
+        tenant_id=auth.tenant_id,
+        query=q,
+        limit=limit,
+        cursor=cursor,
+        domain=domain,
+        file_id=file_id,
+        include_inactive=include_inactive,
+    )
+    return PointSearchResponse.from_search(page, query=q)
+
+
+@router.get("")
+async def list_tenant_points(
+    auth: _RequirePointsRead,
+    repository: _PointRepositoryDep,
+    file_id: uuid.UUID,
+    limit: LimitQuery = DEFAULT_PAGE_SIZE,
+    cursor: str | None = None,
+    include_inactive: bool = False,
+) -> PointListResponse:
+    """A file's points in `order_id` order.
+
+    `file_id` is required rather than optional: the pagination cursor is an
+    `order_id` value, and `order_id` is only unique within one file. Listing
+    across files would silently drop or repeat rows at every page boundary.
+    """
+    page = await list_file_points(
+        repository,
+        tenant_id=auth.tenant_id,
+        file_id=file_id,
+        limit=limit,
+        cursor=cursor,
+        include_inactive=include_inactive,
+    )
+    return PointListResponse.from_page(page)
+
+
+@router.get("/{point_id}")
+async def get_tenant_point(
+    point_id: uuid.UUID,
+    auth: _RequirePointsRead,
+    repository: _PointRepositoryDep,
+    with_vectors: bool = False,
+) -> PointResponse:
+    point = await get_point(
+        repository, tenant_id=auth.tenant_id, point_id=point_id, with_vectors=with_vectors
+    )
+    return PointResponse.from_point(point)
+
+
+@router.delete("/{point_id}")
+async def delete_tenant_point(
+    point_id: uuid.UUID,
+    auth: _RequirePointsWrite,
+    repository: _PointRepositoryDep,
+) -> PointResponse:
+    """Soft-delete one point and relink its neighbours around the gap.
+
+    The point is never removed from Qdrant (ADR-0002): it is flagged
+    `is_active=false` with `deleted_at` set, and its old neighbours are pointed
+    at each other in the same batch, so context-window expansion never walks
+    into it.
+
+    Deleting an already-inactive point is a no-op success rather than a `404` —
+    the response is the point as it stands, so the resulting `version` and
+    `deleted_at` are visible either way.
+    """
+    point = await soft_delete_point(
+        repository,
+        tenant_id=auth.tenant_id,
+        point_id=point_id,
+        actor=f"api_key:{auth.api_key_id}",
+    )
+    return PointResponse.from_point(point)

src/api/schemas/domains.py

@@ -0,0 +1,63 @@
+"""Public request/response models for `/v1/domains` (ADR-0008, ADR-0009).
+
+`tenant_id` appears in none of these: it comes from the authenticated key, and
+accepting it from a body would break the isolation boundary (ADR-0002).
+"""
+
+import uuid
+from datetime import datetime
+
+from pydantic import BaseModel, Field, field_validator
+
+from src.application.domains.models import DomainResult
+
+# Lowercase alphanumerics plus - and _; the key is embedded in every Qdrant
+# payload and filtered on as a keyword, so it stays boring on purpose.
+_DOMAIN_PATTERN = r"^[a-z0-9][a-z0-9_-]*$"
+
+
+class DomainResponse(BaseModel):
+    id: uuid.UUID
+    domain: str
+    display_name: str
+    status: str
+    metadata: dict[str, object]
+    created_at: datetime
+    updated_at: datetime
+
+    @classmethod
+    def from_result(cls, result: DomainResult) -> "DomainResponse":
+        return cls(
+            id=result.id,
+            domain=result.domain,
+            display_name=result.display_name,
+            status=result.status,
+            metadata=result.metadata,
+            created_at=result.created_at,
+            updated_at=result.updated_at,
+        )
+
+
+class DomainListResponse(BaseModel):
+    domains: list[DomainResponse]
+
+
+class CreateDomainRequest(BaseModel):
+    domain: str = Field(min_length=1, max_length=80, pattern=_DOMAIN_PATTERN)
+    display_name: str = Field(min_length=1, max_length=200)
+    metadata: dict[str, object] = Field(default_factory=dict)
+
+    @field_validator("domain")
+    @classmethod
+    def _normalize(cls, value: str) -> str:
+        return value.strip()
+
+
+class UpdateDomainRequest(BaseModel):
+    """`domain` is absent by design — the key is immutable.
+
+    It is denormalized into every point payload and into `source_files`, so
+    renaming it is a migration rather than an edit (ADR-0009).
+    """
+
+    display_name: str = Field(min_length=1, max_length=200)

src/api/schemas/errors.py

@@ -0,0 +1,16 @@
+"""The ADR-0008 error envelope."""
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class ErrorDetail(BaseModel):
+    code: str
+    message: str
+    details: dict[str, Any] = {}
+    request_id: str | None = None
+
+
+class ErrorResponse(BaseModel):
+    error: ErrorDetail

src/api/schemas/files.py

@@ -0,0 +1,64 @@
+"""Public request/response models for `/v1/files` (ADR-0008).
+
+Separate from the SQLAlchemy ORM models and the `application/files` domain
+dataclasses (ADR-0015): this is the shape callers see.
+"""
+
+import uuid
+
+from pydantic import BaseModel
+
+from src.application.files.models import UploadResult
+from src.application.files.status import FileStatusResult
+
+
+class FileUploadResponse(BaseModel):
+    file_id: uuid.UUID
+    ingestion_job_id: uuid.UUID
+    status: str
+    chunks_indexed: int
+
+    @classmethod
+    def from_result(cls, result: UploadResult) -> "FileUploadResponse":
+        return cls(
+            file_id=result.file_id,
+            ingestion_job_id=result.ingestion_job_id,
+            status=result.status,
+            chunks_indexed=result.chunks_indexed,
+        )
+
+
+class FileDeleteResponse(BaseModel):
+    """What `DELETE /v1/files/{file_id}` did.
+
+    `points_soft_deleted` is reported rather than left implicit because the
+    delete is a soft one: nothing is removed from Qdrant, and the count is the
+    only way a caller can tell "deactivated 40 points" from "the file was
+    already deleted" — both of which are successes.
+    """
+
+    file_id: uuid.UUID
+    status: str
+    points_soft_deleted: int
+
+
+class FileStatusResponse(BaseModel):
+    file_id: uuid.UUID
+    source_filename: str
+    domain: str
+    status: str
+    ingestion_job_id: uuid.UUID | None
+    ingestion_status: str | None
+    chunks_indexed: int
+
+    @classmethod
+    def from_result(cls, result: FileStatusResult) -> "FileStatusResponse":
+        return cls(
+            file_id=result.file_id,
+            source_filename=result.source_filename,
+            domain=result.domain,
+            status=result.status,
+            ingestion_job_id=result.ingestion_job_id,
+            ingestion_status=result.ingestion_status,
+            chunks_indexed=result.chunks_indexed,
+        )

src/api/schemas/points.py

@@ -0,0 +1,165 @@
+"""Public request/response models for `/v1/points` (ADR-0002, ADR-0008).
+
+The shape callers see, kept separate from `application/points`' domain models
+(ADR-0015). Two rules are encoded here rather than left to route code:
+
+- **Vectors are opt-in.** `PointResponse` omits them unless the caller asked,
+  so a listing does not ship megabytes of floats nobody reads (ADR-0008).
+- **Server-owned fields are not accepted on input.** The request models simply
+  do not declare `tenant_id`, `version`, or `chunk_index`, and forbid extra
+  keys, so a client that sends one gets `422` from Pydantic instead of having
+  it silently ignored — ADR-0002's isolation rule enforced at the boundary.
+"""
+
+import uuid
+from datetime import datetime
+from typing import Annotated
+
+from fastapi import Query
+from pydantic import BaseModel, ConfigDict, Field
+
+from src.application.points.point import Point
+from src.application.ports.point_repository import PointPage
+
+# A page ceiling the caller cannot raise. Scroll pages are materialized in
+# memory both here and in Qdrant, so an unbounded `limit` is a cheap way for one
+# request to hurt every other tenant sharing the process. Declared once because
+# two routers paginate points -- `/v1/points` and `/v1/files/{file_id}/points` --
+# and a ceiling that differs between them is a ceiling in only one of them.
+DEFAULT_PAGE_SIZE = 50
+MAX_PAGE_SIZE = 200
+
+LimitQuery = Annotated[int, Query(ge=1, le=MAX_PAGE_SIZE)]
+
+
+class PointResponse(BaseModel):
+    point_id: uuid.UUID
+    domain: str
+    file_id: uuid.UUID
+    chunk_id: uuid.UUID
+
+    content: str
+    content_type: str
+    source_filename: str
+    source_type: str
+
+    order_id: float
+    chunk_index: int
+    previous_chunk_id: uuid.UUID | None
+    next_chunk_id: uuid.UUID | None
+
+    is_active: bool
+    deleted_at: datetime | None
+    created_at: datetime
+    updated_at: datetime
+    created_by: str
+    updated_by: str
+
+    version: int
+    content_hash: str
+    embedding_model_version: str
+
+    vectors: dict[str, object] | None = None
+
+    @classmethod
+    def from_point(cls, point: Point) -> "PointResponse":
+        # `tenant_id` is present on `Point` and deliberately absent here: the
+        # caller already knows which tenant it authenticated as, and echoing it
+        # back invites clients to start sending it.
+        return cls.model_validate(point.model_dump(exclude={"tenant_id"}))
+
+
+class PointListResponse(BaseModel):
+    """A page of points plus the cursor for the next one.
+
+    Cursor-based rather than `limit`/`offset`: an offset cursor silently skips
+    or repeats rows when a concurrent insert shifts positions, which is exactly
+    the pagination defect plan 002 requires a test for.
+    """
+
+    points: list[PointResponse]
+    next_cursor: str | None = None
+
+    @classmethod
+    def from_page(cls, page: PointPage) -> "PointListResponse":
+        return cls(
+            points=[PointResponse.from_point(point) for point in page.points],
+            next_cursor=page.next_cursor,
+        )
+
+
+class PointCountResponse(BaseModel):
+    count: int
+
+
+class PointSearchResponse(PointListResponse):
+    """Results of a **keyword** match, not of semantic retrieval.
+
+    Named and documented so it cannot be mistaken for ADR-0003's hybrid
+    retrieval: these points matched a full-text filter on `content`, they are
+    not ranked by relevance, and there is no score to report. Anything that
+    wants ranked results wants the agent retrieval path in plan 003.
+    """
+
+    query: str
+
+    @classmethod
+    def from_search(cls, page: PointPage, *, query: str) -> "PointSearchResponse":
+        return cls(
+            query=query,
+            points=[PointResponse.from_point(point) for point in page.points],
+            next_cursor=page.next_cursor,
+        )
+
+
+class PointCreateRequest(BaseModel):
+    """Create one point. The server assigns identity, ordering, and provenance.
+
+    `after_point_id` positions the new point rather than a raw `order_id`: the
+    caller says where in the sequence it goes and the server computes the
+    fractional key and relinks neighbours, which a client-supplied `order_id`
+    could not do correctly (ADR-0002). `None` means "at the start of the file".
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    file_id: uuid.UUID
+    content: str = Field(min_length=1)
+    content_type: str = "paragraph"
+    after_point_id: uuid.UUID | None = None
+
+
+class PointReplaceRequest(BaseModel):
+    """Replace a point's content under a version guard.
+
+    `version` here is the *expected* version, not a value being written — the
+    optimistic-concurrency precondition. A mismatch is `409`.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    content: str = Field(min_length=1)
+    content_type: str | None = None
+    version: int
+
+
+class PointPayloadPatchRequest(BaseModel):
+    """Payload-only update of caller-writable fields, under a version guard."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    payload: dict[str, object]
+    version: int
+
+
+class PointReorderRequest(BaseModel):
+    """Move a point to sit immediately after `after_point_id`.
+
+    `None` moves it to the front of the file. Expressed as a neighbour rather
+    than an `order_id` for the same reason as `PointCreateRequest`.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    after_point_id: uuid.UUID | None = None
+    version: int

src/application/auth/__init__.py

@@ -0,0 +1,29 @@
+"""API-key authentication and tenant resolution (ADR-0008).
+
+`resolve_auth_context` is the entry point: it takes a bearer token and
+returns a trusted `AuthContext`. Everything downstream of the FastAPI
+boundary receives `tenant_id` only through that context — never from a
+request body, query string, or object metadata.
+"""
+
+from src.application.auth.context import AuthContext
+from src.application.auth.errors import (
+    AuthError,
+    InvalidApiKeyError,
+    MissingScopeError,
+    TenantInactiveError,
+)
+from src.application.auth.keys import generate_api_key, hash_secret, verify_secret
+from src.application.auth.service import resolve_auth_context
+
+__all__ = [
+    "AuthContext",
+    "AuthError",
+    "InvalidApiKeyError",
+    "MissingScopeError",
+    "TenantInactiveError",
+    "generate_api_key",
+    "hash_secret",
+    "resolve_auth_context",
+    "verify_secret",
+]

src/application/auth/context.py

@@ -0,0 +1,16 @@
+"""The trusted request-scoped auth/tenant context (ADR-0008)."""
+
+import uuid
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class AuthContext:
+    tenant_id: uuid.UUID
+    tenant_slug: str
+    api_key_id: uuid.UUID
+    scopes: frozenset[str]
+    actor_type: str
+
+    def has_scope(self, scope: str) -> bool:
+        return scope in self.scopes or "admin" in self.scopes

src/application/auth/errors.py

@@ -0,0 +1,22 @@
+"""Auth failures (ADR-0008). No HTTP knowledge here — `src/api/errors.py` maps
+these to status codes.
+"""
+
+
+class AuthError(Exception):
+    """Base class for auth failures."""
+
+
+class InvalidApiKeyError(AuthError):
+    """The bearer token is missing, malformed, unknown, revoked, or expired.
+
+    Maps to `401`.
+    """
+
+
+class TenantInactiveError(AuthError):
+    """The key's tenant is suspended or deleted. Maps to `401`."""
+
+
+class MissingScopeError(AuthError):
+    """The key is valid but lacks a scope the route requires. Maps to `403`."""

src/application/auth/keys.py

@@ -0,0 +1,39 @@
+"""API-key generation and hashing (ADR-0008, ADR-0009).
+
+Keys are `sk_{prefix}_{secret}`. `prefix` is non-secret and indexed
+(`api_keys.key_prefix`); `secret` is 256 bits of `secrets.token_urlsafe`
+entropy, stored only as a SHA-256 hash. A random 256-bit secret does not
+benefit from a slow password-hashing KDF the way a human-chosen password
+does — the cost that defends against dictionary/brute-force guessing over a
+low-entropy input has nothing to defend here, and would only tax every
+request. Comparison is constant-time to avoid a hash-timing oracle.
+"""
+
+import hashlib
+import hmac
+import secrets
+
+_PREFIX_LENGTH = 16
+
+
+def generate_api_key() -> tuple[str, str, str]:
+    """Return `(key_prefix, secret, full_key)` for a newly issued key."""
+    key_prefix = secrets.token_hex(_PREFIX_LENGTH // 2)
+    secret = secrets.token_urlsafe(32)
+    return key_prefix, secret, f"sk_{key_prefix}_{secret}"
+
+
+def parse_api_key(full_key: str) -> tuple[str, str] | None:
+    """Return `(key_prefix, secret)`, or `None` if the token is malformed."""
+    parts = full_key.split("_", 2)
+    if len(parts) != 3 or parts[0] != "sk" or not parts[1] or not parts[2]:
+        return None
+    return parts[1], parts[2]
+
+
+def hash_secret(secret: str) -> str:
+    return hashlib.sha256(secret.encode("utf-8")).hexdigest()
+
+
+def verify_secret(secret: str, key_hash: str) -> bool:
+    return hmac.compare_digest(hash_secret(secret), key_hash)

src/application/auth/service.py

@@ -0,0 +1,86 @@
+"""Resolve a bearer token to a trusted `AuthContext` (ADR-0008).
+
+This opens and releases its own session rather than borrowing a
+request-scoped one, so auth resolution never pins a pool connection across
+the rest of the request — including the ADR-0017 ingestion work phase, which
+must run with no Postgres session held open at all.
+"""
+
+from datetime import UTC, datetime
+
+import structlog
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.application.auth.context import AuthContext
+from src.application.auth.errors import InvalidApiKeyError, TenantInactiveError
+from src.application.auth.keys import parse_api_key, verify_secret
+from src.infrastructure.postgres.repositories import api_keys as api_keys_repo
+from src.infrastructure.postgres.repositories import tenants as tenants_repo
+
+logger = structlog.get_logger(__name__)
+
+
+async def resolve_auth_context(
+    sessionmaker: async_sessionmaker[AsyncSession], bearer_token: str
+) -> AuthContext:
+    """Resolve a bearer token, logging the outcome either way (ADR-0011).
+
+    This runs on every authenticated request, so `auth.failed` is the one
+    event most likely to matter first when diagnosing a client integration
+    issue -- and the reason string alone (never logged; it can echo back
+    attacker-supplied key material) is not enough to tell a malformed token
+    apart from a revoked one without this.
+    """
+    parsed = parse_api_key(bearer_token)
+    if parsed is None:
+        logger.warning("auth.failed", reason="malformed_key")
+        raise InvalidApiKeyError("malformed API key")
+    key_prefix, secret = parsed
+
+    async with sessionmaker() as session:
+        api_key = await api_keys_repo.get_by_prefix(session, key_prefix)
+        if api_key is None or not verify_secret(secret, api_key.key_hash):
+            logger.warning("auth.failed", reason="unknown_key", key_prefix=key_prefix)
+            raise InvalidApiKeyError("unknown API key")
+        if api_key.status != "active":
+            logger.warning(
+                "auth.failed",
+                reason="key_inactive",
+                key_prefix=key_prefix,
+                api_key_id=str(api_key.id),
+                key_status=api_key.status,
+            )
+            raise InvalidApiKeyError(f"API key is {api_key.status}")
+        if api_key.expires_at is not None and api_key.expires_at <= datetime.now(UTC):
+            logger.warning(
+                "auth.failed",
+                reason="key_expired",
+                key_prefix=key_prefix,
+                api_key_id=str(api_key.id),
+            )
+            raise InvalidApiKeyError("API key has expired")
+
+        tenant = await tenants_repo.get_by_id(session, api_key.tenant_id)
+        if tenant is None or tenant.status != "active":
+            logger.warning(
+                "auth.failed",
+                reason="tenant_inactive",
+                key_prefix=key_prefix,
+                api_key_id=str(api_key.id),
+                tenant_id=str(api_key.tenant_id),
+            )
+            raise TenantInactiveError("tenant is not active")
+
+        logger.info(
+            "auth.succeeded",
+            tenant_id=str(tenant.id),
+            api_key_id=str(api_key.id),
+            actor_type=api_key.actor_type,
+        )
+        return AuthContext(
+            tenant_id=tenant.id,
+            tenant_slug=tenant.slug,
+            api_key_id=api_key.id,
+            scopes=frozenset(api_key.scopes),
+            actor_type=api_key.actor_type,
+        )

src/application/domains/__init__.py

@@ -0,0 +1,27 @@
+"""Tenant-domain management and the upload-time allowlist check (ADR-0009)."""
+
+from src.application.domains.errors import (
+    DomainAlreadyExistsError,
+    DomainsError,
+    UnknownDomainError,
+)
+from src.application.domains.models import DomainResult
+from src.application.domains.service import (
+    create_domain,
+    ensure_domain_allowed,
+    list_domains,
+    set_domain_status,
+    update_domain,
+)
+
+__all__ = [
+    "DomainAlreadyExistsError",
+    "DomainResult",
+    "DomainsError",
+    "UnknownDomainError",
+    "create_domain",
+    "ensure_domain_allowed",
+    "list_domains",
+    "set_domain_status",
+    "update_domain",
+]

src/application/domains/errors.py

@@ -0,0 +1,21 @@
+"""Domain-management failures (ADR-0009). No HTTP knowledge here —
+`src/api/errors.py` maps these to status codes.
+"""
+
+
+class DomainsError(Exception):
+    """Base class for tenant-domain failures."""
+
+
+class UnknownDomainError(DomainsError):
+    """The upload named a domain the tenant has not registered, or one that is
+    disabled. Maps to `400`.
+
+    Rejecting is the whole point: an unrecognized `domain` would otherwise
+    create a new Qdrant partition silently, and a file in a partition nothing
+    queries is invisible rather than failed (ADR-0009).
+    """
+
+
+class DomainAlreadyExistsError(DomainsError):
+    """The tenant already has a domain with this key. Maps to `409`."""

src/application/domains/models.py

@@ -0,0 +1,16 @@
+"""Transport-agnostic results for the domain-management service."""
+
+import uuid
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass(frozen=True)
+class DomainResult:
+    id: uuid.UUID
+    domain: str
+    display_name: str
+    status: str
+    metadata: dict[str, object]
+    created_at: datetime
+    updated_at: datetime

src/application/domains/service.py

@@ -0,0 +1,164 @@
+"""Tenant-domain management (ADR-0009).
+
+A tenant's domain set is per-tenant and varies in size — one may run 14
+insurance lines, another 6 — so it is data, not an enum.
+
+`ensure_domain_allowed` is the reason this package exists: it is the strict
+allowlist check the upload path runs before anything is written. Everything
+else here is the management surface the calling backend uses to populate that
+allowlist, under its own `domains:write` scope so an upload key cannot create
+partitions.
+
+`tenant_id` is always a required parameter taken from `AuthContext`, never from
+a request body (ADR-0002).
+"""
+
+import uuid
+
+import structlog
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.application.domains.errors import DomainAlreadyExistsError, UnknownDomainError
+from src.application.domains.models import DomainResult
+from src.infrastructure.postgres.models.tenant_domain import TenantDomain
+from src.infrastructure.postgres.repositories import tenant_domains as repo
+
+logger = structlog.get_logger(__name__)
+
+
+async def _flush_and_refresh(session: AsyncSession, tenant_domain: TenantDomain) -> None:
+    """Materialize server-generated columns before the row leaves the session.
+
+    `updated_at` is `onupdate=func.now()`, so after an UPDATE its value lives in
+    the database, not in the instance. Reading it later would trigger a lazy
+    load outside any greenlet context (`MissingGreenlet`), so it is fetched here
+    while the session is still open.
+    """
+    await session.flush()
+    await session.refresh(tenant_domain)
+
+
+def _to_result(tenant_domain: TenantDomain) -> DomainResult:
+    return DomainResult(
+        id=tenant_domain.id,
+        domain=tenant_domain.domain,
+        display_name=tenant_domain.display_name,
+        status=tenant_domain.status,
+        metadata=tenant_domain.metadata_,
+        created_at=tenant_domain.created_at,
+        updated_at=tenant_domain.updated_at,
+    )
+
+
+async def ensure_domain_allowed(
+    session: AsyncSession, *, tenant_id: uuid.UUID, domain: str
+) -> None:
+    """Raise `UnknownDomainError` unless the tenant has this domain active.
+
+    Takes a session rather than a sessionmaker: the upload path calls this
+    inside its existing txn A, so the check costs no extra connection and
+    cannot pass and then go stale before the row is written.
+
+    Logs the rejection here rather than at the call site: this runs before any
+    `ingestion_jobs` row exists, so `upload_source_file`'s job-level
+    `ingestion.job.failed` event (ADR-0011) never fires for it -- without a log
+    here, a rejected upload would leave no operational trace at all.
+    """
+    tenant_domain = await repo.get(session, tenant_id=tenant_id, domain=domain)
+    if tenant_domain is None:
+        logger.warning(
+            "domain.rejected", tenant_id=str(tenant_id), domain=domain, reason="unregistered"
+        )
+        raise UnknownDomainError(
+            f"domain '{domain}' is not registered for this tenant; "
+            f"create it via POST /v1/domains before uploading to it"
+        )
+    if tenant_domain.status != "active":
+        logger.warning(
+            "domain.rejected", tenant_id=str(tenant_id), domain=domain, reason="disabled"
+        )
+        raise UnknownDomainError(f"domain '{domain}' is disabled for this tenant")
+
+
+async def list_domains(
+    sessionmaker: async_sessionmaker[AsyncSession],
+    *,
+    tenant_id: uuid.UUID,
+    include_disabled: bool = False,
+) -> list[DomainResult]:
+    async with sessionmaker() as session:
+        found = await repo.list_for_tenant(
+            session, tenant_id=tenant_id, include_disabled=include_disabled
+        )
+    return [_to_result(item) for item in found]
+
+
+async def create_domain(
+    sessionmaker: async_sessionmaker[AsyncSession],
+    *,
+    tenant_id: uuid.UUID,
+    domain: str,
+    display_name: str,
+    metadata: dict[str, object] | None = None,
+) -> DomainResult:
+    async with sessionmaker() as session:
+        if await repo.get(session, tenant_id=tenant_id, domain=domain) is not None:
+            raise DomainAlreadyExistsError(f"domain '{domain}' already exists for this tenant")
+        created = repo.create(
+            session,
+            tenant_id=tenant_id,
+            domain=domain,
+            display_name=display_name,
+            metadata=metadata,
+        )
+        await session.commit()
+
+    logger.info("domain.created", tenant_id=str(tenant_id), domain=domain)
+    return _to_result(created)
+
+
+async def update_domain(
+    sessionmaker: async_sessionmaker[AsyncSession],
+    *,
+    tenant_id: uuid.UUID,
+    domain: str,
+    display_name: str,
+) -> DomainResult:
+    """Only the label is mutable — see `repo.update_display_name`."""
+    async with sessionmaker() as session:
+        found = await repo.get(session, tenant_id=tenant_id, domain=domain)
+        if found is None:
+            raise UnknownDomainError(f"domain '{domain}' is not registered for this tenant")
+        repo.update_display_name(found, display_name=display_name)
+        await _flush_and_refresh(session, found)
+        await session.commit()
+        result = _to_result(found)
+
+    logger.info("domain.updated", tenant_id=str(tenant_id), domain=domain)
+    return result
+
+
+async def set_domain_status(
+    sessionmaker: async_sessionmaker[AsyncSession],
+    *,
+    tenant_id: uuid.UUID,
+    domain: str,
+    status: str,
+) -> DomainResult:
+    """Disable or re-enable a domain.
+
+    Disabling blocks new uploads and hides the domain from pickers. It does not
+    touch the points already indexed under it — removing those needs the
+    tenant-erasure workflow plan 001 defers.
+    """
+    async with sessionmaker() as session:
+        found = await repo.get(session, tenant_id=tenant_id, domain=domain)
+        if found is None:
+            raise UnknownDomainError(f"domain '{domain}' is not registered for this tenant")
+        repo.set_status(found, status=status)
+        await _flush_and_refresh(session, found)
+        await session.commit()
+        result = _to_result(found)
+
+    logger.info("domain.status_changed", tenant_id=str(tenant_id), domain=domain, status=status)
+    return result

src/application/files/__init__.py

@@ -0,0 +1,19 @@
+"""Source-file upload and status use cases (ADR-0008, ADR-0009, ADR-0017)."""
+
+from src.application.files.errors import FilesError, FileTooLargeError, InvalidUploadError
+from src.application.files.models import UploadResult, ValidatedUpload
+from src.application.files.status import FileStatusResult, get_file_status
+from src.application.files.upload import upload_source_file
+from src.application.files.validation import validate_and_hash_upload
+
+__all__ = [
+    "FileStatusResult",
+    "FileTooLargeError",
+    "FilesError",
+    "InvalidUploadError",
+    "UploadResult",
+    "ValidatedUpload",
+    "get_file_status",
+    "upload_source_file",
+    "validate_and_hash_upload",
+]

src/application/files/deletion.py

@@ -0,0 +1,81 @@
+"""`DELETE /v1/files/{file_id}` — retire a file and deactivate its points.
+
+Two stores have to agree here, and the phase boundaries are the same ones
+ingestion uses (ADR-0017): a short Postgres transaction to authorize, then the
+Qdrant work with **no session held**, then a short transaction to record the
+outcome. Holding a session across the sweep would pin a pool connection for the
+length of a multi-page delete.
+
+The order — points first, Postgres second — is deliberate. If the sweep dies
+half way, the row stays `active` and a retried `DELETE` finishes the job, since
+the sweep only ever looks at points that are still active. The reverse order
+would leave a row marked deleted while its points are still live and still
+retrievable by the agent, which is the failure that actually matters.
+"""
+
+import uuid
+from datetime import UTC, datetime
+from time import perf_counter
+
+import structlog
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.application.files.errors import SourceFileNotFoundError
+from src.application.points.deletion import soft_delete_file_points
+from src.application.ports.point_repository import PointRepository
+from src.infrastructure.postgres.repositories import source_files as source_files_repo
+
+logger = structlog.get_logger(__name__)
+
+
+async def delete_source_file(
+    sessionmaker: async_sessionmaker[AsyncSession],
+    repository: PointRepository,
+    *,
+    tenant_id: uuid.UUID,
+    source_file_id: uuid.UUID,
+    actor: str,
+) -> int:
+    """Soft-delete a file: every active point, then the `source_files` row.
+
+    Returns how many points the sweep deactivated. Raises
+    `SourceFileNotFoundError` (`404`) when the file is not this tenant's — the
+    check happens before anything is written, so a probe for another tenant's
+    file id cannot deactivate a single point.
+
+    Idempotent: a second call finds no active points and a row already marked
+    `soft_deleted`, and returns `0`.
+    """
+    started = perf_counter()
+    async with sessionmaker() as session:
+        source_file = await source_files_repo.get_by_id(
+            session, tenant_id=tenant_id, source_file_id=source_file_id
+        )
+        if source_file is None:
+            raise SourceFileNotFoundError(f"file {source_file_id} not found")
+
+    points_soft_deleted = await soft_delete_file_points(
+        repository, tenant_id=tenant_id, file_id=source_file_id, actor=actor
+    )
+
+    async with sessionmaker() as session:
+        source_file = await source_files_repo.get_by_id(
+            session, tenant_id=tenant_id, source_file_id=source_file_id
+        )
+        if source_file is None:
+            raise SourceFileNotFoundError(f"file {source_file_id} not found")
+        source_files_repo.mark_soft_deleted(source_file, deleted_at=datetime.now(UTC))
+        await session.commit()
+
+    logger.info(
+        "files.soft_deleted",
+        tenant_id=str(tenant_id),
+        file_id=str(source_file_id),
+        points_soft_deleted=points_soft_deleted,
+        actor=actor,
+        # End to end, including both Postgres transactions. Comparing it with
+        # the sweep's own `duration_ms` on `points.file_soft_deleted` is what
+        # separates a slow Qdrant from a slow database.
+        duration_ms=round((perf_counter() - started) * 1000, 2),
+    )
+    return points_soft_deleted

src/application/files/errors.py

@@ -0,0 +1,25 @@
+"""Upload-validation failures (ADR-0008). No HTTP knowledge here —
+`src/api/errors.py` maps these to status codes.
+"""
+
+
+class FilesError(Exception):
+    """Base class for file-upload failures."""
+
+
+class InvalidUploadError(FilesError):
+    """Missing domain, empty file, or content that doesn't match its
+    declared extension. Maps to `400`.
+    """
+
+
+class FileTooLargeError(FilesError):
+    """The upload exceeds `INGESTION_MAX_UPLOAD_SIZE_MB`. Maps to `413`."""
+
+
+class SourceFileNotFoundError(FilesError):
+    """No such source file *within the requesting tenant*. Maps to `404`.
+
+    Same non-disclosure rule as points (ADR-0016): a cross-tenant file id and a
+    nonexistent one are indistinguishable to the caller, so this is never `403`.
+    """

src/application/files/models.py

@@ -0,0 +1,26 @@
+"""Domain models for the upload use case (ADR-0008, ADR-0009)."""
+
+import uuid
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ValidatedUpload:
+    """The result of extension/content validation, before any I/O."""
+
+    source_type: str
+    content_type: str
+    content_sha256: str
+
+
+@dataclass(frozen=True)
+class UploadResult:
+    """What `upload_source_file` returns; the route maps this to `FileUploadResponse`."""
+
+    file_id: uuid.UUID
+    ingestion_job_id: uuid.UUID
+    status: str
+    chunks_indexed: int
+    is_new_attempt: bool
+    """`False` when an identical active upload already succeeded and no new
+    ingestion attempt was made (route returns `200`, not `201`)."""

src/application/files/status.py

@@ -0,0 +1,52 @@
+"""`GET /v1/files/{file_id}` read model (ADR-0008, ADR-0009)."""
+
+import uuid
+from dataclasses import dataclass
+
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.infrastructure.postgres.repositories import ingestion_jobs as jobs_repo
+from src.infrastructure.postgres.repositories import source_files as source_files_repo
+
+
+@dataclass(frozen=True)
+class FileStatusResult:
+    file_id: uuid.UUID
+    source_filename: str
+    domain: str
+    status: str
+    ingestion_job_id: uuid.UUID | None
+    ingestion_status: str | None
+    chunks_indexed: int
+
+
+async def get_file_status(
+    sessionmaker: async_sessionmaker[AsyncSession],
+    *,
+    tenant_id: uuid.UUID,
+    source_file_id: uuid.UUID,
+) -> FileStatusResult | None:
+    """Returns `None` when the file doesn't exist under this tenant — the
+    route maps that to `404`, never `403` (ADR-0016: cross-tenant access
+    returns 404).
+    """
+    async with sessionmaker() as session:
+        source_file = await source_files_repo.get_by_id(
+            session, tenant_id=tenant_id, source_file_id=source_file_id
+        )
+        if source_file is None:
+            return None
+
+        latest_job = await jobs_repo.get_latest_for_source_file(
+            session, tenant_id=tenant_id, source_file_id=source_file.id
+        )
+
+        return FileStatusResult(
+            file_id=source_file.id,
+            source_filename=source_file.source_filename,
+            domain=source_file.domain,
+            status=source_file.status,
+            ingestion_job_id=latest_job.id if latest_job else None,
+            ingestion_status=latest_job.status if latest_job else None,
+            chunks_indexed=latest_job.points_created if latest_job else 0,
+        )

src/application/files/storage_keys.py

@@ -0,0 +1,11 @@
+"""Object-storage key derivation (ADR-0013).
+
+Object keys are internal identifiers, never the caller-supplied filename.
+Pure and synchronous.
+"""
+
+import uuid
+
+
+def source_file_object_key(tenant_id: uuid.UUID, source_file_id: uuid.UUID) -> str:
+    return f"tenants/{tenant_id}/source-files/{source_file_id}/original"

src/application/files/upload.py

@@ -0,0 +1,366 @@
+"""`POST /v1/files` orchestration: the ADR-0017 three-phase upload.
+
+This service owns two separate short-lived sessions/transactions rather than
+one request-scoped session, because the request is two units of work
+(ADR-0012, ADR-0017):
+
+    txn A (short):  source_files [+ ingestion_jobs(status='running')], commit
+    no txn:          store bytes in MinIO, parse/chunk (threads),
+                     embed dense+sparse (bounded/batched)
+    txn B (short):  ingestion_jobs -> succeeded/failed, append event, commit
+
+No Postgres session is open during phase 2. A failure at any point between
+txn A and txn B still leaves a durable, inspectable `failed` job — never a
+job stuck in `running`. The whole request additionally holds one of
+`INGESTION_MAX_CONCURRENCY` process-wide slots (`503` when exhausted) and
+phase 2 is bounded by `INGESTION_TIMEOUT_SECONDS` (`504`) (ADR-0017, plan 001
+Phase 4).
+
+Phase 2 ends by upserting the embedded chunks as tenant-scoped Qdrant points
+(`src/application/points/`), so a successful upload is searchable by the time
+the `201` returns. The collection those points land in is provisioned by a
+deployment step, not by this path — see `src/cli/qdrant_bootstrap.py`.
+"""
+
+import uuid
+from collections.abc import Sequence
+
+import structlog
+from anyio import CapacityLimiter, Semaphore, fail_after, to_thread
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.application.auth.context import AuthContext
+from src.application.domains import ensure_domain_allowed
+from src.application.files.errors import InvalidUploadError
+from src.application.files.models import UploadResult
+from src.application.files.storage_keys import source_file_object_key
+from src.application.files.validation import validate_and_hash_upload
+from src.application.ingestion import (
+    ChunkTooLargeError,
+    DocumentParseError,
+    UnsupportedSourceTypeError,
+    parse_and_chunk_document,
+)
+from src.application.ingestion.bounds import acquire_ingestion_slot, enforce_chunk_limit
+from src.application.ingestion.embedding import embed_chunks
+from src.application.ingestion.errors import (
+    ChunkLimitExceededError,
+    EmbedderError,
+    IngestionTimeoutError,
+    PointIndexingError,
+)
+from src.application.points import index_chunks
+from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
+from src.application.ports.object_storage import ObjectStorage
+from src.application.ports.point_storage import PointStorage
+from src.config import ChunkingSettings, IngestionSettings, QdrantSettings
+from src.infrastructure.postgres.repositories import ingestion_jobs as jobs_repo
+from src.infrastructure.postgres.repositories import source_files as source_files_repo
+
+logger = structlog.get_logger(__name__)
+
+
+async def _mark_job_failed(
+    sessionmaker: async_sessionmaker[AsyncSession],
+    *,
+    tenant_id: uuid.UUID,
+    ingestion_job_id: uuid.UUID,
+    error_code: str,
+    error_message: str,
+) -> None:
+    """Write the terminal `failed` job row and emit its log event together.
+
+    Every failure branch below calls this, so logging here once closes every
+    branch at once rather than duplicating a `logger.warning` at each call
+    site (CLAUDE.md, "prefer deep modules") -- previously only
+    `storage_upload_failed` and `timeout` did that ad hoc, and
+    `parse_failed`/`chunk_limit_exceeded`/`embedding_failed`/`index_failed`
+    logged nothing at all: visible in `ingestion_job_events` but invisible to
+    log-based alerting (ADR-0011).
+    """
+    async with sessionmaker() as session:
+        job = await jobs_repo.mark_terminal(
+            session,
+            tenant_id=tenant_id,
+            ingestion_job_id=ingestion_job_id,
+            status="failed",
+            error_code=error_code,
+            error_message=error_message,
+        )
+        if job is not None:
+            jobs_repo.append_event(
+                session,
+                tenant_id=tenant_id,
+                ingestion_job_id=ingestion_job_id,
+                level="error",
+                stage="received",
+                message=error_message,
+            )
+        await session.commit()
+
+    logger.warning(
+        "ingestion.job.failed",
+        tenant_id=str(tenant_id),
+        ingestion_job_id=str(ingestion_job_id),
+        error_code=error_code,
+        error_message=error_message,
+    )
+
+
+async def upload_source_file(
+    *,
+    sessionmaker: async_sessionmaker[AsyncSession],
+    storage: ObjectStorage,
+    point_storage: PointStorage,
+    auth: AuthContext,
+    domain: str,
+    filename: str,
+    data: bytes,
+    ingestion_settings: IngestionSettings,
+    chunking_settings: ChunkingSettings,
+    qdrant_settings: QdrantSettings,
+    thread_limiter: CapacityLimiter,
+    concurrency_limiter: Semaphore,
+    dense_embedders: Sequence[DenseEmbedder],
+    sparse_embedder: SparseEmbedder,
+) -> UploadResult:
+    domain = domain.strip()
+    if not domain:
+        raise InvalidUploadError("domain is required")
+
+    validated = await to_thread.run_sync(
+        lambda: validate_and_hash_upload(
+            filename=filename, data=data, max_size_bytes=ingestion_settings.max_upload_size_bytes
+        ),
+        limiter=thread_limiter,
+    )
+
+    async with acquire_ingestion_slot(concurrency_limiter):
+        async with sessionmaker() as session:
+            # Strict allowlist, checked inside txn A before anything is written
+            # (ADR-0009). An unregistered domain would otherwise create a new
+            # Qdrant partition silently, leaving the file invisible to
+            # retrieval rather than failing.
+            await ensure_domain_allowed(session, tenant_id=auth.tenant_id, domain=domain)
+
+            existing = await source_files_repo.find_active_by_content_hash(
+                session,
+                tenant_id=auth.tenant_id,
+                domain=domain,
+                content_sha256=validated.content_sha256,
+            )
+
+            if existing is not None:
+                latest_job = await jobs_repo.get_latest_for_source_file(
+                    session, tenant_id=auth.tenant_id, source_file_id=existing.id
+                )
+                if latest_job is not None and latest_job.status == "succeeded":
+                    logger.info(
+                        "files.upload.duplicate",
+                        tenant_id=str(auth.tenant_id),
+                        file_id=str(existing.id),
+                    )
+                    return UploadResult(
+                        file_id=existing.id,
+                        ingestion_job_id=latest_job.id,
+                        status=latest_job.status,
+                        chunks_indexed=latest_job.points_created,
+                        is_new_attempt=False,
+                    )
+                source_file_id = existing.id
+                object_key = existing.storage_uri or source_file_object_key(
+                    auth.tenant_id, source_file_id
+                )
+            else:
+                source_file_id = uuid.uuid4()
+                object_key = source_file_object_key(auth.tenant_id, source_file_id)
+                source_files_repo.create(
+                    session,
+                    source_file_id=source_file_id,
+                    tenant_id=auth.tenant_id,
+                    domain=domain,
+                    source_filename=filename,
+                    source_type=validated.source_type,
+                    content_sha256=validated.content_sha256,
+                    byte_size=len(data),
+                    storage_uri=object_key,
+                    created_by_api_key_id=auth.api_key_id,
+                )
+                # `ingestion_jobs.source_file_id` FKs to this row; flush so the
+                # insert below sees it, since the two mapped classes carry no
+                # ORM relationship for the unit of work to order by itself.
+                await session.flush()
+
+            job = jobs_repo.create_running(
+                session,
+                tenant_id=auth.tenant_id,
+                source_file_id=source_file_id,
+                requested_by_api_key_id=auth.api_key_id,
+                chunking_strategy=chunking_settings.strategy,
+            )
+            jobs_repo.append_event(
+                session,
+                tenant_id=auth.tenant_id,
+                ingestion_job_id=job.id,
+                level="info",
+                stage="received",
+                message="upload accepted, storing object",
+            )
+            await session.commit()
+            ingestion_job_id = job.id
+
+        logger.info(
+            "ingestion.job.started",
+            tenant_id=str(auth.tenant_id),
+            ingestion_job_id=str(ingestion_job_id),
+            file_id=str(source_file_id),
+            domain=domain,
+            source_type=validated.source_type,
+        )
+
+        # Phase 2: no Postgres session open across this work (ADR-0017),
+        # bounded end-to-end by INGESTION_TIMEOUT_SECONDS.
+        try:
+            with fail_after(ingestion_settings.timeout_seconds):
+                try:
+                    await storage.put_object(
+                        key=object_key, data=data, content_type=validated.content_type
+                    )
+                except Exception as exc:
+                    await _mark_job_failed(
+                        sessionmaker,
+                        tenant_id=auth.tenant_id,
+                        ingestion_job_id=ingestion_job_id,
+                        error_code="storage_upload_failed",
+                        error_message=f"failed to store object: {exc}",
+                    )
+                    raise
+
+                try:
+                    chunks = await parse_and_chunk_document(
+                        data,
+                        source_type=validated.source_type,
+                        file_id=source_file_id,
+                        settings=chunking_settings,
+                        limiter=thread_limiter,
+                    )
+                except (DocumentParseError, UnsupportedSourceTypeError, ChunkTooLargeError) as exc:
+                    await _mark_job_failed(
+                        sessionmaker,
+                        tenant_id=auth.tenant_id,
+                        ingestion_job_id=ingestion_job_id,
+                        error_code="parse_failed",
+                        error_message=str(exc),
+                    )
+                    raise
+
+                try:
+                    enforce_chunk_limit(chunks, max_chunks=ingestion_settings.max_chunks_per_file)
+                except ChunkLimitExceededError as exc:
+                    await _mark_job_failed(
+                        sessionmaker,
+                        tenant_id=auth.tenant_id,
+                        ingestion_job_id=ingestion_job_id,
+                        error_code="chunk_limit_exceeded",
+                        error_message=str(exc),
+                    )
+                    raise
+
+                try:
+                    embedded = await embed_chunks(
+                        chunks,
+                        dense_embedders=dense_embedders,
+                        sparse_embedder=sparse_embedder,
+                        settings=ingestion_settings,
+                        thread_limiter=thread_limiter,
+                    )
+                except EmbedderError as exc:
+                    await _mark_job_failed(
+                        sessionmaker,
+                        tenant_id=auth.tenant_id,
+                        ingestion_job_id=ingestion_job_id,
+                        error_code="embedding_failed",
+                        error_message=str(exc),
+                    )
+                    raise
+
+                try:
+                    indexed = await index_chunks(
+                        embedded,
+                        storage=point_storage,
+                        tenant_id=auth.tenant_id,
+                        domain=domain,
+                        file_id=source_file_id,
+                        source_filename=filename,
+                        source_type=validated.source_type,
+                        actor=f"api_key:{auth.api_key_id}",
+                        dense_embedders=dense_embedders,
+                        sparse_embedder=sparse_embedder,
+                        settings=qdrant_settings,
+                        thread_limiter=thread_limiter,
+                    )
+                except PointIndexingError as exc:
+                    await _mark_job_failed(
+                        sessionmaker,
+                        tenant_id=auth.tenant_id,
+                        ingestion_job_id=ingestion_job_id,
+                        error_code="index_failed",
+                        error_message=str(exc),
+                    )
+                    raise
+        except TimeoutError:
+            await _mark_job_failed(
+                sessionmaker,
+                tenant_id=auth.tenant_id,
+                ingestion_job_id=ingestion_job_id,
+                error_code="timeout",
+                error_message=f"ingestion exceeded {ingestion_settings.timeout_seconds}s",
+            )
+            raise IngestionTimeoutError(
+                f"ingestion exceeded {ingestion_settings.timeout_seconds}s"
+            ) from None
+
+        async with sessionmaker() as session:
+            await jobs_repo.mark_terminal(
+                session,
+                tenant_id=auth.tenant_id,
+                ingestion_job_id=ingestion_job_id,
+                status="succeeded",
+                # An upsert with deterministic ids cannot tell an insert from
+                # an overwrite, so every written point is reported here and
+                # `points_updated` stays 0 rather than being guessed at.
+                points_created=indexed.points_upserted,
+                points_soft_deleted=indexed.points_soft_deleted,
+            )
+            jobs_repo.append_event(
+                session,
+                tenant_id=auth.tenant_id,
+                ingestion_job_id=ingestion_job_id,
+                level="info",
+                stage="completed",
+                message="chunks parsed, embedded, and indexed",
+                details={
+                    "chunks_parsed": len(chunks),
+                    "chunks_embedded": len(embedded),
+                    "points_upserted": indexed.points_upserted,
+                    "points_soft_deleted": indexed.points_soft_deleted,
+                },
+            )
+            await session.commit()
+
+    logger.info(
+        "ingestion.job.completed",
+        tenant_id=str(auth.tenant_id),
+        ingestion_job_id=str(ingestion_job_id),
+        file_id=str(source_file_id),
+        chunks_parsed=len(chunks),
+        points_upserted=indexed.points_upserted,
+        points_soft_deleted=indexed.points_soft_deleted,
+    )
+    return UploadResult(
+        file_id=source_file_id,
+        ingestion_job_id=ingestion_job_id,
+        status="succeeded",
+        chunks_indexed=indexed.points_upserted,
+        is_new_attempt=True,
+    )

src/application/files/validation.py

@@ -0,0 +1,55 @@
+"""Upload extension/content-type/size validation (ADR-0008).
+
+Pure and synchronous: no I/O. `content_sha256` computation lives here too —
+hashing is blocking CPU work (ADR-0017), so the caller runs this whole
+function through `anyio.to_thread.run_sync` with the ingestion
+`CapacityLimiter`, the same rule applied to parsing/chunking.
+"""
+
+import hashlib
+
+from src.application.files.errors import FileTooLargeError, InvalidUploadError
+from src.application.files.models import ValidatedUpload
+from src.application.ingestion.errors import UnsupportedSourceTypeError
+
+_CONTENT_TYPES = {
+    "csv": "text/csv",
+    "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+}
+_OOXML_MAGIC = b"PK\x03\x04"
+
+
+def _source_type_from_filename(filename: str) -> str:
+    suffix = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    if suffix == "doc":
+        raise UnsupportedSourceTypeError(
+            "legacy .doc is not ingestible until an out-of-process conversion "
+            "service exists (ADR-0018)"
+        )
+    if suffix not in _CONTENT_TYPES:
+        raise UnsupportedSourceTypeError(f"'.{suffix}' is not an ingestible file type")
+    return suffix
+
+
+def validate_and_hash_upload(*, filename: str, data: bytes, max_size_bytes: int) -> ValidatedUpload:
+    source_type = _source_type_from_filename(filename)
+
+    if not data:
+        raise InvalidUploadError("uploaded file is empty")
+    if len(data) > max_size_bytes:
+        raise FileTooLargeError(
+            f"upload is {len(data)} bytes, over the {max_size_bytes}-byte limit"
+        )
+
+    is_ooxml = data[:4] == _OOXML_MAGIC
+    if source_type in ("docx", "xlsx") and not is_ooxml:
+        raise InvalidUploadError(f"content does not match the declared .{source_type} extension")
+    if source_type == "csv" and is_ooxml:
+        raise InvalidUploadError("content does not match the declared .csv extension")
+
+    return ValidatedUpload(
+        source_type=source_type,
+        content_type=_CONTENT_TYPES[source_type],
+        content_sha256=hashlib.sha256(data).hexdigest(),
+    )

src/application/ingestion/__init__.py

@@ -0,0 +1,51 @@
+"""Document parsing and fixed-size chunking (ADR-0004, ADR-0018).
+
+`parse_and_chunk_document` is the entry point callers outside this package
+should use: it dispatches on source type and owns the
+`anyio.to_thread.run_sync` + `CapacityLimiter` offload required by ADR-0017.
+The individual parsers and `chunk_document` are pure, synchronous, and
+exported mainly for their own unit tests — calling them directly from an
+`async def` route or service is the defect ADR-0017 warns about.
+"""
+
+from src.application.ingestion.chunking import chunk_document, chunk_id_for, split_by_tokens
+from src.application.ingestion.docx_parser import parse_docx
+from src.application.ingestion.errors import (
+    ChunkLimitExceededError,
+    ChunkTooLargeError,
+    DocumentParseError,
+    IngestionError,
+    UnsupportedSourceTypeError,
+)
+from src.application.ingestion.models import (
+    Chunk,
+    ContentType,
+    ParsedDocument,
+    StructuralUnit,
+)
+from src.application.ingestion.normalization import normalize_persian_text
+from src.application.ingestion.pipeline import parse_and_chunk_document
+from src.application.ingestion.spreadsheet_parser import parse_csv, parse_xlsx
+from src.application.ingestion.tokenizer import count_tokens, get_encoder
+
+__all__ = [
+    "Chunk",
+    "ChunkLimitExceededError",
+    "ChunkTooLargeError",
+    "ContentType",
+    "DocumentParseError",
+    "IngestionError",
+    "ParsedDocument",
+    "StructuralUnit",
+    "UnsupportedSourceTypeError",
+    "chunk_document",
+    "chunk_id_for",
+    "count_tokens",
+    "get_encoder",
+    "normalize_persian_text",
+    "parse_and_chunk_document",
+    "parse_csv",
+    "parse_docx",
+    "parse_xlsx",
+    "split_by_tokens",
+]

src/application/ingestion/bounds.py

@@ -0,0 +1,48 @@
+"""Request bounds for inline ingestion (ADR-0017).
+
+Three independent bounds, each mapping to its own status code: the chunk
+ceiling (`413`, checked before embedding starts), process-wide concurrency
+(`503` + `Retry-After`, rejected rather than queued), and the work-phase
+deadline (`504`, and the caller must still write a terminal job status).
+"""
+
+from collections.abc import AsyncIterator, Sequence
+from contextlib import asynccontextmanager
+
+from anyio import Semaphore, WouldBlock
+
+from src.application.ingestion.errors import ChunkLimitExceededError, IngestionAtCapacityError
+from src.application.ingestion.models import Chunk
+
+
+def enforce_chunk_limit(chunks: Sequence[Chunk], *, max_chunks: int) -> None:
+    """Raise `ChunkLimitExceededError` if `chunks` exceeds `max_chunks`.
+
+    Call this immediately after parsing/chunking and before any embedding
+    call — the ceiling must be discovered up front, not mid-batch.
+    """
+    if len(chunks) > max_chunks:
+        raise ChunkLimitExceededError(
+            f"document produced {len(chunks)} chunks, over the {max_chunks}-chunk limit"
+        )
+
+
+@asynccontextmanager
+async def acquire_ingestion_slot(limiter: Semaphore) -> AsyncIterator[None]:
+    """Hold one of `INGESTION_MAX_CONCURRENCY` process-wide slots for the block.
+
+    `limiter` is an `anyio.Semaphore` created once in the lifespan. Rejects
+    immediately with `IngestionAtCapacityError` when the process is already at
+    capacity, rather than queueing the request behind an unbounded wait
+    (ADR-0017) — the semaphore's own async `acquire()` would do the latter.
+    """
+    try:
+        limiter.acquire_nowait()
+    except WouldBlock:
+        raise IngestionAtCapacityError(
+            "ingestion is at capacity; retry after the configured backoff"
+        ) from None
+    try:
+        yield
+    finally:
+        limiter.release()

src/application/ingestion/chunking.py

@@ -0,0 +1,129 @@
+"""Fixed-size chunking with overlap (ADR-0018).
+
+DOCX markdown is split on token windows; spreadsheet rows are already atomic
+and bypass the splitter, falling through it only when a single row exceeds the
+embedding model's sequence length.
+"""
+
+import uuid
+
+from src.application.ingestion.errors import ChunkTooLargeError, DocumentParseError
+from src.application.ingestion.models import Chunk, ContentType, ParsedDocument
+from src.application.ingestion.tokenizer import count_tokens, get_encoder
+from src.config import ChunkingSettings
+
+# Fixed namespace so chunk ids stay stable across processes and releases.
+CHUNK_ID_NAMESPACE = uuid.UUID("6f9619ff-8b86-d011-b42d-00c04fc964ff")
+
+
+def chunk_id_for(file_id: uuid.UUID, chunk_index: int) -> uuid.UUID:
+    """Return the deterministic point id for a chunk (ADR-0001).
+
+    Derived from `file_id` and the immutable ingestion ordinal, so re-ingesting
+    a file upserts its points instead of duplicating them.
+    """
+    return uuid.uuid5(CHUNK_ID_NAMESPACE, f"{file_id}:{chunk_index}")
+
+
+def split_by_tokens(text: str, *, chunk_size: int, overlap: int, encoding_name: str) -> list[str]:
+    """Split text into overlapping windows of at most `chunk_size` tokens."""
+    if overlap >= chunk_size:
+        raise ValueError(f"overlap ({overlap}) must be smaller than chunk_size ({chunk_size})")
+
+    encoder = get_encoder(encoding_name)
+    tokens = encoder.encode(text)
+    if len(tokens) <= chunk_size:
+        return [text]
+
+    windows: list[str] = []
+    start = 0
+    while start < len(tokens):
+        end = min(start + chunk_size, len(tokens))
+        windows.append(encoder.decode(tokens[start:end]))
+        if end >= len(tokens):
+            break
+        start = end - overlap
+
+    return windows
+
+
+def _split_oversized(text: str, settings: ChunkingSettings) -> list[str]:
+    """Split a row only if it exceeds the cap; otherwise keep it atomic."""
+    if count_tokens(text, settings.encoding_name) <= settings.max_chunk_tokens:
+        return [text]
+    return split_by_tokens(
+        text,
+        chunk_size=settings.chunk_size,
+        overlap=settings.chunk_overlap,
+        encoding_name=settings.encoding_name,
+    )
+
+
+def _content_units(
+    parsed: ParsedDocument, settings: ChunkingSettings
+) -> list[tuple[str, ContentType]]:
+    """Reduce a parsed document's structural units to ordered text pieces.
+
+    Prose is cut into token windows; a table row is atomic and survives whole
+    unless it alone exceeds the model's sequence length (ADR-0004).
+    """
+    if not parsed.units:
+        raise DocumentParseError("Parsed document has no structural units")
+
+    pieces: list[tuple[str, ContentType]] = []
+    for unit in parsed.units:
+        if unit.content_type is ContentType.PARAGRAPH:
+            windows = split_by_tokens(
+                unit.text,
+                chunk_size=settings.chunk_size,
+                overlap=settings.chunk_overlap,
+                encoding_name=settings.encoding_name,
+            )
+        else:
+            windows = _split_oversized(unit.text, settings)
+        pieces.extend((window, unit.content_type) for window in windows)
+
+    return pieces
+
+
+def chunk_document(
+    parsed: ParsedDocument,
+    *,
+    file_id: uuid.UUID,
+    settings: ChunkingSettings,
+) -> list[Chunk]:
+    """Turn a parsed document into ordered, neighbor-linked chunks."""
+    units = [
+        (text.strip(), content_type) for text, content_type in _content_units(parsed, settings)
+    ]
+    # Drop blanks *before* assigning indices: an index gap would break the
+    # previous/next chain that retrieval-time expansion walks.
+    units = [(text, content_type) for text, content_type in units if text]
+
+    chunks: list[Chunk] = []
+    for index, (text, content_type) in enumerate(units):
+        token_count = count_tokens(text, settings.encoding_name)
+        if token_count > settings.max_chunk_tokens:
+            raise ChunkTooLargeError(
+                f"chunk {index} is {token_count} tokens, over the "
+                f"{settings.max_chunk_tokens}-token cap"
+            )
+        chunks.append(
+            Chunk(
+                chunk_id=chunk_id_for(file_id, index),
+                chunk_index=index,
+                order_id=float(index + 1),
+                content=text,
+                content_type=content_type,
+                token_count=token_count,
+                character_count=len(text),
+            )
+        )
+
+    for position, chunk in enumerate(chunks):
+        if position > 0:
+            chunk.previous_chunk_id = chunks[position - 1].chunk_id
+        if position < len(chunks) - 1:
+            chunk.next_chunk_id = chunks[position + 1].chunk_id
+
+    return chunks

src/application/ingestion/docx_parser.py

@@ -0,0 +1,165 @@
+"""DOCX parsing into ordered structural units (ADR-0004, ADR-0018).
+
+The body is walked in document order and decomposed into structural units
+before any chunking runs: runs of flowing prose become `PARAGRAPH` units the
+splitter cuts into token windows, and data-table rows become atomic
+`TABLE_ROW` units.
+
+The one classification this makes is between a *data* table and a table used
+as page layout, and it is made structurally rather than by inspecting content:
+a data cell fits inside a chunk by definition, so a table holding a cell that
+alone exceeds `chunk_size`, or a cell containing nested tables, is a layout
+container whose cells are prose.
+"""
+
+import io
+
+from docx import Document
+from docx.document import Document as DocxDocument
+from docx.oxml.table import CT_Tbl
+from docx.oxml.text.paragraph import CT_P
+from docx.table import Table, _Cell
+from docx.text.paragraph import Paragraph
+
+from src.application.ingestion.errors import DocumentParseError
+from src.application.ingestion.models import (
+    ContentType,
+    ParsedDocument,
+    StructuralUnit,
+)
+from src.application.ingestion.normalization import normalize_persian_text
+from src.application.ingestion.tabular import clean_rows, render_rows
+from src.application.ingestion.tokenizer import count_tokens
+from src.config import ChunkingSettings
+
+_NORMAL_STYLE = "Normal"
+
+
+def heading_level_from_style(style_name: str) -> int | None:
+    """Return the heading level of a paragraph style, or None for body text.
+
+    Word stores the styleId (`Heading1`), not the friendly name (`Heading 1`),
+    so both spellings must resolve. Only real Word styles count -- no heading
+    is ever inferred from the text itself (ADR-0018).
+    """
+    normalized = style_name.strip().lower().replace(" ", "")
+    if not normalized.startswith("heading"):
+        return None
+    suffix = normalized.removeprefix("heading")
+    return int(suffix) if suffix.isdigit() else None
+
+
+def _paragraph_style(paragraph: Paragraph) -> str:
+    style = paragraph.style.name if paragraph.style is not None else None
+    return style or _NORMAL_STYLE
+
+
+def _is_layout_table(table: Table, settings: ChunkingSettings) -> bool:
+    """Whether a table is page layout rather than data.
+
+    Structural, not a content heuristic: a data cell is small enough to be a
+    chunk, so a cell that alone overflows `chunk_size` -- or that nests another
+    table -- holds a document, not a field. In the sample corpus this separates
+    by two orders of magnitude (32-171 tokens for data tables against 54,007
+    for a cell containing a whole sub-document).
+    """
+    for row in table.rows:
+        for cell in row.cells:
+            if cell.tables:
+                return True
+            if count_tokens(cell.text, settings.encoding_name) > settings.chunk_size:
+                return True
+    return False
+
+
+def _cell_prose(cell: _Cell, settings: ChunkingSettings) -> list[StructuralUnit]:
+    """Extract a layout cell's contents as units, recursing into nested tables."""
+    units: list[StructuralUnit] = []
+    for block in _iter_block_items(cell, settings):
+        units.append(block)
+    return units
+
+
+def _table_units(table: Table, settings: ChunkingSettings) -> list[StructuralUnit]:
+    """Convert a table to structural units."""
+    if _is_layout_table(table, settings):
+        units: list[StructuralUnit] = []
+        # Walk the physical `w:tc` elements rather than `row.cells`, which
+        # repeats a merged cell once per grid position it spans. Identity
+        # tracking is not an option here: lxml builds element proxies on
+        # demand, so `id()` is neither stable nor unique across them.
+        for row in table.rows:
+            for tc in row._tr.tc_lst:
+                units.extend(_cell_prose(_Cell(tc, table), settings))
+        return units
+
+    rows = clean_rows([[cell.text for cell in row.cells] for row in table.rows])
+    return [
+        StructuralUnit(text=text, content_type=ContentType.TABLE_ROW) for text in render_rows(rows)
+    ]
+
+
+def _iter_block_items(
+    container: DocxDocument | _Cell, settings: ChunkingSettings
+) -> list[StructuralUnit]:
+    """Walk a body or cell in document order, emitting structural units.
+
+    Consecutive paragraphs accumulate into one prose unit rather than becoming
+    one unit each: ADR-0004 treats "the whole remaining run of paragraphs" as a
+    single prose block, so the splitter sees flowing text instead of a series
+    of one-sentence fragments.
+    """
+    element = container.element.body if isinstance(container, DocxDocument) else container._tc
+
+    units: list[StructuralUnit] = []
+    prose: list[str] = []
+
+    def flush() -> None:
+        if prose:
+            units.append(
+                StructuralUnit(text="\n\n".join(prose), content_type=ContentType.PARAGRAPH)
+            )
+            prose.clear()
+
+    for child in element:
+        if isinstance(child, CT_P):
+            paragraph = Paragraph(child, container)
+            # `Paragraph.text` includes hyperlink text, which a raw `w:r` walk
+            # silently drops.
+            text = normalize_persian_text(paragraph.text)
+            if not text:
+                continue
+            level = heading_level_from_style(_paragraph_style(paragraph))
+            prose.append(f"{'#' * level} {text}" if level else text)
+        elif isinstance(child, CT_Tbl):
+            table_units = _table_units(Table(child, container), settings)
+            # A layout table is prose; keep it in the surrounding prose block
+            # instead of fragmenting the document around it.
+            if table_units and all(
+                unit.content_type is ContentType.PARAGRAPH for unit in table_units
+            ):
+                prose.extend(unit.text for unit in table_units)
+            else:
+                flush()
+                units.extend(table_units)
+
+    flush()
+    return units
+
+
+def parse_docx(data: bytes, settings: ChunkingSettings) -> ParsedDocument:
+    """Parse DOCX bytes into ordered structural units."""
+    try:
+        doc = Document(io.BytesIO(data))
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open DOCX: {exc}") from exc
+
+    units = _iter_block_items(doc, settings)
+    if not units:
+        raise DocumentParseError("Document contains no text content")
+
+    return ParsedDocument(
+        units=units,
+        markdown="\n\n".join(unit.text for unit in units),
+        block_count=len(units),
+    )

src/application/ingestion/embedding.py

@@ -0,0 +1,112 @@
+"""The one caller-facing entry point for embedding chunks (ADR-0001, ADR-0017).
+
+`embed_chunks` is the only version of this step callers should reach for: it
+owns batching, the `embed_concurrency` semaphore bounding in-flight dense
+batches, and the `anyio.to_thread.run_sync` + `CapacityLimiter` offload for
+the blocking BM25 pipeline. Composing these correctly at every call site is
+exactly the obligation a deep module absorbs once (see CLAUDE.md's "prefer
+deep modules").
+
+Per-provider text shaping — task prefixes, `keep_alive`, request payload —
+belongs to the adapters in `src/infrastructure/embedding/`, not here. This
+module knows only that an embedder turns texts into vectors.
+"""
+
+import asyncio
+from collections.abc import Sequence
+from functools import partial
+
+from anyio import CapacityLimiter, to_thread
+
+from src.application.ingestion.errors import EmbedderError
+from src.application.ingestion.models import Chunk, EmbeddedChunk
+from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
+from src.config import IngestionSettings
+
+
+def _batches(texts: Sequence[str], size: int) -> list[Sequence[str]]:
+    return [texts[i : i + size] for i in range(0, len(texts), size)]
+
+
+async def _embed_dense_bounded(
+    embedder: DenseEmbedder,
+    batch: Sequence[str],
+    *,
+    semaphore: asyncio.Semaphore,
+) -> list[list[float]]:
+    async with semaphore:
+        try:
+            return await embedder.embed_batch(batch)
+        except Exception as exc:
+            raise EmbedderError(f"{embedder.name} embedding batch failed: {exc}") from exc
+
+
+async def _embed_dense_all(
+    embedder: DenseEmbedder,
+    texts: Sequence[str],
+    *,
+    batch_size: int,
+    semaphore: asyncio.Semaphore,
+) -> list[list[float]]:
+    batches = _batches(texts, batch_size)
+    results = await asyncio.gather(
+        *(_embed_dense_bounded(embedder, batch, semaphore=semaphore) for batch in batches)
+    )
+    return [vector for batch_result in results for vector in batch_result]
+
+
+def _embed_sparse_sync(embedder: SparseEmbedder, texts: Sequence[str]):
+    try:
+        return embedder.embed_batch(texts)
+    except Exception as exc:
+        raise EmbedderError(f"{embedder.name} embedding batch failed: {exc}") from exc
+
+
+async def embed_chunks(
+    chunks: Sequence[Chunk],
+    *,
+    dense_embedders: Sequence[DenseEmbedder],
+    sparse_embedder: SparseEmbedder,
+    settings: IngestionSettings,
+    thread_limiter: CapacityLimiter,
+) -> list[EmbeddedChunk]:
+    """Embed every chunk into all dense vectors plus the sparse vector.
+
+    Dense embedders run concurrently with each other; each one's batches are
+    concurrent among themselves too, bounded by one `embed_concurrency`
+    semaphore shared across all dense embedders (ADR-0017: the limit exists
+    for both providers' rate limits and the self-hosted server's capacity —
+    not a per-provider budget). The sparse (BM25) pass is blocking and runs
+    once, off the event loop.
+
+    Raises `EmbedderError` (502) if any embedder call fails.
+    """
+    if not chunks:
+        return []
+
+    texts = [chunk.content for chunk in chunks]
+    semaphore = asyncio.Semaphore(settings.embed_concurrency)
+
+    dense_task = asyncio.gather(
+        *(
+            _embed_dense_all(
+                embedder, texts, batch_size=settings.embed_batch_size, semaphore=semaphore
+            )
+            for embedder in dense_embedders
+        )
+    )
+    sparse_task = to_thread.run_sync(
+        partial(_embed_sparse_sync, sparse_embedder, texts), limiter=thread_limiter
+    )
+    dense_results, sparse_vectors = await asyncio.gather(dense_task, sparse_task)
+
+    dense_by_name = {
+        embedder.name: vectors
+        for embedder, vectors in zip(dense_embedders, dense_results, strict=True)
+    }
+
+    embedded: list[EmbeddedChunk] = []
+    for index, chunk in enumerate(chunks):
+        dense = {name: vectors[index] for name, vectors in dense_by_name.items()}
+        embedded.append(EmbeddedChunk(chunk=chunk, dense=dense, sparse=sparse_vectors[index]))
+    return embedded

src/application/ingestion/errors.py

@@ -0,0 +1,73 @@
+"""Errors raised by the parsing and chunking pipeline (ADR-0018).
+
+These carry no HTTP knowledge — the API layer maps them to status codes
+(ADR-0015: `application/` contains no FastAPI request objects).
+"""
+
+
+class IngestionError(Exception):
+    """Base class for ingestion failures."""
+
+
+class DocumentParseError(IngestionError):
+    """A source file could not be decoded, opened, or yielded no text.
+
+    Maps to `400` per ADR-0017 ("unparseable file → 400").
+    """
+
+
+class UnsupportedSourceTypeError(IngestionError):
+    """A source file's type is not ingestible in this version.
+
+    Maps to `415`. `.doc` lands here until an out-of-process conversion
+    service exists (ADR-0018).
+    """
+
+
+class ChunkLimitExceededError(IngestionError):
+    """A document produced more chunks than `INGESTION_MAX_CHUNKS_PER_FILE`.
+
+    Maps to `413` per ADR-0017.
+    """
+
+
+class ChunkTooLargeError(IngestionError):
+    """A chunk exceeded the embedding model's sequence length.
+
+    This is an internal invariant violation, not a user error: the splitter is
+    supposed to make it impossible. It exists because the failure it guards
+    against is silent — `nomic-embed-text-v2-moe` truncates over-long input
+    without raising (ADR-0004).
+    """
+
+
+class EmbedderError(IngestionError):
+    """A dense or sparse embedder call failed (transport error, non-2xx, or
+    a malformed response).
+
+    Maps to `502` per ADR-0017.
+    """
+
+
+class PointIndexingError(IngestionError):
+    """Upserting or soft-deleting Qdrant points failed.
+
+    Maps to `502` — like `EmbedderError`, this is an upstream dependency
+    failing, not a malformed request. Kept distinct from `EmbedderError` so the
+    job's `error_code` says which dependency broke.
+    """
+
+
+class IngestionAtCapacityError(IngestionError):
+    """`INGESTION_MAX_CONCURRENCY` in-process ingestions are already running.
+
+    Maps to `503` with `Retry-After`, not a queued wait (ADR-0017).
+    """
+
+
+class IngestionTimeoutError(IngestionError):
+    """The work phase (parse/embed/upsert) exceeded `INGESTION_TIMEOUT_SECONDS`.
+
+    Maps to `504`. The caller must still write a terminal `failed` job status
+    before this propagates (ADR-0017).
+    """

src/application/ingestion/models.py

@@ -0,0 +1,91 @@
+"""Domain models for parsing and chunking (ADR-0001, ADR-0004, ADR-0018)."""
+
+import uuid
+from enum import StrEnum
+
+from pydantic import BaseModel, Field
+
+
+class ContentType(StrEnum):
+    """ADR-0004's finalized `content_type` value set.
+
+    v1 emits `PARAGRAPH` and `TABLE_ROW` only; `QA_PAIR` and `IMAGE_CAPTION`
+    are defined but not produced yet (ADR-0018).
+    """
+
+    PARAGRAPH = "paragraph"
+    TABLE_ROW = "table_row"
+    QA_PAIR = "qa_pair"
+    IMAGE_CAPTION = "image_caption"
+
+
+class StructuralUnit(BaseModel):
+    """One structural unit of a document, in reading order (ADR-0004).
+
+    A document is decomposed into these *before* any chunking runs, because
+    the two kinds are chunked differently:
+
+    - `PARAGRAPH` is a run of flowing prose; the fixed-size splitter cuts it
+      into token windows.
+    - `TABLE_ROW` is already atomic; it becomes one chunk, and is split only
+      when a single row is too large for the embedding model.
+    """
+
+    text: str
+    content_type: ContentType
+
+
+class ParsedDocument(BaseModel):
+    """The output of a parser, before chunking.
+
+    `units` is the content, in document order. `markdown` is the same content
+    rendered as one string, for eyeballing a parse; nothing chunks from it.
+    """
+
+    units: list[StructuralUnit] = Field(default_factory=list)
+    markdown: str | None = None
+    block_count: int = 0
+
+
+class Chunk(BaseModel):
+    """One indexable unit of a document.
+
+    `chunk_id` is a deterministic UUIDv5 of `file_id` and `chunk_index`
+    (ADR-0001), so re-ingesting a file upserts its points rather than
+    duplicating them.
+    """
+
+    chunk_id: uuid.UUID
+    chunk_index: int
+    order_id: float
+    content: str
+    content_type: ContentType
+    previous_chunk_id: uuid.UUID | None = None
+    next_chunk_id: uuid.UUID | None = None
+    token_count: int
+    character_count: int
+
+
+class SparseVector(BaseModel):
+    """A sparse (term-index -> weight) vector, Qdrant's `modifier="idf"` shape.
+
+    Kept free of the `qdrant_client` SDK (ADR-0015: ports carry no infra
+    imports) — `src/infrastructure/qdrant/` converts this to the SDK's own
+    `SparseVector` type at upsert time (Phase 5).
+    """
+
+    indices: list[int]
+    values: list[float]
+
+
+class EmbeddedChunk(BaseModel):
+    """A chunk plus every vector it will be upserted with (ADR-0001).
+
+    `dense` is keyed by named-vector name (`dense_nomic`, `dense_openai`).
+    `late_interaction` is deliberately absent — not computed at ingest
+    (ADR-0017).
+    """
+
+    chunk: Chunk
+    dense: dict[str, list[float]]
+    sparse: SparseVector

src/application/ingestion/normalization.py

@@ -0,0 +1,73 @@
+"""Persian text normalization (ADR-0018).
+
+Applied to every extracted text block before chunking, for all source formats.
+
+The problem this solves is silent: Persian authored on mixed Arabic/Persian
+keyboards contains both U+06A9 and U+0643 for "k", both U+06CC and U+064A for
+"y". Those are distinct codepoints and therefore distinct tokens to every
+embedding model, so the same word embeds two different ways depending on which
+key the author pressed.
+
+Letter folding only -- digits and punctuation are left as authored, because
+chunk content is what citations render back to the reader and Western digits
+inside Persian prose read as wrong.
+"""
+
+import re
+import unicodedata
+
+# Both tables are written as codepoints rather than character literals. Arabic
+# letterforms are visually indistinguishable from one another (and alef from a
+# Latin "l") in a monospace editor -- which is the very confusion this module
+# exists to resolve -- and literals would render right-to-left, visually
+# reordering the source line.
+#
+# `str.translate` accepts an ordinal->ordinal mapping directly, and an ordinal
+# mapped to None is deleted.
+
+_ARABIC_KAF = 0x0643
+_ARABIC_YEH = 0x064A
+_ALEF_MAKSURA = 0x0649
+_ALEF_HAMZA_ABOVE = 0x0623
+_ALEF_HAMZA_BELOW = 0x0625
+_NOT_SIGN = 0x00AC
+
+_PERSIAN_KEHEH = 0x06A9
+_PERSIAN_YEH = 0x06CC
+_ALEF = 0x0627
+_SPACE = 0x0020
+
+_LETTER_FOLDING: dict[int, int] = {
+    _ARABIC_KAF: _PERSIAN_KEHEH,
+    _ARABIC_YEH: _PERSIAN_YEH,
+    _ALEF_MAKSURA: _PERSIAN_YEH,
+    _ALEF_HAMZA_ABOVE: _ALEF,
+    _ALEF_HAMZA_BELOW: _ALEF,
+    # A soft-hyphen artifact from documents exported by older Word versions.
+    _NOT_SIGN: _SPACE,
+}
+
+_TATWEEL = 0x0640
+_SUPERSCRIPT_ALEF = 0x0670
+_HARAKAT = range(0x064B, 0x0660)
+
+# Applied after NFKC, which can itself decompose presentation forms into a
+# base letter plus a combining mark.
+_MARK_REMOVAL: dict[int, int | None] = dict.fromkeys(_HARAKAT)
+_MARK_REMOVAL[_TATWEEL] = None
+_MARK_REMOVAL[_SUPERSCRIPT_ALEF] = None
+
+_WHITESPACE = re.compile(r"\s+")
+
+
+def normalize_persian_text(text: str) -> str:
+    """Fold Arabic letterforms to Persian and collapse whitespace.
+
+    Call this per text block, **before** blocks are assembled into a document.
+    The whitespace collapse maps `\\n` to a space, so running it over assembled
+    markdown would flatten every heading and paragraph onto one line.
+    """
+    text = text.translate(_LETTER_FOLDING)
+    text = unicodedata.normalize("NFKC", text)
+    text = text.translate(_MARK_REMOVAL)
+    return _WHITESPACE.sub(" ", text).strip()

src/application/ingestion/pipeline.py

@@ -0,0 +1,65 @@
+"""The one caller-facing entry point for parsing and chunking (ADR-0017).
+
+`parse_docx`/`parse_csv`/`parse_xlsx`/`chunk_document` are blocking, pure
+functions; calling any of them directly from an `async def` route or service
+is the defect ADR-0017 names explicitly ("one large `python-docx` parse would
+stall every concurrent request"). `parse_and_chunk_document` is the only
+version of this pipeline callers should reach for: it owns source-type
+dispatch and the `anyio.to_thread.run_sync` + `CapacityLimiter` offload, so
+that obligation cannot be forgotten at a call site.
+"""
+
+import uuid
+from functools import partial
+
+from anyio import CapacityLimiter, to_thread
+
+from src.application.ingestion.chunking import chunk_document
+from src.application.ingestion.docx_parser import parse_docx
+from src.application.ingestion.errors import UnsupportedSourceTypeError
+from src.application.ingestion.models import Chunk, ParsedDocument
+from src.application.ingestion.spreadsheet_parser import parse_csv, parse_xlsx
+from src.config import ChunkingSettings
+
+_PARSERS = {"csv", "xlsx", "docx"}
+
+
+def _parse(data: bytes, source_type: str, settings: ChunkingSettings) -> ParsedDocument:
+    if source_type == "docx":
+        return parse_docx(data, settings)
+    if source_type == "xlsx":
+        return parse_xlsx(data)
+    if source_type == "csv":
+        return parse_csv(data)
+    raise UnsupportedSourceTypeError(f"'{source_type}' is not an ingestible source type")
+
+
+def _parse_and_chunk(
+    data: bytes, source_type: str, file_id: uuid.UUID, settings: ChunkingSettings
+) -> list[Chunk]:
+    parsed = _parse(data, source_type, settings)
+    return chunk_document(parsed, file_id=file_id, settings=settings)
+
+
+async def parse_and_chunk_document(
+    data: bytes,
+    *,
+    source_type: str,
+    file_id: uuid.UUID,
+    settings: ChunkingSettings,
+    limiter: CapacityLimiter,
+) -> list[Chunk]:
+    """Parse and chunk a document off the event loop, bounded by `limiter`.
+
+    Raises `UnsupportedSourceTypeError` (415), `DocumentParseError` (400), or
+    `ChunkTooLargeError` — see `src/application/ingestion/errors.py`. Callers
+    map these to status codes; this module carries no HTTP knowledge
+    (ADR-0015). The `max_chunks_per_file` ceiling (413) is enforced by the
+    caller, not here — see Phase 4 of plan 001.
+    """
+    if source_type not in _PARSERS:
+        raise UnsupportedSourceTypeError(f"'{source_type}' is not an ingestible source type")
+    return await to_thread.run_sync(
+        partial(_parse_and_chunk, data, source_type, file_id, settings),
+        limiter=limiter,
+    )

src/application/ingestion/spreadsheet_parser.py

@@ -0,0 +1,121 @@
+"""CSV and XLSX parsing: row = chunk (ADR-0004, ADR-0018).
+
+Both formats reduce to rows and hand them to the shared renderer in
+`tabular`, so a Q&A sheet and a branch directory go through one code path with
+no shape detection.
+"""
+
+import csv
+import io
+
+import openpyxl
+from openpyxl.worksheet.worksheet import Worksheet
+
+from src.application.ingestion.errors import DocumentParseError
+from src.application.ingestion.models import ContentType, ParsedDocument, StructuralUnit
+from src.application.ingestion.tabular import Row, clean_cell, clean_rows, render_rows
+
+# Farsi exports from older Excel are frequently cp1256 (Windows Arabic).
+_ENCODINGS = ("utf-8-sig", "utf-8", "cp1256")
+
+_SNIFF_BYTES = 8192
+
+
+def _decode(data: bytes) -> str:
+    for encoding in _ENCODINGS:
+        try:
+            return data.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+    raise DocumentParseError(f"Could not decode file as any of: {', '.join(_ENCODINGS)}")
+
+
+def _to_units(rendered: list[str]) -> list[StructuralUnit]:
+    return [StructuralUnit(text=text, content_type=ContentType.TABLE_ROW) for text in rendered]
+
+
+def parse_csv(data: bytes) -> ParsedDocument:
+    """Parse CSV bytes into one structural unit per row."""
+    text = _decode(data)
+
+    try:
+        dialect = csv.Sniffer().sniff(text[:_SNIFF_BYTES])
+        reader = csv.reader(io.StringIO(text), dialect)
+    except csv.Error:
+        # A single-column file has no delimiter to find; that is not an error.
+        reader = csv.reader(io.StringIO(text))
+
+    rows = clean_rows(reader)
+    if not rows:
+        raise DocumentParseError("File contains no rows")
+
+    units = _to_units(render_rows(rows))
+    if not units:
+        raise DocumentParseError("File contains no data rows")
+
+    return ParsedDocument(units=units, block_count=len(units))
+
+
+def _forward_fill_merges(worksheet: Worksheet) -> dict[tuple[int, int], str]:
+    """Map every cell of a merged range to the range's value.
+
+    openpyxl stores a merged range's value only in its top-left cell; the rest
+    read as None. Without this a branch row inherits nothing from the province
+    cell merged above it and silently loses that field (ADR-0004).
+
+    Iterate the range collection itself and read corners via `bounds`: `.ranges`
+    is a set subclass and `.min_row` and friends are descriptors, neither of
+    which resolves to an int for a type checker.
+    """
+    filled: dict[tuple[int, int], str] = {}
+
+    for merged in worksheet.merged_cells:
+        min_col, min_row, max_col, max_row = merged.bounds
+        value = clean_cell(worksheet.cell(row=min_row, column=min_col).value)
+        if not value:
+            continue
+        for row in range(min_row, max_row + 1):
+            for column in range(min_col, max_col + 1):
+                filled[(row, column)] = value
+
+    return filled
+
+
+def _sheet_rows(worksheet: Worksheet) -> list[Row]:
+    """Read a worksheet into normalized text rows, merges resolved."""
+    merged = _forward_fill_merges(worksheet)
+    rows: list[Row] = []
+
+    for row_index, row in enumerate(worksheet.iter_rows(), start=1):
+        values = [
+            merged.get((row_index, column_index), clean_cell(cell.value))
+            for column_index, cell in enumerate(row, start=1)
+        ]
+        if any(values):
+            rows.append(values)
+
+    return rows
+
+
+def parse_xlsx(data: bytes) -> ParsedDocument:
+    """Parse XLSX bytes into one structural unit per row, across all sheets."""
+    try:
+        workbook = openpyxl.load_workbook(io.BytesIO(data), data_only=True)
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open XLSX: {exc}") from exc
+
+    units: list[StructuralUnit] = []
+    try:
+        for worksheet in workbook.worksheets:
+            rows = _sheet_rows(worksheet)
+            # The dead second sheet seen throughout the sample corpus.
+            if not rows:
+                continue
+            units.extend(_to_units(render_rows(rows)))
+    finally:
+        workbook.close()
+
+    if not units:
+        raise DocumentParseError("Workbook contains no data rows")
+
+    return ParsedDocument(units=units, block_count=len(units))

src/application/ingestion/tabular.py

@@ -0,0 +1,167 @@
+"""Shared row-to-chunk rendering for every tabular source (ADR-0004, ADR-0018).
+
+A table row is an atomic structural unit regardless of the container it
+arrived in -- a docx table, an xlsx sheet, or a csv file -- so one renderer
+serves all three and a Q&A sheet needs no special case against a branch
+directory:
+
+    q: ...                 ردیف: 1
+    a: ...                 استان: اردبیل
+                           شعبه: پارس آباد
+
+The header rules exist because real tables are not uniform. Of the tables in
+the sample corpus, some carry a header row, one is a bare list of values with
+no header at all, and one is page decoration. The guiding constraint is
+therefore: **never invent structure that is not provably there, and never
+discard a row.** A wrongly-detected header turns every chunk into nonsense
+(`80: 70`), which is worse than an unlabeled row.
+"""
+
+from collections.abc import Iterable, Sequence
+
+from src.application.ingestion.normalization import normalize_persian_text
+
+Row = Sequence[str]
+
+# A header cell is a label, not a sentence.
+MAX_HEADER_CELL_LENGTH = 80
+
+# How much shorter a header cell must be than the column beneath it before
+# length alone is taken as evidence of a header (the `q`/`a` case, where
+# one-character labels sit above paragraph-long answers).
+_HEADER_LENGTH_RATIO = 3.0
+
+
+def clean_cell(value: object) -> str:
+    """Normalize a cell to text; None and blank cells become the empty string.
+
+    `object` rather than a union: a spreadsheet cell holds whatever the
+    workbook stored -- str, int, float, bool, datetime, a formula error -- and
+    every one of them is handled the same way, by rendering it.
+    """
+    if value is None:
+        return ""
+    return normalize_persian_text(str(value))
+
+
+def _is_numeric(text: str) -> bool:
+    return bool(text) and text.replace(",", "").replace(".", "").replace("-", "").isdigit()
+
+
+def _looks_like_title_row(row: Row) -> bool:
+    """A merged title spanning the sheet resolves to one value, or repeats it."""
+    populated = [cell for cell in row if cell]
+    return len(populated) < 2 or len(set(populated)) == 1
+
+
+def has_header(rows: Sequence[Row]) -> bool:
+    """Whether the first row labels the columns beneath it.
+
+    Decided by comparing row 0 against the column below it, not by how row 0
+    looks on its own: a label row is *inconsistent* with its data (text above
+    numbers, or a short label above long prose), while a data row is
+    consistent with the rows that follow. This is the test `csv.Sniffer`
+    uses, and it is a property of the table rather than a pattern borrowed
+    from one document.
+    """
+    if len(rows) < 2:
+        return False
+
+    candidate, data = rows[0], rows[1:]
+    if not all(cell for cell in candidate[: len(data[0])] if cell) and _looks_like_title_row(
+        candidate
+    ):
+        return False
+    if any(len(cell) > MAX_HEADER_CELL_LENGTH for cell in candidate):
+        return False
+
+    for index, label in enumerate(candidate):
+        if not label:
+            continue
+        column = [row[index] for row in data if index < len(row) and row[index]]
+        if not column:
+            continue
+
+        # A text label above a numeric column.
+        if not _is_numeric(label) and all(_is_numeric(value) for value in column):
+            return True
+
+        # A short label above a column of much longer values.
+        mean_length = sum(len(value) for value in column) / len(column)
+        if mean_length > len(label) * _HEADER_LENGTH_RATIO:
+            return True
+
+    return False
+
+
+def strip_title_rows(rows: Sequence[Row]) -> Sequence[Row]:
+    """Drop leading merged-title and blank rows.
+
+    Structural, not a content guess: these rows are the artifact of a merged
+    range spanning the sheet width, so they carry one value across many cells.
+    Only *leading* rows are dropped, so no data row is ever lost.
+    """
+    start = 0
+    while start < len(rows) and _looks_like_title_row(rows[start]):
+        start += 1
+    return rows[start:]
+
+
+def _column_name(header: Row, index: int) -> str:
+    """Return a header label, falling back positionally past the header width."""
+    if index < len(header) and header[index]:
+        return header[index]
+    return f"column_{index + 1}"
+
+
+def _dedupe_horizontal_merge(row: Row) -> list[str]:
+    """Collapse the repeats a horizontally merged cell produces.
+
+    Both python-docx and openpyxl report a merged cell once per grid column it
+    spans, so an unlabeled row would otherwise repeat the same value.
+    """
+    collapsed: list[str] = []
+    for cell in row:
+        if cell and (not collapsed or collapsed[-1] != cell):
+            collapsed.append(cell)
+    return collapsed
+
+
+def render_rows(rows: Sequence[Row]) -> list[str]:
+    """Render table rows as text, one string per row.
+
+    With a provable header each row becomes `"{header}: {value}"` lines, which
+    makes it self-describing. Without one, cells are joined with `" | "` --
+    unlabeled, but never mislabeled.
+    """
+    rows = strip_title_rows(rows)
+    if not rows:
+        return []
+
+    if not has_header(rows):
+        return [text for row in rows if (text := " | ".join(_dedupe_horizontal_merge(row)))]
+
+    header, data = rows[0], rows[1:]
+    # A header merged vertically across two rows resolves to the same text in
+    # the row below it; that duplicate is the header, not data.
+    while data and list(data[0]) == list(header):
+        data = data[1:]
+
+    rendered: list[str] = []
+    for row in data:
+        lines = [
+            f"{_column_name(header, index)}: {value}" for index, value in enumerate(row) if value
+        ]
+        if lines:
+            rendered.append("\n".join(lines))
+    return rendered
+
+
+def clean_rows(rows: Iterable[Iterable[object]]) -> list[Row]:
+    """Normalize every cell and drop rows that are entirely empty."""
+    cleaned: list[Row] = []
+    for row in rows:
+        values = [clean_cell(cell) for cell in row]
+        if any(values):
+            cleaned.append(values)
+    return cleaned

src/application/ingestion/tokenizer.py

@@ -0,0 +1,33 @@
+"""Token counting for chunk sizing (ADR-0018).
+
+`cl100k_base` is a deliberate proxy for the embedding models' own tokenizers.
+`text-embedding-3-large` has an 8191-token window and never binds;
+`nomic-embed-text-v2-moe`'s 512-token sequence length is the only real
+constraint. cl100k tokenizes Persian inefficiently while nomic's multilingual
+tokenizer does not, so a cl100k count reliably over-estimates the nomic count --
+safe in the conservative direction, without shipping a second tokenizer and its
+model download into the ingestion path.
+"""
+
+from functools import lru_cache
+
+import tiktoken
+
+
+@lru_cache(maxsize=4)
+def get_encoder(encoding_name: str) -> tiktoken.Encoding:
+    """Return a cached tiktoken encoder.
+
+    Deliberately not a module-level constant: `tiktoken` fetches the BPE
+    vocabulary over the network the first time an encoding is used, and
+    ADR-0012 forbids external resource setup as an import-time side effect.
+    The lifespan warms this at startup so a process fails fast at boot rather
+    than inside the first ingestion request. Set `TIKTOKEN_CACHE_DIR` to a
+    pre-populated directory for offline deployments.
+    """
+    return tiktoken.get_encoding(encoding_name)
+
+
+def count_tokens(text: str, encoding_name: str) -> int:
+    """Return the number of tokens `text` encodes to."""
+    return len(get_encoder(encoding_name).encode(text))

src/application/points/__init__.py

@@ -0,0 +1,18 @@
+"""Ingestion-generated Qdrant point CRUD (ADR-0001, ADR-0002).
+
+`index_chunks` is the entry point callers outside this package should use: it
+dispatches payload construction, batching, bounded-concurrency upserts, and the
+post-success soft-delete sweep. `build_chunk_payload` and the batching helpers
+stay internal, exported mainly for their own unit tests.
+
+The `/v1/points` surface lives here too, in its own modules with their own
+entry points: `queries.py` for the read paths and `deletion.py` for soft delete
+with neighbour relinking. They share this package because they share ADR-0001's
+payload schema, not because they share a caller — `index_chunks` writes a whole
+file at once, while those serve one admin edit at a time.
+"""
+
+from src.application.points.indexing import IndexingResult, index_chunks
+from src.application.points.models import ChunkPoint
+
+__all__ = ["ChunkPoint", "IndexingResult", "index_chunks"]

src/application/points/deletion.py

@@ -0,0 +1,253 @@
+"""Soft delete for `/v1/points` and for a whole file's points (ADR-0002).
+
+The caller-facing entry points are `soft_delete_point` and
+`soft_delete_file_points`. Routers call these; `patches_for_removal` and the
+planning helpers stay internal, because getting a delete right is exactly the
+composition a caller should not have to reassemble: read the point, load its
+neighbours, compute the patches still missing, send them in **one** batch,
+verify they landed, and retry against fresh versions if they did not.
+
+Why the retry exists. Qdrant has no multi-point transaction, so a batch whose
+second operation loses a version race applies its first operation anyway — and
+a filtered `set_payload` that matched nothing still reports success. That
+combination means "did my write land?" is only answerable by reading back, and
+a single-shot delete would be able to leave the deactivation applied and a
+neighbour's pointer stale. Since `patches_for_removal` plans from current state
+towards a fixed end state, simply re-planning emits precisely the patches that
+did not land, so the loop converges instead of re-doing work. Only exhausting
+the attempts raises `PointVersionConflictError` (`409`).
+
+Deleting an already-inactive point falls out of the same machinery rather than
+needing a special case: its neighbours were relinked by the first delete, so the
+plan is empty and the call is a no-op success — not a `404`, and not a second
+relink.
+"""
+
+import uuid
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
+from time import perf_counter
+
+import structlog
+
+from src.application.points.errors import PointVersionConflictError
+from src.application.points.point import Point, PointNotFoundError
+from src.application.points.relinking import (
+    neighbour_ids,
+    patch_for_deactivation,
+    patches_for_removal,
+)
+from src.application.ports.point_repository import PayloadPatch, PointRepository
+
+logger = structlog.get_logger(__name__)
+
+# Three plan-apply rounds, then a final verifying plan. Each round only re-emits
+# what a concurrent writer displaced, so a caller that legitimately needs more
+# than this is contending on the same points continuously and deserves the
+# `409` rather than an unbounded loop inside a request.
+_MAX_ATTEMPTS = 3
+
+# One sweep page. Matches ADR-0002's 100-operation batch cap, so a page of
+# points is always expressible as a single `points/batch` request.
+_SWEEP_BATCH_SIZE = 100
+
+# A hard ceiling on sweep rounds, so a file being concurrently re-ingested while
+# it is deleted cannot spin here for the life of the request.
+_MAX_SWEEP_ROUNDS = 1_000
+
+
+def _elapsed_ms(started: float) -> float:
+    """Wall-clock milliseconds since `started` (ADR-0011's `duration_ms`).
+
+    Worth carrying on these events even though the relinking itself is O(1):
+    what a delete actually spends is Qdrant round trips, and the whole-file
+    sweep spends a number of them proportional to the file's length. Timing the
+    operation is the only way to tell a slow store from a contended one, which
+    `rounds` on the same event then disambiguates.
+    """
+    return round((perf_counter() - started) * 1000, 2)
+
+
+async def soft_delete_point(
+    repository: PointRepository,
+    *,
+    tenant_id: uuid.UUID,
+    point_id: uuid.UUID,
+    actor: str,
+) -> Point:
+    """Deactivate one point and relink its neighbours around the gap.
+
+    Returns the point as it now stands. Raises `PointNotFoundError` (`404`) if
+    it is not this tenant's — the same non-disclosure rule the read paths
+    follow — or `PointVersionConflictError` (`409`) if concurrent writers keep
+    displacing the plan.
+    """
+    started = perf_counter()
+    rounds = 0
+    point, patches = await _plan_removal(
+        repository, tenant_id=tenant_id, point_id=point_id, actor=actor
+    )
+
+    for _ in range(_MAX_ATTEMPTS):
+        if not patches:
+            break
+        await repository.apply_patches(tenant_id=tenant_id, patches=patches)
+        rounds += 1
+        # The next plan doubles as verification: anything that did not land is
+        # still missing from the end state and comes back as a patch.
+        point, patches = await _plan_removal(
+            repository, tenant_id=tenant_id, point_id=point_id, actor=actor
+        )
+
+    if patches:
+        logger.warning(
+            "points.soft_delete.conflict",
+            tenant_id=str(tenant_id),
+            point_id=str(point_id),
+            file_id=str(point.file_id),
+            unsettled_points=[str(patch.point_id) for patch in patches],
+            rounds=rounds,
+            duration_ms=_elapsed_ms(started),
+        )
+        raise PointVersionConflictError(
+            f"point {point_id} could not be soft-deleted under concurrent modification"
+        )
+
+    if rounds:
+        logger.info(
+            "points.soft_deleted",
+            tenant_id=str(tenant_id),
+            point_id=str(point_id),
+            file_id=str(point.file_id),
+            version=point.version,
+            actor=actor,
+            # `rounds` is 1 unless a concurrent writer forced a re-plan, so a
+            # rising value here is contention, not slow relinking.
+            rounds=rounds,
+            duration_ms=_elapsed_ms(started),
+        )
+    return point
+
+
+async def soft_delete_file_points(
+    repository: PointRepository,
+    *,
+    tenant_id: uuid.UUID,
+    file_id: uuid.UUID,
+    actor: str,
+) -> int:
+    """Deactivate every active point of one file, in batches.
+
+    No relinking: the whole file leaves the sequence at once, so no surviving
+    active point can be left pointing at a deactivated one, and the chain is
+    preserved intact for whoever reads the deleted file later.
+
+    Returns how many points were active when the sweep reached them. Each round
+    re-lists from the start rather than paging with a cursor — deactivated
+    points drop straight out of the default listing, so the listing itself is
+    the progress check, and a round that attempts the exact same ids as the one
+    before it made no progress and raises `PointVersionConflictError`.
+    """
+    started = perf_counter()
+    swept: set[uuid.UUID] = set()
+    previous_attempt: frozenset[uuid.UUID] = frozenset()
+    rounds = 0
+
+    for _ in range(_MAX_SWEEP_ROUNDS):
+        page = await repository.list_by_file(
+            tenant_id=tenant_id, file_id=file_id, limit=_SWEEP_BATCH_SIZE
+        )
+        if not page.points:
+            if swept:
+                logger.info(
+                    "points.file_soft_deleted",
+                    tenant_id=str(tenant_id),
+                    file_id=str(file_id),
+                    points_soft_deleted=len(swept),
+                    actor=actor,
+                    # Two Qdrant round trips per round, so this is the delete
+                    # path whose cost tracks the size of the file.
+                    rounds=rounds,
+                    duration_ms=_elapsed_ms(started),
+                )
+            return len(swept)
+
+        attempt = frozenset(point.point_id for point in page.points)
+        if attempt == previous_attempt:
+            logger.warning(
+                "points.file_soft_delete.conflict",
+                tenant_id=str(tenant_id),
+                file_id=str(file_id),
+                unsettled_points=[str(point_id) for point_id in sorted(attempt, key=str)],
+                rounds=rounds,
+                duration_ms=_elapsed_ms(started),
+            )
+            raise PointVersionConflictError(
+                f"file {file_id} could not be soft-deleted under concurrent modification"
+            )
+        previous_attempt = attempt
+
+        now = datetime.now(UTC)
+        await repository.apply_patches(
+            tenant_id=tenant_id,
+            patches=[patch_for_deactivation(point, actor=actor, now=now) for point in page.points],
+        )
+        swept |= attempt
+        rounds += 1
+
+    logger.warning(
+        "points.file_soft_delete.conflict",
+        tenant_id=str(tenant_id),
+        file_id=str(file_id),
+        reason="sweep_rounds_exhausted",
+        rounds=rounds,
+        duration_ms=_elapsed_ms(started),
+    )
+    raise PointVersionConflictError(f"file {file_id} still had active points after the sweep")
+
+
+async def _plan_removal(
+    repository: PointRepository,
+    *,
+    tenant_id: uuid.UUID,
+    point_id: uuid.UUID,
+    actor: str,
+) -> tuple[Point, tuple[PayloadPatch, ...]]:
+    point = await repository.get(tenant_id=tenant_id, point_id=point_id)
+    if point is None:
+        raise PointNotFoundError(f"point {point_id} not found")
+
+    neighbours = await _load_neighbours(repository, tenant_id=tenant_id, point=point)
+    _warn_on_missing_neighbours(point, neighbours, tenant_id=tenant_id)
+    patches = patches_for_removal(point, neighbours, actor=actor, now=datetime.now(UTC))
+    return point, patches
+
+
+async def _load_neighbours(
+    repository: PointRepository, *, tenant_id: uuid.UUID, point: Point
+) -> dict[uuid.UUID, Point]:
+    wanted: Sequence[uuid.UUID] = neighbour_ids(point)
+    if not wanted:
+        return {}
+    found = await repository.get_many(tenant_id=tenant_id, point_ids=wanted)
+    return {neighbour.point_id: neighbour for neighbour in found}
+
+
+def _warn_on_missing_neighbours(
+    point: Point, neighbours: Mapping[uuid.UUID, Point], *, tenant_id: uuid.UUID
+) -> None:
+    """A pointer naming a point that is not there means the chain is already broken.
+
+    Worth a log line rather than an exception: the delete can still complete the
+    part of the relink that does exist, and refusing would leave the caller with
+    a point it cannot remove through any endpoint.
+    """
+    missing = [pointer for pointer in neighbour_ids(point) if pointer not in neighbours]
+    if missing:
+        logger.warning(
+            "points.relink.neighbour_missing",
+            tenant_id=str(tenant_id),
+            point_id=str(point.point_id),
+            file_id=str(point.file_id),
+            missing_neighbours=[str(pointer) for pointer in missing],
+        )

src/application/points/errors.py

@@ -0,0 +1,19 @@
+"""Mutation failures for the `/v1/points` write paths (ADR-0002).
+
+No HTTP knowledge here — `src/api/errors.py` owns the status mapping. Absence
+lives on `PointNotFoundError` in `point.py`, next to the model whose read paths
+raise it; this module is for the failures only a *write* can produce.
+"""
+
+
+class PointVersionConflictError(Exception):
+    """A version-guarded write could not be landed against a moving target.
+
+    Raised when the service has re-read, recomputed, and re-applied its patches
+    the allowed number of times and the desired state still has not settled —
+    something else is writing the same points concurrently. Maps to `409`.
+
+    This is not "the guard fired once": a single stale guard is expected and is
+    retried, because Qdrant reports success for a filtered `set_payload` that
+    matched nothing. It means the retries were exhausted.
+    """

src/application/points/indexing.py

@@ -0,0 +1,203 @@
+"""The one caller-facing entry point for indexing embedded chunks (ADR-0001, ADR-0017).
+
+`index_chunks` is the only version of this step callers should reach for. It
+owns the whole composition a correct upsert needs:
+
+- building ADR-0001's payload for every chunk, with `tenant_id`/`domain` taken
+  from server-derived context;
+- offloading that (and the per-chunk content hashing) to a thread, since it is
+  blocking CPU work (ADR-0017);
+- batching at `QDRANT_UPSERT_BATCH_SIZE` inside ADR-0001's 64-256 band;
+- bounding in-flight batches with an `asyncio.Semaphore` rather than an
+  unbounded `gather` (ADR-0017);
+- running the soft-delete sweep for a shortened file **only after every batch
+  has succeeded**.
+
+That last ordering is the point, not an implementation detail — see
+`_deactivate_stale` below. `build_chunk_payload` and `_batches` stay internal;
+pushing that composition onto every call site is exactly the obligation a deep
+module absorbs once (CLAUDE.md, "prefer deep modules").
+"""
+
+import asyncio
+import uuid
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+
+from anyio import CapacityLimiter, to_thread
+
+from src.application.ingestion.errors import PointIndexingError
+from src.application.ingestion.models import EmbeddedChunk
+from src.application.points.models import ChunkPoint
+from src.application.points.payload import build_chunk_payload
+from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
+from src.application.ports.point_storage import PointStorage
+from src.config import QdrantSettings
+
+
+@dataclass(frozen=True)
+class IndexingResult:
+    """What one indexing pass wrote.
+
+    `points_upserted` counts points written, not points *created* — a
+    deterministic-id upsert cannot distinguish an insert from an overwrite, so
+    the ingestion job reports this as `points_created` and leaves
+    `points_updated` at zero rather than guessing.
+    """
+
+    points_upserted: int
+    points_soft_deleted: int
+
+
+def _embedding_model_version(
+    dense_embedders: Sequence[DenseEmbedder], sparse_embedder: SparseEmbedder
+) -> str:
+    """Compose the `embedding_model_version` payload value (ADR-0001).
+
+    Sorted so the string is stable regardless of the order the embedders were
+    wired in — an unstable value would make "which chunks need re-embedding?"
+    unanswerable, which is the field's only reason to exist.
+    """
+    versions = sorted(
+        [embedder.model_version for embedder in dense_embedders] + [sparse_embedder.model_version]
+    )
+    return "+".join(versions)
+
+
+def _batches(points: Sequence[ChunkPoint], size: int) -> list[Sequence[ChunkPoint]]:
+    return [points[i : i + size] for i in range(0, len(points), size)]
+
+
+def _build_points(
+    embedded: Sequence[EmbeddedChunk],
+    *,
+    tenant_id: uuid.UUID,
+    domain: str,
+    file_id: uuid.UUID,
+    source_filename: str,
+    source_type: str,
+    actor: str,
+    embedding_model_version: str,
+    indexed_at: datetime,
+) -> list[ChunkPoint]:
+    """Blocking: hashes every chunk's content. Always called through a thread."""
+    return [
+        ChunkPoint(
+            point_id=item.chunk.chunk_id,
+            dense=item.dense,
+            sparse=item.sparse,
+            payload=build_chunk_payload(
+                item.chunk,
+                tenant_id=tenant_id,
+                domain=domain,
+                file_id=file_id,
+                source_filename=source_filename,
+                source_type=source_type,
+                actor=actor,
+                embedding_model_version=embedding_model_version,
+                indexed_at=indexed_at,
+            ),
+        )
+        for item in embedded
+    ]
+
+
+async def _upsert_bounded(
+    storage: PointStorage, batch: Sequence[ChunkPoint], *, semaphore: asyncio.Semaphore
+) -> None:
+    async with semaphore:
+        try:
+            await storage.upsert_points(batch)
+        except Exception as exc:
+            raise PointIndexingError(f"upserting {len(batch)} points failed: {exc}") from exc
+
+
+async def _deactivate_stale(
+    storage: PointStorage,
+    *,
+    tenant_id: uuid.UUID,
+    file_id: uuid.UUID,
+    from_chunk_index: int,
+    actor: str,
+    deleted_at: datetime,
+) -> int:
+    """Soft-delete points left over from a longer previous version of this file.
+
+    Chunk indices are contiguous from 0, so "index >= the new chunk count" is
+    exactly the set of points the new version no longer produces.
+
+    This runs **only after every upsert has succeeded**, and that ordering is
+    what keeps a failed attempt from damaging a working index. ADR-0001's
+    deterministic point ids mean a re-ingestion overwrites in place, so literal
+    atomic replacement is not available; what *is* guaranteed is that a failed
+    attempt never removes content (it can only leave a prefix updated), and that
+    a retry converges to the correct state. See ADR-0017.
+    """
+    try:
+        return await storage.deactivate_points_from_index(
+            tenant_id=tenant_id,
+            file_id=file_id,
+            from_chunk_index=from_chunk_index,
+            deleted_at=deleted_at,
+            updated_by=actor,
+        )
+    except Exception as exc:
+        raise PointIndexingError(f"soft-deleting stale points failed: {exc}") from exc
+
+
+async def index_chunks(
+    embedded: Sequence[EmbeddedChunk],
+    *,
+    storage: PointStorage,
+    tenant_id: uuid.UUID,
+    domain: str,
+    file_id: uuid.UUID,
+    source_filename: str,
+    source_type: str,
+    actor: str,
+    dense_embedders: Sequence[DenseEmbedder],
+    sparse_embedder: SparseEmbedder,
+    settings: QdrantSettings,
+    thread_limiter: CapacityLimiter,
+) -> IndexingResult:
+    """Upsert every embedded chunk as a tenant-scoped point, then sweep leftovers.
+
+    Raises `PointIndexingError` (502) if any batch or the sweep fails.
+    """
+    if not embedded:
+        return IndexingResult(points_upserted=0, points_soft_deleted=0)
+
+    indexed_at = datetime.now(UTC)
+    points = await to_thread.run_sync(
+        lambda: _build_points(
+            embedded,
+            tenant_id=tenant_id,
+            domain=domain,
+            file_id=file_id,
+            source_filename=source_filename,
+            source_type=source_type,
+            actor=actor,
+            embedding_model_version=_embedding_model_version(dense_embedders, sparse_embedder),
+            indexed_at=indexed_at,
+        ),
+        limiter=thread_limiter,
+    )
+
+    semaphore = asyncio.Semaphore(settings.upsert_concurrency)
+    await asyncio.gather(
+        *(
+            _upsert_bounded(storage, batch, semaphore=semaphore)
+            for batch in _batches(points, settings.upsert_batch_size)
+        )
+    )
+
+    soft_deleted = await _deactivate_stale(
+        storage,
+        tenant_id=tenant_id,
+        file_id=file_id,
+        from_chunk_index=len(points),
+        actor=actor,
+        deleted_at=indexed_at,
+    )
+    return IndexingResult(points_upserted=len(points), points_soft_deleted=soft_deleted)

src/application/points/models.py

@@ -0,0 +1,29 @@
+"""Domain models for Qdrant points (ADR-0001).
+
+Deliberately free of the `qdrant_client` SDK: `src/infrastructure/qdrant/`
+converts these to `PointStruct`/`models.SparseVector` at upsert time
+(ADR-0015 — application code and ports carry no infrastructure imports).
+"""
+
+import uuid
+
+from pydantic import BaseModel, Field
+
+from src.application.ingestion.models import SparseVector
+
+
+class ChunkPoint(BaseModel):
+    """One chunk, ready to upsert: its id, its named vectors, and its payload.
+
+    `point_id` is the chunk's deterministic UUIDv5 (`chunk_id_for`), so
+    re-ingesting a file overwrites its points rather than duplicating them
+    (ADR-0001).
+
+    `dense` is keyed by named-vector name (`dense_nomic`, `dense_openai`).
+    `late_interaction` is absent — ADR-0017 does not compute it at ingest.
+    """
+
+    point_id: uuid.UUID
+    dense: dict[str, list[float]]
+    sparse: SparseVector
+    payload: dict[str, object] = Field(default_factory=dict)

src/application/points/payload.py

@@ -0,0 +1,70 @@
+"""Builds ADR-0001's point payload from a chunk plus its ingestion context.
+
+Internal to `src/application/points/` — callers use `index_chunks`, which owns
+composing this with batching and the deactivation sweep. Exported for its own
+unit tests, not as a surface to build payloads by hand.
+"""
+
+import uuid
+from datetime import datetime
+from hashlib import sha256
+
+from src.application.ingestion.models import Chunk
+
+
+def _optional_id(value: uuid.UUID | None) -> str | None:
+    return str(value) if value is not None else None
+
+
+def build_chunk_payload(
+    chunk: Chunk,
+    *,
+    tenant_id: uuid.UUID,
+    domain: str,
+    file_id: uuid.UUID,
+    source_filename: str,
+    source_type: str,
+    actor: str,
+    embedding_model_version: str,
+    indexed_at: datetime,
+) -> dict[str, object]:
+    """Return ADR-0001's payload for one chunk.
+
+    `tenant_id` and `domain` are passed in from the server-derived `AuthContext`
+    and the validated request — never from anything the client could assert as
+    authority (ADR-0002's non-negotiable isolation rule).
+
+    UUIDs are serialized as strings because the `tenant_id`/`domain`/`file_id`/
+    `previous_chunk_id`/`next_chunk_id` payload indexes are *keyword* indexes;
+    a native UUID would not match a keyword filter.
+
+    **Known gap — `version` is always written as `1`.** ADR-0002 uses this field
+    for optimistic concurrency between ingestion and manual `/v1/points` edits,
+    which needs a read-check-write (one read per point). Ingestion is
+    authoritative for its own file today, so writing `1` is safe until
+    `/v1/points` exists; plan 002 owns closing this.
+    """
+    timestamp = indexed_at.isoformat()
+    return {
+        "tenant_id": str(tenant_id),
+        "domain": domain,
+        "file_id": str(file_id),
+        "chunk_id": str(chunk.chunk_id),
+        "content": chunk.content,
+        "content_type": chunk.content_type.value,
+        "source_filename": source_filename,
+        "source_type": source_type,
+        "order_id": chunk.order_id,
+        "chunk_index": chunk.chunk_index,
+        "previous_chunk_id": _optional_id(chunk.previous_chunk_id),
+        "next_chunk_id": _optional_id(chunk.next_chunk_id),
+        "is_active": True,
+        "deleted_at": None,
+        "created_at": timestamp,
+        "updated_at": timestamp,
+        "created_by": actor,
+        "updated_by": actor,
+        "version": 1,
+        "content_hash": sha256(chunk.content.encode("utf-8")).hexdigest(),
+        "embedding_model_version": embedding_model_version,
+    }

src/application/points/point.py

@@ -0,0 +1,126 @@
+"""The caller-facing point model for `/v1/points` (ADR-0001, ADR-0002).
+
+`ChunkPoint` in `models.py` is the *write* shape ingestion upserts: an id, its
+named vectors, and an opaque payload dict. This module is the *read/edit* shape
+the `/v1/points` surface works in, where the payload's individual fields matter
+and the distinction between what a caller may write and what the server owns is
+a security boundary rather than a convention.
+
+That split is the reason this is a model and not a dict. ADR-0002's isolation
+rule ("never accepted as client-supplied input") and its optimistic-concurrency
+guard both fail open if a caller can smuggle `tenant_id` or `version` through a
+payload update, so the writable field set is enumerated in one place here and
+every write path validates against it.
+"""
+
+import uuid
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Self
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# Fields the server derives and a caller may never set, patch, or override.
+# `tenant_id` is authority, `version` is the concurrency guard, `chunk_index`
+# derives the point id, and the rest are provenance the server timestamps.
+SERVER_OWNED_FIELDS: frozenset[str] = frozenset(
+    {
+        "tenant_id",
+        "version",
+        "chunk_index",
+        "chunk_id",
+        "is_active",
+        "deleted_at",
+        "created_at",
+        "created_by",
+        "updated_at",
+        "updated_by",
+        "content_hash",
+        "embedding_model_version",
+    }
+)
+
+# Fields a caller may supply on create, replace, or payload patch. `order_id`
+# is writable on create but moves only through `PATCH /v1/points/{id}/order`
+# afterwards, because a bare `order_id` write would not relink neighbours.
+CALLER_WRITABLE_FIELDS: frozenset[str] = frozenset(
+    {
+        "content",
+        "content_type",
+        "domain",
+        "source_filename",
+        "source_type",
+        "order_id",
+        "previous_chunk_id",
+        "next_chunk_id",
+    }
+)
+
+
+class Point(BaseModel):
+    """One Qdrant point, read back with its ADR-0001 payload fields typed.
+
+    Vectors are deliberately absent: ADR-0008 returns them only when explicitly
+    requested, and every read path that does not ask for them should not pay to
+    deserialize them.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    point_id: uuid.UUID
+
+    tenant_id: uuid.UUID
+    domain: str
+    file_id: uuid.UUID
+    chunk_id: uuid.UUID
+
+    content: str
+    content_type: str
+    source_filename: str
+    source_type: str
+
+    order_id: float
+    chunk_index: int
+    previous_chunk_id: uuid.UUID | None = None
+    next_chunk_id: uuid.UUID | None = None
+
+    is_active: bool = True
+    deleted_at: datetime | None = None
+
+    created_at: datetime
+    updated_at: datetime
+    created_by: str
+    updated_by: str
+
+    version: int
+    content_hash: str
+    embedding_model_version: str
+
+    # Only populated when the caller explicitly asked for vectors.
+    vectors: dict[str, object] | None = Field(default=None)
+
+    @classmethod
+    def from_payload(
+        cls,
+        point_id: uuid.UUID,
+        payload: Mapping[str, object],
+        *,
+        vectors: Mapping[str, object] | None = None,
+    ) -> Self:
+        """Build a `Point` from a raw Qdrant payload dict.
+
+        Lives here rather than in the Qdrant adapter so the payload field names
+        are declared once, next to the model that mirrors them. The adapter
+        stays responsible for talking to the SDK, not for knowing ADR-0001's
+        schema twice.
+        """
+        return cls.model_validate({**payload, "point_id": point_id, "vectors": vectors})
+
+
+class PointNotFoundError(LookupError):
+    """No such point *within the requesting tenant*.
+
+    Routes map this to `404`, never `403` — a caller must not be able to probe
+    for the existence of another tenant's point ids (ADR-0016). The error
+    deliberately carries no hint about which of the two cases occurred.
+    """

src/application/points/queries.py

@@ -0,0 +1,115 @@
+"""Read paths for `/v1/points` (ADR-0002, ADR-0008).
+
+The caller-facing entry points for point reads. Routers call these; they never
+touch the `PointRepository` directly, and never build a filter.
+
+This module is thin on purpose but not empty, and the two things it does own are
+exactly the ones a route would otherwise get wrong:
+
+- **`tenant_id` always comes from the caller's `AuthContext`.** Every function
+  takes it as a required keyword and hands it to the repository. Nothing here
+  reads a tenant from a query string or body.
+- **A keyword query is Persian-normalized before it reaches the index.**
+  Ingestion letter-folds chunk content (`normalize_persian_text`, ADR-0018), so
+  stored text contains Persian yeh/keheh. A query typed on an Arabic keyboard
+  carries U+064A/U+0643 and would match nothing at all — a silent empty result,
+  not an error. Folding the query the same way is what makes the two comparable.
+
+Reads emit no log events. The request middleware already records every call, and
+ADR-0011 reserves `INFO` for lifecycle events rather than per-read volume;
+mutations get their own events when those paths land.
+"""
+
+import uuid
+
+from src.application.ingestion.normalization import normalize_persian_text
+from src.application.points.point import Point, PointNotFoundError
+from src.application.ports.point_repository import PointPage, PointRepository
+
+
+async def get_point(
+    repository: PointRepository,
+    *,
+    tenant_id: uuid.UUID,
+    point_id: uuid.UUID,
+    with_vectors: bool = False,
+) -> Point:
+    """One point, or `PointNotFoundError` if it is not this tenant's.
+
+    Raises rather than returning `None` so a route cannot forget the check and
+    serve `200 null`. "Absent" and "another tenant's" are the same outcome by
+    design (ADR-0016: cross-tenant access is `404`, never `403`).
+    """
+    point = await repository.get(tenant_id=tenant_id, point_id=point_id, with_vectors=with_vectors)
+    if point is None:
+        raise PointNotFoundError(f"point {point_id} not found")
+    return point
+
+
+async def list_file_points(
+    repository: PointRepository,
+    *,
+    tenant_id: uuid.UUID,
+    file_id: uuid.UUID,
+    limit: int,
+    cursor: str | None = None,
+    include_inactive: bool = False,
+) -> PointPage:
+    """One file's points in display (`order_id`) order.
+
+    An unknown or foreign `file_id` yields an empty page rather than an error:
+    the two are indistinguishable to the caller, which is the same
+    non-disclosure property `get_point` gets from raising.
+    """
+    return await repository.list_by_file(
+        tenant_id=tenant_id,
+        file_id=file_id,
+        limit=limit,
+        cursor=cursor,
+        include_inactive=include_inactive,
+    )
+
+
+async def count_points(
+    repository: PointRepository,
+    *,
+    tenant_id: uuid.UUID,
+    domain: str | None = None,
+    file_id: uuid.UUID | None = None,
+    include_inactive: bool = False,
+) -> int:
+    return await repository.count(
+        tenant_id=tenant_id,
+        domain=domain,
+        file_id=file_id,
+        include_inactive=include_inactive,
+    )
+
+
+async def search_points(
+    repository: PointRepository,
+    *,
+    tenant_id: uuid.UUID,
+    query: str,
+    limit: int,
+    cursor: str | None = None,
+    domain: str | None = None,
+    file_id: uuid.UUID | None = None,
+    include_inactive: bool = False,
+) -> PointPage:
+    """Keyword match on `content`, within this tenant.
+
+    **Not semantic retrieval.** Qdrant's full-text index filters rather than
+    scores, so results carry no relevance ranking and their order is
+    unspecified. Ranked retrieval is ADR-0003's hybrid path in plan 003; this
+    function must not grow a semantic mode (ADR-0002).
+    """
+    return await repository.keyword_search(
+        tenant_id=tenant_id,
+        query=normalize_persian_text(query),
+        limit=limit,
+        cursor=cursor,
+        domain=domain,
+        file_id=file_id,
+        include_inactive=include_inactive,
+    )

src/application/points/relinking.py

@@ -0,0 +1,135 @@
+"""Adjacency-pointer maintenance for a point leaving a file's sequence.
+
+ADR-0001 keeps `previous_chunk_id`/`next_chunk_id` on every point so ADR-0003's
+context-window expansion can walk a file in O(1) steps. ADR-0002 makes keeping
+them correct an obligation of every operation that changes a point's position:
+a partial relink is a defect, not a degraded-but-acceptable outcome.
+
+The function below is the primitive that obligation reduces to. It is pure, and
+it is written as **"what is still missing between the state I just read and the
+state I want"** rather than "the patches a delete implies". That framing is what
+makes the caller's retry loop correct: re-planning after a partial apply emits
+exactly the patches that did not land, and re-planning after a completed delete
+emits nothing at all. The three cases the plan calls out — a normal delete, a
+second delete of an already-inactive point, and recovery from a half-applied
+batch — are then one code path instead of three.
+
+Note what is deliberately *not* patched: the departing point's own
+`previous_chunk_id`/`next_chunk_id`. Nothing active points at it once its
+neighbours are relinked, so those pointers are unreachable rather than stale,
+and leaving them records where the point sat — which is what a later restore or
+an audit reader would need. `src/application/points/deletion.py` relies on that
+when it re-plans: the departing point's pointers are the only surviving record
+of which two neighbours have to be joined.
+"""
+
+import uuid
+from collections.abc import Mapping
+from datetime import datetime
+
+from src.application.points.point import Point
+from src.application.ports.point_repository import PayloadPatch
+
+
+def _optional_id(value: uuid.UUID | None) -> str | None:
+    return str(value) if value is not None else None
+
+
+def _provenance(point: Point, *, actor: str, now: datetime) -> dict[str, object]:
+    """The fields every mutation writes: who, when, and the next version.
+
+    Bumping `version` on a relinked *neighbour* is intentional. The neighbour's
+    payload really did change, so a concurrent editor holding the old version
+    must get a `409` rather than overwrite the pointer we just fixed.
+    """
+    return {
+        "updated_at": now.isoformat(),
+        "updated_by": actor,
+        "version": point.version + 1,
+    }
+
+
+def neighbour_ids(point: Point) -> tuple[uuid.UUID, ...]:
+    """The ids `patches_for_removal` needs loaded, skipping the nulls."""
+    return tuple(
+        pointer for pointer in (point.previous_chunk_id, point.next_chunk_id) if pointer is not None
+    )
+
+
+def patches_for_removal(
+    point: Point,
+    neighbours: Mapping[uuid.UUID, Point],
+    *,
+    actor: str,
+    now: datetime,
+) -> tuple[PayloadPatch, ...]:
+    """The patches still needed to remove `point` from its file's sequence.
+
+    Returns an empty tuple when the removal is already complete, which the
+    caller reads as both "converged" and "this was a no-op".
+
+    A neighbour absent from `neighbours` is skipped rather than patched blind:
+    its id came from the departing point's payload, so a missing one means the
+    chain was already broken, and inventing a patch for a point that is not
+    there would not fix it. The caller logs that case.
+    """
+    patches: list[PayloadPatch] = []
+
+    if point.is_active:
+        patches.append(
+            PayloadPatch(
+                point_id=point.point_id,
+                payload={
+                    "is_active": False,
+                    "deleted_at": now.isoformat(),
+                    **_provenance(point, actor=actor, now=now),
+                },
+                expected_version=point.version,
+            )
+        )
+
+    previous = neighbours.get(point.previous_chunk_id) if point.previous_chunk_id else None
+    if previous is not None and previous.next_chunk_id != point.next_chunk_id:
+        patches.append(
+            PayloadPatch(
+                point_id=previous.point_id,
+                payload={
+                    "next_chunk_id": _optional_id(point.next_chunk_id),
+                    **_provenance(previous, actor=actor, now=now),
+                },
+                expected_version=previous.version,
+            )
+        )
+
+    following = neighbours.get(point.next_chunk_id) if point.next_chunk_id else None
+    if following is not None and following.previous_chunk_id != point.previous_chunk_id:
+        patches.append(
+            PayloadPatch(
+                point_id=following.point_id,
+                payload={
+                    "previous_chunk_id": _optional_id(point.previous_chunk_id),
+                    **_provenance(following, actor=actor, now=now),
+                },
+                expected_version=following.version,
+            )
+        )
+
+    return tuple(patches)
+
+
+def patch_for_deactivation(point: Point, *, actor: str, now: datetime) -> PayloadPatch:
+    """Deactivate one point without touching any pointer.
+
+    Used by the whole-file sweep, where every point in the file leaves at once:
+    no active point survives to dangle, so there is no neighbour to relink and
+    the chain stays intact for a later reader of the deactivated file.
+    """
+    return PayloadPatch(
+        point_id=point.point_id,
+        payload={
+            "is_active": False,
+            "deleted_at": now.isoformat(),
+            **_provenance(point, actor=actor, now=now),
+        },
+        expected_version=point.version,
+    )

src/application/ports/__init__.py

@@ -0,0 +1,7 @@
+"""Narrow contracts for external side effects (ADR-0015).
+
+Ports exist for external side effects/persistence that need a swappable or
+fake-able boundary — not as a blanket wrapper around every database access.
+`object_storage.py` is one: MinIO is a real external system with its own
+failure modes, and ADR-0016 requires a hand-written fake for it in tests.
+"""

src/application/ports/embedding.py

@@ -0,0 +1,63 @@
+"""Embedding ports (ADR-0001, ADR-0017).
+
+`src/infrastructure/embedding/` holds the production adapters; tests use
+scripted fakes (ADR-0016). Application code depends on these Protocols, not
+on `httpx`/provider SDKs directly.
+"""
+
+from collections.abc import Sequence
+from typing import Protocol
+
+from src.application.ingestion.models import SparseVector
+
+
+class DenseEmbedder(Protocol):
+    """One named dense vector's embedding client (`dense_nomic`/`dense_openai`).
+
+    `embed_batch` is a single batched network call — callers own concurrency
+    bounding (ADR-0017's `embed_concurrency` semaphore), not this Protocol.
+    """
+
+    name: str
+    model_version: str
+    """Identifies the model that produced these vectors (ADR-0001).
+
+    Written into every point's `embedding_model_version` payload field, which
+    exists so a future model swap can tell which chunks need re-embedding. The
+    embedder is what knows this, so it is reported here rather than
+    reconstructed from configuration at the call site.
+    """
+
+    async def embed_batch(self, texts: Sequence[str]) -> list[list[float]]:
+        """Return one vector per input text, same order. Raises `EmbedderError`
+        (see `src/application/ingestion/errors.py`) on transport/response
+        failure.
+        """
+        ...
+
+
+class SparseEmbedder(Protocol):
+    """The `sparse` (BM25) vector's embedding client.
+
+    Blocking/CPU-bound (ADR-0017): callers offload it via
+    `anyio.to_thread.run_sync` with the ingestion `CapacityLimiter`, not call
+    it directly from an `async def`.
+    """
+
+    name: str
+    model_version: str
+    """Identifies the analyzer/parameters that produced these vectors.
+
+    Same purpose as `DenseEmbedder.model_version`; for BM25 the "model" is the
+    analyzer choice (ADR-0005), which is equally a re-embedding trigger.
+    """
+
+    def embed_batch(self, texts: Sequence[str], *, query: bool = False) -> list[SparseVector]:
+        """Return one sparse vector per input text, same order.
+
+        `query=True` selects the query-side weighting, which omits document
+        length normalization. Ingestion always passes `False`; the flag exists
+        so retrieval (ADR-0003) encodes queries through this same port rather
+        than growing a second, silently divergent implementation.
+        """
+        ...

src/application/ports/object_storage.py

@@ -0,0 +1,14 @@
+"""The object-storage port (ADR-0013).
+
+`src/infrastructure/minio/storage.py` is the production adapter; tests use a
+hand-written fake (ADR-0016). Application code depends on this Protocol, not
+on the `minio` SDK.
+"""
+
+from typing import Protocol
+
+
+class ObjectStorage(Protocol):
+    async def put_object(self, *, key: str, data: bytes, content_type: str) -> None:
+        """Store `data` privately under `key`. Overwrites an existing object."""
+        ...

src/application/ports/point_repository.py

@@ -0,0 +1,131 @@
+"""The point read/edit port for `/v1/points` (ADR-0002, ADR-0015).
+
+Separate from `PointStorage`, which stays exactly the two bulk operations
+ingestion performs. Reads, single-point edits, reordering, and keyword search
+have a different caller, a different failure vocabulary, and a different
+tenant-filter obligation, so they get their own port rather than accreting onto
+the ingestion one.
+
+`tenant_id` is a required keyword argument on **every** method. That is not
+style: ADR-0002's isolation rule has to hold on every code path that touches the
+collection, and an optional tenant filter is one forgotten argument away from a
+cross-tenant read. Making it required moves that from a review question to a
+type error.
+
+`src/infrastructure/qdrant/point_repository.py` is the production adapter;
+`tests.fakes.FakePointRepository` is the test double.
+"""
+
+import uuid
+from collections.abc import Sequence
+from typing import Protocol
+
+from pydantic import BaseModel
+
+from src.application.points.point import Point
+
+
+class PointPage(BaseModel):
+    """One page of points plus the cursor that continues it.
+
+    `next_cursor` is opaque to callers and encoded by the adapter: ordered
+    scrolls and keyword searches paginate by different Qdrant mechanisms, and
+    neither is a plain integer offset. `None` means the listing is exhausted.
+    """
+
+    points: tuple[Point, ...]
+    next_cursor: str | None = None
+
+
+class PayloadPatch(BaseModel):
+    """Set these payload fields on one point, optionally guarded by `version`.
+
+    When `expected_version` is set, the adapter attaches it to the operation's
+    filter, so a concurrent write that has already moved the version on means
+    this patch matches nothing rather than clobbering it. The guard is what
+    makes a lost update impossible; detecting that it fired is the service's
+    job (see `apply_patches`).
+    """
+
+    point_id: uuid.UUID
+    payload: dict[str, object]
+    expected_version: int | None = None
+
+
+class PointRepository(Protocol):
+    async def get(
+        self, *, tenant_id: uuid.UUID, point_id: uuid.UUID, with_vectors: bool = False
+    ) -> Point | None:
+        """One point, or `None` if it does not exist *under this tenant*.
+
+        The two cases are deliberately indistinguishable — the route maps both
+        to `404` so a caller cannot probe for another tenant's point ids.
+        """
+        ...
+
+    async def get_many(
+        self, *, tenant_id: uuid.UUID, point_ids: Sequence[uuid.UUID]
+    ) -> tuple[Point, ...]:
+        """The subset of `point_ids` that exists under this tenant.
+
+        Order is not guaranteed and missing ids are silently absent: callers are
+        neighbour-relinking and batch precondition checks, both of which match
+        on id rather than position.
+        """
+        ...
+
+    async def list_by_file(
+        self,
+        *,
+        tenant_id: uuid.UUID,
+        file_id: uuid.UUID,
+        limit: int,
+        cursor: str | None = None,
+        include_inactive: bool = False,
+    ) -> PointPage:
+        """One file's points in `order_id` order (ADR-0008's `scroll`).
+
+        Scoped to a single file because the cursor is an `order_id` value, and
+        `order_id` is only unique within a file.
+        """
+        ...
+
+    async def count(
+        self,
+        *,
+        tenant_id: uuid.UUID,
+        domain: str | None = None,
+        file_id: uuid.UUID | None = None,
+        include_inactive: bool = False,
+    ) -> int: ...
+
+    async def keyword_search(
+        self,
+        *,
+        tenant_id: uuid.UUID,
+        query: str,
+        limit: int,
+        cursor: str | None = None,
+        domain: str | None = None,
+        file_id: uuid.UUID | None = None,
+        include_inactive: bool = False,
+    ) -> PointPage:
+        """Full-text payload match on `content`, plus structured filters.
+
+        Keyword matching, **not** semantic retrieval (ADR-0002). Qdrant's
+        full-text index is a filter, not a scorer, so results carry no relevance
+        ranking and their order is unspecified.
+        """
+        ...
+
+    async def apply_patches(self, *, tenant_id: uuid.UUID, patches: Sequence[PayloadPatch]) -> None:
+        """Apply every patch in one Qdrant `points/batch` request.
+
+        Qdrant has no multi-point transaction, so this is not atomic and does
+        not pretend to be. ADR-0002's all-or-nothing rule is implemented one
+        layer up as validate-every-precondition-then-apply; the per-patch
+        `expected_version` guard here is what makes the residual window safe,
+        turning a lost update into a no-op the service can detect rather than a
+        silent clobber.
+        """
+        ...

src/application/ports/point_storage.py

@@ -0,0 +1,45 @@
+"""The point-storage port (ADR-0001, ADR-0015).
+
+`src/infrastructure/qdrant/points.py` is the production adapter; tests use a
+hand-written fake (ADR-0016). Application code depends on this Protocol, not on
+the `qdrant_client` SDK.
+
+Deliberately narrow: exactly the two operations ingestion performs. Reads,
+single-point edits, reordering, and keyword search are plan 002's `/v1/points`
+surface and belong on a port of their own rather than accreting here.
+"""
+
+import uuid
+from collections.abc import Sequence
+from datetime import datetime
+from typing import Protocol
+
+from src.application.points.models import ChunkPoint
+
+
+class PointStorage(Protocol):
+    async def upsert_points(self, points: Sequence[ChunkPoint]) -> None:
+        """Upsert one batch of points.
+
+        Callers own batching and concurrency bounding (ADR-0017's
+        `upsert_concurrency` semaphore), not this Protocol — the same division
+        `DenseEmbedder.embed_batch` uses.
+        """
+        ...
+
+    async def deactivate_points_from_index(
+        self,
+        *,
+        tenant_id: uuid.UUID,
+        file_id: uuid.UUID,
+        from_chunk_index: int,
+        deleted_at: datetime,
+        updated_by: str,
+    ) -> int:
+        """Soft-delete this file's points at or past `from_chunk_index`.
+
+        Sets `is_active=false`/`deleted_at` rather than removing the points
+        (ADR-0002: delete is soft by default). Tenant-filtered — a `file_id`
+        alone is never sufficient authority. Returns how many points matched.
+        """
+        ...

src/application/tenants/__init__.py

@@ -0,0 +1,9 @@
+"""Operator-run tenant provisioning (ADR-0008, ADR-0009)."""
+
+from src.application.tenants.provisioning import (
+    DEFAULT_SCOPES,
+    ProvisionResult,
+    provision_tenant,
+)
+
+__all__ = ["DEFAULT_SCOPES", "ProvisionResult", "provision_tenant"]

src/application/tenants/provisioning.py

@@ -0,0 +1,129 @@
+"""Provision a tenant, its first API key, and its domains (ADR-0008, ADR-0009).
+
+Nothing in the HTTP surface can bootstrap a tenant: every `/v1` route needs a
+key, and a key can only exist once a tenant does. That chicken-and-egg is why
+this is an operator-run deployment step (`src/cli/provision_tenant.py`) rather
+than an endpoint — the same reasoning that keeps `alembic upgrade head` and
+`qdrant_bootstrap` off the request path.
+
+This is the package's only caller-facing entry point. It owns the whole
+composition — tenant reuse-or-create, key generation and hashing, domain
+registration, and the single transaction the three share — so a caller cannot
+get the order wrong or commit a key whose tenant never landed (CLAUDE.md,
+"prefer deep modules"). The plaintext key is returned exactly once and is never
+logged (ADR-0011 forbids plaintext keys in logs); only its non-secret
+`key_prefix` appears in the event.
+"""
+
+import uuid
+from dataclasses import dataclass
+
+import structlog
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from src.application.auth.keys import generate_api_key, hash_secret
+from src.infrastructure.postgres.repositories import api_keys as api_keys_repo
+from src.infrastructure.postgres.repositories import tenant_domains as domains_repo
+from src.infrastructure.postgres.repositories import tenants as tenants_repo
+
+logger = structlog.get_logger(__name__)
+
+DEFAULT_SCOPES = (
+    "files:write",
+    "domains:read",
+    "domains:write",
+    "points:read",
+    "points:write",
+)
+
+
+@dataclass(frozen=True)
+class ProvisionResult:
+    tenant_id: uuid.UUID
+    tenant_slug: str
+    tenant_created: bool
+    api_key_id: uuid.UUID
+    api_key_prefix: str
+    api_key: str
+    """The plaintext bearer token. Only ever returned here — never stored, never logged."""
+    domains_created: tuple[str, ...]
+    domains_existing: tuple[str, ...]
+
+
+async def provision_tenant(
+    sessionmaker: async_sessionmaker[AsyncSession],
+    *,
+    slug: str,
+    name: str | None = None,
+    key_name: str = "bootstrap",
+    scopes: tuple[str, ...] = DEFAULT_SCOPES,
+    domains: tuple[str, ...] = (),
+    actor_type: str = "backend",
+) -> ProvisionResult:
+    """Create (or reuse) the tenant, issue a key, and register `domains`.
+
+    Re-running with the same `slug` reuses the tenant and its existing domains
+    rather than failing, so an operator can add a key to a live tenant with the
+    same command they used to create it. A *new* key is issued on every run —
+    keys are write-once by construction (only the hash is stored), so there is
+    nothing to return for an existing one.
+    """
+    key_prefix, secret, full_key = generate_api_key()
+
+    async with sessionmaker() as session:
+        tenant = await tenants_repo.get_by_slug(session, slug)
+        tenant_created = tenant is None
+        if tenant is None:
+            tenant = tenants_repo.create(session, slug=slug, name=name or slug)
+            # `api_keys.tenant_id` and `tenant_domains.tenant_id` FK to this row
+            # and the mapped classes carry no ORM relationship for the unit of
+            # work to order by itself, so the insert has to land first.
+            await session.flush()
+
+        api_key = api_keys_repo.create(
+            session,
+            tenant_id=tenant.id,
+            name=key_name,
+            key_prefix=key_prefix,
+            key_hash=hash_secret(secret),
+            scopes=list(scopes),
+            actor_type=actor_type,
+            created_by="cli:provision_tenant",
+        )
+
+        created: list[str] = []
+        existing: list[str] = []
+        for domain in domains:
+            if await domains_repo.get(session, tenant_id=tenant.id, domain=domain) is not None:
+                existing.append(domain)
+                continue
+            domains_repo.create(session, tenant_id=tenant.id, domain=domain, display_name=domain)
+            created.append(domain)
+
+        await session.flush()
+        tenant_id, api_key_id, tenant_slug = tenant.id, api_key.id, tenant.slug
+        await session.commit()
+
+    if tenant_created:
+        logger.info("tenant.provisioned", tenant_id=str(tenant_id), tenant_slug=tenant_slug)
+    for domain in created:
+        logger.info("domain.created", tenant_id=str(tenant_id), domain=domain)
+    logger.info(
+        "api_key.provisioned",
+        tenant_id=str(tenant_id),
+        api_key_id=str(api_key_id),
+        key_prefix=key_prefix,
+        scopes=list(scopes),
+        actor_type=actor_type,
+    )
+
+    return ProvisionResult(
+        tenant_id=tenant_id,
+        tenant_slug=tenant_slug,
+        tenant_created=tenant_created,
+        api_key_id=api_key_id,
+        api_key_prefix=key_prefix,
+        api_key=full_key,
+        domains_created=tuple(created),
+        domains_existing=tuple(existing),
+    )