feat(ingestion): add bounded, benchmark-aligned embedding execution

Why:
- Plan 001 Phase 4 needs batched, concurrency-bounded embedding wired into
  the inline upload path, with process-wide capacity/timeout/chunk-limit
  guards (ADR-0017).
- The BM25 analyzer and dense-model config are ported from the `emet`
  evaluation lab, which benchmarked them against the real Farsi corpus
  (bm25-fa-norm-stop; nomic-embed-text-v2-moe at 768-dim; text-embedding-3-large
  at native 3072-dim), closing open items in ADR-0001/ADR-0005.

Changes:
- New: embedding ports, orchestration (embed_chunks), request-bounds
  helpers, and dense/sparse adapters (analyzers.py, bm25.py,
  openai_compatible.py).
- upload.py now parses/chunks/embeds inline behind INGESTION_MAX_CONCURRENCY
  (503), INGESTION_TIMEOUT_SECONDS (504), and the chunk-count ceiling (413);
  every failure path still writes a terminal job row.
- Lifespan builds and warms both dense embedders at startup (fail-soft) and
  creates the sparse embedder and concurrency semaphore.
- httpx moves from dev to main dependencies (adapters use it directly).

Impact:
- Qdrant point upserts are still Phase 5 -- chunks_indexed stays 0.
- New EMBEDDING_* env vars documented in .env.example; safe defaults.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/api/errors.py

@@ -17,10 +17,22 @@ from src.application.auth.errors import (
     TenantInactiveError,
 )
 from src.application.files.errors import FileTooLargeError, InvalidUploadError
-from src.application.ingestion.errors import DocumentParseError, UnsupportedSourceTypeError
+from src.application.ingestion.errors import (
+    ChunkLimitExceededError,
+    DocumentParseError,
+    EmbedderError,
+    IngestionAtCapacityError,
+    IngestionTimeoutError,
+    UnsupportedSourceTypeError,
+)
 
 logger = structlog.get_logger(__name__)
 
+# A fixed backoff hint, not a computed retry budget: ADR-0017 rejects a
+# request outright at capacity rather than queueing it, so there is no
+# in-process estimate of when a slot will free up to report instead.
+_CAPACITY_RETRY_AFTER_SECONDS = 1
+
 # (exception type, status code, stable error code)
 _MAPPING: tuple[tuple[type[Exception], int, str], ...] = (
     (InvalidApiKeyError, status.HTTP_401_UNAUTHORIZED, "invalid_api_key"),
@@ -30,6 +42,9 @@ _MAPPING: tuple[tuple[type[Exception], int, str], ...] = (
     (DocumentParseError, status.HTTP_400_BAD_REQUEST, "validation_error"),
     (UnsupportedSourceTypeError, status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, "unsupported_media_type"),
     (FileTooLargeError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
+    (ChunkLimitExceededError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
+    (EmbedderError, status.HTTP_502_BAD_GATEWAY, "embedder_error"),
+    (IngestionTimeoutError, status.HTTP_504_GATEWAY_TIMEOUT, "ingestion_timeout"),
 )
 
 
@@ -66,6 +81,14 @@ def register_exception_handlers(app: FastAPI) -> None:
 
         app.add_exception_handler(exc_type, _handler)
 
+    @app.exception_handler(IngestionAtCapacityError)
+    def _capacity_handler(request: Request, exc: IngestionAtCapacityError) -> JSONResponse:
+        return JSONResponse(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            content=_envelope("ingestion_at_capacity", str(exc), _request_id(request)),
+            headers={"Retry-After": str(_CAPACITY_RETRY_AFTER_SECONDS)},
+        )
+
     @app.exception_handler(RequestValidationError)
     def _validation_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
         return JSONResponse(

src/api/routers/files.py

@@ -5,9 +5,10 @@ touch MinIO/Qdrant, or otherwise carry ingestion business logic (ADR-0015).
 """
 
 import uuid
+from collections.abc import Sequence
 from typing import Annotated
 
-from anyio import CapacityLimiter
+from anyio import CapacityLimiter, Semaphore
 from fastapi import APIRouter, Depends, Form, HTTPException, Response, UploadFile, status
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 
@@ -16,12 +17,16 @@ from src.api.schemas.files import FileStatusResponse, FileUploadResponse
 from src.application.auth.context import AuthContext
 from src.application.files.status import get_file_status
 from src.application.files.upload import upload_source_file
+from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
 from src.application.ports.object_storage import ObjectStorage
 from src.bootstrap.dependencies import (
+    get_dense_embedders,
+    get_ingestion_concurrency_limiter,
     get_ingestion_limiter,
     get_object_storage,
     get_sessionmaker,
     get_settings,
+    get_sparse_embedder,
 )
 from src.config import Settings
 
@@ -32,6 +37,9 @@ _SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessi
 _ObjectStorageDep = Annotated[ObjectStorage, Depends(get_object_storage)]
 _SettingsDep = Annotated[Settings, Depends(get_settings)]
 _IngestionLimiterDep = Annotated[CapacityLimiter, Depends(get_ingestion_limiter)]
+_ConcurrencyLimiterDep = Annotated[Semaphore, Depends(get_ingestion_concurrency_limiter)]
+_DenseEmbeddersDep = Annotated[Sequence[DenseEmbedder], Depends(get_dense_embedders)]
+_SparseEmbedderDep = Annotated[SparseEmbedder, Depends(get_sparse_embedder)]
 
 
 @router.post("", status_code=status.HTTP_201_CREATED)
@@ -44,6 +52,9 @@ async def upload_file(
     storage: _ObjectStorageDep,
     settings: _SettingsDep,
     limiter: _IngestionLimiterDep,
+    concurrency_limiter: _ConcurrencyLimiterDep,
+    dense_embedders: _DenseEmbeddersDep,
+    sparse_embedder: _SparseEmbedderDep,
 ) -> FileUploadResponse:
     data = await file.read()
     result = await upload_source_file(
@@ -53,9 +64,12 @@ async def upload_file(
         domain=domain,
         filename=file.filename or "",
         data=data,
-        max_upload_size_bytes=settings.ingestion.max_upload_size_bytes,
-        chunking_strategy=settings.chunking.strategy,
-        validation_limiter=limiter,
+        ingestion_settings=settings.ingestion,
+        chunking_settings=settings.chunking,
+        thread_limiter=limiter,
+        concurrency_limiter=concurrency_limiter,
+        dense_embedders=dense_embedders,
+        sparse_embedder=sparse_embedder,
     )
     if not result.is_new_attempt:
         response.status_code = status.HTTP_200_OK