feat(ingestion): add bounded, benchmark-aligned embedding execution
Why: - Plan 001 Phase 4 needs batched, concurrency-bounded embedding wired into the inline upload path, with process-wide capacity/timeout/chunk-limit guards (ADR-0017). - The BM25 analyzer and dense-model config are ported from the `emet` evaluation lab, which benchmarked them against the real Farsi corpus (bm25-fa-norm-stop; nomic-embed-text-v2-moe at 768-dim; text-embedding-3-large at native 3072-dim), closing open items in ADR-0001/ADR-0005. Changes: - New: embedding ports, orchestration (embed_chunks), request-bounds helpers, and dense/sparse adapters (analyzers.py, bm25.py, openai_compatible.py). - upload.py now parses/chunks/embeds inline behind INGESTION_MAX_CONCURRENCY (503), INGESTION_TIMEOUT_SECONDS (504), and the chunk-count ceiling (413); every failure path still writes a terminal job row. - Lifespan builds and warms both dense embedders at startup (fail-soft) and creates the sparse embedder and concurrency semaphore. - httpx moves from dev to main dependencies (adapters use it directly). Impact: - Qdrant point upserts are still Phase 5 -- chunks_indexed stays 0. - New EMBEDDING_* env vars documented in .env.example; safe defaults. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,12 +1,13 @@
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from anyio import CapacityLimiter
|
||||
from anyio import CapacityLimiter, Semaphore
|
||||
from fastapi import Request
|
||||
from minio import Minio
|
||||
from qdrant_client import AsyncQdrantClient
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||||
|
||||
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||
from src.application.ports.object_storage import ObjectStorage
|
||||
from src.config import Settings
|
||||
|
||||
@@ -20,6 +21,9 @@ class AppResources:
|
||||
qdrant_client: AsyncQdrantClient
|
||||
object_storage: ObjectStorage
|
||||
ingestion_limiter: CapacityLimiter
|
||||
dense_embedders: Sequence[DenseEmbedder]
|
||||
sparse_embedder: SparseEmbedder
|
||||
ingestion_concurrency_limiter: Semaphore
|
||||
|
||||
|
||||
def _resources(request: Request) -> AppResources:
|
||||
@@ -46,6 +50,18 @@ def get_ingestion_limiter(request: Request) -> CapacityLimiter:
|
||||
return _resources(request).ingestion_limiter
|
||||
|
||||
|
||||
def get_dense_embedders(request: Request) -> Sequence[DenseEmbedder]:
|
||||
return _resources(request).dense_embedders
|
||||
|
||||
|
||||
def get_sparse_embedder(request: Request) -> SparseEmbedder:
|
||||
return _resources(request).sparse_embedder
|
||||
|
||||
|
||||
def get_ingestion_concurrency_limiter(request: Request) -> Semaphore:
|
||||
return _resources(request).ingestion_concurrency_limiter
|
||||
|
||||
|
||||
def get_sessionmaker(request: Request) -> async_sessionmaker[AsyncSession]:
|
||||
"""The session *factory*, not a request-scoped session.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user