feat(ingestion): add bounded, benchmark-aligned embedding execution
Why: - Plan 001 Phase 4 needs batched, concurrency-bounded embedding wired into the inline upload path, with process-wide capacity/timeout/chunk-limit guards (ADR-0017). - The BM25 analyzer and dense-model config are ported from the `emet` evaluation lab, which benchmarked them against the real Farsi corpus (bm25-fa-norm-stop; nomic-embed-text-v2-moe at 768-dim; text-embedding-3-large at native 3072-dim), closing open items in ADR-0001/ADR-0005. Changes: - New: embedding ports, orchestration (embed_chunks), request-bounds helpers, and dense/sparse adapters (analyzers.py, bm25.py, openai_compatible.py). - upload.py now parses/chunks/embeds inline behind INGESTION_MAX_CONCURRENCY (503), INGESTION_TIMEOUT_SECONDS (504), and the chunk-count ceiling (413); every failure path still writes a terminal job row. - Lifespan builds and warms both dense embedders at startup (fail-soft) and creates the sparse embedder and concurrency semaphore. - httpx moves from dev to main dependencies (adapters use it directly). Impact: - Qdrant point upserts are still Phase 5 -- chunks_indexed stays 0. - New EMBEDDING_* env vars documented in .env.example; safe defaults. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
48
src/application/ingestion/bounds.py
Normal file
48
src/application/ingestion/bounds.py
Normal file
@@ -0,0 +1,48 @@
|
||||
"""Request bounds for inline ingestion (ADR-0017).
|
||||
|
||||
Three independent bounds, each mapping to its own status code: the chunk
|
||||
ceiling (`413`, checked before embedding starts), process-wide concurrency
|
||||
(`503` + `Retry-After`, rejected rather than queued), and the work-phase
|
||||
deadline (`504`, and the caller must still write a terminal job status).
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from anyio import Semaphore, WouldBlock
|
||||
|
||||
from src.application.ingestion.errors import ChunkLimitExceededError, IngestionAtCapacityError
|
||||
from src.application.ingestion.models import Chunk
|
||||
|
||||
|
||||
def enforce_chunk_limit(chunks: Sequence[Chunk], *, max_chunks: int) -> None:
|
||||
"""Raise `ChunkLimitExceededError` if `chunks` exceeds `max_chunks`.
|
||||
|
||||
Call this immediately after parsing/chunking and before any embedding
|
||||
call — the ceiling must be discovered up front, not mid-batch.
|
||||
"""
|
||||
if len(chunks) > max_chunks:
|
||||
raise ChunkLimitExceededError(
|
||||
f"document produced {len(chunks)} chunks, over the {max_chunks}-chunk limit"
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def acquire_ingestion_slot(limiter: Semaphore) -> AsyncIterator[None]:
|
||||
"""Hold one of `INGESTION_MAX_CONCURRENCY` process-wide slots for the block.
|
||||
|
||||
`limiter` is an `anyio.Semaphore` created once in the lifespan. Rejects
|
||||
immediately with `IngestionAtCapacityError` when the process is already at
|
||||
capacity, rather than queueing the request behind an unbounded wait
|
||||
(ADR-0017) — the semaphore's own async `acquire()` would do the latter.
|
||||
"""
|
||||
try:
|
||||
limiter.acquire_nowait()
|
||||
except WouldBlock:
|
||||
raise IngestionAtCapacityError(
|
||||
"ingestion is at capacity; retry after the configured backoff"
|
||||
) from None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
limiter.release()
|
||||
Reference in New Issue
Block a user