feat(ingestion): add bounded, benchmark-aligned embedding execution
Why: - Plan 001 Phase 4 needs batched, concurrency-bounded embedding wired into the inline upload path, with process-wide capacity/timeout/chunk-limit guards (ADR-0017). - The BM25 analyzer and dense-model config are ported from the `emet` evaluation lab, which benchmarked them against the real Farsi corpus (bm25-fa-norm-stop; nomic-embed-text-v2-moe at 768-dim; text-embedding-3-large at native 3072-dim), closing open items in ADR-0001/ADR-0005. Changes: - New: embedding ports, orchestration (embed_chunks), request-bounds helpers, and dense/sparse adapters (analyzers.py, bm25.py, openai_compatible.py). - upload.py now parses/chunks/embeds inline behind INGESTION_MAX_CONCURRENCY (503), INGESTION_TIMEOUT_SECONDS (504), and the chunk-count ceiling (413); every failure path still writes a terminal job row. - Lifespan builds and warms both dense embedders at startup (fail-soft) and creates the sparse embedder and concurrency semaphore. - httpx moves from dev to main dependencies (adapters use it directly). Impact: - Qdrant point upserts are still Phase 5 -- chunks_indexed stays 0. - New EMBEDDING_* env vars documented in .env.example; safe defaults. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
78
tests/unit/application/test_bounds.py
Normal file
78
tests/unit/application/test_bounds.py
Normal file
@@ -0,0 +1,78 @@
|
||||
"""Request bounds for inline ingestion: chunk ceiling and the process-wide
|
||||
concurrency gate (ADR-0017, plan 001 Phase 4).
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from anyio import Semaphore
|
||||
|
||||
from src.application.ingestion.bounds import acquire_ingestion_slot, enforce_chunk_limit
|
||||
from src.application.ingestion.errors import ChunkLimitExceededError, IngestionAtCapacityError
|
||||
from src.application.ingestion.models import Chunk, ContentType
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _chunk(index: int) -> Chunk:
|
||||
return Chunk(
|
||||
chunk_id=uuid.uuid4(),
|
||||
chunk_index=index,
|
||||
order_id=float(index + 1),
|
||||
content="x",
|
||||
content_type=ContentType.PARAGRAPH,
|
||||
token_count=1,
|
||||
character_count=1,
|
||||
)
|
||||
|
||||
|
||||
def test_enforce_chunk_limit_within_bound_does_not_raise() -> None:
|
||||
enforce_chunk_limit([_chunk(0), _chunk(1)], max_chunks=2)
|
||||
|
||||
|
||||
def test_enforce_chunk_limit_over_bound_raises() -> None:
|
||||
with pytest.raises(ChunkLimitExceededError):
|
||||
enforce_chunk_limit([_chunk(0), _chunk(1), _chunk(2)], max_chunks=2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acquire_ingestion_slot_allows_up_to_the_limit() -> None:
|
||||
limiter = Semaphore(2)
|
||||
|
||||
async with acquire_ingestion_slot(limiter), acquire_ingestion_slot(limiter):
|
||||
pass # two concurrent holders within a limit of two: no rejection
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acquire_ingestion_slot_rejects_beyond_the_limit() -> None:
|
||||
limiter = Semaphore(1)
|
||||
|
||||
async with acquire_ingestion_slot(limiter):
|
||||
with pytest.raises(IngestionAtCapacityError):
|
||||
async with acquire_ingestion_slot(limiter):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acquire_ingestion_slot_releases_on_exit() -> None:
|
||||
limiter = Semaphore(1)
|
||||
|
||||
async with acquire_ingestion_slot(limiter):
|
||||
pass
|
||||
|
||||
# The slot from the first `async with` must be released by the time it
|
||||
# exits, or every subsequent request would see permanent capacity loss.
|
||||
async with acquire_ingestion_slot(limiter):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acquire_ingestion_slot_releases_after_body_raises() -> None:
|
||||
limiter = Semaphore(1)
|
||||
|
||||
with pytest.raises(ValueError, match="boom"):
|
||||
async with acquire_ingestion_slot(limiter):
|
||||
raise ValueError("boom")
|
||||
|
||||
async with acquire_ingestion_slot(limiter):
|
||||
pass
|
||||
Reference in New Issue
Block a user