feat(ingestion): add bounded, benchmark-aligned embedding execution

Why:
- Plan 001 Phase 4 needs batched, concurrency-bounded embedding wired into
  the inline upload path, with process-wide capacity/timeout/chunk-limit
  guards (ADR-0017).
- The BM25 analyzer and dense-model config are ported from the `emet`
  evaluation lab, which benchmarked them against the real Farsi corpus
  (bm25-fa-norm-stop; nomic-embed-text-v2-moe at 768-dim; text-embedding-3-large
  at native 3072-dim), closing open items in ADR-0001/ADR-0005.

Changes:
- New: embedding ports, orchestration (embed_chunks), request-bounds
  helpers, and dense/sparse adapters (analyzers.py, bm25.py,
  openai_compatible.py).
- upload.py now parses/chunks/embeds inline behind INGESTION_MAX_CONCURRENCY
  (503), INGESTION_TIMEOUT_SECONDS (504), and the chunk-count ceiling (413);
  every failure path still writes a terminal job row.
- Lifespan builds and warms both dense embedders at startup (fail-soft) and
  creates the sparse embedder and concurrency semaphore.
- httpx moves from dev to main dependencies (adapters use it directly).

Impact:
- Qdrant point upserts are still Phase 5 -- chunks_indexed stays 0.
- New EMBEDDING_* env vars documented in .env.example; safe defaults.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-19 17:13:32 +03:30
parent aa6d595424
commit 5c0a5938f8
33 changed files with 2455 additions and 536 deletions

View File

@@ -0,0 +1,78 @@
"""Request bounds for inline ingestion: chunk ceiling and the process-wide
concurrency gate (ADR-0017, plan 001 Phase 4).
"""
import uuid
import pytest
from anyio import Semaphore
from src.application.ingestion.bounds import acquire_ingestion_slot, enforce_chunk_limit
from src.application.ingestion.errors import ChunkLimitExceededError, IngestionAtCapacityError
from src.application.ingestion.models import Chunk, ContentType
pytestmark = pytest.mark.unit
def _chunk(index: int) -> Chunk:
return Chunk(
chunk_id=uuid.uuid4(),
chunk_index=index,
order_id=float(index + 1),
content="x",
content_type=ContentType.PARAGRAPH,
token_count=1,
character_count=1,
)
def test_enforce_chunk_limit_within_bound_does_not_raise() -> None:
enforce_chunk_limit([_chunk(0), _chunk(1)], max_chunks=2)
def test_enforce_chunk_limit_over_bound_raises() -> None:
with pytest.raises(ChunkLimitExceededError):
enforce_chunk_limit([_chunk(0), _chunk(1), _chunk(2)], max_chunks=2)
@pytest.mark.asyncio
async def test_acquire_ingestion_slot_allows_up_to_the_limit() -> None:
limiter = Semaphore(2)
async with acquire_ingestion_slot(limiter), acquire_ingestion_slot(limiter):
pass # two concurrent holders within a limit of two: no rejection
@pytest.mark.asyncio
async def test_acquire_ingestion_slot_rejects_beyond_the_limit() -> None:
limiter = Semaphore(1)
async with acquire_ingestion_slot(limiter):
with pytest.raises(IngestionAtCapacityError):
async with acquire_ingestion_slot(limiter):
pass
@pytest.mark.asyncio
async def test_acquire_ingestion_slot_releases_on_exit() -> None:
limiter = Semaphore(1)
async with acquire_ingestion_slot(limiter):
pass
# The slot from the first `async with` must be released by the time it
# exits, or every subsequent request would see permanent capacity loss.
async with acquire_ingestion_slot(limiter):
pass
@pytest.mark.asyncio
async def test_acquire_ingestion_slot_releases_after_body_raises() -> None:
limiter = Semaphore(1)
with pytest.raises(ValueError, match="boom"):
async with acquire_ingestion_slot(limiter):
raise ValueError("boom")
async with acquire_ingestion_slot(limiter):
pass

View File

@@ -0,0 +1,211 @@
"""Batching, concurrency bounding, and failure translation for `embed_chunks`
(ADR-0001, ADR-0017, plan 001 Phase 4).
"""
import asyncio
import uuid
from collections.abc import Sequence
from dataclasses import dataclass, field
import pytest
from anyio import CapacityLimiter
from src.application.ingestion.embedding import embed_chunks
from src.application.ingestion.errors import EmbedderError
from src.application.ingestion.models import Chunk, ContentType, SparseVector
from src.config import IngestionSettings
pytestmark = [pytest.mark.unit, pytest.mark.asyncio]
def _chunk(index: int, content: str = "hello world") -> Chunk:
return Chunk(
chunk_id=uuid.uuid4(),
chunk_index=index,
order_id=float(index + 1),
content=content,
content_type=ContentType.PARAGRAPH,
token_count=2,
character_count=len(content),
)
@dataclass
class _TrackingDenseEmbedder:
"""Records each batch's texts and the peak number of batches in flight
at once, to prove concurrency is bounded, not serial.
"""
name: str
dimensions: int = 3
batches: list[list[str]] = field(default_factory=list)
in_flight: int = 0
peak_in_flight: int = 0
async def embed_batch(self, texts: Sequence[str]) -> list[list[float]]:
self.in_flight += 1
self.peak_in_flight = max(self.peak_in_flight, self.in_flight)
self.batches.append(list(texts))
await asyncio.sleep(0) # yield so overlapping calls can interleave
self.in_flight -= 1
return [[0.0] * self.dimensions for _ in texts]
@dataclass
class _FailingDenseEmbedder:
name: str
async def embed_batch(self, texts: Sequence[str]) -> list[list[float]]:
raise RuntimeError("boom")
@dataclass
class _StubSparseEmbedder:
name: str = "sparse"
calls: list[list[str]] = field(default_factory=list)
def embed_batch(self, texts: Sequence[str], *, query: bool = False) -> list[SparseVector]:
self.calls.append(list(texts))
return [SparseVector(indices=[i], values=[1.0]) for i in range(len(texts))]
@dataclass
class _FailingSparseEmbedder:
name: str = "sparse"
def embed_batch(self, texts: Sequence[str], *, query: bool = False) -> list[SparseVector]:
raise RuntimeError("boom")
@pytest.fixture
def settings() -> IngestionSettings:
return IngestionSettings(embed_batch_size=2, embed_concurrency=2)
@pytest.fixture
def thread_limiter() -> CapacityLimiter:
return CapacityLimiter(4)
async def test_embed_chunks_empty_input_returns_empty(
settings: IngestionSettings, thread_limiter: CapacityLimiter
) -> None:
result = await embed_chunks(
[],
dense_embedders=[_TrackingDenseEmbedder(name="dense_nomic")],
sparse_embedder=_StubSparseEmbedder(),
settings=settings,
thread_limiter=thread_limiter,
)
assert result == []
async def test_embed_chunks_batches_before_parallelizing(
settings: IngestionSettings, thread_limiter: CapacityLimiter
) -> None:
chunks = [_chunk(i) for i in range(5)]
embedder = _TrackingDenseEmbedder(name="dense_nomic")
await embed_chunks(
chunks,
dense_embedders=[embedder],
sparse_embedder=_StubSparseEmbedder(),
settings=settings,
thread_limiter=thread_limiter,
)
# embed_batch_size=2 over 5 chunks -> 3 batches (2, 2, 1), never one call
# per chunk.
assert len(embedder.batches) == 3
assert [len(batch) for batch in embedder.batches] == [2, 2, 1]
async def test_embed_chunks_bounds_concurrency_by_embed_concurrency(
thread_limiter: CapacityLimiter,
) -> None:
settings = IngestionSettings(embed_batch_size=1, embed_concurrency=2)
chunks = [_chunk(i) for i in range(6)]
embedder = _TrackingDenseEmbedder(name="dense_nomic")
await embed_chunks(
chunks,
dense_embedders=[embedder],
sparse_embedder=_StubSparseEmbedder(),
settings=settings,
thread_limiter=thread_limiter,
)
assert embedder.peak_in_flight <= settings.embed_concurrency
assert embedder.peak_in_flight > 1 # proves it isn't serial either
async def test_embed_chunks_passes_chunk_text_through_unmodified(
settings: IngestionSettings, thread_limiter: CapacityLimiter
) -> None:
"""Text shaping (task prefixes, `keep_alive`) is the adapter's job, not
this module's -- see `src/infrastructure/embedding/openai_compatible.py`.
Orchestration here must stay provider-agnostic.
"""
chunks = [_chunk(0, content="salam")]
nomic = _TrackingDenseEmbedder(name="dense_nomic")
openai = _TrackingDenseEmbedder(name="dense_openai")
await embed_chunks(
chunks,
dense_embedders=[nomic, openai],
sparse_embedder=_StubSparseEmbedder(),
settings=settings,
thread_limiter=thread_limiter,
)
assert nomic.batches[0] == ["salam"]
assert openai.batches[0] == ["salam"]
async def test_embed_chunks_assembles_dense_and_sparse_per_chunk_in_order(
settings: IngestionSettings, thread_limiter: CapacityLimiter
) -> None:
chunks = [_chunk(0, "a"), _chunk(1, "b")]
nomic = _TrackingDenseEmbedder(name="dense_nomic", dimensions=3)
result = await embed_chunks(
chunks,
dense_embedders=[nomic],
sparse_embedder=_StubSparseEmbedder(),
settings=settings,
thread_limiter=thread_limiter,
)
assert [item.chunk.chunk_id for item in result] == [c.chunk_id for c in chunks]
assert all(len(item.dense["dense_nomic"]) == 3 for item in result)
assert all("sparse" not in item.dense for item in result)
async def test_embed_chunks_dense_failure_raises_embedder_error(
settings: IngestionSettings, thread_limiter: CapacityLimiter
) -> None:
chunks = [_chunk(0)]
with pytest.raises(EmbedderError):
await embed_chunks(
chunks,
dense_embedders=[_FailingDenseEmbedder(name="dense_nomic")],
sparse_embedder=_StubSparseEmbedder(),
settings=settings,
thread_limiter=thread_limiter,
)
async def test_embed_chunks_sparse_failure_raises_embedder_error(
settings: IngestionSettings, thread_limiter: CapacityLimiter
) -> None:
chunks = [_chunk(0)]
with pytest.raises(EmbedderError):
await embed_chunks(
chunks,
dense_embedders=[_TrackingDenseEmbedder(name="dense_nomic")],
sparse_embedder=_FailingSparseEmbedder(),
settings=settings,
thread_limiter=thread_limiter,
)

View File

@@ -2,7 +2,9 @@ import pytest
from fastapi import FastAPI
from httpx import AsyncClient
from src.bootstrap.lifespan import _warm_dense_embedders
from src.config import Settings
from tests.fakes import FakeDenseEmbedder
pytestmark = [pytest.mark.unit, pytest.mark.asyncio]
@@ -17,3 +19,32 @@ async def test_lifespan_binds_resources_to_app_state(app: FastAPI, client: Async
resources = app.state.resources
assert isinstance(resources.settings, Settings)
assert resources.minio_client is not None
assert len(resources.dense_embedders) == 2
assert {e.name for e in resources.dense_embedders} == {"dense_nomic", "dense_openai"}
assert resources.sparse_embedder.name == "sparse"
assert resources.ingestion_concurrency_limiter is not None
async def test_warm_dense_embedders_calls_every_embedder() -> None:
"""Pays the model-load cost at boot instead of on a user's first upload:
a cold nomic load outruns INGESTION_TIMEOUT_SECONDS entirely.
"""
embedders = [FakeDenseEmbedder(name="dense_nomic"), FakeDenseEmbedder(name="dense_openai")]
await _warm_dense_embedders(embedders)
assert all(e.calls for e in embedders)
async def test_warm_dense_embedders_survives_an_unreachable_embedder() -> None:
"""A down embedder must not stop the process booting -- otherwise the
service cannot come up to report its own health. `/readyz` owns that
signal, not startup.
"""
failing = FakeDenseEmbedder(name="dense_nomic", fail_next=True)
healthy = FakeDenseEmbedder(name="dense_openai")
await _warm_dense_embedders([failing, healthy])
# The failure is swallowed *and* does not abort the remaining warm-ups.
assert healthy.calls

View File

View File

@@ -0,0 +1,170 @@
"""The `bm25-fa-norm-stop` sparse embedder (ADR-0001, ADR-0005).
These assert the analyzer/weighting behaviour benchmarked in the `emet`
evaluation lab. A change that makes one of these fail is a change that
invalidates that benchmark, not just a failing test.
"""
import pytest
from src.config import SparseEmbeddingSettings
from src.infrastructure.embedding.analyzers import analyze
from src.infrastructure.embedding.bm25 import Bm25SparseEmbedder, _token_index
pytestmark = pytest.mark.unit
@pytest.fixture
def settings() -> SparseEmbeddingSettings:
return SparseEmbeddingSettings()
@pytest.fixture
def embedder(settings: SparseEmbeddingSettings) -> Bm25SparseEmbedder:
return Bm25SparseEmbedder(settings)
# --- analyzer ------------------------------------------------------------
def test_analyze_keeps_digits_as_tokens() -> None:
"""Policy numbers, dates, and amounts are exactly what a lexical index
should match on. An earlier implementation dropped every digit.
"""
tokens = analyze("بیمه‌نامه شماره ۱۲۳۴۵ صادر شد")
assert "12345" in tokens
def test_analyze_folds_eastern_digits_to_ascii() -> None:
"""The same number must match however it was authored."""
assert analyze("۹۹۸۸۷۷") == analyze("٩٩٨٨٧٧") == analyze("998877")
def test_analyze_splits_on_zwnj() -> None:
"""ZWNJ joins compounds visually but they are separate index terms."""
assert analyze("آتش‌سوزی") == ["آتش", "سوزی"]
def test_analyze_folds_arabic_letterforms_to_persian() -> None:
# Arabic kaf (U+0643) vs. Persian keheh (U+06A9): the same word typed on
# two different keyboards must produce the same term.
assert analyze("كتاب") == analyze("کتاب")
def test_analyze_drops_persian_stopwords() -> None:
assert analyze("این کتاب و آن مداد") == analyze("کتاب مداد")
def test_analyze_drops_english_stopwords() -> None:
"""The corpus is mixed-script, so the list carries English too."""
assert analyze("the policy is valid") == ["policy", "valid"]
def test_analyze_lowercases_latin() -> None:
assert analyze("POLICY Number") == ["policy", "number"]
def test_analyze_rejects_unknown_analyzer() -> None:
with pytest.raises(ValueError, match="Unknown analyzer"):
analyze("متن", "fa_norm_stem")
# --- token indexing ------------------------------------------------------
def test_token_index_is_stable_across_calls() -> None:
assert _token_index("کتاب") == _token_index("کتاب")
def test_token_index_is_pinned_to_known_values() -> None:
"""A golden test. These indices are baked into every stored sparse vector,
so changing the hash silently orphans the whole index -- a re-ingestion,
not a deploy. Ingest-time and query-time encoding must agree forever.
"""
assert _token_index("کتاب") == 1701064151
assert _token_index("policy") == 741331709
assert _token_index("12345") == 1232178634
def test_token_index_fits_signed_int32() -> None:
for token in ("کتاب", "policy", "12345", "بیمه", "x" * 200):
assert 0 <= _token_index(token) < 2**31 - 1
def test_token_index_distinguishes_different_tokens() -> None:
assert _token_index("کتاب") != _token_index("مداد")
# --- vector construction -------------------------------------------------
def test_embed_batch_returns_one_vector_per_text(embedder: Bm25SparseEmbedder) -> None:
assert len(embedder.embed_batch(["سلام دنیا", "یک تست دیگر"])) == 2
def test_embed_batch_empty_text_returns_empty_vector(embedder: Bm25SparseEmbedder) -> None:
(vector,) = embedder.embed_batch([""])
assert vector.indices == []
assert vector.values == []
def test_embed_batch_all_stopwords_returns_empty_vector(embedder: Bm25SparseEmbedder) -> None:
(vector,) = embedder.embed_batch(["و در به از که"])
assert vector.indices == []
def test_embed_batch_emits_tokens_in_sorted_order(embedder: Bm25SparseEmbedder) -> None:
"""Deterministic output keeps re-ingestion byte-stable."""
text = "مداد کتاب دفتر"
expected = [_token_index(token) for token in sorted(analyze(text))]
(vector,) = embedder.embed_batch([text])
assert vector.indices == expected
def test_embed_batch_applies_bm25_saturation_not_raw_counts(
embedder: Bm25SparseEmbedder,
) -> None:
"""Weight must be sublinear in term frequency: tripling a term must not
triple its weight, which is the whole point of the `k` parameter.
"""
(once,) = embedder.embed_batch(["کتاب"])
(thrice,) = embedder.embed_batch(["کتاب کتاب کتاب"])
assert thrice.values[0] > once.values[0]
assert thrice.values[0] < 3 * once.values[0]
def test_embed_batch_query_side_omits_length_normalization(
embedder: Bm25SparseEmbedder, settings: SparseEmbeddingSettings
) -> None:
text = "کتاب مداد دفتر خودکار"
(document,) = embedder.embed_batch([text], query=False)
(query,) = embedder.embed_batch([text], query=True)
assert document.indices == query.indices # same terms, same hashing
assert document.values != query.values
k = settings.k
assert query.values[0] == pytest.approx(1.0 * (k + 1.0) / (1.0 + k))
def test_embed_batch_short_document_outweighs_long_one(
embedder: Bm25SparseEmbedder,
) -> None:
"""The `b` term discounts a term appearing in a longer document."""
(short,) = embedder.embed_batch(["کتاب"])
(long,) = embedder.embed_batch(["کتاب " + " ".join(f"واژه{i}" for i in range(200))])
short_weight = short.values[short.indices.index(_token_index("کتاب"))]
long_weight = long.values[long.indices.index(_token_index("کتاب"))]
assert short_weight > long_weight
def test_embed_batch_respects_configured_parameters() -> None:
"""k/b/avg_len come from settings, so they can be retuned without a code
change -- and so a retune is visibly a config decision.
"""
default = Bm25SparseEmbedder(SparseEmbeddingSettings())
tuned = Bm25SparseEmbedder(SparseEmbeddingSettings(k=2.5, b=0.2, avg_len=64.0))
text = "کتاب کتاب مداد"
assert default.embed_batch([text])[0].values != tuned.embed_batch([text])[0].values

View File

@@ -0,0 +1,170 @@
"""`OpenAICompatibleEmbedder` against a mocked `/embeddings` endpoint.
Backs both `dense_nomic` and `dense_openai` (ADR-0001) -- `httpx.MockTransport`
stands in for the real self-hosted/OpenAI server so this stays a unit test
with no network dependency.
"""
import json
from collections.abc import Callable
import httpx
import pytest
from src.infrastructure.embedding.openai_compatible import (
OpenAICompatibleEmbedder,
is_ollama_base_url,
)
pytestmark = pytest.mark.unit
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.AsyncClient:
return httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="http://embedder")
@pytest.mark.asyncio
async def test_embed_batch_returns_vectors_in_input_order() -> None:
def handler(request: httpx.Request) -> httpx.Response:
# Respond out of order to prove the adapter re-sorts by `index`.
return httpx.Response(
200,
json={
"data": [
{"index": 1, "embedding": [0.2, 0.2]},
{"index": 0, "embedding": [0.1, 0.1]},
],
"model": "test-model",
},
)
embedder = OpenAICompatibleEmbedder(_client(handler), name="dense_nomic", model="test-model")
vectors = await embedder.embed_batch(["first", "second"])
assert vectors == [[0.1, 0.1], [0.2, 0.2]]
@pytest.mark.asyncio
async def test_embed_batch_sends_model_and_input() -> None:
captured: dict[str, object] = {}
def handler(request: httpx.Request) -> httpx.Response:
captured.update(json.loads(request.read()))
return httpx.Response(
200, json={"data": [{"index": 0, "embedding": [0.0]}], "model": "test-model"}
)
embedder = OpenAICompatibleEmbedder(_client(handler), name="dense_openai", model="test-model")
await embedder.embed_batch(["only text"])
assert captured["model"] == "test-model"
assert captured["input"] == ["only text"]
assert "dimensions" not in captured
@pytest.mark.asyncio
async def test_embed_batch_sends_dimensions_when_configured() -> None:
captured: dict[str, object] = {}
def handler(request: httpx.Request) -> httpx.Response:
captured.update(json.loads(request.read()))
return httpx.Response(
200, json={"data": [{"index": 0, "embedding": [0.0] * 256}], "model": "test-model"}
)
embedder = OpenAICompatibleEmbedder(
_client(handler), name="dense_openai", model="test-model", dimensions=256
)
await embedder.embed_batch(["only text"])
assert captured["dimensions"] == 256
@pytest.mark.asyncio
async def test_embed_batch_raises_on_non_2xx_response() -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(500, json={"error": "boom"})
embedder = OpenAICompatibleEmbedder(_client(handler), name="dense_nomic", model="test-model")
with pytest.raises(httpx.HTTPStatusError):
await embedder.embed_batch(["text"])
@pytest.mark.asyncio
async def test_embed_batch_sends_no_prefix_by_default() -> None:
"""emet parity: the benchmarked run used no task prefix (ADR-0004)."""
captured: dict[str, object] = {}
def handler(request: httpx.Request) -> httpx.Response:
captured.update(json.loads(request.read()))
return httpx.Response(200, json={"data": [{"index": 0, "embedding": [0.0]}], "model": "m"})
embedder = OpenAICompatibleEmbedder(_client(handler), name="dense_nomic", model="m")
await embedder.embed_batch(["سلام"])
assert captured["input"] == ["سلام"]
@pytest.mark.asyncio
async def test_embed_batch_applies_document_prefix_when_configured() -> None:
captured: dict[str, object] = {}
def handler(request: httpx.Request) -> httpx.Response:
captured.update(json.loads(request.read()))
return httpx.Response(200, json={"data": [{"index": 0, "embedding": [0.0]}], "model": "m"})
embedder = OpenAICompatibleEmbedder(
_client(handler), name="dense_nomic", model="m", document_prefix="search_document: "
)
await embedder.embed_batch(["سلام"])
assert captured["input"] == ["search_document: سلام"]
@pytest.mark.asyncio
async def test_embed_batch_sends_keep_alive_when_configured() -> None:
"""Keeps an Ollama-hosted model resident; a cold load outruns the
ingestion timeout entirely.
"""
captured: dict[str, object] = {}
def handler(request: httpx.Request) -> httpx.Response:
captured.update(json.loads(request.read()))
return httpx.Response(200, json={"data": [{"index": 0, "embedding": [0.0]}], "model": "m"})
embedder = OpenAICompatibleEmbedder(
_client(handler), name="dense_nomic", model="m", keep_alive="30m"
)
await embedder.embed_batch(["text"])
assert captured["keep_alive"] == "30m"
@pytest.mark.asyncio
async def test_embed_batch_omits_keep_alive_when_not_configured() -> None:
"""OpenAI would reject an unknown field, so it must not be sent there."""
captured: dict[str, object] = {}
def handler(request: httpx.Request) -> httpx.Response:
captured.update(json.loads(request.read()))
return httpx.Response(200, json={"data": [{"index": 0, "embedding": [0.0]}], "model": "m"})
embedder = OpenAICompatibleEmbedder(_client(handler), name="dense_openai", model="m")
await embedder.embed_batch(["text"])
assert "keep_alive" not in captured
@pytest.mark.parametrize(
("base_url", "expected"),
[
("http://192.168.10.10:11435/v1", True),
("http://127.0.0.1:11434/v1", True),
("http://ollama.internal/v1", True),
("https://api.openai.com/v1", False),
("http://127.0.0.1:8081/v1", False),
],
)
def test_is_ollama_base_url(base_url: str, expected: bool) -> None:
assert is_ollama_base_url(base_url) is expected