feat(qdrant): add tenant-scoped point storage for ingestion
Why: - Ingested chunks need to become searchable Qdrant points before the upload response returns, with tenant/domain isolation and a safe re-ingestion story per ADR-0001/0017. Changes: - src/application/points/: index_chunks() is the sole entry point, owning payload construction, batched/bounded-concurrency upserts (upsert_concurrency semaphore), and a soft-delete sweep for points a shorter re-ingestion leaves behind. The sweep runs only after every upsert in the attempt succeeds, so a failed attempt can leave a stale prefix but never removes content from a working index. - PointStorage port (application/ports/) + QdrantPointStorage adapter (infrastructure/qdrant/points.py), keeping the qdrant_client SDK out of application code per ADR-0015. - FakePointStorage test double for exercising the ordering/idempotency guarantees without a real Qdrant.
This commit is contained in:
203
src/application/points/indexing.py
Normal file
203
src/application/points/indexing.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""The one caller-facing entry point for indexing embedded chunks (ADR-0001, ADR-0017).
|
||||
|
||||
`index_chunks` is the only version of this step callers should reach for. It
|
||||
owns the whole composition a correct upsert needs:
|
||||
|
||||
- building ADR-0001's payload for every chunk, with `tenant_id`/`domain` taken
|
||||
from server-derived context;
|
||||
- offloading that (and the per-chunk content hashing) to a thread, since it is
|
||||
blocking CPU work (ADR-0017);
|
||||
- batching at `QDRANT_UPSERT_BATCH_SIZE` inside ADR-0001's 64-256 band;
|
||||
- bounding in-flight batches with an `asyncio.Semaphore` rather than an
|
||||
unbounded `gather` (ADR-0017);
|
||||
- running the soft-delete sweep for a shortened file **only after every batch
|
||||
has succeeded**.
|
||||
|
||||
That last ordering is the point, not an implementation detail — see
|
||||
`_deactivate_stale` below. `build_chunk_payload` and `_batches` stay internal;
|
||||
pushing that composition onto every call site is exactly the obligation a deep
|
||||
module absorbs once (CLAUDE.md, "prefer deep modules").
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from anyio import CapacityLimiter, to_thread
|
||||
|
||||
from src.application.ingestion.errors import PointIndexingError
|
||||
from src.application.ingestion.models import EmbeddedChunk
|
||||
from src.application.points.models import ChunkPoint
|
||||
from src.application.points.payload import build_chunk_payload
|
||||
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||
from src.application.ports.point_storage import PointStorage
|
||||
from src.config import QdrantSettings
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IndexingResult:
|
||||
"""What one indexing pass wrote.
|
||||
|
||||
`points_upserted` counts points written, not points *created* — a
|
||||
deterministic-id upsert cannot distinguish an insert from an overwrite, so
|
||||
the ingestion job reports this as `points_created` and leaves
|
||||
`points_updated` at zero rather than guessing.
|
||||
"""
|
||||
|
||||
points_upserted: int
|
||||
points_soft_deleted: int
|
||||
|
||||
|
||||
def _embedding_model_version(
|
||||
dense_embedders: Sequence[DenseEmbedder], sparse_embedder: SparseEmbedder
|
||||
) -> str:
|
||||
"""Compose the `embedding_model_version` payload value (ADR-0001).
|
||||
|
||||
Sorted so the string is stable regardless of the order the embedders were
|
||||
wired in — an unstable value would make "which chunks need re-embedding?"
|
||||
unanswerable, which is the field's only reason to exist.
|
||||
"""
|
||||
versions = sorted(
|
||||
[embedder.model_version for embedder in dense_embedders] + [sparse_embedder.model_version]
|
||||
)
|
||||
return "+".join(versions)
|
||||
|
||||
|
||||
def _batches(points: Sequence[ChunkPoint], size: int) -> list[Sequence[ChunkPoint]]:
|
||||
return [points[i : i + size] for i in range(0, len(points), size)]
|
||||
|
||||
|
||||
def _build_points(
|
||||
embedded: Sequence[EmbeddedChunk],
|
||||
*,
|
||||
tenant_id: uuid.UUID,
|
||||
domain: str,
|
||||
file_id: uuid.UUID,
|
||||
source_filename: str,
|
||||
source_type: str,
|
||||
actor: str,
|
||||
embedding_model_version: str,
|
||||
indexed_at: datetime,
|
||||
) -> list[ChunkPoint]:
|
||||
"""Blocking: hashes every chunk's content. Always called through a thread."""
|
||||
return [
|
||||
ChunkPoint(
|
||||
point_id=item.chunk.chunk_id,
|
||||
dense=item.dense,
|
||||
sparse=item.sparse,
|
||||
payload=build_chunk_payload(
|
||||
item.chunk,
|
||||
tenant_id=tenant_id,
|
||||
domain=domain,
|
||||
file_id=file_id,
|
||||
source_filename=source_filename,
|
||||
source_type=source_type,
|
||||
actor=actor,
|
||||
embedding_model_version=embedding_model_version,
|
||||
indexed_at=indexed_at,
|
||||
),
|
||||
)
|
||||
for item in embedded
|
||||
]
|
||||
|
||||
|
||||
async def _upsert_bounded(
|
||||
storage: PointStorage, batch: Sequence[ChunkPoint], *, semaphore: asyncio.Semaphore
|
||||
) -> None:
|
||||
async with semaphore:
|
||||
try:
|
||||
await storage.upsert_points(batch)
|
||||
except Exception as exc:
|
||||
raise PointIndexingError(f"upserting {len(batch)} points failed: {exc}") from exc
|
||||
|
||||
|
||||
async def _deactivate_stale(
|
||||
storage: PointStorage,
|
||||
*,
|
||||
tenant_id: uuid.UUID,
|
||||
file_id: uuid.UUID,
|
||||
from_chunk_index: int,
|
||||
actor: str,
|
||||
deleted_at: datetime,
|
||||
) -> int:
|
||||
"""Soft-delete points left over from a longer previous version of this file.
|
||||
|
||||
Chunk indices are contiguous from 0, so "index >= the new chunk count" is
|
||||
exactly the set of points the new version no longer produces.
|
||||
|
||||
This runs **only after every upsert has succeeded**, and that ordering is
|
||||
what keeps a failed attempt from damaging a working index. ADR-0001's
|
||||
deterministic point ids mean a re-ingestion overwrites in place, so literal
|
||||
atomic replacement is not available; what *is* guaranteed is that a failed
|
||||
attempt never removes content (it can only leave a prefix updated), and that
|
||||
a retry converges to the correct state. See ADR-0017.
|
||||
"""
|
||||
try:
|
||||
return await storage.deactivate_points_from_index(
|
||||
tenant_id=tenant_id,
|
||||
file_id=file_id,
|
||||
from_chunk_index=from_chunk_index,
|
||||
deleted_at=deleted_at,
|
||||
updated_by=actor,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise PointIndexingError(f"soft-deleting stale points failed: {exc}") from exc
|
||||
|
||||
|
||||
async def index_chunks(
|
||||
embedded: Sequence[EmbeddedChunk],
|
||||
*,
|
||||
storage: PointStorage,
|
||||
tenant_id: uuid.UUID,
|
||||
domain: str,
|
||||
file_id: uuid.UUID,
|
||||
source_filename: str,
|
||||
source_type: str,
|
||||
actor: str,
|
||||
dense_embedders: Sequence[DenseEmbedder],
|
||||
sparse_embedder: SparseEmbedder,
|
||||
settings: QdrantSettings,
|
||||
thread_limiter: CapacityLimiter,
|
||||
) -> IndexingResult:
|
||||
"""Upsert every embedded chunk as a tenant-scoped point, then sweep leftovers.
|
||||
|
||||
Raises `PointIndexingError` (502) if any batch or the sweep fails.
|
||||
"""
|
||||
if not embedded:
|
||||
return IndexingResult(points_upserted=0, points_soft_deleted=0)
|
||||
|
||||
indexed_at = datetime.now(UTC)
|
||||
points = await to_thread.run_sync(
|
||||
lambda: _build_points(
|
||||
embedded,
|
||||
tenant_id=tenant_id,
|
||||
domain=domain,
|
||||
file_id=file_id,
|
||||
source_filename=source_filename,
|
||||
source_type=source_type,
|
||||
actor=actor,
|
||||
embedding_model_version=_embedding_model_version(dense_embedders, sparse_embedder),
|
||||
indexed_at=indexed_at,
|
||||
),
|
||||
limiter=thread_limiter,
|
||||
)
|
||||
|
||||
semaphore = asyncio.Semaphore(settings.upsert_concurrency)
|
||||
await asyncio.gather(
|
||||
*(
|
||||
_upsert_bounded(storage, batch, semaphore=semaphore)
|
||||
for batch in _batches(points, settings.upsert_batch_size)
|
||||
)
|
||||
)
|
||||
|
||||
soft_deleted = await _deactivate_stale(
|
||||
storage,
|
||||
tenant_id=tenant_id,
|
||||
file_id=file_id,
|
||||
from_chunk_index=len(points),
|
||||
actor=actor,
|
||||
deleted_at=indexed_at,
|
||||
)
|
||||
return IndexingResult(points_upserted=len(points), points_soft_deleted=soft_deleted)
|
||||
Reference in New Issue
Block a user