"""The one caller-facing entry point for indexing embedded chunks (ADR-0001, ADR-0017). `index_chunks` is the only version of this step callers should reach for. It owns the whole composition a correct upsert needs: - building ADR-0001's payload for every chunk, with `tenant_id`/`domain` taken from server-derived context; - offloading that (and the per-chunk content hashing) to a thread, since it is blocking CPU work (ADR-0017); - batching at `QDRANT_UPSERT_BATCH_SIZE` inside ADR-0001's 64-256 band; - bounding in-flight batches with an `asyncio.Semaphore` rather than an unbounded `gather` (ADR-0017); - running the soft-delete sweep for a shortened file **only after every batch has succeeded**. That last ordering is the point, not an implementation detail — see `_deactivate_stale` below. `build_chunk_payload` and `_batches` stay internal; pushing that composition onto every call site is exactly the obligation a deep module absorbs once (CLAUDE.md, "prefer deep modules"). """ import asyncio import uuid from collections.abc import Sequence from dataclasses import dataclass from datetime import UTC, datetime from anyio import CapacityLimiter, to_thread from src.application.ingestion.errors import PointIndexingError from src.application.ingestion.models import EmbeddedChunk from src.application.points.models import ChunkPoint from src.application.points.payload import build_chunk_payload from src.application.ports.embedding import DenseEmbedder, SparseEmbedder from src.application.ports.point_storage import PointStorage from src.config import QdrantSettings @dataclass(frozen=True) class IndexingResult: """What one indexing pass wrote. `points_upserted` counts points written, not points *created* — a deterministic-id upsert cannot distinguish an insert from an overwrite, so the ingestion job reports this as `points_created` and leaves `points_updated` at zero rather than guessing. """ points_upserted: int points_soft_deleted: int def _embedding_model_version( dense_embedders: Sequence[DenseEmbedder], sparse_embedder: SparseEmbedder ) -> str: """Compose the `embedding_model_version` payload value (ADR-0001). Sorted so the string is stable regardless of the order the embedders were wired in — an unstable value would make "which chunks need re-embedding?" unanswerable, which is the field's only reason to exist. """ versions = sorted( [embedder.model_version for embedder in dense_embedders] + [sparse_embedder.model_version] ) return "+".join(versions) def _batches(points: Sequence[ChunkPoint], size: int) -> list[Sequence[ChunkPoint]]: return [points[i : i + size] for i in range(0, len(points), size)] def _build_points( embedded: Sequence[EmbeddedChunk], *, tenant_id: uuid.UUID, domain: str, file_id: uuid.UUID, source_filename: str, source_type: str, actor: str, embedding_model_version: str, indexed_at: datetime, ) -> list[ChunkPoint]: """Blocking: hashes every chunk's content. Always called through a thread.""" return [ ChunkPoint( point_id=item.chunk.chunk_id, dense=item.dense, sparse=item.sparse, payload=build_chunk_payload( item.chunk, tenant_id=tenant_id, domain=domain, file_id=file_id, source_filename=source_filename, source_type=source_type, actor=actor, embedding_model_version=embedding_model_version, indexed_at=indexed_at, ), ) for item in embedded ] async def _upsert_bounded( storage: PointStorage, batch: Sequence[ChunkPoint], *, semaphore: asyncio.Semaphore ) -> None: async with semaphore: try: await storage.upsert_points(batch) except Exception as exc: raise PointIndexingError(f"upserting {len(batch)} points failed: {exc}") from exc async def _deactivate_stale( storage: PointStorage, *, tenant_id: uuid.UUID, file_id: uuid.UUID, from_chunk_index: int, actor: str, deleted_at: datetime, ) -> int: """Soft-delete points left over from a longer previous version of this file. Chunk indices are contiguous from 0, so "index >= the new chunk count" is exactly the set of points the new version no longer produces. This runs **only after every upsert has succeeded**, and that ordering is what keeps a failed attempt from damaging a working index. ADR-0001's deterministic point ids mean a re-ingestion overwrites in place, so literal atomic replacement is not available; what *is* guaranteed is that a failed attempt never removes content (it can only leave a prefix updated), and that a retry converges to the correct state. See ADR-0017. """ try: return await storage.deactivate_points_from_index( tenant_id=tenant_id, file_id=file_id, from_chunk_index=from_chunk_index, deleted_at=deleted_at, updated_by=actor, ) except Exception as exc: raise PointIndexingError(f"soft-deleting stale points failed: {exc}") from exc async def index_chunks( embedded: Sequence[EmbeddedChunk], *, storage: PointStorage, tenant_id: uuid.UUID, domain: str, file_id: uuid.UUID, source_filename: str, source_type: str, actor: str, dense_embedders: Sequence[DenseEmbedder], sparse_embedder: SparseEmbedder, settings: QdrantSettings, thread_limiter: CapacityLimiter, ) -> IndexingResult: """Upsert every embedded chunk as a tenant-scoped point, then sweep leftovers. Raises `PointIndexingError` (502) if any batch or the sweep fails. """ if not embedded: return IndexingResult(points_upserted=0, points_soft_deleted=0) indexed_at = datetime.now(UTC) points = await to_thread.run_sync( lambda: _build_points( embedded, tenant_id=tenant_id, domain=domain, file_id=file_id, source_filename=source_filename, source_type=source_type, actor=actor, embedding_model_version=_embedding_model_version(dense_embedders, sparse_embedder), indexed_at=indexed_at, ), limiter=thread_limiter, ) semaphore = asyncio.Semaphore(settings.upsert_concurrency) await asyncio.gather( *( _upsert_bounded(storage, batch, semaphore=semaphore) for batch in _batches(points, settings.upsert_batch_size) ) ) soft_deleted = await _deactivate_stale( storage, tenant_id=tenant_id, file_id=file_id, from_chunk_index=len(points), actor=actor, deleted_at=indexed_at, ) return IndexingResult(points_upserted=len(points), points_soft_deleted=soft_deleted)