chatbot_v3/src/application/points/indexing.py

"""The one caller-facing entry point for indexing embedded chunks (ADR-0001, ADR-0017).

`index_chunks` is the only version of this step callers should reach for. It
owns the whole composition a correct upsert needs:

- building ADR-0001's payload for every chunk, with `tenant_id`/`domain` taken
  from server-derived context;
- offloading that (and the per-chunk content hashing) to a thread, since it is
  blocking CPU work (ADR-0017);
- batching at `QDRANT_UPSERT_BATCH_SIZE` inside ADR-0001's 64-256 band;
- bounding in-flight batches with an `asyncio.Semaphore` rather than an
  unbounded `gather` (ADR-0017);
- running the soft-delete sweep for a shortened file **only after every batch
  has succeeded**.

That last ordering is the point, not an implementation detail — see
`_deactivate_stale` below. `build_chunk_payload` and `_batches` stay internal;
pushing that composition onto every call site is exactly the obligation a deep
module absorbs once (CLAUDE.md, "prefer deep modules").
"""

import asyncio
import uuid
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import UTC, datetime

from anyio import CapacityLimiter, to_thread

from src.application.ingestion.errors import PointIndexingError
from src.application.ingestion.models import EmbeddedChunk
from src.application.points.models import ChunkPoint
from src.application.points.payload import build_chunk_payload
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
from src.application.ports.point_storage import PointStorage
from src.config import QdrantSettings


@dataclass(frozen=True)
class IndexingResult:
    """What one indexing pass wrote.

    `points_upserted` counts points written, not points *created* — a
    deterministic-id upsert cannot distinguish an insert from an overwrite, so
    the ingestion job reports this as `points_created` and leaves
    `points_updated` at zero rather than guessing.
    """

    points_upserted: int
    points_soft_deleted: int


def _embedding_model_version(
    dense_embedders: Sequence[DenseEmbedder], sparse_embedder: SparseEmbedder
) -> str:
    """Compose the `embedding_model_version` payload value (ADR-0001).

    Sorted so the string is stable regardless of the order the embedders were
    wired in — an unstable value would make "which chunks need re-embedding?"
    unanswerable, which is the field's only reason to exist.
    """
    versions = sorted(
        [embedder.model_version for embedder in dense_embedders] + [sparse_embedder.model_version]
    )
    return "+".join(versions)


def _batches(points: Sequence[ChunkPoint], size: int) -> list[Sequence[ChunkPoint]]:
    return [points[i : i + size] for i in range(0, len(points), size)]


def _build_points(
    embedded: Sequence[EmbeddedChunk],
    *,
    tenant_id: uuid.UUID,
    domain: str,
    file_id: uuid.UUID,
    source_filename: str,
    source_type: str,
    actor: str,
    embedding_model_version: str,
    indexed_at: datetime,
) -> list[ChunkPoint]:
    """Blocking: hashes every chunk's content. Always called through a thread."""
    return [
        ChunkPoint(
            point_id=item.chunk.chunk_id,
            dense=item.dense,
            sparse=item.sparse,
            payload=build_chunk_payload(
                item.chunk,
                tenant_id=tenant_id,
                domain=domain,
                file_id=file_id,
                source_filename=source_filename,
                source_type=source_type,
                actor=actor,
                embedding_model_version=embedding_model_version,
                indexed_at=indexed_at,
            ),
        )
        for item in embedded
    ]


async def _upsert_bounded(
    storage: PointStorage, batch: Sequence[ChunkPoint], *, semaphore: asyncio.Semaphore
) -> None:
    async with semaphore:
        try:
            await storage.upsert_points(batch)
        except Exception as exc:
            raise PointIndexingError(f"upserting {len(batch)} points failed: {exc}") from exc


async def _deactivate_stale(
    storage: PointStorage,
    *,
    tenant_id: uuid.UUID,
    file_id: uuid.UUID,
    from_chunk_index: int,
    actor: str,
    deleted_at: datetime,
) -> int:
    """Soft-delete points left over from a longer previous version of this file.

    Chunk indices are contiguous from 0, so "index >= the new chunk count" is
    exactly the set of points the new version no longer produces.

    This runs **only after every upsert has succeeded**, and that ordering is
    what keeps a failed attempt from damaging a working index. ADR-0001's
    deterministic point ids mean a re-ingestion overwrites in place, so literal
    atomic replacement is not available; what *is* guaranteed is that a failed
    attempt never removes content (it can only leave a prefix updated), and that
    a retry converges to the correct state. See ADR-0017.
    """
    try:
        return await storage.deactivate_points_from_index(
            tenant_id=tenant_id,
            file_id=file_id,
            from_chunk_index=from_chunk_index,
            deleted_at=deleted_at,
            updated_by=actor,
        )
    except Exception as exc:
        raise PointIndexingError(f"soft-deleting stale points failed: {exc}") from exc


async def index_chunks(
    embedded: Sequence[EmbeddedChunk],
    *,
    storage: PointStorage,
    tenant_id: uuid.UUID,
    domain: str,
    file_id: uuid.UUID,
    source_filename: str,
    source_type: str,
    actor: str,
    dense_embedders: Sequence[DenseEmbedder],
    sparse_embedder: SparseEmbedder,
    settings: QdrantSettings,
    thread_limiter: CapacityLimiter,
) -> IndexingResult:
    """Upsert every embedded chunk as a tenant-scoped point, then sweep leftovers.

    Raises `PointIndexingError` (502) if any batch or the sweep fails.
    """
    if not embedded:
        return IndexingResult(points_upserted=0, points_soft_deleted=0)

    indexed_at = datetime.now(UTC)
    points = await to_thread.run_sync(
        lambda: _build_points(
            embedded,
            tenant_id=tenant_id,
            domain=domain,
            file_id=file_id,
            source_filename=source_filename,
            source_type=source_type,
            actor=actor,
            embedding_model_version=_embedding_model_version(dense_embedders, sparse_embedder),
            indexed_at=indexed_at,
        ),
        limiter=thread_limiter,
    )

    semaphore = asyncio.Semaphore(settings.upsert_concurrency)
    await asyncio.gather(
        *(
            _upsert_bounded(storage, batch, semaphore=semaphore)
            for batch in _batches(points, settings.upsert_batch_size)
        )
    )

    soft_deleted = await _deactivate_stale(
        storage,
        tenant_id=tenant_id,
        file_id=file_id,
        from_chunk_index=len(points),
        actor=actor,
        deleted_at=indexed_at,
    )
    return IndexingResult(points_upserted=len(points), points_soft_deleted=soft_deleted)