chatbot_v3/tests/integration/qdrant/test_collection.py

"""`ensure_chunks_collection` against a real Qdrant (ADR-0001).

The assertions that matter most here are the ones for schema properties that
fail *silently* in production: the sparse `modifier=idf` and the pinned dense
dimensions.
"""

import pytest
from qdrant_client import AsyncQdrantClient, models

from src.config import QdrantSettings
from src.infrastructure.qdrant.collection import (
    CollectionSchemaMismatchError,
    ensure_chunks_collection,
)

pytestmark = [
    pytest.mark.integration,
    pytest.mark.qdrant,
    pytest.mark.asyncio(loop_scope="session"),
]


async def test_ensure_chunks_collection_creates_all_four_named_vectors(
    qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
    created = await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)

    assert created is True
    info = await qdrant_client.get_collection(qdrant_settings.collection)
    vectors = info.config.params.vectors
    assert isinstance(vectors, dict)
    assert vectors["dense_nomic"].size == 768
    assert vectors["dense_openai"].size == 3072
    assert vectors["late_interaction"].size == 128
    assert vectors["late_interaction"].multivector_config is not None
    assert vectors["late_interaction"].on_disk is True


async def test_ensure_chunks_collection_sets_the_sparse_idf_modifier(
    qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
    """Without this, Qdrant applies no IDF and lexical retrieval silently
    degrades -- no error, no warning (ADR-0005).
    """
    await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)

    info = await qdrant_client.get_collection(qdrant_settings.collection)
    sparse = info.config.params.sparse_vectors
    assert sparse is not None
    assert sparse["sparse"].modifier == models.Modifier.IDF


async def test_ensure_chunks_collection_creates_the_payload_indexes(
    qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
    await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)

    info = await qdrant_client.get_collection(qdrant_settings.collection)
    schema = info.payload_schema
    assert set(schema) >= {
        "tenant_id",
        "domain",
        "file_id",
        "order_id",
        "previous_chunk_id",
        "next_chunk_id",
        "content",
        "is_active",
        "chunk_index",
    }
    # order_id must be numeric: Qdrant's Range/order_by reject keyword payloads.
    assert schema["order_id"].data_type == models.PayloadSchemaType.FLOAT
    assert schema["tenant_id"].data_type == models.PayloadSchemaType.KEYWORD
    # `content` must be TEXT, not KEYWORD: ADR-0002's keyword search is a
    # full-text match on it, and a keyword index would only match the entire
    # chunk verbatim -- which never happens and would fail silently.
    assert schema["content"].data_type == models.PayloadSchemaType.TEXT
    assert schema["is_active"].data_type == models.PayloadSchemaType.BOOL
    assert schema["chunk_index"].data_type == models.PayloadSchemaType.INTEGER


async def test_ensure_chunks_collection_adds_a_missing_index_to_a_live_collection(
    qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
    """Payload indexes are additive, unlike vector config.

    This is the operational claim the runbook makes when a new index ships:
    re-running the bootstrap against an existing collection adds it in place, so
    plan 002's `content`/`is_active`/`chunk_index` indexes do not require
    recreating a collection that already holds a tenant's points.
    """
    await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
    await qdrant_client.delete_payload_index(
        collection_name=qdrant_settings.collection, field_name="content"
    )
    info = await qdrant_client.get_collection(qdrant_settings.collection)
    assert "content" not in info.payload_schema

    assert not await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)

    info = await qdrant_client.get_collection(qdrant_settings.collection)
    assert info.payload_schema["content"].data_type == models.PayloadSchemaType.TEXT


async def test_ensure_chunks_collection_is_idempotent(
    qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
    assert await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)

    assert not await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)


async def test_ensure_chunks_collection_rejects_a_mismatched_existing_collection(
    qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
    """A wrong-dimension collection must fail loudly, not be silently accepted."""
    await qdrant_client.create_collection(
        collection_name=qdrant_settings.collection,
        vectors_config={
            "dense_nomic": models.VectorParams(size=384, distance=models.Distance.COSINE),
            "dense_openai": models.VectorParams(size=3072, distance=models.Distance.COSINE),
            "late_interaction": models.VectorParams(size=128, distance=models.Distance.COSINE),
        },
        sparse_vectors_config={"sparse": models.SparseVectorParams(modifier=models.Modifier.IDF)},
    )

    with pytest.raises(CollectionSchemaMismatchError, match="768"):
        await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)


async def test_ensure_chunks_collection_rejects_a_collection_without_the_idf_modifier(
    qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
    await qdrant_client.create_collection(
        collection_name=qdrant_settings.collection,
        vectors_config={
            "dense_nomic": models.VectorParams(size=768, distance=models.Distance.COSINE),
            "dense_openai": models.VectorParams(size=3072, distance=models.Distance.COSINE),
            "late_interaction": models.VectorParams(size=128, distance=models.Distance.COSINE),
        },
        sparse_vectors_config={"sparse": models.SparseVectorParams()},
    )

    with pytest.raises(CollectionSchemaMismatchError, match="idf"):
        await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)