5e0addcc55
Why: - The chunks collection needs four named vectors (dense_nomic, dense_openai, sparse, late_interaction) and payload indexes defined at creation time per ADR-0001; sparse/multivector fields cannot be added to an existing collection without recreating it, so schema drift here is expensive. - Creating it at FastAPI startup would mirror the DDL-at-boot anti-pattern ADR-0009 already rejects for Postgres and ADR-0012 rejects for LangGraph's setup(), so it is a deployment step instead. Changes: - src/infrastructure/qdrant/collection.py: ensure_chunks_collection(), idempotent and schema-verifying (raises on dimension/modifier mismatch rather than silently accepting a misconfigured collection). - src/cli/qdrant_bootstrap.py: the operator entry point (python -m src.cli.qdrant_bootstrap). - QdrantSettings gains collection/upsert_batch_size/upsert_concurrency. Impact: - Deployments must run the new bootstrap command before the first upload; see ADR-0001's new "Collection provisioning" section.
115 lines
4.6 KiB
Python
115 lines
4.6 KiB
Python
"""`ensure_chunks_collection` against a real Qdrant (ADR-0001).
|
|
|
|
The assertions that matter most here are the ones for schema properties that
|
|
fail *silently* in production: the sparse `modifier=idf` and the pinned dense
|
|
dimensions.
|
|
"""
|
|
|
|
import pytest
|
|
from qdrant_client import AsyncQdrantClient, models
|
|
|
|
from src.config import QdrantSettings
|
|
from src.infrastructure.qdrant.collection import (
|
|
CollectionSchemaMismatchError,
|
|
ensure_chunks_collection,
|
|
)
|
|
|
|
pytestmark = [
|
|
pytest.mark.integration,
|
|
pytest.mark.qdrant,
|
|
pytest.mark.asyncio(loop_scope="session"),
|
|
]
|
|
|
|
|
|
async def test_ensure_chunks_collection_creates_all_four_named_vectors(
|
|
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
|
) -> None:
|
|
created = await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
|
|
|
assert created is True
|
|
info = await qdrant_client.get_collection(qdrant_settings.collection)
|
|
vectors = info.config.params.vectors
|
|
assert isinstance(vectors, dict)
|
|
assert vectors["dense_nomic"].size == 768
|
|
assert vectors["dense_openai"].size == 3072
|
|
assert vectors["late_interaction"].size == 128
|
|
assert vectors["late_interaction"].multivector_config is not None
|
|
assert vectors["late_interaction"].on_disk is True
|
|
|
|
|
|
async def test_ensure_chunks_collection_sets_the_sparse_idf_modifier(
|
|
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
|
) -> None:
|
|
"""Without this, Qdrant applies no IDF and lexical retrieval silently
|
|
degrades -- no error, no warning (ADR-0005).
|
|
"""
|
|
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
|
|
|
info = await qdrant_client.get_collection(qdrant_settings.collection)
|
|
sparse = info.config.params.sparse_vectors
|
|
assert sparse is not None
|
|
assert sparse["sparse"].modifier == models.Modifier.IDF
|
|
|
|
|
|
async def test_ensure_chunks_collection_creates_the_payload_indexes(
|
|
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
|
) -> None:
|
|
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
|
|
|
info = await qdrant_client.get_collection(qdrant_settings.collection)
|
|
schema = info.payload_schema
|
|
assert set(schema) >= {
|
|
"tenant_id",
|
|
"domain",
|
|
"file_id",
|
|
"order_id",
|
|
"previous_chunk_id",
|
|
"next_chunk_id",
|
|
}
|
|
# order_id must be numeric: Qdrant's Range/order_by reject keyword payloads.
|
|
assert schema["order_id"].data_type == models.PayloadSchemaType.FLOAT
|
|
assert schema["tenant_id"].data_type == models.PayloadSchemaType.KEYWORD
|
|
|
|
|
|
async def test_ensure_chunks_collection_is_idempotent(
|
|
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
|
) -> None:
|
|
assert await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
|
|
|
assert not await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
|
|
|
|
|
async def test_ensure_chunks_collection_rejects_a_mismatched_existing_collection(
|
|
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
|
) -> None:
|
|
"""A wrong-dimension collection must fail loudly, not be silently accepted."""
|
|
await qdrant_client.create_collection(
|
|
collection_name=qdrant_settings.collection,
|
|
vectors_config={
|
|
"dense_nomic": models.VectorParams(size=384, distance=models.Distance.COSINE),
|
|
"dense_openai": models.VectorParams(size=3072, distance=models.Distance.COSINE),
|
|
"late_interaction": models.VectorParams(size=128, distance=models.Distance.COSINE),
|
|
},
|
|
sparse_vectors_config={"sparse": models.SparseVectorParams(modifier=models.Modifier.IDF)},
|
|
)
|
|
|
|
with pytest.raises(CollectionSchemaMismatchError, match="768"):
|
|
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
|
|
|
|
|
async def test_ensure_chunks_collection_rejects_a_collection_without_the_idf_modifier(
|
|
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
|
) -> None:
|
|
await qdrant_client.create_collection(
|
|
collection_name=qdrant_settings.collection,
|
|
vectors_config={
|
|
"dense_nomic": models.VectorParams(size=768, distance=models.Distance.COSINE),
|
|
"dense_openai": models.VectorParams(size=3072, distance=models.Distance.COSINE),
|
|
"late_interaction": models.VectorParams(size=128, distance=models.Distance.COSINE),
|
|
},
|
|
sparse_vectors_config={"sparse": models.SparseVectorParams()},
|
|
)
|
|
|
|
with pytest.raises(CollectionSchemaMismatchError, match="idf"):
|
|
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|