feat(qdrant): provision the chunks collection as an explicit deployment step
Why: - The chunks collection needs four named vectors (dense_nomic, dense_openai, sparse, late_interaction) and payload indexes defined at creation time per ADR-0001; sparse/multivector fields cannot be added to an existing collection without recreating it, so schema drift here is expensive. - Creating it at FastAPI startup would mirror the DDL-at-boot anti-pattern ADR-0009 already rejects for Postgres and ADR-0012 rejects for LangGraph's setup(), so it is a deployment step instead. Changes: - src/infrastructure/qdrant/collection.py: ensure_chunks_collection(), idempotent and schema-verifying (raises on dimension/modifier mismatch rather than silently accepting a misconfigured collection). - src/cli/qdrant_bootstrap.py: the operator entry point (python -m src.cli.qdrant_bootstrap). - QdrantSettings gains collection/upsert_batch_size/upsert_concurrency. Impact: - Deployments must run the new bootstrap command before the first upload; see ADR-0001's new "Collection provisioning" section.
This commit is contained in:
114
tests/integration/qdrant/test_collection.py
Normal file
114
tests/integration/qdrant/test_collection.py
Normal file
@@ -0,0 +1,114 @@
|
||||
"""`ensure_chunks_collection` against a real Qdrant (ADR-0001).
|
||||
|
||||
The assertions that matter most here are the ones for schema properties that
|
||||
fail *silently* in production: the sparse `modifier=idf` and the pinned dense
|
||||
dimensions.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from qdrant_client import AsyncQdrantClient, models
|
||||
|
||||
from src.config import QdrantSettings
|
||||
from src.infrastructure.qdrant.collection import (
|
||||
CollectionSchemaMismatchError,
|
||||
ensure_chunks_collection,
|
||||
)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.qdrant,
|
||||
pytest.mark.asyncio(loop_scope="session"),
|
||||
]
|
||||
|
||||
|
||||
async def test_ensure_chunks_collection_creates_all_four_named_vectors(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
created = await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
|
||||
assert created is True
|
||||
info = await qdrant_client.get_collection(qdrant_settings.collection)
|
||||
vectors = info.config.params.vectors
|
||||
assert isinstance(vectors, dict)
|
||||
assert vectors["dense_nomic"].size == 768
|
||||
assert vectors["dense_openai"].size == 3072
|
||||
assert vectors["late_interaction"].size == 128
|
||||
assert vectors["late_interaction"].multivector_config is not None
|
||||
assert vectors["late_interaction"].on_disk is True
|
||||
|
||||
|
||||
async def test_ensure_chunks_collection_sets_the_sparse_idf_modifier(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
"""Without this, Qdrant applies no IDF and lexical retrieval silently
|
||||
degrades -- no error, no warning (ADR-0005).
|
||||
"""
|
||||
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
|
||||
info = await qdrant_client.get_collection(qdrant_settings.collection)
|
||||
sparse = info.config.params.sparse_vectors
|
||||
assert sparse is not None
|
||||
assert sparse["sparse"].modifier == models.Modifier.IDF
|
||||
|
||||
|
||||
async def test_ensure_chunks_collection_creates_the_payload_indexes(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
|
||||
info = await qdrant_client.get_collection(qdrant_settings.collection)
|
||||
schema = info.payload_schema
|
||||
assert set(schema) >= {
|
||||
"tenant_id",
|
||||
"domain",
|
||||
"file_id",
|
||||
"order_id",
|
||||
"previous_chunk_id",
|
||||
"next_chunk_id",
|
||||
}
|
||||
# order_id must be numeric: Qdrant's Range/order_by reject keyword payloads.
|
||||
assert schema["order_id"].data_type == models.PayloadSchemaType.FLOAT
|
||||
assert schema["tenant_id"].data_type == models.PayloadSchemaType.KEYWORD
|
||||
|
||||
|
||||
async def test_ensure_chunks_collection_is_idempotent(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
assert await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
|
||||
assert not await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
|
||||
|
||||
async def test_ensure_chunks_collection_rejects_a_mismatched_existing_collection(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
"""A wrong-dimension collection must fail loudly, not be silently accepted."""
|
||||
await qdrant_client.create_collection(
|
||||
collection_name=qdrant_settings.collection,
|
||||
vectors_config={
|
||||
"dense_nomic": models.VectorParams(size=384, distance=models.Distance.COSINE),
|
||||
"dense_openai": models.VectorParams(size=3072, distance=models.Distance.COSINE),
|
||||
"late_interaction": models.VectorParams(size=128, distance=models.Distance.COSINE),
|
||||
},
|
||||
sparse_vectors_config={"sparse": models.SparseVectorParams(modifier=models.Modifier.IDF)},
|
||||
)
|
||||
|
||||
with pytest.raises(CollectionSchemaMismatchError, match="768"):
|
||||
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
|
||||
|
||||
async def test_ensure_chunks_collection_rejects_a_collection_without_the_idf_modifier(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
await qdrant_client.create_collection(
|
||||
collection_name=qdrant_settings.collection,
|
||||
vectors_config={
|
||||
"dense_nomic": models.VectorParams(size=768, distance=models.Distance.COSINE),
|
||||
"dense_openai": models.VectorParams(size=3072, distance=models.Distance.COSINE),
|
||||
"late_interaction": models.VectorParams(size=128, distance=models.Distance.COSINE),
|
||||
},
|
||||
sparse_vectors_config={"sparse": models.SparseVectorParams()},
|
||||
)
|
||||
|
||||
with pytest.raises(CollectionSchemaMismatchError, match="idf"):
|
||||
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
Reference in New Issue
Block a user