feat(qdrant): provision the chunks collection as an explicit deployment step

Why:
- The chunks collection needs four named vectors (dense_nomic, dense_openai,
  sparse, late_interaction) and payload indexes defined at creation time per
  ADR-0001; sparse/multivector fields cannot be added to an existing
  collection without recreating it, so schema drift here is expensive.
- Creating it at FastAPI startup would mirror the DDL-at-boot anti-pattern
  ADR-0009 already rejects for Postgres and ADR-0012 rejects for LangGraph's
  setup(), so it is a deployment step instead.

Changes:
- src/infrastructure/qdrant/collection.py: ensure_chunks_collection(),
  idempotent and schema-verifying (raises on dimension/modifier mismatch
  rather than silently accepting a misconfigured collection).
- src/cli/qdrant_bootstrap.py: the operator entry point
  (python -m src.cli.qdrant_bootstrap).
- QdrantSettings gains collection/upsert_batch_size/upsert_concurrency.

Impact:
- Deployments must run the new bootstrap command before the first upload;
  see ADR-0001's new "Collection provisioning" section.
This commit is contained in:
Ali Zarinkolah
2026-08-20 18:15:21 +03:30
parent e8fb41af87
commit 5e0addcc55
8 changed files with 456 additions and 2 deletions

View File

@@ -0,0 +1,52 @@
import uuid
from collections.abc import AsyncIterator, Iterator
import pytest
import pytest_asyncio
from qdrant_client import AsyncQdrantClient
from testcontainers.community.qdrant import QdrantContainer
from src.config import QdrantSettings
from src.infrastructure.qdrant.client import create_client
# Pinned to match the `qdrant-client` major/minor in pyproject.toml. The
# testcontainers default image trails it far enough that the client emits an
# incompatibility warning, and testing against a version we do not deploy is
# the wrong signal anyway.
_QDRANT_IMAGE = "qdrant/qdrant:v1.19.0"
@pytest.fixture(scope="session")
def qdrant_container() -> Iterator[QdrantContainer]:
with QdrantContainer(image=_QDRANT_IMAGE) as container:
yield container
@pytest.fixture(scope="session")
def qdrant_url(qdrant_container: QdrantContainer) -> str:
"""The container's REST URL, pinned to IPv4.
Same gotcha as postgres_url/minio_settings (see their conftests):
testcontainers reports the host as `localhost`, which resolves to `::1`
first, but Docker publishes the mapped port on IPv4 only. The IPv6 SYN is
dropped rather than refused, so the client hangs until its timeout instead
of falling back to the second address -- the connection does not fail, it
hangs.
"""
host = qdrant_container.get_container_host_ip().replace("localhost", "127.0.0.1")
return f"http://{host}:{qdrant_container.get_exposed_port(6333)}"
@pytest.fixture
def qdrant_settings(qdrant_url: str) -> QdrantSettings:
"""Settings naming a collection unique to this test (ADR-0016 isolation)."""
return QdrantSettings(url=qdrant_url, collection=f"chunks_{uuid.uuid4().hex}")
@pytest_asyncio.fixture(loop_scope="session")
async def qdrant_client(qdrant_settings: QdrantSettings) -> AsyncIterator[AsyncQdrantClient]:
client = create_client(qdrant_settings)
try:
yield client
finally:
await client.close()

View File

@@ -0,0 +1,114 @@
"""`ensure_chunks_collection` against a real Qdrant (ADR-0001).
The assertions that matter most here are the ones for schema properties that
fail *silently* in production: the sparse `modifier=idf` and the pinned dense
dimensions.
"""
import pytest
from qdrant_client import AsyncQdrantClient, models
from src.config import QdrantSettings
from src.infrastructure.qdrant.collection import (
CollectionSchemaMismatchError,
ensure_chunks_collection,
)
pytestmark = [
pytest.mark.integration,
pytest.mark.qdrant,
pytest.mark.asyncio(loop_scope="session"),
]
async def test_ensure_chunks_collection_creates_all_four_named_vectors(
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
created = await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
assert created is True
info = await qdrant_client.get_collection(qdrant_settings.collection)
vectors = info.config.params.vectors
assert isinstance(vectors, dict)
assert vectors["dense_nomic"].size == 768
assert vectors["dense_openai"].size == 3072
assert vectors["late_interaction"].size == 128
assert vectors["late_interaction"].multivector_config is not None
assert vectors["late_interaction"].on_disk is True
async def test_ensure_chunks_collection_sets_the_sparse_idf_modifier(
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
"""Without this, Qdrant applies no IDF and lexical retrieval silently
degrades -- no error, no warning (ADR-0005).
"""
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
info = await qdrant_client.get_collection(qdrant_settings.collection)
sparse = info.config.params.sparse_vectors
assert sparse is not None
assert sparse["sparse"].modifier == models.Modifier.IDF
async def test_ensure_chunks_collection_creates_the_payload_indexes(
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
info = await qdrant_client.get_collection(qdrant_settings.collection)
schema = info.payload_schema
assert set(schema) >= {
"tenant_id",
"domain",
"file_id",
"order_id",
"previous_chunk_id",
"next_chunk_id",
}
# order_id must be numeric: Qdrant's Range/order_by reject keyword payloads.
assert schema["order_id"].data_type == models.PayloadSchemaType.FLOAT
assert schema["tenant_id"].data_type == models.PayloadSchemaType.KEYWORD
async def test_ensure_chunks_collection_is_idempotent(
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
assert await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
assert not await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
async def test_ensure_chunks_collection_rejects_a_mismatched_existing_collection(
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
"""A wrong-dimension collection must fail loudly, not be silently accepted."""
await qdrant_client.create_collection(
collection_name=qdrant_settings.collection,
vectors_config={
"dense_nomic": models.VectorParams(size=384, distance=models.Distance.COSINE),
"dense_openai": models.VectorParams(size=3072, distance=models.Distance.COSINE),
"late_interaction": models.VectorParams(size=128, distance=models.Distance.COSINE),
},
sparse_vectors_config={"sparse": models.SparseVectorParams(modifier=models.Modifier.IDF)},
)
with pytest.raises(CollectionSchemaMismatchError, match="768"):
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
async def test_ensure_chunks_collection_rejects_a_collection_without_the_idf_modifier(
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
await qdrant_client.create_collection(
collection_name=qdrant_settings.collection,
vectors_config={
"dense_nomic": models.VectorParams(size=768, distance=models.Distance.COSINE),
"dense_openai": models.VectorParams(size=3072, distance=models.Distance.COSINE),
"late_interaction": models.VectorParams(size=128, distance=models.Distance.COSINE),
},
sparse_vectors_config={"sparse": models.SparseVectorParams()},
)
with pytest.raises(CollectionSchemaMismatchError, match="idf"):
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)