diff --git a/src/infrastructure/qdrant/collection.py b/src/infrastructure/qdrant/collection.py index 309616f..91e3907 100644 --- a/src/infrastructure/qdrant/collection.py +++ b/src/infrastructure/qdrant/collection.py @@ -80,14 +80,25 @@ def _sparse_vectors_config() -> dict[str, models.SparseVectorParams]: return {SPARSE_VECTOR: models.SparseVectorParams(modifier=models.Modifier.IDF)} +# `content`'s full-text index backs ADR-0002's keyword search. The +# `multilingual` tokenizer is the one that segments Persian correctly; `word` +# splits on non-alphanumerics, which mis-handles ZWNJ-joined compounds. No +# stemmer or stopword list is configured: `content` is already letter-folded by +# `normalize_persian_text` at ingest (ADR-0018), and the *ranked* Farsi lexical +# path is the benchmarked BM25 sparse vector, not this index. This one exists +# for exact keyword/filter matching, which ADR-0002 keeps deliberately distinct +# from retrieval. +_CONTENT_INDEX = models.TextIndexParams( + type=models.TextIndexType.TEXT, + tokenizer=models.TokenizerType.MULTILINGUAL, + lowercase=True, +) + # (field name, schema). `order_id` is float because Qdrant's `Range` conditions # and `order_by` only support numeric/datetime payloads -- a keyword key could # only be sorted client-side after fetching every chunk (ADR-0001). -# -# The full-text index on `content` is deliberately absent: it belongs to plan -# 002's keyword search, and payload indexes -- unlike vector config -- can be -# added to a live collection later. -_PAYLOAD_INDEXES: tuple[tuple[str, models.PayloadSchemaType | models.KeywordIndexParams], ...] = ( +_PayloadIndexSchema = models.PayloadSchemaType | models.KeywordIndexParams | models.TextIndexParams +_PAYLOAD_INDEXES: tuple[tuple[str, _PayloadIndexSchema], ...] = ( # `is_tenant` co-locates a tenant's vectors on disk for sequential reads, # which is the whole point of payload-partitioned multitenancy. ( @@ -99,6 +110,13 @@ _PAYLOAD_INDEXES: tuple[tuple[str, models.PayloadSchemaType | models.KeywordInde ("order_id", models.PayloadSchemaType.FLOAT), ("previous_chunk_id", models.PayloadSchemaType.KEYWORD), ("next_chunk_id", models.PayloadSchemaType.KEYWORD), + ("content", _CONTENT_INDEX), + # Every read path filters on `is_active` (ADR-0002 implies `is_active: true` + # unless the caller opts in), and the re-ingestion sweep and `/v1/points` + # both range over `chunk_index`. Both were unindexed while ingestion was the + # only reader; plan 002 makes them hot. + ("is_active", models.PayloadSchemaType.BOOL), + ("chunk_index", models.PayloadSchemaType.INTEGER), ) diff --git a/tests/integration/qdrant/test_collection.py b/tests/integration/qdrant/test_collection.py index 1cdc266..b8a9f9c 100644 --- a/tests/integration/qdrant/test_collection.py +++ b/tests/integration/qdrant/test_collection.py @@ -65,10 +65,42 @@ async def test_ensure_chunks_collection_creates_the_payload_indexes( "order_id", "previous_chunk_id", "next_chunk_id", + "content", + "is_active", + "chunk_index", } # order_id must be numeric: Qdrant's Range/order_by reject keyword payloads. assert schema["order_id"].data_type == models.PayloadSchemaType.FLOAT assert schema["tenant_id"].data_type == models.PayloadSchemaType.KEYWORD + # `content` must be TEXT, not KEYWORD: ADR-0002's keyword search is a + # full-text match on it, and a keyword index would only match the entire + # chunk verbatim -- which never happens and would fail silently. + assert schema["content"].data_type == models.PayloadSchemaType.TEXT + assert schema["is_active"].data_type == models.PayloadSchemaType.BOOL + assert schema["chunk_index"].data_type == models.PayloadSchemaType.INTEGER + + +async def test_ensure_chunks_collection_adds_a_missing_index_to_a_live_collection( + qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings +) -> None: + """Payload indexes are additive, unlike vector config. + + This is the operational claim the runbook makes when a new index ships: + re-running the bootstrap against an existing collection adds it in place, so + plan 002's `content`/`is_active`/`chunk_index` indexes do not require + recreating a collection that already holds a tenant's points. + """ + await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection) + await qdrant_client.delete_payload_index( + collection_name=qdrant_settings.collection, field_name="content" + ) + info = await qdrant_client.get_collection(qdrant_settings.collection) + assert "content" not in info.payload_schema + + assert not await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection) + + info = await qdrant_client.get_collection(qdrant_settings.collection) + assert info.payload_schema["content"].data_type == models.PayloadSchemaType.TEXT async def test_ensure_chunks_collection_is_idempotent(