feat(qdrant): index content, is_active, and chunk_index on the chunks collection
Why: - ADR-0002's keyword search needs a full-text index on content, which collection.py deliberately deferred to plan 002. is_active and chunk_index were unindexed while ingestion was the only reader; every /v1/points read path filters on them. Changes: - content gets a TEXT index with the multilingual tokenizer, which segments Persian correctly where the word tokenizer mishandles ZWNJ-joined compounds. No stemmer or stopword list: content is already letter-folded by normalize_persian_text at ingest, and the ranked Farsi lexical path is the benchmarked BM25 sparse vector, not this index. - Tests assert content is TEXT rather than KEYWORD -- a keyword index would only match an entire chunk verbatim, which never happens and fails silently. - Adds a test that a missing index is added to an already-live collection. Impact: - Requires re-running `python -m src.cli.qdrant_bootstrap`. Payload indexes are additive, so no collection rebuild and no re-embedding. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -65,10 +65,42 @@ async def test_ensure_chunks_collection_creates_the_payload_indexes(
|
||||
"order_id",
|
||||
"previous_chunk_id",
|
||||
"next_chunk_id",
|
||||
"content",
|
||||
"is_active",
|
||||
"chunk_index",
|
||||
}
|
||||
# order_id must be numeric: Qdrant's Range/order_by reject keyword payloads.
|
||||
assert schema["order_id"].data_type == models.PayloadSchemaType.FLOAT
|
||||
assert schema["tenant_id"].data_type == models.PayloadSchemaType.KEYWORD
|
||||
# `content` must be TEXT, not KEYWORD: ADR-0002's keyword search is a
|
||||
# full-text match on it, and a keyword index would only match the entire
|
||||
# chunk verbatim -- which never happens and would fail silently.
|
||||
assert schema["content"].data_type == models.PayloadSchemaType.TEXT
|
||||
assert schema["is_active"].data_type == models.PayloadSchemaType.BOOL
|
||||
assert schema["chunk_index"].data_type == models.PayloadSchemaType.INTEGER
|
||||
|
||||
|
||||
async def test_ensure_chunks_collection_adds_a_missing_index_to_a_live_collection(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
"""Payload indexes are additive, unlike vector config.
|
||||
|
||||
This is the operational claim the runbook makes when a new index ships:
|
||||
re-running the bootstrap against an existing collection adds it in place, so
|
||||
plan 002's `content`/`is_active`/`chunk_index` indexes do not require
|
||||
recreating a collection that already holds a tenant's points.
|
||||
"""
|
||||
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
await qdrant_client.delete_payload_index(
|
||||
collection_name=qdrant_settings.collection, field_name="content"
|
||||
)
|
||||
info = await qdrant_client.get_collection(qdrant_settings.collection)
|
||||
assert "content" not in info.payload_schema
|
||||
|
||||
assert not await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
|
||||
info = await qdrant_client.get_collection(qdrant_settings.collection)
|
||||
assert info.payload_schema["content"].data_type == models.PayloadSchemaType.TEXT
|
||||
|
||||
|
||||
async def test_ensure_chunks_collection_is_idempotent(
|
||||
|
||||
Reference in New Issue
Block a user