feat(qdrant): index content, is_active, and chunk_index on the chunks collection

Why:
- ADR-0002's keyword search needs a full-text index on content, which
  collection.py deliberately deferred to plan 002. is_active and chunk_index
  were unindexed while ingestion was the only reader; every /v1/points read path
  filters on them.

Changes:
- content gets a TEXT index with the multilingual tokenizer, which segments
  Persian correctly where the word tokenizer mishandles ZWNJ-joined compounds.
  No stemmer or stopword list: content is already letter-folded by
  normalize_persian_text at ingest, and the ranked Farsi lexical path is the
  benchmarked BM25 sparse vector, not this index.
- Tests assert content is TEXT rather than KEYWORD -- a keyword index would only
  match an entire chunk verbatim, which never happens and fails silently.
- Adds a test that a missing index is added to an already-live collection.

Impact:
- Requires re-running `python -m src.cli.qdrant_bootstrap`. Payload indexes are
  additive, so no collection rebuild and no re-embedding.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-22 13:09:26 +03:30
parent ac3810182d
commit 5e935e5895
2 changed files with 55 additions and 5 deletions

View File

@@ -65,10 +65,42 @@ async def test_ensure_chunks_collection_creates_the_payload_indexes(
"order_id",
"previous_chunk_id",
"next_chunk_id",
"content",
"is_active",
"chunk_index",
}
# order_id must be numeric: Qdrant's Range/order_by reject keyword payloads.
assert schema["order_id"].data_type == models.PayloadSchemaType.FLOAT
assert schema["tenant_id"].data_type == models.PayloadSchemaType.KEYWORD
# `content` must be TEXT, not KEYWORD: ADR-0002's keyword search is a
# full-text match on it, and a keyword index would only match the entire
# chunk verbatim -- which never happens and would fail silently.
assert schema["content"].data_type == models.PayloadSchemaType.TEXT
assert schema["is_active"].data_type == models.PayloadSchemaType.BOOL
assert schema["chunk_index"].data_type == models.PayloadSchemaType.INTEGER
async def test_ensure_chunks_collection_adds_a_missing_index_to_a_live_collection(
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
) -> None:
"""Payload indexes are additive, unlike vector config.
This is the operational claim the runbook makes when a new index ships:
re-running the bootstrap against an existing collection adds it in place, so
plan 002's `content`/`is_active`/`chunk_index` indexes do not require
recreating a collection that already holds a tenant's points.
"""
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
await qdrant_client.delete_payload_index(
collection_name=qdrant_settings.collection, field_name="content"
)
info = await qdrant_client.get_collection(qdrant_settings.collection)
assert "content" not in info.payload_schema
assert not await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
info = await qdrant_client.get_collection(qdrant_settings.collection)
assert info.payload_schema["content"].data_type == models.PayloadSchemaType.TEXT
async def test_ensure_chunks_collection_is_idempotent(