feat(qdrant): add tenant-scoped point storage for ingestion
Why: - Ingested chunks need to become searchable Qdrant points before the upload response returns, with tenant/domain isolation and a safe re-ingestion story per ADR-0001/0017. Changes: - src/application/points/: index_chunks() is the sole entry point, owning payload construction, batched/bounded-concurrency upserts (upsert_concurrency semaphore), and a soft-delete sweep for points a shorter re-ingestion leaves behind. The sweep runs only after every upsert in the attempt succeeds, so a failed attempt can leave a stale prefix but never removes content from a working index. - PointStorage port (application/ports/) + QdrantPointStorage adapter (infrastructure/qdrant/points.py), keeping the qdrant_client SDK out of application code per ADR-0015. - FakePointStorage test double for exercising the ordering/idempotency guarantees without a real Qdrant.
This commit is contained in:
Ali Zarinkolah
182
tests/integration/qdrant/test_points.py
Normal file
182
tests/integration/qdrant/test_points.py
Normal file
@@ -0,0 +1,182 @@
|
||||
"""`QdrantPointStorage` against a real Qdrant (ADR-0001, ADR-0002).
|
||||
|
||||
Reads here go through the raw client rather than the port: `PointStorage` is
|
||||
deliberately write-only, because point reads are plan 002's `/v1/points`
|
||||
surface. The reads below are the test's own verification, not a preview of an
|
||||
API this slice ships.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from qdrant_client import AsyncQdrantClient, models
|
||||
|
||||
from src.application.ingestion.chunking import chunk_id_for
|
||||
from src.application.ingestion.models import SparseVector
|
||||
from src.application.points.models import ChunkPoint
|
||||
from src.config import QdrantSettings
|
||||
from src.infrastructure.qdrant.collection import (
|
||||
DENSE_NOMIC_DIMENSIONS,
|
||||
DENSE_OPENAI_DIMENSIONS,
|
||||
ensure_chunks_collection,
|
||||
)
|
||||
from src.infrastructure.qdrant.points import QdrantPointStorage
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.integration,
|
||||
pytest.mark.qdrant,
|
||||
pytest.mark.asyncio(loop_scope="session"),
|
||||
]
|
||||
|
||||
|
||||
def _point(tenant_id: uuid.UUID, file_id: uuid.UUID, chunk_index: int) -> ChunkPoint:
|
||||
return ChunkPoint(
|
||||
point_id=chunk_id_for(file_id, chunk_index),
|
||||
dense={
|
||||
"dense_nomic": [0.1] * DENSE_NOMIC_DIMENSIONS,
|
||||
"dense_openai": [0.2] * DENSE_OPENAI_DIMENSIONS,
|
||||
},
|
||||
sparse=SparseVector(indices=[1, 2], values=[0.5, 0.25]),
|
||||
payload={
|
||||
"tenant_id": str(tenant_id),
|
||||
"domain": "fire",
|
||||
"file_id": str(file_id),
|
||||
"chunk_id": str(chunk_id_for(file_id, chunk_index)),
|
||||
"chunk_index": chunk_index,
|
||||
"order_id": float(chunk_index + 1),
|
||||
"content": f"chunk {chunk_index}",
|
||||
"is_active": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _storage(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> QdrantPointStorage:
|
||||
await ensure_chunks_collection(qdrant_client, collection=qdrant_settings.collection)
|
||||
return QdrantPointStorage(qdrant_client, collection=qdrant_settings.collection)
|
||||
|
||||
|
||||
async def _count_for_tenant(
|
||||
client: AsyncQdrantClient, collection: str, tenant_id: uuid.UUID
|
||||
) -> int:
|
||||
result = await client.count(
|
||||
collection_name=collection,
|
||||
count_filter=models.Filter(
|
||||
must=[
|
||||
models.FieldCondition(
|
||||
key="tenant_id", match=models.MatchValue(value=str(tenant_id))
|
||||
)
|
||||
]
|
||||
),
|
||||
exact=True,
|
||||
)
|
||||
return result.count
|
||||
|
||||
|
||||
async def test_upsert_points_stores_points_readable_under_the_owning_tenant_filter(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
storage = await _storage(qdrant_client, qdrant_settings)
|
||||
tenant_id, file_id = uuid.uuid4(), uuid.uuid4()
|
||||
|
||||
await storage.upsert_points([_point(tenant_id, file_id, i) for i in range(3)])
|
||||
|
||||
assert await _count_for_tenant(qdrant_client, qdrant_settings.collection, tenant_id) == 3
|
||||
|
||||
|
||||
async def test_upsert_points_are_invisible_to_another_tenants_filter(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
"""The Qdrant-level form of "cross-tenant access finds nothing" (ADR-0002)."""
|
||||
storage = await _storage(qdrant_client, qdrant_settings)
|
||||
owner, other, file_id = uuid.uuid4(), uuid.uuid4(), uuid.uuid4()
|
||||
|
||||
await storage.upsert_points([_point(owner, file_id, i) for i in range(3)])
|
||||
|
||||
assert await _count_for_tenant(qdrant_client, qdrant_settings.collection, other) == 0
|
||||
|
||||
|
||||
async def test_upsert_points_is_idempotent_for_deterministic_ids(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
storage = await _storage(qdrant_client, qdrant_settings)
|
||||
tenant_id, file_id = uuid.uuid4(), uuid.uuid4()
|
||||
points = [_point(tenant_id, file_id, i) for i in range(4)]
|
||||
|
||||
await storage.upsert_points(points)
|
||||
await storage.upsert_points(points)
|
||||
|
||||
assert await _count_for_tenant(qdrant_client, qdrant_settings.collection, tenant_id) == 4
|
||||
|
||||
|
||||
async def test_deactivate_points_from_index_soft_deletes_only_the_tail(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
storage = await _storage(qdrant_client, qdrant_settings)
|
||||
tenant_id, file_id = uuid.uuid4(), uuid.uuid4()
|
||||
await storage.upsert_points([_point(tenant_id, file_id, i) for i in range(5)])
|
||||
|
||||
deactivated = await storage.deactivate_points_from_index(
|
||||
tenant_id=tenant_id,
|
||||
file_id=file_id,
|
||||
from_chunk_index=2,
|
||||
deleted_at=datetime.now(UTC),
|
||||
updated_by="api_key:test",
|
||||
)
|
||||
|
||||
assert deactivated == 3
|
||||
records, _ = await qdrant_client.scroll(
|
||||
collection_name=qdrant_settings.collection,
|
||||
scroll_filter=models.Filter(
|
||||
must=[models.FieldCondition(key="file_id", match=models.MatchValue(value=str(file_id)))]
|
||||
),
|
||||
limit=10,
|
||||
with_payload=True,
|
||||
)
|
||||
by_index = {
|
||||
record.payload["chunk_index"]: record.payload["is_active"]
|
||||
for record in records
|
||||
if record.payload is not None
|
||||
}
|
||||
assert by_index == {0: True, 1: True, 2: False, 3: False, 4: False}
|
||||
# Soft delete, not removal -- the points stay for audit (ADR-0002).
|
||||
assert await _count_for_tenant(qdrant_client, qdrant_settings.collection, tenant_id) == 5
|
||||
|
||||
|
||||
async def test_deactivate_points_from_index_does_not_touch_another_tenants_points(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
"""A file_id alone is never authority to mutate (ADR-0002)."""
|
||||
storage = await _storage(qdrant_client, qdrant_settings)
|
||||
owner, other, file_id = uuid.uuid4(), uuid.uuid4(), uuid.uuid4()
|
||||
await storage.upsert_points([_point(owner, file_id, i) for i in range(3)])
|
||||
|
||||
deactivated = await storage.deactivate_points_from_index(
|
||||
tenant_id=other,
|
||||
file_id=file_id,
|
||||
from_chunk_index=0,
|
||||
deleted_at=datetime.now(UTC),
|
||||
updated_by="api_key:intruder",
|
||||
)
|
||||
|
||||
assert deactivated == 0
|
||||
|
||||
|
||||
async def test_deactivate_points_from_index_returns_zero_when_nothing_is_stale(
|
||||
qdrant_client: AsyncQdrantClient, qdrant_settings: QdrantSettings
|
||||
) -> None:
|
||||
storage = await _storage(qdrant_client, qdrant_settings)
|
||||
tenant_id, file_id = uuid.uuid4(), uuid.uuid4()
|
||||
await storage.upsert_points([_point(tenant_id, file_id, i) for i in range(3)])
|
||||
|
||||
deactivated = await storage.deactivate_points_from_index(
|
||||
tenant_id=tenant_id,
|
||||
file_id=file_id,
|
||||
from_chunk_index=3,
|
||||
deleted_at=datetime.now(UTC),
|
||||
updated_by="api_key:test",
|
||||
)
|
||||
|
||||
assert deactivated == 0
|
||||
Reference in New Issue
Block a user