feat(ingestion): index embedded chunks into Qdrant on upload
Why: - POST /v1/files was reporting chunks_indexed=0/points_created=0 unconditionally — chunks were parsed and embedded but never written to Qdrant, so nothing was actually searchable after upload. Changes: - upload_source_file() now calls index_chunks() after embedding, inside the same INGESTION_TIMEOUT_SECONDS window, and marks the job failed (error_code=index_failed, 502) if it raises. - Job counters (points_created, points_soft_deleted) and the response's chunks_indexed now reflect the real indexing result instead of a hardcoded zero. - Wired PointStorage through AppResources/lifespan/the files router. Impact: - A successful upload is now searchable in Qdrant by the time 201 returns.
This commit is contained in:
Ali Zarinkolah
@@ -13,11 +13,21 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from src.application.auth.context import AuthContext
|
||||
from src.application.files.models import UploadResult
|
||||
from src.application.files.upload import upload_source_file
|
||||
from src.application.ingestion.errors import IngestionAtCapacityError, IngestionTimeoutError
|
||||
from src.application.ingestion.errors import (
|
||||
IngestionAtCapacityError,
|
||||
IngestionTimeoutError,
|
||||
PointIndexingError,
|
||||
)
|
||||
from src.application.ports.object_storage import ObjectStorage
|
||||
from src.config import ChunkingSettings, IngestionSettings
|
||||
from src.application.ports.point_storage import PointStorage
|
||||
from src.config import ChunkingSettings, IngestionSettings, QdrantSettings
|
||||
from src.infrastructure.postgres.models.ingestion_job import IngestionJob
|
||||
from tests.fakes import FakeDenseEmbedder, FakeObjectStorage, FakeSparseEmbedder
|
||||
from tests.fakes import (
|
||||
FakeDenseEmbedder,
|
||||
FakeObjectStorage,
|
||||
FakePointStorage,
|
||||
FakeSparseEmbedder,
|
||||
)
|
||||
from tests.support.factories import create_api_key, create_tenant
|
||||
|
||||
pytestmark = [
|
||||
@@ -34,6 +44,7 @@ async def _upload(
|
||||
sessionmaker: async_sessionmaker[AsyncSession],
|
||||
storage: ObjectStorage,
|
||||
auth: AuthContext,
|
||||
point_storage: PointStorage | None = None,
|
||||
domain: str = "general",
|
||||
filename: str = "report.csv",
|
||||
data: bytes = _CSV_BYTES,
|
||||
@@ -41,12 +52,14 @@ async def _upload(
|
||||
return await upload_source_file(
|
||||
sessionmaker=sessionmaker,
|
||||
storage=storage,
|
||||
point_storage=point_storage if point_storage is not None else FakePointStorage(),
|
||||
auth=auth,
|
||||
domain=domain,
|
||||
filename=filename,
|
||||
data=data,
|
||||
ingestion_settings=IngestionSettings(),
|
||||
chunking_settings=ChunkingSettings(),
|
||||
qdrant_settings=QdrantSettings(),
|
||||
thread_limiter=CapacityLimiter(2),
|
||||
concurrency_limiter=Semaphore(2),
|
||||
dense_embedders=[
|
||||
@@ -83,7 +96,7 @@ async def test_upload_source_file_commits_running_job_before_storage_write(
|
||||
result = await _upload(sessionmaker=db_sessionmaker, storage=storage, auth=auth)
|
||||
|
||||
assert result.status == "succeeded"
|
||||
assert result.chunks_indexed == 0
|
||||
assert result.chunks_indexed == 1
|
||||
assert result.is_new_attempt
|
||||
|
||||
async with db_sessionmaker() as verify_session:
|
||||
@@ -170,12 +183,14 @@ async def test_upload_source_file_timeout_writes_failed_job_and_raises(
|
||||
await upload_source_file(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
point_storage=FakePointStorage(),
|
||||
auth=auth,
|
||||
domain="general",
|
||||
filename="report.csv",
|
||||
data=_CSV_BYTES,
|
||||
ingestion_settings=IngestionSettings(timeout_seconds=0.05),
|
||||
chunking_settings=ChunkingSettings(),
|
||||
qdrant_settings=QdrantSettings(),
|
||||
thread_limiter=CapacityLimiter(2),
|
||||
concurrency_limiter=Semaphore(2),
|
||||
dense_embedders=[slow_embedder, FakeDenseEmbedder(name="dense_openai")],
|
||||
@@ -212,12 +227,14 @@ async def test_upload_source_file_at_capacity_rejects_before_any_job_row(
|
||||
await upload_source_file(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
point_storage=FakePointStorage(),
|
||||
auth=auth,
|
||||
domain="general",
|
||||
filename="report.csv",
|
||||
data=_CSV_BYTES,
|
||||
ingestion_settings=IngestionSettings(),
|
||||
chunking_settings=ChunkingSettings(),
|
||||
qdrant_settings=QdrantSettings(),
|
||||
thread_limiter=CapacityLimiter(2),
|
||||
concurrency_limiter=concurrency_limiter,
|
||||
dense_embedders=[
|
||||
@@ -238,3 +255,143 @@ async def test_upload_source_file_at_capacity_rejects_before_any_job_row(
|
||||
.all()
|
||||
)
|
||||
assert jobs == []
|
||||
|
||||
|
||||
async def test_upload_source_file_indexes_points_and_records_real_counters(
|
||||
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
||||
) -> None:
|
||||
"""The Phase 5 payoff: a successful upload is searchable, and the job row
|
||||
says how many points it wrote rather than a hardcoded zero.
|
||||
"""
|
||||
auth = await _auth_for(db_session)
|
||||
point_storage = FakePointStorage()
|
||||
|
||||
result = await _upload(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
auth=auth,
|
||||
point_storage=point_storage,
|
||||
data=b"name,value\nfirst,1\nsecond,2\nthird,3\n",
|
||||
)
|
||||
|
||||
assert result.status == "succeeded"
|
||||
assert result.chunks_indexed == len(point_storage.points)
|
||||
assert result.chunks_indexed > 0
|
||||
|
||||
async with db_sessionmaker() as verify_session:
|
||||
job = await verify_session.get(IngestionJob, result.ingestion_job_id)
|
||||
assert job is not None
|
||||
assert job.points_created == result.chunks_indexed
|
||||
# An upsert cannot tell an insert from an overwrite, so this stays 0.
|
||||
assert job.points_updated == 0
|
||||
|
||||
|
||||
async def test_upload_source_file_indexes_points_under_the_authenticated_tenant(
|
||||
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
||||
) -> None:
|
||||
auth = await _auth_for(db_session)
|
||||
point_storage = FakePointStorage()
|
||||
|
||||
result = await _upload(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
auth=auth,
|
||||
point_storage=point_storage,
|
||||
domain="fire",
|
||||
)
|
||||
|
||||
payloads = [point.payload for point in point_storage.points.values()]
|
||||
assert payloads
|
||||
for payload in payloads:
|
||||
assert payload["tenant_id"] == str(auth.tenant_id)
|
||||
assert payload["domain"] == "fire"
|
||||
assert payload["file_id"] == str(result.file_id)
|
||||
assert payload["created_by"] == f"api_key:{auth.api_key_id}"
|
||||
|
||||
|
||||
async def test_upload_source_file_index_failure_writes_terminal_failed_job(
|
||||
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
||||
) -> None:
|
||||
auth = await _auth_for(db_session)
|
||||
point_storage = FakePointStorage(fail_on_batch=0)
|
||||
|
||||
with pytest.raises(PointIndexingError):
|
||||
await _upload(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
auth=auth,
|
||||
point_storage=point_storage,
|
||||
)
|
||||
|
||||
async with db_sessionmaker() as verify_session:
|
||||
jobs = (await verify_session.execute(select(IngestionJob))).scalars().all()
|
||||
job = next(job for job in jobs if job.tenant_id == auth.tenant_id)
|
||||
assert job.status == "failed"
|
||||
assert job.error_code == "index_failed"
|
||||
assert job.completed_at is not None
|
||||
|
||||
|
||||
async def test_upload_source_file_failed_index_does_not_soft_delete_existing_points(
|
||||
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
||||
) -> None:
|
||||
"""A failed attempt must never remove content from a working index."""
|
||||
auth = await _auth_for(db_session)
|
||||
point_storage = FakePointStorage()
|
||||
await _upload(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
auth=auth,
|
||||
point_storage=point_storage,
|
||||
data=b"name,value\nfirst,1\nsecond,2\n",
|
||||
)
|
||||
# fail_on_batch indexes into upsert_batches, which accumulates across
|
||||
# uploads -- reset it so "batch 0" means the retry's first batch.
|
||||
point_storage.deactivate_calls.clear()
|
||||
point_storage.upsert_batches.clear()
|
||||
point_storage.fail_on_batch = 0
|
||||
|
||||
with pytest.raises(PointIndexingError):
|
||||
await _upload(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
auth=auth,
|
||||
point_storage=point_storage,
|
||||
data=b"name,value\nonly,1\n",
|
||||
)
|
||||
|
||||
assert point_storage.deactivate_calls == []
|
||||
assert all(point.payload["is_active"] is True for point in point_storage.points.values())
|
||||
|
||||
|
||||
async def test_upload_source_file_retry_after_index_failure_produces_no_duplicate_points(
|
||||
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
||||
) -> None:
|
||||
auth = await _auth_for(db_session)
|
||||
point_storage = FakePointStorage(fail_on_batch=0)
|
||||
|
||||
with pytest.raises(PointIndexingError):
|
||||
await _upload(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
auth=auth,
|
||||
point_storage=point_storage,
|
||||
)
|
||||
|
||||
point_storage.fail_on_batch = None
|
||||
retry = await _upload(
|
||||
sessionmaker=db_sessionmaker,
|
||||
storage=FakeObjectStorage(),
|
||||
auth=auth,
|
||||
point_storage=point_storage,
|
||||
)
|
||||
|
||||
assert retry.status == "succeeded"
|
||||
assert len(point_storage.points) == retry.chunks_indexed
|
||||
|
||||
async with db_sessionmaker() as verify_session:
|
||||
jobs = (await verify_session.execute(select(IngestionJob))).scalars().all()
|
||||
tenant_jobs = [job for job in jobs if job.tenant_id == auth.tenant_id]
|
||||
# A terminal job never returns to `running` (ADR-0017); the retry is a new row.
|
||||
assert len(tenant_jobs) == 2
|
||||
assert {job.status for job in tenant_jobs} == {"failed", "succeeded"}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user