feat(ingestion): index embedded chunks into Qdrant on upload
Why: - POST /v1/files was reporting chunks_indexed=0/points_created=0 unconditionally — chunks were parsed and embedded but never written to Qdrant, so nothing was actually searchable after upload. Changes: - upload_source_file() now calls index_chunks() after embedding, inside the same INGESTION_TIMEOUT_SECONDS window, and marks the job failed (error_code=index_failed, 502) if it raises. - Job counters (points_created, points_soft_deleted) and the response's chunks_indexed now reflect the real indexing result instead of a hardcoded zero. - Wired PointStorage through AppResources/lifespan/the files router. Impact: - A successful upload is now searchable in Qdrant by the time 201 returns.
This commit is contained in:
@@ -16,9 +16,10 @@ job stuck in `running`. The whole request additionally holds one of
|
||||
phase 2 is bounded by `INGESTION_TIMEOUT_SECONDS` (`504`) (ADR-0017, plan 001
|
||||
Phase 4).
|
||||
|
||||
Qdrant point upserts are Phase 5 work, not implemented here: this phase
|
||||
parses, chunks, and embeds, so a successful job still reports
|
||||
`chunks_indexed=0` — nothing is searchable yet.
|
||||
Phase 2 ends by upserting the embedded chunks as tenant-scoped Qdrant points
|
||||
(`src/application/points/`), so a successful upload is searchable by the time
|
||||
the `201` returns. The collection those points land in is provisioned by a
|
||||
deployment step, not by this path — see `src/cli/qdrant_bootstrap.py`.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
@@ -45,10 +46,13 @@ from src.application.ingestion.errors import (
|
||||
ChunkLimitExceededError,
|
||||
EmbedderError,
|
||||
IngestionTimeoutError,
|
||||
PointIndexingError,
|
||||
)
|
||||
from src.application.points import index_chunks
|
||||
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||
from src.application.ports.object_storage import ObjectStorage
|
||||
from src.config import ChunkingSettings, IngestionSettings
|
||||
from src.application.ports.point_storage import PointStorage
|
||||
from src.config import ChunkingSettings, IngestionSettings, QdrantSettings
|
||||
from src.infrastructure.postgres.repositories import ingestion_jobs as jobs_repo
|
||||
from src.infrastructure.postgres.repositories import source_files as source_files_repo
|
||||
|
||||
@@ -88,12 +92,14 @@ async def upload_source_file(
|
||||
*,
|
||||
sessionmaker: async_sessionmaker[AsyncSession],
|
||||
storage: ObjectStorage,
|
||||
point_storage: PointStorage,
|
||||
auth: AuthContext,
|
||||
domain: str,
|
||||
filename: str,
|
||||
data: bytes,
|
||||
ingestion_settings: IngestionSettings,
|
||||
chunking_settings: ChunkingSettings,
|
||||
qdrant_settings: QdrantSettings,
|
||||
thread_limiter: CapacityLimiter,
|
||||
concurrency_limiter: Semaphore,
|
||||
dense_embedders: Sequence[DenseEmbedder],
|
||||
@@ -249,6 +255,31 @@ async def upload_source_file(
|
||||
error_message=str(exc),
|
||||
)
|
||||
raise
|
||||
|
||||
try:
|
||||
indexed = await index_chunks(
|
||||
embedded,
|
||||
storage=point_storage,
|
||||
tenant_id=auth.tenant_id,
|
||||
domain=domain,
|
||||
file_id=source_file_id,
|
||||
source_filename=filename,
|
||||
source_type=validated.source_type,
|
||||
actor=f"api_key:{auth.api_key_id}",
|
||||
dense_embedders=dense_embedders,
|
||||
sparse_embedder=sparse_embedder,
|
||||
settings=qdrant_settings,
|
||||
thread_limiter=thread_limiter,
|
||||
)
|
||||
except PointIndexingError as exc:
|
||||
await _mark_job_failed(
|
||||
sessionmaker,
|
||||
tenant_id=auth.tenant_id,
|
||||
ingestion_job_id=ingestion_job_id,
|
||||
error_code="index_failed",
|
||||
error_message=str(exc),
|
||||
)
|
||||
raise
|
||||
except TimeoutError:
|
||||
logger.warning(
|
||||
"files.upload.timeout",
|
||||
@@ -273,7 +304,11 @@ async def upload_source_file(
|
||||
tenant_id=auth.tenant_id,
|
||||
ingestion_job_id=ingestion_job_id,
|
||||
status="succeeded",
|
||||
points_created=0,
|
||||
# An upsert with deterministic ids cannot tell an insert from
|
||||
# an overwrite, so every written point is reported here and
|
||||
# `points_updated` stays 0 rather than being guessed at.
|
||||
points_created=indexed.points_upserted,
|
||||
points_soft_deleted=indexed.points_soft_deleted,
|
||||
)
|
||||
jobs_repo.append_event(
|
||||
session,
|
||||
@@ -281,8 +316,13 @@ async def upload_source_file(
|
||||
ingestion_job_id=ingestion_job_id,
|
||||
level="info",
|
||||
stage="completed",
|
||||
message="chunks parsed and embedded; Qdrant indexing not yet implemented",
|
||||
details={"chunks_parsed": len(chunks), "chunks_embedded": len(embedded)},
|
||||
message="chunks parsed, embedded, and indexed",
|
||||
details={
|
||||
"chunks_parsed": len(chunks),
|
||||
"chunks_embedded": len(embedded),
|
||||
"points_upserted": indexed.points_upserted,
|
||||
"points_soft_deleted": indexed.points_soft_deleted,
|
||||
},
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
@@ -291,11 +331,12 @@ async def upload_source_file(
|
||||
tenant_id=str(auth.tenant_id),
|
||||
file_id=str(source_file_id),
|
||||
ingestion_job_id=str(ingestion_job_id),
|
||||
points_indexed=indexed.points_upserted,
|
||||
)
|
||||
return UploadResult(
|
||||
file_id=source_file_id,
|
||||
ingestion_job_id=ingestion_job_id,
|
||||
status="succeeded",
|
||||
chunks_indexed=0,
|
||||
chunks_indexed=indexed.points_upserted,
|
||||
is_new_attempt=True,
|
||||
)
|
||||
|
||||
@@ -49,6 +49,15 @@ class EmbedderError(IngestionError):
|
||||
"""
|
||||
|
||||
|
||||
class PointIndexingError(IngestionError):
|
||||
"""Upserting or soft-deleting Qdrant points failed.
|
||||
|
||||
Maps to `502` — like `EmbedderError`, this is an upstream dependency
|
||||
failing, not a malformed request. Kept distinct from `EmbedderError` so the
|
||||
job's `error_code` says which dependency broke.
|
||||
"""
|
||||
|
||||
|
||||
class IngestionAtCapacityError(IngestionError):
|
||||
"""`INGESTION_MAX_CONCURRENCY` in-process ingestions are already running.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user