feat(ingestion): index embedded chunks into Qdrant on upload

Why:
- POST /v1/files was reporting chunks_indexed=0/points_created=0 unconditionally
  — chunks were parsed and embedded but never written to Qdrant, so nothing
  was actually searchable after upload.

Changes:
- upload_source_file() now calls index_chunks() after embedding, inside the
  same INGESTION_TIMEOUT_SECONDS window, and marks the job failed
  (error_code=index_failed, 502) if it raises.
- Job counters (points_created, points_soft_deleted) and the response's
  chunks_indexed now reflect the real indexing result instead of a hardcoded
  zero.
- Wired PointStorage through AppResources/lifespan/the files router.

Impact:
- A successful upload is now searchable in Qdrant by the time 201 returns.
This commit is contained in:
Ali Zarinkolah
2026-08-20 18:17:39 +03:30
parent d00d436e5c
commit cc915f0f1a
11 changed files with 304 additions and 24 deletions

View File

@@ -16,9 +16,10 @@ job stuck in `running`. The whole request additionally holds one of
phase 2 is bounded by `INGESTION_TIMEOUT_SECONDS` (`504`) (ADR-0017, plan 001
Phase 4).
Qdrant point upserts are Phase 5 work, not implemented here: this phase
parses, chunks, and embeds, so a successful job still reports
`chunks_indexed=0` — nothing is searchable yet.
Phase 2 ends by upserting the embedded chunks as tenant-scoped Qdrant points
(`src/application/points/`), so a successful upload is searchable by the time
the `201` returns. The collection those points land in is provisioned by a
deployment step, not by this path — see `src/cli/qdrant_bootstrap.py`.
"""
import uuid
@@ -45,10 +46,13 @@ from src.application.ingestion.errors import (
ChunkLimitExceededError,
EmbedderError,
IngestionTimeoutError,
PointIndexingError,
)
from src.application.points import index_chunks
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
from src.application.ports.object_storage import ObjectStorage
from src.config import ChunkingSettings, IngestionSettings
from src.application.ports.point_storage import PointStorage
from src.config import ChunkingSettings, IngestionSettings, QdrantSettings
from src.infrastructure.postgres.repositories import ingestion_jobs as jobs_repo
from src.infrastructure.postgres.repositories import source_files as source_files_repo
@@ -88,12 +92,14 @@ async def upload_source_file(
*,
sessionmaker: async_sessionmaker[AsyncSession],
storage: ObjectStorage,
point_storage: PointStorage,
auth: AuthContext,
domain: str,
filename: str,
data: bytes,
ingestion_settings: IngestionSettings,
chunking_settings: ChunkingSettings,
qdrant_settings: QdrantSettings,
thread_limiter: CapacityLimiter,
concurrency_limiter: Semaphore,
dense_embedders: Sequence[DenseEmbedder],
@@ -249,6 +255,31 @@ async def upload_source_file(
error_message=str(exc),
)
raise
try:
indexed = await index_chunks(
embedded,
storage=point_storage,
tenant_id=auth.tenant_id,
domain=domain,
file_id=source_file_id,
source_filename=filename,
source_type=validated.source_type,
actor=f"api_key:{auth.api_key_id}",
dense_embedders=dense_embedders,
sparse_embedder=sparse_embedder,
settings=qdrant_settings,
thread_limiter=thread_limiter,
)
except PointIndexingError as exc:
await _mark_job_failed(
sessionmaker,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
error_code="index_failed",
error_message=str(exc),
)
raise
except TimeoutError:
logger.warning(
"files.upload.timeout",
@@ -273,7 +304,11 @@ async def upload_source_file(
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
status="succeeded",
points_created=0,
# An upsert with deterministic ids cannot tell an insert from
# an overwrite, so every written point is reported here and
# `points_updated` stays 0 rather than being guessed at.
points_created=indexed.points_upserted,
points_soft_deleted=indexed.points_soft_deleted,
)
jobs_repo.append_event(
session,
@@ -281,8 +316,13 @@ async def upload_source_file(
ingestion_job_id=ingestion_job_id,
level="info",
stage="completed",
message="chunks parsed and embedded; Qdrant indexing not yet implemented",
details={"chunks_parsed": len(chunks), "chunks_embedded": len(embedded)},
message="chunks parsed, embedded, and indexed",
details={
"chunks_parsed": len(chunks),
"chunks_embedded": len(embedded),
"points_upserted": indexed.points_upserted,
"points_soft_deleted": indexed.points_soft_deleted,
},
)
await session.commit()
@@ -291,11 +331,12 @@ async def upload_source_file(
tenant_id=str(auth.tenant_id),
file_id=str(source_file_id),
ingestion_job_id=str(ingestion_job_id),
points_indexed=indexed.points_upserted,
)
return UploadResult(
file_id=source_file_id,
ingestion_job_id=ingestion_job_id,
status="succeeded",
chunks_indexed=0,
chunks_indexed=indexed.points_upserted,
is_new_attempt=True,
)

View File

@@ -49,6 +49,15 @@ class EmbedderError(IngestionError):
"""
class PointIndexingError(IngestionError):
"""Upserting or soft-deleting Qdrant points failed.
Maps to `502` — like `EmbedderError`, this is an upstream dependency
failing, not a malformed request. Kept distinct from `EmbedderError` so the
job's `error_code` says which dependency broke.
"""
class IngestionAtCapacityError(IngestionError):
"""`INGESTION_MAX_CONCURRENCY` in-process ingestions are already running.