azk/chatbot_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ingestion): index embedded chunks into Qdrant on upload

Browse Source 
Why:
- POST /v1/files was reporting chunks_indexed=0/points_created=0 unconditionally
  — chunks were parsed and embedded but never written to Qdrant, so nothing
  was actually searchable after upload.

Changes:
- upload_source_file() now calls index_chunks() after embedding, inside the
  same INGESTION_TIMEOUT_SECONDS window, and marks the job failed
  (error_code=index_failed, 502) if it raises.
- Job counters (points_created, points_soft_deleted) and the response's
  chunks_indexed now reflect the real indexing result instead of a hardcoded
  zero.
- Wired PointStorage through AppResources/lifespan/the files router.

Impact:
- A successful upload is now searchable in Qdrant by the time 201 returns.
This commit is contained in:
Ali Zarinkolah
2026-08-20 18:17:39 +03:30
parent d00d436e5c
commit cc915f0f1a
11 changed files with 304 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/api/errors.py
View File

@@ -23,6 +23,7 @@ from src.application.ingestion.errors import (
EmbedderError,
IngestionAtCapacityError,
IngestionTimeoutError,
PointIndexingError,
UnsupportedSourceTypeError,
)
@@ -44,6 +45,7 @@ _MAPPING: tuple[tuple[type[Exception], int, str], ...] = (
(FileTooLargeError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
(ChunkLimitExceededError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
(EmbedderError, status.HTTP_502_BAD_GATEWAY, "embedder_error"),
(PointIndexingError, status.HTTP_502_BAD_GATEWAY, "index_error"),
(IngestionTimeoutError, status.HTTP_504_GATEWAY_TIMEOUT, "ingestion_timeout"),
)

6
src/api/routers/files.py
View File

@@ -19,11 +19,13 @@ from src.application.files.status import get_file_status
from src.application.files.upload import upload_source_file
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
from src.application.ports.object_storage import ObjectStorage
from src.application.ports.point_storage import PointStorage
from src.bootstrap.dependencies import (
get_dense_embedders,
get_ingestion_concurrency_limiter,
get_ingestion_limiter,
get_object_storage,
get_point_storage,
get_sessionmaker,
get_settings,
get_sparse_embedder,
@@ -35,6 +37,7 @@ router = APIRouter(prefix="/files", tags=["files"])
_RequireFilesWrite = Annotated[AuthContext, Depends(require_scope("files:write"))]
_SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)]
_ObjectStorageDep = Annotated[ObjectStorage, Depends(get_object_storage)]
_PointStorageDep = Annotated[PointStorage, Depends(get_point_storage)]
_SettingsDep = Annotated[Settings, Depends(get_settings)]
_IngestionLimiterDep = Annotated[CapacityLimiter, Depends(get_ingestion_limiter)]
_ConcurrencyLimiterDep = Annotated[Semaphore, Depends(get_ingestion_concurrency_limiter)]
@@ -50,6 +53,7 @@ async def upload_file(
auth: _RequireFilesWrite,
sessionmaker: _SessionmakerDep,
storage: _ObjectStorageDep,
point_storage: _PointStorageDep,
settings: _SettingsDep,
limiter: _IngestionLimiterDep,
concurrency_limiter: _ConcurrencyLimiterDep,
@@ -60,12 +64,14 @@ async def upload_file(
result = await upload_source_file(
sessionmaker=sessionmaker,
storage=storage,
point_storage=point_storage,
auth=auth,
domain=domain,
filename=file.filename or "",
data=data,
ingestion_settings=settings.ingestion,
chunking_settings=settings.chunking,
qdrant_settings=settings.qdrant,
thread_limiter=limiter,
concurrency_limiter=concurrency_limiter,
dense_embedders=dense_embedders,

57
src/application/files/upload.py
View File

@@ -16,9 +16,10 @@ job stuck in `running`. The whole request additionally holds one of
phase 2 is bounded by `INGESTION_TIMEOUT_SECONDS` (`504`) (ADR-0017, plan 001
Phase 4).
Qdrant point upserts are Phase 5 work, not implemented here: this phase
parses, chunks, and embeds, so a successful job still reports
`chunks_indexed=0` — nothing is searchable yet.
Phase 2 ends by upserting the embedded chunks as tenant-scoped Qdrant points
(`src/application/points/`), so a successful upload is searchable by the time
the `201` returns. The collection those points land in is provisioned by a
deployment step, not by this path — see `src/cli/qdrant_bootstrap.py`.
"""
import uuid
@@ -45,10 +46,13 @@ from src.application.ingestion.errors import (
ChunkLimitExceededError,
EmbedderError,
IngestionTimeoutError,
PointIndexingError,
)
from src.application.points import index_chunks
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
from src.application.ports.object_storage import ObjectStorage
from src.config import ChunkingSettings, IngestionSettings
from src.application.ports.point_storage import PointStorage
from src.config import ChunkingSettings, IngestionSettings, QdrantSettings
from src.infrastructure.postgres.repositories import ingestion_jobs as jobs_repo
from src.infrastructure.postgres.repositories import source_files as source_files_repo
@@ -88,12 +92,14 @@ async def upload_source_file(
*,
sessionmaker: async_sessionmaker[AsyncSession],
storage: ObjectStorage,
point_storage: PointStorage,
auth: AuthContext,
domain: str,
filename: str,
data: bytes,
ingestion_settings: IngestionSettings,
chunking_settings: ChunkingSettings,
qdrant_settings: QdrantSettings,
thread_limiter: CapacityLimiter,
concurrency_limiter: Semaphore,
dense_embedders: Sequence[DenseEmbedder],
@@ -249,6 +255,31 @@ async def upload_source_file(
error_message=str(exc),
)
raise
try:
indexed = await index_chunks(
embedded,
storage=point_storage,
tenant_id=auth.tenant_id,
domain=domain,
file_id=source_file_id,
source_filename=filename,
source_type=validated.source_type,
actor=f"api_key:{auth.api_key_id}",
dense_embedders=dense_embedders,
sparse_embedder=sparse_embedder,
settings=qdrant_settings,
thread_limiter=thread_limiter,
)
except PointIndexingError as exc:
await _mark_job_failed(
sessionmaker,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
error_code="index_failed",
error_message=str(exc),
)
raise
except TimeoutError:
logger.warning(
"files.upload.timeout",
@@ -273,7 +304,11 @@ async def upload_source_file(
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
status="succeeded",
points_created=0,
# An upsert with deterministic ids cannot tell an insert from
# an overwrite, so every written point is reported here and
# `points_updated` stays 0 rather than being guessed at.
points_created=indexed.points_upserted,
points_soft_deleted=indexed.points_soft_deleted,
)
jobs_repo.append_event(
session,
@@ -281,8 +316,13 @@ async def upload_source_file(
ingestion_job_id=ingestion_job_id,
level="info",
stage="completed",
message="chunks parsed and embedded; Qdrant indexing not yet implemented",
details={"chunks_parsed": len(chunks), "chunks_embedded": len(embedded)},
message="chunks parsed, embedded, and indexed",
details={
"chunks_parsed": len(chunks),
"chunks_embedded": len(embedded),
"points_upserted": indexed.points_upserted,
"points_soft_deleted": indexed.points_soft_deleted,
},
)
await session.commit()
@@ -291,11 +331,12 @@ async def upload_source_file(
tenant_id=str(auth.tenant_id),
file_id=str(source_file_id),
ingestion_job_id=str(ingestion_job_id),
points_indexed=indexed.points_upserted,
)
return UploadResult(
file_id=source_file_id,
ingestion_job_id=ingestion_job_id,
status="succeeded",
chunks_indexed=0,
chunks_indexed=indexed.points_upserted,
is_new_attempt=True,
)

9
src/application/ingestion/errors.py
View File

@@ -49,6 +49,15 @@ class EmbedderError(IngestionError):
"""
class PointIndexingError(IngestionError):
"""Upserting or soft-deleting Qdrant points failed.
Maps to `502` — like `EmbedderError`, this is an upstream dependency
failing, not a malformed request. Kept distinct from `EmbedderError` so the
job's `error_code` says which dependency broke.
"""
class IngestionAtCapacityError(IngestionError):
"""`INGESTION_MAX_CONCURRENCY` in-process ingestions are already running.

6
src/bootstrap/dependencies.py
View File

@@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
from src.application.ports.object_storage import ObjectStorage
from src.application.ports.point_storage import PointStorage
from src.config import Settings
@@ -20,6 +21,7 @@ class AppResources:
minio_client: Minio
qdrant_client: AsyncQdrantClient
object_storage: ObjectStorage
point_storage: PointStorage
ingestion_limiter: CapacityLimiter
dense_embedders: Sequence[DenseEmbedder]
sparse_embedder: SparseEmbedder
@@ -46,6 +48,10 @@ def get_object_storage(request: Request) -> ObjectStorage:
return _resources(request).object_storage
def get_point_storage(request: Request) -> PointStorage:
return _resources(request).point_storage
def get_ingestion_limiter(request: Request) -> CapacityLimiter:
return _resources(request).ingestion_limiter

9
src/bootstrap/lifespan.py
View File

@@ -20,6 +20,7 @@ from src.infrastructure.minio.storage import MinioObjectStorage
from src.infrastructure.observability.logging import configure_logging
from src.infrastructure.postgres.database import create_engine, create_sessionmaker
from src.infrastructure.qdrant.client import create_client as create_qdrant_client
from src.infrastructure.qdrant.points import QdrantPointStorage
logger = structlog.get_logger(__name__)
@@ -77,6 +78,13 @@ def create_lifespan(
logger.info("lifespan.minio.client.created")
qdrant_client = create_qdrant_client(resolved_settings.qdrant)
# No collection DDL here: `ensure_chunks_collection` is a deployment
# step (`python -m src.cli.qdrant_bootstrap`), for the same reason
# ADR-0009 keeps Alembic out of startup and ADR-0012 makes LangGraph's
# `.setup()` a deployment step.
point_storage = QdrantPointStorage(
qdrant_client, collection=resolved_settings.qdrant.collection
)
logger.info("lifespan.qdrant.client.created")
nomic_settings = resolved_settings.embedding.nomic
@@ -136,6 +144,7 @@ def create_lifespan(
minio_client=minio_client,
qdrant_client=qdrant_client,
object_storage=object_storage,
point_storage=point_storage,
ingestion_limiter=ingestion_limiter,
dense_embedders=dense_embedders,
sparse_embedder=sparse_embedder,