feat(ingestion): index embedded chunks into Qdrant on upload

Why:
- POST /v1/files was reporting chunks_indexed=0/points_created=0 unconditionally
  — chunks were parsed and embedded but never written to Qdrant, so nothing
  was actually searchable after upload.

Changes:
- upload_source_file() now calls index_chunks() after embedding, inside the
  same INGESTION_TIMEOUT_SECONDS window, and marks the job failed
  (error_code=index_failed, 502) if it raises.
- Job counters (points_created, points_soft_deleted) and the response's
  chunks_indexed now reflect the real indexing result instead of a hardcoded
  zero.
- Wired PointStorage through AppResources/lifespan/the files router.

Impact:
- A successful upload is now searchable in Qdrant by the time 201 returns.
This commit is contained in:
Ali Zarinkolah
2026-08-20 18:17:39 +03:30
parent d00d436e5c
commit cc915f0f1a
11 changed files with 304 additions and 24 deletions

View File

@@ -19,11 +19,13 @@ from src.application.files.status import get_file_status
from src.application.files.upload import upload_source_file
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
from src.application.ports.object_storage import ObjectStorage
from src.application.ports.point_storage import PointStorage
from src.bootstrap.dependencies import (
get_dense_embedders,
get_ingestion_concurrency_limiter,
get_ingestion_limiter,
get_object_storage,
get_point_storage,
get_sessionmaker,
get_settings,
get_sparse_embedder,
@@ -35,6 +37,7 @@ router = APIRouter(prefix="/files", tags=["files"])
_RequireFilesWrite = Annotated[AuthContext, Depends(require_scope("files:write"))]
_SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)]
_ObjectStorageDep = Annotated[ObjectStorage, Depends(get_object_storage)]
_PointStorageDep = Annotated[PointStorage, Depends(get_point_storage)]
_SettingsDep = Annotated[Settings, Depends(get_settings)]
_IngestionLimiterDep = Annotated[CapacityLimiter, Depends(get_ingestion_limiter)]
_ConcurrencyLimiterDep = Annotated[Semaphore, Depends(get_ingestion_concurrency_limiter)]
@@ -50,6 +53,7 @@ async def upload_file(
auth: _RequireFilesWrite,
sessionmaker: _SessionmakerDep,
storage: _ObjectStorageDep,
point_storage: _PointStorageDep,
settings: _SettingsDep,
limiter: _IngestionLimiterDep,
concurrency_limiter: _ConcurrencyLimiterDep,
@@ -60,12 +64,14 @@ async def upload_file(
result = await upload_source_file(
sessionmaker=sessionmaker,
storage=storage,
point_storage=point_storage,
auth=auth,
domain=domain,
filename=file.filename or "",
data=data,
ingestion_settings=settings.ingestion,
chunking_settings=settings.chunking,
qdrant_settings=settings.qdrant,
thread_limiter=limiter,
concurrency_limiter=concurrency_limiter,
dense_embedders=dense_embedders,