feat(ingestion): index embedded chunks into Qdrant on upload
Why: - POST /v1/files was reporting chunks_indexed=0/points_created=0 unconditionally — chunks were parsed and embedded but never written to Qdrant, so nothing was actually searchable after upload. Changes: - upload_source_file() now calls index_chunks() after embedding, inside the same INGESTION_TIMEOUT_SECONDS window, and marks the job failed (error_code=index_failed, 502) if it raises. - Job counters (points_created, points_soft_deleted) and the response's chunks_indexed now reflect the real indexing result instead of a hardcoded zero. - Wired PointStorage through AppResources/lifespan/the files router. Impact: - A successful upload is now searchable in Qdrant by the time 201 returns.
This commit is contained in:
@@ -23,6 +23,7 @@ from src.application.ingestion.errors import (
|
||||
EmbedderError,
|
||||
IngestionAtCapacityError,
|
||||
IngestionTimeoutError,
|
||||
PointIndexingError,
|
||||
UnsupportedSourceTypeError,
|
||||
)
|
||||
|
||||
@@ -44,6 +45,7 @@ _MAPPING: tuple[tuple[type[Exception], int, str], ...] = (
|
||||
(FileTooLargeError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
|
||||
(ChunkLimitExceededError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
|
||||
(EmbedderError, status.HTTP_502_BAD_GATEWAY, "embedder_error"),
|
||||
(PointIndexingError, status.HTTP_502_BAD_GATEWAY, "index_error"),
|
||||
(IngestionTimeoutError, status.HTTP_504_GATEWAY_TIMEOUT, "ingestion_timeout"),
|
||||
)
|
||||
|
||||
|
||||
@@ -19,11 +19,13 @@ from src.application.files.status import get_file_status
|
||||
from src.application.files.upload import upload_source_file
|
||||
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||
from src.application.ports.object_storage import ObjectStorage
|
||||
from src.application.ports.point_storage import PointStorage
|
||||
from src.bootstrap.dependencies import (
|
||||
get_dense_embedders,
|
||||
get_ingestion_concurrency_limiter,
|
||||
get_ingestion_limiter,
|
||||
get_object_storage,
|
||||
get_point_storage,
|
||||
get_sessionmaker,
|
||||
get_settings,
|
||||
get_sparse_embedder,
|
||||
@@ -35,6 +37,7 @@ router = APIRouter(prefix="/files", tags=["files"])
|
||||
_RequireFilesWrite = Annotated[AuthContext, Depends(require_scope("files:write"))]
|
||||
_SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)]
|
||||
_ObjectStorageDep = Annotated[ObjectStorage, Depends(get_object_storage)]
|
||||
_PointStorageDep = Annotated[PointStorage, Depends(get_point_storage)]
|
||||
_SettingsDep = Annotated[Settings, Depends(get_settings)]
|
||||
_IngestionLimiterDep = Annotated[CapacityLimiter, Depends(get_ingestion_limiter)]
|
||||
_ConcurrencyLimiterDep = Annotated[Semaphore, Depends(get_ingestion_concurrency_limiter)]
|
||||
@@ -50,6 +53,7 @@ async def upload_file(
|
||||
auth: _RequireFilesWrite,
|
||||
sessionmaker: _SessionmakerDep,
|
||||
storage: _ObjectStorageDep,
|
||||
point_storage: _PointStorageDep,
|
||||
settings: _SettingsDep,
|
||||
limiter: _IngestionLimiterDep,
|
||||
concurrency_limiter: _ConcurrencyLimiterDep,
|
||||
@@ -60,12 +64,14 @@ async def upload_file(
|
||||
result = await upload_source_file(
|
||||
sessionmaker=sessionmaker,
|
||||
storage=storage,
|
||||
point_storage=point_storage,
|
||||
auth=auth,
|
||||
domain=domain,
|
||||
filename=file.filename or "",
|
||||
data=data,
|
||||
ingestion_settings=settings.ingestion,
|
||||
chunking_settings=settings.chunking,
|
||||
qdrant_settings=settings.qdrant,
|
||||
thread_limiter=limiter,
|
||||
concurrency_limiter=concurrency_limiter,
|
||||
dense_embedders=dense_embedders,
|
||||
|
||||
Reference in New Issue
Block a user