Ali Zarinkolah
5c0a5938f8
Why: - Plan 001 Phase 4 needs batched, concurrency-bounded embedding wired into the inline upload path, with process-wide capacity/timeout/chunk-limit guards (ADR-0017). - The BM25 analyzer and dense-model config are ported from the `emet` evaluation lab, which benchmarked them against the real Farsi corpus (bm25-fa-norm-stop; nomic-embed-text-v2-moe at 768-dim; text-embedding-3-large at native 3072-dim), closing open items in ADR-0001/ADR-0005. Changes: - New: embedding ports, orchestration (embed_chunks), request-bounds helpers, and dense/sparse adapters (analyzers.py, bm25.py, openai_compatible.py). - upload.py now parses/chunks/embeds inline behind INGESTION_MAX_CONCURRENCY (503), INGESTION_TIMEOUT_SECONDS (504), and the chunk-count ceiling (413); every failure path still writes a terminal job row. - Lifespan builds and warms both dense embedders at startup (fail-soft) and creates the sparse embedder and concurrency semaphore. - httpx moves from dev to main dependencies (adapters use it directly). Impact: - Qdrant point upserts are still Phase 5 -- chunks_indexed stays 0. - New EMBEDDING_* env vars documented in .env.example; safe defaults. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
89 lines
3.3 KiB
Python
89 lines
3.3 KiB
Python
"""`POST /v1/files`, `GET /v1/files/{file_id}` (ADR-0008).
|
|
|
|
Routes adapt HTTP to `application/files` calls; they do not parse, hash,
|
|
touch MinIO/Qdrant, or otherwise carry ingestion business logic (ADR-0015).
|
|
"""
|
|
|
|
import uuid
|
|
from collections.abc import Sequence
|
|
from typing import Annotated
|
|
|
|
from anyio import CapacityLimiter, Semaphore
|
|
from fastapi import APIRouter, Depends, Form, HTTPException, Response, UploadFile, status
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
|
|
from src.api.dependencies.auth import require_scope
|
|
from src.api.schemas.files import FileStatusResponse, FileUploadResponse
|
|
from src.application.auth.context import AuthContext
|
|
from src.application.files.status import get_file_status
|
|
from src.application.files.upload import upload_source_file
|
|
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
|
from src.application.ports.object_storage import ObjectStorage
|
|
from src.bootstrap.dependencies import (
|
|
get_dense_embedders,
|
|
get_ingestion_concurrency_limiter,
|
|
get_ingestion_limiter,
|
|
get_object_storage,
|
|
get_sessionmaker,
|
|
get_settings,
|
|
get_sparse_embedder,
|
|
)
|
|
from src.config import Settings
|
|
|
|
router = APIRouter(prefix="/files", tags=["files"])
|
|
|
|
_RequireFilesWrite = Annotated[AuthContext, Depends(require_scope("files:write"))]
|
|
_SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)]
|
|
_ObjectStorageDep = Annotated[ObjectStorage, Depends(get_object_storage)]
|
|
_SettingsDep = Annotated[Settings, Depends(get_settings)]
|
|
_IngestionLimiterDep = Annotated[CapacityLimiter, Depends(get_ingestion_limiter)]
|
|
_ConcurrencyLimiterDep = Annotated[Semaphore, Depends(get_ingestion_concurrency_limiter)]
|
|
_DenseEmbeddersDep = Annotated[Sequence[DenseEmbedder], Depends(get_dense_embedders)]
|
|
_SparseEmbedderDep = Annotated[SparseEmbedder, Depends(get_sparse_embedder)]
|
|
|
|
|
|
@router.post("", status_code=status.HTTP_201_CREATED)
|
|
async def upload_file(
|
|
response: Response,
|
|
file: UploadFile,
|
|
domain: Annotated[str, Form()],
|
|
auth: _RequireFilesWrite,
|
|
sessionmaker: _SessionmakerDep,
|
|
storage: _ObjectStorageDep,
|
|
settings: _SettingsDep,
|
|
limiter: _IngestionLimiterDep,
|
|
concurrency_limiter: _ConcurrencyLimiterDep,
|
|
dense_embedders: _DenseEmbeddersDep,
|
|
sparse_embedder: _SparseEmbedderDep,
|
|
) -> FileUploadResponse:
|
|
data = await file.read()
|
|
result = await upload_source_file(
|
|
sessionmaker=sessionmaker,
|
|
storage=storage,
|
|
auth=auth,
|
|
domain=domain,
|
|
filename=file.filename or "",
|
|
data=data,
|
|
ingestion_settings=settings.ingestion,
|
|
chunking_settings=settings.chunking,
|
|
thread_limiter=limiter,
|
|
concurrency_limiter=concurrency_limiter,
|
|
dense_embedders=dense_embedders,
|
|
sparse_embedder=sparse_embedder,
|
|
)
|
|
if not result.is_new_attempt:
|
|
response.status_code = status.HTTP_200_OK
|
|
return FileUploadResponse.from_result(result)
|
|
|
|
|
|
@router.get("/{file_id}")
|
|
async def get_file(
|
|
file_id: uuid.UUID,
|
|
auth: _RequireFilesWrite,
|
|
sessionmaker: _SessionmakerDep,
|
|
) -> FileStatusResponse:
|
|
result = await get_file_status(sessionmaker, tenant_id=auth.tenant_id, source_file_id=file_id)
|
|
if result is None:
|
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="file not found")
|
|
return FileStatusResponse.from_result(result)
|