azk/chatbot_v3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ingestion): add bounded, benchmark-aligned embedding execution

Browse Source 
Why:
- Plan 001 Phase 4 needs batched, concurrency-bounded embedding wired into
  the inline upload path, with process-wide capacity/timeout/chunk-limit
  guards (ADR-0017).
- The BM25 analyzer and dense-model config are ported from the `emet`
  evaluation lab, which benchmarked them against the real Farsi corpus
  (bm25-fa-norm-stop; nomic-embed-text-v2-moe at 768-dim; text-embedding-3-large
  at native 3072-dim), closing open items in ADR-0001/ADR-0005.

Changes:
- New: embedding ports, orchestration (embed_chunks), request-bounds
  helpers, and dense/sparse adapters (analyzers.py, bm25.py,
  openai_compatible.py).
- upload.py now parses/chunks/embeds inline behind INGESTION_MAX_CONCURRENCY
  (503), INGESTION_TIMEOUT_SECONDS (504), and the chunk-count ceiling (413);
  every failure path still writes a terminal job row.
- Lifespan builds and warms both dense embedders at startup (fail-soft) and
  creates the sparse embedder and concurrency semaphore.
- httpx moves from dev to main dependencies (adapters use it directly).

Impact:
- Qdrant point upserts are still Phase 5 -- chunks_indexed stays 0.
- New EMBEDDING_* env vars documented in .env.example; safe defaults.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Ali Zarin Kolah
2026-08-19 17:13:32 +03:30
parent aa6d595424
commit 5c0a5938f8
33 changed files with 2455 additions and 536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
src/application/files/upload.py
View File

@@ -5,21 +5,27 @@ one request-scoped session, because the request is two units of work
(ADR-0012, ADR-0017):
txn A (short): source_files [+ ingestion_jobs(status='running')], commit
no txn: store bytes in MinIO
no txn: store bytes in MinIO, parse/chunk (threads),
embed dense+sparse (bounded/batched)
txn B (short): ingestion_jobs -> succeeded/failed, append event, commit
No Postgres session is open during the MinIO write. A storage failure between
No Postgres session is open during phase 2. A failure at any point between
txn A and txn B still leaves a durable, inspectable `failed` job — never a
job stuck in `running`.
job stuck in `running`. The whole request additionally holds one of
`INGESTION_MAX_CONCURRENCY` process-wide slots (`503` when exhausted) and
phase 2 is bounded by `INGESTION_TIMEOUT_SECONDS` (`504`) (ADR-0017, plan 001
Phase 4).
Parsing/chunking/Qdrant indexing are Phase 4/5 work, not implemented here:
this phase stores bytes only, so a successful job reports `chunks_indexed=0`.
Qdrant point upserts are Phase 5 work, not implemented here: this phase
parses, chunks, and embeds, so a successful job still reports
`chunks_indexed=0` — nothing is searchable yet.
"""
import uuid
from collections.abc import Sequence
import structlog
from anyio import CapacityLimiter, to_thread
from anyio import CapacityLimiter, Semaphore, fail_after, to_thread
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from src.application.auth.context import AuthContext
@@ -27,7 +33,22 @@ from src.application.files.errors import InvalidUploadError
from src.application.files.models import UploadResult
from src.application.files.storage_keys import source_file_object_key
from src.application.files.validation import validate_and_hash_upload
from src.application.ingestion import (
ChunkTooLargeError,
DocumentParseError,
UnsupportedSourceTypeError,
parse_and_chunk_document,
)
from src.application.ingestion.bounds import acquire_ingestion_slot, enforce_chunk_limit
from src.application.ingestion.embedding import embed_chunks
from src.application.ingestion.errors import (
ChunkLimitExceededError,
EmbedderError,
IngestionTimeoutError,
)
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
from src.application.ports.object_storage import ObjectStorage
from src.config import ChunkingSettings, IngestionSettings
from src.infrastructure.postgres.repositories import ingestion_jobs as jobs_repo
from src.infrastructure.postgres.repositories import source_files as source_files_repo
@@ -71,9 +92,12 @@ async def upload_source_file(
domain: str,
filename: str,
data: bytes,
max_upload_size_bytes: int,
chunking_strategy: str,
validation_limiter: CapacityLimiter,
ingestion_settings: IngestionSettings,
chunking_settings: ChunkingSettings,
thread_limiter: CapacityLimiter,
concurrency_limiter: Semaphore,
dense_embedders: Sequence[DenseEmbedder],
sparse_embedder: SparseEmbedder,
) -> UploadResult:
domain = domain.strip()
if not domain:
@@ -81,114 +105,186 @@ async def upload_source_file(
validated = await to_thread.run_sync(
lambda: validate_and_hash_upload(
filename=filename, data=data, max_size_bytes=max_upload_size_bytes
filename=filename, data=data, max_size_bytes=ingestion_settings.max_upload_size_bytes
),
limiter=validation_limiter,
limiter=thread_limiter,
)
async with sessionmaker() as session:
existing = await source_files_repo.find_active_by_content_hash(
session,
tenant_id=auth.tenant_id,
domain=domain,
content_sha256=validated.content_sha256,
)
if existing is not None:
latest_job = await jobs_repo.get_latest_for_source_file(
session, tenant_id=auth.tenant_id, source_file_id=existing.id
)
if latest_job is not None and latest_job.status == "succeeded":
logger.info(
"files.upload.duplicate",
tenant_id=str(auth.tenant_id),
file_id=str(existing.id),
)
return UploadResult(
file_id=existing.id,
ingestion_job_id=latest_job.id,
status=latest_job.status,
chunks_indexed=latest_job.points_created,
is_new_attempt=False,
)
source_file_id = existing.id
object_key = existing.storage_uri or source_file_object_key(
auth.tenant_id, source_file_id
)
else:
source_file_id = uuid.uuid4()
object_key = source_file_object_key(auth.tenant_id, source_file_id)
source_files_repo.create(
async with acquire_ingestion_slot(concurrency_limiter):
async with sessionmaker() as session:
existing = await source_files_repo.find_active_by_content_hash(
session,
source_file_id=source_file_id,
tenant_id=auth.tenant_id,
domain=domain,
source_filename=filename,
source_type=validated.source_type,
content_sha256=validated.content_sha256,
byte_size=len(data),
storage_uri=object_key,
created_by_api_key_id=auth.api_key_id,
)
# `ingestion_jobs.source_file_id` FKs to this row; flush so the
# insert below sees it, since the two mapped classes carry no
# ORM relationship for the unit of work to order by itself.
await session.flush()
job = jobs_repo.create_running(
session,
tenant_id=auth.tenant_id,
source_file_id=source_file_id,
requested_by_api_key_id=auth.api_key_id,
chunking_strategy=chunking_strategy,
)
jobs_repo.append_event(
session,
tenant_id=auth.tenant_id,
ingestion_job_id=job.id,
level="info",
stage="received",
message="upload accepted, storing object",
)
await session.commit()
ingestion_job_id = job.id
if existing is not None:
latest_job = await jobs_repo.get_latest_for_source_file(
session, tenant_id=auth.tenant_id, source_file_id=existing.id
)
if latest_job is not None and latest_job.status == "succeeded":
logger.info(
"files.upload.duplicate",
tenant_id=str(auth.tenant_id),
file_id=str(existing.id),
)
return UploadResult(
file_id=existing.id,
ingestion_job_id=latest_job.id,
status=latest_job.status,
chunks_indexed=latest_job.points_created,
is_new_attempt=False,
)
source_file_id = existing.id
object_key = existing.storage_uri or source_file_object_key(
auth.tenant_id, source_file_id
)
else:
source_file_id = uuid.uuid4()
object_key = source_file_object_key(auth.tenant_id, source_file_id)
source_files_repo.create(
session,
source_file_id=source_file_id,
tenant_id=auth.tenant_id,
domain=domain,
source_filename=filename,
source_type=validated.source_type,
content_sha256=validated.content_sha256,
byte_size=len(data),
storage_uri=object_key,
created_by_api_key_id=auth.api_key_id,
)
# `ingestion_jobs.source_file_id` FKs to this row; flush so the
# insert below sees it, since the two mapped classes carry no
# ORM relationship for the unit of work to order by itself.
await session.flush()
# Phase 2: no Postgres session open across this work (ADR-0017).
try:
await storage.put_object(key=object_key, data=data, content_type=validated.content_type)
except Exception as exc:
logger.warning(
"files.upload.storage_failed",
tenant_id=str(auth.tenant_id),
file_id=str(source_file_id),
ingestion_job_id=str(ingestion_job_id),
)
await _mark_job_failed(
sessionmaker,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
error_code="storage_upload_failed",
error_message=f"failed to store object: {exc}",
)
raise
job = jobs_repo.create_running(
session,
tenant_id=auth.tenant_id,
source_file_id=source_file_id,
requested_by_api_key_id=auth.api_key_id,
chunking_strategy=chunking_settings.strategy,
)
jobs_repo.append_event(
session,
tenant_id=auth.tenant_id,
ingestion_job_id=job.id,
level="info",
stage="received",
message="upload accepted, storing object",
)
await session.commit()
ingestion_job_id = job.id
async with sessionmaker() as session:
await jobs_repo.mark_terminal(
session,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
status="succeeded",
points_created=0,
)
jobs_repo.append_event(
session,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
level="info",
stage="completed",
message="object stored; parsing/embedding/indexing not yet implemented",
)
await session.commit()
# Phase 2: no Postgres session open across this work (ADR-0017),
# bounded end-to-end by INGESTION_TIMEOUT_SECONDS.
try:
with fail_after(ingestion_settings.timeout_seconds):
try:
await storage.put_object(
key=object_key, data=data, content_type=validated.content_type
)
except Exception as exc:
logger.warning(
"files.upload.storage_failed",
tenant_id=str(auth.tenant_id),
file_id=str(source_file_id),
ingestion_job_id=str(ingestion_job_id),
)
await _mark_job_failed(
sessionmaker,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
error_code="storage_upload_failed",
error_message=f"failed to store object: {exc}",
)
raise
try:
chunks = await parse_and_chunk_document(
data,
source_type=validated.source_type,
file_id=source_file_id,
settings=chunking_settings,
limiter=thread_limiter,
)
except (DocumentParseError, UnsupportedSourceTypeError, ChunkTooLargeError) as exc:
await _mark_job_failed(
sessionmaker,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
error_code="parse_failed",
error_message=str(exc),
)
raise
try:
enforce_chunk_limit(chunks, max_chunks=ingestion_settings.max_chunks_per_file)
except ChunkLimitExceededError as exc:
await _mark_job_failed(
sessionmaker,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
error_code="chunk_limit_exceeded",
error_message=str(exc),
)
raise
try:
embedded = await embed_chunks(
chunks,
dense_embedders=dense_embedders,
sparse_embedder=sparse_embedder,
settings=ingestion_settings,
thread_limiter=thread_limiter,
)
except EmbedderError as exc:
await _mark_job_failed(
sessionmaker,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
error_code="embedding_failed",
error_message=str(exc),
)
raise
except TimeoutError:
logger.warning(
"files.upload.timeout",
tenant_id=str(auth.tenant_id),
file_id=str(source_file_id),
ingestion_job_id=str(ingestion_job_id),
)
await _mark_job_failed(
sessionmaker,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
error_code="timeout",
error_message=f"ingestion exceeded {ingestion_settings.timeout_seconds}s",
)
raise IngestionTimeoutError(
f"ingestion exceeded {ingestion_settings.timeout_seconds}s"
) from None
async with sessionmaker() as session:
await jobs_repo.mark_terminal(
session,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
status="succeeded",
points_created=0,
)
jobs_repo.append_event(
session,
tenant_id=auth.tenant_id,
ingestion_job_id=ingestion_job_id,
level="info",
stage="completed",
message="chunks parsed and embedded; Qdrant indexing not yet implemented",
details={"chunks_parsed": len(chunks), "chunks_embedded": len(embedded)},
)
await session.commit()
logger.info(
"files.upload.succeeded",