cc915f0f1a
Why: - POST /v1/files was reporting chunks_indexed=0/points_created=0 unconditionally — chunks were parsed and embedded but never written to Qdrant, so nothing was actually searchable after upload. Changes: - upload_source_file() now calls index_chunks() after embedding, inside the same INGESTION_TIMEOUT_SECONDS window, and marks the job failed (error_code=index_failed, 502) if it raises. - Job counters (points_created, points_soft_deleted) and the response's chunks_indexed now reflect the real indexing result instead of a hardcoded zero. - Wired PointStorage through AppResources/lifespan/the files router. Impact: - A successful upload is now searchable in Qdrant by the time 201 returns.
398 lines
14 KiB
Python
398 lines
14 KiB
Python
"""ADR-0017's two-transaction upload shape, exercised against real Postgres.
|
|
|
|
Object storage and the embedders are faked (`tests/fakes.py`) -- they're
|
|
ports, not the thing under test here. `MinioObjectStorage` itself is covered
|
|
in `tests/integration/minio/`.
|
|
"""
|
|
|
|
import pytest
|
|
from anyio import CapacityLimiter, Semaphore
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
|
|
from src.application.auth.context import AuthContext
|
|
from src.application.files.models import UploadResult
|
|
from src.application.files.upload import upload_source_file
|
|
from src.application.ingestion.errors import (
|
|
IngestionAtCapacityError,
|
|
IngestionTimeoutError,
|
|
PointIndexingError,
|
|
)
|
|
from src.application.ports.object_storage import ObjectStorage
|
|
from src.application.ports.point_storage import PointStorage
|
|
from src.config import ChunkingSettings, IngestionSettings, QdrantSettings
|
|
from src.infrastructure.postgres.models.ingestion_job import IngestionJob
|
|
from tests.fakes import (
|
|
FakeDenseEmbedder,
|
|
FakeObjectStorage,
|
|
FakePointStorage,
|
|
FakeSparseEmbedder,
|
|
)
|
|
from tests.support.factories import create_api_key, create_tenant
|
|
|
|
pytestmark = [
|
|
pytest.mark.integration,
|
|
pytest.mark.postgres,
|
|
pytest.mark.asyncio(loop_scope="session"),
|
|
]
|
|
|
|
_CSV_BYTES = b"name,value\nfirst,1\n"
|
|
|
|
|
|
async def _upload(
|
|
*,
|
|
sessionmaker: async_sessionmaker[AsyncSession],
|
|
storage: ObjectStorage,
|
|
auth: AuthContext,
|
|
point_storage: PointStorage | None = None,
|
|
domain: str = "general",
|
|
filename: str = "report.csv",
|
|
data: bytes = _CSV_BYTES,
|
|
) -> UploadResult:
|
|
return await upload_source_file(
|
|
sessionmaker=sessionmaker,
|
|
storage=storage,
|
|
point_storage=point_storage if point_storage is not None else FakePointStorage(),
|
|
auth=auth,
|
|
domain=domain,
|
|
filename=filename,
|
|
data=data,
|
|
ingestion_settings=IngestionSettings(),
|
|
chunking_settings=ChunkingSettings(),
|
|
qdrant_settings=QdrantSettings(),
|
|
thread_limiter=CapacityLimiter(2),
|
|
concurrency_limiter=Semaphore(2),
|
|
dense_embedders=[
|
|
FakeDenseEmbedder(name="dense_nomic"),
|
|
FakeDenseEmbedder(name="dense_openai"),
|
|
],
|
|
sparse_embedder=FakeSparseEmbedder(),
|
|
)
|
|
|
|
|
|
async def _auth_for(db_session: AsyncSession) -> AuthContext:
|
|
tenant = await create_tenant(db_session)
|
|
api_key, _ = await create_api_key(db_session, tenant=tenant)
|
|
await db_session.commit()
|
|
return AuthContext(
|
|
tenant_id=tenant.id,
|
|
tenant_slug=tenant.slug,
|
|
api_key_id=api_key.id,
|
|
scopes=frozenset({"files:write"}),
|
|
actor_type="backend",
|
|
)
|
|
|
|
|
|
async def test_upload_source_file_commits_running_job_before_storage_write(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
"""After phase 1 commits, a *separate* session must already see the
|
|
running job -- proving txn A committed and released before phase 2 work,
|
|
per ADR-0017.
|
|
"""
|
|
auth = await _auth_for(db_session)
|
|
storage = FakeObjectStorage()
|
|
|
|
result = await _upload(sessionmaker=db_sessionmaker, storage=storage, auth=auth)
|
|
|
|
assert result.status == "succeeded"
|
|
assert result.chunks_indexed == 1
|
|
assert result.is_new_attempt
|
|
|
|
async with db_sessionmaker() as verify_session:
|
|
job = await verify_session.get(IngestionJob, result.ingestion_job_id)
|
|
assert job is not None
|
|
assert job.status == "succeeded"
|
|
assert storage.objects # bytes were actually written
|
|
|
|
|
|
async def test_upload_source_file_duplicate_hash_returns_existing_without_reingesting(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
auth = await _auth_for(db_session)
|
|
storage = FakeObjectStorage()
|
|
|
|
first = await _upload(sessionmaker=db_sessionmaker, storage=storage, auth=auth)
|
|
second = await _upload(sessionmaker=db_sessionmaker, storage=storage, auth=auth)
|
|
|
|
assert second.file_id == first.file_id
|
|
assert second.ingestion_job_id == first.ingestion_job_id
|
|
assert not second.is_new_attempt
|
|
assert len(storage.objects) == 1 # no second write
|
|
|
|
|
|
async def test_upload_source_file_retries_after_failed_job(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
"""A duplicate upload whose last attempt failed must get a fresh job,
|
|
not be treated as already-succeeded (ADR-0017: safe to run more than
|
|
once).
|
|
"""
|
|
auth = await _auth_for(db_session)
|
|
failing_storage = FakeObjectStorage(fail_next=True)
|
|
|
|
with pytest.raises(OSError):
|
|
await _upload(sessionmaker=db_sessionmaker, storage=failing_storage, auth=auth)
|
|
|
|
async with db_sessionmaker() as verify_session:
|
|
jobs = (
|
|
(
|
|
await verify_session.execute(
|
|
select(IngestionJob).where(IngestionJob.tenant_id == auth.tenant_id)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(jobs) == 1
|
|
assert jobs[0].status == "failed"
|
|
|
|
retry = await _upload(sessionmaker=db_sessionmaker, storage=FakeObjectStorage(), auth=auth)
|
|
|
|
assert retry.status == "succeeded"
|
|
assert retry.is_new_attempt
|
|
assert retry.file_id == jobs[0].source_file_id
|
|
|
|
|
|
async def test_upload_source_file_is_tenant_isolated_for_identical_content(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
"""Two different tenants uploading byte-identical files must not collide
|
|
on the (tenant_id, domain, content_sha256) idempotency key.
|
|
"""
|
|
auth_a = await _auth_for(db_session)
|
|
auth_b = await _auth_for(db_session)
|
|
|
|
result_a = await _upload(sessionmaker=db_sessionmaker, storage=FakeObjectStorage(), auth=auth_a)
|
|
result_b = await _upload(sessionmaker=db_sessionmaker, storage=FakeObjectStorage(), auth=auth_b)
|
|
|
|
assert result_a.file_id != result_b.file_id
|
|
|
|
|
|
async def test_upload_source_file_timeout_writes_failed_job_and_raises(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
"""A work phase that outruns `INGESTION_TIMEOUT_SECONDS` must still leave
|
|
a terminal `failed` job -- never one stuck in `running` (ADR-0017, plan
|
|
001 Phase 4).
|
|
"""
|
|
auth = await _auth_for(db_session)
|
|
slow_embedder = FakeDenseEmbedder(name="dense_nomic", delay_seconds=10)
|
|
|
|
with pytest.raises(IngestionTimeoutError):
|
|
await upload_source_file(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
point_storage=FakePointStorage(),
|
|
auth=auth,
|
|
domain="general",
|
|
filename="report.csv",
|
|
data=_CSV_BYTES,
|
|
ingestion_settings=IngestionSettings(timeout_seconds=0.05),
|
|
chunking_settings=ChunkingSettings(),
|
|
qdrant_settings=QdrantSettings(),
|
|
thread_limiter=CapacityLimiter(2),
|
|
concurrency_limiter=Semaphore(2),
|
|
dense_embedders=[slow_embedder, FakeDenseEmbedder(name="dense_openai")],
|
|
sparse_embedder=FakeSparseEmbedder(),
|
|
)
|
|
|
|
async with db_sessionmaker() as verify_session:
|
|
jobs = (
|
|
(
|
|
await verify_session.execute(
|
|
select(IngestionJob).where(IngestionJob.tenant_id == auth.tenant_id)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(jobs) == 1
|
|
assert jobs[0].status == "failed"
|
|
assert jobs[0].error_code == "timeout"
|
|
|
|
|
|
async def test_upload_source_file_at_capacity_rejects_before_any_job_row(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
"""`INGESTION_MAX_CONCURRENCY` rejects outright rather than queuing
|
|
(ADR-0017) -- and does so before txn A, so a rejected request leaves no
|
|
job row behind at all.
|
|
"""
|
|
auth = await _auth_for(db_session)
|
|
concurrency_limiter = Semaphore(1)
|
|
concurrency_limiter.acquire_nowait() # simulate the one slot already in use
|
|
|
|
with pytest.raises(IngestionAtCapacityError):
|
|
await upload_source_file(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
point_storage=FakePointStorage(),
|
|
auth=auth,
|
|
domain="general",
|
|
filename="report.csv",
|
|
data=_CSV_BYTES,
|
|
ingestion_settings=IngestionSettings(),
|
|
chunking_settings=ChunkingSettings(),
|
|
qdrant_settings=QdrantSettings(),
|
|
thread_limiter=CapacityLimiter(2),
|
|
concurrency_limiter=concurrency_limiter,
|
|
dense_embedders=[
|
|
FakeDenseEmbedder(name="dense_nomic"),
|
|
FakeDenseEmbedder(name="dense_openai"),
|
|
],
|
|
sparse_embedder=FakeSparseEmbedder(),
|
|
)
|
|
|
|
async with db_sessionmaker() as verify_session:
|
|
jobs = (
|
|
(
|
|
await verify_session.execute(
|
|
select(IngestionJob).where(IngestionJob.tenant_id == auth.tenant_id)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert jobs == []
|
|
|
|
|
|
async def test_upload_source_file_indexes_points_and_records_real_counters(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
"""The Phase 5 payoff: a successful upload is searchable, and the job row
|
|
says how many points it wrote rather than a hardcoded zero.
|
|
"""
|
|
auth = await _auth_for(db_session)
|
|
point_storage = FakePointStorage()
|
|
|
|
result = await _upload(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
auth=auth,
|
|
point_storage=point_storage,
|
|
data=b"name,value\nfirst,1\nsecond,2\nthird,3\n",
|
|
)
|
|
|
|
assert result.status == "succeeded"
|
|
assert result.chunks_indexed == len(point_storage.points)
|
|
assert result.chunks_indexed > 0
|
|
|
|
async with db_sessionmaker() as verify_session:
|
|
job = await verify_session.get(IngestionJob, result.ingestion_job_id)
|
|
assert job is not None
|
|
assert job.points_created == result.chunks_indexed
|
|
# An upsert cannot tell an insert from an overwrite, so this stays 0.
|
|
assert job.points_updated == 0
|
|
|
|
|
|
async def test_upload_source_file_indexes_points_under_the_authenticated_tenant(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
auth = await _auth_for(db_session)
|
|
point_storage = FakePointStorage()
|
|
|
|
result = await _upload(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
auth=auth,
|
|
point_storage=point_storage,
|
|
domain="fire",
|
|
)
|
|
|
|
payloads = [point.payload for point in point_storage.points.values()]
|
|
assert payloads
|
|
for payload in payloads:
|
|
assert payload["tenant_id"] == str(auth.tenant_id)
|
|
assert payload["domain"] == "fire"
|
|
assert payload["file_id"] == str(result.file_id)
|
|
assert payload["created_by"] == f"api_key:{auth.api_key_id}"
|
|
|
|
|
|
async def test_upload_source_file_index_failure_writes_terminal_failed_job(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
auth = await _auth_for(db_session)
|
|
point_storage = FakePointStorage(fail_on_batch=0)
|
|
|
|
with pytest.raises(PointIndexingError):
|
|
await _upload(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
auth=auth,
|
|
point_storage=point_storage,
|
|
)
|
|
|
|
async with db_sessionmaker() as verify_session:
|
|
jobs = (await verify_session.execute(select(IngestionJob))).scalars().all()
|
|
job = next(job for job in jobs if job.tenant_id == auth.tenant_id)
|
|
assert job.status == "failed"
|
|
assert job.error_code == "index_failed"
|
|
assert job.completed_at is not None
|
|
|
|
|
|
async def test_upload_source_file_failed_index_does_not_soft_delete_existing_points(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
"""A failed attempt must never remove content from a working index."""
|
|
auth = await _auth_for(db_session)
|
|
point_storage = FakePointStorage()
|
|
await _upload(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
auth=auth,
|
|
point_storage=point_storage,
|
|
data=b"name,value\nfirst,1\nsecond,2\n",
|
|
)
|
|
# fail_on_batch indexes into upsert_batches, which accumulates across
|
|
# uploads -- reset it so "batch 0" means the retry's first batch.
|
|
point_storage.deactivate_calls.clear()
|
|
point_storage.upsert_batches.clear()
|
|
point_storage.fail_on_batch = 0
|
|
|
|
with pytest.raises(PointIndexingError):
|
|
await _upload(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
auth=auth,
|
|
point_storage=point_storage,
|
|
data=b"name,value\nonly,1\n",
|
|
)
|
|
|
|
assert point_storage.deactivate_calls == []
|
|
assert all(point.payload["is_active"] is True for point in point_storage.points.values())
|
|
|
|
|
|
async def test_upload_source_file_retry_after_index_failure_produces_no_duplicate_points(
|
|
db_session: AsyncSession, db_sessionmaker: async_sessionmaker[AsyncSession]
|
|
) -> None:
|
|
auth = await _auth_for(db_session)
|
|
point_storage = FakePointStorage(fail_on_batch=0)
|
|
|
|
with pytest.raises(PointIndexingError):
|
|
await _upload(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
auth=auth,
|
|
point_storage=point_storage,
|
|
)
|
|
|
|
point_storage.fail_on_batch = None
|
|
retry = await _upload(
|
|
sessionmaker=db_sessionmaker,
|
|
storage=FakeObjectStorage(),
|
|
auth=auth,
|
|
point_storage=point_storage,
|
|
)
|
|
|
|
assert retry.status == "succeeded"
|
|
assert len(point_storage.points) == retry.chunks_indexed
|
|
|
|
async with db_sessionmaker() as verify_session:
|
|
jobs = (await verify_session.execute(select(IngestionJob))).scalars().all()
|
|
tenant_jobs = [job for job in jobs if job.tenant_id == auth.tenant_id]
|
|
# A terminal job never returns to `running` (ADR-0017); the retry is a new row.
|
|
assert len(tenant_jobs) == 2
|
|
assert {job.status for job in tenant_jobs} == {"failed", "succeeded"}
|
|
|