"""The ingestion vertical slice end to end (plan 001, Phase 6). Every test here asserts a reliability invariant ADR-0016 lists as a reusable contract, not a happy path: the bound maps to its status code, the failure still writes a terminal job row, the retry converges, and the tenant filter holds. Failures additionally assert the ADR-0008 error envelope, because the status code alone is not the contract a client integrates against. """ import asyncio import uuid import pytest from httpx import Response from qdrant_client import AsyncQdrantClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from src.infrastructure.postgres.models.ingestion_job import IngestionJob from tests.e2e.conftest import ( CORRUPT_DOCX_BYTES, Principal, StackFactory, count_active_points, provision_principal, upload_payload, ) pytestmark = [ pytest.mark.e2e, pytest.mark.postgres, pytest.mark.minio, pytest.mark.qdrant, pytest.mark.asyncio(loop_scope="session"), ] async def _latest_job(session: AsyncSession, *, tenant_id: uuid.UUID) -> IngestionJob: """The tenant's most recent job row, read fresh. `populate_existing` matters: this session shares a connection with the app's sessions, so without it the identity map can answer from a row loaded before the request under test committed its update. """ result = await session.execute( select(IngestionJob) .where(IngestionJob.tenant_id == tenant_id) .order_by(IngestionJob.created_at.desc(), IngestionJob.id) .execution_options(populate_existing=True) ) return result.scalars().first() or pytest.fail("no ingestion job was written") def _envelope(response: Response) -> dict[str, object]: """The ADR-0008 error body. Asserting the code here, not just the HTTP status, is the difference between testing the contract and testing the number.""" error: dict[str, object] = response.json()["error"] assert error["request_id"], "every error envelope carries its correlation id" return error async def test_upload_succeeds_and_indexes_points_under_the_tenant_filter( make_stack: StackFactory, principal: Principal, qdrant_client: AsyncQdrantClient ) -> None: stack = await make_stack() response = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) assert response.status_code == 201 body = response.json() assert body["status"] == "succeeded" assert body["chunks_indexed"] > 0 indexed = await count_active_points( qdrant_client, stack.collection, tenant_id=principal.tenant_id ) assert indexed == body["chunks_indexed"] async def test_get_file_reports_the_terminal_status_after_upload( make_stack: StackFactory, principal: Principal ) -> None: stack = await make_stack() created = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) status = await stack.client.get( f"/v1/files/{created.json()['file_id']}", headers=principal.headers ) assert status.status_code == 200 assert status.json()["ingestion_status"] == "succeeded" assert status.json()["domain"] == principal.domain async def test_upload_duplicate_content_returns_the_existing_job_without_reingesting( make_stack: StackFactory, principal: Principal, qdrant_client: AsyncQdrantClient ) -> None: stack = await make_stack() first = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) points_after_first = await count_active_points( qdrant_client, stack.collection, tenant_id=principal.tenant_id ) second = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) assert first.status_code == 201 # 200, not 201: nothing was created the second time (ADR-0017 idempotency). assert second.status_code == 200 assert second.json()["file_id"] == first.json()["file_id"] assert second.json()["ingestion_job_id"] == first.json()["ingestion_job_id"] assert ( await count_active_points(qdrant_client, stack.collection, tenant_id=principal.tenant_id) == points_after_first ) async def test_upload_retried_after_a_failed_job_succeeds_without_duplicate_points( make_stack: StackFactory, principal: Principal, db_session: AsyncSession, qdrant_client: AsyncQdrantClient, ) -> None: """ADR-0017, "Re-running an ingestion stays safe": a failed attempt is retried by re-uploading the same bytes, and deterministic point ids make the retry converge instead of duplicating. """ stack = await make_stack() stack.dense_embedders[0].fail_next = True failed = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) failed_job = await _latest_job(db_session, tenant_id=principal.tenant_id) retried = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) assert failed.status_code == 502 assert failed_job.status == "failed" assert failed_job.error_code == "embedding_failed" assert retried.status_code == 201 assert retried.json()["status"] == "succeeded" # Same source file, new attempt -- a terminal job is never reused or # transitioned back to running. assert retried.json()["ingestion_job_id"] != str(failed_job.id) indexed = await count_active_points( qdrant_client, stack.collection, tenant_id=principal.tenant_id ) assert indexed == retried.json()["chunks_indexed"] async def test_get_file_for_another_tenants_file_returns_404_not_403( make_stack: StackFactory, principal: Principal, db_session: AsyncSession ) -> None: """404, never 403: a 403 would confirm the file exists to a stranger.""" stack = await make_stack() created = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) other = await provision_principal(db_session) response = await stack.client.get( f"/v1/files/{created.json()['file_id']}", headers=other.headers ) assert response.status_code == 404 assert _envelope(response)["code"] == "not_found" async def test_upload_to_an_unregistered_domain_is_rejected_before_any_job_is_written( make_stack: StackFactory, principal: Principal, db_session: AsyncSession ) -> None: stack = await make_stack() response = await stack.client.post( "/v1/files", **upload_payload(domain="never-registered"), headers=principal.headers ) assert response.status_code == 400 assert _envelope(response)["code"] == "unknown_domain" jobs = await db_session.execute( select(IngestionJob).where(IngestionJob.tenant_id == principal.tenant_id) ) assert jobs.scalars().all() == [] async def test_upload_at_capacity_is_rejected_with_503_and_retry_after( make_stack: StackFactory, committed_sessionmaker: async_sessionmaker[AsyncSession] ) -> None: """ADR-0017 rejects rather than queues, so the second concurrent upload must fail fast with a backoff hint instead of waiting for a slot. This is the one test that needs two requests genuinely in flight at once, so it runs on `committed_sessionmaker` -- see that fixture for why the shared-connection default cannot serve concurrent sessions. """ async with committed_sessionmaker() as session: principal = await provision_principal(session) stack = await make_stack( ingestion={"max_concurrency": 1}, dense_delay_seconds=0.4, sessionmaker=committed_sessionmaker, ) first, second = await asyncio.gather( stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ), stack.client.post( "/v1/files", **upload_payload(domain=principal.domain, filename="other.csv", data=b"a,b\n1,2\n"), headers=principal.headers, ), ) statuses = sorted([first.status_code, second.status_code]) assert statuses == [201, 503] rejected = first if first.status_code == 503 else second assert _envelope(rejected)["code"] == "ingestion_at_capacity" assert rejected.headers["Retry-After"] == "1" async def test_upload_exceeding_the_timeout_returns_504_and_writes_a_terminal_job( make_stack: StackFactory, principal: Principal, db_session: AsyncSession ) -> None: """A timeout must never leave a job stuck in `running` (ADR-0016).""" stack = await make_stack(ingestion={"timeout_seconds": 0.05}, dense_delay_seconds=0.5) response = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) assert response.status_code == 504 assert _envelope(response)["code"] == "ingestion_timeout" job = await _latest_job(db_session, tenant_id=principal.tenant_id) assert job.status == "failed" assert job.error_code == "timeout" async def test_upload_of_an_unparseable_document_returns_400_and_a_failed_job( make_stack: StackFactory, principal: Principal, db_session: AsyncSession ) -> None: stack = await make_stack() response = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain, filename="broken.docx", data=CORRUPT_DOCX_BYTES), headers=principal.headers, ) assert response.status_code == 400 assert _envelope(response)["code"] == "validation_error" job = await _latest_job(db_session, tenant_id=principal.tenant_id) assert job.status == "failed" assert job.error_code == "parse_failed" async def test_upload_returns_502_when_qdrant_indexing_fails( make_stack: StackFactory, principal: Principal, db_session: AsyncSession ) -> None: """A real Qdrant error, not a fake: the collection the deployment step should have created is missing, which is the failure an operator actually hits when `qdrant_bootstrap` was skipped. """ stack = await make_stack( collection=f"never_bootstrapped_{uuid.uuid4().hex}", bootstrap_collection=False ) response = await stack.client.post( "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers ) assert response.status_code == 502 assert _envelope(response)["code"] == "index_error" job = await _latest_job(db_session, tenant_id=principal.tenant_id) assert job.status == "failed" assert job.error_code == "index_failed" async def test_upload_without_the_files_write_scope_is_forbidden( make_stack: StackFactory, db_session: AsyncSession ) -> None: stack = await make_stack() reader = await provision_principal(db_session, scopes=["domains:read"]) response = await stack.client.post( "/v1/files", **upload_payload(domain=reader.domain), headers=reader.headers ) assert response.status_code == 403 assert _envelope(response)["code"] == "missing_scope" async def test_readyz_reports_every_dependency_ready_against_real_services( make_stack: StackFactory, ) -> None: """`/readyz` is dependency readiness, `/healthz` is process health.""" stack = await make_stack() ready = await stack.client.get("/readyz") healthy = await stack.client.get("/healthz") assert healthy.status_code == 200 assert ready.status_code == 200 assert ready.json() == {"postgres": True, "minio": True, "qdrant": True} async def test_readyz_is_unready_when_the_chunks_collection_is_missing( make_stack: StackFactory, ) -> None: """A reachable but unbootstrapped Qdrant is deliberately not ready -- uploads to it would 502 (see the indexing test above). """ stack = await make_stack( collection=f"never_bootstrapped_{uuid.uuid4().hex}", bootstrap_collection=False ) response = await stack.client.get("/readyz") assert response.status_code == 503 assert response.json()["qdrant"] is False