chatbot_v3/tests/e2e/test_ingestion_slice.py

"""The ingestion vertical slice end to end (plan 001, Phase 6).

Every test here asserts a reliability invariant ADR-0016 lists as a reusable
contract, not a happy path: the bound maps to its status code, the failure
still writes a terminal job row, the retry converges, and the tenant filter
holds. Failures additionally assert the ADR-0008 error envelope, because the
status code alone is not the contract a client integrates against.
"""

import asyncio
import uuid

import pytest
from httpx import Response
from qdrant_client import AsyncQdrantClient
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker

from src.infrastructure.postgres.models.ingestion_job import IngestionJob
from tests.e2e.conftest import (
    CORRUPT_DOCX_BYTES,
    Principal,
    StackFactory,
    count_active_points,
    provision_principal,
    upload_payload,
)

pytestmark = [
    pytest.mark.e2e,
    pytest.mark.postgres,
    pytest.mark.minio,
    pytest.mark.qdrant,
    pytest.mark.asyncio(loop_scope="session"),
]


async def _latest_job(session: AsyncSession, *, tenant_id: uuid.UUID) -> IngestionJob:
    """The tenant's most recent job row, read fresh.

    `populate_existing` matters: this session shares a connection with the
    app's sessions, so without it the identity map can answer from a row
    loaded before the request under test committed its update.
    """
    result = await session.execute(
        select(IngestionJob)
        .where(IngestionJob.tenant_id == tenant_id)
        .order_by(IngestionJob.created_at.desc(), IngestionJob.id)
        .execution_options(populate_existing=True)
    )
    return result.scalars().first() or pytest.fail("no ingestion job was written")


def _envelope(response: Response) -> dict[str, object]:
    """The ADR-0008 error body. Asserting the code here, not just the HTTP
    status, is the difference between testing the contract and testing the
    number."""
    error: dict[str, object] = response.json()["error"]
    assert error["request_id"], "every error envelope carries its correlation id"
    return error


async def test_upload_succeeds_and_indexes_points_under_the_tenant_filter(
    make_stack: StackFactory, principal: Principal, qdrant_client: AsyncQdrantClient
) -> None:
    stack = await make_stack()

    response = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )

    assert response.status_code == 201
    body = response.json()
    assert body["status"] == "succeeded"
    assert body["chunks_indexed"] > 0
    indexed = await count_active_points(
        qdrant_client, stack.collection, tenant_id=principal.tenant_id
    )
    assert indexed == body["chunks_indexed"]


async def test_get_file_reports_the_terminal_status_after_upload(
    make_stack: StackFactory, principal: Principal
) -> None:
    stack = await make_stack()
    created = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )

    status = await stack.client.get(
        f"/v1/files/{created.json()['file_id']}", headers=principal.headers
    )

    assert status.status_code == 200
    assert status.json()["ingestion_status"] == "succeeded"
    assert status.json()["domain"] == principal.domain


async def test_upload_duplicate_content_returns_the_existing_job_without_reingesting(
    make_stack: StackFactory, principal: Principal, qdrant_client: AsyncQdrantClient
) -> None:
    stack = await make_stack()
    first = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )
    points_after_first = await count_active_points(
        qdrant_client, stack.collection, tenant_id=principal.tenant_id
    )

    second = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )

    assert first.status_code == 201
    # 200, not 201: nothing was created the second time (ADR-0017 idempotency).
    assert second.status_code == 200
    assert second.json()["file_id"] == first.json()["file_id"]
    assert second.json()["ingestion_job_id"] == first.json()["ingestion_job_id"]
    assert (
        await count_active_points(qdrant_client, stack.collection, tenant_id=principal.tenant_id)
        == points_after_first
    )


async def test_upload_retried_after_a_failed_job_succeeds_without_duplicate_points(
    make_stack: StackFactory,
    principal: Principal,
    db_session: AsyncSession,
    qdrant_client: AsyncQdrantClient,
) -> None:
    """ADR-0017, "Re-running an ingestion stays safe": a failed attempt is
    retried by re-uploading the same bytes, and deterministic point ids make
    the retry converge instead of duplicating.
    """
    stack = await make_stack()
    stack.dense_embedders[0].fail_next = True

    failed = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )
    failed_job = await _latest_job(db_session, tenant_id=principal.tenant_id)
    retried = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )

    assert failed.status_code == 502
    assert failed_job.status == "failed"
    assert failed_job.error_code == "embedding_failed"
    assert retried.status_code == 201
    assert retried.json()["status"] == "succeeded"
    # Same source file, new attempt -- a terminal job is never reused or
    # transitioned back to running.
    assert retried.json()["ingestion_job_id"] != str(failed_job.id)
    indexed = await count_active_points(
        qdrant_client, stack.collection, tenant_id=principal.tenant_id
    )
    assert indexed == retried.json()["chunks_indexed"]


async def test_get_file_for_another_tenants_file_returns_404_not_403(
    make_stack: StackFactory, principal: Principal, db_session: AsyncSession
) -> None:
    """404, never 403: a 403 would confirm the file exists to a stranger."""
    stack = await make_stack()
    created = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )
    other = await provision_principal(db_session)

    response = await stack.client.get(
        f"/v1/files/{created.json()['file_id']}", headers=other.headers
    )

    assert response.status_code == 404
    assert _envelope(response)["code"] == "not_found"


async def test_upload_to_an_unregistered_domain_is_rejected_before_any_job_is_written(
    make_stack: StackFactory, principal: Principal, db_session: AsyncSession
) -> None:
    stack = await make_stack()

    response = await stack.client.post(
        "/v1/files", **upload_payload(domain="never-registered"), headers=principal.headers
    )

    assert response.status_code == 400
    assert _envelope(response)["code"] == "unknown_domain"
    jobs = await db_session.execute(
        select(IngestionJob).where(IngestionJob.tenant_id == principal.tenant_id)
    )
    assert jobs.scalars().all() == []


async def test_upload_at_capacity_is_rejected_with_503_and_retry_after(
    make_stack: StackFactory, committed_sessionmaker: async_sessionmaker[AsyncSession]
) -> None:
    """ADR-0017 rejects rather than queues, so the second concurrent upload
    must fail fast with a backoff hint instead of waiting for a slot.

    This is the one test that needs two requests genuinely in flight at once,
    so it runs on `committed_sessionmaker` -- see that fixture for why the
    shared-connection default cannot serve concurrent sessions.
    """
    async with committed_sessionmaker() as session:
        principal = await provision_principal(session)
    stack = await make_stack(
        ingestion={"max_concurrency": 1},
        dense_delay_seconds=0.4,
        sessionmaker=committed_sessionmaker,
    )

    first, second = await asyncio.gather(
        stack.client.post(
            "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
        ),
        stack.client.post(
            "/v1/files",
            **upload_payload(domain=principal.domain, filename="other.csv", data=b"a,b\n1,2\n"),
            headers=principal.headers,
        ),
    )

    statuses = sorted([first.status_code, second.status_code])
    assert statuses == [201, 503]
    rejected = first if first.status_code == 503 else second
    assert _envelope(rejected)["code"] == "ingestion_at_capacity"
    assert rejected.headers["Retry-After"] == "1"


async def test_upload_exceeding_the_timeout_returns_504_and_writes_a_terminal_job(
    make_stack: StackFactory, principal: Principal, db_session: AsyncSession
) -> None:
    """A timeout must never leave a job stuck in `running` (ADR-0016)."""
    stack = await make_stack(ingestion={"timeout_seconds": 0.05}, dense_delay_seconds=0.5)

    response = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )

    assert response.status_code == 504
    assert _envelope(response)["code"] == "ingestion_timeout"
    job = await _latest_job(db_session, tenant_id=principal.tenant_id)
    assert job.status == "failed"
    assert job.error_code == "timeout"


async def test_upload_of_an_unparseable_document_returns_400_and_a_failed_job(
    make_stack: StackFactory, principal: Principal, db_session: AsyncSession
) -> None:
    stack = await make_stack()

    response = await stack.client.post(
        "/v1/files",
        **upload_payload(domain=principal.domain, filename="broken.docx", data=CORRUPT_DOCX_BYTES),
        headers=principal.headers,
    )

    assert response.status_code == 400
    assert _envelope(response)["code"] == "validation_error"
    job = await _latest_job(db_session, tenant_id=principal.tenant_id)
    assert job.status == "failed"
    assert job.error_code == "parse_failed"


async def test_upload_returns_502_when_qdrant_indexing_fails(
    make_stack: StackFactory, principal: Principal, db_session: AsyncSession
) -> None:
    """A real Qdrant error, not a fake: the collection the deployment step
    should have created is missing, which is the failure an operator actually
    hits when `qdrant_bootstrap` was skipped.
    """
    stack = await make_stack(
        collection=f"never_bootstrapped_{uuid.uuid4().hex}", bootstrap_collection=False
    )

    response = await stack.client.post(
        "/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
    )

    assert response.status_code == 502
    assert _envelope(response)["code"] == "index_error"
    job = await _latest_job(db_session, tenant_id=principal.tenant_id)
    assert job.status == "failed"
    assert job.error_code == "index_failed"


async def test_upload_without_the_files_write_scope_is_forbidden(
    make_stack: StackFactory, db_session: AsyncSession
) -> None:
    stack = await make_stack()
    reader = await provision_principal(db_session, scopes=["domains:read"])

    response = await stack.client.post(
        "/v1/files", **upload_payload(domain=reader.domain), headers=reader.headers
    )

    assert response.status_code == 403
    assert _envelope(response)["code"] == "missing_scope"


async def test_readyz_reports_every_dependency_ready_against_real_services(
    make_stack: StackFactory,
) -> None:
    """`/readyz` is dependency readiness, `/healthz` is process health."""
    stack = await make_stack()

    ready = await stack.client.get("/readyz")
    healthy = await stack.client.get("/healthz")

    assert healthy.status_code == 200
    assert ready.status_code == 200
    assert ready.json() == {"postgres": True, "minio": True, "qdrant": True}


async def test_readyz_is_unready_when_the_chunks_collection_is_missing(
    make_stack: StackFactory,
) -> None:
    """A reachable but unbootstrapped Qdrant is deliberately not ready --
    uploads to it would 502 (see the indexing test above).
    """
    stack = await make_stack(
        collection=f"never_bootstrapped_{uuid.uuid4().hex}", bootstrap_collection=False
    )

    response = await stack.client.get("/readyz")

    assert response.status_code == 503
    assert response.json()["qdrant"] is False