"""The vertical slice against real Postgres, MinIO, and Qdrant (ADR-0016). These are the ADR-0016 end-to-end tests: "a small vertical-slice acceptance contract through the real application composition". They run on Testcontainers, in the default `uv run pytest` run, because ADR-0016 makes Testcontainers "the standard automated integration-test resource mechanism" and reserves Docker Compose for manual validation and the separate serialized smoke test of the *running web process* (`tests/e2e/test_compose_smoke.py`). What is real here: routing, auth, scopes, the domain allowlist, the ADR-0008 error envelope, the two-transaction upload, MinIO object writes, BM25 sparse embedding, and Qdrant upserts into a per-test collection created by the production `ensure_chunks_collection`. What is faked, and only this: the two dense embedders. They are paid/remote network calls, and ADR-0016 forbids routine runs from touching a live provider. Their fakes return the pinned 768/3072 dimensions, because the real collection rejects anything else. """ import uuid from collections.abc import AsyncIterator, Callable, Coroutine from contextlib import AsyncExitStack from dataclasses import dataclass from typing import Any import pytest import pytest_asyncio from asgi_lifespan import LifespanManager from httpx import ASGITransport, AsyncClient from qdrant_client import AsyncQdrantClient, models from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker from src.bootstrap.dependencies import get_dense_embedders, get_sessionmaker from src.config import MinioSettings, PostgresSettings, QdrantSettings, Settings from src.infrastructure.postgres.database import create_sessionmaker from src.infrastructure.qdrant.collection import ( DENSE_NOMIC_DIMENSIONS, DENSE_OPENAI_DIMENSIONS, ensure_chunks_collection, ) from src.main import create_app from tests.fakes import FakeDenseEmbedder from tests.support.factories import create_api_key, create_tenant, create_tenant_domain CSV_BYTES = b"question,answer\nhow do I file a claim,call the branch\nwhat is covered,see policy\n" # Past validation's OOXML magic-byte check, so the failure lands in the DOCX # parser rather than in upload validation -- the branch this exercises. CORRUPT_DOCX_BYTES = b"PK\x03\x04" + b"not really a docx" * 8 @dataclass class E2EStack: """One configured app instance plus the handles a test asserts against.""" client: AsyncClient collection: str dense_embedders: list[FakeDenseEmbedder] StackFactory = Callable[..., Coroutine[Any, Any, E2EStack]] @dataclass(frozen=True) class Principal: """A provisioned tenant with a bearer token and a registered domain. Plain values, not the ORM `Tenant`: tests re-read rows the app committed on a shared connection, and any expiry/refresh of a held ORM instance would lazy-load outside a greenlet context (`MissingGreenlet`). """ tenant_id: uuid.UUID tenant_slug: str token: str domain: str @property def headers(self) -> dict[str, str]: return {"Authorization": f"Bearer {self.token}"} @pytest.fixture def e2e_settings( postgres_settings: PostgresSettings, minio_settings: MinioSettings, qdrant_settings: QdrantSettings, ) -> Settings: """Settings wired to this test's containers. Every nested settings object is passed explicitly. `src/config.py` cascades `.env` into each nested class, so omitting one would let a developer's real `EMBEDDING_NOMIC_BASE_URL` (a colleague's Ollama box) into a routine test run. The embedder URLs below point at a closed port for the same reason: the lifespan warms the *real* embedders before the fakes are substituted, and connection-refused is both instant and free. """ return Settings( postgres=postgres_settings, minio=minio_settings, qdrant=qdrant_settings, embedding={ "nomic": {"base_url": "http://127.0.0.1:1/v1", "timeout_seconds": 0.5}, "openai": {"base_url": "http://127.0.0.1:1/v1", "timeout_seconds": 0.5}, }, app={"readiness_check_timeout_seconds": 2.0}, ) @pytest_asyncio.fixture(loop_scope="session") async def make_stack( e2e_settings: Settings, db_sessionmaker: async_sessionmaker[AsyncSession], qdrant_client: AsyncQdrantClient, ) -> AsyncIterator[StackFactory]: """Build an app under test, optionally with tightened ingestion bounds. A factory rather than a fixture because the capacity and timeout tests need their own `INGESTION_*` values, and those are read at app construction. """ async with AsyncExitStack() as exit_stack: async def _make( *, ingestion: dict[str, object] | None = None, collection: str | None = None, dense_delay_seconds: float = 0.0, bootstrap_collection: bool = True, sessionmaker: async_sessionmaker[AsyncSession] | None = None, ) -> E2EStack: resolved_collection = collection or e2e_settings.qdrant.collection if bootstrap_collection: await ensure_chunks_collection(qdrant_client, collection=resolved_collection) settings = e2e_settings.model_copy( update={ "qdrant": e2e_settings.qdrant.model_copy( update={"collection": resolved_collection} ), "ingestion": e2e_settings.ingestion.model_copy(update=ingestion or {}), } ) dense = [ FakeDenseEmbedder( name="dense_nomic", dimensions=DENSE_NOMIC_DIMENSIONS, value=0.1, delay_seconds=dense_delay_seconds, ), FakeDenseEmbedder( name="dense_openai", dimensions=DENSE_OPENAI_DIMENSIONS, value=0.2, delay_seconds=dense_delay_seconds, ), ] app = create_app(settings) # The session factory is the test's, so factory-created tenants are # visible to the app and every write rolls back at teardown. Object # storage, point storage, and the sparse embedder stay real. resolved_sessionmaker = sessionmaker or db_sessionmaker app.dependency_overrides[get_sessionmaker] = lambda: resolved_sessionmaker app.dependency_overrides[get_dense_embedders] = lambda: dense manager = await exit_stack.enter_async_context(LifespanManager(app)) client = await exit_stack.enter_async_context( AsyncClient( transport=ASGITransport(app=manager.app), base_url="http://test", timeout=30.0, ) ) return E2EStack(client=client, collection=resolved_collection, dense_embedders=dense) yield _make @pytest_asyncio.fixture(loop_scope="session") async def committed_sessionmaker( postgres_engine: AsyncEngine, ) -> async_sessionmaker[AsyncSession]: """A session factory whose writes really commit, one connection each. The default `db_sessionmaker` pins every session to a single connection inside one rolled-back transaction, which is what isolates a test's writes -- but savepoints on a shared connection cannot interleave, so any test that issues genuinely *concurrent* requests deadlocks or fails with `InvalidSavepointSpecificationError`. Those tests use this instead and rely on their uuid-keyed tenant for isolation (ADR-0016 allows either). """ return create_sessionmaker(postgres_engine) @pytest_asyncio.fixture(loop_scope="session") async def principal(db_session: AsyncSession) -> Principal: """A tenant with an upload-scoped key and one registered domain. Registering the domain is not optional: `POST /v1/files` rejects an unregistered one with `400 unknown_domain` before anything else runs (ADR-0009). """ return await provision_principal(db_session) async def provision_principal( session: AsyncSession, *, domain: str = "fire", scopes: list[str] | None = None ) -> Principal: tenant = await create_tenant(session) _, token = await create_api_key(session, tenant=tenant, scopes=scopes or ["files:write"]) await create_tenant_domain(session, tenant=tenant, domain=domain) await session.commit() return Principal(tenant_id=tenant.id, tenant_slug=tenant.slug, token=token, domain=domain) def upload_payload( *, domain: str, filename: str = "faq.csv", data: bytes = CSV_BYTES ) -> dict[str, Any]: return {"files": {"file": (filename, data, "text/csv")}, "data": {"domain": domain}} async def count_active_points( client: AsyncQdrantClient, collection: str, *, tenant_id: uuid.UUID ) -> int: """Active points for one tenant, read through the raw client. `PointStorage` is deliberately write-only (point reads are plan 002's `/v1/points`), so the verification read here uses the SDK directly. """ result = await client.count( collection_name=collection, count_filter=models.Filter( must=[ models.FieldCondition( key="tenant_id", match=models.MatchValue(value=str(tenant_id)) ), models.FieldCondition(key="is_active", match=models.MatchValue(value=True)), ] ), exact=True, ) return result.count