chatbot_v3/tests/e2e/conftest.py

"""The vertical slice against real Postgres, MinIO, and Qdrant (ADR-0016).

These are the ADR-0016 end-to-end tests: "a small vertical-slice acceptance
contract through the real application composition". They run on
Testcontainers, in the default `uv run pytest` run, because ADR-0016 makes
Testcontainers "the standard automated integration-test resource mechanism"
and reserves Docker Compose for manual validation and the separate serialized
smoke test of the *running web process* (`tests/e2e/test_compose_smoke.py`).

What is real here: routing, auth, scopes, the domain allowlist, the ADR-0008
error envelope, the two-transaction upload, MinIO object writes, BM25 sparse
embedding, and Qdrant upserts into a per-test collection created by the
production `ensure_chunks_collection`.

What is faked, and only this: the two dense embedders. They are paid/remote
network calls, and ADR-0016 forbids routine runs from touching a live
provider. Their fakes return the pinned 768/3072 dimensions, because the real
collection rejects anything else.
"""

import uuid
from collections.abc import AsyncIterator, Callable, Coroutine
from contextlib import AsyncExitStack
from dataclasses import dataclass
from typing import Any

import pytest
import pytest_asyncio
from asgi_lifespan import LifespanManager
from httpx import ASGITransport, AsyncClient
from qdrant_client import AsyncQdrantClient, models
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker

from src.bootstrap.dependencies import get_dense_embedders, get_sessionmaker
from src.config import MinioSettings, PostgresSettings, QdrantSettings, Settings
from src.infrastructure.postgres.database import create_sessionmaker
from src.infrastructure.qdrant.collection import (
    DENSE_NOMIC_DIMENSIONS,
    DENSE_OPENAI_DIMENSIONS,
    ensure_chunks_collection,
)
from src.main import create_app
from tests.fakes import FakeDenseEmbedder
from tests.support.factories import create_api_key, create_tenant, create_tenant_domain

CSV_BYTES = b"question,answer\nhow do I file a claim,call the branch\nwhat is covered,see policy\n"

# Past validation's OOXML magic-byte check, so the failure lands in the DOCX
# parser rather than in upload validation -- the branch this exercises.
CORRUPT_DOCX_BYTES = b"PK\x03\x04" + b"not really a docx" * 8


@dataclass
class E2EStack:
    """One configured app instance plus the handles a test asserts against."""

    client: AsyncClient
    collection: str
    dense_embedders: list[FakeDenseEmbedder]


StackFactory = Callable[..., Coroutine[Any, Any, E2EStack]]


@dataclass(frozen=True)
class Principal:
    """A provisioned tenant with a bearer token and a registered domain.

    Plain values, not the ORM `Tenant`: tests re-read rows the app committed on
    a shared connection, and any expiry/refresh of a held ORM instance would
    lazy-load outside a greenlet context (`MissingGreenlet`).
    """

    tenant_id: uuid.UUID
    tenant_slug: str
    token: str
    domain: str

    @property
    def headers(self) -> dict[str, str]:
        return {"Authorization": f"Bearer {self.token}"}


@pytest.fixture
def e2e_settings(
    postgres_settings: PostgresSettings,
    minio_settings: MinioSettings,
    qdrant_settings: QdrantSettings,
) -> Settings:
    """Settings wired to this test's containers.

    Every nested settings object is passed explicitly. `src/config.py` cascades
    `.env` into each nested class, so omitting one would let a developer's real
    `EMBEDDING_NOMIC_BASE_URL` (a colleague's Ollama box) into a routine test
    run. The embedder URLs below point at a closed port for the same reason:
    the lifespan warms the *real* embedders before the fakes are substituted,
    and connection-refused is both instant and free.
    """
    return Settings(
        postgres=postgres_settings,
        minio=minio_settings,
        qdrant=qdrant_settings,
        embedding={
            "nomic": {"base_url": "http://127.0.0.1:1/v1", "timeout_seconds": 0.5},
            "openai": {"base_url": "http://127.0.0.1:1/v1", "timeout_seconds": 0.5},
        },
        app={"readiness_check_timeout_seconds": 2.0},
    )


@pytest_asyncio.fixture(loop_scope="session")
async def make_stack(
    e2e_settings: Settings,
    db_sessionmaker: async_sessionmaker[AsyncSession],
    qdrant_client: AsyncQdrantClient,
) -> AsyncIterator[StackFactory]:
    """Build an app under test, optionally with tightened ingestion bounds.

    A factory rather than a fixture because the capacity and timeout tests need
    their own `INGESTION_*` values, and those are read at app construction.
    """
    async with AsyncExitStack() as exit_stack:

        async def _make(
            *,
            ingestion: dict[str, object] | None = None,
            collection: str | None = None,
            dense_delay_seconds: float = 0.0,
            bootstrap_collection: bool = True,
            sessionmaker: async_sessionmaker[AsyncSession] | None = None,
        ) -> E2EStack:
            resolved_collection = collection or e2e_settings.qdrant.collection
            if bootstrap_collection:
                await ensure_chunks_collection(qdrant_client, collection=resolved_collection)

            settings = e2e_settings.model_copy(
                update={
                    "qdrant": e2e_settings.qdrant.model_copy(
                        update={"collection": resolved_collection}
                    ),
                    "ingestion": e2e_settings.ingestion.model_copy(update=ingestion or {}),
                }
            )

            dense = [
                FakeDenseEmbedder(
                    name="dense_nomic",
                    dimensions=DENSE_NOMIC_DIMENSIONS,
                    value=0.1,
                    delay_seconds=dense_delay_seconds,
                ),
                FakeDenseEmbedder(
                    name="dense_openai",
                    dimensions=DENSE_OPENAI_DIMENSIONS,
                    value=0.2,
                    delay_seconds=dense_delay_seconds,
                ),
            ]

            app = create_app(settings)
            # The session factory is the test's, so factory-created tenants are
            # visible to the app and every write rolls back at teardown. Object
            # storage, point storage, and the sparse embedder stay real.
            resolved_sessionmaker = sessionmaker or db_sessionmaker
            app.dependency_overrides[get_sessionmaker] = lambda: resolved_sessionmaker
            app.dependency_overrides[get_dense_embedders] = lambda: dense

            manager = await exit_stack.enter_async_context(LifespanManager(app))
            client = await exit_stack.enter_async_context(
                AsyncClient(
                    transport=ASGITransport(app=manager.app),
                    base_url="http://test",
                    timeout=30.0,
                )
            )
            return E2EStack(client=client, collection=resolved_collection, dense_embedders=dense)

        yield _make


@pytest_asyncio.fixture(loop_scope="session")
async def committed_sessionmaker(
    postgres_engine: AsyncEngine,
) -> async_sessionmaker[AsyncSession]:
    """A session factory whose writes really commit, one connection each.

    The default `db_sessionmaker` pins every session to a single connection
    inside one rolled-back transaction, which is what isolates a test's writes
    -- but savepoints on a shared connection cannot interleave, so any test
    that issues genuinely *concurrent* requests deadlocks or fails with
    `InvalidSavepointSpecificationError`. Those tests use this instead and rely
    on their uuid-keyed tenant for isolation (ADR-0016 allows either).
    """
    return create_sessionmaker(postgres_engine)


@pytest_asyncio.fixture(loop_scope="session")
async def principal(db_session: AsyncSession) -> Principal:
    """A tenant with an upload-scoped key and one registered domain.

    Registering the domain is not optional: `POST /v1/files` rejects an
    unregistered one with `400 unknown_domain` before anything else runs
    (ADR-0009).
    """
    return await provision_principal(db_session)


async def provision_principal(
    session: AsyncSession, *, domain: str = "fire", scopes: list[str] | None = None
) -> Principal:
    tenant = await create_tenant(session)
    _, token = await create_api_key(session, tenant=tenant, scopes=scopes or ["files:write"])
    await create_tenant_domain(session, tenant=tenant, domain=domain)
    await session.commit()
    return Principal(tenant_id=tenant.id, tenant_slug=tenant.slug, token=token, domain=domain)


def upload_payload(
    *, domain: str, filename: str = "faq.csv", data: bytes = CSV_BYTES
) -> dict[str, Any]:
    return {"files": {"file": (filename, data, "text/csv")}, "data": {"domain": domain}}


async def count_active_points(
    client: AsyncQdrantClient, collection: str, *, tenant_id: uuid.UUID
) -> int:
    """Active points for one tenant, read through the raw client.

    `PointStorage` is deliberately write-only (point reads are plan 002's
    `/v1/points`), so the verification read here uses the SDK directly.
    """
    result = await client.count(
        collection_name=collection,
        count_filter=models.Filter(
            must=[
                models.FieldCondition(
                    key="tenant_id", match=models.MatchValue(value=str(tenant_id))
                ),
                models.FieldCondition(key="is_active", match=models.MatchValue(value=True)),
            ]
        ),
        exact=True,
    )
    return result.count