diff --git a/scripts/smoke.sh b/scripts/smoke.sh new file mode 100755 index 0000000..aad6418 --- /dev/null +++ b/scripts/smoke.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Serialized operational smoke test of the running web process (ADR-0016, plan +# 001 Phase 6). +# +# ./scripts/smoke.sh +# +# Brings up the Compose stack, runs both deployment steps for real, provisions +# a tenant, starts uvicorn, and drives `tests/e2e/test_compose_smoke.py` +# against it over a socket. This is the only Compose-based test: every other +# test uses Testcontainers and an in-process ASGI transport, which is exactly +# what makes this one worth having -- it is the only thing that exercises the +# deployment steps, the real logging configuration, and a real HTTP server. +# +# Not part of `uv run pytest`: the smoke test skips itself unless SMOKE_BASE_URL +# is set, so this script is the only way it runs. Run it before a release. +# +# Leaves the Compose stack running (it is the local dev stack); only the uvicorn +# process and the temporary log file are cleaned up. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +PORT="${SMOKE_PORT:-8021}" +SLUG="smoke-$(date +%s)" +DOMAIN="smoke" +LOG_FILE="$(mktemp -t smoke-app-log.XXXXXX.jsonl)" +CONSOLE_LOG="$(mktemp -t smoke-app-console.XXXXXX.log)" +APP_PID="" + +cleanup() { + if [[ -n "${APP_PID}" ]] && kill -0 "${APP_PID}" 2>/dev/null; then + kill "${APP_PID}" 2>/dev/null || true + wait "${APP_PID}" 2>/dev/null || true + fi + rm -f "${LOG_FILE}" "${CONSOLE_LOG}" +} +trap cleanup EXIT + +if [[ ! -f .env ]]; then + echo "no .env found; copy .env.example first (see docs/runbook.md)" >&2 + exit 1 +fi + +echo "==> starting Postgres, MinIO, Qdrant" +docker compose up -d --wait + +echo "==> applying deployment steps" +uv run alembic upgrade head +uv run python -m src.cli.qdrant_bootstrap + +echo "==> provisioning tenant '${SLUG}'" +PROVISION_OUTPUT="$(uv run python -m src.cli.provision_tenant \ + --slug "${SLUG}" --domain "${DOMAIN}" --scopes files:write 2>/dev/null)" +API_KEY="$(printf '%s\n' "${PROVISION_OUTPUT}" | sed -n 's/^api_key=//p')" +if [[ -z "${API_KEY}" ]]; then + echo "provisioning did not return an api_key" >&2 + exit 1 +fi + +echo "==> starting the web process on port ${PORT}" +# JSON to a file sink, because the smoke test asserts the real ADR-0011 log +# output -- the one thing no in-process test can check. +LOG_JSON_FORMAT=true LOG_FILE_PATH="${LOG_FILE}" \ + uv run python -m uvicorn src.main:app --host 127.0.0.1 --port "${PORT}" \ + >"${CONSOLE_LOG}" 2>&1 & +APP_PID=$! + +echo "==> waiting for /readyz" +for _ in $(seq 1 60); do + if curl -fsS "http://127.0.0.1:${PORT}/readyz" >/dev/null 2>&1; then + break + fi + if ! kill -0 "${APP_PID}" 2>/dev/null; then + echo "the web process exited before becoming ready:" >&2 + tail -20 "${CONSOLE_LOG}" >&2 + exit 1 + fi + sleep 1 +done + +if ! curl -fsS "http://127.0.0.1:${PORT}/readyz" >/dev/null 2>&1; then + # Most often an unbootstrapped Qdrant or an unreachable embedder host; the + # runbook's health/readiness section covers reading this. + echo "the web process never became ready:" >&2 + tail -20 "${CONSOLE_LOG}" >&2 + exit 1 +fi + +echo "==> running the smoke test" +SMOKE_BASE_URL="http://127.0.0.1:${PORT}" \ +SMOKE_API_KEY="${API_KEY}" \ +SMOKE_DOMAIN="${DOMAIN}" \ +SMOKE_LOG_PATH="${LOG_FILE}" \ +SMOKE_QDRANT_URL="${QDRANT_URL:-http://127.0.0.1:6343}" \ +SMOKE_QDRANT_COLLECTION="${QDRANT_COLLECTION:-chunks}" \ + uv run python -m pytest tests/e2e/test_compose_smoke.py -q + +echo "==> smoke test passed" diff --git a/tests/e2e/test_compose_smoke.py b/tests/e2e/test_compose_smoke.py new file mode 100644 index 0000000..1fba0a4 --- /dev/null +++ b/tests/e2e/test_compose_smoke.py @@ -0,0 +1,143 @@ +"""Operational smoke test against the *running web process* (ADR-0016). + +Different in kind from every other test in this repo. Everything else drives +the app in-process through `ASGITransport`; this drives a real uvicorn process +over a real socket, backed by the Docker Compose stack, after the two +deployment steps (`alembic upgrade head`, `src.cli.qdrant_bootstrap`) have run +for real. ADR-0016 reserves Compose for exactly this: "manual local validation +and a later, serialized operational smoke test of the running web process", +run "as a serialized pre-release or scheduled gate". + +It is therefore **skipped unless `SMOKE_BASE_URL` is set**, so a plain +`uv run pytest` never touches Docker Compose. Run it through `scripts/smoke.sh`, +which brings the stack up, provisions a tenant, starts the process, and exports +the environment below. + +Because the app runs in its own process, this is also the only place in the +suite where the real `configure_logging()` is in effect -- +`tests/conftest.py::_no_real_logging_configuration` no-ops it everywhere else +to keep global structlog state out of other tests. So the ADR-0011 log sink is +asserted here and nowhere else. +""" + +import json +import os +import uuid +from collections.abc import AsyncIterator +from pathlib import Path + +import pytest +import pytest_asyncio +from httpx import AsyncClient +from qdrant_client import AsyncQdrantClient, models + +BASE_URL = os.environ.get("SMOKE_BASE_URL") +API_KEY = os.environ.get("SMOKE_API_KEY", "") +DOMAIN = os.environ.get("SMOKE_DOMAIN", "smoke") +LOG_PATH = os.environ.get("SMOKE_LOG_PATH", "") +QDRANT_URL = os.environ.get("SMOKE_QDRANT_URL", "http://127.0.0.1:6343") +QDRANT_COLLECTION = os.environ.get("SMOKE_QDRANT_COLLECTION", "chunks") + +pytestmark = [ + pytest.mark.e2e, + pytest.mark.slow, + pytest.mark.asyncio, + pytest.mark.skipif( + not BASE_URL, + reason="SMOKE_BASE_URL is unset; run this through scripts/smoke.sh", + ), + # Well past the global 10s budget: this drives a real process over a + # socket, and the upload does real embedding work end to end. + pytest.mark.timeout(120), +] + +# Unique per run so a re-run against the same persistent Compose volumes is a +# new file rather than an idempotent duplicate-hash hit (ADR-0017). +_CSV = f"question,answer\nsmoke run {uuid.uuid4().hex},indexed\n".encode() + + +@pytest_asyncio.fixture +async def client() -> AsyncIterator[AsyncClient]: + async with AsyncClient(base_url=BASE_URL or "", timeout=120.0) as async_client: + yield async_client + + +async def test_running_process_reports_healthy_and_ready() -> None: + """Readiness must be green *after* the deployment steps, not before -- + an unbootstrapped Qdrant is deliberately unready. + """ + async with AsyncClient(base_url=BASE_URL or "", timeout=30.0) as client: + healthz = await client.get("/healthz") + readyz = await client.get("/readyz") + + assert healthz.status_code == 200 + assert healthz.json() == {"status": "ok"} + assert readyz.status_code == 200, readyz.text + assert readyz.json() == {"postgres": True, "minio": True, "qdrant": True} + + +async def test_upload_through_the_running_process_indexes_retrievable_points( + client: AsyncClient, +) -> None: + """The whole slice against the deployed shape: HTTP in, Qdrant points out.""" + upload = await client.post( + "/v1/files", + files={"file": ("smoke.csv", _CSV, "text/csv")}, + data={"domain": DOMAIN}, + headers={"Authorization": f"Bearer {API_KEY}"}, + ) + assert upload.status_code == 201, upload.text + body = upload.json() + assert body["status"] == "succeeded" + assert body["chunks_indexed"] > 0 + + status = await client.get( + f"/v1/files/{body['file_id']}", headers={"Authorization": f"Bearer {API_KEY}"} + ) + assert status.status_code == 200 + assert status.json()["ingestion_status"] == "succeeded" + + qdrant = AsyncQdrantClient(url=QDRANT_URL) + try: + count = await qdrant.count( + collection_name=QDRANT_COLLECTION, + count_filter=models.Filter( + must=[ + models.FieldCondition( + key="file_id", match=models.MatchValue(value=body["file_id"]) + ), + models.FieldCondition(key="is_active", match=models.MatchValue(value=True)), + ] + ), + exact=True, + ) + finally: + await qdrant.close() + + assert count.count == body["chunks_indexed"] + + +async def test_running_process_writes_structured_ingestion_logs() -> None: + """The real ADR-0011 sink, which only a separate process exercises. + + An upload that succeeds while emitting nothing parseable is an upload no + operator can investigate -- and the runbook's failure procedure is written + against these exact event names. + """ + if not LOG_PATH: + pytest.skip("SMOKE_LOG_PATH is unset; scripts/smoke.sh normally provides it") + + lines = Path(LOG_PATH).read_text("utf-8").splitlines() + events = [json.loads(line) for line in lines if line.startswith("{")] + + by_name = {event.get("event") for event in events} + assert "ingestion.job.started" in by_name + assert "ingestion.job.completed" in by_name + + completed = next(event for event in events if event.get("event") == "ingestion.job.completed") + assert completed["tenant_id"] + assert completed["ingestion_job_id"] + assert completed["points_upserted"] > 0 + # Static process context bound once at startup (ADR-0011). + assert completed["env"] + assert completed["service_version"]