Compare commits
42 Commits
5cdfb70085
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 0b1932f716 | |||
| 73bdac0da2 | |||
| b58f4630f3 | |||
| b25c15fefa | |||
| ba7921dd4e | |||
| 802bae429d | |||
| 43932b6562 | |||
| 3b9434faf4 | |||
| 5251990444 | |||
| 062fdd7ac1 | |||
| 923ac8e5d6 | |||
| 4da30f9983 | |||
| 5e935e5895 | |||
| ac3810182d | |||
|
|
3d9269e54f | ||
|
|
7c1fe79f1c | ||
|
|
1b873e5a6f | ||
|
|
133f565704 | ||
|
|
c7a5b69c0a | ||
|
|
2c72688440 | ||
|
|
012b44d5f2 | ||
|
|
ac779dec7e | ||
|
|
9e8987968c | ||
|
|
e9e83b3a26 | ||
|
|
fa933b08ff | ||
|
|
cc915f0f1a | ||
|
|
d00d436e5c | ||
|
|
58ca6109d1 | ||
|
|
5e0addcc55 | ||
|
|
e8fb41af87 | ||
| 9a4b173b95 | |||
| 5c0a5938f8 | |||
| aa6d595424 | |||
| 07b50d6987 | |||
| 9858e27c2d | |||
| e70ad13b10 | |||
| 3bced65926 | |||
| c9cf7b368b | |||
| e97ce6e5f3 | |||
| 3803d9c79a | |||
| 94684d97ae | |||
| 7753651dd6 |
8
.claude/agents/Explore.md
Normal file
8
.claude/agents/Explore.md
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
---
|
||||||
|
name: Explore
|
||||||
|
description: Fast, read-only codebase search
|
||||||
|
model: sonnet
|
||||||
|
effort: low
|
||||||
|
tools: Read, Grep, Glob, Bash, WebFetch, WebSearch
|
||||||
|
maxTurns: 20
|
||||||
|
---
|
||||||
52
.env.example
52
.env.example
@@ -9,12 +9,19 @@
|
|||||||
|
|
||||||
# Application
|
# Application
|
||||||
APP_ENV=local
|
APP_ENV=local
|
||||||
APP_MAX_UPLOAD_SIZE_MB=25
|
|
||||||
APP_READINESS_CHECK_TIMEOUT_SECONDS=2.0
|
APP_READINESS_CHECK_TIMEOUT_SECONDS=2.0
|
||||||
|
# Set by CI/CD at build/deploy time; never computed at runtime.
|
||||||
|
APP_SERVICE_VERSION=dev
|
||||||
|
|
||||||
# Logging
|
# Logging
|
||||||
LOG_LEVEL=INFO
|
LOG_LEVEL=INFO
|
||||||
LOG_JSON_FORMAT=false
|
LOG_JSON_FORMAT=false
|
||||||
|
# Optional second sink, always JSON regardless of LOG_JSON_FORMAT. Local dev
|
||||||
|
# only -- leave unset in production, where stdout/stderr collection is
|
||||||
|
# preferred over an in-container log file.
|
||||||
|
# LOG_FILE_PATH=logs/app.log
|
||||||
|
LOG_FILE_MAX_BYTES=10485760
|
||||||
|
LOG_FILE_BACKUP_COUNT=5
|
||||||
|
|
||||||
# Postgres (application database, separate from Langfuse's Postgres)
|
# Postgres (application database, separate from Langfuse's Postgres)
|
||||||
# Use 127.0.0.1 rather than localhost: some environments resolve localhost to
|
# Use 127.0.0.1 rather than localhost: some environments resolve localhost to
|
||||||
@@ -37,6 +44,7 @@ MINIO_BUCKET=chatbot-source-files
|
|||||||
INGESTION_MAX_CONCURRENCY=4
|
INGESTION_MAX_CONCURRENCY=4
|
||||||
INGESTION_THREAD_POOL_SIZE=8
|
INGESTION_THREAD_POOL_SIZE=8
|
||||||
INGESTION_TIMEOUT_SECONDS=120.0
|
INGESTION_TIMEOUT_SECONDS=120.0
|
||||||
|
INGESTION_MAX_UPLOAD_SIZE_MB=25
|
||||||
INGESTION_MAX_CHUNKS_PER_FILE=5000
|
INGESTION_MAX_CHUNKS_PER_FILE=5000
|
||||||
INGESTION_EMBED_BATCH_SIZE=128
|
INGESTION_EMBED_BATCH_SIZE=128
|
||||||
INGESTION_EMBED_CONCURRENCY=4
|
INGESTION_EMBED_CONCURRENCY=4
|
||||||
@@ -44,6 +52,48 @@ INGESTION_EMBED_CONCURRENCY=4
|
|||||||
# Qdrant
|
# Qdrant
|
||||||
QDRANT_URL=http://127.0.0.1:6343
|
QDRANT_URL=http://127.0.0.1:6343
|
||||||
QDRANT_API_KEY=
|
QDRANT_API_KEY=
|
||||||
|
QDRANT_COLLECTION=chunks
|
||||||
|
QDRANT_UPSERT_BATCH_SIZE=128
|
||||||
|
QDRANT_UPSERT_CONCURRENCY=4
|
||||||
|
|
||||||
|
# Dense embedders (ADR-0001). Both speak an OpenAI-compatible /embeddings
|
||||||
|
# endpoint, so one adapter serves both. Models and endpoints are the ones the
|
||||||
|
# `emet` evaluation lab benchmarked as winners on the Farsi corpus.
|
||||||
|
#
|
||||||
|
# dense_nomic runs behind Ollama's OpenAI-compat shim, which accepts any
|
||||||
|
# non-empty API key. KEEP_ALIVE holds the model resident: a cold load of
|
||||||
|
# nomic-embed-text-v2-moe takes >150s, well past INGESTION_TIMEOUT_SECONDS,
|
||||||
|
# so an idle-then-upload would otherwise 504.
|
||||||
|
EMBEDDING_NOMIC_BASE_URL=http://192.168.10.10:11435/v1
|
||||||
|
EMBEDDING_NOMIC_MODEL=nomic-embed-text-v2-moe
|
||||||
|
EMBEDDING_NOMIC_API_KEY=sk-not-set
|
||||||
|
EMBEDDING_NOMIC_KEEP_ALIVE=30m
|
||||||
|
EMBEDDING_NOMIC_TIMEOUT_SECONDS=30.0
|
||||||
|
# Empty = emet parity. The model card specifies `search_document: ` (ADR-0004),
|
||||||
|
# but the benchmark ran without it and the prefix shifts the vector a lot
|
||||||
|
# (cosine 0.57 on identical text) — so if you set this, the query side must
|
||||||
|
# send `search_query: ` to match, or retrieval gets worse rather than better.
|
||||||
|
EMBEDDING_NOMIC_DOCUMENT_PREFIX=
|
||||||
|
|
||||||
|
# Leave DIMENSIONS empty for text-embedding-3-large's native 3072, which is
|
||||||
|
# what was benchmarked. Setting it truncates via Matryoshka and is a
|
||||||
|
# re-embedding migration, not a config tweak.
|
||||||
|
EMBEDDING_OPENAI_BASE_URL=https://api.openai.com/v1
|
||||||
|
EMBEDDING_OPENAI_MODEL=text-embedding-3-large
|
||||||
|
EMBEDDING_OPENAI_API_KEY=
|
||||||
|
EMBEDDING_OPENAI_DIMENSIONS=
|
||||||
|
EMBEDDING_OPENAI_DOCUMENT_PREFIX=
|
||||||
|
EMBEDDING_OPENAI_TIMEOUT_SECONDS=30.0
|
||||||
|
|
||||||
|
# Sparse BM25 (ADR-0001, ADR-0005): the benchmarked `bm25-fa-norm-stop`.
|
||||||
|
# k/b saturation is applied client-side; IDF comes from Qdrant's
|
||||||
|
# modifier="idf" on the sparse vector field. AVG_LEN is the average document
|
||||||
|
# length in analyzer tokens — emet's placeholder, worth recalibrating from
|
||||||
|
# real corpus statistics.
|
||||||
|
EMBEDDING_SPARSE_ANALYZER=fa_norm_stop
|
||||||
|
EMBEDDING_SPARSE_K=1.2
|
||||||
|
EMBEDDING_SPARSE_B=0.75
|
||||||
|
EMBEDDING_SPARSE_AVG_LEN=256.0
|
||||||
|
|
||||||
# Parsing and chunking (ADR-0018).
|
# Parsing and chunking (ADR-0018).
|
||||||
# max_chunk_tokens is nomic-embed-text-v2-moe's sequence length; text past it
|
# max_chunk_tokens is nomic-embed-text-v2-moe's sequence length; text past it
|
||||||
|
|||||||
186
CLAUDE.md
186
CLAUDE.md
@@ -7,11 +7,127 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
|||||||
This repo is ADR-driven and early in implementation. Working today: the FastAPI
|
This repo is ADR-driven and early in implementation. Working today: the FastAPI
|
||||||
app factory and lifespan wiring (`src/bootstrap/`), `/healthz` and `/readyz`,
|
app factory and lifespan wiring (`src/bootstrap/`), `/healthz` and `/readyz`,
|
||||||
structlog config, Postgres/MinIO/Qdrant clients (`src/infrastructure/`), five
|
structlog config, Postgres/MinIO/Qdrant clients (`src/infrastructure/`), five
|
||||||
SQLAlchemy models with one Alembic migration, and document parsing plus
|
SQLAlchemy models with one Alembic migration, document parsing plus fixed-size
|
||||||
fixed-size chunking (`src/application/ingestion/`). Not built yet: API-key auth,
|
chunking (`src/application/ingestion/`), API-key auth, `POST`/`GET /v1/files`
|
||||||
`POST /v1/files` (so `/v1` currently exposes no routes), repositories, embedding
|
with durable two-phase job creation (`src/application/files/`), and bounded
|
||||||
adapters, Qdrant collection bootstrap, and `src/agent/`. That maps to plan 001
|
inline embedding: `dense_nomic`/`dense_openai` adapters over an
|
||||||
Phase 1 done, Phase 2 partly done, and the parsing half of Phase 5.
|
OpenAI-compatible HTTP client and a `bm25-fa-norm-stop` sparse adapter
|
||||||
|
(`src/infrastructure/embedding/`), wired into the upload path behind
|
||||||
|
`INGESTION_MAX_CONCURRENCY` (`503`), `INGESTION_TIMEOUT_SECONDS` (`504`), and
|
||||||
|
the chunk-count ceiling (`413`). The embedding configuration is **ported from
|
||||||
|
the `emet` evaluation lab** (`~/code/talie/emet`), which benchmarked these
|
||||||
|
models and analyzers on the real Farsi corpus — the analyzer and BM25 weights
|
||||||
|
are verified token-for-token against it, so treat them as a measured artifact
|
||||||
|
and re-benchmark rather than tune them in place (ADR-0005). Also working: the
|
||||||
|
`chunks` collection bootstrap (`src/infrastructure/qdrant/collection.py`, run as
|
||||||
|
a deployment step via `uv run python -m src.cli.qdrant_bootstrap` — never at
|
||||||
|
startup) and tenant-scoped point upserts (`src/application/points/` behind the
|
||||||
|
`PointStorage` port), so an upload is searchable by the time `201` returns.
|
||||||
|
Also working: `tenant_domains` plus `/v1/domains` (`src/application/domains/`),
|
||||||
|
a strict per-tenant allowlist — `POST /v1/files` rejects an unregistered or
|
||||||
|
disabled `domain` with `400` before anything is written, and domain management
|
||||||
|
sits behind its own `domains:read`/`domains:write` scopes, never `files:write`.
|
||||||
|
Also working: the operator runbook (`docs/runbook.md`), tenant/API-key/domain
|
||||||
|
provisioning (`uv run python -m src.cli.provision_tenant` — the third deployment
|
||||||
|
step, since nothing over HTTP can create the first tenant), a Testcontainers
|
||||||
|
e2e suite in the default pytest run (`tests/e2e/test_ingestion_slice.py`:
|
||||||
|
duplicate upload, retry after failure, tenant isolation, capacity, timeout,
|
||||||
|
parse and Qdrant failure), and the one Compose-based test — `scripts/smoke.sh`
|
||||||
|
driving `tests/e2e/test_compose_smoke.py` against a real uvicorn process, which
|
||||||
|
skips itself unless `SMOKE_BASE_URL` is set. That maps to plan 001 Phases 1-6
|
||||||
|
done.
|
||||||
|
|
||||||
|
Plan 002 (`/v1/points` CRUD and keyword search) is **Phases 1-3 done**. Phase 1
|
||||||
|
landed the `PointRepository` port (`src/application/ports/point_repository.py`)
|
||||||
|
with its `Point` read model (`src/application/points/point.py`), the Qdrant adapter
|
||||||
|
(`src/infrastructure/qdrant/point_repository.py`), request/response schemas
|
||||||
|
(`src/api/schemas/points.py`), and lifespan wiring. This port is **separate from
|
||||||
|
`PointStorage`**, which stays exactly the two bulk operations ingestion
|
||||||
|
performs — reads, single-point edits, and keyword search have a different caller
|
||||||
|
and a different tenant-filter obligation, so do not accrete them onto the
|
||||||
|
ingestion port. `tenant_id` is a required keyword argument on every
|
||||||
|
`PointRepository` method by design; keep it that way, because it is what turns a
|
||||||
|
forgotten tenant filter into a type error. The `chunks` collection also gained
|
||||||
|
full-text `content`, `is_active`, and `chunk_index` payload indexes, so a
|
||||||
|
deployed environment needs `qdrant_bootstrap` re-run (indexes are additive — no
|
||||||
|
rebuild, no re-embedding).
|
||||||
|
|
||||||
|
Two adapter mechanics there are load-bearing and easy to "simplify" into bugs:
|
||||||
|
reads go through `scroll` with a `HasIdCondition` rather than `retrieve` (which
|
||||||
|
takes no filter, and would move the tenant check to *after* Qdrant answered),
|
||||||
|
and ordered listing paginates by `order_id` value rather than offset (Qdrant
|
||||||
|
returns no page offset under `order_by`, and an offset cursor skips or repeats
|
||||||
|
rows under a concurrent insert).
|
||||||
|
|
||||||
|
Phase 2 added the **read routes**: `GET /v1/points/{point_id}`,
|
||||||
|
`GET /v1/points?file_id=...`, `GET /v1/points/count`, `GET /v1/points/search`,
|
||||||
|
and `GET /v1/files/{file_id}/points`, over `src/application/points/queries.py`
|
||||||
|
(`src/api/routers/points.py`). All are gated on `points:read`, which — with
|
||||||
|
`points:write` — is now in `DEFAULT_SCOPES`; `GET /v1/files/{file_id}/points`
|
||||||
|
uses `points:read` rather than `files:write`, so the scope follows the data
|
||||||
|
rather than the URL prefix. `PointNotFoundError` maps to `404` in
|
||||||
|
`src/api/errors.py`, never `403`. Three route-level rules are load-bearing:
|
||||||
|
`/count` and `/search` are declared **before** `/{point_id}` (FastAPI matches in
|
||||||
|
declaration order, so reordering them makes `/v1/points/count` a `422`),
|
||||||
|
`file_id` is **required** on the listing (the cursor is an `order_id` value and
|
||||||
|
`order_id` is unique only within one file), and `search_points` folds the query
|
||||||
|
with `normalize_persian_text` before matching, because ingestion letter-folds
|
||||||
|
content and an unfolded Arabic-keyboard query would return an empty result set
|
||||||
|
silently rather than erroring (ADR-0002).
|
||||||
|
|
||||||
|
Phase 3 added **soft delete**: `DELETE /v1/points/{point_id}` and
|
||||||
|
`DELETE /v1/files/{file_id}`, over `src/application/points/deletion.py` (with
|
||||||
|
the pure relinking primitive in `src/application/points/relinking.py`) and
|
||||||
|
`src/application/files/deletion.py`. Both are gated on `points:write` — the
|
||||||
|
file route included, since the data it destroys is points. Nothing is ever
|
||||||
|
removed from Qdrant.
|
||||||
|
|
||||||
|
Four rules there are load-bearing, and three of them look like complications
|
||||||
|
until the concurrency is taken seriously:
|
||||||
|
|
||||||
|
- `patches_for_removal` computes **what is still missing between the state just
|
||||||
|
read and the desired end state**, not "the patches a delete implies". That is
|
||||||
|
what makes a normal delete, a second delete of an already-inactive point (a
|
||||||
|
no-op success, never `404`), and recovery from a half-applied batch one code
|
||||||
|
path. Rewriting it as a straight-line "deactivate, patch prev, patch next"
|
||||||
|
breaks all three.
|
||||||
|
- Qdrant has no multi-point transaction and reports success for a filtered
|
||||||
|
`set_payload` that matched nothing, so a batch whose second operation loses a
|
||||||
|
version race applies its first anyway. `soft_delete_point` therefore re-plans
|
||||||
|
and re-applies up to three times, verifying by read-back, and only then raises
|
||||||
|
`PointVersionConflictError` (`409`). A single-shot delete would be able to
|
||||||
|
leave a stale pointer, which ADR-0002 calls a defect.
|
||||||
|
- A soft-deleted point **keeps its own** `previous_chunk_id`/`next_chunk_id`;
|
||||||
|
only the surviving neighbours are rewritten. Those pointers are unreachable
|
||||||
|
rather than stale, they are the only record of where the point sat, and the
|
||||||
|
retry re-plans from them. The whole-file sweep follows from the same rule:
|
||||||
|
every point leaves at once, so no survivor can dangle and no pointer is
|
||||||
|
touched at all.
|
||||||
|
- `DELETE /v1/files/{file_id}` marks the `source_files` row `soft_deleted`
|
||||||
|
**after** the point sweep, in its own short transaction (no session is held
|
||||||
|
across the Qdrant work). Order matters: a half-finished sweep leaves the row
|
||||||
|
`active` and a retried `DELETE` finishes it, and retiring the row is what
|
||||||
|
makes a later re-upload of the same bytes re-ingest instead of matching
|
||||||
|
`find_active_by_content_hash` and returning a file whose points are gone.
|
||||||
|
|
||||||
|
Audit rows are still Phase 4/6 work; Phase 3 emits log events only
|
||||||
|
(`points.soft_deleted`, `files.soft_deleted`, `points.relink.neighbour_missing`,
|
||||||
|
and the two `*.conflict` warnings). The completion and conflict events carry
|
||||||
|
ADR-0011's `duration_ms` plus `rounds`, and the pair is what makes them
|
||||||
|
diagnostic: relinking itself is O(1) (that is what the adjacency pointers buy),
|
||||||
|
so a single-point delete costs a fixed ~5 Qdrant round trips and a `rounds`
|
||||||
|
above 1 means contention, not a slow store. The whole-file sweep is the one
|
||||||
|
whose cost scales — two round trips per 100-point page.
|
||||||
|
|
||||||
|
Also worth knowing before touching the points tests: `tests/support/point_contract.py`
|
||||||
|
holds **one** scenario suite run against both `FakePointRepository` (unit) and
|
||||||
|
`QdrantPointRepository` (integration), so new repository behaviour belongs there
|
||||||
|
rather than in one of the two runners — that is what keeps the fake from drifting
|
||||||
|
more permissive than the real store.
|
||||||
|
|
||||||
|
Not built yet: plan 002 Phases 4-6 — create/replace/patch, reorder and batch,
|
||||||
|
the `api_request_logs`/`point_audit_events` tables, and the runbook section on
|
||||||
|
inspecting and repairing a file's pointer chain — and `src/agent/`.
|
||||||
|
|
||||||
Architecture decisions live in `docs/adr/` (18 ADRs plus the 0000 template;
|
Architecture decisions live in `docs/adr/` (18 ADRs plus the 0000 template;
|
||||||
0001–0004 are `Accepted` — 0004 amended by 0018; 0014 is `Superseded by 0017`;
|
0001–0004 are `Accepted` — 0004 amended by 0018; 0014 is `Superseded by 0017`;
|
||||||
@@ -113,6 +229,31 @@ MinIO/Qdrant/SQLAlchemy client-construction code. Use ports only for
|
|||||||
external side effects/persistence — not around pure local functions.
|
external side effects/persistence — not around pure local functions.
|
||||||
(ADR-0015)
|
(ADR-0015)
|
||||||
|
|
||||||
|
### Prefer deep modules over shallow ones
|
||||||
|
|
||||||
|
When a package exposes several small pure functions that a caller must
|
||||||
|
compose correctly every time (right dispatch, right order, right
|
||||||
|
thread/async offload), give it one entry point that owns that composition,
|
||||||
|
and keep the small functions internal — exported only where their own unit
|
||||||
|
tests need them. A shallow interface (one whose surface is nearly as complex
|
||||||
|
as its implementation) pushes a correctness obligation onto every call site;
|
||||||
|
a deep one absorbs it once. Apply the deletion test when unsure: if deleting
|
||||||
|
the wrapper would concentrate the composition logic back into every caller
|
||||||
|
rather than just relocate it, the wrapper is worth having.
|
||||||
|
|
||||||
|
Worked example: `src/application/ingestion/` exposes `parse_and_chunk_document`
|
||||||
|
as its only caller-facing entry point. It dispatches on source type and owns
|
||||||
|
the `anyio.to_thread.run_sync` + `CapacityLimiter` offload ADR-0017 requires;
|
||||||
|
`parse_docx`/`parse_csv`/`parse_xlsx`/`chunk_document` stay in the package,
|
||||||
|
exported mainly for their own tests, not for outside callers to reach for
|
||||||
|
directly. `src/application/points/` follows the same shape: `index_chunks` is
|
||||||
|
the only caller-facing entry point, owning payload construction, batching,
|
||||||
|
the `upsert_concurrency` semaphore, and the ordering rule that the soft-delete
|
||||||
|
sweep runs only after every upsert succeeds; `build_chunk_payload` stays
|
||||||
|
internal. Follow this pattern in `application/` as new packages are added
|
||||||
|
there — `retrieval/`, `threads/` — rather than exposing their internals as the
|
||||||
|
primary surface.
|
||||||
|
|
||||||
### Resource lifetime rules (ADR-0012)
|
### Resource lifetime rules (ADR-0012)
|
||||||
|
|
||||||
- Application-lifetime objects (SQLAlchemy engine/sessionmaker, Qdrant client,
|
- Application-lifetime objects (SQLAlchemy engine/sessionmaker, Qdrant client,
|
||||||
@@ -190,6 +331,11 @@ content_sha256)` idempotency, no terminal job returning to `running`.
|
|||||||
|
|
||||||
### Postgres conventions (ADR-0009)
|
### Postgres conventions (ADR-0009)
|
||||||
|
|
||||||
|
`domain` is never free-form: it must match an `active` `tenant_domains` row for
|
||||||
|
the authenticated tenant (ADR-0009). Domain sets are per-tenant and vary in
|
||||||
|
size. The key itself is immutable — it is denormalized into every Qdrant point
|
||||||
|
payload and into `source_files`, so renaming it is a migration, not an edit.
|
||||||
|
|
||||||
UUID primary keys (app-generated), `timestamptz` for all timestamps,
|
UUID primary keys (app-generated), `timestamptz` for all timestamps,
|
||||||
`Numeric(18, 8)` for money (never floats), `JSONB` for flexible metadata but
|
`Numeric(18, 8)` for money (never floats), `JSONB` for flexible metadata but
|
||||||
typed/indexed columns for query-critical fields, string status columns with
|
typed/indexed columns for query-critical fields, string status columns with
|
||||||
@@ -206,7 +352,24 @@ Postgres remains system of record for tenants, API keys, audit, jobs,
|
|||||||
`graph_runs`, `llm_calls`/`llm_pricing`. Correlate the two via `request_id`,
|
`graph_runs`, `llm_calls`/`llm_pricing`. Correlate the two via `request_id`,
|
||||||
`tenant_id`, `thread_id`, `run_id`. Use `structlog` with stable event names
|
`tenant_id`, `thread_id`, `run_id`. Use `structlog` with stable event names
|
||||||
and structured fields (`logger.info("graph.run.completed", ...)`), not
|
and structured fields (`logger.info("graph.run.completed", ...)`), not
|
||||||
interpolated prose; JSON logs by default in production.
|
interpolated prose; JSON logs by default in production, plus an optional
|
||||||
|
local-only JSON file sink independent of the console renderer (`LOG_FILE_PATH`).
|
||||||
|
|
||||||
|
**Add logging in the same change that adds the code, not as a follow-up.**
|
||||||
|
When you add a new service-level entry point (an `application/` function a
|
||||||
|
route calls directly, an ingestion phase, a mutation) or a new failure branch
|
||||||
|
inside one, add its `logger.*` event in that same diff, using ADR-0011's
|
||||||
|
level/event-naming table. Deferring it means re-deriving the failure modes and
|
||||||
|
field names later from code that no longer has them in working memory — as
|
||||||
|
happened with `src/application/files/upload.py`, where four failure branches
|
||||||
|
(`parse_failed`, `chunk_limit_exceeded`, `embedding_failed`, `index_failed`)
|
||||||
|
shipped with no log event and had to be retrofitted.
|
||||||
|
|
||||||
|
This does not mean logging every function. Pure functions, models, schemas,
|
||||||
|
and repositories (`infrastructure/postgres/repositories/`) stay silent by
|
||||||
|
convention — the caller that turns their result into a business-meaningful
|
||||||
|
outcome (job succeeded, upload rejected, domain disabled) is where the event
|
||||||
|
belongs, not the row-level function underneath it.
|
||||||
|
|
||||||
## Testing (ADR-0016)
|
## Testing (ADR-0016)
|
||||||
|
|
||||||
@@ -220,9 +383,14 @@ interpolated prose; JSON logs by default in production.
|
|||||||
- Test naming: `test_<unit>_<scenario>_<outcome>`, Arrange–Act–Assert.
|
- Test naming: `test_<unit>_<scenario>_<outcome>`, Arrange–Act–Assert.
|
||||||
- Layout mirrors architecture: `tests/unit/{application,agent}`,
|
- Layout mirrors architecture: `tests/unit/{application,agent}`,
|
||||||
`tests/integration/{postgres,minio,qdrant}`, `tests/e2e/`.
|
`tests/integration/{postgres,minio,qdrant}`, `tests/e2e/`.
|
||||||
- Integration tests use **Testcontainers** (never a developer's local
|
- Integration **and e2e** tests use **Testcontainers** (never a developer's
|
||||||
services or Langfuse-owned storage/credentials) — this is the standard
|
local services or Langfuse-owned storage/credentials) — this is the standard
|
||||||
automated mechanism, not Docker Compose. Isolate data per test via unique
|
automated mechanism, not Docker Compose. Compose is reserved for exactly one
|
||||||
|
thing: the serialized operational smoke test of the *running web process*
|
||||||
|
(`scripts/smoke.sh`), which is gated out of `uv run pytest`. Shared container
|
||||||
|
fixtures live in `tests/support/containers.py`, registered from the root
|
||||||
|
`tests/conftest.py` via `pytest_plugins` (a non-root conftest cannot declare
|
||||||
|
it). Isolate data per test via unique
|
||||||
keys/queue/collection names; parallel integration execution is disabled
|
keys/queue/collection names; parallel integration execution is disabled
|
||||||
until fixture isolation is proven safe.
|
until fixture isolation is proven safe.
|
||||||
- Pytest never calls a live/paid model provider in routine runs — that's
|
- Pytest never calls a live/paid model provider in routine runs — that's
|
||||||
|
|||||||
43
README.md
43
README.md
@@ -2,6 +2,49 @@
|
|||||||
|
|
||||||
Architecture decisions live in [`docs/adr`](docs/adr). The first implementation
|
Architecture decisions live in [`docs/adr`](docs/adr). The first implementation
|
||||||
milestone is documented in the [ingestion vertical-slice plan](docs/plans/001-ingestion-vertical-slice.md).
|
milestone is documented in the [ingestion vertical-slice plan](docs/plans/001-ingestion-vertical-slice.md).
|
||||||
|
Day-to-day operation — tuning the ingestion bounds, the proxy timeout
|
||||||
|
requirement, and how to investigate or retry a failed upload — is the
|
||||||
|
[operator runbook](docs/runbook.md).
|
||||||
|
|
||||||
|
## Provisioning the datastores
|
||||||
|
|
||||||
|
Both schema steps run as explicit deployment steps. The application performs no
|
||||||
|
DDL at startup — not for Postgres (ADR-0009) and not for Qdrant (ADR-0001,
|
||||||
|
"Collection provisioning").
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d # Postgres, MinIO, Qdrant
|
||||||
|
uv run alembic upgrade head # Postgres schema
|
||||||
|
uv run python -m src.cli.qdrant_bootstrap # the `chunks` collection
|
||||||
|
uv run fastapi dev src/main.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Nothing over HTTP can create the first tenant — every `/v1` route needs an API
|
||||||
|
key, and a key cannot exist before its tenant. One command issues both, plus any
|
||||||
|
domains, printing the key once (only its hash is stored):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run python -m src.cli.provision_tenant --slug acme --domain fire
|
||||||
|
```
|
||||||
|
|
||||||
|
Before a tenant can upload, its domains must be registered — `POST /v1/files`
|
||||||
|
rejects an unregistered or disabled `domain` with `400`. The calling backend
|
||||||
|
manages them over `/v1/domains` using a key with the `domains:write` scope:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8000/v1/domains \
|
||||||
|
-H "Authorization: Bearer $API_KEY" \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d '{"domain": "fire", "display_name": "Fire insurance"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
Both bootstrap commands are idempotent and safe to re-run. `qdrant_bootstrap` verifies an
|
||||||
|
existing collection against the pinned schema and exits non-zero on a mismatch,
|
||||||
|
rather than leaving a silently degraded sparse index in place.
|
||||||
|
|
||||||
|
`./scripts/smoke.sh` verifies the whole path — Compose up, both deployment
|
||||||
|
steps, provisioning, an upload through the running web process to indexed Qdrant
|
||||||
|
points. See the [runbook](docs/runbook.md#12-verifying-a-deployment).
|
||||||
|
|
||||||
## Local Langfuse
|
## Local Langfuse
|
||||||
|
|
||||||
|
|||||||
48
alembic/versions/41335d162de8_create_tenant_domains.py
Normal file
48
alembic/versions/41335d162de8_create_tenant_domains.py
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
"""create tenant_domains
|
||||||
|
|
||||||
|
Revision ID: 41335d162de8
|
||||||
|
Revises: bfc6c81c2542
|
||||||
|
Create Date: 2026-08-20 17:48:29.443293
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '41335d162de8'
|
||||||
|
down_revision: Union[str, Sequence[str], None] = 'bfc6c81c2542'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Upgrade schema."""
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.create_table('tenant_domains',
|
||||||
|
sa.Column('id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('domain', sa.String(length=80), nullable=False),
|
||||||
|
sa.Column('display_name', sa.String(length=200), nullable=False),
|
||||||
|
sa.Column('status', sa.String(length=20), server_default='active', nullable=False),
|
||||||
|
sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default='{}', nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
|
sa.Column('disabled_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.CheckConstraint("status IN ('active', 'disabled')", name='ck_tenant_domains_status'),
|
||||||
|
sa.ForeignKeyConstraint(['tenant_id'], ['tenants.id'], ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
sa.UniqueConstraint('tenant_id', 'domain', name='uq_tenant_domains_tenant_id_domain')
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_tenant_domains_tenant_id'), 'tenant_domains', ['tenant_id'], unique=False)
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Downgrade schema."""
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.drop_index(op.f('ix_tenant_domains_tenant_id'), table_name='tenant_domains')
|
||||||
|
op.drop_table('tenant_domains')
|
||||||
|
# ### end Alembic commands ###
|
||||||
@@ -49,7 +49,7 @@ One collection, e.g. `chunks`, shared by all tenants and domains.
|
|||||||
| Name | Type | Purpose | Notes |
|
| Name | Type | Purpose | Notes |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `dense_nomic` | dense vector | primary semantic similarity (multilingual, incl. Persian) | `nomic-embed-text-v2-moe`, 768-dim ([0004](0004-docx-csv-chunking-strategy.md)) |
|
| `dense_nomic` | dense vector | primary semantic similarity (multilingual, incl. Persian) | `nomic-embed-text-v2-moe`, 768-dim ([0004](0004-docx-csv-chunking-strategy.md)) |
|
||||||
| `dense_openai` | dense vector | second semantic signal | OpenAI large embedding model (e.g. `text-embedding-3-large`), dimension per OpenAI's `dimensions` param (TBD — full 3072 vs. a truncated size) |
|
| `dense_openai` | dense vector | second semantic signal | `text-embedding-3-large` at its **native 3072 dimensions** — the `dimensions` param is deliberately left unset (see below) |
|
||||||
| `sparse` | sparse vector | lexical/keyword-sensitive retrieval | `bm25-fa-norm-stop` — Qdrant FastEmbed's BM25 sparse encoder configured for Persian (stopword removal + normalization), not a separately trained model |
|
| `sparse` | sparse vector | lexical/keyword-sensitive retrieval | `bm25-fa-norm-stop` — Qdrant FastEmbed's BM25 sparse encoder configured for Persian (stopword removal + normalization), not a separately trained model |
|
||||||
| `late_interaction` | multivector | reserved for late-interaction rerank ([0003](0003-agent-hybrid-retrieval.md)) | `jina-colbert-v2` ([0005](0005-reranking-model-and-sparse-analyzer-selection.md)), `comparator: max_sim`, `hnsw_config: m=0` (rerank-only, never independently ANN-searched), stored **on disk** |
|
| `late_interaction` | multivector | reserved for late-interaction rerank ([0003](0003-agent-hybrid-retrieval.md)) | `jina-colbert-v2` ([0005](0005-reranking-model-and-sparse-analyzer-selection.md)), `comparator: max_sim`, `hnsw_config: m=0` (rerank-only, never independently ANN-searched), stored **on disk** |
|
||||||
|
|
||||||
@@ -61,6 +61,35 @@ dense/sparse query latency. Two dense vectors are provisioned deliberately —
|
|||||||
`dense_nomic` and `dense_openai` are two independent semantic signals, both
|
`dense_nomic` and `dense_openai` are two independent semantic signals, both
|
||||||
prefetched and fused at query time (ADR-0003), not a primary/fallback pair.
|
prefetched and fused at query time (ADR-0003), not a primary/fallback pair.
|
||||||
|
|
||||||
|
#### Dense model endpoints and dimensions (resolved by the `emet` benchmark)
|
||||||
|
|
||||||
|
Both dense models are reached over the **same OpenAI-compatible
|
||||||
|
`/embeddings` API**, so one adapter
|
||||||
|
(`src/infrastructure/embedding/openai_compatible.py`) serves both named
|
||||||
|
vectors with different configuration:
|
||||||
|
|
||||||
|
| Named vector | Model | Endpoint | Dimensions |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `dense_nomic` | `nomic-embed-text-v2-moe` | self-hosted Ollama OpenAI-compat shim | **768** (verified against the live endpoint) |
|
||||||
|
| `dense_openai` | `text-embedding-3-large` | OpenAI hosted API | **3072** (native; `dimensions` unset) |
|
||||||
|
|
||||||
|
`dense_openai`'s dimension was previously listed as an open dependency. It is
|
||||||
|
now pinned to the native 3072, because that is the configuration the `emet`
|
||||||
|
lab benchmarked — it never passed a `dimensions` argument. Setting it later
|
||||||
|
would truncate via Matryoshka and is a **re-embedding migration, not a config
|
||||||
|
tweak**, exactly as the negative consequence below warns.
|
||||||
|
|
||||||
|
Two operational notes about the self-hosted embedder, both learned by
|
||||||
|
measurement rather than assumption:
|
||||||
|
|
||||||
|
- **Cold load exceeds 150s**, far beyond `INGESTION_TIMEOUT_SECONDS`, so an
|
||||||
|
idle-then-upload would return `504`. Mitigated on both ends: Ollama's
|
||||||
|
`keep_alive` keeps the model resident, and the FastAPI lifespan warms each
|
||||||
|
dense embedder at startup (fail-soft — a down embedder must not block boot).
|
||||||
|
- **Once warm it is fast**: ~0.30s for one input and ~0.34s for a batch of 16.
|
||||||
|
Batching is therefore nearly free, which is what keeps ADR-0017's inline
|
||||||
|
ingestion viable.
|
||||||
|
|
||||||
### Multitenancy / indexing config
|
### Multitenancy / indexing config
|
||||||
|
|
||||||
- HNSW: `m: 0` (disable the global index) + `payload_m: 16`, per Qdrant's
|
- HNSW: `m: 0` (disable the global index) + `payload_m: 16`, per Qdrant's
|
||||||
@@ -79,6 +108,38 @@ prefetched and fused at query time (ADR-0003), not a primary/fallback pair.
|
|||||||
- Payload index on `previous_chunk_id` / `next_chunk_id`: keyword index,
|
- Payload index on `previous_chunk_id` / `next_chunk_id`: keyword index,
|
||||||
used for O(1) adjacency retrieval (see below).
|
used for O(1) adjacency retrieval (see below).
|
||||||
|
|
||||||
|
### Collection provisioning
|
||||||
|
|
||||||
|
The collection is created by an explicit **deployment step**, not by application
|
||||||
|
startup and not lazily on first write:
|
||||||
|
|
||||||
|
uv run python -m src.cli.qdrant_bootstrap
|
||||||
|
|
||||||
|
Creating a collection is DDL, and this project already keeps DDL out of the boot
|
||||||
|
and request paths: [0009](0009-postgres-sqlalchemy-alembic-schema.md) requires
|
||||||
|
Alembic for Postgres schema and forbids `create_all()` at startup, and
|
||||||
|
[0012](0012-application-resource-lifetime-and-dependency-ownership.md) makes
|
||||||
|
LangGraph's `.setup()` a deployment step for the same reason. Neither ADR named
|
||||||
|
Qdrant explicitly; this section closes that gap rather than letting the placement
|
||||||
|
be decided by whichever code happened to need it first.
|
||||||
|
|
||||||
|
Doing it in the FastAPI lifespan was rejected: it couples process boot to Qdrant
|
||||||
|
being reachable (which is `/readyz`'s job, not boot's), races across replicas,
|
||||||
|
and turns a misconfigured collection into a silent skip. Doing it lazily on first
|
||||||
|
upsert was rejected for putting DDL on a user request and hiding the
|
||||||
|
misconfiguration until traffic arrives.
|
||||||
|
|
||||||
|
`ensure_chunks_collection` is idempotent and **verifying**: against an existing
|
||||||
|
collection it compares the dense dimensions and the sparse `modifier` to the
|
||||||
|
pinned values and fails loudly on divergence. That check is the point of making
|
||||||
|
the step explicit — both properties degrade silently in production if wrong (a
|
||||||
|
missing `modifier="idf"` produces no error, just unweighted lexical retrieval).
|
||||||
|
|
||||||
|
Payload indexes are (re)created on every run, since unlike vector configuration
|
||||||
|
they can be added to a live collection. The full-text index on `content` is
|
||||||
|
therefore deferred to the keyword-search work in
|
||||||
|
[0002](0002-chunk-crud-and-search-api.md), not created here.
|
||||||
|
|
||||||
### Payload schema
|
### Payload schema
|
||||||
|
|
||||||
This schema is now decided for the fields below. Additional document-context
|
This schema is now decided for the fields below. Additional document-context
|
||||||
@@ -95,7 +156,7 @@ involves format-specific tradeoffs not yet made.
|
|||||||
| `chunk_id` | keyword | stable identifier for a single chunk |
|
| `chunk_id` | keyword | stable identifier for a single chunk |
|
||||||
| `content_type` | keyword | classification of the chunk's content; exact value set (e.g. `paragraph`, `table_row`, `heading`) to be finalized alongside the chunking-strategy ADR |
|
| `content_type` | keyword | classification of the chunk's content; exact value set (e.g. `paragraph`, `table_row`, `heading`) to be finalized alongside the chunking-strategy ADR |
|
||||||
| `source_filename` | keyword | original uploaded filename |
|
| `source_filename` | keyword | original uploaded filename |
|
||||||
| `source_type` | keyword (`docx` \| `csv`) | which parser produced this chunk |
|
| `source_type` | keyword (`docx` \| `xlsx` \| `csv`) | which parser produced this chunk — `xlsx` added by [0018](0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md) |
|
||||||
| `order_id` | float (see below) | chunk's *display* position within the file; mutable so the backend can reorder/insert chunks |
|
| `order_id` | float (see below) | chunk's *display* position within the file; mutable so the backend can reorder/insert chunks |
|
||||||
| `chunk_index` | integer | chunk's *original ingestion* ordinal — immutable, used to derive the deterministic point ID below (kept separate from `order_id` precisely because `order_id` can change) |
|
| `chunk_index` | integer | chunk's *original ingestion* ordinal — immutable, used to derive the deterministic point ID below (kept separate from `order_id` precisely because `order_id` can change) |
|
||||||
| `previous_chunk_id` | keyword, nullable | `chunk_id` of the preceding chunk in display order (`null` for the first chunk in a file) — O(1) adjacency pointer for context-window expansion in ADR-0003 |
|
| `previous_chunk_id` | keyword, nullable | `chunk_id` of the preceding chunk in display order (`null` for the first chunk in a file) — O(1) adjacency pointer for context-window expansion in ADR-0003 |
|
||||||
@@ -107,7 +168,7 @@ involves format-specific tradeoffs not yet made.
|
|||||||
| `updated_at` | datetime | last modification timestamp |
|
| `updated_at` | datetime | last modification timestamp |
|
||||||
| `created_by` | keyword | user/service that created the chunk |
|
| `created_by` | keyword | user/service that created the chunk |
|
||||||
| `updated_by` | keyword | user/service that last modified the chunk |
|
| `updated_by` | keyword | user/service that last modified the chunk |
|
||||||
| `version` | integer | optimistic-concurrency counter, used in ADR-0002 |
|
| `version` | integer | optimistic-concurrency counter, used in ADR-0002. Ingestion currently writes `1` unconditionally: the read-check-write that makes the guard meaningful costs one read per point and belongs with the `/v1/points` write paths, so plan 002 owns it. Safe while ingestion is the only writer of a file's points; it would clobber a concurrent manual edit's counter once `/v1/points` ships. |
|
||||||
| `content_hash` | keyword | hash of the chunk's raw text; lets re-ingestion detect unchanged content and skip re-embedding it |
|
| `content_hash` | keyword | hash of the chunk's raw text; lets re-ingestion detect unchanged content and skip re-embedding it |
|
||||||
| `embedding_model_version` | keyword | identifies which embedding model(s) produced this chunk's vectors; needed to know which chunks require re-embedding after a future model swap |
|
| `embedding_model_version` | keyword | identifies which embedding model(s) produced this chunk's vectors; needed to know which chunks require re-embedding after a future model swap |
|
||||||
|
|
||||||
@@ -212,9 +273,14 @@ them — see ADR-0002 for how reorder/insert/delete operations keep
|
|||||||
ingestion time and both are queried at retrieval time — roughly double
|
ingestion time and both are queried at retrieval time — roughly double
|
||||||
the dense embedding cost/latency of a single-dense-vector design, plus an
|
the dense embedding cost/latency of a single-dense-vector design, plus an
|
||||||
external network dependency on OpenAI's API in the ingestion path.
|
external network dependency on OpenAI's API in the ingestion path.
|
||||||
- `dense_openai`'s exact output dimension is still an open dependency that
|
- ~~`dense_openai`'s exact output dimension is still an open dependency~~ —
|
||||||
should be pinned before ingestion is implemented — changing it later is a
|
**resolved**: pinned to the native 3072 (see "Dense model endpoints and
|
||||||
re-embedding migration, not a config tweak.
|
dimensions" above). The warning still stands for any future change:
|
||||||
|
re-dimensioning is a re-embedding migration, not a config tweak.
|
||||||
|
- The `sparse` vector must be created with `modifier="idf"`. The client
|
||||||
|
computes only BM25's term-frequency saturation; without that modifier
|
||||||
|
Qdrant applies no IDF at all and lexical retrieval silently degrades
|
||||||
|
(ADR-0005).
|
||||||
- `jina-colbert-v2` ([0005](0005-reranking-model-and-sparse-analyzer-selection.md))
|
- `jina-colbert-v2` ([0005](0005-reranking-model-and-sparse-analyzer-selection.md))
|
||||||
adds a hard GPU dependency to ingestion (not just query time, since the
|
adds a hard GPU dependency to ingestion (not just query time, since the
|
||||||
document-side multivector is computed here) and its commercial license is
|
document-side multivector is computed here) and its commercial license is
|
||||||
|
|||||||
@@ -75,6 +75,51 @@ retrieval used by the AI agent in ADR-0003; the two "search" concepts serve
|
|||||||
different callers (a human/admin managing chunks vs. an agent retrieving
|
different callers (a human/admin managing chunks vs. an agent retrieving
|
||||||
context) and should not be conflated in the API or in future discussion.
|
context) and should not be conflated in the API or in future discussion.
|
||||||
|
|
||||||
|
Two properties follow from the index being a *filter*: results carry no
|
||||||
|
relevance score, and their order is unspecified. The API therefore returns
|
||||||
|
neither a score field nor a ranked list, and callers must not read the array
|
||||||
|
order as relevance. A caller that wants ranking wants ADR-0003's path.
|
||||||
|
|
||||||
|
#### The query is normalized the way ingested content was
|
||||||
|
|
||||||
|
`normalize_persian_text` (ADR-0018) folds Arabic letterforms to their Persian
|
||||||
|
equivalents — U+064A to U+06CC, U+0643 to U+06A9 — on every text block before
|
||||||
|
chunking, so stored `content` is uniformly Persian-formed. A query string is
|
||||||
|
not chunk content and never passes through that path, so a term typed on an
|
||||||
|
Arabic keyboard reaches the index as a different codepoint sequence than the
|
||||||
|
document it should match.
|
||||||
|
|
||||||
|
The service therefore applies the same folding to the query before matching.
|
||||||
|
Without it the endpoint fails in the worst available way: an exact-looking
|
||||||
|
query returns an empty result set, with no error, no warning, and nothing in
|
||||||
|
the logs to distinguish "no such term" from "the term is spelled with the
|
||||||
|
other yeh". Note this is a *query-side* transformation only — it changes what
|
||||||
|
is compared, never what is stored.
|
||||||
|
|
||||||
|
This does not extend to stemming or synonyms. Qdrant's full-text index offers
|
||||||
|
neither, and adding a Farsi analyzer here would duplicate the benchmarked BM25
|
||||||
|
sparse pipeline (ADR-0005) in a code path that is not benchmarked against
|
||||||
|
anything.
|
||||||
|
|
||||||
|
#### Listing is scoped to one file, and paginates by `order_id`
|
||||||
|
|
||||||
|
`GET /points?file_id=...` requires `file_id` rather than treating it as one
|
||||||
|
optional filter among several, and its pagination cursor is an `order_id`
|
||||||
|
value rather than an offset. Both follow from `order_id` being per-file:
|
||||||
|
|
||||||
|
- A cursor is only meaningful against a totally ordered key. `order_id` orders
|
||||||
|
points within one file and says nothing across files, so an unscoped listing
|
||||||
|
has no stable sort to paginate along.
|
||||||
|
- An offset cursor is wrong even within one file. Insert, reorder, and delete
|
||||||
|
all shift positions, so a page-two request issued after a concurrent insert
|
||||||
|
ahead of the cursor would repeat a row already returned — silently. Ranging
|
||||||
|
on `order_id > cursor` is unaffected: the reader has passed that value, and a
|
||||||
|
point inserted behind it was already served.
|
||||||
|
|
||||||
|
The second point depends on `order_id` being unique within a file, which the
|
||||||
|
gap-exhaustion rule below preserves by rejecting a reorder whose computed gap
|
||||||
|
would collapse onto a neighbour value.
|
||||||
|
|
||||||
### Delete is soft by default
|
### Delete is soft by default
|
||||||
|
|
||||||
`DELETE /points/{point_id}` and `DELETE /points?file_id=...` set
|
`DELETE /points/{point_id}` and `DELETE /points?file_id=...` set
|
||||||
@@ -105,6 +150,41 @@ Qdrant's `update_filter`, giving an optimistic-concurrency-style guard
|
|||||||
against races between a concurrent ingestion re-run (ADR-0001) and a manual
|
against races between a concurrent ingestion re-run (ADR-0001) and a manual
|
||||||
edit through this API.
|
edit through this API.
|
||||||
|
|
||||||
|
### Re-ingestion versus manual edits
|
||||||
|
|
||||||
|
A file can be re-uploaded after someone has hand-edited one of its points
|
||||||
|
through this API. **The newly ingested file wins.** Ingestion is authoritative
|
||||||
|
for the content of the file it ingested; a manual edit is a correction that
|
||||||
|
survives only until the source document is replaced.
|
||||||
|
|
||||||
|
Concretely:
|
||||||
|
|
||||||
|
- A point that still exists in the new version (same `file_id` +
|
||||||
|
`chunk_index`, hence the same deterministic point ID) is **overwritten in
|
||||||
|
place**. Ingestion performs a read-check-write so `version` is incremented
|
||||||
|
from whatever the manual edit left it at, rather than reset to `1`.
|
||||||
|
- A point from the previous ingestion that is **absent** from the new version
|
||||||
|
is flagged `is_active: false` with `deleted_at` set. It is never removed
|
||||||
|
from Qdrant — the soft-delete rule above applies to re-ingestion exactly as
|
||||||
|
it applies to `DELETE`.
|
||||||
|
- A manually created point (`POST /points`) is assigned a `chunk_index` past
|
||||||
|
the ingested range, so the same sweep deactivates it on the next upload of
|
||||||
|
its file. This is the intended consequence of "the new file wins", not an
|
||||||
|
accident of the sweep's bounds.
|
||||||
|
|
||||||
|
Because the point ID is derived from the immutable `chunk_index`, an
|
||||||
|
overwritten point cannot hold both the manual edit and the new file's content.
|
||||||
|
The clobbered content is therefore recorded in `point_audit_events`
|
||||||
|
(ADR-0009) as a `reingest_overwrite` operation carrying `before_version`, so
|
||||||
|
the edit is recoverable from the audit trail even though it is no longer a
|
||||||
|
live point.
|
||||||
|
|
||||||
|
Rejected alternative: preserving manual edits by having ingestion skip points
|
||||||
|
with `version > 1`. It breaks the guarantee that a successful upload leaves
|
||||||
|
Qdrant matching the uploaded document, and it needs a second, separate rule
|
||||||
|
for edited points that no longer exist in the new version — two divergent
|
||||||
|
notions of authority over one file.
|
||||||
|
|
||||||
### Re-embedding on content edit
|
### Re-embedding on content edit
|
||||||
|
|
||||||
`PUT /points/{point_id}` can change `content`, which leaves the stored
|
`PUT /points/{point_id}` can change `content`, which leaves the stored
|
||||||
|
|||||||
@@ -62,6 +62,23 @@ from its model card: 768-dim output, Matryoshka-truncatable down to 256;
|
|||||||
every embedded string — `search_document: ` at ingestion time, `search_query: `
|
every embedded string — `search_document: ` at ingestion time, `search_query: `
|
||||||
on the agent's query side (ADR-0003).
|
on the agent's query side (ADR-0003).
|
||||||
|
|
||||||
|
> **Amendment — the task prefix is currently not applied.** The `emet`
|
||||||
|
> benchmark that selected this model ran *without* any prefix: its Ollama
|
||||||
|
> deployment's template is a bare `{{ .Prompt }}` passthrough that injects
|
||||||
|
> nothing, which was verified directly against the running endpoint. The
|
||||||
|
> prefix is not cosmetic — embedding the same Persian text with and without
|
||||||
|
> `search_document: ` yields a cosine of only **0.5741** — so applying it at
|
||||||
|
> ingest while the query side omits `search_query: ` would make retrieval
|
||||||
|
> *worse* than using neither.
|
||||||
|
>
|
||||||
|
> Implementation therefore defaults `EMBEDDING_NOMIC_DOCUMENT_PREFIX` to
|
||||||
|
> empty, matching the measured configuration, and exposes it as config so the
|
||||||
|
> prefixed variant is a one-line experiment rather than a code change. The
|
||||||
|
> model card remains the reason to expect prefixing to help; what is missing
|
||||||
|
> is evidence on *this* corpus. Turning it on is a paired change — ingest and
|
||||||
|
> query must move together — and should be settled by an emet run that
|
||||||
|
> measures the pair, not by an unmeasured edit here.
|
||||||
|
|
||||||
## Decision
|
## Decision
|
||||||
|
|
||||||
### Parsing order: structural extraction before chunking
|
### Parsing order: structural extraction before chunking
|
||||||
|
|||||||
@@ -2,9 +2,10 @@
|
|||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
Proposed — the fusion/rerank *shape* and reranker model are decided; the
|
Proposed — the fusion/rerank *shape*, the reranker model, and (as of the
|
||||||
final BM25 analyzer and the commercial license status of the reranker are
|
`emet` benchmark, see "Benchmark outcome" below) the **BM25 analyzer** are
|
||||||
still open per the follow-up items below.
|
decided. The commercial license status of the reranker remains open per the
|
||||||
|
follow-up items below.
|
||||||
|
|
||||||
## Context
|
## Context
|
||||||
|
|
||||||
@@ -100,6 +101,50 @@ entirely to the analyzer stage, not the ranking formula:
|
|||||||
consistent with Farsi's high density of function words (ezafe particles,
|
consistent with Farsi's high density of function words (ezafe particles,
|
||||||
prepositions, common verbs) adding TF/IDF noise if left in.
|
prepositions, common verbs) adding TF/IDF noise if left in.
|
||||||
|
|
||||||
|
### 3a. Benchmark outcome: `bm25-fa-norm-stop` confirmed, and where the BM25 math runs
|
||||||
|
|
||||||
|
The `emet` evaluation lab (`~/code/talie/emet`) ran the four-variant
|
||||||
|
comparison above against the real Farsi corpus and confirmed
|
||||||
|
**`bm25-fa-norm-stop`** as the winner. It is the only sparse variant promoted
|
||||||
|
into emet's hybrid matrix (`emet/hybrid.yaml`). This closes follow-up item 4
|
||||||
|
below.
|
||||||
|
|
||||||
|
The winning analyzer is a specific, reproducible artifact, ported into
|
||||||
|
`src/infrastructure/embedding/analyzers.py` and verified token-for-token
|
||||||
|
against emet's implementation. Its details are load-bearing:
|
||||||
|
|
||||||
|
- Unicode **NFC** (not NFKC), then ZWNJ → space, then Persian/Arabic-Indic
|
||||||
|
digits → ASCII, then `ي→ی ك→ک ة→ه ؤ→و إ→ا أ→ا`.
|
||||||
|
- Tokenizer `[^\W_]+`, which **keeps digits**. This matters for an insurance
|
||||||
|
corpus: policy numbers, dates, and amounts are exactly the terms lexical
|
||||||
|
retrieval should match, and the digit folding above means a query in ASCII
|
||||||
|
digits matches a document authored in Persian ones.
|
||||||
|
- A 51-entry stopword set (40 Persian/Arabic + 11 English, the corpus being
|
||||||
|
mixed-script). Deliberately not a full `hazm` list.
|
||||||
|
- No stemming — `fa_norm_stem` was the losing arm.
|
||||||
|
|
||||||
|
**The BM25 formula is split across two systems, deliberately.** The client
|
||||||
|
applies term-frequency saturation, including the `k`/`b` document-length
|
||||||
|
normalization; **IDF is supplied by Qdrant** via `modifier="idf"` on the
|
||||||
|
sparse vector field, computed from collection-wide statistics rather than
|
||||||
|
from a fixed client-side corpus.
|
||||||
|
|
||||||
|
That split is a correctness trap worth stating plainly: a `chunks` collection
|
||||||
|
created *without* `modifier="idf"` will score these vectors as saturated term
|
||||||
|
frequencies with no IDF weighting at all — no error, no warning, just
|
||||||
|
materially worse lexical retrieval. The collection bootstrap must set it.
|
||||||
|
|
||||||
|
Document and query encoding are asymmetric in exactly one term: documents
|
||||||
|
carry the `b` length normalization, queries do not (standard BM25 practice).
|
||||||
|
Both sides must therefore encode through the same implementation, which is
|
||||||
|
why the sparse port carries a `query` flag rather than leaving retrieval to
|
||||||
|
grow a second, silently divergent encoder.
|
||||||
|
|
||||||
|
Term → sparse-index mapping is `blake2b(token, digest_size=8) % (2**31 - 1)`,
|
||||||
|
a pure hash with no vocabulary table, so it needs no shared state and stays
|
||||||
|
identical across processes and between ingest and query time. Changing the
|
||||||
|
hash orphans every stored sparse vector: that is a re-ingestion, not a deploy.
|
||||||
|
|
||||||
### 4. BM25 parameters: keep `k=1.2`, `b=0.75`; tune analyzer, not formula
|
### 4. BM25 parameters: keep `k=1.2`, `b=0.75`; tune analyzer, not formula
|
||||||
|
|
||||||
These are standard, well-validated defaults (Trotman, Puurula & Burgess,
|
These are standard, well-validated defaults (Trotman, Puurula & Burgess,
|
||||||
@@ -159,8 +204,21 @@ comparison and `b` sweep in the follow-ups below.
|
|||||||
3. Run an ablation: single dense model + sparse + rerank vs. the current
|
3. Run an ablation: single dense model + sparse + rerank vs. the current
|
||||||
dual-dense-model + sparse + rerank setup, on real Farsi queries, to
|
dual-dense-model + sparse + rerank setup, on real Farsi queries, to
|
||||||
justify (or drop) the second dense vector (`dense_openai`).
|
justify (or drop) the second dense vector (`dense_openai`).
|
||||||
4. Compare `bm25-fa-norm-stop` vs. `bm25-fa-norm-stem` in isolation to
|
4. ~~Compare `bm25-fa-norm-stop` vs. `bm25-fa-norm-stem` in isolation~~ —
|
||||||
determine whether gains come from stopword removal, stemming, or both.
|
**done**, see "Benchmark outcome" above. `fa_norm_stop` won; stemming was
|
||||||
|
not adopted.
|
||||||
5. Sweep BM25 `b` (e.g. 0.5–0.9) for the winning analyzer, since document
|
5. Sweep BM25 `b` (e.g. 0.5–0.9) for the winning analyzer, since document
|
||||||
length varies significantly across the corpus (short chat messages vs.
|
length varies significantly across the corpus (short chat messages vs.
|
||||||
long articles) and `0.75` is a generic default, not corpus-tuned.
|
long articles) and `0.75` is a generic default, not corpus-tuned.
|
||||||
|
6. **Recalibrate `avg_len`.** The client-side `b` term needs an average
|
||||||
|
document length in *analyzer tokens*. The ported value (256.0) is emet's
|
||||||
|
own placeholder, and emet measured it over short Q&A records rather than
|
||||||
|
this service's ~400-token chunks, so it is very likely miscalibrated here.
|
||||||
|
Exposed as `EMBEDDING_SPARSE_AVG_LEN` so it can be corrected from real
|
||||||
|
corpus statistics without a code change.
|
||||||
|
7. **Re-benchmark the analyzer with diacritic stripping.** `fa_norm_stop`
|
||||||
|
does not remove harakat or tatweel, so `ســلام` and `سلام` are distinct
|
||||||
|
terms. `src/application/ingestion/normalization.py` already strips both
|
||||||
|
for chunk *content*; extending that to the analyzer is plausibly an
|
||||||
|
improvement but would deviate from the measured configuration, so it
|
||||||
|
belongs in an emet run rather than an unmeasured edit.
|
||||||
|
|||||||
@@ -98,13 +98,16 @@ One row per customer/tenant.
|
|||||||
| `slug` | Stable short name, unique, human-readable. |
|
| `slug` | Stable short name, unique, human-readable. |
|
||||||
| `name` | Display name. |
|
| `name` | Display name. |
|
||||||
| `status` | `active` \| `suspended` \| `deleted`. Suspended tenants authenticate to a clear error but cannot run work. |
|
| `status` | `active` \| `suspended` \| `deleted`. Suspended tenants authenticate to a clear error but cannot run work. |
|
||||||
| `settings` | JSONB for tenant-level feature flags/limits (max upload size, enabled file types, allowed domains, etc.). |
|
| `settings` | JSONB for tenant-level feature flags/limits (max upload size, enabled file types, etc.). Allowed domains were previously listed here as well; they live in `tenant_domains` instead, per this ADR's own rule that query-critical fields get typed columns — `domain` is validated on every upload and filtered on every query. |
|
||||||
| `created_at`, `updated_at`, `deleted_at` | Audit/soft-delete timestamps. |
|
| `created_at`, `updated_at`, `deleted_at` | Audit/soft-delete timestamps. |
|
||||||
|
|
||||||
#### `tenant_domains`
|
#### `tenant_domains`
|
||||||
|
|
||||||
Optional but recommended. Validates the `domain` values used throughout Qdrant
|
**Required.** (Previously "optional but recommended"; implemented and made
|
||||||
payloads (`car`, `fire`, etc.) per tenant.
|
mandatory alongside `/v1/domains`.) Validates the `domain` values used
|
||||||
|
throughout Qdrant payloads (`car`, `fire`, etc.) per tenant. Domain sets are
|
||||||
|
per-tenant and differ in size — one tenant may run 14 insurance lines and
|
||||||
|
another 6 — so this is data, not an enum.
|
||||||
|
|
||||||
| Column | Notes |
|
| Column | Notes |
|
||||||
|---|---|
|
|---|---|
|
||||||
@@ -116,7 +119,37 @@ payloads (`car`, `fire`, etc.) per tenant.
|
|||||||
| `metadata` | JSONB for domain-specific ingestion/retrieval settings. |
|
| `metadata` | JSONB for domain-specific ingestion/retrieval settings. |
|
||||||
|
|
||||||
This prevents arbitrary caller-supplied domains from silently creating new
|
This prevents arbitrary caller-supplied domains from silently creating new
|
||||||
partitions in Qdrant.
|
partitions in Qdrant. The failure it guards against is quiet: a typo such as
|
||||||
|
`fier` for `fire` produces no error anywhere — the file is stored, parsed,
|
||||||
|
embedded, and indexed into a partition retrieval never queries, so it is
|
||||||
|
invisible rather than failed.
|
||||||
|
|
||||||
|
##### Enforcement and management
|
||||||
|
|
||||||
|
- **Strict allowlist.** `POST /v1/files` rejects a domain with no `active` row
|
||||||
|
for the tenant (`400`, error code `unknown_domain`). There is no auto-create
|
||||||
|
on first use: that would record the typo rather than prevent it. The check
|
||||||
|
runs inside the upload's first transaction, before any MinIO object, job row,
|
||||||
|
or Qdrant point is written.
|
||||||
|
- **Managed over the API, not by an operator.** `/v1/domains` (list, create,
|
||||||
|
update, disable, enable) is the surface the calling backend uses. Domains are
|
||||||
|
created by an explicit, scoped call rather than as a side effect of an upload
|
||||||
|
— that distinction, not who makes the call, is what "strict" means here.
|
||||||
|
- **Its own scope.** `domains:read`/`domains:write`, deliberately separate from
|
||||||
|
`files:write`. Folding domain creation into the upload scope would let an
|
||||||
|
upload key create partitions again, which is the exact hole this closes.
|
||||||
|
`api_keys.scopes` is already a free JSONB list, so this needs no schema change.
|
||||||
|
- **`tenant_id` stays derived from the API key.** One key per tenant; nothing
|
||||||
|
request-suppliable. A platform key acting across tenants would need a real
|
||||||
|
actor model and is not adopted.
|
||||||
|
- **`domain` is immutable; `display_name` is not.** The key is denormalized into
|
||||||
|
every Qdrant point payload and into `source_files`, so renaming it means
|
||||||
|
rewriting all of them — a migration, not a `PATCH`. The update schema
|
||||||
|
therefore has no `domain` field.
|
||||||
|
- **Disable is not delete.** `status='disabled'` blocks new uploads and hides
|
||||||
|
the domain from listings, leaving already-indexed points intact and
|
||||||
|
retrievable. Actual removal needs the retention/erasure workflow this ADR and
|
||||||
|
plan 001 defer.
|
||||||
|
|
||||||
#### `api_keys`
|
#### `api_keys`
|
||||||
|
|
||||||
|
|||||||
@@ -70,17 +70,30 @@ logger.info(
|
|||||||
Do not build log messages by interpolating operational metadata into prose.
|
Do not build log messages by interpolating operational metadata into prose.
|
||||||
Prefer fields over long strings because fields are queryable.
|
Prefer fields over long strings because fields are queryable.
|
||||||
|
|
||||||
### Emit JSON logs by default in production
|
### Emit JSON logs by default in production; console and file are independent sinks locally
|
||||||
|
|
||||||
Production logs are JSON on stdout so process managers, container runtimes, and
|
Production logs are JSON on stdout so process managers, container runtimes, and
|
||||||
log collectors can ingest them directly. Local development may use a colored
|
log collectors can ingest them directly. This does not change.
|
||||||
console renderer controlled by configuration.
|
|
||||||
|
|
||||||
File logging is optional and mainly for local development. If enabled, it must
|
Locally, stdout and an optional file are two **independent, simultaneous**
|
||||||
use explicit rotation settings such as `maxBytes` and `backupCount`. Do not rely
|
handlers on the same logger, not a single renderer chosen by a flag — the same
|
||||||
on a default `RotatingFileHandler` with no rotation parameters. In containerized
|
structlog event fans out to both:
|
||||||
production, stdout/stderr collection is preferred over writing `logs/app.log`
|
|
||||||
inside the application container.
|
- **Console handler**: always on, `structlog.dev.ConsoleRenderer(colors=True)`.
|
||||||
|
This is what a developer reads while the process runs, so it stays
|
||||||
|
human-readable regardless of whether file logging is also enabled.
|
||||||
|
- **File handler**: off by default, enabled by setting `LOG_FILE_PATH`. Always
|
||||||
|
renders JSON (`structlog.processors.JSONRenderer()`), independent of the
|
||||||
|
console handler's renderer, so a saved log is machine-parseable even though
|
||||||
|
the terminal output next to it is not. Must use explicit rotation
|
||||||
|
(`RotatingFileHandler` with `maxBytes`/`backupCount` — never an unrotated
|
||||||
|
handler).
|
||||||
|
|
||||||
|
In containerized production, stdout/stderr collection remains preferred over
|
||||||
|
writing `logs/app.log` inside the application container, so `LOG_FILE_PATH` is
|
||||||
|
expected to be unset there; the file handler exists for local development,
|
||||||
|
where reading a colored terminal *and* keeping a JSON trail to grep/parse later
|
||||||
|
are both useful at once.
|
||||||
|
|
||||||
### Configure stdlib and structlog together
|
### Configure stdlib and structlog together
|
||||||
|
|
||||||
@@ -188,6 +201,36 @@ Notes:
|
|||||||
- `structlog.contextvars.merge_contextvars` ensures request-bound fields appear
|
- `structlog.contextvars.merge_contextvars` ensures request-bound fields appear
|
||||||
on both structlog and stdlib logs processed through the formatter.
|
on both structlog and stdlib logs processed through the formatter.
|
||||||
|
|
||||||
|
### Bind process-level environment context once at startup
|
||||||
|
|
||||||
|
Deployment identity — which build is running, in which environment, on which
|
||||||
|
instance — answers a different question than request correlation: "is this
|
||||||
|
issue specific to one deployment / one region / one instance?" rather than "is
|
||||||
|
this issue specific to one request?" It does not vary per request, so it must
|
||||||
|
not go through `structlog.contextvars`, which `RequestIdMiddleware` clears on
|
||||||
|
every request; a value bound there before the first request would be wiped the
|
||||||
|
moment that middleware runs.
|
||||||
|
|
||||||
|
Instead, add a static structlog **processor** — a plain closure over values read
|
||||||
|
once at `configure_logging()` time — so it runs on every event regardless of
|
||||||
|
request context:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def _bind_environment(settings: AppLimitSettings):
|
||||||
|
def processor(logger, method_name, event_dict):
|
||||||
|
event_dict["env"] = settings.env
|
||||||
|
event_dict["service_version"] = settings.service_version
|
||||||
|
return event_dict
|
||||||
|
|
||||||
|
return processor
|
||||||
|
```
|
||||||
|
|
||||||
|
`service_version` should be the deployed commit SHA or release tag (e.g. from a
|
||||||
|
`GIT_SHA`/`APP_VERSION` build-time env var — not computed at runtime by
|
||||||
|
shelling out to `git`). This makes "is this only happening on the new
|
||||||
|
deployment?" answerable directly from logs, without cross-referencing a
|
||||||
|
separate deployment record.
|
||||||
|
|
||||||
### Bind request context with contextvars
|
### Bind request context with contextvars
|
||||||
|
|
||||||
At FastAPI ingress, clear stale context, bind request identifiers, and return the
|
At FastAPI ingress, clear stale context, bind request identifiers, and return the
|
||||||
|
|||||||
@@ -159,8 +159,25 @@ retry, and phase 2 has no transaction protecting it:
|
|||||||
return the existing file/job rather than re-ingesting (plan 001).
|
return the existing file/job rather than re-ingesting (plan 001).
|
||||||
- `tenant_id` comes from `AuthContext`, never from the request body.
|
- `tenant_id` comes from `AuthContext`, never from the request body.
|
||||||
- A terminal job is never transitioned back to `running`.
|
- A terminal job is never transitioned back to `running`.
|
||||||
- Qdrant points from a failed attempt do not replace the previous successful
|
- A failed attempt never *removes* content from a working index. The
|
||||||
index; replacement happens only after a successful attempt.
|
soft-delete sweep that retires a shortened file's leftover points runs only
|
||||||
|
after every upsert in the attempt has succeeded.
|
||||||
|
|
||||||
|
This is deliberately weaker than "replacement happens only after a successful
|
||||||
|
attempt", which an earlier revision of this ADR claimed. That guarantee is not
|
||||||
|
achievable alongside ADR-0001's deterministic point ids: those ids are exactly
|
||||||
|
what makes a retry idempotent, and they also mean a re-ingestion overwrites
|
||||||
|
points **in place**, so a crash partway through leaves a prefix updated and the
|
||||||
|
remainder still on the old content. Buying literal atomicity would mean
|
||||||
|
generation-suffixed ids and an activation flip, which contradicts ADR-0001 and
|
||||||
|
ADR-0002's stable point ids. Staging the new points as `is_active=false` and
|
||||||
|
flipping them on success is strictly worse — the in-place overwrite would
|
||||||
|
deactivate the previously live points, silently emptying a working index if the
|
||||||
|
attempt were interrupted.
|
||||||
|
|
||||||
|
What holds instead: the index is never emptied, never partially deleted, and a
|
||||||
|
retry converges — deterministic ids rewrite every point and the sweep re-runs,
|
||||||
|
reaching the exact correct state.
|
||||||
|
|
||||||
### Failures are HTTP failures
|
### Failures are HTTP failures
|
||||||
|
|
||||||
|
|||||||
@@ -125,9 +125,12 @@ them:
|
|||||||
- Use `(tenant_id, domain, content_sha256)` to recognize identical uploads.
|
- Use `(tenant_id, domain, content_sha256)` to recognize identical uploads.
|
||||||
- An identical active upload should return the existing source-file/job reference
|
- An identical active upload should return the existing source-file/job reference
|
||||||
rather than create a duplicate ingestion.
|
rather than create a duplicate ingestion.
|
||||||
- A changed upload creates a new ingestion job. Existing active Qdrant points are
|
- A changed upload creates a new ingestion job. A failed re-ingestion never
|
||||||
replaced only after the new job completes successfully, so a failed re-ingestion
|
removes a working index: the soft-delete sweep for a shortened file runs only
|
||||||
does not remove a working index.
|
after every upsert has succeeded. Because ADR-0001's point ids are
|
||||||
|
deterministic, upserts overwrite in place, so an interrupted attempt can leave
|
||||||
|
a prefix updated — it cannot empty or partially delete the index, and a retry
|
||||||
|
converges. See ADR-0017, "Re-running an ingestion stays safe".
|
||||||
- Preserve the original filename in Postgres metadata. MinIO object keys remain
|
- Preserve the original filename in Postgres metadata. MinIO object keys remain
|
||||||
internal ID-based paths.
|
internal ID-based paths.
|
||||||
|
|
||||||
@@ -247,6 +250,16 @@ code and a terminal job row.
|
|||||||
|
|
||||||
### Phase 5: Ingestion execution and Qdrant Chunk/Point CRUD
|
### Phase 5: Ingestion execution and Qdrant Chunk/Point CRUD
|
||||||
|
|
||||||
|
> **Carried forward from Phase 4 — the `chunks` collection must create the
|
||||||
|
> `sparse` vector with `modifier="idf"`.** The BM25 adapter computes only
|
||||||
|
> term-frequency saturation client-side; IDF comes from Qdrant's
|
||||||
|
> collection-wide statistics. Omit the modifier and there is no error and no
|
||||||
|
> warning — sparse scoring silently loses its IDF term and lexical retrieval
|
||||||
|
> degrades. See ADR-0005, "Benchmark outcome".
|
||||||
|
>
|
||||||
|
> Collection creation must also use the pinned dimensions from ADR-0001:
|
||||||
|
> `dense_nomic` 768, `dense_openai` 3072.
|
||||||
|
|
||||||
1. Implement the ingestion service called by the route, using the
|
1. Implement the ingestion service called by the route, using the
|
||||||
application-lifetime database, MinIO, Qdrant, model, and logging clients.
|
application-lifetime database, MinIO, Qdrant, model, and logging clients.
|
||||||
2. Validate the persisted records before fetching the MinIO object.
|
2. Validate the persisted records before fetching the MinIO object.
|
||||||
@@ -262,6 +275,11 @@ code and a terminal job row.
|
|||||||
tenant-filtered upserts, terminal state persistence, retrying an upload, and
|
tenant-filtered upserts, terminal state persistence, retrying an upload, and
|
||||||
parser/Qdrant failure handling.
|
parser/Qdrant failure handling.
|
||||||
|
|
||||||
|
The `chunks` collection itself is provisioned by a deployment step —
|
||||||
|
`uv run python -m src.cli.qdrant_bootstrap` — not by FastAPI startup, for the
|
||||||
|
same reason ADR-0009 keeps Alembic out of startup and ADR-0012 makes LangGraph's
|
||||||
|
`.setup()` a deployment step. See ADR-0001, "Collection provisioning".
|
||||||
|
|
||||||
**Exit criteria:** a successful upload returns `201` with a terminal status, and
|
**Exit criteria:** a successful upload returns `201` with a terminal status, and
|
||||||
its points are retrievable only under the owning tenant's Qdrant filter. A forced
|
its points are retrievable only under the owning tenant's Qdrant filter. A forced
|
||||||
failure mid-ingestion produces a `failed` job and the right HTTP status, and
|
failure mid-ingestion produces a `failed` job and the right HTTP status, and
|
||||||
@@ -271,17 +289,28 @@ retrying the upload produces a correct final state without duplicate chunks.
|
|||||||
|
|
||||||
1. Add an operator runbook covering local startup, migrations, MinIO bucket
|
1. Add an operator runbook covering local startup, migrations, MinIO bucket
|
||||||
setup, the run command, ingestion-bound tuning, the proxy/client timeout
|
setup, the run command, ingestion-bound tuning, the proxy/client timeout
|
||||||
requirement, and how to retry a failed ingestion.
|
requirement, and how to retry a failed ingestion. — `docs/runbook.md`.
|
||||||
2. Add a serialized Compose-based operational smoke test covering upload through
|
2. Add a serialized Compose-based operational smoke test covering upload through
|
||||||
indexed points against the running web process. Testcontainers remains the
|
indexed points against the running web process. Testcontainers remains the
|
||||||
standard pytest mechanism for individual adapter integration tests.
|
standard pytest mechanism for individual adapter integration tests. —
|
||||||
|
`scripts/smoke.sh` driving `tests/e2e/test_compose_smoke.py`, which skips
|
||||||
|
itself unless `SMOKE_BASE_URL` is set so `uv run pytest` never invokes
|
||||||
|
Compose.
|
||||||
3. Add end-to-end tests for duplicate upload, retrying a failed upload, tenant
|
3. Add end-to-end tests for duplicate upload, retrying a failed upload, tenant
|
||||||
isolation, capacity/timeout rejection, and failed parser/Qdrant behavior.
|
isolation, capacity/timeout rejection, and failed parser/Qdrant behavior. —
|
||||||
|
`tests/e2e/test_ingestion_slice.py`, on Testcontainers, in the default suite.
|
||||||
4. Add health/readiness checks that distinguish process health from dependency
|
4. Add health/readiness checks that distinguish process health from dependency
|
||||||
readiness.
|
readiness. — `/healthz` and `/readyz`; `/readyz` additionally requires the
|
||||||
|
`chunks` collection to exist, since a reachable but unbootstrapped Qdrant
|
||||||
|
would `502` on the first upload.
|
||||||
5. Update the README with local-start instructions and links to ADRs, this plan,
|
5. Update the README with local-start instructions and links to ADRs, this plan,
|
||||||
and the operations runbook.
|
and the operations runbook.
|
||||||
|
|
||||||
|
Provisioning a tenant and its first API key turned out to be a prerequisite for
|
||||||
|
1 and 2 rather than a separate milestone: nothing over HTTP can create the first
|
||||||
|
tenant, so `src/cli/provision_tenant.py` was added alongside the other two
|
||||||
|
deployment-step commands.
|
||||||
|
|
||||||
**Exit criteria:** a new developer can start the stack, apply migrations, upload a
|
**Exit criteria:** a new developer can start the stack, apply migrations, upload a
|
||||||
a document, observe the job through completion, and understand how to investigate or
|
a document, observe the job through completion, and understand how to investigate or
|
||||||
retry a failure.
|
retry a failure.
|
||||||
|
|||||||
@@ -15,8 +15,9 @@ are; this document defines order, scope, and verification criteria.
|
|||||||
|
|
||||||
## Prerequisite
|
## Prerequisite
|
||||||
|
|
||||||
Plan 001 must be complete through **Phase 5** before Phase 3 of this plan
|
Plan 001 is complete through Phase 6, so this prerequisite is satisfied. It
|
||||||
starts. Specifically this plan depends on: the `chunks` collection and its
|
required plan 001 through **Phase 5** before Phase 3 of this plan starts.
|
||||||
|
Specifically this plan depends on: the `chunks` collection and its
|
||||||
payload indexes actually existing, API-key authentication and `AuthContext`
|
payload indexes actually existing, API-key authentication and `AuthContext`
|
||||||
tenant derivation, the application-lifetime Qdrant client from the FastAPI
|
tenant derivation, the application-lifetime Qdrant client from the FastAPI
|
||||||
lifespan, and the request-lifetime `AsyncSession` wiring. Phases 1–2 below
|
lifespan, and the request-lifetime `AsyncSession` wiring. Phases 1–2 below
|
||||||
@@ -71,7 +72,9 @@ project owner accepts them, and update the ADR rather than diverging silently.
|
|||||||
(bulk soft delete of a file's points), from ADR-0008.
|
(bulk soft delete of a file's points), from ADR-0008.
|
||||||
- Soft delete as the default for every delete path, with neighbor relinking.
|
- Soft delete as the default for every delete path, with neighbor relinking.
|
||||||
- Optimistic concurrency on every mutating path via the `version` payload field.
|
- Optimistic concurrency on every mutating path via the `version` payload field.
|
||||||
- Audit rows in Postgres for mutating operations.
|
- Audit rows in Postgres for mutating operations: both ADR-0009 tables,
|
||||||
|
`api_request_logs` (one row per API call, written from the request middleware)
|
||||||
|
and `point_audit_events` with the real `api_request_log_id` foreign key.
|
||||||
- Automated tests for tenant isolation, pointer integrity, concurrency
|
- Automated tests for tenant isolation, pointer integrity, concurrency
|
||||||
conflicts, and pagination.
|
conflicts, and pagination.
|
||||||
|
|
||||||
@@ -113,40 +116,22 @@ project owner accepts them, and update the ADR rather than diverging silently.
|
|||||||
9. Routers contain no Qdrant SDK calls and no filter construction. The Qdrant
|
9. Routers contain no Qdrant SDK calls and no filter construction. The Qdrant
|
||||||
client is injected from the lifespan (ADR-0012).
|
client is injected from the lifespan (ADR-0012).
|
||||||
|
|
||||||
## Decisions needed before the affected phase
|
## Decisions resolved before implementation
|
||||||
|
|
||||||
### Re-embedding on content edit (blocks Phase 4)
|
An earlier revision of this plan listed three open decisions here. All are now
|
||||||
|
settled, and one further question this plan deferred to a Phase 6 test has been
|
||||||
|
settled too. They are recorded in the ADRs — these lines are a pointer, not a
|
||||||
|
second source of truth.
|
||||||
|
|
||||||
`PUT /v1/points/{point_id}` can change `content`. The stored vectors then no
|
| Question | Resolution | Recorded in |
|
||||||
longer match the text. Three options, in order of preference:
|
|---|---|---|
|
||||||
|
| Re-embedding on content edit | Re-embed inline, reusing ingestion's ports and bounds and its `502`/`504` codes. The re-embed happens *before* the version-guarded write, so a stale edit still `409`s rather than re-embedding for nothing. | ADR-0002, "Re-embedding on content edit" |
|
||||||
|
| Fractional-key exhaustion | No renormalize endpoint in this slice. Log `points.order_id.gap_low` under a safety threshold; reject with `409` and a distinct error code if the gap would collapse onto a neighbor value. Recovery is a runbook operation. | ADR-0002, "`order_id` gap exhaustion" |
|
||||||
|
| Batch semantics | All-or-nothing, capped at 100 operations. Every operation's `version` precondition is validated before any is applied; one failure rejects the whole request and nothing reaches Qdrant. | ADR-0002, "`POST /points/batch` semantics" |
|
||||||
|
| Re-ingestion versus manual edits | The newly uploaded file wins. Surviving points are overwritten in place with an incremented `version`; points absent from the new version are flagged inactive, never removed; manually created points sit past the ingested `chunk_index` range and are swept by the same rule. Clobbered content is recorded in `point_audit_events` as `reingest_overwrite`. | ADR-0002, "Re-ingestion versus manual edits" |
|
||||||
|
|
||||||
1. **Re-embed inline** on content change, reusing plan 001's embedding ports and
|
Phase 6's cross-slice end-to-end test therefore *verifies* the re-ingestion rule
|
||||||
bounds. Consistent, but puts embedder latency and `502`/`504` failure modes
|
rather than forcing the decision.
|
||||||
on an admin edit path.
|
|
||||||
2. **Require caller-supplied vectors** when content changes, and reject the edit
|
|
||||||
otherwise. Simple and honest, but pushes model knowledge to the client.
|
|
||||||
3. **Mark the point stale** (a payload flag) and re-embed later. Needs
|
|
||||||
background work, which ADR-0017 currently rules out.
|
|
||||||
|
|
||||||
Default to (1) for parity with ingestion, with the same batch/semaphore bounds
|
|
||||||
and the same status codes. Record whichever is chosen in ADR-0002 before
|
|
||||||
implementing Phase 4 — this is a real behavioral contract, not an
|
|
||||||
implementation detail.
|
|
||||||
|
|
||||||
### Fractional-key exhaustion
|
|
||||||
|
|
||||||
ADR-0001 notes float keys eventually need renormalization. Decide now whether
|
|
||||||
this slice ships a renormalize path (an internal operation rewriting a file's
|
|
||||||
`order_id` values to `1000, 2000, 3000, ...`) or explicitly defers it with a
|
|
||||||
logged warning when the gap between neighbors falls under a threshold. Deferring
|
|
||||||
is acceptable; silently producing unrepresentable gaps is not.
|
|
||||||
|
|
||||||
### Batch semantics
|
|
||||||
|
|
||||||
`POST /v1/points/batch` must define, in the API schema and the tests: whether
|
|
||||||
operations are all-or-nothing, what happens when operation 3 of 5 fails a
|
|
||||||
version check, and the maximum operation count per request. Decide before
|
|
||||||
Phase 5; do not let the answer be "whatever Qdrant happened to do."
|
|
||||||
|
|
||||||
## Build order
|
## Build order
|
||||||
|
|
||||||
@@ -251,8 +236,8 @@ of mutations.
|
|||||||
ordering behavior, isolated per test by unique collection or tenant keys.
|
ordering behavior, isolated per test by unique collection or tenant keys.
|
||||||
2. An end-to-end test crossing plan 001 and this slice: ingest a CSV, list its
|
2. An end-to-end test crossing plan 001 and this slice: ingest a CSV, list its
|
||||||
points, reorder one, soft-delete another, re-upload the same file, and assert
|
points, reorder one, soft-delete another, re-upload the same file, and assert
|
||||||
the manual edits interact with re-ingestion exactly as ADR-0001/0002 specify.
|
the manual edits interact with re-ingestion exactly as ADR-0002's
|
||||||
If that interaction is not yet decided, this test is what forces the decision.
|
"Re-ingestion versus manual edits" specifies.
|
||||||
3. Structured logging at the mutation boundary with stable event names
|
3. Structured logging at the mutation boundary with stable event names
|
||||||
(`points.updated`, `points.reordered`, `points.soft_deleted`) carrying
|
(`points.updated`, `points.reordered`, `points.soft_deleted`) carrying
|
||||||
`request_id`, `tenant_id`, `file_id`, and the resulting version.
|
`request_id`, `tenant_id`, `file_id`, and the resulting version.
|
||||||
|
|||||||
303
docs/runbook.md
Normal file
303
docs/runbook.md
Normal file
@@ -0,0 +1,303 @@
|
|||||||
|
# Operator runbook
|
||||||
|
|
||||||
|
How to start this service, configure its ingestion bounds, and investigate or
|
||||||
|
retry a failed upload. Architecture rationale lives in [`docs/adr/`](adr/); the
|
||||||
|
implementation milestone is
|
||||||
|
[plan 001](plans/001-ingestion-vertical-slice.md). This document covers
|
||||||
|
operating what those describe.
|
||||||
|
|
||||||
|
The service is a **single process with no background work**. `POST /v1/files`
|
||||||
|
parses, chunks, embeds, and indexes inline and returns a terminal result
|
||||||
|
(ADR-0017). There is no queue, no worker, and no automatic retry — the caller
|
||||||
|
owns the retry decision, which makes the request's duration a deployment
|
||||||
|
constraint. That fact drives most of this document.
|
||||||
|
|
||||||
|
## 1. Prerequisites and local startup
|
||||||
|
|
||||||
|
Docker, and [`uv`](https://docs.astral.sh/uv/) with Python 3.13.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp .env.example .env # non-secret local defaults; .env is gitignored
|
||||||
|
uv sync
|
||||||
|
docker compose up -d --wait
|
||||||
|
```
|
||||||
|
|
||||||
|
`docker-compose.yml` runs Postgres (`127.0.0.1:5433`), MinIO
|
||||||
|
(`127.0.0.1:9100`, console `9101`), and Qdrant (`127.0.0.1:6343`). It is the
|
||||||
|
local development stack and says so in its header — it is not a production
|
||||||
|
deployment.
|
||||||
|
|
||||||
|
## 2. Deployment steps
|
||||||
|
|
||||||
|
Two schema steps run **before** the application, never at startup: FastAPI
|
||||||
|
performs no DDL, for Postgres (ADR-0009) or for Qdrant (ADR-0001, "Collection
|
||||||
|
provisioning"). Both commands and their reasoning are in the README's
|
||||||
|
[Provisioning the datastores](../README.md#provisioning-the-datastores)
|
||||||
|
section:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run alembic upgrade head # Postgres schema
|
||||||
|
uv run python -m src.cli.qdrant_bootstrap # the `chunks` collection
|
||||||
|
```
|
||||||
|
|
||||||
|
Both are idempotent. `qdrant_bootstrap` verifies an existing collection against
|
||||||
|
the pinned schema and **exits non-zero on a mismatch** rather than leaving a
|
||||||
|
silently degraded sparse index in place — the `sparse` vector's
|
||||||
|
`modifier="idf"` and the pinned dense dimensions (768 / 3072) fail silently if
|
||||||
|
wrong, which is why they are checked rather than assumed.
|
||||||
|
|
||||||
|
Run both again after every deploy that ships a migration or a collection-schema
|
||||||
|
change.
|
||||||
|
|
||||||
|
## 3. MinIO bucket
|
||||||
|
|
||||||
|
Under Compose the bucket already exists: the `app-minio` service's entrypoint
|
||||||
|
runs `mkdir -p /data/${MINIO_BUCKET:-chatbot-source-files}` before starting the
|
||||||
|
server, so first boot creates it. Nothing else needs to be done locally.
|
||||||
|
|
||||||
|
Outside Compose, create the bucket named by `MINIO_BUCKET` before the first
|
||||||
|
upload — the application never creates it. It must stay **private**; ADR-0013
|
||||||
|
keeps source bytes non-public and this slice ships no download API or presigned
|
||||||
|
URLs.
|
||||||
|
|
||||||
|
## 4. Provisioning a tenant, an API key, and its domains
|
||||||
|
|
||||||
|
Nothing over HTTP can bootstrap a tenant: every `/v1` route needs an API key,
|
||||||
|
and a key cannot exist before its tenant. So the first key is issued by an
|
||||||
|
operator command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run python -m src.cli.provision_tenant \
|
||||||
|
--slug acme --domain fire --domain life --scopes files:write,domains:read
|
||||||
|
```
|
||||||
|
|
||||||
|
It prints `api_key=sk_...` **once**. Postgres stores only its SHA-256 hash
|
||||||
|
(ADR-0009), so a lost key is reissued by re-running the command, never
|
||||||
|
recovered. Structured logs carry only the non-secret `key_prefix` — a plaintext
|
||||||
|
key must never reach a log sink (ADR-0011).
|
||||||
|
|
||||||
|
Re-running with the same `--slug` reuses the tenant and any domains it already
|
||||||
|
has, and issues an **additional** key. Both keys stay valid; this adds a key, it
|
||||||
|
does not rotate one.
|
||||||
|
|
||||||
|
Scopes are the security boundary between uploading, reading chunks, and
|
||||||
|
managing the allowlist. Give an upload client `files:write` only. `domains:write`
|
||||||
|
lets its holder create new domains, which is exactly what the allowlist exists to
|
||||||
|
prevent an upload key from doing, and `points:read` lets its holder read the text
|
||||||
|
of every chunk of every file — so an upload-only key gets neither.
|
||||||
|
|
||||||
|
| Scope | Grants |
|
||||||
|
|---|---|
|
||||||
|
| `files:write` | Upload a document and read its ingestion status. |
|
||||||
|
| `points:read` | Read, list, count, and keyword-search this tenant's points, including `GET /v1/files/{file_id}/points`. |
|
||||||
|
| `points:write` | Create, edit, reorder, and soft-delete points (plan 002 Phases 3-5; no route uses it yet). |
|
||||||
|
| `domains:read` / `domains:write` | Inspect and manage the domain allowlist. |
|
||||||
|
| `admin` | Satisfies every scope check. |
|
||||||
|
|
||||||
|
The command's `--scopes` default issues all of the above except `admin`, which
|
||||||
|
suits a first operator key; narrow it explicitly for per-client keys.
|
||||||
|
|
||||||
|
### Domains after the first one
|
||||||
|
|
||||||
|
`POST /v1/files` rejects an unregistered or disabled `domain` with `400`
|
||||||
|
(`unknown_domain`) before anything is written. Ongoing domain management is the
|
||||||
|
`/v1/domains` API, under `domains:read` / `domains:write`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8000/v1/domains \
|
||||||
|
-H "Authorization: Bearer $API_KEY" -H 'Content-Type: application/json' \
|
||||||
|
-d '{"domain": "fire", "display_name": "Fire insurance"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
The `domain` key itself is immutable — it is denormalized into every Qdrant
|
||||||
|
point payload and into `source_files`, so renaming it is a migration, not an
|
||||||
|
edit (ADR-0009). Disabling a domain blocks new uploads; it does not delete
|
||||||
|
existing points.
|
||||||
|
|
||||||
|
## 5. Running the service
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run fastapi dev src/main.py # local, reload
|
||||||
|
uv run uvicorn src.main:app --host 0.0.0.0 --port 8000 # deployed shape
|
||||||
|
```
|
||||||
|
|
||||||
|
Run more than one worker/replica only after reading §6: ingestion bounds are
|
||||||
|
**per process**, so `INGESTION_MAX_CONCURRENCY` multiplies by the number of
|
||||||
|
processes.
|
||||||
|
|
||||||
|
## 6. Ingestion bounds and tuning
|
||||||
|
|
||||||
|
Every bound is enforced server-side and maps to a status code. All are in
|
||||||
|
`.env.example`. Ingestion is CPU- and network-bound in the request, so these are
|
||||||
|
the numbers that decide whether the service degrades gracefully or falls over.
|
||||||
|
|
||||||
|
| Setting | Bounds | On breach | Size it against |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `INGESTION_MAX_CONCURRENCY` | Ingestions in flight **per process** | `503` + `Retry-After` | Memory per in-flight upload (whole file plus its chunks and vectors are resident) and the embedder's capacity. Rejecting is deliberate: ADR-0017 refuses rather than queues. |
|
||||||
|
| `INGESTION_THREAD_POOL_SIZE` | Threads for blocking work (parse, chunk, hash, BM25, the sync `minio` SDK) | — (waits) | CPU cores. It exists to stop ingestion exhausting Starlette's own thread pool, so keep it below the total thread budget. |
|
||||||
|
| `INGESTION_TIMEOUT_SECONDS` | The whole work phase | `504`, job marked `failed` | The slowest legitimate document, plus headroom. See §7 — this must stay under every read timeout in front of it. |
|
||||||
|
| `INGESTION_MAX_UPLOAD_SIZE_MB` | Bytes accepted | `413` | Memory: the upload is read fully into the process before any work starts. |
|
||||||
|
| `INGESTION_MAX_CHUNKS_PER_FILE` | Chunks per file, checked before embedding | `413` | Embedder cost/time per chunk × `INGESTION_TIMEOUT_SECONDS`. This is the real defence against one pathological file eating a slot. |
|
||||||
|
| `INGESTION_EMBED_BATCH_SIZE` | Texts per embedder request | `502` on embedder failure | The provider's per-request limits. Batch before parallelizing. |
|
||||||
|
| `INGESTION_EMBED_CONCURRENCY` | Concurrent embed batches | `502` | Provider rate limits and the self-hosted embedder's throughput. Never unbounded. |
|
||||||
|
| `QDRANT_UPSERT_BATCH_SIZE` / `_CONCURRENCY` | Points per upsert and concurrent upserts | `502` (`index_error`) | Qdrant's ingest capacity; the batch size stays in ADR-0001's 64–256 band. |
|
||||||
|
|
||||||
|
Two settings that look like tuning knobs but are not:
|
||||||
|
|
||||||
|
- **`EMBEDDING_NOMIC_KEEP_ALIVE`** holds the self-hosted model resident. A cold
|
||||||
|
load of `nomic-embed-text-v2-moe` takes over 150 s — longer than any sane
|
||||||
|
`INGESTION_TIMEOUT_SECONDS` — so an idle period followed by an upload would
|
||||||
|
otherwise `504`. The lifespan also warms both dense embedders at startup for
|
||||||
|
the same reason.
|
||||||
|
- **The BM25 analyzer and weights** (`EMBEDDING_SPARSE_*`) are a measured
|
||||||
|
artifact ported from the `emet` evaluation lab, verified token-for-token
|
||||||
|
against it (ADR-0005). Re-benchmark; do not tune them in place.
|
||||||
|
|
||||||
|
## 7. The proxy and client read-timeout requirement
|
||||||
|
|
||||||
|
**Every read timeout in front of this service must exceed
|
||||||
|
`INGESTION_TIMEOUT_SECONDS`.** That includes the reverse proxy / ingress, any
|
||||||
|
load balancer, and the calling backend's own HTTP client.
|
||||||
|
|
||||||
|
If a proxy times out first, the client gets that proxy's error, the upload keeps
|
||||||
|
running in the process, and the caller learns nothing about the outcome from the
|
||||||
|
response. The job row still reaches a terminal status, so
|
||||||
|
`GET /v1/files/{file_id}` remains the way to find out what happened — but the
|
||||||
|
response contract is broken for that request. ADR-0017 names this the main cost
|
||||||
|
of inline ingestion.
|
||||||
|
|
||||||
|
A workable local ordering: client read timeout > proxy read timeout >
|
||||||
|
`INGESTION_TIMEOUT_SECONDS`.
|
||||||
|
|
||||||
|
## 8. Health and readiness
|
||||||
|
|
||||||
|
| Endpoint | Question it answers | Use for |
|
||||||
|
|---|---|---|
|
||||||
|
| `GET /healthz` | Is the process alive? | Liveness probes / restart policy. Never depends on Postgres, MinIO, or Qdrant. |
|
||||||
|
| `GET /readyz` | Can it actually serve? | Load-balancer admission and post-deploy gating. `200` with each dependency `true`, `503` if any is `false`. |
|
||||||
|
|
||||||
|
`/readyz` checks Postgres, MinIO, and Qdrant reachability **and** that the
|
||||||
|
`chunks` collection exists. A reachable-but-unbootstrapped Qdrant reports
|
||||||
|
`{"qdrant": false}` on purpose: uploads to it would fail with `502`, so it is
|
||||||
|
not ready, and this is how a skipped `qdrant_bootstrap` surfaces at deploy time
|
||||||
|
instead of on a user's first upload.
|
||||||
|
|
||||||
|
## 9. Investigating a failure
|
||||||
|
|
||||||
|
Logs are structured (`structlog`, JSON in production) with stable event names —
|
||||||
|
grep the event name, not prose (ADR-0011). Set `LOG_FILE_PATH` for a local
|
||||||
|
JSON file sink alongside the console renderer; leave it unset in production,
|
||||||
|
where stdout collection is preferred.
|
||||||
|
|
||||||
|
**Correlate by `request_id`.** Every request has one, echoed in the
|
||||||
|
`X-Request-Id` response header and included in every error envelope, and bound
|
||||||
|
into every log line emitted while handling that request. A client reporting a
|
||||||
|
failed upload should quote it. `tenant_id`, `file_id`, and `ingestion_job_id`
|
||||||
|
are the other join keys.
|
||||||
|
|
||||||
|
Events worth knowing:
|
||||||
|
|
||||||
|
| Event | Level | Means |
|
||||||
|
|---|---|---|
|
||||||
|
| `ingestion.job.started` | info | Txn A committed; work phase beginning. Carries `tenant_id`, `ingestion_job_id`, `file_id`, `domain`, `source_type`. |
|
||||||
|
| `ingestion.job.completed` | info | Terminal success, with `chunks_parsed`, `points_upserted`, `points_soft_deleted`. |
|
||||||
|
| `ingestion.job.failed` | warning | Terminal failure. **`error_code` says which stage**: `storage_upload_failed`, `parse_failed`, `chunk_limit_exceeded`, `embedding_failed`, `index_failed`, `timeout`. |
|
||||||
|
| `files.upload.duplicate` | info | Identical content already ingested; the existing file/job was returned and nothing was re-ingested. |
|
||||||
|
| `domain.rejected` | warning | Upload refused before any row was written; `reason` is `unregistered` or `disabled`. |
|
||||||
|
| `auth.failed` | warning | `reason` is `malformed_key`, `unknown_key`, `key_inactive`, `key_expired`, or `tenant_inactive`. Never contains key material. |
|
||||||
|
| `auth.succeeded` | info | Carries `tenant_id`, `api_key_id`, `actor_type`. |
|
||||||
|
| `lifespan.embedder.warm_failed` | warning | An embedder was unreachable at startup. Boot continues by design — `/readyz` and the first upload are where this bites. |
|
||||||
|
| `qdrant.bootstrap.schema_mismatch` | error | The existing collection diverges from the pinned schema. The bootstrap exits non-zero; do not start the app against it. |
|
||||||
|
| `api.unhandled_exception` | error | A bug: an exception with no mapping to the error envelope. Always worth a look. |
|
||||||
|
|
||||||
|
A `503` (`ingestion_at_capacity`) is rejected before a job row exists, so it
|
||||||
|
appears in the access log and metrics, not in `ingestion_jobs`.
|
||||||
|
|
||||||
|
### The durable record
|
||||||
|
|
||||||
|
Logs may roll; `ingestion_jobs` and `ingestion_job_events` do not. For one file:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT id, status, error_code, error_message, points_created, points_soft_deleted,
|
||||||
|
created_at, updated_at
|
||||||
|
FROM ingestion_jobs
|
||||||
|
WHERE tenant_id = :tenant_id AND source_file_id = :file_id
|
||||||
|
ORDER BY created_at DESC;
|
||||||
|
|
||||||
|
SELECT stage, level, message, details, created_at
|
||||||
|
FROM ingestion_job_events
|
||||||
|
WHERE tenant_id = :tenant_id AND ingestion_job_id = :ingestion_job_id
|
||||||
|
ORDER BY created_at;
|
||||||
|
```
|
||||||
|
|
||||||
|
Recent failures across a tenant:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT error_code, count(*), max(created_at)
|
||||||
|
FROM ingestion_jobs
|
||||||
|
WHERE tenant_id = :tenant_id AND status = 'failed' AND created_at > now() - interval '1 day'
|
||||||
|
GROUP BY error_code ORDER BY 2 DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
`GET /v1/files/{file_id}` reports the same terminal status over HTTP, scoped to
|
||||||
|
the owning tenant — a file belonging to another tenant returns `404`, not `403`.
|
||||||
|
|
||||||
|
## 10. Retrying a failed ingestion
|
||||||
|
|
||||||
|
**Re-upload the same bytes.** There is no retry endpoint and no automatic retry;
|
||||||
|
the client owns that decision (ADR-0017).
|
||||||
|
|
||||||
|
What that guarantees:
|
||||||
|
|
||||||
|
- Identical content with a **succeeded** job is recognized by
|
||||||
|
`(tenant_id, domain, content_sha256)` and returned as-is with `200` — no
|
||||||
|
re-ingestion, no duplicate points.
|
||||||
|
- Identical content whose last job **failed** starts a fresh job against the
|
||||||
|
same `source_files` row. A terminal job is never moved back to `running`.
|
||||||
|
- Point ids are deterministic from `file_id` + `chunk_index` (ADR-0001), so the
|
||||||
|
retry **overwrites in place** — it cannot duplicate chunks.
|
||||||
|
- A failed attempt never empties or partially deletes a working index: the
|
||||||
|
soft-delete sweep that retires a shortened file's leftover points runs only
|
||||||
|
after every upsert has succeeded. An interrupted attempt can leave a prefix
|
||||||
|
updated; a retry converges (ADR-0017, "Re-running an ingestion stays safe").
|
||||||
|
|
||||||
|
Fix the cause first — the `error_code` says where to look:
|
||||||
|
|
||||||
|
| `error_code` | Usual cause |
|
||||||
|
|---|---|
|
||||||
|
| `parse_failed` | The file is corrupt or is not really the type its extension claims. Retrying identical bytes will fail identically. |
|
||||||
|
| `chunk_limit_exceeded` | The file is genuinely too large for one inline ingestion. Split it, or raise `INGESTION_MAX_CHUNKS_PER_FILE` knowing what §6 says about the timeout. |
|
||||||
|
| `embedding_failed` | The embedder is down, rate-limiting, or unauthenticated. Fix it, then retry — this one usually succeeds unchanged. |
|
||||||
|
| `index_failed` | Qdrant is down, or the collection is missing (run `qdrant_bootstrap`). |
|
||||||
|
| `timeout` | The work exceeded `INGESTION_TIMEOUT_SECONDS`. Check whether the embedder was cold (see `lifespan.embedder.warm_failed` and `KEEP_ALIVE`) before raising the bound. |
|
||||||
|
| `storage_upload_failed` | MinIO is unreachable or the bucket is missing (§3). |
|
||||||
|
|
||||||
|
## 11. What to alert on
|
||||||
|
|
||||||
|
ADR-0017's own triggers for moving ingestion back off the request path. These
|
||||||
|
are the numbers that say the inline design has stopped fitting:
|
||||||
|
|
||||||
|
- **p95 ingestion duration** approaching `INGESTION_TIMEOUT_SECONDS`.
|
||||||
|
- **`503` and `504` rates** ceasing to be negligible.
|
||||||
|
- **Jobs stuck in `running` past the timeout** — every handled failure writes a
|
||||||
|
terminal status, so a non-zero count here means the process died mid-request:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT count(*) FROM ingestion_jobs
|
||||||
|
WHERE status = 'running' AND created_at < now() - interval '5 minutes';
|
||||||
|
```
|
||||||
|
|
||||||
|
Also worth alerting: any `qdrant.bootstrap.schema_mismatch`, a sustained
|
||||||
|
`/readyz` `503`, and any `api.unhandled_exception`.
|
||||||
|
|
||||||
|
## 12. Verifying a deployment
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/smoke.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Brings up Compose, runs both deployment steps, provisions a throwaway tenant,
|
||||||
|
starts the web process, and drives an upload through to indexed Qdrant points
|
||||||
|
against the **running process** — including asserting the structured log output
|
||||||
|
from §9. It is the only Compose-based test; everything else runs on
|
||||||
|
Testcontainers under `uv run pytest` (ADR-0016). Run it before a release.
|
||||||
@@ -9,6 +9,7 @@ dependencies = [
|
|||||||
"anyio>=4.11.0",
|
"anyio>=4.11.0",
|
||||||
"asyncpg>=0.31.0",
|
"asyncpg>=0.31.0",
|
||||||
"fastapi[standard]==0.141.1",
|
"fastapi[standard]==0.141.1",
|
||||||
|
"httpx>=0.28.1",
|
||||||
"langgraph>=1.2.10",
|
"langgraph>=1.2.10",
|
||||||
"minio>=7.2.20",
|
"minio>=7.2.20",
|
||||||
"openpyxl>=3.1.5",
|
"openpyxl>=3.1.5",
|
||||||
@@ -23,7 +24,6 @@ dependencies = [
|
|||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
"asgi-lifespan>=2.1.0",
|
"asgi-lifespan>=2.1.0",
|
||||||
"httpx>=0.28.1",
|
|
||||||
"pytest>=8.3.5",
|
"pytest>=8.3.5",
|
||||||
"pytest-asyncio>=0.25.3",
|
"pytest-asyncio>=0.25.3",
|
||||||
"pytest-cov>=6.0.0",
|
"pytest-cov>=6.0.0",
|
||||||
@@ -37,6 +37,11 @@ dev = [
|
|||||||
testpaths = ["tests"]
|
testpaths = ["tests"]
|
||||||
asyncio_mode = "strict"
|
asyncio_mode = "strict"
|
||||||
timeout = 10
|
timeout = 10
|
||||||
|
# Bound the test function only, not fixture setup. Testcontainers' container
|
||||||
|
# startup is charged to whichever test first pulls a session-scoped container
|
||||||
|
# fixture; on a cold Docker cache that is ~25s and would trip the 10s budget
|
||||||
|
# for every integration test, regardless of how fast the test itself is.
|
||||||
|
timeout_func_only = true
|
||||||
markers = [
|
markers = [
|
||||||
"unit: fast tests with no external services",
|
"unit: fast tests with no external services",
|
||||||
"integration: tests against a real disposable service",
|
"integration: tests against a real disposable service",
|
||||||
|
|||||||
98
scripts/smoke.sh
Executable file
98
scripts/smoke.sh
Executable file
@@ -0,0 +1,98 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Serialized operational smoke test of the running web process (ADR-0016, plan
|
||||||
|
# 001 Phase 6).
|
||||||
|
#
|
||||||
|
# ./scripts/smoke.sh
|
||||||
|
#
|
||||||
|
# Brings up the Compose stack, runs both deployment steps for real, provisions
|
||||||
|
# a tenant, starts uvicorn, and drives `tests/e2e/test_compose_smoke.py`
|
||||||
|
# against it over a socket. This is the only Compose-based test: every other
|
||||||
|
# test uses Testcontainers and an in-process ASGI transport, which is exactly
|
||||||
|
# what makes this one worth having -- it is the only thing that exercises the
|
||||||
|
# deployment steps, the real logging configuration, and a real HTTP server.
|
||||||
|
#
|
||||||
|
# Not part of `uv run pytest`: the smoke test skips itself unless SMOKE_BASE_URL
|
||||||
|
# is set, so this script is the only way it runs. Run it before a release.
|
||||||
|
#
|
||||||
|
# Leaves the Compose stack running (it is the local dev stack); only the uvicorn
|
||||||
|
# process and the temporary log file are cleaned up.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
|
||||||
|
PORT="${SMOKE_PORT:-8021}"
|
||||||
|
SLUG="smoke-$(date +%s)"
|
||||||
|
DOMAIN="smoke"
|
||||||
|
LOG_FILE="$(mktemp -t smoke-app-log.XXXXXX.jsonl)"
|
||||||
|
CONSOLE_LOG="$(mktemp -t smoke-app-console.XXXXXX.log)"
|
||||||
|
APP_PID=""
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [[ -n "${APP_PID}" ]] && kill -0 "${APP_PID}" 2>/dev/null; then
|
||||||
|
kill "${APP_PID}" 2>/dev/null || true
|
||||||
|
wait "${APP_PID}" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "${LOG_FILE}" "${CONSOLE_LOG}"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
if [[ ! -f .env ]]; then
|
||||||
|
echo "no .env found; copy .env.example first (see docs/runbook.md)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> starting Postgres, MinIO, Qdrant"
|
||||||
|
docker compose up -d --wait
|
||||||
|
|
||||||
|
echo "==> applying deployment steps"
|
||||||
|
uv run alembic upgrade head
|
||||||
|
uv run python -m src.cli.qdrant_bootstrap
|
||||||
|
|
||||||
|
echo "==> provisioning tenant '${SLUG}'"
|
||||||
|
PROVISION_OUTPUT="$(uv run python -m src.cli.provision_tenant \
|
||||||
|
--slug "${SLUG}" --domain "${DOMAIN}" --scopes files:write 2>/dev/null)"
|
||||||
|
API_KEY="$(printf '%s\n' "${PROVISION_OUTPUT}" | sed -n 's/^api_key=//p')"
|
||||||
|
if [[ -z "${API_KEY}" ]]; then
|
||||||
|
echo "provisioning did not return an api_key" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> starting the web process on port ${PORT}"
|
||||||
|
# JSON to a file sink, because the smoke test asserts the real ADR-0011 log
|
||||||
|
# output -- the one thing no in-process test can check.
|
||||||
|
LOG_JSON_FORMAT=true LOG_FILE_PATH="${LOG_FILE}" \
|
||||||
|
uv run python -m uvicorn src.main:app --host 127.0.0.1 --port "${PORT}" \
|
||||||
|
>"${CONSOLE_LOG}" 2>&1 &
|
||||||
|
APP_PID=$!
|
||||||
|
|
||||||
|
echo "==> waiting for /readyz"
|
||||||
|
for _ in $(seq 1 60); do
|
||||||
|
if curl -fsS "http://127.0.0.1:${PORT}/readyz" >/dev/null 2>&1; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if ! kill -0 "${APP_PID}" 2>/dev/null; then
|
||||||
|
echo "the web process exited before becoming ready:" >&2
|
||||||
|
tail -20 "${CONSOLE_LOG}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
if ! curl -fsS "http://127.0.0.1:${PORT}/readyz" >/dev/null 2>&1; then
|
||||||
|
# Most often an unbootstrapped Qdrant or an unreachable embedder host; the
|
||||||
|
# runbook's health/readiness section covers reading this.
|
||||||
|
echo "the web process never became ready:" >&2
|
||||||
|
tail -20 "${CONSOLE_LOG}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> running the smoke test"
|
||||||
|
SMOKE_BASE_URL="http://127.0.0.1:${PORT}" \
|
||||||
|
SMOKE_API_KEY="${API_KEY}" \
|
||||||
|
SMOKE_DOMAIN="${DOMAIN}" \
|
||||||
|
SMOKE_LOG_PATH="${LOG_FILE}" \
|
||||||
|
SMOKE_QDRANT_URL="${QDRANT_URL:-http://127.0.0.1:6343}" \
|
||||||
|
SMOKE_QDRANT_COLLECTION="${QDRANT_COLLECTION:-chunks}" \
|
||||||
|
uv run python -m pytest tests/e2e/test_compose_smoke.py -q
|
||||||
|
|
||||||
|
echo "==> smoke test passed"
|
||||||
38
src/api/dependencies/auth.py
Normal file
38
src/api/dependencies/auth.py
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
"""Auth dependencies (ADR-0008): resolve `AuthContext` from a bearer token,
|
||||||
|
then gate routes on scope.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Awaitable, Callable
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import Depends
|
||||||
|
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.application.auth.context import AuthContext
|
||||||
|
from src.application.auth.errors import InvalidApiKeyError, MissingScopeError
|
||||||
|
from src.application.auth.service import resolve_auth_context
|
||||||
|
from src.bootstrap.dependencies import get_sessionmaker
|
||||||
|
|
||||||
|
_bearer_scheme = HTTPBearer(auto_error=False)
|
||||||
|
|
||||||
|
|
||||||
|
async def get_auth_context(
|
||||||
|
credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(_bearer_scheme)],
|
||||||
|
sessionmaker: Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)],
|
||||||
|
) -> AuthContext:
|
||||||
|
if credentials is None:
|
||||||
|
raise InvalidApiKeyError("missing Authorization header")
|
||||||
|
return await resolve_auth_context(sessionmaker, credentials.credentials)
|
||||||
|
|
||||||
|
|
||||||
|
AuthContextDep = Annotated[AuthContext, Depends(get_auth_context)]
|
||||||
|
|
||||||
|
|
||||||
|
def require_scope(scope: str) -> Callable[[AuthContext], Awaitable[AuthContext]]:
|
||||||
|
async def _dependency(auth: AuthContextDep) -> AuthContext:
|
||||||
|
if not auth.has_scope(scope):
|
||||||
|
raise MissingScopeError(f"missing required scope '{scope}'")
|
||||||
|
return auth
|
||||||
|
|
||||||
|
return _dependency
|
||||||
138
src/api/errors.py
Normal file
138
src/api/errors.py
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
"""Maps application exceptions to the ADR-0008 error envelope.
|
||||||
|
|
||||||
|
This is the single place that knows the exception-type -> status-code
|
||||||
|
mapping; application/infrastructure code never imports FastAPI or raises
|
||||||
|
`HTTPException` (ADR-0015).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from fastapi import FastAPI, Request, status
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||||
|
|
||||||
|
from src.application.auth.errors import (
|
||||||
|
InvalidApiKeyError,
|
||||||
|
MissingScopeError,
|
||||||
|
TenantInactiveError,
|
||||||
|
)
|
||||||
|
from src.application.domains.errors import DomainAlreadyExistsError, UnknownDomainError
|
||||||
|
from src.application.files.errors import (
|
||||||
|
FileTooLargeError,
|
||||||
|
InvalidUploadError,
|
||||||
|
SourceFileNotFoundError,
|
||||||
|
)
|
||||||
|
from src.application.ingestion.errors import (
|
||||||
|
ChunkLimitExceededError,
|
||||||
|
DocumentParseError,
|
||||||
|
EmbedderError,
|
||||||
|
IngestionAtCapacityError,
|
||||||
|
IngestionTimeoutError,
|
||||||
|
PointIndexingError,
|
||||||
|
UnsupportedSourceTypeError,
|
||||||
|
)
|
||||||
|
from src.application.points.errors import PointVersionConflictError
|
||||||
|
from src.application.points.point import PointNotFoundError
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
# A fixed backoff hint, not a computed retry budget: ADR-0017 rejects a
|
||||||
|
# request outright at capacity rather than queueing it, so there is no
|
||||||
|
# in-process estimate of when a slot will free up to report instead.
|
||||||
|
_CAPACITY_RETRY_AFTER_SECONDS = 1
|
||||||
|
|
||||||
|
# (exception type, status code, stable error code)
|
||||||
|
_MAPPING: tuple[tuple[type[Exception], int, str], ...] = (
|
||||||
|
(InvalidApiKeyError, status.HTTP_401_UNAUTHORIZED, "invalid_api_key"),
|
||||||
|
(TenantInactiveError, status.HTTP_401_UNAUTHORIZED, "tenant_not_found"),
|
||||||
|
(MissingScopeError, status.HTTP_403_FORBIDDEN, "missing_scope"),
|
||||||
|
(InvalidUploadError, status.HTTP_400_BAD_REQUEST, "validation_error"),
|
||||||
|
# 404, never 403: a cross-tenant point id must be indistinguishable from a
|
||||||
|
# nonexistent one, or the API becomes an existence oracle (ADR-0016).
|
||||||
|
(PointNotFoundError, status.HTTP_404_NOT_FOUND, "not_found"),
|
||||||
|
(SourceFileNotFoundError, status.HTTP_404_NOT_FOUND, "not_found"),
|
||||||
|
# Not "the version guard fired once" — that is retried. This is the service
|
||||||
|
# giving up after repeated re-plans, i.e. a genuinely contended point.
|
||||||
|
(PointVersionConflictError, status.HTTP_409_CONFLICT, "conflict"),
|
||||||
|
(UnknownDomainError, status.HTTP_400_BAD_REQUEST, "unknown_domain"),
|
||||||
|
(DomainAlreadyExistsError, status.HTTP_409_CONFLICT, "conflict"),
|
||||||
|
(DocumentParseError, status.HTTP_400_BAD_REQUEST, "validation_error"),
|
||||||
|
(UnsupportedSourceTypeError, status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, "unsupported_media_type"),
|
||||||
|
(FileTooLargeError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
|
||||||
|
(ChunkLimitExceededError, status.HTTP_413_CONTENT_TOO_LARGE, "payload_too_large"),
|
||||||
|
(EmbedderError, status.HTTP_502_BAD_GATEWAY, "embedder_error"),
|
||||||
|
(PointIndexingError, status.HTTP_502_BAD_GATEWAY, "index_error"),
|
||||||
|
(IngestionTimeoutError, status.HTTP_504_GATEWAY_TIMEOUT, "ingestion_timeout"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _request_id(request: Request) -> str | None:
|
||||||
|
return getattr(request.state, "request_id", None)
|
||||||
|
|
||||||
|
|
||||||
|
def _envelope(
|
||||||
|
code: str, message: str, request_id: str | None, details: dict[str, object] | None = None
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"error": {
|
||||||
|
"code": code,
|
||||||
|
"message": message,
|
||||||
|
"details": details or {},
|
||||||
|
"request_id": request_id,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def register_exception_handlers(app: FastAPI) -> None:
|
||||||
|
for exc_type, status_code, error_code in _MAPPING:
|
||||||
|
|
||||||
|
def _handler(
|
||||||
|
request: Request,
|
||||||
|
exc: Exception,
|
||||||
|
status_code: int = status_code,
|
||||||
|
error_code: str = error_code,
|
||||||
|
) -> JSONResponse:
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status_code,
|
||||||
|
content=_envelope(error_code, str(exc), _request_id(request)),
|
||||||
|
)
|
||||||
|
|
||||||
|
app.add_exception_handler(exc_type, _handler)
|
||||||
|
|
||||||
|
@app.exception_handler(IngestionAtCapacityError)
|
||||||
|
def _capacity_handler(request: Request, exc: IngestionAtCapacityError) -> JSONResponse:
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||||
|
content=_envelope("ingestion_at_capacity", str(exc), _request_id(request)),
|
||||||
|
headers={"Retry-After": str(_CAPACITY_RETRY_AFTER_SECONDS)},
|
||||||
|
)
|
||||||
|
|
||||||
|
@app.exception_handler(RequestValidationError)
|
||||||
|
def _validation_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
content=_envelope(
|
||||||
|
"validation_error",
|
||||||
|
"request validation failed",
|
||||||
|
_request_id(request),
|
||||||
|
details={"errors": exc.errors()},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@app.exception_handler(StarletteHTTPException)
|
||||||
|
def _http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||||
|
code = "not_found" if exc.status_code == status.HTTP_404_NOT_FOUND else "http_error"
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=exc.status_code,
|
||||||
|
content=_envelope(code, str(exc.detail), _request_id(request)),
|
||||||
|
)
|
||||||
|
|
||||||
|
@app.exception_handler(Exception)
|
||||||
|
def _unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||||
|
logger.exception("api.unhandled_exception", path=request.url.path)
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||||
|
content=_envelope(
|
||||||
|
"internal_error", "an unexpected error occurred", _request_id(request)
|
||||||
|
),
|
||||||
|
)
|
||||||
38
src/api/middleware.py
Normal file
38
src/api/middleware.py
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
"""Per-request correlation id (ADR-0008, ADR-0011).
|
||||||
|
|
||||||
|
Every request gets a `request_id`: reused from an incoming `X-Request-Id` if
|
||||||
|
the caller supplied one, otherwise generated. It is bound into structlog's
|
||||||
|
contextvars so every log line emitted while handling the request carries it,
|
||||||
|
stored on `request.state` for exception handlers, and echoed back in the
|
||||||
|
response header.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Awaitable, Callable
|
||||||
|
from typing import override
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from starlette.middleware.base import BaseHTTPMiddleware
|
||||||
|
from starlette.requests import Request
|
||||||
|
from starlette.responses import Response
|
||||||
|
|
||||||
|
_HEADER = "X-Request-Id"
|
||||||
|
|
||||||
|
|
||||||
|
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||||
|
@override
|
||||||
|
async def dispatch(
|
||||||
|
self, request: Request, call_next: Callable[[Request], Awaitable[Response]]
|
||||||
|
) -> Response:
|
||||||
|
request_id = request.headers.get(_HEADER) or str(uuid.uuid4())
|
||||||
|
request.state.request_id = request_id
|
||||||
|
|
||||||
|
structlog.contextvars.clear_contextvars()
|
||||||
|
structlog.contextvars.bind_contextvars(request_id=request_id)
|
||||||
|
try:
|
||||||
|
response = await call_next(request)
|
||||||
|
finally:
|
||||||
|
structlog.contextvars.clear_contextvars()
|
||||||
|
|
||||||
|
response.headers[_HEADER] = request_id
|
||||||
|
return response
|
||||||
@@ -1,3 +1,10 @@
|
|||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
from src.api.routers.domains import router as domains_router
|
||||||
|
from src.api.routers.files import router as files_router
|
||||||
|
from src.api.routers.points import router as points_router
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
router.include_router(domains_router)
|
||||||
|
router.include_router(files_router)
|
||||||
|
router.include_router(points_router)
|
||||||
|
|||||||
110
src/api/routers/domains.py
Normal file
110
src/api/routers/domains.py
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
"""`/v1/domains` (ADR-0008, ADR-0009).
|
||||||
|
|
||||||
|
The management surface for a tenant's domain allowlist, used by the calling
|
||||||
|
backend rather than by an operator with a psql prompt.
|
||||||
|
|
||||||
|
Gated on `domains:read`/`domains:write`, deliberately **not** on `files:write`:
|
||||||
|
if an upload key could create domains, the allowlist would no longer prevent a
|
||||||
|
typo'd `domain` from creating a Qdrant partition, which is its only purpose.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, status
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.api.dependencies.auth import require_scope
|
||||||
|
from src.api.schemas.domains import (
|
||||||
|
CreateDomainRequest,
|
||||||
|
DomainListResponse,
|
||||||
|
DomainResponse,
|
||||||
|
UpdateDomainRequest,
|
||||||
|
)
|
||||||
|
from src.application.auth.context import AuthContext
|
||||||
|
from src.application.domains import (
|
||||||
|
create_domain,
|
||||||
|
list_domains,
|
||||||
|
set_domain_status,
|
||||||
|
update_domain,
|
||||||
|
)
|
||||||
|
from src.bootstrap.dependencies import get_sessionmaker
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/domains", tags=["domains"])
|
||||||
|
|
||||||
|
_RequireDomainsRead = Annotated[AuthContext, Depends(require_scope("domains:read"))]
|
||||||
|
_RequireDomainsWrite = Annotated[AuthContext, Depends(require_scope("domains:write"))]
|
||||||
|
_SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("")
|
||||||
|
async def list_tenant_domains(
|
||||||
|
auth: _RequireDomainsRead,
|
||||||
|
sessionmaker: _SessionmakerDep,
|
||||||
|
include_disabled: bool = False,
|
||||||
|
) -> DomainListResponse:
|
||||||
|
results = await list_domains(
|
||||||
|
sessionmaker, tenant_id=auth.tenant_id, include_disabled=include_disabled
|
||||||
|
)
|
||||||
|
return DomainListResponse(domains=[DomainResponse.from_result(item) for item in results])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||||
|
async def create_tenant_domain(
|
||||||
|
request: CreateDomainRequest,
|
||||||
|
auth: _RequireDomainsWrite,
|
||||||
|
sessionmaker: _SessionmakerDep,
|
||||||
|
) -> DomainResponse:
|
||||||
|
result = await create_domain(
|
||||||
|
sessionmaker,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
domain=request.domain,
|
||||||
|
display_name=request.display_name,
|
||||||
|
metadata=request.metadata,
|
||||||
|
)
|
||||||
|
return DomainResponse.from_result(result)
|
||||||
|
|
||||||
|
|
||||||
|
@router.patch("/{domain}")
|
||||||
|
async def update_tenant_domain(
|
||||||
|
domain: str,
|
||||||
|
request: UpdateDomainRequest,
|
||||||
|
auth: _RequireDomainsWrite,
|
||||||
|
sessionmaker: _SessionmakerDep,
|
||||||
|
) -> DomainResponse:
|
||||||
|
result = await update_domain(
|
||||||
|
sessionmaker,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
display_name=request.display_name,
|
||||||
|
)
|
||||||
|
return DomainResponse.from_result(result)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{domain}")
|
||||||
|
async def disable_tenant_domain(
|
||||||
|
domain: str,
|
||||||
|
auth: _RequireDomainsWrite,
|
||||||
|
sessionmaker: _SessionmakerDep,
|
||||||
|
) -> DomainResponse:
|
||||||
|
"""Disable, not delete.
|
||||||
|
|
||||||
|
Blocks new uploads and drops the domain from pickers while leaving the
|
||||||
|
points already indexed under it intact and retrievable. Actually removing
|
||||||
|
them needs the tenant-erasure workflow plan 001 defers.
|
||||||
|
"""
|
||||||
|
result = await set_domain_status(
|
||||||
|
sessionmaker, tenant_id=auth.tenant_id, domain=domain, status="disabled"
|
||||||
|
)
|
||||||
|
return DomainResponse.from_result(result)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{domain}/enable")
|
||||||
|
async def enable_tenant_domain(
|
||||||
|
domain: str,
|
||||||
|
auth: _RequireDomainsWrite,
|
||||||
|
sessionmaker: _SessionmakerDep,
|
||||||
|
) -> DomainResponse:
|
||||||
|
result = await set_domain_status(
|
||||||
|
sessionmaker, tenant_id=auth.tenant_id, domain=domain, status="active"
|
||||||
|
)
|
||||||
|
return DomainResponse.from_result(result)
|
||||||
157
src/api/routers/files.py
Normal file
157
src/api/routers/files.py
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
"""`POST /v1/files`, `GET /v1/files/{file_id}`, `DELETE /v1/files/{file_id}` (ADR-0008).
|
||||||
|
|
||||||
|
Routes adapt HTTP to `application/files` calls; they do not parse, hash,
|
||||||
|
touch MinIO/Qdrant, or otherwise carry ingestion business logic (ADR-0015).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from anyio import CapacityLimiter, Semaphore
|
||||||
|
from fastapi import APIRouter, Depends, Form, HTTPException, Response, UploadFile, status
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.api.dependencies.auth import require_scope
|
||||||
|
from src.api.schemas.files import FileDeleteResponse, FileStatusResponse, FileUploadResponse
|
||||||
|
from src.api.schemas.points import DEFAULT_PAGE_SIZE, LimitQuery, PointListResponse
|
||||||
|
from src.application.auth.context import AuthContext
|
||||||
|
from src.application.files.deletion import delete_source_file
|
||||||
|
from src.application.files.status import get_file_status
|
||||||
|
from src.application.files.upload import upload_source_file
|
||||||
|
from src.application.points.queries import list_file_points
|
||||||
|
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||||
|
from src.application.ports.object_storage import ObjectStorage
|
||||||
|
from src.application.ports.point_repository import PointRepository
|
||||||
|
from src.application.ports.point_storage import PointStorage
|
||||||
|
from src.bootstrap.dependencies import (
|
||||||
|
get_dense_embedders,
|
||||||
|
get_ingestion_concurrency_limiter,
|
||||||
|
get_ingestion_limiter,
|
||||||
|
get_object_storage,
|
||||||
|
get_point_repository,
|
||||||
|
get_point_storage,
|
||||||
|
get_sessionmaker,
|
||||||
|
get_settings,
|
||||||
|
get_sparse_embedder,
|
||||||
|
)
|
||||||
|
from src.config import Settings
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/files", tags=["files"])
|
||||||
|
|
||||||
|
_RequireFilesWrite = Annotated[AuthContext, Depends(require_scope("files:write"))]
|
||||||
|
_RequirePointsRead = Annotated[AuthContext, Depends(require_scope("points:read"))]
|
||||||
|
_RequirePointsWrite = Annotated[AuthContext, Depends(require_scope("points:write"))]
|
||||||
|
_SessionmakerDep = Annotated[async_sessionmaker[AsyncSession], Depends(get_sessionmaker)]
|
||||||
|
_ObjectStorageDep = Annotated[ObjectStorage, Depends(get_object_storage)]
|
||||||
|
_PointStorageDep = Annotated[PointStorage, Depends(get_point_storage)]
|
||||||
|
_PointRepositoryDep = Annotated[PointRepository, Depends(get_point_repository)]
|
||||||
|
_SettingsDep = Annotated[Settings, Depends(get_settings)]
|
||||||
|
_IngestionLimiterDep = Annotated[CapacityLimiter, Depends(get_ingestion_limiter)]
|
||||||
|
_ConcurrencyLimiterDep = Annotated[Semaphore, Depends(get_ingestion_concurrency_limiter)]
|
||||||
|
_DenseEmbeddersDep = Annotated[Sequence[DenseEmbedder], Depends(get_dense_embedders)]
|
||||||
|
_SparseEmbedderDep = Annotated[SparseEmbedder, Depends(get_sparse_embedder)]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||||
|
async def upload_file(
|
||||||
|
response: Response,
|
||||||
|
file: UploadFile,
|
||||||
|
domain: Annotated[str, Form()],
|
||||||
|
auth: _RequireFilesWrite,
|
||||||
|
sessionmaker: _SessionmakerDep,
|
||||||
|
storage: _ObjectStorageDep,
|
||||||
|
point_storage: _PointStorageDep,
|
||||||
|
settings: _SettingsDep,
|
||||||
|
limiter: _IngestionLimiterDep,
|
||||||
|
concurrency_limiter: _ConcurrencyLimiterDep,
|
||||||
|
dense_embedders: _DenseEmbeddersDep,
|
||||||
|
sparse_embedder: _SparseEmbedderDep,
|
||||||
|
) -> FileUploadResponse:
|
||||||
|
data = await file.read()
|
||||||
|
result = await upload_source_file(
|
||||||
|
sessionmaker=sessionmaker,
|
||||||
|
storage=storage,
|
||||||
|
point_storage=point_storage,
|
||||||
|
auth=auth,
|
||||||
|
domain=domain,
|
||||||
|
filename=file.filename or "",
|
||||||
|
data=data,
|
||||||
|
ingestion_settings=settings.ingestion,
|
||||||
|
chunking_settings=settings.chunking,
|
||||||
|
qdrant_settings=settings.qdrant,
|
||||||
|
thread_limiter=limiter,
|
||||||
|
concurrency_limiter=concurrency_limiter,
|
||||||
|
dense_embedders=dense_embedders,
|
||||||
|
sparse_embedder=sparse_embedder,
|
||||||
|
)
|
||||||
|
if not result.is_new_attempt:
|
||||||
|
response.status_code = status.HTTP_200_OK
|
||||||
|
return FileUploadResponse.from_result(result)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}")
|
||||||
|
async def get_file(
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
auth: _RequireFilesWrite,
|
||||||
|
sessionmaker: _SessionmakerDep,
|
||||||
|
) -> FileStatusResponse:
|
||||||
|
result = await get_file_status(sessionmaker, tenant_id=auth.tenant_id, source_file_id=file_id)
|
||||||
|
if result is None:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="file not found")
|
||||||
|
return FileStatusResponse.from_result(result)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{file_id}")
|
||||||
|
async def delete_file(
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
auth: _RequirePointsWrite,
|
||||||
|
sessionmaker: _SessionmakerDep,
|
||||||
|
repository: _PointRepositoryDep,
|
||||||
|
) -> FileDeleteResponse:
|
||||||
|
"""Soft-delete a file: every active point, then the `source_files` row.
|
||||||
|
|
||||||
|
Gated on `points:write` rather than `files:write` for the same reason as the
|
||||||
|
listing above — the data this destroys is points. Nothing is removed from
|
||||||
|
Qdrant (ADR-0002); the points are flagged inactive and the row is marked
|
||||||
|
`soft_deleted`, which is also what makes a later re-upload of the same bytes
|
||||||
|
ingest afresh instead of matching the duplicate path.
|
||||||
|
|
||||||
|
Deleting an already-deleted file is a success reporting `0` points.
|
||||||
|
"""
|
||||||
|
points_soft_deleted = await delete_source_file(
|
||||||
|
sessionmaker,
|
||||||
|
repository,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
source_file_id=file_id,
|
||||||
|
actor=f"api_key:{auth.api_key_id}",
|
||||||
|
)
|
||||||
|
return FileDeleteResponse(
|
||||||
|
file_id=file_id, status="soft_deleted", points_soft_deleted=points_soft_deleted
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{file_id}/points")
|
||||||
|
async def list_points_for_file(
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
auth: _RequirePointsRead,
|
||||||
|
repository: _PointRepositoryDep,
|
||||||
|
limit: LimitQuery = DEFAULT_PAGE_SIZE,
|
||||||
|
cursor: str | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointListResponse:
|
||||||
|
"""The same listing as `GET /v1/points?file_id=...`, addressed by file.
|
||||||
|
|
||||||
|
Gated on `points:read`, not `files:write`: the resource being read is the
|
||||||
|
file's chunks, so the scope follows the data rather than the URL prefix. An
|
||||||
|
upload-only key must not become a way to read every chunk of every file.
|
||||||
|
"""
|
||||||
|
page = await list_file_points(
|
||||||
|
repository,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
file_id=file_id,
|
||||||
|
limit=limit,
|
||||||
|
cursor=cursor,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
return PointListResponse.from_page(page)
|
||||||
@@ -23,7 +23,9 @@ async def readyz(request: Request, response: Response) -> dict[str, bool]:
|
|||||||
postgres_ready, minio_ready, qdrant_ready = await asyncio.gather(
|
postgres_ready, minio_ready, qdrant_ready = await asyncio.gather(
|
||||||
ping_postgres(resources.db_engine, timeout),
|
ping_postgres(resources.db_engine, timeout),
|
||||||
ping_minio(resources.minio_client, timeout),
|
ping_minio(resources.minio_client, timeout),
|
||||||
ping_qdrant(resources.qdrant_client, timeout),
|
ping_qdrant(
|
||||||
|
resources.qdrant_client, timeout, collection=resources.settings.qdrant.collection
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
|
|||||||
162
src/api/routers/points.py
Normal file
162
src/api/routers/points.py
Normal file
@@ -0,0 +1,162 @@
|
|||||||
|
"""`/v1/points` read and soft-delete paths (ADR-0002, ADR-0008).
|
||||||
|
|
||||||
|
Routes adapt HTTP to `application/points` calls. They build no Qdrant filters
|
||||||
|
and hold no CRUD semantics (ADR-0015), and they never read a tenant from the
|
||||||
|
request — `auth.tenant_id` is the only source, which is what makes ADR-0002's
|
||||||
|
isolation rule structural rather than a habit.
|
||||||
|
|
||||||
|
**Route order is load-bearing.** `/count` and `/search` are declared before
|
||||||
|
`/{point_id}`. FastAPI matches in declaration order, so with `/{point_id}` first
|
||||||
|
a request for `/v1/points/count` would try to parse `"count"` as a UUID and
|
||||||
|
fail with `422` instead of counting anything. The failure is loud but confusing,
|
||||||
|
and it comes back the moment someone reorders these for tidiness.
|
||||||
|
|
||||||
|
Gated on `points:read`, separately from `files:write`: a key that can upload
|
||||||
|
documents should not thereby be able to read every chunk of every file, and
|
||||||
|
plan 002's mutating paths will want `points:write` distinct again.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Query
|
||||||
|
|
||||||
|
from src.api.dependencies.auth import require_scope
|
||||||
|
from src.api.schemas.points import (
|
||||||
|
DEFAULT_PAGE_SIZE,
|
||||||
|
LimitQuery,
|
||||||
|
PointCountResponse,
|
||||||
|
PointListResponse,
|
||||||
|
PointResponse,
|
||||||
|
PointSearchResponse,
|
||||||
|
)
|
||||||
|
from src.application.auth.context import AuthContext
|
||||||
|
from src.application.points.deletion import soft_delete_point
|
||||||
|
from src.application.points.queries import (
|
||||||
|
count_points,
|
||||||
|
get_point,
|
||||||
|
list_file_points,
|
||||||
|
search_points,
|
||||||
|
)
|
||||||
|
from src.application.ports.point_repository import PointRepository
|
||||||
|
from src.bootstrap.dependencies import get_point_repository
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/points", tags=["points"])
|
||||||
|
|
||||||
|
_RequirePointsRead = Annotated[AuthContext, Depends(require_scope("points:read"))]
|
||||||
|
_RequirePointsWrite = Annotated[AuthContext, Depends(require_scope("points:write"))]
|
||||||
|
_PointRepositoryDep = Annotated[PointRepository, Depends(get_point_repository)]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/count")
|
||||||
|
async def count_tenant_points(
|
||||||
|
auth: _RequirePointsRead,
|
||||||
|
repository: _PointRepositoryDep,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointCountResponse:
|
||||||
|
count = await count_points(
|
||||||
|
repository,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
return PointCountResponse(count=count)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/search")
|
||||||
|
async def search_tenant_points(
|
||||||
|
auth: _RequirePointsRead,
|
||||||
|
repository: _PointRepositoryDep,
|
||||||
|
q: Annotated[str, Query(min_length=1)],
|
||||||
|
limit: LimitQuery = DEFAULT_PAGE_SIZE,
|
||||||
|
cursor: str | None = None,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointSearchResponse:
|
||||||
|
"""Keyword search over point content — **not** semantic retrieval.
|
||||||
|
|
||||||
|
Matches Qdrant's full-text payload index on `content`, combined with the
|
||||||
|
structured filters below. Results are unranked: the index filters rather
|
||||||
|
than scores, so there is no relevance order and no score to return. Callers
|
||||||
|
wanting ranked answers want the agent retrieval path (plan 003), not this.
|
||||||
|
"""
|
||||||
|
page = await search_points(
|
||||||
|
repository,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
query=q,
|
||||||
|
limit=limit,
|
||||||
|
cursor=cursor,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
return PointSearchResponse.from_search(page, query=q)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("")
|
||||||
|
async def list_tenant_points(
|
||||||
|
auth: _RequirePointsRead,
|
||||||
|
repository: _PointRepositoryDep,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
limit: LimitQuery = DEFAULT_PAGE_SIZE,
|
||||||
|
cursor: str | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointListResponse:
|
||||||
|
"""A file's points in `order_id` order.
|
||||||
|
|
||||||
|
`file_id` is required rather than optional: the pagination cursor is an
|
||||||
|
`order_id` value, and `order_id` is only unique within one file. Listing
|
||||||
|
across files would silently drop or repeat rows at every page boundary.
|
||||||
|
"""
|
||||||
|
page = await list_file_points(
|
||||||
|
repository,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
file_id=file_id,
|
||||||
|
limit=limit,
|
||||||
|
cursor=cursor,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
return PointListResponse.from_page(page)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{point_id}")
|
||||||
|
async def get_tenant_point(
|
||||||
|
point_id: uuid.UUID,
|
||||||
|
auth: _RequirePointsRead,
|
||||||
|
repository: _PointRepositoryDep,
|
||||||
|
with_vectors: bool = False,
|
||||||
|
) -> PointResponse:
|
||||||
|
point = await get_point(
|
||||||
|
repository, tenant_id=auth.tenant_id, point_id=point_id, with_vectors=with_vectors
|
||||||
|
)
|
||||||
|
return PointResponse.from_point(point)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/{point_id}")
|
||||||
|
async def delete_tenant_point(
|
||||||
|
point_id: uuid.UUID,
|
||||||
|
auth: _RequirePointsWrite,
|
||||||
|
repository: _PointRepositoryDep,
|
||||||
|
) -> PointResponse:
|
||||||
|
"""Soft-delete one point and relink its neighbours around the gap.
|
||||||
|
|
||||||
|
The point is never removed from Qdrant (ADR-0002): it is flagged
|
||||||
|
`is_active=false` with `deleted_at` set, and its old neighbours are pointed
|
||||||
|
at each other in the same batch, so context-window expansion never walks
|
||||||
|
into it.
|
||||||
|
|
||||||
|
Deleting an already-inactive point is a no-op success rather than a `404` —
|
||||||
|
the response is the point as it stands, so the resulting `version` and
|
||||||
|
`deleted_at` are visible either way.
|
||||||
|
"""
|
||||||
|
point = await soft_delete_point(
|
||||||
|
repository,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
point_id=point_id,
|
||||||
|
actor=f"api_key:{auth.api_key_id}",
|
||||||
|
)
|
||||||
|
return PointResponse.from_point(point)
|
||||||
63
src/api/schemas/domains.py
Normal file
63
src/api/schemas/domains.py
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
"""Public request/response models for `/v1/domains` (ADR-0008, ADR-0009).
|
||||||
|
|
||||||
|
`tenant_id` appears in none of these: it comes from the authenticated key, and
|
||||||
|
accepting it from a body would break the isolation boundary (ADR-0002).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field, field_validator
|
||||||
|
|
||||||
|
from src.application.domains.models import DomainResult
|
||||||
|
|
||||||
|
# Lowercase alphanumerics plus - and _; the key is embedded in every Qdrant
|
||||||
|
# payload and filtered on as a keyword, so it stays boring on purpose.
|
||||||
|
_DOMAIN_PATTERN = r"^[a-z0-9][a-z0-9_-]*$"
|
||||||
|
|
||||||
|
|
||||||
|
class DomainResponse(BaseModel):
|
||||||
|
id: uuid.UUID
|
||||||
|
domain: str
|
||||||
|
display_name: str
|
||||||
|
status: str
|
||||||
|
metadata: dict[str, object]
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_result(cls, result: DomainResult) -> "DomainResponse":
|
||||||
|
return cls(
|
||||||
|
id=result.id,
|
||||||
|
domain=result.domain,
|
||||||
|
display_name=result.display_name,
|
||||||
|
status=result.status,
|
||||||
|
metadata=result.metadata,
|
||||||
|
created_at=result.created_at,
|
||||||
|
updated_at=result.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DomainListResponse(BaseModel):
|
||||||
|
domains: list[DomainResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class CreateDomainRequest(BaseModel):
|
||||||
|
domain: str = Field(min_length=1, max_length=80, pattern=_DOMAIN_PATTERN)
|
||||||
|
display_name: str = Field(min_length=1, max_length=200)
|
||||||
|
metadata: dict[str, object] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
@field_validator("domain")
|
||||||
|
@classmethod
|
||||||
|
def _normalize(cls, value: str) -> str:
|
||||||
|
return value.strip()
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateDomainRequest(BaseModel):
|
||||||
|
"""`domain` is absent by design — the key is immutable.
|
||||||
|
|
||||||
|
It is denormalized into every point payload and into `source_files`, so
|
||||||
|
renaming it is a migration rather than an edit (ADR-0009).
|
||||||
|
"""
|
||||||
|
|
||||||
|
display_name: str = Field(min_length=1, max_length=200)
|
||||||
16
src/api/schemas/errors.py
Normal file
16
src/api/schemas/errors.py
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
"""The ADR-0008 error envelope."""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
|
||||||
|
class ErrorDetail(BaseModel):
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
details: dict[str, Any] = {}
|
||||||
|
request_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ErrorResponse(BaseModel):
|
||||||
|
error: ErrorDetail
|
||||||
64
src/api/schemas/files.py
Normal file
64
src/api/schemas/files.py
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
"""Public request/response models for `/v1/files` (ADR-0008).
|
||||||
|
|
||||||
|
Separate from the SQLAlchemy ORM models and the `application/files` domain
|
||||||
|
dataclasses (ADR-0015): this is the shape callers see.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from src.application.files.models import UploadResult
|
||||||
|
from src.application.files.status import FileStatusResult
|
||||||
|
|
||||||
|
|
||||||
|
class FileUploadResponse(BaseModel):
|
||||||
|
file_id: uuid.UUID
|
||||||
|
ingestion_job_id: uuid.UUID
|
||||||
|
status: str
|
||||||
|
chunks_indexed: int
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_result(cls, result: UploadResult) -> "FileUploadResponse":
|
||||||
|
return cls(
|
||||||
|
file_id=result.file_id,
|
||||||
|
ingestion_job_id=result.ingestion_job_id,
|
||||||
|
status=result.status,
|
||||||
|
chunks_indexed=result.chunks_indexed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FileDeleteResponse(BaseModel):
|
||||||
|
"""What `DELETE /v1/files/{file_id}` did.
|
||||||
|
|
||||||
|
`points_soft_deleted` is reported rather than left implicit because the
|
||||||
|
delete is a soft one: nothing is removed from Qdrant, and the count is the
|
||||||
|
only way a caller can tell "deactivated 40 points" from "the file was
|
||||||
|
already deleted" — both of which are successes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
file_id: uuid.UUID
|
||||||
|
status: str
|
||||||
|
points_soft_deleted: int
|
||||||
|
|
||||||
|
|
||||||
|
class FileStatusResponse(BaseModel):
|
||||||
|
file_id: uuid.UUID
|
||||||
|
source_filename: str
|
||||||
|
domain: str
|
||||||
|
status: str
|
||||||
|
ingestion_job_id: uuid.UUID | None
|
||||||
|
ingestion_status: str | None
|
||||||
|
chunks_indexed: int
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_result(cls, result: FileStatusResult) -> "FileStatusResponse":
|
||||||
|
return cls(
|
||||||
|
file_id=result.file_id,
|
||||||
|
source_filename=result.source_filename,
|
||||||
|
domain=result.domain,
|
||||||
|
status=result.status,
|
||||||
|
ingestion_job_id=result.ingestion_job_id,
|
||||||
|
ingestion_status=result.ingestion_status,
|
||||||
|
chunks_indexed=result.chunks_indexed,
|
||||||
|
)
|
||||||
165
src/api/schemas/points.py
Normal file
165
src/api/schemas/points.py
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
"""Public request/response models for `/v1/points` (ADR-0002, ADR-0008).
|
||||||
|
|
||||||
|
The shape callers see, kept separate from `application/points`' domain models
|
||||||
|
(ADR-0015). Two rules are encoded here rather than left to route code:
|
||||||
|
|
||||||
|
- **Vectors are opt-in.** `PointResponse` omits them unless the caller asked,
|
||||||
|
so a listing does not ship megabytes of floats nobody reads (ADR-0008).
|
||||||
|
- **Server-owned fields are not accepted on input.** The request models simply
|
||||||
|
do not declare `tenant_id`, `version`, or `chunk_index`, and forbid extra
|
||||||
|
keys, so a client that sends one gets `422` from Pydantic instead of having
|
||||||
|
it silently ignored — ADR-0002's isolation rule enforced at the boundary.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
from fastapi import Query
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
from src.application.points.point import Point
|
||||||
|
from src.application.ports.point_repository import PointPage
|
||||||
|
|
||||||
|
# A page ceiling the caller cannot raise. Scroll pages are materialized in
|
||||||
|
# memory both here and in Qdrant, so an unbounded `limit` is a cheap way for one
|
||||||
|
# request to hurt every other tenant sharing the process. Declared once because
|
||||||
|
# two routers paginate points -- `/v1/points` and `/v1/files/{file_id}/points` --
|
||||||
|
# and a ceiling that differs between them is a ceiling in only one of them.
|
||||||
|
DEFAULT_PAGE_SIZE = 50
|
||||||
|
MAX_PAGE_SIZE = 200
|
||||||
|
|
||||||
|
LimitQuery = Annotated[int, Query(ge=1, le=MAX_PAGE_SIZE)]
|
||||||
|
|
||||||
|
|
||||||
|
class PointResponse(BaseModel):
|
||||||
|
point_id: uuid.UUID
|
||||||
|
domain: str
|
||||||
|
file_id: uuid.UUID
|
||||||
|
chunk_id: uuid.UUID
|
||||||
|
|
||||||
|
content: str
|
||||||
|
content_type: str
|
||||||
|
source_filename: str
|
||||||
|
source_type: str
|
||||||
|
|
||||||
|
order_id: float
|
||||||
|
chunk_index: int
|
||||||
|
previous_chunk_id: uuid.UUID | None
|
||||||
|
next_chunk_id: uuid.UUID | None
|
||||||
|
|
||||||
|
is_active: bool
|
||||||
|
deleted_at: datetime | None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
created_by: str
|
||||||
|
updated_by: str
|
||||||
|
|
||||||
|
version: int
|
||||||
|
content_hash: str
|
||||||
|
embedding_model_version: str
|
||||||
|
|
||||||
|
vectors: dict[str, object] | None = None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_point(cls, point: Point) -> "PointResponse":
|
||||||
|
# `tenant_id` is present on `Point` and deliberately absent here: the
|
||||||
|
# caller already knows which tenant it authenticated as, and echoing it
|
||||||
|
# back invites clients to start sending it.
|
||||||
|
return cls.model_validate(point.model_dump(exclude={"tenant_id"}))
|
||||||
|
|
||||||
|
|
||||||
|
class PointListResponse(BaseModel):
|
||||||
|
"""A page of points plus the cursor for the next one.
|
||||||
|
|
||||||
|
Cursor-based rather than `limit`/`offset`: an offset cursor silently skips
|
||||||
|
or repeats rows when a concurrent insert shifts positions, which is exactly
|
||||||
|
the pagination defect plan 002 requires a test for.
|
||||||
|
"""
|
||||||
|
|
||||||
|
points: list[PointResponse]
|
||||||
|
next_cursor: str | None = None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_page(cls, page: PointPage) -> "PointListResponse":
|
||||||
|
return cls(
|
||||||
|
points=[PointResponse.from_point(point) for point in page.points],
|
||||||
|
next_cursor=page.next_cursor,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PointCountResponse(BaseModel):
|
||||||
|
count: int
|
||||||
|
|
||||||
|
|
||||||
|
class PointSearchResponse(PointListResponse):
|
||||||
|
"""Results of a **keyword** match, not of semantic retrieval.
|
||||||
|
|
||||||
|
Named and documented so it cannot be mistaken for ADR-0003's hybrid
|
||||||
|
retrieval: these points matched a full-text filter on `content`, they are
|
||||||
|
not ranked by relevance, and there is no score to report. Anything that
|
||||||
|
wants ranked results wants the agent retrieval path in plan 003.
|
||||||
|
"""
|
||||||
|
|
||||||
|
query: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_search(cls, page: PointPage, *, query: str) -> "PointSearchResponse":
|
||||||
|
return cls(
|
||||||
|
query=query,
|
||||||
|
points=[PointResponse.from_point(point) for point in page.points],
|
||||||
|
next_cursor=page.next_cursor,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PointCreateRequest(BaseModel):
|
||||||
|
"""Create one point. The server assigns identity, ordering, and provenance.
|
||||||
|
|
||||||
|
`after_point_id` positions the new point rather than a raw `order_id`: the
|
||||||
|
caller says where in the sequence it goes and the server computes the
|
||||||
|
fractional key and relinks neighbours, which a client-supplied `order_id`
|
||||||
|
could not do correctly (ADR-0002). `None` means "at the start of the file".
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
file_id: uuid.UUID
|
||||||
|
content: str = Field(min_length=1)
|
||||||
|
content_type: str = "paragraph"
|
||||||
|
after_point_id: uuid.UUID | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class PointReplaceRequest(BaseModel):
|
||||||
|
"""Replace a point's content under a version guard.
|
||||||
|
|
||||||
|
`version` here is the *expected* version, not a value being written — the
|
||||||
|
optimistic-concurrency precondition. A mismatch is `409`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
content: str = Field(min_length=1)
|
||||||
|
content_type: str | None = None
|
||||||
|
version: int
|
||||||
|
|
||||||
|
|
||||||
|
class PointPayloadPatchRequest(BaseModel):
|
||||||
|
"""Payload-only update of caller-writable fields, under a version guard."""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
payload: dict[str, object]
|
||||||
|
version: int
|
||||||
|
|
||||||
|
|
||||||
|
class PointReorderRequest(BaseModel):
|
||||||
|
"""Move a point to sit immediately after `after_point_id`.
|
||||||
|
|
||||||
|
`None` moves it to the front of the file. Expressed as a neighbour rather
|
||||||
|
than an `order_id` for the same reason as `PointCreateRequest`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
after_point_id: uuid.UUID | None = None
|
||||||
|
version: int
|
||||||
29
src/application/auth/__init__.py
Normal file
29
src/application/auth/__init__.py
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
"""API-key authentication and tenant resolution (ADR-0008).
|
||||||
|
|
||||||
|
`resolve_auth_context` is the entry point: it takes a bearer token and
|
||||||
|
returns a trusted `AuthContext`. Everything downstream of the FastAPI
|
||||||
|
boundary receives `tenant_id` only through that context — never from a
|
||||||
|
request body, query string, or object metadata.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from src.application.auth.context import AuthContext
|
||||||
|
from src.application.auth.errors import (
|
||||||
|
AuthError,
|
||||||
|
InvalidApiKeyError,
|
||||||
|
MissingScopeError,
|
||||||
|
TenantInactiveError,
|
||||||
|
)
|
||||||
|
from src.application.auth.keys import generate_api_key, hash_secret, verify_secret
|
||||||
|
from src.application.auth.service import resolve_auth_context
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"AuthContext",
|
||||||
|
"AuthError",
|
||||||
|
"InvalidApiKeyError",
|
||||||
|
"MissingScopeError",
|
||||||
|
"TenantInactiveError",
|
||||||
|
"generate_api_key",
|
||||||
|
"hash_secret",
|
||||||
|
"resolve_auth_context",
|
||||||
|
"verify_secret",
|
||||||
|
]
|
||||||
16
src/application/auth/context.py
Normal file
16
src/application/auth/context.py
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
"""The trusted request-scoped auth/tenant context (ADR-0008)."""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AuthContext:
|
||||||
|
tenant_id: uuid.UUID
|
||||||
|
tenant_slug: str
|
||||||
|
api_key_id: uuid.UUID
|
||||||
|
scopes: frozenset[str]
|
||||||
|
actor_type: str
|
||||||
|
|
||||||
|
def has_scope(self, scope: str) -> bool:
|
||||||
|
return scope in self.scopes or "admin" in self.scopes
|
||||||
22
src/application/auth/errors.py
Normal file
22
src/application/auth/errors.py
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
"""Auth failures (ADR-0008). No HTTP knowledge here — `src/api/errors.py` maps
|
||||||
|
these to status codes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class AuthError(Exception):
|
||||||
|
"""Base class for auth failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidApiKeyError(AuthError):
|
||||||
|
"""The bearer token is missing, malformed, unknown, revoked, or expired.
|
||||||
|
|
||||||
|
Maps to `401`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TenantInactiveError(AuthError):
|
||||||
|
"""The key's tenant is suspended or deleted. Maps to `401`."""
|
||||||
|
|
||||||
|
|
||||||
|
class MissingScopeError(AuthError):
|
||||||
|
"""The key is valid but lacks a scope the route requires. Maps to `403`."""
|
||||||
39
src/application/auth/keys.py
Normal file
39
src/application/auth/keys.py
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
"""API-key generation and hashing (ADR-0008, ADR-0009).
|
||||||
|
|
||||||
|
Keys are `sk_{prefix}_{secret}`. `prefix` is non-secret and indexed
|
||||||
|
(`api_keys.key_prefix`); `secret` is 256 bits of `secrets.token_urlsafe`
|
||||||
|
entropy, stored only as a SHA-256 hash. A random 256-bit secret does not
|
||||||
|
benefit from a slow password-hashing KDF the way a human-chosen password
|
||||||
|
does — the cost that defends against dictionary/brute-force guessing over a
|
||||||
|
low-entropy input has nothing to defend here, and would only tax every
|
||||||
|
request. Comparison is constant-time to avoid a hash-timing oracle.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
_PREFIX_LENGTH = 16
|
||||||
|
|
||||||
|
|
||||||
|
def generate_api_key() -> tuple[str, str, str]:
|
||||||
|
"""Return `(key_prefix, secret, full_key)` for a newly issued key."""
|
||||||
|
key_prefix = secrets.token_hex(_PREFIX_LENGTH // 2)
|
||||||
|
secret = secrets.token_urlsafe(32)
|
||||||
|
return key_prefix, secret, f"sk_{key_prefix}_{secret}"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_api_key(full_key: str) -> tuple[str, str] | None:
|
||||||
|
"""Return `(key_prefix, secret)`, or `None` if the token is malformed."""
|
||||||
|
parts = full_key.split("_", 2)
|
||||||
|
if len(parts) != 3 or parts[0] != "sk" or not parts[1] or not parts[2]:
|
||||||
|
return None
|
||||||
|
return parts[1], parts[2]
|
||||||
|
|
||||||
|
|
||||||
|
def hash_secret(secret: str) -> str:
|
||||||
|
return hashlib.sha256(secret.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def verify_secret(secret: str, key_hash: str) -> bool:
|
||||||
|
return hmac.compare_digest(hash_secret(secret), key_hash)
|
||||||
86
src/application/auth/service.py
Normal file
86
src/application/auth/service.py
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
"""Resolve a bearer token to a trusted `AuthContext` (ADR-0008).
|
||||||
|
|
||||||
|
This opens and releases its own session rather than borrowing a
|
||||||
|
request-scoped one, so auth resolution never pins a pool connection across
|
||||||
|
the rest of the request — including the ADR-0017 ingestion work phase, which
|
||||||
|
must run with no Postgres session held open at all.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.application.auth.context import AuthContext
|
||||||
|
from src.application.auth.errors import InvalidApiKeyError, TenantInactiveError
|
||||||
|
from src.application.auth.keys import parse_api_key, verify_secret
|
||||||
|
from src.infrastructure.postgres.repositories import api_keys as api_keys_repo
|
||||||
|
from src.infrastructure.postgres.repositories import tenants as tenants_repo
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def resolve_auth_context(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession], bearer_token: str
|
||||||
|
) -> AuthContext:
|
||||||
|
"""Resolve a bearer token, logging the outcome either way (ADR-0011).
|
||||||
|
|
||||||
|
This runs on every authenticated request, so `auth.failed` is the one
|
||||||
|
event most likely to matter first when diagnosing a client integration
|
||||||
|
issue -- and the reason string alone (never logged; it can echo back
|
||||||
|
attacker-supplied key material) is not enough to tell a malformed token
|
||||||
|
apart from a revoked one without this.
|
||||||
|
"""
|
||||||
|
parsed = parse_api_key(bearer_token)
|
||||||
|
if parsed is None:
|
||||||
|
logger.warning("auth.failed", reason="malformed_key")
|
||||||
|
raise InvalidApiKeyError("malformed API key")
|
||||||
|
key_prefix, secret = parsed
|
||||||
|
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
api_key = await api_keys_repo.get_by_prefix(session, key_prefix)
|
||||||
|
if api_key is None or not verify_secret(secret, api_key.key_hash):
|
||||||
|
logger.warning("auth.failed", reason="unknown_key", key_prefix=key_prefix)
|
||||||
|
raise InvalidApiKeyError("unknown API key")
|
||||||
|
if api_key.status != "active":
|
||||||
|
logger.warning(
|
||||||
|
"auth.failed",
|
||||||
|
reason="key_inactive",
|
||||||
|
key_prefix=key_prefix,
|
||||||
|
api_key_id=str(api_key.id),
|
||||||
|
key_status=api_key.status,
|
||||||
|
)
|
||||||
|
raise InvalidApiKeyError(f"API key is {api_key.status}")
|
||||||
|
if api_key.expires_at is not None and api_key.expires_at <= datetime.now(UTC):
|
||||||
|
logger.warning(
|
||||||
|
"auth.failed",
|
||||||
|
reason="key_expired",
|
||||||
|
key_prefix=key_prefix,
|
||||||
|
api_key_id=str(api_key.id),
|
||||||
|
)
|
||||||
|
raise InvalidApiKeyError("API key has expired")
|
||||||
|
|
||||||
|
tenant = await tenants_repo.get_by_id(session, api_key.tenant_id)
|
||||||
|
if tenant is None or tenant.status != "active":
|
||||||
|
logger.warning(
|
||||||
|
"auth.failed",
|
||||||
|
reason="tenant_inactive",
|
||||||
|
key_prefix=key_prefix,
|
||||||
|
api_key_id=str(api_key.id),
|
||||||
|
tenant_id=str(api_key.tenant_id),
|
||||||
|
)
|
||||||
|
raise TenantInactiveError("tenant is not active")
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"auth.succeeded",
|
||||||
|
tenant_id=str(tenant.id),
|
||||||
|
api_key_id=str(api_key.id),
|
||||||
|
actor_type=api_key.actor_type,
|
||||||
|
)
|
||||||
|
return AuthContext(
|
||||||
|
tenant_id=tenant.id,
|
||||||
|
tenant_slug=tenant.slug,
|
||||||
|
api_key_id=api_key.id,
|
||||||
|
scopes=frozenset(api_key.scopes),
|
||||||
|
actor_type=api_key.actor_type,
|
||||||
|
)
|
||||||
27
src/application/domains/__init__.py
Normal file
27
src/application/domains/__init__.py
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
"""Tenant-domain management and the upload-time allowlist check (ADR-0009)."""
|
||||||
|
|
||||||
|
from src.application.domains.errors import (
|
||||||
|
DomainAlreadyExistsError,
|
||||||
|
DomainsError,
|
||||||
|
UnknownDomainError,
|
||||||
|
)
|
||||||
|
from src.application.domains.models import DomainResult
|
||||||
|
from src.application.domains.service import (
|
||||||
|
create_domain,
|
||||||
|
ensure_domain_allowed,
|
||||||
|
list_domains,
|
||||||
|
set_domain_status,
|
||||||
|
update_domain,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"DomainAlreadyExistsError",
|
||||||
|
"DomainResult",
|
||||||
|
"DomainsError",
|
||||||
|
"UnknownDomainError",
|
||||||
|
"create_domain",
|
||||||
|
"ensure_domain_allowed",
|
||||||
|
"list_domains",
|
||||||
|
"set_domain_status",
|
||||||
|
"update_domain",
|
||||||
|
]
|
||||||
21
src/application/domains/errors.py
Normal file
21
src/application/domains/errors.py
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
"""Domain-management failures (ADR-0009). No HTTP knowledge here —
|
||||||
|
`src/api/errors.py` maps these to status codes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class DomainsError(Exception):
|
||||||
|
"""Base class for tenant-domain failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class UnknownDomainError(DomainsError):
|
||||||
|
"""The upload named a domain the tenant has not registered, or one that is
|
||||||
|
disabled. Maps to `400`.
|
||||||
|
|
||||||
|
Rejecting is the whole point: an unrecognized `domain` would otherwise
|
||||||
|
create a new Qdrant partition silently, and a file in a partition nothing
|
||||||
|
queries is invisible rather than failed (ADR-0009).
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class DomainAlreadyExistsError(DomainsError):
|
||||||
|
"""The tenant already has a domain with this key. Maps to `409`."""
|
||||||
16
src/application/domains/models.py
Normal file
16
src/application/domains/models.py
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
"""Transport-agnostic results for the domain-management service."""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DomainResult:
|
||||||
|
id: uuid.UUID
|
||||||
|
domain: str
|
||||||
|
display_name: str
|
||||||
|
status: str
|
||||||
|
metadata: dict[str, object]
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
164
src/application/domains/service.py
Normal file
164
src/application/domains/service.py
Normal file
@@ -0,0 +1,164 @@
|
|||||||
|
"""Tenant-domain management (ADR-0009).
|
||||||
|
|
||||||
|
A tenant's domain set is per-tenant and varies in size — one may run 14
|
||||||
|
insurance lines, another 6 — so it is data, not an enum.
|
||||||
|
|
||||||
|
`ensure_domain_allowed` is the reason this package exists: it is the strict
|
||||||
|
allowlist check the upload path runs before anything is written. Everything
|
||||||
|
else here is the management surface the calling backend uses to populate that
|
||||||
|
allowlist, under its own `domains:write` scope so an upload key cannot create
|
||||||
|
partitions.
|
||||||
|
|
||||||
|
`tenant_id` is always a required parameter taken from `AuthContext`, never from
|
||||||
|
a request body (ADR-0002).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.application.domains.errors import DomainAlreadyExistsError, UnknownDomainError
|
||||||
|
from src.application.domains.models import DomainResult
|
||||||
|
from src.infrastructure.postgres.models.tenant_domain import TenantDomain
|
||||||
|
from src.infrastructure.postgres.repositories import tenant_domains as repo
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def _flush_and_refresh(session: AsyncSession, tenant_domain: TenantDomain) -> None:
|
||||||
|
"""Materialize server-generated columns before the row leaves the session.
|
||||||
|
|
||||||
|
`updated_at` is `onupdate=func.now()`, so after an UPDATE its value lives in
|
||||||
|
the database, not in the instance. Reading it later would trigger a lazy
|
||||||
|
load outside any greenlet context (`MissingGreenlet`), so it is fetched here
|
||||||
|
while the session is still open.
|
||||||
|
"""
|
||||||
|
await session.flush()
|
||||||
|
await session.refresh(tenant_domain)
|
||||||
|
|
||||||
|
|
||||||
|
def _to_result(tenant_domain: TenantDomain) -> DomainResult:
|
||||||
|
return DomainResult(
|
||||||
|
id=tenant_domain.id,
|
||||||
|
domain=tenant_domain.domain,
|
||||||
|
display_name=tenant_domain.display_name,
|
||||||
|
status=tenant_domain.status,
|
||||||
|
metadata=tenant_domain.metadata_,
|
||||||
|
created_at=tenant_domain.created_at,
|
||||||
|
updated_at=tenant_domain.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def ensure_domain_allowed(
|
||||||
|
session: AsyncSession, *, tenant_id: uuid.UUID, domain: str
|
||||||
|
) -> None:
|
||||||
|
"""Raise `UnknownDomainError` unless the tenant has this domain active.
|
||||||
|
|
||||||
|
Takes a session rather than a sessionmaker: the upload path calls this
|
||||||
|
inside its existing txn A, so the check costs no extra connection and
|
||||||
|
cannot pass and then go stale before the row is written.
|
||||||
|
|
||||||
|
Logs the rejection here rather than at the call site: this runs before any
|
||||||
|
`ingestion_jobs` row exists, so `upload_source_file`'s job-level
|
||||||
|
`ingestion.job.failed` event (ADR-0011) never fires for it -- without a log
|
||||||
|
here, a rejected upload would leave no operational trace at all.
|
||||||
|
"""
|
||||||
|
tenant_domain = await repo.get(session, tenant_id=tenant_id, domain=domain)
|
||||||
|
if tenant_domain is None:
|
||||||
|
logger.warning(
|
||||||
|
"domain.rejected", tenant_id=str(tenant_id), domain=domain, reason="unregistered"
|
||||||
|
)
|
||||||
|
raise UnknownDomainError(
|
||||||
|
f"domain '{domain}' is not registered for this tenant; "
|
||||||
|
f"create it via POST /v1/domains before uploading to it"
|
||||||
|
)
|
||||||
|
if tenant_domain.status != "active":
|
||||||
|
logger.warning(
|
||||||
|
"domain.rejected", tenant_id=str(tenant_id), domain=domain, reason="disabled"
|
||||||
|
)
|
||||||
|
raise UnknownDomainError(f"domain '{domain}' is disabled for this tenant")
|
||||||
|
|
||||||
|
|
||||||
|
async def list_domains(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
include_disabled: bool = False,
|
||||||
|
) -> list[DomainResult]:
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
found = await repo.list_for_tenant(
|
||||||
|
session, tenant_id=tenant_id, include_disabled=include_disabled
|
||||||
|
)
|
||||||
|
return [_to_result(item) for item in found]
|
||||||
|
|
||||||
|
|
||||||
|
async def create_domain(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str,
|
||||||
|
display_name: str,
|
||||||
|
metadata: dict[str, object] | None = None,
|
||||||
|
) -> DomainResult:
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
if await repo.get(session, tenant_id=tenant_id, domain=domain) is not None:
|
||||||
|
raise DomainAlreadyExistsError(f"domain '{domain}' already exists for this tenant")
|
||||||
|
created = repo.create(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
display_name=display_name,
|
||||||
|
metadata=metadata,
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
logger.info("domain.created", tenant_id=str(tenant_id), domain=domain)
|
||||||
|
return _to_result(created)
|
||||||
|
|
||||||
|
|
||||||
|
async def update_domain(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str,
|
||||||
|
display_name: str,
|
||||||
|
) -> DomainResult:
|
||||||
|
"""Only the label is mutable — see `repo.update_display_name`."""
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
found = await repo.get(session, tenant_id=tenant_id, domain=domain)
|
||||||
|
if found is None:
|
||||||
|
raise UnknownDomainError(f"domain '{domain}' is not registered for this tenant")
|
||||||
|
repo.update_display_name(found, display_name=display_name)
|
||||||
|
await _flush_and_refresh(session, found)
|
||||||
|
await session.commit()
|
||||||
|
result = _to_result(found)
|
||||||
|
|
||||||
|
logger.info("domain.updated", tenant_id=str(tenant_id), domain=domain)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def set_domain_status(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str,
|
||||||
|
status: str,
|
||||||
|
) -> DomainResult:
|
||||||
|
"""Disable or re-enable a domain.
|
||||||
|
|
||||||
|
Disabling blocks new uploads and hides the domain from pickers. It does not
|
||||||
|
touch the points already indexed under it — removing those needs the
|
||||||
|
tenant-erasure workflow plan 001 defers.
|
||||||
|
"""
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
found = await repo.get(session, tenant_id=tenant_id, domain=domain)
|
||||||
|
if found is None:
|
||||||
|
raise UnknownDomainError(f"domain '{domain}' is not registered for this tenant")
|
||||||
|
repo.set_status(found, status=status)
|
||||||
|
await _flush_and_refresh(session, found)
|
||||||
|
await session.commit()
|
||||||
|
result = _to_result(found)
|
||||||
|
|
||||||
|
logger.info("domain.status_changed", tenant_id=str(tenant_id), domain=domain, status=status)
|
||||||
|
return result
|
||||||
19
src/application/files/__init__.py
Normal file
19
src/application/files/__init__.py
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
"""Source-file upload and status use cases (ADR-0008, ADR-0009, ADR-0017)."""
|
||||||
|
|
||||||
|
from src.application.files.errors import FilesError, FileTooLargeError, InvalidUploadError
|
||||||
|
from src.application.files.models import UploadResult, ValidatedUpload
|
||||||
|
from src.application.files.status import FileStatusResult, get_file_status
|
||||||
|
from src.application.files.upload import upload_source_file
|
||||||
|
from src.application.files.validation import validate_and_hash_upload
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FileStatusResult",
|
||||||
|
"FileTooLargeError",
|
||||||
|
"FilesError",
|
||||||
|
"InvalidUploadError",
|
||||||
|
"UploadResult",
|
||||||
|
"ValidatedUpload",
|
||||||
|
"get_file_status",
|
||||||
|
"upload_source_file",
|
||||||
|
"validate_and_hash_upload",
|
||||||
|
]
|
||||||
81
src/application/files/deletion.py
Normal file
81
src/application/files/deletion.py
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
"""`DELETE /v1/files/{file_id}` — retire a file and deactivate its points.
|
||||||
|
|
||||||
|
Two stores have to agree here, and the phase boundaries are the same ones
|
||||||
|
ingestion uses (ADR-0017): a short Postgres transaction to authorize, then the
|
||||||
|
Qdrant work with **no session held**, then a short transaction to record the
|
||||||
|
outcome. Holding a session across the sweep would pin a pool connection for the
|
||||||
|
length of a multi-page delete.
|
||||||
|
|
||||||
|
The order — points first, Postgres second — is deliberate. If the sweep dies
|
||||||
|
half way, the row stays `active` and a retried `DELETE` finishes the job, since
|
||||||
|
the sweep only ever looks at points that are still active. The reverse order
|
||||||
|
would leave a row marked deleted while its points are still live and still
|
||||||
|
retrievable by the agent, which is the failure that actually matters.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from time import perf_counter
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.application.files.errors import SourceFileNotFoundError
|
||||||
|
from src.application.points.deletion import soft_delete_file_points
|
||||||
|
from src.application.ports.point_repository import PointRepository
|
||||||
|
from src.infrastructure.postgres.repositories import source_files as source_files_repo
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def delete_source_file(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
repository: PointRepository,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
source_file_id: uuid.UUID,
|
||||||
|
actor: str,
|
||||||
|
) -> int:
|
||||||
|
"""Soft-delete a file: every active point, then the `source_files` row.
|
||||||
|
|
||||||
|
Returns how many points the sweep deactivated. Raises
|
||||||
|
`SourceFileNotFoundError` (`404`) when the file is not this tenant's — the
|
||||||
|
check happens before anything is written, so a probe for another tenant's
|
||||||
|
file id cannot deactivate a single point.
|
||||||
|
|
||||||
|
Idempotent: a second call finds no active points and a row already marked
|
||||||
|
`soft_deleted`, and returns `0`.
|
||||||
|
"""
|
||||||
|
started = perf_counter()
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
source_file = await source_files_repo.get_by_id(
|
||||||
|
session, tenant_id=tenant_id, source_file_id=source_file_id
|
||||||
|
)
|
||||||
|
if source_file is None:
|
||||||
|
raise SourceFileNotFoundError(f"file {source_file_id} not found")
|
||||||
|
|
||||||
|
points_soft_deleted = await soft_delete_file_points(
|
||||||
|
repository, tenant_id=tenant_id, file_id=source_file_id, actor=actor
|
||||||
|
)
|
||||||
|
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
source_file = await source_files_repo.get_by_id(
|
||||||
|
session, tenant_id=tenant_id, source_file_id=source_file_id
|
||||||
|
)
|
||||||
|
if source_file is None:
|
||||||
|
raise SourceFileNotFoundError(f"file {source_file_id} not found")
|
||||||
|
source_files_repo.mark_soft_deleted(source_file, deleted_at=datetime.now(UTC))
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"files.soft_deleted",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
file_id=str(source_file_id),
|
||||||
|
points_soft_deleted=points_soft_deleted,
|
||||||
|
actor=actor,
|
||||||
|
# End to end, including both Postgres transactions. Comparing it with
|
||||||
|
# the sweep's own `duration_ms` on `points.file_soft_deleted` is what
|
||||||
|
# separates a slow Qdrant from a slow database.
|
||||||
|
duration_ms=round((perf_counter() - started) * 1000, 2),
|
||||||
|
)
|
||||||
|
return points_soft_deleted
|
||||||
25
src/application/files/errors.py
Normal file
25
src/application/files/errors.py
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
"""Upload-validation failures (ADR-0008). No HTTP knowledge here —
|
||||||
|
`src/api/errors.py` maps these to status codes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class FilesError(Exception):
|
||||||
|
"""Base class for file-upload failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidUploadError(FilesError):
|
||||||
|
"""Missing domain, empty file, or content that doesn't match its
|
||||||
|
declared extension. Maps to `400`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class FileTooLargeError(FilesError):
|
||||||
|
"""The upload exceeds `INGESTION_MAX_UPLOAD_SIZE_MB`. Maps to `413`."""
|
||||||
|
|
||||||
|
|
||||||
|
class SourceFileNotFoundError(FilesError):
|
||||||
|
"""No such source file *within the requesting tenant*. Maps to `404`.
|
||||||
|
|
||||||
|
Same non-disclosure rule as points (ADR-0016): a cross-tenant file id and a
|
||||||
|
nonexistent one are indistinguishable to the caller, so this is never `403`.
|
||||||
|
"""
|
||||||
26
src/application/files/models.py
Normal file
26
src/application/files/models.py
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
"""Domain models for the upload use case (ADR-0008, ADR-0009)."""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ValidatedUpload:
|
||||||
|
"""The result of extension/content validation, before any I/O."""
|
||||||
|
|
||||||
|
source_type: str
|
||||||
|
content_type: str
|
||||||
|
content_sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UploadResult:
|
||||||
|
"""What `upload_source_file` returns; the route maps this to `FileUploadResponse`."""
|
||||||
|
|
||||||
|
file_id: uuid.UUID
|
||||||
|
ingestion_job_id: uuid.UUID
|
||||||
|
status: str
|
||||||
|
chunks_indexed: int
|
||||||
|
is_new_attempt: bool
|
||||||
|
"""`False` when an identical active upload already succeeded and no new
|
||||||
|
ingestion attempt was made (route returns `200`, not `201`)."""
|
||||||
52
src/application/files/status.py
Normal file
52
src/application/files/status.py
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
"""`GET /v1/files/{file_id}` read model (ADR-0008, ADR-0009)."""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.infrastructure.postgres.repositories import ingestion_jobs as jobs_repo
|
||||||
|
from src.infrastructure.postgres.repositories import source_files as source_files_repo
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FileStatusResult:
|
||||||
|
file_id: uuid.UUID
|
||||||
|
source_filename: str
|
||||||
|
domain: str
|
||||||
|
status: str
|
||||||
|
ingestion_job_id: uuid.UUID | None
|
||||||
|
ingestion_status: str | None
|
||||||
|
chunks_indexed: int
|
||||||
|
|
||||||
|
|
||||||
|
async def get_file_status(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
source_file_id: uuid.UUID,
|
||||||
|
) -> FileStatusResult | None:
|
||||||
|
"""Returns `None` when the file doesn't exist under this tenant — the
|
||||||
|
route maps that to `404`, never `403` (ADR-0016: cross-tenant access
|
||||||
|
returns 404).
|
||||||
|
"""
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
source_file = await source_files_repo.get_by_id(
|
||||||
|
session, tenant_id=tenant_id, source_file_id=source_file_id
|
||||||
|
)
|
||||||
|
if source_file is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
latest_job = await jobs_repo.get_latest_for_source_file(
|
||||||
|
session, tenant_id=tenant_id, source_file_id=source_file.id
|
||||||
|
)
|
||||||
|
|
||||||
|
return FileStatusResult(
|
||||||
|
file_id=source_file.id,
|
||||||
|
source_filename=source_file.source_filename,
|
||||||
|
domain=source_file.domain,
|
||||||
|
status=source_file.status,
|
||||||
|
ingestion_job_id=latest_job.id if latest_job else None,
|
||||||
|
ingestion_status=latest_job.status if latest_job else None,
|
||||||
|
chunks_indexed=latest_job.points_created if latest_job else 0,
|
||||||
|
)
|
||||||
11
src/application/files/storage_keys.py
Normal file
11
src/application/files/storage_keys.py
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
"""Object-storage key derivation (ADR-0013).
|
||||||
|
|
||||||
|
Object keys are internal identifiers, never the caller-supplied filename.
|
||||||
|
Pure and synchronous.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
|
def source_file_object_key(tenant_id: uuid.UUID, source_file_id: uuid.UUID) -> str:
|
||||||
|
return f"tenants/{tenant_id}/source-files/{source_file_id}/original"
|
||||||
366
src/application/files/upload.py
Normal file
366
src/application/files/upload.py
Normal file
@@ -0,0 +1,366 @@
|
|||||||
|
"""`POST /v1/files` orchestration: the ADR-0017 three-phase upload.
|
||||||
|
|
||||||
|
This service owns two separate short-lived sessions/transactions rather than
|
||||||
|
one request-scoped session, because the request is two units of work
|
||||||
|
(ADR-0012, ADR-0017):
|
||||||
|
|
||||||
|
txn A (short): source_files [+ ingestion_jobs(status='running')], commit
|
||||||
|
no txn: store bytes in MinIO, parse/chunk (threads),
|
||||||
|
embed dense+sparse (bounded/batched)
|
||||||
|
txn B (short): ingestion_jobs -> succeeded/failed, append event, commit
|
||||||
|
|
||||||
|
No Postgres session is open during phase 2. A failure at any point between
|
||||||
|
txn A and txn B still leaves a durable, inspectable `failed` job — never a
|
||||||
|
job stuck in `running`. The whole request additionally holds one of
|
||||||
|
`INGESTION_MAX_CONCURRENCY` process-wide slots (`503` when exhausted) and
|
||||||
|
phase 2 is bounded by `INGESTION_TIMEOUT_SECONDS` (`504`) (ADR-0017, plan 001
|
||||||
|
Phase 4).
|
||||||
|
|
||||||
|
Phase 2 ends by upserting the embedded chunks as tenant-scoped Qdrant points
|
||||||
|
(`src/application/points/`), so a successful upload is searchable by the time
|
||||||
|
the `201` returns. The collection those points land in is provisioned by a
|
||||||
|
deployment step, not by this path — see `src/cli/qdrant_bootstrap.py`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from anyio import CapacityLimiter, Semaphore, fail_after, to_thread
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.application.auth.context import AuthContext
|
||||||
|
from src.application.domains import ensure_domain_allowed
|
||||||
|
from src.application.files.errors import InvalidUploadError
|
||||||
|
from src.application.files.models import UploadResult
|
||||||
|
from src.application.files.storage_keys import source_file_object_key
|
||||||
|
from src.application.files.validation import validate_and_hash_upload
|
||||||
|
from src.application.ingestion import (
|
||||||
|
ChunkTooLargeError,
|
||||||
|
DocumentParseError,
|
||||||
|
UnsupportedSourceTypeError,
|
||||||
|
parse_and_chunk_document,
|
||||||
|
)
|
||||||
|
from src.application.ingestion.bounds import acquire_ingestion_slot, enforce_chunk_limit
|
||||||
|
from src.application.ingestion.embedding import embed_chunks
|
||||||
|
from src.application.ingestion.errors import (
|
||||||
|
ChunkLimitExceededError,
|
||||||
|
EmbedderError,
|
||||||
|
IngestionTimeoutError,
|
||||||
|
PointIndexingError,
|
||||||
|
)
|
||||||
|
from src.application.points import index_chunks
|
||||||
|
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||||
|
from src.application.ports.object_storage import ObjectStorage
|
||||||
|
from src.application.ports.point_storage import PointStorage
|
||||||
|
from src.config import ChunkingSettings, IngestionSettings, QdrantSettings
|
||||||
|
from src.infrastructure.postgres.repositories import ingestion_jobs as jobs_repo
|
||||||
|
from src.infrastructure.postgres.repositories import source_files as source_files_repo
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def _mark_job_failed(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
ingestion_job_id: uuid.UUID,
|
||||||
|
error_code: str,
|
||||||
|
error_message: str,
|
||||||
|
) -> None:
|
||||||
|
"""Write the terminal `failed` job row and emit its log event together.
|
||||||
|
|
||||||
|
Every failure branch below calls this, so logging here once closes every
|
||||||
|
branch at once rather than duplicating a `logger.warning` at each call
|
||||||
|
site (CLAUDE.md, "prefer deep modules") -- previously only
|
||||||
|
`storage_upload_failed` and `timeout` did that ad hoc, and
|
||||||
|
`parse_failed`/`chunk_limit_exceeded`/`embedding_failed`/`index_failed`
|
||||||
|
logged nothing at all: visible in `ingestion_job_events` but invisible to
|
||||||
|
log-based alerting (ADR-0011).
|
||||||
|
"""
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
job = await jobs_repo.mark_terminal(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
status="failed",
|
||||||
|
error_code=error_code,
|
||||||
|
error_message=error_message,
|
||||||
|
)
|
||||||
|
if job is not None:
|
||||||
|
jobs_repo.append_event(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
level="error",
|
||||||
|
stage="received",
|
||||||
|
message=error_message,
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
logger.warning(
|
||||||
|
"ingestion.job.failed",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
ingestion_job_id=str(ingestion_job_id),
|
||||||
|
error_code=error_code,
|
||||||
|
error_message=error_message,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def upload_source_file(
|
||||||
|
*,
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
storage: ObjectStorage,
|
||||||
|
point_storage: PointStorage,
|
||||||
|
auth: AuthContext,
|
||||||
|
domain: str,
|
||||||
|
filename: str,
|
||||||
|
data: bytes,
|
||||||
|
ingestion_settings: IngestionSettings,
|
||||||
|
chunking_settings: ChunkingSettings,
|
||||||
|
qdrant_settings: QdrantSettings,
|
||||||
|
thread_limiter: CapacityLimiter,
|
||||||
|
concurrency_limiter: Semaphore,
|
||||||
|
dense_embedders: Sequence[DenseEmbedder],
|
||||||
|
sparse_embedder: SparseEmbedder,
|
||||||
|
) -> UploadResult:
|
||||||
|
domain = domain.strip()
|
||||||
|
if not domain:
|
||||||
|
raise InvalidUploadError("domain is required")
|
||||||
|
|
||||||
|
validated = await to_thread.run_sync(
|
||||||
|
lambda: validate_and_hash_upload(
|
||||||
|
filename=filename, data=data, max_size_bytes=ingestion_settings.max_upload_size_bytes
|
||||||
|
),
|
||||||
|
limiter=thread_limiter,
|
||||||
|
)
|
||||||
|
|
||||||
|
async with acquire_ingestion_slot(concurrency_limiter):
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
# Strict allowlist, checked inside txn A before anything is written
|
||||||
|
# (ADR-0009). An unregistered domain would otherwise create a new
|
||||||
|
# Qdrant partition silently, leaving the file invisible to
|
||||||
|
# retrieval rather than failing.
|
||||||
|
await ensure_domain_allowed(session, tenant_id=auth.tenant_id, domain=domain)
|
||||||
|
|
||||||
|
existing = await source_files_repo.find_active_by_content_hash(
|
||||||
|
session,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
content_sha256=validated.content_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
if existing is not None:
|
||||||
|
latest_job = await jobs_repo.get_latest_for_source_file(
|
||||||
|
session, tenant_id=auth.tenant_id, source_file_id=existing.id
|
||||||
|
)
|
||||||
|
if latest_job is not None and latest_job.status == "succeeded":
|
||||||
|
logger.info(
|
||||||
|
"files.upload.duplicate",
|
||||||
|
tenant_id=str(auth.tenant_id),
|
||||||
|
file_id=str(existing.id),
|
||||||
|
)
|
||||||
|
return UploadResult(
|
||||||
|
file_id=existing.id,
|
||||||
|
ingestion_job_id=latest_job.id,
|
||||||
|
status=latest_job.status,
|
||||||
|
chunks_indexed=latest_job.points_created,
|
||||||
|
is_new_attempt=False,
|
||||||
|
)
|
||||||
|
source_file_id = existing.id
|
||||||
|
object_key = existing.storage_uri or source_file_object_key(
|
||||||
|
auth.tenant_id, source_file_id
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
source_file_id = uuid.uuid4()
|
||||||
|
object_key = source_file_object_key(auth.tenant_id, source_file_id)
|
||||||
|
source_files_repo.create(
|
||||||
|
session,
|
||||||
|
source_file_id=source_file_id,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
source_filename=filename,
|
||||||
|
source_type=validated.source_type,
|
||||||
|
content_sha256=validated.content_sha256,
|
||||||
|
byte_size=len(data),
|
||||||
|
storage_uri=object_key,
|
||||||
|
created_by_api_key_id=auth.api_key_id,
|
||||||
|
)
|
||||||
|
# `ingestion_jobs.source_file_id` FKs to this row; flush so the
|
||||||
|
# insert below sees it, since the two mapped classes carry no
|
||||||
|
# ORM relationship for the unit of work to order by itself.
|
||||||
|
await session.flush()
|
||||||
|
|
||||||
|
job = jobs_repo.create_running(
|
||||||
|
session,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
source_file_id=source_file_id,
|
||||||
|
requested_by_api_key_id=auth.api_key_id,
|
||||||
|
chunking_strategy=chunking_settings.strategy,
|
||||||
|
)
|
||||||
|
jobs_repo.append_event(
|
||||||
|
session,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=job.id,
|
||||||
|
level="info",
|
||||||
|
stage="received",
|
||||||
|
message="upload accepted, storing object",
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
ingestion_job_id = job.id
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"ingestion.job.started",
|
||||||
|
tenant_id=str(auth.tenant_id),
|
||||||
|
ingestion_job_id=str(ingestion_job_id),
|
||||||
|
file_id=str(source_file_id),
|
||||||
|
domain=domain,
|
||||||
|
source_type=validated.source_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 2: no Postgres session open across this work (ADR-0017),
|
||||||
|
# bounded end-to-end by INGESTION_TIMEOUT_SECONDS.
|
||||||
|
try:
|
||||||
|
with fail_after(ingestion_settings.timeout_seconds):
|
||||||
|
try:
|
||||||
|
await storage.put_object(
|
||||||
|
key=object_key, data=data, content_type=validated.content_type
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
await _mark_job_failed(
|
||||||
|
sessionmaker,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
error_code="storage_upload_failed",
|
||||||
|
error_message=f"failed to store object: {exc}",
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
try:
|
||||||
|
chunks = await parse_and_chunk_document(
|
||||||
|
data,
|
||||||
|
source_type=validated.source_type,
|
||||||
|
file_id=source_file_id,
|
||||||
|
settings=chunking_settings,
|
||||||
|
limiter=thread_limiter,
|
||||||
|
)
|
||||||
|
except (DocumentParseError, UnsupportedSourceTypeError, ChunkTooLargeError) as exc:
|
||||||
|
await _mark_job_failed(
|
||||||
|
sessionmaker,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
error_code="parse_failed",
|
||||||
|
error_message=str(exc),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
try:
|
||||||
|
enforce_chunk_limit(chunks, max_chunks=ingestion_settings.max_chunks_per_file)
|
||||||
|
except ChunkLimitExceededError as exc:
|
||||||
|
await _mark_job_failed(
|
||||||
|
sessionmaker,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
error_code="chunk_limit_exceeded",
|
||||||
|
error_message=str(exc),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
try:
|
||||||
|
embedded = await embed_chunks(
|
||||||
|
chunks,
|
||||||
|
dense_embedders=dense_embedders,
|
||||||
|
sparse_embedder=sparse_embedder,
|
||||||
|
settings=ingestion_settings,
|
||||||
|
thread_limiter=thread_limiter,
|
||||||
|
)
|
||||||
|
except EmbedderError as exc:
|
||||||
|
await _mark_job_failed(
|
||||||
|
sessionmaker,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
error_code="embedding_failed",
|
||||||
|
error_message=str(exc),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
try:
|
||||||
|
indexed = await index_chunks(
|
||||||
|
embedded,
|
||||||
|
storage=point_storage,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=source_file_id,
|
||||||
|
source_filename=filename,
|
||||||
|
source_type=validated.source_type,
|
||||||
|
actor=f"api_key:{auth.api_key_id}",
|
||||||
|
dense_embedders=dense_embedders,
|
||||||
|
sparse_embedder=sparse_embedder,
|
||||||
|
settings=qdrant_settings,
|
||||||
|
thread_limiter=thread_limiter,
|
||||||
|
)
|
||||||
|
except PointIndexingError as exc:
|
||||||
|
await _mark_job_failed(
|
||||||
|
sessionmaker,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
error_code="index_failed",
|
||||||
|
error_message=str(exc),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
except TimeoutError:
|
||||||
|
await _mark_job_failed(
|
||||||
|
sessionmaker,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
error_code="timeout",
|
||||||
|
error_message=f"ingestion exceeded {ingestion_settings.timeout_seconds}s",
|
||||||
|
)
|
||||||
|
raise IngestionTimeoutError(
|
||||||
|
f"ingestion exceeded {ingestion_settings.timeout_seconds}s"
|
||||||
|
) from None
|
||||||
|
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
await jobs_repo.mark_terminal(
|
||||||
|
session,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
status="succeeded",
|
||||||
|
# An upsert with deterministic ids cannot tell an insert from
|
||||||
|
# an overwrite, so every written point is reported here and
|
||||||
|
# `points_updated` stays 0 rather than being guessed at.
|
||||||
|
points_created=indexed.points_upserted,
|
||||||
|
points_soft_deleted=indexed.points_soft_deleted,
|
||||||
|
)
|
||||||
|
jobs_repo.append_event(
|
||||||
|
session,
|
||||||
|
tenant_id=auth.tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
level="info",
|
||||||
|
stage="completed",
|
||||||
|
message="chunks parsed, embedded, and indexed",
|
||||||
|
details={
|
||||||
|
"chunks_parsed": len(chunks),
|
||||||
|
"chunks_embedded": len(embedded),
|
||||||
|
"points_upserted": indexed.points_upserted,
|
||||||
|
"points_soft_deleted": indexed.points_soft_deleted,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"ingestion.job.completed",
|
||||||
|
tenant_id=str(auth.tenant_id),
|
||||||
|
ingestion_job_id=str(ingestion_job_id),
|
||||||
|
file_id=str(source_file_id),
|
||||||
|
chunks_parsed=len(chunks),
|
||||||
|
points_upserted=indexed.points_upserted,
|
||||||
|
points_soft_deleted=indexed.points_soft_deleted,
|
||||||
|
)
|
||||||
|
return UploadResult(
|
||||||
|
file_id=source_file_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
status="succeeded",
|
||||||
|
chunks_indexed=indexed.points_upserted,
|
||||||
|
is_new_attempt=True,
|
||||||
|
)
|
||||||
55
src/application/files/validation.py
Normal file
55
src/application/files/validation.py
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
"""Upload extension/content-type/size validation (ADR-0008).
|
||||||
|
|
||||||
|
Pure and synchronous: no I/O. `content_sha256` computation lives here too —
|
||||||
|
hashing is blocking CPU work (ADR-0017), so the caller runs this whole
|
||||||
|
function through `anyio.to_thread.run_sync` with the ingestion
|
||||||
|
`CapacityLimiter`, the same rule applied to parsing/chunking.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from src.application.files.errors import FileTooLargeError, InvalidUploadError
|
||||||
|
from src.application.files.models import ValidatedUpload
|
||||||
|
from src.application.ingestion.errors import UnsupportedSourceTypeError
|
||||||
|
|
||||||
|
_CONTENT_TYPES = {
|
||||||
|
"csv": "text/csv",
|
||||||
|
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||||
|
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||||
|
}
|
||||||
|
_OOXML_MAGIC = b"PK\x03\x04"
|
||||||
|
|
||||||
|
|
||||||
|
def _source_type_from_filename(filename: str) -> str:
|
||||||
|
suffix = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||||
|
if suffix == "doc":
|
||||||
|
raise UnsupportedSourceTypeError(
|
||||||
|
"legacy .doc is not ingestible until an out-of-process conversion "
|
||||||
|
"service exists (ADR-0018)"
|
||||||
|
)
|
||||||
|
if suffix not in _CONTENT_TYPES:
|
||||||
|
raise UnsupportedSourceTypeError(f"'.{suffix}' is not an ingestible file type")
|
||||||
|
return suffix
|
||||||
|
|
||||||
|
|
||||||
|
def validate_and_hash_upload(*, filename: str, data: bytes, max_size_bytes: int) -> ValidatedUpload:
|
||||||
|
source_type = _source_type_from_filename(filename)
|
||||||
|
|
||||||
|
if not data:
|
||||||
|
raise InvalidUploadError("uploaded file is empty")
|
||||||
|
if len(data) > max_size_bytes:
|
||||||
|
raise FileTooLargeError(
|
||||||
|
f"upload is {len(data)} bytes, over the {max_size_bytes}-byte limit"
|
||||||
|
)
|
||||||
|
|
||||||
|
is_ooxml = data[:4] == _OOXML_MAGIC
|
||||||
|
if source_type in ("docx", "xlsx") and not is_ooxml:
|
||||||
|
raise InvalidUploadError(f"content does not match the declared .{source_type} extension")
|
||||||
|
if source_type == "csv" and is_ooxml:
|
||||||
|
raise InvalidUploadError("content does not match the declared .csv extension")
|
||||||
|
|
||||||
|
return ValidatedUpload(
|
||||||
|
source_type=source_type,
|
||||||
|
content_type=_CONTENT_TYPES[source_type],
|
||||||
|
content_sha256=hashlib.sha256(data).hexdigest(),
|
||||||
|
)
|
||||||
@@ -1,9 +1,11 @@
|
|||||||
"""Document parsing and fixed-size chunking (ADR-0004, ADR-0018).
|
"""Document parsing and fixed-size chunking (ADR-0004, ADR-0018).
|
||||||
|
|
||||||
Everything here is pure and synchronous: no I/O, no ports, no SDK clients
|
`parse_and_chunk_document` is the entry point callers outside this package
|
||||||
(ADR-0015 reserves ports for external side effects). Parsing and chunking are
|
should use: it dispatches on source type and owns the
|
||||||
blocking CPU work, so callers run them through `anyio.to_thread.run_sync` with
|
`anyio.to_thread.run_sync` + `CapacityLimiter` offload required by ADR-0017.
|
||||||
the ingestion `CapacityLimiter` rather than on the event loop (ADR-0017).
|
The individual parsers and `chunk_document` are pure, synchronous, and
|
||||||
|
exported mainly for their own unit tests — calling them directly from an
|
||||||
|
`async def` route or service is the defect ADR-0017 warns about.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from src.application.ingestion.chunking import chunk_document, chunk_id_for, split_by_tokens
|
from src.application.ingestion.chunking import chunk_document, chunk_id_for, split_by_tokens
|
||||||
@@ -22,6 +24,7 @@ from src.application.ingestion.models import (
|
|||||||
StructuralUnit,
|
StructuralUnit,
|
||||||
)
|
)
|
||||||
from src.application.ingestion.normalization import normalize_persian_text
|
from src.application.ingestion.normalization import normalize_persian_text
|
||||||
|
from src.application.ingestion.pipeline import parse_and_chunk_document
|
||||||
from src.application.ingestion.spreadsheet_parser import parse_csv, parse_xlsx
|
from src.application.ingestion.spreadsheet_parser import parse_csv, parse_xlsx
|
||||||
from src.application.ingestion.tokenizer import count_tokens, get_encoder
|
from src.application.ingestion.tokenizer import count_tokens, get_encoder
|
||||||
|
|
||||||
@@ -40,6 +43,7 @@ __all__ = [
|
|||||||
"count_tokens",
|
"count_tokens",
|
||||||
"get_encoder",
|
"get_encoder",
|
||||||
"normalize_persian_text",
|
"normalize_persian_text",
|
||||||
|
"parse_and_chunk_document",
|
||||||
"parse_csv",
|
"parse_csv",
|
||||||
"parse_docx",
|
"parse_docx",
|
||||||
"parse_xlsx",
|
"parse_xlsx",
|
||||||
|
|||||||
48
src/application/ingestion/bounds.py
Normal file
48
src/application/ingestion/bounds.py
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
"""Request bounds for inline ingestion (ADR-0017).
|
||||||
|
|
||||||
|
Three independent bounds, each mapping to its own status code: the chunk
|
||||||
|
ceiling (`413`, checked before embedding starts), process-wide concurrency
|
||||||
|
(`503` + `Retry-After`, rejected rather than queued), and the work-phase
|
||||||
|
deadline (`504`, and the caller must still write a terminal job status).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import AsyncIterator, Sequence
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
|
from anyio import Semaphore, WouldBlock
|
||||||
|
|
||||||
|
from src.application.ingestion.errors import ChunkLimitExceededError, IngestionAtCapacityError
|
||||||
|
from src.application.ingestion.models import Chunk
|
||||||
|
|
||||||
|
|
||||||
|
def enforce_chunk_limit(chunks: Sequence[Chunk], *, max_chunks: int) -> None:
|
||||||
|
"""Raise `ChunkLimitExceededError` if `chunks` exceeds `max_chunks`.
|
||||||
|
|
||||||
|
Call this immediately after parsing/chunking and before any embedding
|
||||||
|
call — the ceiling must be discovered up front, not mid-batch.
|
||||||
|
"""
|
||||||
|
if len(chunks) > max_chunks:
|
||||||
|
raise ChunkLimitExceededError(
|
||||||
|
f"document produced {len(chunks)} chunks, over the {max_chunks}-chunk limit"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def acquire_ingestion_slot(limiter: Semaphore) -> AsyncIterator[None]:
|
||||||
|
"""Hold one of `INGESTION_MAX_CONCURRENCY` process-wide slots for the block.
|
||||||
|
|
||||||
|
`limiter` is an `anyio.Semaphore` created once in the lifespan. Rejects
|
||||||
|
immediately with `IngestionAtCapacityError` when the process is already at
|
||||||
|
capacity, rather than queueing the request behind an unbounded wait
|
||||||
|
(ADR-0017) — the semaphore's own async `acquire()` would do the latter.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
limiter.acquire_nowait()
|
||||||
|
except WouldBlock:
|
||||||
|
raise IngestionAtCapacityError(
|
||||||
|
"ingestion is at capacity; retry after the configured backoff"
|
||||||
|
) from None
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
limiter.release()
|
||||||
112
src/application/ingestion/embedding.py
Normal file
112
src/application/ingestion/embedding.py
Normal file
@@ -0,0 +1,112 @@
|
|||||||
|
"""The one caller-facing entry point for embedding chunks (ADR-0001, ADR-0017).
|
||||||
|
|
||||||
|
`embed_chunks` is the only version of this step callers should reach for: it
|
||||||
|
owns batching, the `embed_concurrency` semaphore bounding in-flight dense
|
||||||
|
batches, and the `anyio.to_thread.run_sync` + `CapacityLimiter` offload for
|
||||||
|
the blocking BM25 pipeline. Composing these correctly at every call site is
|
||||||
|
exactly the obligation a deep module absorbs once (see CLAUDE.md's "prefer
|
||||||
|
deep modules").
|
||||||
|
|
||||||
|
Per-provider text shaping — task prefixes, `keep_alive`, request payload —
|
||||||
|
belongs to the adapters in `src/infrastructure/embedding/`, not here. This
|
||||||
|
module knows only that an embedder turns texts into vectors.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
from anyio import CapacityLimiter, to_thread
|
||||||
|
|
||||||
|
from src.application.ingestion.errors import EmbedderError
|
||||||
|
from src.application.ingestion.models import Chunk, EmbeddedChunk
|
||||||
|
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||||
|
from src.config import IngestionSettings
|
||||||
|
|
||||||
|
|
||||||
|
def _batches(texts: Sequence[str], size: int) -> list[Sequence[str]]:
|
||||||
|
return [texts[i : i + size] for i in range(0, len(texts), size)]
|
||||||
|
|
||||||
|
|
||||||
|
async def _embed_dense_bounded(
|
||||||
|
embedder: DenseEmbedder,
|
||||||
|
batch: Sequence[str],
|
||||||
|
*,
|
||||||
|
semaphore: asyncio.Semaphore,
|
||||||
|
) -> list[list[float]]:
|
||||||
|
async with semaphore:
|
||||||
|
try:
|
||||||
|
return await embedder.embed_batch(batch)
|
||||||
|
except Exception as exc:
|
||||||
|
raise EmbedderError(f"{embedder.name} embedding batch failed: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
async def _embed_dense_all(
|
||||||
|
embedder: DenseEmbedder,
|
||||||
|
texts: Sequence[str],
|
||||||
|
*,
|
||||||
|
batch_size: int,
|
||||||
|
semaphore: asyncio.Semaphore,
|
||||||
|
) -> list[list[float]]:
|
||||||
|
batches = _batches(texts, batch_size)
|
||||||
|
results = await asyncio.gather(
|
||||||
|
*(_embed_dense_bounded(embedder, batch, semaphore=semaphore) for batch in batches)
|
||||||
|
)
|
||||||
|
return [vector for batch_result in results for vector in batch_result]
|
||||||
|
|
||||||
|
|
||||||
|
def _embed_sparse_sync(embedder: SparseEmbedder, texts: Sequence[str]):
|
||||||
|
try:
|
||||||
|
return embedder.embed_batch(texts)
|
||||||
|
except Exception as exc:
|
||||||
|
raise EmbedderError(f"{embedder.name} embedding batch failed: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
async def embed_chunks(
|
||||||
|
chunks: Sequence[Chunk],
|
||||||
|
*,
|
||||||
|
dense_embedders: Sequence[DenseEmbedder],
|
||||||
|
sparse_embedder: SparseEmbedder,
|
||||||
|
settings: IngestionSettings,
|
||||||
|
thread_limiter: CapacityLimiter,
|
||||||
|
) -> list[EmbeddedChunk]:
|
||||||
|
"""Embed every chunk into all dense vectors plus the sparse vector.
|
||||||
|
|
||||||
|
Dense embedders run concurrently with each other; each one's batches are
|
||||||
|
concurrent among themselves too, bounded by one `embed_concurrency`
|
||||||
|
semaphore shared across all dense embedders (ADR-0017: the limit exists
|
||||||
|
for both providers' rate limits and the self-hosted server's capacity —
|
||||||
|
not a per-provider budget). The sparse (BM25) pass is blocking and runs
|
||||||
|
once, off the event loop.
|
||||||
|
|
||||||
|
Raises `EmbedderError` (502) if any embedder call fails.
|
||||||
|
"""
|
||||||
|
if not chunks:
|
||||||
|
return []
|
||||||
|
|
||||||
|
texts = [chunk.content for chunk in chunks]
|
||||||
|
semaphore = asyncio.Semaphore(settings.embed_concurrency)
|
||||||
|
|
||||||
|
dense_task = asyncio.gather(
|
||||||
|
*(
|
||||||
|
_embed_dense_all(
|
||||||
|
embedder, texts, batch_size=settings.embed_batch_size, semaphore=semaphore
|
||||||
|
)
|
||||||
|
for embedder in dense_embedders
|
||||||
|
)
|
||||||
|
)
|
||||||
|
sparse_task = to_thread.run_sync(
|
||||||
|
partial(_embed_sparse_sync, sparse_embedder, texts), limiter=thread_limiter
|
||||||
|
)
|
||||||
|
dense_results, sparse_vectors = await asyncio.gather(dense_task, sparse_task)
|
||||||
|
|
||||||
|
dense_by_name = {
|
||||||
|
embedder.name: vectors
|
||||||
|
for embedder, vectors in zip(dense_embedders, dense_results, strict=True)
|
||||||
|
}
|
||||||
|
|
||||||
|
embedded: list[EmbeddedChunk] = []
|
||||||
|
for index, chunk in enumerate(chunks):
|
||||||
|
dense = {name: vectors[index] for name, vectors in dense_by_name.items()}
|
||||||
|
embedded.append(EmbeddedChunk(chunk=chunk, dense=dense, sparse=sparse_vectors[index]))
|
||||||
|
return embedded
|
||||||
@@ -39,3 +39,35 @@ class ChunkTooLargeError(IngestionError):
|
|||||||
against is silent — `nomic-embed-text-v2-moe` truncates over-long input
|
against is silent — `nomic-embed-text-v2-moe` truncates over-long input
|
||||||
without raising (ADR-0004).
|
without raising (ADR-0004).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class EmbedderError(IngestionError):
|
||||||
|
"""A dense or sparse embedder call failed (transport error, non-2xx, or
|
||||||
|
a malformed response).
|
||||||
|
|
||||||
|
Maps to `502` per ADR-0017.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class PointIndexingError(IngestionError):
|
||||||
|
"""Upserting or soft-deleting Qdrant points failed.
|
||||||
|
|
||||||
|
Maps to `502` — like `EmbedderError`, this is an upstream dependency
|
||||||
|
failing, not a malformed request. Kept distinct from `EmbedderError` so the
|
||||||
|
job's `error_code` says which dependency broke.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionAtCapacityError(IngestionError):
|
||||||
|
"""`INGESTION_MAX_CONCURRENCY` in-process ingestions are already running.
|
||||||
|
|
||||||
|
Maps to `503` with `Retry-After`, not a queued wait (ADR-0017).
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionTimeoutError(IngestionError):
|
||||||
|
"""The work phase (parse/embed/upsert) exceeded `INGESTION_TIMEOUT_SECONDS`.
|
||||||
|
|
||||||
|
Maps to `504`. The caller must still write a terminal `failed` job status
|
||||||
|
before this propagates (ADR-0017).
|
||||||
|
"""
|
||||||
|
|||||||
@@ -64,3 +64,28 @@ class Chunk(BaseModel):
|
|||||||
next_chunk_id: uuid.UUID | None = None
|
next_chunk_id: uuid.UUID | None = None
|
||||||
token_count: int
|
token_count: int
|
||||||
character_count: int
|
character_count: int
|
||||||
|
|
||||||
|
|
||||||
|
class SparseVector(BaseModel):
|
||||||
|
"""A sparse (term-index -> weight) vector, Qdrant's `modifier="idf"` shape.
|
||||||
|
|
||||||
|
Kept free of the `qdrant_client` SDK (ADR-0015: ports carry no infra
|
||||||
|
imports) — `src/infrastructure/qdrant/` converts this to the SDK's own
|
||||||
|
`SparseVector` type at upsert time (Phase 5).
|
||||||
|
"""
|
||||||
|
|
||||||
|
indices: list[int]
|
||||||
|
values: list[float]
|
||||||
|
|
||||||
|
|
||||||
|
class EmbeddedChunk(BaseModel):
|
||||||
|
"""A chunk plus every vector it will be upserted with (ADR-0001).
|
||||||
|
|
||||||
|
`dense` is keyed by named-vector name (`dense_nomic`, `dense_openai`).
|
||||||
|
`late_interaction` is deliberately absent — not computed at ingest
|
||||||
|
(ADR-0017).
|
||||||
|
"""
|
||||||
|
|
||||||
|
chunk: Chunk
|
||||||
|
dense: dict[str, list[float]]
|
||||||
|
sparse: SparseVector
|
||||||
|
|||||||
65
src/application/ingestion/pipeline.py
Normal file
65
src/application/ingestion/pipeline.py
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
"""The one caller-facing entry point for parsing and chunking (ADR-0017).
|
||||||
|
|
||||||
|
`parse_docx`/`parse_csv`/`parse_xlsx`/`chunk_document` are blocking, pure
|
||||||
|
functions; calling any of them directly from an `async def` route or service
|
||||||
|
is the defect ADR-0017 names explicitly ("one large `python-docx` parse would
|
||||||
|
stall every concurrent request"). `parse_and_chunk_document` is the only
|
||||||
|
version of this pipeline callers should reach for: it owns source-type
|
||||||
|
dispatch and the `anyio.to_thread.run_sync` + `CapacityLimiter` offload, so
|
||||||
|
that obligation cannot be forgotten at a call site.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
from anyio import CapacityLimiter, to_thread
|
||||||
|
|
||||||
|
from src.application.ingestion.chunking import chunk_document
|
||||||
|
from src.application.ingestion.docx_parser import parse_docx
|
||||||
|
from src.application.ingestion.errors import UnsupportedSourceTypeError
|
||||||
|
from src.application.ingestion.models import Chunk, ParsedDocument
|
||||||
|
from src.application.ingestion.spreadsheet_parser import parse_csv, parse_xlsx
|
||||||
|
from src.config import ChunkingSettings
|
||||||
|
|
||||||
|
_PARSERS = {"csv", "xlsx", "docx"}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse(data: bytes, source_type: str, settings: ChunkingSettings) -> ParsedDocument:
|
||||||
|
if source_type == "docx":
|
||||||
|
return parse_docx(data, settings)
|
||||||
|
if source_type == "xlsx":
|
||||||
|
return parse_xlsx(data)
|
||||||
|
if source_type == "csv":
|
||||||
|
return parse_csv(data)
|
||||||
|
raise UnsupportedSourceTypeError(f"'{source_type}' is not an ingestible source type")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_and_chunk(
|
||||||
|
data: bytes, source_type: str, file_id: uuid.UUID, settings: ChunkingSettings
|
||||||
|
) -> list[Chunk]:
|
||||||
|
parsed = _parse(data, source_type, settings)
|
||||||
|
return chunk_document(parsed, file_id=file_id, settings=settings)
|
||||||
|
|
||||||
|
|
||||||
|
async def parse_and_chunk_document(
|
||||||
|
data: bytes,
|
||||||
|
*,
|
||||||
|
source_type: str,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
settings: ChunkingSettings,
|
||||||
|
limiter: CapacityLimiter,
|
||||||
|
) -> list[Chunk]:
|
||||||
|
"""Parse and chunk a document off the event loop, bounded by `limiter`.
|
||||||
|
|
||||||
|
Raises `UnsupportedSourceTypeError` (415), `DocumentParseError` (400), or
|
||||||
|
`ChunkTooLargeError` — see `src/application/ingestion/errors.py`. Callers
|
||||||
|
map these to status codes; this module carries no HTTP knowledge
|
||||||
|
(ADR-0015). The `max_chunks_per_file` ceiling (413) is enforced by the
|
||||||
|
caller, not here — see Phase 4 of plan 001.
|
||||||
|
"""
|
||||||
|
if source_type not in _PARSERS:
|
||||||
|
raise UnsupportedSourceTypeError(f"'{source_type}' is not an ingestible source type")
|
||||||
|
return await to_thread.run_sync(
|
||||||
|
partial(_parse_and_chunk, data, source_type, file_id, settings),
|
||||||
|
limiter=limiter,
|
||||||
|
)
|
||||||
18
src/application/points/__init__.py
Normal file
18
src/application/points/__init__.py
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
"""Ingestion-generated Qdrant point CRUD (ADR-0001, ADR-0002).
|
||||||
|
|
||||||
|
`index_chunks` is the entry point callers outside this package should use: it
|
||||||
|
dispatches payload construction, batching, bounded-concurrency upserts, and the
|
||||||
|
post-success soft-delete sweep. `build_chunk_payload` and the batching helpers
|
||||||
|
stay internal, exported mainly for their own unit tests.
|
||||||
|
|
||||||
|
The `/v1/points` surface lives here too, in its own modules with their own
|
||||||
|
entry points: `queries.py` for the read paths and `deletion.py` for soft delete
|
||||||
|
with neighbour relinking. They share this package because they share ADR-0001's
|
||||||
|
payload schema, not because they share a caller — `index_chunks` writes a whole
|
||||||
|
file at once, while those serve one admin edit at a time.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from src.application.points.indexing import IndexingResult, index_chunks
|
||||||
|
from src.application.points.models import ChunkPoint
|
||||||
|
|
||||||
|
__all__ = ["ChunkPoint", "IndexingResult", "index_chunks"]
|
||||||
253
src/application/points/deletion.py
Normal file
253
src/application/points/deletion.py
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
"""Soft delete for `/v1/points` and for a whole file's points (ADR-0002).
|
||||||
|
|
||||||
|
The caller-facing entry points are `soft_delete_point` and
|
||||||
|
`soft_delete_file_points`. Routers call these; `patches_for_removal` and the
|
||||||
|
planning helpers stay internal, because getting a delete right is exactly the
|
||||||
|
composition a caller should not have to reassemble: read the point, load its
|
||||||
|
neighbours, compute the patches still missing, send them in **one** batch,
|
||||||
|
verify they landed, and retry against fresh versions if they did not.
|
||||||
|
|
||||||
|
Why the retry exists. Qdrant has no multi-point transaction, so a batch whose
|
||||||
|
second operation loses a version race applies its first operation anyway — and
|
||||||
|
a filtered `set_payload` that matched nothing still reports success. That
|
||||||
|
combination means "did my write land?" is only answerable by reading back, and
|
||||||
|
a single-shot delete would be able to leave the deactivation applied and a
|
||||||
|
neighbour's pointer stale. Since `patches_for_removal` plans from current state
|
||||||
|
towards a fixed end state, simply re-planning emits precisely the patches that
|
||||||
|
did not land, so the loop converges instead of re-doing work. Only exhausting
|
||||||
|
the attempts raises `PointVersionConflictError` (`409`).
|
||||||
|
|
||||||
|
Deleting an already-inactive point falls out of the same machinery rather than
|
||||||
|
needing a special case: its neighbours were relinked by the first delete, so the
|
||||||
|
plan is empty and the call is a no-op success — not a `404`, and not a second
|
||||||
|
relink.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from time import perf_counter
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
|
||||||
|
from src.application.points.errors import PointVersionConflictError
|
||||||
|
from src.application.points.point import Point, PointNotFoundError
|
||||||
|
from src.application.points.relinking import (
|
||||||
|
neighbour_ids,
|
||||||
|
patch_for_deactivation,
|
||||||
|
patches_for_removal,
|
||||||
|
)
|
||||||
|
from src.application.ports.point_repository import PayloadPatch, PointRepository
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
# Three plan-apply rounds, then a final verifying plan. Each round only re-emits
|
||||||
|
# what a concurrent writer displaced, so a caller that legitimately needs more
|
||||||
|
# than this is contending on the same points continuously and deserves the
|
||||||
|
# `409` rather than an unbounded loop inside a request.
|
||||||
|
_MAX_ATTEMPTS = 3
|
||||||
|
|
||||||
|
# One sweep page. Matches ADR-0002's 100-operation batch cap, so a page of
|
||||||
|
# points is always expressible as a single `points/batch` request.
|
||||||
|
_SWEEP_BATCH_SIZE = 100
|
||||||
|
|
||||||
|
# A hard ceiling on sweep rounds, so a file being concurrently re-ingested while
|
||||||
|
# it is deleted cannot spin here for the life of the request.
|
||||||
|
_MAX_SWEEP_ROUNDS = 1_000
|
||||||
|
|
||||||
|
|
||||||
|
def _elapsed_ms(started: float) -> float:
|
||||||
|
"""Wall-clock milliseconds since `started` (ADR-0011's `duration_ms`).
|
||||||
|
|
||||||
|
Worth carrying on these events even though the relinking itself is O(1):
|
||||||
|
what a delete actually spends is Qdrant round trips, and the whole-file
|
||||||
|
sweep spends a number of them proportional to the file's length. Timing the
|
||||||
|
operation is the only way to tell a slow store from a contended one, which
|
||||||
|
`rounds` on the same event then disambiguates.
|
||||||
|
"""
|
||||||
|
return round((perf_counter() - started) * 1000, 2)
|
||||||
|
|
||||||
|
|
||||||
|
async def soft_delete_point(
|
||||||
|
repository: PointRepository,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
point_id: uuid.UUID,
|
||||||
|
actor: str,
|
||||||
|
) -> Point:
|
||||||
|
"""Deactivate one point and relink its neighbours around the gap.
|
||||||
|
|
||||||
|
Returns the point as it now stands. Raises `PointNotFoundError` (`404`) if
|
||||||
|
it is not this tenant's — the same non-disclosure rule the read paths
|
||||||
|
follow — or `PointVersionConflictError` (`409`) if concurrent writers keep
|
||||||
|
displacing the plan.
|
||||||
|
"""
|
||||||
|
started = perf_counter()
|
||||||
|
rounds = 0
|
||||||
|
point, patches = await _plan_removal(
|
||||||
|
repository, tenant_id=tenant_id, point_id=point_id, actor=actor
|
||||||
|
)
|
||||||
|
|
||||||
|
for _ in range(_MAX_ATTEMPTS):
|
||||||
|
if not patches:
|
||||||
|
break
|
||||||
|
await repository.apply_patches(tenant_id=tenant_id, patches=patches)
|
||||||
|
rounds += 1
|
||||||
|
# The next plan doubles as verification: anything that did not land is
|
||||||
|
# still missing from the end state and comes back as a patch.
|
||||||
|
point, patches = await _plan_removal(
|
||||||
|
repository, tenant_id=tenant_id, point_id=point_id, actor=actor
|
||||||
|
)
|
||||||
|
|
||||||
|
if patches:
|
||||||
|
logger.warning(
|
||||||
|
"points.soft_delete.conflict",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
point_id=str(point_id),
|
||||||
|
file_id=str(point.file_id),
|
||||||
|
unsettled_points=[str(patch.point_id) for patch in patches],
|
||||||
|
rounds=rounds,
|
||||||
|
duration_ms=_elapsed_ms(started),
|
||||||
|
)
|
||||||
|
raise PointVersionConflictError(
|
||||||
|
f"point {point_id} could not be soft-deleted under concurrent modification"
|
||||||
|
)
|
||||||
|
|
||||||
|
if rounds:
|
||||||
|
logger.info(
|
||||||
|
"points.soft_deleted",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
point_id=str(point_id),
|
||||||
|
file_id=str(point.file_id),
|
||||||
|
version=point.version,
|
||||||
|
actor=actor,
|
||||||
|
# `rounds` is 1 unless a concurrent writer forced a re-plan, so a
|
||||||
|
# rising value here is contention, not slow relinking.
|
||||||
|
rounds=rounds,
|
||||||
|
duration_ms=_elapsed_ms(started),
|
||||||
|
)
|
||||||
|
return point
|
||||||
|
|
||||||
|
|
||||||
|
async def soft_delete_file_points(
|
||||||
|
repository: PointRepository,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
actor: str,
|
||||||
|
) -> int:
|
||||||
|
"""Deactivate every active point of one file, in batches.
|
||||||
|
|
||||||
|
No relinking: the whole file leaves the sequence at once, so no surviving
|
||||||
|
active point can be left pointing at a deactivated one, and the chain is
|
||||||
|
preserved intact for whoever reads the deleted file later.
|
||||||
|
|
||||||
|
Returns how many points were active when the sweep reached them. Each round
|
||||||
|
re-lists from the start rather than paging with a cursor — deactivated
|
||||||
|
points drop straight out of the default listing, so the listing itself is
|
||||||
|
the progress check, and a round that attempts the exact same ids as the one
|
||||||
|
before it made no progress and raises `PointVersionConflictError`.
|
||||||
|
"""
|
||||||
|
started = perf_counter()
|
||||||
|
swept: set[uuid.UUID] = set()
|
||||||
|
previous_attempt: frozenset[uuid.UUID] = frozenset()
|
||||||
|
rounds = 0
|
||||||
|
|
||||||
|
for _ in range(_MAX_SWEEP_ROUNDS):
|
||||||
|
page = await repository.list_by_file(
|
||||||
|
tenant_id=tenant_id, file_id=file_id, limit=_SWEEP_BATCH_SIZE
|
||||||
|
)
|
||||||
|
if not page.points:
|
||||||
|
if swept:
|
||||||
|
logger.info(
|
||||||
|
"points.file_soft_deleted",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
file_id=str(file_id),
|
||||||
|
points_soft_deleted=len(swept),
|
||||||
|
actor=actor,
|
||||||
|
# Two Qdrant round trips per round, so this is the delete
|
||||||
|
# path whose cost tracks the size of the file.
|
||||||
|
rounds=rounds,
|
||||||
|
duration_ms=_elapsed_ms(started),
|
||||||
|
)
|
||||||
|
return len(swept)
|
||||||
|
|
||||||
|
attempt = frozenset(point.point_id for point in page.points)
|
||||||
|
if attempt == previous_attempt:
|
||||||
|
logger.warning(
|
||||||
|
"points.file_soft_delete.conflict",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
file_id=str(file_id),
|
||||||
|
unsettled_points=[str(point_id) for point_id in sorted(attempt, key=str)],
|
||||||
|
rounds=rounds,
|
||||||
|
duration_ms=_elapsed_ms(started),
|
||||||
|
)
|
||||||
|
raise PointVersionConflictError(
|
||||||
|
f"file {file_id} could not be soft-deleted under concurrent modification"
|
||||||
|
)
|
||||||
|
previous_attempt = attempt
|
||||||
|
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
await repository.apply_patches(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
patches=[patch_for_deactivation(point, actor=actor, now=now) for point in page.points],
|
||||||
|
)
|
||||||
|
swept |= attempt
|
||||||
|
rounds += 1
|
||||||
|
|
||||||
|
logger.warning(
|
||||||
|
"points.file_soft_delete.conflict",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
file_id=str(file_id),
|
||||||
|
reason="sweep_rounds_exhausted",
|
||||||
|
rounds=rounds,
|
||||||
|
duration_ms=_elapsed_ms(started),
|
||||||
|
)
|
||||||
|
raise PointVersionConflictError(f"file {file_id} still had active points after the sweep")
|
||||||
|
|
||||||
|
|
||||||
|
async def _plan_removal(
|
||||||
|
repository: PointRepository,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
point_id: uuid.UUID,
|
||||||
|
actor: str,
|
||||||
|
) -> tuple[Point, tuple[PayloadPatch, ...]]:
|
||||||
|
point = await repository.get(tenant_id=tenant_id, point_id=point_id)
|
||||||
|
if point is None:
|
||||||
|
raise PointNotFoundError(f"point {point_id} not found")
|
||||||
|
|
||||||
|
neighbours = await _load_neighbours(repository, tenant_id=tenant_id, point=point)
|
||||||
|
_warn_on_missing_neighbours(point, neighbours, tenant_id=tenant_id)
|
||||||
|
patches = patches_for_removal(point, neighbours, actor=actor, now=datetime.now(UTC))
|
||||||
|
return point, patches
|
||||||
|
|
||||||
|
|
||||||
|
async def _load_neighbours(
|
||||||
|
repository: PointRepository, *, tenant_id: uuid.UUID, point: Point
|
||||||
|
) -> dict[uuid.UUID, Point]:
|
||||||
|
wanted: Sequence[uuid.UUID] = neighbour_ids(point)
|
||||||
|
if not wanted:
|
||||||
|
return {}
|
||||||
|
found = await repository.get_many(tenant_id=tenant_id, point_ids=wanted)
|
||||||
|
return {neighbour.point_id: neighbour for neighbour in found}
|
||||||
|
|
||||||
|
|
||||||
|
def _warn_on_missing_neighbours(
|
||||||
|
point: Point, neighbours: Mapping[uuid.UUID, Point], *, tenant_id: uuid.UUID
|
||||||
|
) -> None:
|
||||||
|
"""A pointer naming a point that is not there means the chain is already broken.
|
||||||
|
|
||||||
|
Worth a log line rather than an exception: the delete can still complete the
|
||||||
|
part of the relink that does exist, and refusing would leave the caller with
|
||||||
|
a point it cannot remove through any endpoint.
|
||||||
|
"""
|
||||||
|
missing = [pointer for pointer in neighbour_ids(point) if pointer not in neighbours]
|
||||||
|
if missing:
|
||||||
|
logger.warning(
|
||||||
|
"points.relink.neighbour_missing",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
point_id=str(point.point_id),
|
||||||
|
file_id=str(point.file_id),
|
||||||
|
missing_neighbours=[str(pointer) for pointer in missing],
|
||||||
|
)
|
||||||
19
src/application/points/errors.py
Normal file
19
src/application/points/errors.py
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
"""Mutation failures for the `/v1/points` write paths (ADR-0002).
|
||||||
|
|
||||||
|
No HTTP knowledge here — `src/api/errors.py` owns the status mapping. Absence
|
||||||
|
lives on `PointNotFoundError` in `point.py`, next to the model whose read paths
|
||||||
|
raise it; this module is for the failures only a *write* can produce.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class PointVersionConflictError(Exception):
|
||||||
|
"""A version-guarded write could not be landed against a moving target.
|
||||||
|
|
||||||
|
Raised when the service has re-read, recomputed, and re-applied its patches
|
||||||
|
the allowed number of times and the desired state still has not settled —
|
||||||
|
something else is writing the same points concurrently. Maps to `409`.
|
||||||
|
|
||||||
|
This is not "the guard fired once": a single stale guard is expected and is
|
||||||
|
retried, because Qdrant reports success for a filtered `set_payload` that
|
||||||
|
matched nothing. It means the retries were exhausted.
|
||||||
|
"""
|
||||||
203
src/application/points/indexing.py
Normal file
203
src/application/points/indexing.py
Normal file
@@ -0,0 +1,203 @@
|
|||||||
|
"""The one caller-facing entry point for indexing embedded chunks (ADR-0001, ADR-0017).
|
||||||
|
|
||||||
|
`index_chunks` is the only version of this step callers should reach for. It
|
||||||
|
owns the whole composition a correct upsert needs:
|
||||||
|
|
||||||
|
- building ADR-0001's payload for every chunk, with `tenant_id`/`domain` taken
|
||||||
|
from server-derived context;
|
||||||
|
- offloading that (and the per-chunk content hashing) to a thread, since it is
|
||||||
|
blocking CPU work (ADR-0017);
|
||||||
|
- batching at `QDRANT_UPSERT_BATCH_SIZE` inside ADR-0001's 64-256 band;
|
||||||
|
- bounding in-flight batches with an `asyncio.Semaphore` rather than an
|
||||||
|
unbounded `gather` (ADR-0017);
|
||||||
|
- running the soft-delete sweep for a shortened file **only after every batch
|
||||||
|
has succeeded**.
|
||||||
|
|
||||||
|
That last ordering is the point, not an implementation detail — see
|
||||||
|
`_deactivate_stale` below. `build_chunk_payload` and `_batches` stay internal;
|
||||||
|
pushing that composition onto every call site is exactly the obligation a deep
|
||||||
|
module absorbs once (CLAUDE.md, "prefer deep modules").
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from anyio import CapacityLimiter, to_thread
|
||||||
|
|
||||||
|
from src.application.ingestion.errors import PointIndexingError
|
||||||
|
from src.application.ingestion.models import EmbeddedChunk
|
||||||
|
from src.application.points.models import ChunkPoint
|
||||||
|
from src.application.points.payload import build_chunk_payload
|
||||||
|
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||||
|
from src.application.ports.point_storage import PointStorage
|
||||||
|
from src.config import QdrantSettings
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class IndexingResult:
|
||||||
|
"""What one indexing pass wrote.
|
||||||
|
|
||||||
|
`points_upserted` counts points written, not points *created* — a
|
||||||
|
deterministic-id upsert cannot distinguish an insert from an overwrite, so
|
||||||
|
the ingestion job reports this as `points_created` and leaves
|
||||||
|
`points_updated` at zero rather than guessing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
points_upserted: int
|
||||||
|
points_soft_deleted: int
|
||||||
|
|
||||||
|
|
||||||
|
def _embedding_model_version(
|
||||||
|
dense_embedders: Sequence[DenseEmbedder], sparse_embedder: SparseEmbedder
|
||||||
|
) -> str:
|
||||||
|
"""Compose the `embedding_model_version` payload value (ADR-0001).
|
||||||
|
|
||||||
|
Sorted so the string is stable regardless of the order the embedders were
|
||||||
|
wired in — an unstable value would make "which chunks need re-embedding?"
|
||||||
|
unanswerable, which is the field's only reason to exist.
|
||||||
|
"""
|
||||||
|
versions = sorted(
|
||||||
|
[embedder.model_version for embedder in dense_embedders] + [sparse_embedder.model_version]
|
||||||
|
)
|
||||||
|
return "+".join(versions)
|
||||||
|
|
||||||
|
|
||||||
|
def _batches(points: Sequence[ChunkPoint], size: int) -> list[Sequence[ChunkPoint]]:
|
||||||
|
return [points[i : i + size] for i in range(0, len(points), size)]
|
||||||
|
|
||||||
|
|
||||||
|
def _build_points(
|
||||||
|
embedded: Sequence[EmbeddedChunk],
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
source_filename: str,
|
||||||
|
source_type: str,
|
||||||
|
actor: str,
|
||||||
|
embedding_model_version: str,
|
||||||
|
indexed_at: datetime,
|
||||||
|
) -> list[ChunkPoint]:
|
||||||
|
"""Blocking: hashes every chunk's content. Always called through a thread."""
|
||||||
|
return [
|
||||||
|
ChunkPoint(
|
||||||
|
point_id=item.chunk.chunk_id,
|
||||||
|
dense=item.dense,
|
||||||
|
sparse=item.sparse,
|
||||||
|
payload=build_chunk_payload(
|
||||||
|
item.chunk,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
source_filename=source_filename,
|
||||||
|
source_type=source_type,
|
||||||
|
actor=actor,
|
||||||
|
embedding_model_version=embedding_model_version,
|
||||||
|
indexed_at=indexed_at,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for item in embedded
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def _upsert_bounded(
|
||||||
|
storage: PointStorage, batch: Sequence[ChunkPoint], *, semaphore: asyncio.Semaphore
|
||||||
|
) -> None:
|
||||||
|
async with semaphore:
|
||||||
|
try:
|
||||||
|
await storage.upsert_points(batch)
|
||||||
|
except Exception as exc:
|
||||||
|
raise PointIndexingError(f"upserting {len(batch)} points failed: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
async def _deactivate_stale(
|
||||||
|
storage: PointStorage,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
from_chunk_index: int,
|
||||||
|
actor: str,
|
||||||
|
deleted_at: datetime,
|
||||||
|
) -> int:
|
||||||
|
"""Soft-delete points left over from a longer previous version of this file.
|
||||||
|
|
||||||
|
Chunk indices are contiguous from 0, so "index >= the new chunk count" is
|
||||||
|
exactly the set of points the new version no longer produces.
|
||||||
|
|
||||||
|
This runs **only after every upsert has succeeded**, and that ordering is
|
||||||
|
what keeps a failed attempt from damaging a working index. ADR-0001's
|
||||||
|
deterministic point ids mean a re-ingestion overwrites in place, so literal
|
||||||
|
atomic replacement is not available; what *is* guaranteed is that a failed
|
||||||
|
attempt never removes content (it can only leave a prefix updated), and that
|
||||||
|
a retry converges to the correct state. See ADR-0017.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return await storage.deactivate_points_from_index(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
file_id=file_id,
|
||||||
|
from_chunk_index=from_chunk_index,
|
||||||
|
deleted_at=deleted_at,
|
||||||
|
updated_by=actor,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise PointIndexingError(f"soft-deleting stale points failed: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
|
async def index_chunks(
|
||||||
|
embedded: Sequence[EmbeddedChunk],
|
||||||
|
*,
|
||||||
|
storage: PointStorage,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
source_filename: str,
|
||||||
|
source_type: str,
|
||||||
|
actor: str,
|
||||||
|
dense_embedders: Sequence[DenseEmbedder],
|
||||||
|
sparse_embedder: SparseEmbedder,
|
||||||
|
settings: QdrantSettings,
|
||||||
|
thread_limiter: CapacityLimiter,
|
||||||
|
) -> IndexingResult:
|
||||||
|
"""Upsert every embedded chunk as a tenant-scoped point, then sweep leftovers.
|
||||||
|
|
||||||
|
Raises `PointIndexingError` (502) if any batch or the sweep fails.
|
||||||
|
"""
|
||||||
|
if not embedded:
|
||||||
|
return IndexingResult(points_upserted=0, points_soft_deleted=0)
|
||||||
|
|
||||||
|
indexed_at = datetime.now(UTC)
|
||||||
|
points = await to_thread.run_sync(
|
||||||
|
lambda: _build_points(
|
||||||
|
embedded,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
source_filename=source_filename,
|
||||||
|
source_type=source_type,
|
||||||
|
actor=actor,
|
||||||
|
embedding_model_version=_embedding_model_version(dense_embedders, sparse_embedder),
|
||||||
|
indexed_at=indexed_at,
|
||||||
|
),
|
||||||
|
limiter=thread_limiter,
|
||||||
|
)
|
||||||
|
|
||||||
|
semaphore = asyncio.Semaphore(settings.upsert_concurrency)
|
||||||
|
await asyncio.gather(
|
||||||
|
*(
|
||||||
|
_upsert_bounded(storage, batch, semaphore=semaphore)
|
||||||
|
for batch in _batches(points, settings.upsert_batch_size)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
soft_deleted = await _deactivate_stale(
|
||||||
|
storage,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
file_id=file_id,
|
||||||
|
from_chunk_index=len(points),
|
||||||
|
actor=actor,
|
||||||
|
deleted_at=indexed_at,
|
||||||
|
)
|
||||||
|
return IndexingResult(points_upserted=len(points), points_soft_deleted=soft_deleted)
|
||||||
29
src/application/points/models.py
Normal file
29
src/application/points/models.py
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
"""Domain models for Qdrant points (ADR-0001).
|
||||||
|
|
||||||
|
Deliberately free of the `qdrant_client` SDK: `src/infrastructure/qdrant/`
|
||||||
|
converts these to `PointStruct`/`models.SparseVector` at upsert time
|
||||||
|
(ADR-0015 — application code and ports carry no infrastructure imports).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from src.application.ingestion.models import SparseVector
|
||||||
|
|
||||||
|
|
||||||
|
class ChunkPoint(BaseModel):
|
||||||
|
"""One chunk, ready to upsert: its id, its named vectors, and its payload.
|
||||||
|
|
||||||
|
`point_id` is the chunk's deterministic UUIDv5 (`chunk_id_for`), so
|
||||||
|
re-ingesting a file overwrites its points rather than duplicating them
|
||||||
|
(ADR-0001).
|
||||||
|
|
||||||
|
`dense` is keyed by named-vector name (`dense_nomic`, `dense_openai`).
|
||||||
|
`late_interaction` is absent — ADR-0017 does not compute it at ingest.
|
||||||
|
"""
|
||||||
|
|
||||||
|
point_id: uuid.UUID
|
||||||
|
dense: dict[str, list[float]]
|
||||||
|
sparse: SparseVector
|
||||||
|
payload: dict[str, object] = Field(default_factory=dict)
|
||||||
70
src/application/points/payload.py
Normal file
70
src/application/points/payload.py
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
"""Builds ADR-0001's point payload from a chunk plus its ingestion context.
|
||||||
|
|
||||||
|
Internal to `src/application/points/` — callers use `index_chunks`, which owns
|
||||||
|
composing this with batching and the deactivation sweep. Exported for its own
|
||||||
|
unit tests, not as a surface to build payloads by hand.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
from hashlib import sha256
|
||||||
|
|
||||||
|
from src.application.ingestion.models import Chunk
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_id(value: uuid.UUID | None) -> str | None:
|
||||||
|
return str(value) if value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def build_chunk_payload(
|
||||||
|
chunk: Chunk,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
source_filename: str,
|
||||||
|
source_type: str,
|
||||||
|
actor: str,
|
||||||
|
embedding_model_version: str,
|
||||||
|
indexed_at: datetime,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Return ADR-0001's payload for one chunk.
|
||||||
|
|
||||||
|
`tenant_id` and `domain` are passed in from the server-derived `AuthContext`
|
||||||
|
and the validated request — never from anything the client could assert as
|
||||||
|
authority (ADR-0002's non-negotiable isolation rule).
|
||||||
|
|
||||||
|
UUIDs are serialized as strings because the `tenant_id`/`domain`/`file_id`/
|
||||||
|
`previous_chunk_id`/`next_chunk_id` payload indexes are *keyword* indexes;
|
||||||
|
a native UUID would not match a keyword filter.
|
||||||
|
|
||||||
|
**Known gap — `version` is always written as `1`.** ADR-0002 uses this field
|
||||||
|
for optimistic concurrency between ingestion and manual `/v1/points` edits,
|
||||||
|
which needs a read-check-write (one read per point). Ingestion is
|
||||||
|
authoritative for its own file today, so writing `1` is safe until
|
||||||
|
`/v1/points` exists; plan 002 owns closing this.
|
||||||
|
"""
|
||||||
|
timestamp = indexed_at.isoformat()
|
||||||
|
return {
|
||||||
|
"tenant_id": str(tenant_id),
|
||||||
|
"domain": domain,
|
||||||
|
"file_id": str(file_id),
|
||||||
|
"chunk_id": str(chunk.chunk_id),
|
||||||
|
"content": chunk.content,
|
||||||
|
"content_type": chunk.content_type.value,
|
||||||
|
"source_filename": source_filename,
|
||||||
|
"source_type": source_type,
|
||||||
|
"order_id": chunk.order_id,
|
||||||
|
"chunk_index": chunk.chunk_index,
|
||||||
|
"previous_chunk_id": _optional_id(chunk.previous_chunk_id),
|
||||||
|
"next_chunk_id": _optional_id(chunk.next_chunk_id),
|
||||||
|
"is_active": True,
|
||||||
|
"deleted_at": None,
|
||||||
|
"created_at": timestamp,
|
||||||
|
"updated_at": timestamp,
|
||||||
|
"created_by": actor,
|
||||||
|
"updated_by": actor,
|
||||||
|
"version": 1,
|
||||||
|
"content_hash": sha256(chunk.content.encode("utf-8")).hexdigest(),
|
||||||
|
"embedding_model_version": embedding_model_version,
|
||||||
|
}
|
||||||
126
src/application/points/point.py
Normal file
126
src/application/points/point.py
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
"""The caller-facing point model for `/v1/points` (ADR-0001, ADR-0002).
|
||||||
|
|
||||||
|
`ChunkPoint` in `models.py` is the *write* shape ingestion upserts: an id, its
|
||||||
|
named vectors, and an opaque payload dict. This module is the *read/edit* shape
|
||||||
|
the `/v1/points` surface works in, where the payload's individual fields matter
|
||||||
|
and the distinction between what a caller may write and what the server owns is
|
||||||
|
a security boundary rather than a convention.
|
||||||
|
|
||||||
|
That split is the reason this is a model and not a dict. ADR-0002's isolation
|
||||||
|
rule ("never accepted as client-supplied input") and its optimistic-concurrency
|
||||||
|
guard both fail open if a caller can smuggle `tenant_id` or `version` through a
|
||||||
|
payload update, so the writable field set is enumerated in one place here and
|
||||||
|
every write path validates against it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Self
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
# Fields the server derives and a caller may never set, patch, or override.
|
||||||
|
# `tenant_id` is authority, `version` is the concurrency guard, `chunk_index`
|
||||||
|
# derives the point id, and the rest are provenance the server timestamps.
|
||||||
|
SERVER_OWNED_FIELDS: frozenset[str] = frozenset(
|
||||||
|
{
|
||||||
|
"tenant_id",
|
||||||
|
"version",
|
||||||
|
"chunk_index",
|
||||||
|
"chunk_id",
|
||||||
|
"is_active",
|
||||||
|
"deleted_at",
|
||||||
|
"created_at",
|
||||||
|
"created_by",
|
||||||
|
"updated_at",
|
||||||
|
"updated_by",
|
||||||
|
"content_hash",
|
||||||
|
"embedding_model_version",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fields a caller may supply on create, replace, or payload patch. `order_id`
|
||||||
|
# is writable on create but moves only through `PATCH /v1/points/{id}/order`
|
||||||
|
# afterwards, because a bare `order_id` write would not relink neighbours.
|
||||||
|
CALLER_WRITABLE_FIELDS: frozenset[str] = frozenset(
|
||||||
|
{
|
||||||
|
"content",
|
||||||
|
"content_type",
|
||||||
|
"domain",
|
||||||
|
"source_filename",
|
||||||
|
"source_type",
|
||||||
|
"order_id",
|
||||||
|
"previous_chunk_id",
|
||||||
|
"next_chunk_id",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Point(BaseModel):
|
||||||
|
"""One Qdrant point, read back with its ADR-0001 payload fields typed.
|
||||||
|
|
||||||
|
Vectors are deliberately absent: ADR-0008 returns them only when explicitly
|
||||||
|
requested, and every read path that does not ask for them should not pay to
|
||||||
|
deserialize them.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = ConfigDict(frozen=True)
|
||||||
|
|
||||||
|
point_id: uuid.UUID
|
||||||
|
|
||||||
|
tenant_id: uuid.UUID
|
||||||
|
domain: str
|
||||||
|
file_id: uuid.UUID
|
||||||
|
chunk_id: uuid.UUID
|
||||||
|
|
||||||
|
content: str
|
||||||
|
content_type: str
|
||||||
|
source_filename: str
|
||||||
|
source_type: str
|
||||||
|
|
||||||
|
order_id: float
|
||||||
|
chunk_index: int
|
||||||
|
previous_chunk_id: uuid.UUID | None = None
|
||||||
|
next_chunk_id: uuid.UUID | None = None
|
||||||
|
|
||||||
|
is_active: bool = True
|
||||||
|
deleted_at: datetime | None = None
|
||||||
|
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
created_by: str
|
||||||
|
updated_by: str
|
||||||
|
|
||||||
|
version: int
|
||||||
|
content_hash: str
|
||||||
|
embedding_model_version: str
|
||||||
|
|
||||||
|
# Only populated when the caller explicitly asked for vectors.
|
||||||
|
vectors: dict[str, object] | None = Field(default=None)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_payload(
|
||||||
|
cls,
|
||||||
|
point_id: uuid.UUID,
|
||||||
|
payload: Mapping[str, object],
|
||||||
|
*,
|
||||||
|
vectors: Mapping[str, object] | None = None,
|
||||||
|
) -> Self:
|
||||||
|
"""Build a `Point` from a raw Qdrant payload dict.
|
||||||
|
|
||||||
|
Lives here rather than in the Qdrant adapter so the payload field names
|
||||||
|
are declared once, next to the model that mirrors them. The adapter
|
||||||
|
stays responsible for talking to the SDK, not for knowing ADR-0001's
|
||||||
|
schema twice.
|
||||||
|
"""
|
||||||
|
return cls.model_validate({**payload, "point_id": point_id, "vectors": vectors})
|
||||||
|
|
||||||
|
|
||||||
|
class PointNotFoundError(LookupError):
|
||||||
|
"""No such point *within the requesting tenant*.
|
||||||
|
|
||||||
|
Routes map this to `404`, never `403` — a caller must not be able to probe
|
||||||
|
for the existence of another tenant's point ids (ADR-0016). The error
|
||||||
|
deliberately carries no hint about which of the two cases occurred.
|
||||||
|
"""
|
||||||
115
src/application/points/queries.py
Normal file
115
src/application/points/queries.py
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
"""Read paths for `/v1/points` (ADR-0002, ADR-0008).
|
||||||
|
|
||||||
|
The caller-facing entry points for point reads. Routers call these; they never
|
||||||
|
touch the `PointRepository` directly, and never build a filter.
|
||||||
|
|
||||||
|
This module is thin on purpose but not empty, and the two things it does own are
|
||||||
|
exactly the ones a route would otherwise get wrong:
|
||||||
|
|
||||||
|
- **`tenant_id` always comes from the caller's `AuthContext`.** Every function
|
||||||
|
takes it as a required keyword and hands it to the repository. Nothing here
|
||||||
|
reads a tenant from a query string or body.
|
||||||
|
- **A keyword query is Persian-normalized before it reaches the index.**
|
||||||
|
Ingestion letter-folds chunk content (`normalize_persian_text`, ADR-0018), so
|
||||||
|
stored text contains Persian yeh/keheh. A query typed on an Arabic keyboard
|
||||||
|
carries U+064A/U+0643 and would match nothing at all — a silent empty result,
|
||||||
|
not an error. Folding the query the same way is what makes the two comparable.
|
||||||
|
|
||||||
|
Reads emit no log events. The request middleware already records every call, and
|
||||||
|
ADR-0011 reserves `INFO` for lifecycle events rather than per-read volume;
|
||||||
|
mutations get their own events when those paths land.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from src.application.ingestion.normalization import normalize_persian_text
|
||||||
|
from src.application.points.point import Point, PointNotFoundError
|
||||||
|
from src.application.ports.point_repository import PointPage, PointRepository
|
||||||
|
|
||||||
|
|
||||||
|
async def get_point(
|
||||||
|
repository: PointRepository,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
point_id: uuid.UUID,
|
||||||
|
with_vectors: bool = False,
|
||||||
|
) -> Point:
|
||||||
|
"""One point, or `PointNotFoundError` if it is not this tenant's.
|
||||||
|
|
||||||
|
Raises rather than returning `None` so a route cannot forget the check and
|
||||||
|
serve `200 null`. "Absent" and "another tenant's" are the same outcome by
|
||||||
|
design (ADR-0016: cross-tenant access is `404`, never `403`).
|
||||||
|
"""
|
||||||
|
point = await repository.get(tenant_id=tenant_id, point_id=point_id, with_vectors=with_vectors)
|
||||||
|
if point is None:
|
||||||
|
raise PointNotFoundError(f"point {point_id} not found")
|
||||||
|
return point
|
||||||
|
|
||||||
|
|
||||||
|
async def list_file_points(
|
||||||
|
repository: PointRepository,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointPage:
|
||||||
|
"""One file's points in display (`order_id`) order.
|
||||||
|
|
||||||
|
An unknown or foreign `file_id` yields an empty page rather than an error:
|
||||||
|
the two are indistinguishable to the caller, which is the same
|
||||||
|
non-disclosure property `get_point` gets from raising.
|
||||||
|
"""
|
||||||
|
return await repository.list_by_file(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
file_id=file_id,
|
||||||
|
limit=limit,
|
||||||
|
cursor=cursor,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def count_points(
|
||||||
|
repository: PointRepository,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> int:
|
||||||
|
return await repository.count(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def search_points(
|
||||||
|
repository: PointRepository,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
query: str,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None = None,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointPage:
|
||||||
|
"""Keyword match on `content`, within this tenant.
|
||||||
|
|
||||||
|
**Not semantic retrieval.** Qdrant's full-text index filters rather than
|
||||||
|
scores, so results carry no relevance ranking and their order is
|
||||||
|
unspecified. Ranked retrieval is ADR-0003's hybrid path in plan 003; this
|
||||||
|
function must not grow a semantic mode (ADR-0002).
|
||||||
|
"""
|
||||||
|
return await repository.keyword_search(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
query=normalize_persian_text(query),
|
||||||
|
limit=limit,
|
||||||
|
cursor=cursor,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
135
src/application/points/relinking.py
Normal file
135
src/application/points/relinking.py
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
"""Adjacency-pointer maintenance for a point leaving a file's sequence.
|
||||||
|
|
||||||
|
ADR-0001 keeps `previous_chunk_id`/`next_chunk_id` on every point so ADR-0003's
|
||||||
|
context-window expansion can walk a file in O(1) steps. ADR-0002 makes keeping
|
||||||
|
them correct an obligation of every operation that changes a point's position:
|
||||||
|
a partial relink is a defect, not a degraded-but-acceptable outcome.
|
||||||
|
|
||||||
|
The function below is the primitive that obligation reduces to. It is pure, and
|
||||||
|
it is written as **"what is still missing between the state I just read and the
|
||||||
|
state I want"** rather than "the patches a delete implies". That framing is what
|
||||||
|
makes the caller's retry loop correct: re-planning after a partial apply emits
|
||||||
|
exactly the patches that did not land, and re-planning after a completed delete
|
||||||
|
emits nothing at all. The three cases the plan calls out — a normal delete, a
|
||||||
|
second delete of an already-inactive point, and recovery from a half-applied
|
||||||
|
batch — are then one code path instead of three.
|
||||||
|
|
||||||
|
Note what is deliberately *not* patched: the departing point's own
|
||||||
|
`previous_chunk_id`/`next_chunk_id`. Nothing active points at it once its
|
||||||
|
neighbours are relinked, so those pointers are unreachable rather than stale,
|
||||||
|
and leaving them records where the point sat — which is what a later restore or
|
||||||
|
an audit reader would need. `src/application/points/deletion.py` relies on that
|
||||||
|
when it re-plans: the departing point's pointers are the only surviving record
|
||||||
|
of which two neighbours have to be joined.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from src.application.points.point import Point
|
||||||
|
from src.application.ports.point_repository import PayloadPatch
|
||||||
|
|
||||||
|
|
||||||
|
def _optional_id(value: uuid.UUID | None) -> str | None:
|
||||||
|
return str(value) if value is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _provenance(point: Point, *, actor: str, now: datetime) -> dict[str, object]:
|
||||||
|
"""The fields every mutation writes: who, when, and the next version.
|
||||||
|
|
||||||
|
Bumping `version` on a relinked *neighbour* is intentional. The neighbour's
|
||||||
|
payload really did change, so a concurrent editor holding the old version
|
||||||
|
must get a `409` rather than overwrite the pointer we just fixed.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"updated_at": now.isoformat(),
|
||||||
|
"updated_by": actor,
|
||||||
|
"version": point.version + 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def neighbour_ids(point: Point) -> tuple[uuid.UUID, ...]:
|
||||||
|
"""The ids `patches_for_removal` needs loaded, skipping the nulls."""
|
||||||
|
return tuple(
|
||||||
|
pointer for pointer in (point.previous_chunk_id, point.next_chunk_id) if pointer is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def patches_for_removal(
|
||||||
|
point: Point,
|
||||||
|
neighbours: Mapping[uuid.UUID, Point],
|
||||||
|
*,
|
||||||
|
actor: str,
|
||||||
|
now: datetime,
|
||||||
|
) -> tuple[PayloadPatch, ...]:
|
||||||
|
"""The patches still needed to remove `point` from its file's sequence.
|
||||||
|
|
||||||
|
Returns an empty tuple when the removal is already complete, which the
|
||||||
|
caller reads as both "converged" and "this was a no-op".
|
||||||
|
|
||||||
|
A neighbour absent from `neighbours` is skipped rather than patched blind:
|
||||||
|
its id came from the departing point's payload, so a missing one means the
|
||||||
|
chain was already broken, and inventing a patch for a point that is not
|
||||||
|
there would not fix it. The caller logs that case.
|
||||||
|
"""
|
||||||
|
patches: list[PayloadPatch] = []
|
||||||
|
|
||||||
|
if point.is_active:
|
||||||
|
patches.append(
|
||||||
|
PayloadPatch(
|
||||||
|
point_id=point.point_id,
|
||||||
|
payload={
|
||||||
|
"is_active": False,
|
||||||
|
"deleted_at": now.isoformat(),
|
||||||
|
**_provenance(point, actor=actor, now=now),
|
||||||
|
},
|
||||||
|
expected_version=point.version,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
previous = neighbours.get(point.previous_chunk_id) if point.previous_chunk_id else None
|
||||||
|
if previous is not None and previous.next_chunk_id != point.next_chunk_id:
|
||||||
|
patches.append(
|
||||||
|
PayloadPatch(
|
||||||
|
point_id=previous.point_id,
|
||||||
|
payload={
|
||||||
|
"next_chunk_id": _optional_id(point.next_chunk_id),
|
||||||
|
**_provenance(previous, actor=actor, now=now),
|
||||||
|
},
|
||||||
|
expected_version=previous.version,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
following = neighbours.get(point.next_chunk_id) if point.next_chunk_id else None
|
||||||
|
if following is not None and following.previous_chunk_id != point.previous_chunk_id:
|
||||||
|
patches.append(
|
||||||
|
PayloadPatch(
|
||||||
|
point_id=following.point_id,
|
||||||
|
payload={
|
||||||
|
"previous_chunk_id": _optional_id(point.previous_chunk_id),
|
||||||
|
**_provenance(following, actor=actor, now=now),
|
||||||
|
},
|
||||||
|
expected_version=following.version,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return tuple(patches)
|
||||||
|
|
||||||
|
|
||||||
|
def patch_for_deactivation(point: Point, *, actor: str, now: datetime) -> PayloadPatch:
|
||||||
|
"""Deactivate one point without touching any pointer.
|
||||||
|
|
||||||
|
Used by the whole-file sweep, where every point in the file leaves at once:
|
||||||
|
no active point survives to dangle, so there is no neighbour to relink and
|
||||||
|
the chain stays intact for a later reader of the deactivated file.
|
||||||
|
"""
|
||||||
|
return PayloadPatch(
|
||||||
|
point_id=point.point_id,
|
||||||
|
payload={
|
||||||
|
"is_active": False,
|
||||||
|
"deleted_at": now.isoformat(),
|
||||||
|
**_provenance(point, actor=actor, now=now),
|
||||||
|
},
|
||||||
|
expected_version=point.version,
|
||||||
|
)
|
||||||
7
src/application/ports/__init__.py
Normal file
7
src/application/ports/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
"""Narrow contracts for external side effects (ADR-0015).
|
||||||
|
|
||||||
|
Ports exist for external side effects/persistence that need a swappable or
|
||||||
|
fake-able boundary — not as a blanket wrapper around every database access.
|
||||||
|
`object_storage.py` is one: MinIO is a real external system with its own
|
||||||
|
failure modes, and ADR-0016 requires a hand-written fake for it in tests.
|
||||||
|
"""
|
||||||
63
src/application/ports/embedding.py
Normal file
63
src/application/ports/embedding.py
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
"""Embedding ports (ADR-0001, ADR-0017).
|
||||||
|
|
||||||
|
`src/infrastructure/embedding/` holds the production adapters; tests use
|
||||||
|
scripted fakes (ADR-0016). Application code depends on these Protocols, not
|
||||||
|
on `httpx`/provider SDKs directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
from src.application.ingestion.models import SparseVector
|
||||||
|
|
||||||
|
|
||||||
|
class DenseEmbedder(Protocol):
|
||||||
|
"""One named dense vector's embedding client (`dense_nomic`/`dense_openai`).
|
||||||
|
|
||||||
|
`embed_batch` is a single batched network call — callers own concurrency
|
||||||
|
bounding (ADR-0017's `embed_concurrency` semaphore), not this Protocol.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
model_version: str
|
||||||
|
"""Identifies the model that produced these vectors (ADR-0001).
|
||||||
|
|
||||||
|
Written into every point's `embedding_model_version` payload field, which
|
||||||
|
exists so a future model swap can tell which chunks need re-embedding. The
|
||||||
|
embedder is what knows this, so it is reported here rather than
|
||||||
|
reconstructed from configuration at the call site.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def embed_batch(self, texts: Sequence[str]) -> list[list[float]]:
|
||||||
|
"""Return one vector per input text, same order. Raises `EmbedderError`
|
||||||
|
(see `src/application/ingestion/errors.py`) on transport/response
|
||||||
|
failure.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
class SparseEmbedder(Protocol):
|
||||||
|
"""The `sparse` (BM25) vector's embedding client.
|
||||||
|
|
||||||
|
Blocking/CPU-bound (ADR-0017): callers offload it via
|
||||||
|
`anyio.to_thread.run_sync` with the ingestion `CapacityLimiter`, not call
|
||||||
|
it directly from an `async def`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
model_version: str
|
||||||
|
"""Identifies the analyzer/parameters that produced these vectors.
|
||||||
|
|
||||||
|
Same purpose as `DenseEmbedder.model_version`; for BM25 the "model" is the
|
||||||
|
analyzer choice (ADR-0005), which is equally a re-embedding trigger.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def embed_batch(self, texts: Sequence[str], *, query: bool = False) -> list[SparseVector]:
|
||||||
|
"""Return one sparse vector per input text, same order.
|
||||||
|
|
||||||
|
`query=True` selects the query-side weighting, which omits document
|
||||||
|
length normalization. Ingestion always passes `False`; the flag exists
|
||||||
|
so retrieval (ADR-0003) encodes queries through this same port rather
|
||||||
|
than growing a second, silently divergent implementation.
|
||||||
|
"""
|
||||||
|
...
|
||||||
14
src/application/ports/object_storage.py
Normal file
14
src/application/ports/object_storage.py
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
"""The object-storage port (ADR-0013).
|
||||||
|
|
||||||
|
`src/infrastructure/minio/storage.py` is the production adapter; tests use a
|
||||||
|
hand-written fake (ADR-0016). Application code depends on this Protocol, not
|
||||||
|
on the `minio` SDK.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectStorage(Protocol):
|
||||||
|
async def put_object(self, *, key: str, data: bytes, content_type: str) -> None:
|
||||||
|
"""Store `data` privately under `key`. Overwrites an existing object."""
|
||||||
|
...
|
||||||
131
src/application/ports/point_repository.py
Normal file
131
src/application/ports/point_repository.py
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
"""The point read/edit port for `/v1/points` (ADR-0002, ADR-0015).
|
||||||
|
|
||||||
|
Separate from `PointStorage`, which stays exactly the two bulk operations
|
||||||
|
ingestion performs. Reads, single-point edits, reordering, and keyword search
|
||||||
|
have a different caller, a different failure vocabulary, and a different
|
||||||
|
tenant-filter obligation, so they get their own port rather than accreting onto
|
||||||
|
the ingestion one.
|
||||||
|
|
||||||
|
`tenant_id` is a required keyword argument on **every** method. That is not
|
||||||
|
style: ADR-0002's isolation rule has to hold on every code path that touches the
|
||||||
|
collection, and an optional tenant filter is one forgotten argument away from a
|
||||||
|
cross-tenant read. Making it required moves that from a review question to a
|
||||||
|
type error.
|
||||||
|
|
||||||
|
`src/infrastructure/qdrant/point_repository.py` is the production adapter;
|
||||||
|
`tests.fakes.FakePointRepository` is the test double.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from src.application.points.point import Point
|
||||||
|
|
||||||
|
|
||||||
|
class PointPage(BaseModel):
|
||||||
|
"""One page of points plus the cursor that continues it.
|
||||||
|
|
||||||
|
`next_cursor` is opaque to callers and encoded by the adapter: ordered
|
||||||
|
scrolls and keyword searches paginate by different Qdrant mechanisms, and
|
||||||
|
neither is a plain integer offset. `None` means the listing is exhausted.
|
||||||
|
"""
|
||||||
|
|
||||||
|
points: tuple[Point, ...]
|
||||||
|
next_cursor: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class PayloadPatch(BaseModel):
|
||||||
|
"""Set these payload fields on one point, optionally guarded by `version`.
|
||||||
|
|
||||||
|
When `expected_version` is set, the adapter attaches it to the operation's
|
||||||
|
filter, so a concurrent write that has already moved the version on means
|
||||||
|
this patch matches nothing rather than clobbering it. The guard is what
|
||||||
|
makes a lost update impossible; detecting that it fired is the service's
|
||||||
|
job (see `apply_patches`).
|
||||||
|
"""
|
||||||
|
|
||||||
|
point_id: uuid.UUID
|
||||||
|
payload: dict[str, object]
|
||||||
|
expected_version: int | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class PointRepository(Protocol):
|
||||||
|
async def get(
|
||||||
|
self, *, tenant_id: uuid.UUID, point_id: uuid.UUID, with_vectors: bool = False
|
||||||
|
) -> Point | None:
|
||||||
|
"""One point, or `None` if it does not exist *under this tenant*.
|
||||||
|
|
||||||
|
The two cases are deliberately indistinguishable — the route maps both
|
||||||
|
to `404` so a caller cannot probe for another tenant's point ids.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
async def get_many(
|
||||||
|
self, *, tenant_id: uuid.UUID, point_ids: Sequence[uuid.UUID]
|
||||||
|
) -> tuple[Point, ...]:
|
||||||
|
"""The subset of `point_ids` that exists under this tenant.
|
||||||
|
|
||||||
|
Order is not guaranteed and missing ids are silently absent: callers are
|
||||||
|
neighbour-relinking and batch precondition checks, both of which match
|
||||||
|
on id rather than position.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
async def list_by_file(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointPage:
|
||||||
|
"""One file's points in `order_id` order (ADR-0008's `scroll`).
|
||||||
|
|
||||||
|
Scoped to a single file because the cursor is an `order_id` value, and
|
||||||
|
`order_id` is only unique within a file.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
async def count(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> int: ...
|
||||||
|
|
||||||
|
async def keyword_search(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
query: str,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None = None,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointPage:
|
||||||
|
"""Full-text payload match on `content`, plus structured filters.
|
||||||
|
|
||||||
|
Keyword matching, **not** semantic retrieval (ADR-0002). Qdrant's
|
||||||
|
full-text index is a filter, not a scorer, so results carry no relevance
|
||||||
|
ranking and their order is unspecified.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
async def apply_patches(self, *, tenant_id: uuid.UUID, patches: Sequence[PayloadPatch]) -> None:
|
||||||
|
"""Apply every patch in one Qdrant `points/batch` request.
|
||||||
|
|
||||||
|
Qdrant has no multi-point transaction, so this is not atomic and does
|
||||||
|
not pretend to be. ADR-0002's all-or-nothing rule is implemented one
|
||||||
|
layer up as validate-every-precondition-then-apply; the per-patch
|
||||||
|
`expected_version` guard here is what makes the residual window safe,
|
||||||
|
turning a lost update into a no-op the service can detect rather than a
|
||||||
|
silent clobber.
|
||||||
|
"""
|
||||||
|
...
|
||||||
45
src/application/ports/point_storage.py
Normal file
45
src/application/ports/point_storage.py
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
"""The point-storage port (ADR-0001, ADR-0015).
|
||||||
|
|
||||||
|
`src/infrastructure/qdrant/points.py` is the production adapter; tests use a
|
||||||
|
hand-written fake (ADR-0016). Application code depends on this Protocol, not on
|
||||||
|
the `qdrant_client` SDK.
|
||||||
|
|
||||||
|
Deliberately narrow: exactly the two operations ingestion performs. Reads,
|
||||||
|
single-point edits, reordering, and keyword search are plan 002's `/v1/points`
|
||||||
|
surface and belong on a port of their own rather than accreting here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
from src.application.points.models import ChunkPoint
|
||||||
|
|
||||||
|
|
||||||
|
class PointStorage(Protocol):
|
||||||
|
async def upsert_points(self, points: Sequence[ChunkPoint]) -> None:
|
||||||
|
"""Upsert one batch of points.
|
||||||
|
|
||||||
|
Callers own batching and concurrency bounding (ADR-0017's
|
||||||
|
`upsert_concurrency` semaphore), not this Protocol — the same division
|
||||||
|
`DenseEmbedder.embed_batch` uses.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
async def deactivate_points_from_index(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
from_chunk_index: int,
|
||||||
|
deleted_at: datetime,
|
||||||
|
updated_by: str,
|
||||||
|
) -> int:
|
||||||
|
"""Soft-delete this file's points at or past `from_chunk_index`.
|
||||||
|
|
||||||
|
Sets `is_active=false`/`deleted_at` rather than removing the points
|
||||||
|
(ADR-0002: delete is soft by default). Tenant-filtered — a `file_id`
|
||||||
|
alone is never sufficient authority. Returns how many points matched.
|
||||||
|
"""
|
||||||
|
...
|
||||||
9
src/application/tenants/__init__.py
Normal file
9
src/application/tenants/__init__.py
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
"""Operator-run tenant provisioning (ADR-0008, ADR-0009)."""
|
||||||
|
|
||||||
|
from src.application.tenants.provisioning import (
|
||||||
|
DEFAULT_SCOPES,
|
||||||
|
ProvisionResult,
|
||||||
|
provision_tenant,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = ["DEFAULT_SCOPES", "ProvisionResult", "provision_tenant"]
|
||||||
129
src/application/tenants/provisioning.py
Normal file
129
src/application/tenants/provisioning.py
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
"""Provision a tenant, its first API key, and its domains (ADR-0008, ADR-0009).
|
||||||
|
|
||||||
|
Nothing in the HTTP surface can bootstrap a tenant: every `/v1` route needs a
|
||||||
|
key, and a key can only exist once a tenant does. That chicken-and-egg is why
|
||||||
|
this is an operator-run deployment step (`src/cli/provision_tenant.py`) rather
|
||||||
|
than an endpoint — the same reasoning that keeps `alembic upgrade head` and
|
||||||
|
`qdrant_bootstrap` off the request path.
|
||||||
|
|
||||||
|
This is the package's only caller-facing entry point. It owns the whole
|
||||||
|
composition — tenant reuse-or-create, key generation and hashing, domain
|
||||||
|
registration, and the single transaction the three share — so a caller cannot
|
||||||
|
get the order wrong or commit a key whose tenant never landed (CLAUDE.md,
|
||||||
|
"prefer deep modules"). The plaintext key is returned exactly once and is never
|
||||||
|
logged (ADR-0011 forbids plaintext keys in logs); only its non-secret
|
||||||
|
`key_prefix` appears in the event.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.application.auth.keys import generate_api_key, hash_secret
|
||||||
|
from src.infrastructure.postgres.repositories import api_keys as api_keys_repo
|
||||||
|
from src.infrastructure.postgres.repositories import tenant_domains as domains_repo
|
||||||
|
from src.infrastructure.postgres.repositories import tenants as tenants_repo
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_SCOPES = (
|
||||||
|
"files:write",
|
||||||
|
"domains:read",
|
||||||
|
"domains:write",
|
||||||
|
"points:read",
|
||||||
|
"points:write",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ProvisionResult:
|
||||||
|
tenant_id: uuid.UUID
|
||||||
|
tenant_slug: str
|
||||||
|
tenant_created: bool
|
||||||
|
api_key_id: uuid.UUID
|
||||||
|
api_key_prefix: str
|
||||||
|
api_key: str
|
||||||
|
"""The plaintext bearer token. Only ever returned here — never stored, never logged."""
|
||||||
|
domains_created: tuple[str, ...]
|
||||||
|
domains_existing: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
async def provision_tenant(
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
*,
|
||||||
|
slug: str,
|
||||||
|
name: str | None = None,
|
||||||
|
key_name: str = "bootstrap",
|
||||||
|
scopes: tuple[str, ...] = DEFAULT_SCOPES,
|
||||||
|
domains: tuple[str, ...] = (),
|
||||||
|
actor_type: str = "backend",
|
||||||
|
) -> ProvisionResult:
|
||||||
|
"""Create (or reuse) the tenant, issue a key, and register `domains`.
|
||||||
|
|
||||||
|
Re-running with the same `slug` reuses the tenant and its existing domains
|
||||||
|
rather than failing, so an operator can add a key to a live tenant with the
|
||||||
|
same command they used to create it. A *new* key is issued on every run —
|
||||||
|
keys are write-once by construction (only the hash is stored), so there is
|
||||||
|
nothing to return for an existing one.
|
||||||
|
"""
|
||||||
|
key_prefix, secret, full_key = generate_api_key()
|
||||||
|
|
||||||
|
async with sessionmaker() as session:
|
||||||
|
tenant = await tenants_repo.get_by_slug(session, slug)
|
||||||
|
tenant_created = tenant is None
|
||||||
|
if tenant is None:
|
||||||
|
tenant = tenants_repo.create(session, slug=slug, name=name or slug)
|
||||||
|
# `api_keys.tenant_id` and `tenant_domains.tenant_id` FK to this row
|
||||||
|
# and the mapped classes carry no ORM relationship for the unit of
|
||||||
|
# work to order by itself, so the insert has to land first.
|
||||||
|
await session.flush()
|
||||||
|
|
||||||
|
api_key = api_keys_repo.create(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant.id,
|
||||||
|
name=key_name,
|
||||||
|
key_prefix=key_prefix,
|
||||||
|
key_hash=hash_secret(secret),
|
||||||
|
scopes=list(scopes),
|
||||||
|
actor_type=actor_type,
|
||||||
|
created_by="cli:provision_tenant",
|
||||||
|
)
|
||||||
|
|
||||||
|
created: list[str] = []
|
||||||
|
existing: list[str] = []
|
||||||
|
for domain in domains:
|
||||||
|
if await domains_repo.get(session, tenant_id=tenant.id, domain=domain) is not None:
|
||||||
|
existing.append(domain)
|
||||||
|
continue
|
||||||
|
domains_repo.create(session, tenant_id=tenant.id, domain=domain, display_name=domain)
|
||||||
|
created.append(domain)
|
||||||
|
|
||||||
|
await session.flush()
|
||||||
|
tenant_id, api_key_id, tenant_slug = tenant.id, api_key.id, tenant.slug
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
if tenant_created:
|
||||||
|
logger.info("tenant.provisioned", tenant_id=str(tenant_id), tenant_slug=tenant_slug)
|
||||||
|
for domain in created:
|
||||||
|
logger.info("domain.created", tenant_id=str(tenant_id), domain=domain)
|
||||||
|
logger.info(
|
||||||
|
"api_key.provisioned",
|
||||||
|
tenant_id=str(tenant_id),
|
||||||
|
api_key_id=str(api_key_id),
|
||||||
|
key_prefix=key_prefix,
|
||||||
|
scopes=list(scopes),
|
||||||
|
actor_type=actor_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
return ProvisionResult(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
tenant_slug=tenant_slug,
|
||||||
|
tenant_created=tenant_created,
|
||||||
|
api_key_id=api_key_id,
|
||||||
|
api_key_prefix=key_prefix,
|
||||||
|
api_key=full_key,
|
||||||
|
domains_created=tuple(created),
|
||||||
|
domains_existing=tuple(existing),
|
||||||
|
)
|
||||||
@@ -1,11 +1,16 @@
|
|||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator, Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from anyio import CapacityLimiter, Semaphore
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
from qdrant_client import AsyncQdrantClient
|
from qdrant_client import AsyncQdrantClient
|
||||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.application.ports.embedding import DenseEmbedder, SparseEmbedder
|
||||||
|
from src.application.ports.object_storage import ObjectStorage
|
||||||
|
from src.application.ports.point_repository import PointRepository
|
||||||
|
from src.application.ports.point_storage import PointStorage
|
||||||
from src.config import Settings
|
from src.config import Settings
|
||||||
|
|
||||||
|
|
||||||
@@ -16,6 +21,13 @@ class AppResources:
|
|||||||
db_sessionmaker: async_sessionmaker[AsyncSession]
|
db_sessionmaker: async_sessionmaker[AsyncSession]
|
||||||
minio_client: Minio
|
minio_client: Minio
|
||||||
qdrant_client: AsyncQdrantClient
|
qdrant_client: AsyncQdrantClient
|
||||||
|
object_storage: ObjectStorage
|
||||||
|
point_storage: PointStorage
|
||||||
|
point_repository: PointRepository
|
||||||
|
ingestion_limiter: CapacityLimiter
|
||||||
|
dense_embedders: Sequence[DenseEmbedder]
|
||||||
|
sparse_embedder: SparseEmbedder
|
||||||
|
ingestion_concurrency_limiter: Semaphore
|
||||||
|
|
||||||
|
|
||||||
def _resources(request: Request) -> AppResources:
|
def _resources(request: Request) -> AppResources:
|
||||||
@@ -34,6 +46,45 @@ def get_qdrant_client(request: Request) -> AsyncQdrantClient:
|
|||||||
return _resources(request).qdrant_client
|
return _resources(request).qdrant_client
|
||||||
|
|
||||||
|
|
||||||
|
def get_object_storage(request: Request) -> ObjectStorage:
|
||||||
|
return _resources(request).object_storage
|
||||||
|
|
||||||
|
|
||||||
|
def get_point_storage(request: Request) -> PointStorage:
|
||||||
|
return _resources(request).point_storage
|
||||||
|
|
||||||
|
|
||||||
|
def get_point_repository(request: Request) -> PointRepository:
|
||||||
|
return _resources(request).point_repository
|
||||||
|
|
||||||
|
|
||||||
|
def get_ingestion_limiter(request: Request) -> CapacityLimiter:
|
||||||
|
return _resources(request).ingestion_limiter
|
||||||
|
|
||||||
|
|
||||||
|
def get_dense_embedders(request: Request) -> Sequence[DenseEmbedder]:
|
||||||
|
return _resources(request).dense_embedders
|
||||||
|
|
||||||
|
|
||||||
|
def get_sparse_embedder(request: Request) -> SparseEmbedder:
|
||||||
|
return _resources(request).sparse_embedder
|
||||||
|
|
||||||
|
|
||||||
|
def get_ingestion_concurrency_limiter(request: Request) -> Semaphore:
|
||||||
|
return _resources(request).ingestion_concurrency_limiter
|
||||||
|
|
||||||
|
|
||||||
|
def get_sessionmaker(request: Request) -> async_sessionmaker[AsyncSession]:
|
||||||
|
"""The session *factory*, not a request-scoped session.
|
||||||
|
|
||||||
|
Application services that own more than one transaction in a single
|
||||||
|
request (ADR-0017's two-phase upload) need to open and close sessions
|
||||||
|
themselves rather than borrow one request-scoped session that would
|
||||||
|
otherwise stay open across the whole request.
|
||||||
|
"""
|
||||||
|
return _resources(request).db_sessionmaker
|
||||||
|
|
||||||
|
|
||||||
async def get_db_session(request: Request) -> AsyncIterator[AsyncSession]:
|
async def get_db_session(request: Request) -> AsyncIterator[AsyncSession]:
|
||||||
sessionmaker = _resources(request).db_sessionmaker
|
sessionmaker = _resources(request).db_sessionmaker
|
||||||
async with sessionmaker() as session:
|
async with sessionmaker() as session:
|
||||||
|
|||||||
@@ -1,28 +1,66 @@
|
|||||||
from collections.abc import AsyncIterator, Callable
|
from collections.abc import AsyncIterator, Callable, Sequence
|
||||||
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
||||||
|
|
||||||
|
import httpx
|
||||||
import structlog
|
import structlog
|
||||||
from anyio import to_thread
|
from anyio import CapacityLimiter, Semaphore, to_thread
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
||||||
from src.application.ingestion import get_encoder
|
from src.application.ingestion import get_encoder
|
||||||
|
from src.application.ports.embedding import DenseEmbedder
|
||||||
from src.bootstrap.dependencies import AppResources
|
from src.bootstrap.dependencies import AppResources
|
||||||
from src.config import Settings
|
from src.config import Settings
|
||||||
|
from src.infrastructure.embedding.bm25 import Bm25SparseEmbedder
|
||||||
|
from src.infrastructure.embedding.openai_compatible import (
|
||||||
|
OpenAICompatibleEmbedder,
|
||||||
|
is_ollama_base_url,
|
||||||
|
)
|
||||||
from src.infrastructure.minio.client import create_client as create_minio_client
|
from src.infrastructure.minio.client import create_client as create_minio_client
|
||||||
|
from src.infrastructure.minio.storage import MinioObjectStorage
|
||||||
from src.infrastructure.observability.logging import configure_logging
|
from src.infrastructure.observability.logging import configure_logging
|
||||||
from src.infrastructure.postgres.database import create_engine, create_sessionmaker
|
from src.infrastructure.postgres.database import create_engine, create_sessionmaker
|
||||||
from src.infrastructure.qdrant.client import create_client as create_qdrant_client
|
from src.infrastructure.qdrant.client import create_client as create_qdrant_client
|
||||||
|
from src.infrastructure.qdrant.point_repository import QdrantPointRepository
|
||||||
|
from src.infrastructure.qdrant.points import QdrantPointStorage
|
||||||
|
|
||||||
logger = structlog.get_logger(__name__)
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _auth_headers(api_key: str | None) -> dict[str, str]:
|
||||||
|
"""Bearer header, or none at all when no key is configured.
|
||||||
|
|
||||||
|
Sending an empty `Bearer ` is worse than sending nothing: some gateways
|
||||||
|
treat a malformed credential as an auth failure rather than as anonymous.
|
||||||
|
"""
|
||||||
|
return {"Authorization": f"Bearer {api_key}"} if api_key else {}
|
||||||
|
|
||||||
|
|
||||||
|
async def _warm_dense_embedders(embedders: Sequence[DenseEmbedder]) -> None:
|
||||||
|
"""Force each dense model to load before the first upload needs it.
|
||||||
|
|
||||||
|
Same rationale as the tiktoken warm-up above, but with the opposite
|
||||||
|
failure policy. A self-hosted embedder that has unloaded the model takes
|
||||||
|
minutes to serve its first request — longer than
|
||||||
|
`INGESTION_TIMEOUT_SECONDS` — so paying that once at boot keeps it off a
|
||||||
|
user's upload. Unlike the tokenizer this is best-effort: an embedder that
|
||||||
|
is merely *down* must not stop the process from booting and reporting its
|
||||||
|
own health, and `/readyz` is where that condition belongs.
|
||||||
|
"""
|
||||||
|
for embedder in embedders:
|
||||||
|
try:
|
||||||
|
await embedder.embed_batch(["warmup"])
|
||||||
|
logger.info("lifespan.embedder.warmed", embedder=embedder.name)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("lifespan.embedder.warm_failed", embedder=embedder.name, exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
def create_lifespan(
|
def create_lifespan(
|
||||||
settings: Settings | None = None,
|
settings: Settings | None = None,
|
||||||
) -> Callable[[FastAPI], AbstractAsyncContextManager[None, bool | None]]:
|
) -> Callable[[FastAPI], AbstractAsyncContextManager[None, bool | None]]:
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||||
resolved_settings = settings or Settings()
|
resolved_settings = settings or Settings()
|
||||||
configure_logging(resolved_settings.logging)
|
configure_logging(resolved_settings.logging, resolved_settings.app)
|
||||||
|
|
||||||
# tiktoken fetches its vocabulary over the network on first use, so warm
|
# tiktoken fetches its vocabulary over the network on first use, so warm
|
||||||
# it here: a missing vocabulary should fail the process at boot, not the
|
# it here: a missing vocabulary should fail the process at boot, not the
|
||||||
@@ -41,14 +79,81 @@ def create_lifespan(
|
|||||||
logger.info("lifespan.minio.client.created")
|
logger.info("lifespan.minio.client.created")
|
||||||
|
|
||||||
qdrant_client = create_qdrant_client(resolved_settings.qdrant)
|
qdrant_client = create_qdrant_client(resolved_settings.qdrant)
|
||||||
|
# No collection DDL here: `ensure_chunks_collection` is a deployment
|
||||||
|
# step (`python -m src.cli.qdrant_bootstrap`), for the same reason
|
||||||
|
# ADR-0009 keeps Alembic out of startup and ADR-0012 makes LangGraph's
|
||||||
|
# `.setup()` a deployment step.
|
||||||
|
point_storage = QdrantPointStorage(
|
||||||
|
qdrant_client, collection=resolved_settings.qdrant.collection
|
||||||
|
)
|
||||||
|
point_repository = QdrantPointRepository(
|
||||||
|
qdrant_client, collection=resolved_settings.qdrant.collection
|
||||||
|
)
|
||||||
logger.info("lifespan.qdrant.client.created")
|
logger.info("lifespan.qdrant.client.created")
|
||||||
|
|
||||||
|
nomic_settings = resolved_settings.embedding.nomic
|
||||||
|
nomic_http_client = httpx.AsyncClient(
|
||||||
|
base_url=nomic_settings.base_url,
|
||||||
|
timeout=nomic_settings.timeout_seconds,
|
||||||
|
headers=_auth_headers(nomic_settings.api_key),
|
||||||
|
)
|
||||||
|
openai_settings = resolved_settings.embedding.openai
|
||||||
|
openai_http_client = httpx.AsyncClient(
|
||||||
|
base_url=openai_settings.base_url,
|
||||||
|
timeout=openai_settings.timeout_seconds,
|
||||||
|
headers=_auth_headers(openai_settings.api_key),
|
||||||
|
)
|
||||||
|
dense_embedders = (
|
||||||
|
OpenAICompatibleEmbedder(
|
||||||
|
nomic_http_client,
|
||||||
|
name="dense_nomic",
|
||||||
|
model=nomic_settings.model,
|
||||||
|
document_prefix=nomic_settings.document_prefix,
|
||||||
|
keep_alive=(
|
||||||
|
nomic_settings.keep_alive
|
||||||
|
if is_ollama_base_url(nomic_settings.base_url)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
),
|
||||||
|
OpenAICompatibleEmbedder(
|
||||||
|
openai_http_client,
|
||||||
|
name="dense_openai",
|
||||||
|
model=openai_settings.model,
|
||||||
|
dimensions=openai_settings.dimensions,
|
||||||
|
document_prefix=openai_settings.document_prefix,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
sparse_embedder = Bm25SparseEmbedder(resolved_settings.embedding.sparse)
|
||||||
|
logger.info("lifespan.embedders.created")
|
||||||
|
|
||||||
|
await _warm_dense_embedders(dense_embedders)
|
||||||
|
|
||||||
|
# Bounds how many ingestions run in this process at once (ADR-0017);
|
||||||
|
# a distinct resource from ingestion_limiter, which bounds threads
|
||||||
|
# spent on blocking work within a single ingestion.
|
||||||
|
ingestion_concurrency_limiter = Semaphore(resolved_settings.ingestion.max_concurrency)
|
||||||
|
|
||||||
|
# Bounds threads spent on blocking ingestion work (parsing, chunking,
|
||||||
|
# hashing, the sync minio SDK) so it cannot exhaust Starlette's own
|
||||||
|
# thread pool (ADR-0017).
|
||||||
|
ingestion_limiter = CapacityLimiter(resolved_settings.ingestion.thread_pool_size)
|
||||||
|
object_storage = MinioObjectStorage(
|
||||||
|
minio_client, bucket=resolved_settings.minio.bucket, limiter=ingestion_limiter
|
||||||
|
)
|
||||||
|
|
||||||
app.state.resources = AppResources(
|
app.state.resources = AppResources(
|
||||||
settings=resolved_settings,
|
settings=resolved_settings,
|
||||||
db_engine=db_engine,
|
db_engine=db_engine,
|
||||||
db_sessionmaker=db_sessionmaker,
|
db_sessionmaker=db_sessionmaker,
|
||||||
minio_client=minio_client,
|
minio_client=minio_client,
|
||||||
qdrant_client=qdrant_client,
|
qdrant_client=qdrant_client,
|
||||||
|
object_storage=object_storage,
|
||||||
|
point_storage=point_storage,
|
||||||
|
point_repository=point_repository,
|
||||||
|
ingestion_limiter=ingestion_limiter,
|
||||||
|
dense_embedders=dense_embedders,
|
||||||
|
sparse_embedder=sparse_embedder,
|
||||||
|
ingestion_concurrency_limiter=ingestion_concurrency_limiter,
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -64,4 +169,14 @@ def create_lifespan(
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("lifespan.qdrant.close.failed")
|
logger.exception("lifespan.qdrant.close.failed")
|
||||||
|
|
||||||
|
try:
|
||||||
|
await nomic_http_client.aclose()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("lifespan.embedding.nomic_client.close.failed")
|
||||||
|
|
||||||
|
try:
|
||||||
|
await openai_http_client.aclose()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("lifespan.embedding.openai_client.close.failed")
|
||||||
|
|
||||||
return lifespan
|
return lifespan
|
||||||
|
|||||||
1
src/cli/__init__.py
Normal file
1
src/cli/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
"""Operator entry points that run as deployment steps, not at app startup."""
|
||||||
94
src/cli/provision_tenant.py
Normal file
94
src/cli/provision_tenant.py
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
"""Create a tenant, issue its first API key, and register its domains.
|
||||||
|
|
||||||
|
uv run python -m src.cli.provision_tenant --slug acme --domain fire
|
||||||
|
|
||||||
|
A deployment step, like `alembic upgrade head` and `src.cli.qdrant_bootstrap`.
|
||||||
|
It exists because nothing over HTTP can bootstrap a tenant: every `/v1` route
|
||||||
|
requires an API key, and a key cannot exist before its tenant does.
|
||||||
|
|
||||||
|
The plaintext key is printed to **stdout once** and never stored or logged —
|
||||||
|
Postgres holds only its SHA-256 hash (ADR-0009), so a lost key is reissued by
|
||||||
|
re-running this command, not recovered. Structured logs go to stderr/the log
|
||||||
|
sink and carry only the non-secret `key_prefix` (ADR-0011).
|
||||||
|
|
||||||
|
Re-running with the same `--slug` reuses the tenant and any domains it already
|
||||||
|
has, and issues an additional key.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
|
||||||
|
from src.application.tenants import DEFAULT_SCOPES, provision_tenant
|
||||||
|
from src.config import Settings
|
||||||
|
from src.infrastructure.observability.logging import configure_logging
|
||||||
|
from src.infrastructure.postgres.database import create_engine, create_sessionmaker
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="python -m src.cli.provision_tenant",
|
||||||
|
description="Create a tenant, issue an API key, and register domains.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--slug", required=True, help="URL-safe tenant identifier, e.g. 'acme'")
|
||||||
|
parser.add_argument("--name", default=None, help="Display name (defaults to --slug)")
|
||||||
|
parser.add_argument("--key-name", default="bootstrap", help="Label for the issued API key")
|
||||||
|
parser.add_argument(
|
||||||
|
"--scopes",
|
||||||
|
default=",".join(DEFAULT_SCOPES),
|
||||||
|
help=f"Comma-separated scopes for the key (default: {','.join(DEFAULT_SCOPES)})",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--domain",
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
dest="domains",
|
||||||
|
help="Domain to register; repeatable. Uploads reject an unregistered domain.",
|
||||||
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
async def run(argv: list[str] | None = None, settings: Settings | None = None) -> int:
|
||||||
|
args = _parse_args(argv)
|
||||||
|
resolved = settings or Settings()
|
||||||
|
configure_logging(resolved.logging, resolved.app)
|
||||||
|
|
||||||
|
scopes = tuple(scope.strip() for scope in args.scopes.split(",") if scope.strip())
|
||||||
|
if not scopes:
|
||||||
|
logger.error("tenant.provision.failed", reason="no_scopes")
|
||||||
|
return 2
|
||||||
|
|
||||||
|
engine = create_engine(resolved.postgres)
|
||||||
|
try:
|
||||||
|
result = await provision_tenant(
|
||||||
|
create_sessionmaker(engine),
|
||||||
|
slug=args.slug,
|
||||||
|
name=args.name,
|
||||||
|
key_name=args.key_name,
|
||||||
|
scopes=scopes,
|
||||||
|
domains=tuple(args.domains),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
# stdout, not the logger: this is the one value the operator must copy, and
|
||||||
|
# it must never reach a log sink (ADR-0011).
|
||||||
|
print(f"tenant_id={result.tenant_id}")
|
||||||
|
print(f"tenant_slug={result.tenant_slug}")
|
||||||
|
print(f"api_key_id={result.api_key_id}")
|
||||||
|
print(f"domains={','.join(result.domains_created + result.domains_existing)}")
|
||||||
|
print(f"api_key={result.api_key}")
|
||||||
|
print("Store the api_key now -- only its hash is persisted and it cannot be shown again.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
sys.exit(asyncio.run(run()))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
54
src/cli/qdrant_bootstrap.py
Normal file
54
src/cli/qdrant_bootstrap.py
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
"""Create the `chunks` collection — the Qdrant analogue of `alembic upgrade head`.
|
||||||
|
|
||||||
|
uv run python -m src.cli.qdrant_bootstrap
|
||||||
|
|
||||||
|
A deployment step, deliberately not part of the FastAPI lifespan: collection
|
||||||
|
creation is DDL, which ADR-0009 keeps out of application startup for Postgres
|
||||||
|
and ADR-0012 keeps out of it for LangGraph's `.setup()`. See
|
||||||
|
`src/infrastructure/qdrant/collection.py` for the full reasoning.
|
||||||
|
|
||||||
|
Idempotent and safe to re-run. Exits non-zero if an existing collection
|
||||||
|
diverges from the pinned schema, rather than leaving a silently degraded
|
||||||
|
sparse index behind.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
|
||||||
|
from src.config import Settings
|
||||||
|
from src.infrastructure.observability.logging import configure_logging
|
||||||
|
from src.infrastructure.qdrant.client import create_client
|
||||||
|
from src.infrastructure.qdrant.collection import (
|
||||||
|
CollectionSchemaMismatchError,
|
||||||
|
ensure_chunks_collection,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def bootstrap(settings: Settings | None = None) -> int:
|
||||||
|
resolved = settings or Settings()
|
||||||
|
configure_logging(resolved.logging, resolved.app)
|
||||||
|
client = create_client(resolved.qdrant)
|
||||||
|
try:
|
||||||
|
created = await ensure_chunks_collection(client, collection=resolved.qdrant.collection)
|
||||||
|
except CollectionSchemaMismatchError as exc:
|
||||||
|
logger.error("qdrant.bootstrap.schema_mismatch", error=str(exc))
|
||||||
|
return 1
|
||||||
|
finally:
|
||||||
|
await client.close()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"qdrant.bootstrap.completed", collection=resolved.qdrant.collection, created=created
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
sys.exit(asyncio.run(bootstrap()))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
188
src/config.py
188
src/config.py
@@ -1,9 +1,31 @@
|
|||||||
|
from typing import Any
|
||||||
|
|
||||||
from pydantic import Field, model_validator
|
from pydantic import Field, model_validator
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
|
# pydantic-settings does not cascade `env_file` from a parent `BaseSettings`
|
||||||
|
# to a nested one: each `BaseSettings` subclass only reads `.env` if its own
|
||||||
|
# `model_config` names it. `Settings` has no fields of its own -- only
|
||||||
|
# nested settings objects -- so every nested class below repeats
|
||||||
|
# `env_file=".env"` as its own default, or its fields would silently read
|
||||||
|
# only real process environment variables (fine under Docker Compose, broken
|
||||||
|
# for local `.env`-file development) while still *appearing* to work
|
||||||
|
# whenever a `.env.example` default happens to match the class default.
|
||||||
|
#
|
||||||
|
# That default alone isn't enough to let a caller point `Settings` at a
|
||||||
|
# *different* file (as the test suite does, to parse `.env.example`), since
|
||||||
|
# passing `_env_file=...` to `Settings(...)` only overrides `Settings`'s own
|
||||||
|
# `model_config` -- nested defaults still construct against their own
|
||||||
|
# hardcoded ".env". `Settings.__init__`/`EmbeddingSettings.__init__` below
|
||||||
|
# thread an explicit `_env_file` override through to every nested
|
||||||
|
# constructor so one override actually reaches the whole tree.
|
||||||
|
_UNSET: Any = object()
|
||||||
|
|
||||||
|
|
||||||
class PostgresSettings(BaseSettings):
|
class PostgresSettings(BaseSettings):
|
||||||
model_config = SettingsConfigDict(env_prefix="POSTGRES_", extra="ignore")
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="POSTGRES_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
host: str = "127.0.0.1"
|
host: str = "127.0.0.1"
|
||||||
port: int = 5433
|
port: int = 5433
|
||||||
@@ -17,7 +39,9 @@ class PostgresSettings(BaseSettings):
|
|||||||
|
|
||||||
|
|
||||||
class MinioSettings(BaseSettings):
|
class MinioSettings(BaseSettings):
|
||||||
model_config = SettingsConfigDict(env_prefix="MINIO_", extra="ignore")
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="MINIO_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
endpoint: str = "127.0.0.1:9100"
|
endpoint: str = "127.0.0.1:9100"
|
||||||
access_key: str = "chatbot"
|
access_key: str = "chatbot"
|
||||||
@@ -33,15 +57,22 @@ class IngestionSettings(BaseSettings):
|
|||||||
timeouts, or callers give up on work that is still succeeding.
|
timeouts, or callers give up on work that is still succeeding.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
model_config = SettingsConfigDict(env_prefix="INGESTION_", extra="ignore")
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="INGESTION_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
max_concurrency: int = 4
|
max_concurrency: int = 4
|
||||||
thread_pool_size: int = 8
|
thread_pool_size: int = 8
|
||||||
timeout_seconds: float = 120.0
|
timeout_seconds: float = 120.0
|
||||||
|
max_upload_size_mb: int = 25
|
||||||
max_chunks_per_file: int = 5000
|
max_chunks_per_file: int = 5000
|
||||||
embed_batch_size: int = 128
|
embed_batch_size: int = 128
|
||||||
embed_concurrency: int = 4
|
embed_concurrency: int = 4
|
||||||
|
|
||||||
|
@property
|
||||||
|
def max_upload_size_bytes(self) -> int:
|
||||||
|
return self.max_upload_size_mb * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
class ChunkingSettings(BaseSettings):
|
class ChunkingSettings(BaseSettings):
|
||||||
"""Parsing and chunking parameters (ADR-0018).
|
"""Parsing and chunking parameters (ADR-0018).
|
||||||
@@ -52,7 +83,9 @@ class ChunkingSettings(BaseSettings):
|
|||||||
`search_document: ` task prefix and any heading text carried into a chunk.
|
`search_document: ` task prefix and any heading text carried into a chunk.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
model_config = SettingsConfigDict(env_prefix="CHUNKING_", extra="ignore")
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="CHUNKING_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
strategy: str = "fixed_size"
|
strategy: str = "fixed_size"
|
||||||
chunk_size: int = 400
|
chunk_size: int = 400
|
||||||
@@ -76,25 +109,151 @@ class ChunkingSettings(BaseSettings):
|
|||||||
|
|
||||||
|
|
||||||
class QdrantSettings(BaseSettings):
|
class QdrantSettings(BaseSettings):
|
||||||
model_config = SettingsConfigDict(env_prefix="QDRANT_", extra="ignore")
|
"""Qdrant connection and bulk-upsert bounds (ADR-0001, ADR-0017).
|
||||||
|
|
||||||
|
`collection` names the single shared collection all tenants live in
|
||||||
|
(ADR-0001); it is deliberately configurable so tests can point at a
|
||||||
|
disposable one. The vector *dimensions* are not settings -- they are model
|
||||||
|
facts pinned in `src/infrastructure/qdrant/collection.py`, and changing one
|
||||||
|
is a re-embedding migration.
|
||||||
|
|
||||||
|
`upsert_batch_size` sits inside ADR-0001's 64-256 bulk-upload band, and
|
||||||
|
`upsert_concurrency` bounds in-flight batches so ingestion issues parallel
|
||||||
|
streams rather than an unbounded `gather` (ADR-0017).
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="QDRANT_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
url: str = "http://127.0.0.1:6343"
|
url: str = "http://127.0.0.1:6343"
|
||||||
api_key: str | None = None
|
api_key: str | None = None
|
||||||
|
collection: str = "chunks"
|
||||||
|
upsert_batch_size: int = 128
|
||||||
|
upsert_concurrency: int = 4
|
||||||
|
|
||||||
|
|
||||||
|
class NomicEmbeddingSettings(BaseSettings):
|
||||||
|
"""Self-hosted `nomic-embed-text-v2-moe` server (ADR-0001, 768-dim).
|
||||||
|
|
||||||
|
Talks the OpenAI-compatible `/embeddings` endpoint shape. The default
|
||||||
|
points at the Ollama host the `emet` benchmark used, whose OpenAI-compat
|
||||||
|
shim accepts any non-empty API key.
|
||||||
|
|
||||||
|
`document_prefix` is empty by design. ADR-0004 cites the model card's
|
||||||
|
`search_document: ` requirement, but emet's winning run used no prefix
|
||||||
|
(Ollama's template is a bare passthrough and injects none), and the
|
||||||
|
prefix is not cosmetic -- it moves the vector substantially. Turning it
|
||||||
|
on here obliges the query side to send `search_query: ` too (ADR-0003),
|
||||||
|
so it stays off until an emet run measures the pair together.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="EMBEDDING_NOMIC_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
|
base_url: str = "http://192.168.10.10:11435/v1"
|
||||||
|
model: str = "nomic-embed-text-v2-moe"
|
||||||
|
api_key: str = "sk-not-set"
|
||||||
|
document_prefix: str = ""
|
||||||
|
keep_alive: str = "30m"
|
||||||
|
timeout_seconds: float = 30.0
|
||||||
|
|
||||||
|
|
||||||
|
class OpenaiEmbeddingSettings(BaseSettings):
|
||||||
|
"""OpenAI's hosted embedding API (ADR-0001's second dense signal).
|
||||||
|
|
||||||
|
`dimensions` is unset, so `text-embedding-3-large` returns its native
|
||||||
|
3072 dimensions -- the configuration emet benchmarked. Setting it would
|
||||||
|
truncate via Matryoshka and is a re-embedding migration, not a config
|
||||||
|
tweak.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="EMBEDDING_OPENAI_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
|
base_url: str = "https://api.openai.com/v1"
|
||||||
|
model: str = "text-embedding-3-large"
|
||||||
|
api_key: str | None = None
|
||||||
|
dimensions: int | None = None
|
||||||
|
document_prefix: str = ""
|
||||||
|
timeout_seconds: float = 30.0
|
||||||
|
|
||||||
|
|
||||||
|
class SparseEmbeddingSettings(BaseSettings):
|
||||||
|
"""BM25 sparse-vector parameters (ADR-0001, ADR-0005).
|
||||||
|
|
||||||
|
`analyzer`, `k`, and `b` are the configuration emet benchmarked as
|
||||||
|
`bm25-fa-norm-stop`; changing them invalidates that result. `k`/`b`
|
||||||
|
saturation is applied client-side here, while IDF comes from Qdrant's
|
||||||
|
`modifier="idf"` sparse-vector config at query time.
|
||||||
|
|
||||||
|
`avg_len` is emet's placeholder average document length in analyzer
|
||||||
|
tokens, exposed as a setting so it can be recalibrated from real corpus
|
||||||
|
statistics without a code change.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="EMBEDDING_SPARSE_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
|
analyzer: str = "fa_norm_stop"
|
||||||
|
k: float = 1.2
|
||||||
|
b: float = 0.75
|
||||||
|
avg_len: float = 256.0
|
||||||
|
|
||||||
|
|
||||||
|
class EmbeddingSettings(BaseSettings):
|
||||||
|
model_config = SettingsConfigDict(extra="ignore")
|
||||||
|
|
||||||
|
nomic: NomicEmbeddingSettings = Field(default_factory=NomicEmbeddingSettings)
|
||||||
|
openai: OpenaiEmbeddingSettings = Field(default_factory=OpenaiEmbeddingSettings)
|
||||||
|
sparse: SparseEmbeddingSettings = Field(default_factory=SparseEmbeddingSettings)
|
||||||
|
|
||||||
|
def __init__(self, _env_file: Any = _UNSET, **data: Any) -> None:
|
||||||
|
env_file = ".env" if _env_file is _UNSET else _env_file
|
||||||
|
data.setdefault("nomic", NomicEmbeddingSettings(_env_file=env_file))
|
||||||
|
data.setdefault("openai", OpenaiEmbeddingSettings(_env_file=env_file))
|
||||||
|
data.setdefault("sparse", SparseEmbeddingSettings(_env_file=env_file))
|
||||||
|
super().__init__(**data)
|
||||||
|
|
||||||
|
|
||||||
class AppLimitSettings(BaseSettings):
|
class AppLimitSettings(BaseSettings):
|
||||||
model_config = SettingsConfigDict(env_prefix="APP_", extra="ignore")
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="APP_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
env: str = "local"
|
env: str = "local"
|
||||||
max_upload_size_mb: int = 25
|
|
||||||
readiness_check_timeout_seconds: float = 2.0
|
readiness_check_timeout_seconds: float = 2.0
|
||||||
|
# The deployed commit SHA or release tag (ADR-0011, "Bind process-level
|
||||||
|
# environment context"). Set by CI/CD at build/deploy time -- never
|
||||||
|
# computed at runtime by shelling out to git, which would fail in a
|
||||||
|
# container image with no .git directory.
|
||||||
|
service_version: str = "dev"
|
||||||
|
|
||||||
|
|
||||||
class LoggingSettings(BaseSettings):
|
class LoggingSettings(BaseSettings):
|
||||||
model_config = SettingsConfigDict(env_prefix="LOG_", extra="ignore")
|
"""Logging sinks (ADR-0011).
|
||||||
|
|
||||||
|
`json_format` controls stdout's renderer only. Production sets it `true`
|
||||||
|
so stdout is JSON for the container log collector; local development
|
||||||
|
leaves it `false` for a colored console renderer. `file_path`, when set,
|
||||||
|
is a second, independent handler that always renders JSON regardless of
|
||||||
|
`json_format` -- a developer can read a human console while still keeping
|
||||||
|
a machine-parseable file. Unset in production: stdout/stderr collection is
|
||||||
|
preferred there over a log file inside the container.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_config = SettingsConfigDict(
|
||||||
|
env_prefix="LOG_", extra="ignore", env_file=".env", env_ignore_empty=True
|
||||||
|
)
|
||||||
|
|
||||||
level: str = "INFO"
|
level: str = "INFO"
|
||||||
json_format: bool = False
|
json_format: bool = False
|
||||||
|
file_path: str | None = None
|
||||||
|
file_max_bytes: int = 10 * 1024 * 1024
|
||||||
|
file_backup_count: int = 5
|
||||||
|
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
@@ -105,5 +264,18 @@ class Settings(BaseSettings):
|
|||||||
ingestion: IngestionSettings = Field(default_factory=IngestionSettings)
|
ingestion: IngestionSettings = Field(default_factory=IngestionSettings)
|
||||||
chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
|
chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
|
||||||
qdrant: QdrantSettings = Field(default_factory=QdrantSettings)
|
qdrant: QdrantSettings = Field(default_factory=QdrantSettings)
|
||||||
|
embedding: EmbeddingSettings = Field(default_factory=EmbeddingSettings)
|
||||||
app: AppLimitSettings = Field(default_factory=AppLimitSettings)
|
app: AppLimitSettings = Field(default_factory=AppLimitSettings)
|
||||||
logging: LoggingSettings = Field(default_factory=LoggingSettings)
|
logging: LoggingSettings = Field(default_factory=LoggingSettings)
|
||||||
|
|
||||||
|
def __init__(self, _env_file: Any = _UNSET, **data: Any) -> None:
|
||||||
|
env_file = ".env" if _env_file is _UNSET else _env_file
|
||||||
|
data.setdefault("postgres", PostgresSettings(_env_file=env_file))
|
||||||
|
data.setdefault("minio", MinioSettings(_env_file=env_file))
|
||||||
|
data.setdefault("ingestion", IngestionSettings(_env_file=env_file))
|
||||||
|
data.setdefault("chunking", ChunkingSettings(_env_file=env_file))
|
||||||
|
data.setdefault("qdrant", QdrantSettings(_env_file=env_file))
|
||||||
|
data.setdefault("embedding", EmbeddingSettings(_env_file=env_file))
|
||||||
|
data.setdefault("app", AppLimitSettings(_env_file=env_file))
|
||||||
|
data.setdefault("logging", LoggingSettings(_env_file=env_file))
|
||||||
|
super().__init__(_env_file=env_file, **data)
|
||||||
|
|||||||
0
src/infrastructure/embedding/__init__.py
Normal file
0
src/infrastructure/embedding/__init__.py
Normal file
150
src/infrastructure/embedding/analyzers.py
Normal file
150
src/infrastructure/embedding/analyzers.py
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
"""The `fa_norm_stop` BM25 analyzer (ADR-0001, ADR-0005).
|
||||||
|
|
||||||
|
Ported from the `emet` evaluation lab
|
||||||
|
(`src/chatbot_gh/adapters/sparse/analyzers.py`), which benchmarked four Farsi
|
||||||
|
analyzer variants on the real corpus and found `fa_norm_stop` the best
|
||||||
|
performer. This is a *measured* artifact: changing the normalization,
|
||||||
|
tokenization, or stopword list invalidates that result, so improvements belong
|
||||||
|
in a new emet benchmark run rather than in an edit here.
|
||||||
|
|
||||||
|
Deliberately independent of `src/application/ingestion/normalization.py`.
|
||||||
|
Those solve different problems: `normalize_persian_text` shapes chunk content
|
||||||
|
that gets cited back to the reader, so ADR-0018 has it preserve digits and
|
||||||
|
punctuation as authored. This module shapes index terms nobody ever sees, so
|
||||||
|
it folds digits and diacritics freely. Sharing one function between them would
|
||||||
|
let a display-motivated tweak silently perturb the benchmarked sparse index.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
# Several Arabic letterforms are visually indistinguishable from Latin ones in
|
||||||
|
# a monospace editor (alef from "l", heh from "o"), and literals render
|
||||||
|
# right-to-left, visually reordering the source line. `normalization.py` writes
|
||||||
|
# them as codepoints for that reason; this module follows the same convention.
|
||||||
|
_ZWNJ = 0x200C
|
||||||
|
_ARABIC_YEH = 0x064A
|
||||||
|
_ARABIC_KAF = 0x0643
|
||||||
|
_TEH_MARBUTA = 0x0629
|
||||||
|
_HAMZA_ON_WAW = 0x0624
|
||||||
|
_ALEF_HAMZA_BELOW = 0x0625
|
||||||
|
_ALEF_HAMZA_ABOVE = 0x0623
|
||||||
|
|
||||||
|
_PERSIAN_YEH = 0x06CC
|
||||||
|
_PERSIAN_KEHEH = 0x06A9
|
||||||
|
_HEH = 0x0647
|
||||||
|
_WAW = 0x0648
|
||||||
|
_ALEF = 0x0627
|
||||||
|
_SPACE = 0x0020
|
||||||
|
|
||||||
|
# Persian (U+06F0-U+06F9) and Arabic-Indic (U+0660-U+0669) digits both fold to
|
||||||
|
# ASCII, so the same number matches however it was authored.
|
||||||
|
_EASTERN_DIGITS = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789")
|
||||||
|
|
||||||
|
# ZWNJ becomes a space (splitting compounds into separate terms) and the
|
||||||
|
# Arabic letterforms fold to their Persian equivalents. Every entry is a
|
||||||
|
# single codepoint mapping to a single codepoint over disjoint sources, so
|
||||||
|
# applying them in one pass is equivalent to emet's chained `str.replace`
|
||||||
|
# calls -- provided NFC runs first, since NFC is what composes the hamza
|
||||||
|
# forms this table then folds.
|
||||||
|
_FOLDING: dict[int, int] = {
|
||||||
|
_ZWNJ: _SPACE,
|
||||||
|
_ARABIC_YEH: _PERSIAN_YEH,
|
||||||
|
_ARABIC_KAF: _PERSIAN_KEHEH,
|
||||||
|
_TEH_MARBUTA: _HEH,
|
||||||
|
_HAMZA_ON_WAW: _WAW,
|
||||||
|
_ALEF_HAMZA_BELOW: _ALEF,
|
||||||
|
_ALEF_HAMZA_ABOVE: _ALEF,
|
||||||
|
}
|
||||||
|
_FOLDING.update(_EASTERN_DIGITS)
|
||||||
|
|
||||||
|
# Common Persian/Arabic stopwords (function words + FAQ noise), plus the
|
||||||
|
# English function words that appear in a mixed-script corpus. Kept small and
|
||||||
|
# explicit -- deliberately not a full hazm list. `_HEH_ALEF` is the plural
|
||||||
|
# suffix "ha"; written as a codepoint pair because both of its letters are
|
||||||
|
# Latin-confusable, which is exactly the case Ruff's RUF001 flags.
|
||||||
|
_HEH_ALEF = chr(_HEH) + chr(_ALEF)
|
||||||
|
|
||||||
|
_PERSIAN_STOPWORDS: frozenset[str] = frozenset(
|
||||||
|
{
|
||||||
|
"و",
|
||||||
|
"در",
|
||||||
|
"به",
|
||||||
|
"از",
|
||||||
|
"که",
|
||||||
|
"این",
|
||||||
|
"را",
|
||||||
|
"با",
|
||||||
|
"برای",
|
||||||
|
"آن",
|
||||||
|
"یک",
|
||||||
|
"است",
|
||||||
|
"شد",
|
||||||
|
"شده",
|
||||||
|
"می",
|
||||||
|
"های",
|
||||||
|
_HEH_ALEF,
|
||||||
|
"یا",
|
||||||
|
"تا",
|
||||||
|
"بر",
|
||||||
|
"اگر",
|
||||||
|
"هم",
|
||||||
|
"نیز",
|
||||||
|
"ولی",
|
||||||
|
"اما",
|
||||||
|
"چه",
|
||||||
|
"چون",
|
||||||
|
"روی",
|
||||||
|
"پس",
|
||||||
|
"پیش",
|
||||||
|
"هر",
|
||||||
|
"هیچ",
|
||||||
|
"بود",
|
||||||
|
"باشد",
|
||||||
|
"هست",
|
||||||
|
"نیست",
|
||||||
|
"کند",
|
||||||
|
"کرد",
|
||||||
|
"کردن",
|
||||||
|
"شود",
|
||||||
|
"the",
|
||||||
|
"a",
|
||||||
|
"an",
|
||||||
|
"of",
|
||||||
|
"to",
|
||||||
|
"and",
|
||||||
|
"in",
|
||||||
|
"on",
|
||||||
|
"for",
|
||||||
|
"is",
|
||||||
|
"are",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Word characters minus underscore. Note this KEEPS digits: an insurance
|
||||||
|
# corpus is full of policy numbers, dates, and amounts, and those are exactly
|
||||||
|
# the tokens a lexical index should be able to match on.
|
||||||
|
_TOKEN_RE = re.compile(r"[^\W_]+", re.UNICODE)
|
||||||
|
|
||||||
|
FA_NORM_STOP = "fa_norm_stop"
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_fa(text: str) -> str:
|
||||||
|
return unicodedata.normalize("NFC", text).translate(_FOLDING)
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenize_raw(text: str) -> list[str]:
|
||||||
|
return [m.group(0).lower() for m in _TOKEN_RE.finditer(text)]
|
||||||
|
|
||||||
|
|
||||||
|
def analyze(text: str, analyzer: str = FA_NORM_STOP) -> list[str]:
|
||||||
|
"""Tokenize `text` into sparse-index terms.
|
||||||
|
|
||||||
|
Only `fa_norm_stop` is implemented -- emet's other three variants
|
||||||
|
(`raw`, `fa_norm`, `fa_norm_stem`) lost the benchmark and exist there as
|
||||||
|
experiment arms, not as configurations this service should run.
|
||||||
|
"""
|
||||||
|
if analyzer != FA_NORM_STOP:
|
||||||
|
raise ValueError(f"Unknown analyzer '{analyzer}'")
|
||||||
|
tokens = _tokenize_raw(_normalize_fa(text))
|
||||||
|
return [token for token in tokens if token not in _PERSIAN_STOPWORDS]
|
||||||
93
src/infrastructure/embedding/bm25.py
Normal file
93
src/infrastructure/embedding/bm25.py
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
"""The `bm25-fa-norm-stop` sparse embedder (ADR-0001, ADR-0005).
|
||||||
|
|
||||||
|
Ported from the `emet` evaluation lab
|
||||||
|
(`src/chatbot_gh/adapters/sparse/bm25_embedder.py`), the configuration that
|
||||||
|
won its Farsi analyzer benchmark. Not Qdrant's hosted `Qdrant/bm25` FastEmbed
|
||||||
|
model, whose documented language support omits Farsi (ADR-0005).
|
||||||
|
|
||||||
|
**The BM25 work is split across two systems.** This adapter applies the
|
||||||
|
term-frequency saturation half client-side -- the `k` and `b` parameters,
|
||||||
|
including document-length normalization. IDF is *not* computed here: Qdrant
|
||||||
|
supplies it from collection-wide statistics when the sparse vector field is
|
||||||
|
created with `modifier="idf"`.
|
||||||
|
|
||||||
|
That split is load-bearing. A Qdrant collection created without
|
||||||
|
`modifier="idf"` will silently score these vectors as saturated term
|
||||||
|
frequencies with no IDF weighting at all -- no error, just materially worse
|
||||||
|
lexical retrieval. The collection bootstrap (plan 001 Phase 5) must set it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from hashlib import blake2b
|
||||||
|
|
||||||
|
from src.application.ingestion.models import SparseVector
|
||||||
|
from src.config import SparseEmbeddingSettings
|
||||||
|
from src.infrastructure.embedding.analyzers import analyze
|
||||||
|
|
||||||
|
# Qdrant sparse indices must be non-negative and fit a signed 32-bit int.
|
||||||
|
_INDEX_SPACE = 2**31 - 1
|
||||||
|
|
||||||
|
|
||||||
|
def _token_index(token: str) -> int:
|
||||||
|
"""Map a term to its sparse-vector index.
|
||||||
|
|
||||||
|
A hash rather than a vocabulary table, so the mapping needs no shared
|
||||||
|
state and stays identical across processes, restarts, and — critically —
|
||||||
|
between ingest-time and query-time encoding. `blake2b` rather than
|
||||||
|
`hash()`, which is PYTHONHASHSEED-salted and therefore differs per
|
||||||
|
process.
|
||||||
|
"""
|
||||||
|
digest = blake2b(token.encode("utf-8"), digest_size=8).digest()
|
||||||
|
return int.from_bytes(digest, "big") % _INDEX_SPACE
|
||||||
|
|
||||||
|
|
||||||
|
def text_to_sparse_vector(
|
||||||
|
text: str, *, settings: SparseEmbeddingSettings, query: bool = False
|
||||||
|
) -> SparseVector:
|
||||||
|
"""Encode one text as a BM25-saturated sparse vector (IDF applied by Qdrant).
|
||||||
|
|
||||||
|
Document and query sides differ in exactly one term: documents carry the
|
||||||
|
`b` length normalization, queries do not (standard BM25 practice — a
|
||||||
|
query's own length should not discount its terms).
|
||||||
|
"""
|
||||||
|
tokens = analyze(text, settings.analyzer)
|
||||||
|
if not tokens:
|
||||||
|
return SparseVector(indices=[], values=[])
|
||||||
|
|
||||||
|
frequencies = Counter(tokens)
|
||||||
|
doc_length = float(len(tokens))
|
||||||
|
k = settings.k
|
||||||
|
b = settings.b
|
||||||
|
|
||||||
|
indices: list[int] = []
|
||||||
|
values: list[float] = []
|
||||||
|
# Sorted so the emitted vector is deterministic for a given text, which
|
||||||
|
# keeps re-ingestion byte-stable and makes the output testable.
|
||||||
|
for token, freq in sorted(frequencies.items()):
|
||||||
|
if query:
|
||||||
|
weight = freq * (k + 1.0) / (freq + k)
|
||||||
|
else:
|
||||||
|
weight = freq * (k + 1.0) / (freq + k * (1.0 - b + b * doc_length / settings.avg_len))
|
||||||
|
indices.append(_token_index(token))
|
||||||
|
values.append(float(weight))
|
||||||
|
|
||||||
|
return SparseVector(indices=indices, values=values)
|
||||||
|
|
||||||
|
|
||||||
|
class Bm25SparseEmbedder:
|
||||||
|
"""A `SparseEmbedder` (see `src/application/ports/embedding.py`).
|
||||||
|
|
||||||
|
Pure CPU work with no network calls, so it is blocking: callers offload it
|
||||||
|
via `anyio.to_thread.run_sync` with the ingestion `CapacityLimiter`
|
||||||
|
(ADR-0017), never awaiting it directly on the event loop.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name = "sparse"
|
||||||
|
|
||||||
|
def __init__(self, settings: SparseEmbeddingSettings) -> None:
|
||||||
|
self.model_version = f"bm25-{settings.analyzer}"
|
||||||
|
self._settings = settings
|
||||||
|
|
||||||
|
def embed_batch(self, texts: Sequence[str], *, query: bool = False) -> list[SparseVector]:
|
||||||
|
return [text_to_sparse_vector(text, settings=self._settings, query=query) for text in texts]
|
||||||
82
src/infrastructure/embedding/openai_compatible.py
Normal file
82
src/infrastructure/embedding/openai_compatible.py
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
"""Dense embedding adapter for OpenAI-compatible `/embeddings` endpoints.
|
||||||
|
|
||||||
|
Backs both `dense_nomic` (self-hosted `nomic-embed-text-v2-moe` behind
|
||||||
|
Ollama's OpenAI-compatible shim) and `dense_openai` (OpenAI's hosted API) —
|
||||||
|
both speak the same request/response shape, so one adapter serves both named
|
||||||
|
vectors with different config (ADR-0001).
|
||||||
|
|
||||||
|
Uses `httpx` directly rather than the `openai` SDK. The `emet` benchmark this
|
||||||
|
configuration comes from uses the SDK, but it is a synchronous batch tool;
|
||||||
|
ADR-0017 requires async, semaphore-bounded batches here, and the request shape
|
||||||
|
is small enough that the SDK earns nothing.
|
||||||
|
|
||||||
|
`httpx.AsyncClient` is application-lifetime (ADR-0012): built once in the
|
||||||
|
FastAPI lifespan and passed in, never constructed per call.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
# Ollama's default port, plus the alternate the benchmarked deployment uses.
|
||||||
|
_OLLAMA_PORTS = frozenset({11434, 11435})
|
||||||
|
|
||||||
|
|
||||||
|
def is_ollama_base_url(base_url: str) -> bool:
|
||||||
|
"""Whether `base_url` looks like an Ollama OpenAI-compatible endpoint.
|
||||||
|
|
||||||
|
Ollama unloads an idle model, and reloading `nomic-embed-text-v2-moe`
|
||||||
|
costs well over two minutes — longer than `INGESTION_TIMEOUT_SECONDS`, so
|
||||||
|
a cold upload would 504. `keep_alive` is how the model is kept resident,
|
||||||
|
and it is an Ollama extension, hence the sniffing.
|
||||||
|
"""
|
||||||
|
parsed = urlparse(base_url)
|
||||||
|
host = parsed.hostname or ""
|
||||||
|
return parsed.port in _OLLAMA_PORTS or "ollama" in host.lower()
|
||||||
|
|
||||||
|
|
||||||
|
class OpenAICompatibleEmbedder:
|
||||||
|
"""A `DenseEmbedder` (see `src/application/ports/embedding.py`) over one
|
||||||
|
OpenAI-compatible `/embeddings` endpoint.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
client: httpx.AsyncClient,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
model: str,
|
||||||
|
dimensions: int | None = None,
|
||||||
|
document_prefix: str = "",
|
||||||
|
keep_alive: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.name = name
|
||||||
|
self.model_version = model
|
||||||
|
self._client = client
|
||||||
|
self._model = model
|
||||||
|
self._dimensions = dimensions
|
||||||
|
self._document_prefix = document_prefix
|
||||||
|
self._keep_alive = keep_alive
|
||||||
|
|
||||||
|
async def embed_batch(self, texts: Sequence[str]) -> list[list[float]]:
|
||||||
|
inputs = (
|
||||||
|
[f"{self._document_prefix}{text}" for text in texts]
|
||||||
|
if self._document_prefix
|
||||||
|
else list(texts)
|
||||||
|
)
|
||||||
|
payload: dict[str, object] = {"model": self._model, "input": inputs}
|
||||||
|
if self._dimensions is not None:
|
||||||
|
payload["dimensions"] = self._dimensions
|
||||||
|
if self._keep_alive is not None:
|
||||||
|
payload["keep_alive"] = self._keep_alive
|
||||||
|
|
||||||
|
response = await self._client.post("/embeddings", json=payload)
|
||||||
|
response.raise_for_status()
|
||||||
|
body = response.json()
|
||||||
|
|
||||||
|
# Sort by `index` rather than trusting response order: the contract
|
||||||
|
# guarantees the field, not the ordering, and a silently permuted
|
||||||
|
# batch would attach every vector to the wrong chunk.
|
||||||
|
data = sorted(body["data"], key=lambda item: item["index"])
|
||||||
|
return [item["embedding"] for item in data]
|
||||||
33
src/infrastructure/minio/storage.py
Normal file
33
src/infrastructure/minio/storage.py
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
"""MinIO adapter for the `ObjectStorage` port (ADR-0013, ADR-0017).
|
||||||
|
|
||||||
|
The `minio` SDK is synchronous, so every call runs through
|
||||||
|
`anyio.to_thread.run_sync` bounded by the ingestion `CapacityLimiter` — the
|
||||||
|
same rule ADR-0017 applies to parsing/chunking. Calling the SDK directly from
|
||||||
|
`async def` would block every concurrent request in the process.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import io
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
from anyio import CapacityLimiter, to_thread
|
||||||
|
from minio import Minio
|
||||||
|
|
||||||
|
|
||||||
|
class MinioObjectStorage:
|
||||||
|
def __init__(self, client: Minio, *, bucket: str, limiter: CapacityLimiter) -> None:
|
||||||
|
self._client = client
|
||||||
|
self._bucket = bucket
|
||||||
|
self._limiter = limiter
|
||||||
|
|
||||||
|
async def put_object(self, *, key: str, data: bytes, content_type: str) -> None:
|
||||||
|
await to_thread.run_sync(
|
||||||
|
partial(
|
||||||
|
self._client.put_object,
|
||||||
|
self._bucket,
|
||||||
|
key,
|
||||||
|
io.BytesIO(data),
|
||||||
|
length=len(data),
|
||||||
|
content_type=content_type,
|
||||||
|
),
|
||||||
|
limiter=self._limiter,
|
||||||
|
)
|
||||||
@@ -1,14 +1,50 @@
|
|||||||
|
"""Logging configuration: structlog + stdlib, dual local sinks (ADR-0011).
|
||||||
|
|
||||||
|
Console and an optional file are independent, simultaneous handlers on the
|
||||||
|
same logger, not a single renderer chosen by a flag -- the same structlog
|
||||||
|
event fans out to both. The console handler is always human-readable
|
||||||
|
(`ConsoleRenderer`); the file handler, when enabled via `LOG_FILE_PATH`,
|
||||||
|
always renders JSON regardless of `LOG_JSON_FORMAT`, so a saved log stays
|
||||||
|
machine-parseable even when the terminal next to it is not.
|
||||||
|
|
||||||
|
`LOG_JSON_FORMAT` controls *stdout's* renderer only: production sets it `true`
|
||||||
|
so the container log collector gets JSON; local development leaves it `false`
|
||||||
|
for the colored console. `LOG_FILE_PATH` is expected to be unset in
|
||||||
|
production -- stdout/stderr collection is preferred there over a log file
|
||||||
|
inside the container.
|
||||||
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import logging.config
|
import logging.config
|
||||||
import sys
|
import sys
|
||||||
|
from collections.abc import Callable
|
||||||
|
|
||||||
import structlog
|
import structlog
|
||||||
|
|
||||||
from src.config import LoggingSettings
|
from src.config import AppLimitSettings, LoggingSettings
|
||||||
|
|
||||||
|
|
||||||
def configure_logging(settings: LoggingSettings) -> None:
|
def _bind_environment(settings: AppLimitSettings) -> Callable[..., dict[str, object]]:
|
||||||
|
"""A static processor, not a contextvar: `env`/`service_version` don't
|
||||||
|
vary per request, and a contextvar bound before the first request would
|
||||||
|
be wiped by `RequestIdMiddleware`'s `clear_contextvars()` on that request.
|
||||||
|
Closing over `settings` at configure time makes every event carry them
|
||||||
|
instead, regardless of request context (ADR-0011).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def processor(
|
||||||
|
logger: object, method_name: str, event_dict: dict[str, object]
|
||||||
|
) -> dict[str, object]:
|
||||||
|
event_dict["env"] = settings.env
|
||||||
|
event_dict["service_version"] = settings.service_version
|
||||||
|
return event_dict
|
||||||
|
|
||||||
|
return processor
|
||||||
|
|
||||||
|
|
||||||
|
def configure_logging(settings: LoggingSettings, app_settings: AppLimitSettings) -> None:
|
||||||
shared_processors = [
|
shared_processors = [
|
||||||
|
_bind_environment(app_settings),
|
||||||
structlog.contextvars.merge_contextvars,
|
structlog.contextvars.merge_contextvars,
|
||||||
structlog.stdlib.add_log_level,
|
structlog.stdlib.add_log_level,
|
||||||
structlog.stdlib.add_logger_name,
|
structlog.stdlib.add_logger_name,
|
||||||
@@ -27,55 +63,84 @@ def configure_logging(settings: LoggingSettings) -> None:
|
|||||||
cache_logger_on_first_use=True,
|
cache_logger_on_first_use=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
renderer = (
|
console_renderer = (
|
||||||
structlog.processors.JSONRenderer()
|
structlog.processors.JSONRenderer()
|
||||||
if settings.json_format
|
if settings.json_format
|
||||||
else structlog.dev.ConsoleRenderer(colors=True)
|
else structlog.dev.ConsoleRenderer(colors=True)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
formatters = {
|
||||||
|
"console": {
|
||||||
|
"()": structlog.stdlib.ProcessorFormatter,
|
||||||
|
"processors": [
|
||||||
|
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
|
||||||
|
console_renderer,
|
||||||
|
],
|
||||||
|
"foreign_pre_chain": [
|
||||||
|
structlog.stdlib.ExtraAdder(),
|
||||||
|
*shared_processors,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
handlers: dict[str, dict[str, object]] = {
|
||||||
|
"console": {
|
||||||
|
"class": "logging.StreamHandler",
|
||||||
|
"level": settings.level,
|
||||||
|
"formatter": "console",
|
||||||
|
"stream": sys.stdout,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
root_handlers = ["console"]
|
||||||
|
|
||||||
|
if settings.file_path is not None:
|
||||||
|
# File handler always renders JSON, independent of the console
|
||||||
|
# renderer chosen above -- a saved log stays machine-parseable even
|
||||||
|
# when stdout is the colored, human-readable renderer.
|
||||||
|
formatters["file"] = {
|
||||||
|
"()": structlog.stdlib.ProcessorFormatter,
|
||||||
|
"processors": [
|
||||||
|
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
|
||||||
|
structlog.processors.JSONRenderer(),
|
||||||
|
],
|
||||||
|
"foreign_pre_chain": [
|
||||||
|
structlog.stdlib.ExtraAdder(),
|
||||||
|
*shared_processors,
|
||||||
|
],
|
||||||
|
}
|
||||||
|
handlers["file"] = {
|
||||||
|
"class": "logging.handlers.RotatingFileHandler",
|
||||||
|
"level": settings.level,
|
||||||
|
"formatter": "file",
|
||||||
|
"filename": settings.file_path,
|
||||||
|
"maxBytes": settings.file_max_bytes,
|
||||||
|
"backupCount": settings.file_backup_count,
|
||||||
|
}
|
||||||
|
root_handlers.append("file")
|
||||||
|
|
||||||
logging.config.dictConfig(
|
logging.config.dictConfig(
|
||||||
{
|
{
|
||||||
"version": 1,
|
"version": 1,
|
||||||
"disable_existing_loggers": False,
|
"disable_existing_loggers": False,
|
||||||
"formatters": {
|
"formatters": formatters,
|
||||||
"default": {
|
"handlers": handlers,
|
||||||
"()": structlog.stdlib.ProcessorFormatter,
|
|
||||||
"processors": [
|
|
||||||
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
|
|
||||||
renderer,
|
|
||||||
],
|
|
||||||
"foreign_pre_chain": [
|
|
||||||
structlog.stdlib.ExtraAdder(),
|
|
||||||
*shared_processors,
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"handlers": {
|
|
||||||
"console": {
|
|
||||||
"class": "logging.StreamHandler",
|
|
||||||
"level": settings.level,
|
|
||||||
"formatter": "default",
|
|
||||||
"stream": sys.stdout,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"loggers": {
|
"loggers": {
|
||||||
"": {
|
"": {
|
||||||
"handlers": ["console"],
|
"handlers": root_handlers,
|
||||||
"level": settings.level,
|
"level": settings.level,
|
||||||
"propagate": False,
|
"propagate": False,
|
||||||
},
|
},
|
||||||
"uvicorn": {
|
"uvicorn": {
|
||||||
"handlers": ["console"],
|
"handlers": root_handlers,
|
||||||
"level": settings.level,
|
"level": settings.level,
|
||||||
"propagate": False,
|
"propagate": False,
|
||||||
},
|
},
|
||||||
"uvicorn.access": {
|
"uvicorn.access": {
|
||||||
"handlers": ["console"],
|
"handlers": root_handlers,
|
||||||
"level": settings.level,
|
"level": settings.level,
|
||||||
"propagate": False,
|
"propagate": False,
|
||||||
},
|
},
|
||||||
"sqlalchemy.engine": {
|
"sqlalchemy.engine": {
|
||||||
"handlers": ["console"],
|
"handlers": root_handlers,
|
||||||
"level": "WARNING",
|
"level": "WARNING",
|
||||||
"propagate": False,
|
"propagate": False,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from src.infrastructure.postgres.models.ingestion_job import IngestionJob
|
|||||||
from src.infrastructure.postgres.models.ingestion_job_event import IngestionJobEvent
|
from src.infrastructure.postgres.models.ingestion_job_event import IngestionJobEvent
|
||||||
from src.infrastructure.postgres.models.source_file import SourceFile
|
from src.infrastructure.postgres.models.source_file import SourceFile
|
||||||
from src.infrastructure.postgres.models.tenant import Tenant
|
from src.infrastructure.postgres.models.tenant import Tenant
|
||||||
|
from src.infrastructure.postgres.models.tenant_domain import TenantDomain
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"ApiKey",
|
"ApiKey",
|
||||||
@@ -12,4 +13,5 @@ __all__ = [
|
|||||||
"IngestionJobEvent",
|
"IngestionJobEvent",
|
||||||
"SourceFile",
|
"SourceFile",
|
||||||
"Tenant",
|
"Tenant",
|
||||||
|
"TenantDomain",
|
||||||
]
|
]
|
||||||
|
|||||||
52
src/infrastructure/postgres/models/tenant_domain.py
Normal file
52
src/infrastructure/postgres/models/tenant_domain.py
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, String, UniqueConstraint, func
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from src.infrastructure.postgres.models.base import Base
|
||||||
|
|
||||||
|
TENANT_DOMAIN_STATUSES = ("active", "disabled")
|
||||||
|
|
||||||
|
|
||||||
|
class TenantDomain(Base):
|
||||||
|
"""A domain a tenant is allowed to ingest into (ADR-0009).
|
||||||
|
|
||||||
|
Tenants do not share a domain list — one may run 14 insurance lines and
|
||||||
|
another 6 — so this is a per-tenant table rather than an enum or a global
|
||||||
|
lookup.
|
||||||
|
|
||||||
|
Its purpose is to stop an arbitrary caller-supplied `domain` from silently
|
||||||
|
creating a new Qdrant partition. `domain` is denormalized into every point's
|
||||||
|
payload and into `source_files`, and a typo like `fier` for `fire` produces
|
||||||
|
no error anywhere: the file indexes into a partition retrieval never queries,
|
||||||
|
so it is invisible rather than failed.
|
||||||
|
|
||||||
|
`domain` is the immutable key. Renaming it would mean rewriting every point
|
||||||
|
payload that carries it, which is a migration, not an edit — `display_name`
|
||||||
|
is the mutable human-facing label instead.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "tenant_domains"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("tenant_id", "domain", name="uq_tenant_domains_tenant_id_domain"),
|
||||||
|
CheckConstraint(f"status IN {TENANT_DOMAIN_STATUSES}", name="ck_tenant_domains_status"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||||
|
tenant_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
ForeignKey("tenants.id", ondelete="CASCADE"), index=True
|
||||||
|
)
|
||||||
|
domain: Mapped[str] = mapped_column(String(80))
|
||||||
|
display_name: Mapped[str] = mapped_column(String(200))
|
||||||
|
status: Mapped[str] = mapped_column(String(20), default="active", server_default="active")
|
||||||
|
metadata_: Mapped[dict[str, object]] = mapped_column(
|
||||||
|
"metadata", JSONB, default=dict, server_default="{}"
|
||||||
|
)
|
||||||
|
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||||
|
)
|
||||||
|
disabled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None)
|
||||||
45
src/infrastructure/postgres/repositories/api_keys.py
Normal file
45
src/infrastructure/postgres/repositories/api_keys.py
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
"""API-key lookups and issuance (ADR-0008, ADR-0009).
|
||||||
|
|
||||||
|
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||||
|
commits, rolls back, or closes the session (ADR-0012). Secret comparison
|
||||||
|
happens in `src/application/auth`, not here — this module only fetches rows
|
||||||
|
by their non-secret `key_prefix`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from src.infrastructure.postgres.models.api_key import ApiKey
|
||||||
|
|
||||||
|
|
||||||
|
async def get_by_prefix(session: AsyncSession, key_prefix: str) -> ApiKey | None:
|
||||||
|
result = await session.execute(select(ApiKey).where(ApiKey.key_prefix == key_prefix))
|
||||||
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
def create(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
name: str,
|
||||||
|
key_prefix: str,
|
||||||
|
key_hash: str,
|
||||||
|
scopes: list[str],
|
||||||
|
actor_type: str = "backend",
|
||||||
|
created_by: str | None = None,
|
||||||
|
) -> ApiKey:
|
||||||
|
"""Persist an issued key. The caller hashes the secret; this never sees it."""
|
||||||
|
api_key = ApiKey(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
name=name,
|
||||||
|
key_prefix=key_prefix,
|
||||||
|
key_hash=key_hash,
|
||||||
|
scopes=scopes,
|
||||||
|
actor_type=actor_type,
|
||||||
|
created_by=created_by,
|
||||||
|
)
|
||||||
|
session.add(api_key)
|
||||||
|
return api_key
|
||||||
119
src/infrastructure/postgres/repositories/ingestion_jobs.py
Normal file
119
src/infrastructure/postgres/repositories/ingestion_jobs.py
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
"""`ingestion_jobs`/`ingestion_job_events` persistence (ADR-0009, ADR-0017).
|
||||||
|
|
||||||
|
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||||
|
commits, rolls back, or closes the session (ADR-0012) — the two-transaction
|
||||||
|
shape in `src/application/files/upload.py` depends on that. Every read is
|
||||||
|
tenant-scoped by a required `tenant_id` argument.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import desc, select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from src.infrastructure.postgres.models.ingestion_job import IngestionJob
|
||||||
|
from src.infrastructure.postgres.models.ingestion_job_event import IngestionJobEvent
|
||||||
|
|
||||||
|
|
||||||
|
async def get_by_id(
|
||||||
|
session: AsyncSession, *, tenant_id: uuid.UUID, ingestion_job_id: uuid.UUID
|
||||||
|
) -> IngestionJob | None:
|
||||||
|
result = await session.execute(
|
||||||
|
select(IngestionJob).where(
|
||||||
|
IngestionJob.id == ingestion_job_id, IngestionJob.tenant_id == tenant_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
async def get_latest_for_source_file(
|
||||||
|
session: AsyncSession, *, tenant_id: uuid.UUID, source_file_id: uuid.UUID
|
||||||
|
) -> IngestionJob | None:
|
||||||
|
result = await session.execute(
|
||||||
|
select(IngestionJob)
|
||||||
|
.where(
|
||||||
|
IngestionJob.tenant_id == tenant_id,
|
||||||
|
IngestionJob.source_file_id == source_file_id,
|
||||||
|
)
|
||||||
|
.order_by(desc(IngestionJob.created_at))
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
def create_running(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
source_file_id: uuid.UUID,
|
||||||
|
requested_by_api_key_id: uuid.UUID | None,
|
||||||
|
chunking_strategy: str,
|
||||||
|
) -> IngestionJob:
|
||||||
|
job = IngestionJob(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
source_file_id=source_file_id,
|
||||||
|
requested_by_api_key_id=requested_by_api_key_id,
|
||||||
|
status="running",
|
||||||
|
chunking_strategy=chunking_strategy,
|
||||||
|
started_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
session.add(job)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
async def mark_terminal(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
ingestion_job_id: uuid.UUID,
|
||||||
|
status: str,
|
||||||
|
points_created: int = 0,
|
||||||
|
points_updated: int = 0,
|
||||||
|
points_soft_deleted: int = 0,
|
||||||
|
points_skipped: int = 0,
|
||||||
|
error_code: str | None = None,
|
||||||
|
error_message: str | None = None,
|
||||||
|
) -> IngestionJob | None:
|
||||||
|
"""Move a job from `running` to a terminal status.
|
||||||
|
|
||||||
|
Never reads/writes a job whose current status is already terminal — a
|
||||||
|
terminal job must not transition back to `running` or to a different
|
||||||
|
terminal status (ADR-0017).
|
||||||
|
"""
|
||||||
|
job = await get_by_id(session, tenant_id=tenant_id, ingestion_job_id=ingestion_job_id)
|
||||||
|
if job is None or job.status != "running":
|
||||||
|
return None
|
||||||
|
job.status = status
|
||||||
|
job.completed_at = datetime.now(UTC)
|
||||||
|
job.points_created = points_created
|
||||||
|
job.points_updated = points_updated
|
||||||
|
job.points_soft_deleted = points_soft_deleted
|
||||||
|
job.points_skipped = points_skipped
|
||||||
|
job.error_code = error_code
|
||||||
|
job.error_message = error_message
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
|
def append_event(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
ingestion_job_id: uuid.UUID,
|
||||||
|
level: str,
|
||||||
|
stage: str,
|
||||||
|
message: str,
|
||||||
|
details: dict[str, object] | None = None,
|
||||||
|
) -> IngestionJobEvent:
|
||||||
|
event = IngestionJobEvent(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
ingestion_job_id=ingestion_job_id,
|
||||||
|
level=level,
|
||||||
|
stage=stage,
|
||||||
|
message=message,
|
||||||
|
details=details or {},
|
||||||
|
)
|
||||||
|
session.add(event)
|
||||||
|
return event
|
||||||
86
src/infrastructure/postgres/repositories/source_files.py
Normal file
86
src/infrastructure/postgres/repositories/source_files.py
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
"""`source_files` persistence (ADR-0009).
|
||||||
|
|
||||||
|
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||||
|
commits, rolls back, or closes the session (ADR-0012). Every read is
|
||||||
|
tenant-scoped by a required `tenant_id` argument, so a missing filter is a
|
||||||
|
signature error rather than a cross-tenant leak.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from src.infrastructure.postgres.models.source_file import SourceFile
|
||||||
|
|
||||||
|
|
||||||
|
async def get_by_id(
|
||||||
|
session: AsyncSession, *, tenant_id: uuid.UUID, source_file_id: uuid.UUID
|
||||||
|
) -> SourceFile | None:
|
||||||
|
result = await session.execute(
|
||||||
|
select(SourceFile).where(SourceFile.id == source_file_id, SourceFile.tenant_id == tenant_id)
|
||||||
|
)
|
||||||
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
async def find_active_by_content_hash(
|
||||||
|
session: AsyncSession, *, tenant_id: uuid.UUID, domain: str, content_sha256: str
|
||||||
|
) -> SourceFile | None:
|
||||||
|
result = await session.execute(
|
||||||
|
select(SourceFile).where(
|
||||||
|
SourceFile.tenant_id == tenant_id,
|
||||||
|
SourceFile.domain == domain,
|
||||||
|
SourceFile.content_sha256 == content_sha256,
|
||||||
|
SourceFile.status == "active",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
def create(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
source_file_id: uuid.UUID,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str,
|
||||||
|
source_filename: str,
|
||||||
|
source_type: str,
|
||||||
|
content_sha256: str,
|
||||||
|
byte_size: int,
|
||||||
|
storage_uri: str,
|
||||||
|
created_by_api_key_id: uuid.UUID | None,
|
||||||
|
) -> SourceFile:
|
||||||
|
"""`source_file_id` is caller-generated: the upload service derives the
|
||||||
|
MinIO object key from it before this row exists, so the id has to be
|
||||||
|
chosen up front rather than assigned by the database.
|
||||||
|
"""
|
||||||
|
source_file = SourceFile(
|
||||||
|
id=source_file_id,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
source_filename=source_filename,
|
||||||
|
source_type=source_type,
|
||||||
|
content_sha256=content_sha256,
|
||||||
|
byte_size=byte_size,
|
||||||
|
storage_uri=storage_uri,
|
||||||
|
created_by_api_key_id=created_by_api_key_id,
|
||||||
|
)
|
||||||
|
session.add(source_file)
|
||||||
|
return source_file
|
||||||
|
|
||||||
|
|
||||||
|
def mark_soft_deleted(source_file: SourceFile, *, deleted_at: datetime) -> None:
|
||||||
|
"""Retire a file: `status='soft_deleted'` plus `deleted_at` (ADR-0009).
|
||||||
|
|
||||||
|
Takes the already-loaded row rather than an id, because the caller fetched
|
||||||
|
it under its tenant filter and re-fetching here would be a second place
|
||||||
|
that could forget that filter.
|
||||||
|
|
||||||
|
Retiring the row matters beyond bookkeeping: `find_active_by_content_hash`
|
||||||
|
matches only `active` files, so a re-upload of the same bytes after a delete
|
||||||
|
creates a fresh file and re-ingests it, instead of taking the duplicate path
|
||||||
|
and returning a file whose points have all been deactivated.
|
||||||
|
"""
|
||||||
|
source_file.status = "soft_deleted"
|
||||||
|
source_file.deleted_at = deleted_at
|
||||||
73
src/infrastructure/postgres/repositories/tenant_domains.py
Normal file
73
src/infrastructure/postgres/repositories/tenant_domains.py
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
"""`tenant_domains` persistence (ADR-0009).
|
||||||
|
|
||||||
|
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||||
|
commits, rolls back, or closes the session (ADR-0012). Every read and write is
|
||||||
|
tenant-scoped by a required `tenant_id` argument, so a missing filter is a
|
||||||
|
signature error rather than a cross-tenant leak.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from src.infrastructure.postgres.models.tenant_domain import TenantDomain
|
||||||
|
|
||||||
|
|
||||||
|
async def get(session: AsyncSession, *, tenant_id: uuid.UUID, domain: str) -> TenantDomain | None:
|
||||||
|
result = await session.execute(
|
||||||
|
select(TenantDomain).where(
|
||||||
|
TenantDomain.tenant_id == tenant_id, TenantDomain.domain == domain
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
async def list_for_tenant(
|
||||||
|
session: AsyncSession, *, tenant_id: uuid.UUID, include_disabled: bool = False
|
||||||
|
) -> list[TenantDomain]:
|
||||||
|
statement = select(TenantDomain).where(TenantDomain.tenant_id == tenant_id)
|
||||||
|
if not include_disabled:
|
||||||
|
statement = statement.where(TenantDomain.status == "active")
|
||||||
|
result = await session.execute(statement.order_by(TenantDomain.domain))
|
||||||
|
return list(result.scalars().all())
|
||||||
|
|
||||||
|
|
||||||
|
def create(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str,
|
||||||
|
display_name: str,
|
||||||
|
metadata: dict[str, object] | None = None,
|
||||||
|
) -> TenantDomain:
|
||||||
|
tenant_domain = TenantDomain(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
display_name=display_name,
|
||||||
|
metadata_=metadata or {},
|
||||||
|
)
|
||||||
|
session.add(tenant_domain)
|
||||||
|
return tenant_domain
|
||||||
|
|
||||||
|
|
||||||
|
def update_display_name(tenant_domain: TenantDomain, *, display_name: str) -> TenantDomain:
|
||||||
|
"""`domain` itself is deliberately not updatable.
|
||||||
|
|
||||||
|
It is denormalized into every Qdrant point payload and into `source_files`,
|
||||||
|
so changing the key would mean rewriting all of them — a migration, not an
|
||||||
|
edit. The label is what callers actually want to change.
|
||||||
|
"""
|
||||||
|
tenant_domain.display_name = display_name
|
||||||
|
return tenant_domain
|
||||||
|
|
||||||
|
|
||||||
|
def set_status(tenant_domain: TenantDomain, *, status: str) -> TenantDomain:
|
||||||
|
"""Disable/re-enable a domain. Existing points are untouched either way —
|
||||||
|
disabling blocks new uploads, it is not a delete (ADR-0002).
|
||||||
|
"""
|
||||||
|
tenant_domain.status = status
|
||||||
|
tenant_domain.disabled_at = datetime.now(UTC) if status == "disabled" else None
|
||||||
|
return tenant_domain
|
||||||
27
src/infrastructure/postgres/repositories/tenants.py
Normal file
27
src/infrastructure/postgres/repositories/tenants.py
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
"""Tenant lookups and creation (ADR-0009).
|
||||||
|
|
||||||
|
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||||
|
commits, rolls back, or closes the session (ADR-0012).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from src.infrastructure.postgres.models.tenant import Tenant
|
||||||
|
|
||||||
|
|
||||||
|
async def get_by_id(session: AsyncSession, tenant_id: uuid.UUID) -> Tenant | None:
|
||||||
|
return await session.get(Tenant, tenant_id)
|
||||||
|
|
||||||
|
|
||||||
|
async def get_by_slug(session: AsyncSession, slug: str) -> Tenant | None:
|
||||||
|
result = await session.execute(select(Tenant).where(Tenant.slug == slug))
|
||||||
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
def create(session: AsyncSession, *, slug: str, name: str) -> Tenant:
|
||||||
|
tenant = Tenant(id=uuid.uuid4(), slug=slug, name=name)
|
||||||
|
session.add(tenant)
|
||||||
|
return tenant
|
||||||
@@ -9,10 +9,22 @@ def create_client(settings: QdrantSettings) -> AsyncQdrantClient:
|
|||||||
return AsyncQdrantClient(url=settings.url, api_key=settings.api_key)
|
return AsyncQdrantClient(url=settings.url, api_key=settings.api_key)
|
||||||
|
|
||||||
|
|
||||||
async def ping(client: AsyncQdrantClient, timeout: float) -> bool:
|
async def ping(client: AsyncQdrantClient, timeout: float, *, collection: str) -> bool:
|
||||||
|
"""Whether Qdrant is reachable **and** the `chunks` collection exists.
|
||||||
|
|
||||||
|
Reachability alone is not readiness here. The collection is created by a
|
||||||
|
deployment step (`python -m src.cli.qdrant_bootstrap`, see ADR-0001
|
||||||
|
"Collection provisioning"), so a process can boot against a healthy Qdrant
|
||||||
|
that has no collection at all. Without this check that misconfiguration
|
||||||
|
stays invisible until the first upload fails with a `502` — after the
|
||||||
|
request has already paid for the MinIO write and the embedding round trips.
|
||||||
|
|
||||||
|
This is the Qdrant analogue of an unapplied Alembic migration, and it
|
||||||
|
belongs in `/readyz` for the same reason: it is a dependency-readiness
|
||||||
|
condition, not a process-health one.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
async with asyncio.timeout(timeout):
|
async with asyncio.timeout(timeout):
|
||||||
await client.get_collections()
|
return await client.collection_exists(collection)
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
return True
|
|
||||||
|
|||||||
200
src/infrastructure/qdrant/collection.py
Normal file
200
src/infrastructure/qdrant/collection.py
Normal file
@@ -0,0 +1,200 @@
|
|||||||
|
"""The `chunks` collection schema and its provisioning (ADR-0001).
|
||||||
|
|
||||||
|
**This runs as a deployment step, never at FastAPI startup.** Creating a
|
||||||
|
collection is DDL, and the project already rules DDL out of the request/boot
|
||||||
|
path: ADR-0009 requires Alembic for Postgres schema, and ADR-0012 makes
|
||||||
|
LangGraph's `.setup()` a deployment step. Doing it in the lifespan would also
|
||||||
|
couple boot to Qdrant being reachable (that is `/readyz`'s job), race across
|
||||||
|
replicas, and hide a misconfigured collection until traffic arrives.
|
||||||
|
|
||||||
|
Entry point for operators: `uv run python -m src.cli.qdrant_bootstrap`.
|
||||||
|
|
||||||
|
Two properties of this schema are load-bearing and fail *silently* if wrong,
|
||||||
|
which is why `ensure_chunks_collection` verifies rather than skips:
|
||||||
|
|
||||||
|
- **`sparse` must carry `modifier=IDF`.** `src/infrastructure/embedding/bm25.py`
|
||||||
|
computes only BM25's term-frequency saturation; Qdrant supplies IDF from
|
||||||
|
collection-wide statistics. Without the modifier there is no error and no
|
||||||
|
warning — lexical retrieval just quietly loses its IDF term (ADR-0005).
|
||||||
|
- **The dense dimensions are pinned**: `dense_nomic` 768,
|
||||||
|
`dense_openai` 3072 (native, `dimensions` deliberately unset). They are
|
||||||
|
constants here rather than settings because they are model facts; changing
|
||||||
|
one is a re-embedding migration, not a config tweak (ADR-0001).
|
||||||
|
|
||||||
|
All four named vectors are defined at creation even though `late_interaction`
|
||||||
|
stays unpopulated until ADR-0003's rerank work (ADR-0017 does not compute it at
|
||||||
|
ingest). Sparse and multivector fields cannot be added to an existing
|
||||||
|
collection without recreating it, so deferring them is the one thing this
|
||||||
|
schema cannot afford.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from qdrant_client import AsyncQdrantClient, models
|
||||||
|
|
||||||
|
logger = structlog.get_logger(__name__)
|
||||||
|
|
||||||
|
DENSE_NOMIC_VECTOR = "dense_nomic"
|
||||||
|
DENSE_OPENAI_VECTOR = "dense_openai"
|
||||||
|
SPARSE_VECTOR = "sparse"
|
||||||
|
LATE_INTERACTION_VECTOR = "late_interaction"
|
||||||
|
|
||||||
|
DENSE_NOMIC_DIMENSIONS = 768
|
||||||
|
DENSE_OPENAI_DIMENSIONS = 3072
|
||||||
|
# jina-colbert-v2's per-token output dimension (ADR-0005).
|
||||||
|
LATE_INTERACTION_DIMENSIONS = 128
|
||||||
|
|
||||||
|
|
||||||
|
class CollectionSchemaMismatchError(RuntimeError):
|
||||||
|
"""An existing collection does not match the schema this code expects.
|
||||||
|
|
||||||
|
Raised loudly instead of returning: silently accepting a collection whose
|
||||||
|
dense size or sparse modifier differs is precisely the failure this
|
||||||
|
explicit bootstrap step exists to prevent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _vectors_config() -> dict[str, models.VectorParams]:
|
||||||
|
return {
|
||||||
|
DENSE_NOMIC_VECTOR: models.VectorParams(
|
||||||
|
size=DENSE_NOMIC_DIMENSIONS, distance=models.Distance.COSINE
|
||||||
|
),
|
||||||
|
DENSE_OPENAI_VECTOR: models.VectorParams(
|
||||||
|
size=DENSE_OPENAI_DIMENSIONS, distance=models.Distance.COSINE
|
||||||
|
),
|
||||||
|
# Rerank-only: never independently ANN-searched, so its HNSW graph is
|
||||||
|
# disabled (m=0), and stored on disk so its larger footprint does not
|
||||||
|
# degrade dense/sparse query latency (ADR-0001).
|
||||||
|
LATE_INTERACTION_VECTOR: models.VectorParams(
|
||||||
|
size=LATE_INTERACTION_DIMENSIONS,
|
||||||
|
distance=models.Distance.COSINE,
|
||||||
|
multivector_config=models.MultiVectorConfig(
|
||||||
|
comparator=models.MultiVectorComparator.MAX_SIM
|
||||||
|
),
|
||||||
|
hnsw_config=models.HnswConfigDiff(m=0),
|
||||||
|
on_disk=True,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _sparse_vectors_config() -> dict[str, models.SparseVectorParams]:
|
||||||
|
return {SPARSE_VECTOR: models.SparseVectorParams(modifier=models.Modifier.IDF)}
|
||||||
|
|
||||||
|
|
||||||
|
# `content`'s full-text index backs ADR-0002's keyword search. The
|
||||||
|
# `multilingual` tokenizer is the one that segments Persian correctly; `word`
|
||||||
|
# splits on non-alphanumerics, which mis-handles ZWNJ-joined compounds. No
|
||||||
|
# stemmer or stopword list is configured: `content` is already letter-folded by
|
||||||
|
# `normalize_persian_text` at ingest (ADR-0018), and the *ranked* Farsi lexical
|
||||||
|
# path is the benchmarked BM25 sparse vector, not this index. This one exists
|
||||||
|
# for exact keyword/filter matching, which ADR-0002 keeps deliberately distinct
|
||||||
|
# from retrieval.
|
||||||
|
_CONTENT_INDEX = models.TextIndexParams(
|
||||||
|
type=models.TextIndexType.TEXT,
|
||||||
|
tokenizer=models.TokenizerType.MULTILINGUAL,
|
||||||
|
lowercase=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# (field name, schema). `order_id` is float because Qdrant's `Range` conditions
|
||||||
|
# and `order_by` only support numeric/datetime payloads -- a keyword key could
|
||||||
|
# only be sorted client-side after fetching every chunk (ADR-0001).
|
||||||
|
_PayloadIndexSchema = models.PayloadSchemaType | models.KeywordIndexParams | models.TextIndexParams
|
||||||
|
_PAYLOAD_INDEXES: tuple[tuple[str, _PayloadIndexSchema], ...] = (
|
||||||
|
# `is_tenant` co-locates a tenant's vectors on disk for sequential reads,
|
||||||
|
# which is the whole point of payload-partitioned multitenancy.
|
||||||
|
(
|
||||||
|
"tenant_id",
|
||||||
|
models.KeywordIndexParams(type=models.KeywordIndexType.KEYWORD, is_tenant=True),
|
||||||
|
),
|
||||||
|
("domain", models.PayloadSchemaType.KEYWORD),
|
||||||
|
("file_id", models.PayloadSchemaType.KEYWORD),
|
||||||
|
("order_id", models.PayloadSchemaType.FLOAT),
|
||||||
|
("previous_chunk_id", models.PayloadSchemaType.KEYWORD),
|
||||||
|
("next_chunk_id", models.PayloadSchemaType.KEYWORD),
|
||||||
|
("content", _CONTENT_INDEX),
|
||||||
|
# Every read path filters on `is_active` (ADR-0002 implies `is_active: true`
|
||||||
|
# unless the caller opts in), and the re-ingestion sweep and `/v1/points`
|
||||||
|
# both range over `chunk_index`. Both were unindexed while ingestion was the
|
||||||
|
# only reader; plan 002 makes them hot.
|
||||||
|
("is_active", models.PayloadSchemaType.BOOL),
|
||||||
|
("chunk_index", models.PayloadSchemaType.INTEGER),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_existing(collection: str, info: models.CollectionInfo) -> None:
|
||||||
|
params = info.config.params
|
||||||
|
vectors = params.vectors
|
||||||
|
if not isinstance(vectors, dict):
|
||||||
|
raise CollectionSchemaMismatchError(
|
||||||
|
f"collection {collection!r} has an unnamed dense vector; ADR-0001 requires "
|
||||||
|
f"named vectors and this cannot be fixed without recreating the collection"
|
||||||
|
)
|
||||||
|
|
||||||
|
for name, expected_size in (
|
||||||
|
(DENSE_NOMIC_VECTOR, DENSE_NOMIC_DIMENSIONS),
|
||||||
|
(DENSE_OPENAI_VECTOR, DENSE_OPENAI_DIMENSIONS),
|
||||||
|
(LATE_INTERACTION_VECTOR, LATE_INTERACTION_DIMENSIONS),
|
||||||
|
):
|
||||||
|
existing = vectors.get(name)
|
||||||
|
if existing is None:
|
||||||
|
raise CollectionSchemaMismatchError(
|
||||||
|
f"collection {collection!r} is missing the {name!r} vector"
|
||||||
|
)
|
||||||
|
if existing.size != expected_size:
|
||||||
|
raise CollectionSchemaMismatchError(
|
||||||
|
f"collection {collection!r} has {name!r} at {existing.size} dimensions, "
|
||||||
|
f"expected {expected_size}; re-dimensioning is a re-embedding migration"
|
||||||
|
)
|
||||||
|
|
||||||
|
sparse = (params.sparse_vectors or {}).get(SPARSE_VECTOR)
|
||||||
|
if sparse is None:
|
||||||
|
raise CollectionSchemaMismatchError(
|
||||||
|
f"collection {collection!r} is missing the {SPARSE_VECTOR!r} vector; sparse "
|
||||||
|
f"vectors cannot be added without recreating the collection"
|
||||||
|
)
|
||||||
|
if sparse.modifier != models.Modifier.IDF:
|
||||||
|
raise CollectionSchemaMismatchError(
|
||||||
|
f"collection {collection!r} has {SPARSE_VECTOR!r} with modifier "
|
||||||
|
f"{sparse.modifier!r}, expected 'idf'; without it Qdrant applies no IDF "
|
||||||
|
f"and lexical retrieval silently degrades (ADR-0005)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def ensure_chunks_collection(client: AsyncQdrantClient, *, collection: str) -> bool:
|
||||||
|
"""Create the `chunks` collection and its payload indexes if absent.
|
||||||
|
|
||||||
|
Idempotent: an existing collection is verified against the pinned schema
|
||||||
|
and left alone. Returns whether it created the collection.
|
||||||
|
|
||||||
|
Raises `CollectionSchemaMismatchError` if an existing collection diverges.
|
||||||
|
"""
|
||||||
|
if await client.collection_exists(collection):
|
||||||
|
_verify_existing(collection, await client.get_collection(collection))
|
||||||
|
logger.info("qdrant.collection.verified", collection=collection)
|
||||||
|
# Payload indexes are additive and idempotent, so (re)creating them
|
||||||
|
# here is what lets an index be added to an already-live collection.
|
||||||
|
await _create_payload_indexes(client, collection=collection)
|
||||||
|
return False
|
||||||
|
|
||||||
|
await client.create_collection(
|
||||||
|
collection_name=collection,
|
||||||
|
vectors_config=_vectors_config(),
|
||||||
|
sparse_vectors_config=_sparse_vectors_config(),
|
||||||
|
# m=0 disables the global index; payload_m builds per-tenant graphs
|
||||||
|
# instead, per Qdrant's multitenant guidance (ADR-0001).
|
||||||
|
hnsw_config=models.HnswConfigDiff(m=0, payload_m=16),
|
||||||
|
)
|
||||||
|
logger.info("qdrant.collection.created", collection=collection)
|
||||||
|
await _create_payload_indexes(client, collection=collection)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
async def _create_payload_indexes(client: AsyncQdrantClient, *, collection: str) -> None:
|
||||||
|
for field_name, field_schema in _PAYLOAD_INDEXES:
|
||||||
|
await client.create_payload_index(
|
||||||
|
collection_name=collection, field_name=field_name, field_schema=field_schema
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"qdrant.collection.payload_indexes.ensured",
|
||||||
|
collection=collection,
|
||||||
|
fields=[name for name, _ in _PAYLOAD_INDEXES],
|
||||||
|
)
|
||||||
247
src/infrastructure/qdrant/point_repository.py
Normal file
247
src/infrastructure/qdrant/point_repository.py
Normal file
@@ -0,0 +1,247 @@
|
|||||||
|
"""Qdrant adapter for the `PointRepository` port (ADR-0002, ADR-0015).
|
||||||
|
|
||||||
|
Every Qdrant filter for the `/v1/points` surface is built here. Routers and
|
||||||
|
application services never import `qdrant_client` — they pass a `tenant_id` and
|
||||||
|
get `Point` models back.
|
||||||
|
|
||||||
|
Two mechanics are worth reading before changing anything:
|
||||||
|
|
||||||
|
**Reads go through `scroll`, not `retrieve`.** `retrieve` fetches by id and
|
||||||
|
takes no filter, which would force the tenant check to happen *after* Qdrant
|
||||||
|
answered — exactly the "check it in Python afterwards" shape ADR-0002's
|
||||||
|
isolation rule exists to prevent. `scroll` with a `HasIdCondition` plus the
|
||||||
|
tenant condition pushes the check server-side, so a foreign id returns an empty
|
||||||
|
page rather than a row this code has to remember to reject.
|
||||||
|
|
||||||
|
**Ordered listing paginates by `order_id` value, not by offset.** Qdrant does
|
||||||
|
not return a `next_page_offset` when `order_by` is set, and an offset-based
|
||||||
|
cursor would skip or repeat rows when a concurrent insert shifts positions
|
||||||
|
underneath the reader. Ranging on `order_id > cursor` is stable instead: a point
|
||||||
|
inserted ahead of the cursor was already returned, and one inserted after it
|
||||||
|
shows up on a later page. This relies on `order_id` being unique within a file,
|
||||||
|
which ADR-0002 guarantees by rejecting a reorder whose gap would collapse onto a
|
||||||
|
neighbour value.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from qdrant_client import AsyncQdrantClient, models
|
||||||
|
|
||||||
|
from src.application.points.point import Point
|
||||||
|
from src.application.ports.point_repository import PayloadPatch, PointPage
|
||||||
|
|
||||||
|
# Qdrant's scroll returns `(records, next_page_offset)`; a record's id is a
|
||||||
|
# `str | int` union in the SDK, and every id this service writes is a UUID.
|
||||||
|
type _ScrollRecord = models.Record
|
||||||
|
|
||||||
|
|
||||||
|
def _uuid_of(record: _ScrollRecord) -> uuid.UUID:
|
||||||
|
return uuid.UUID(str(record.id))
|
||||||
|
|
||||||
|
|
||||||
|
def _to_point(record: _ScrollRecord, *, with_vectors: bool) -> Point:
|
||||||
|
vectors = record.vector if with_vectors and isinstance(record.vector, dict) else None
|
||||||
|
return Point.from_payload(_uuid_of(record), record.payload or {}, vectors=vectors)
|
||||||
|
|
||||||
|
|
||||||
|
def _conditions(
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> list[models.Condition]:
|
||||||
|
"""The base filter every point query carries.
|
||||||
|
|
||||||
|
`tenant_id` is unconditional and first. `is_active` is added unless the
|
||||||
|
caller explicitly opted into inactive points, which is ADR-0002's
|
||||||
|
"`is_active: true` implied" rule expressed once rather than per method.
|
||||||
|
"""
|
||||||
|
conditions: list[models.Condition] = [
|
||||||
|
models.FieldCondition(key="tenant_id", match=models.MatchValue(value=str(tenant_id)))
|
||||||
|
]
|
||||||
|
if domain is not None:
|
||||||
|
conditions.append(
|
||||||
|
models.FieldCondition(key="domain", match=models.MatchValue(value=domain))
|
||||||
|
)
|
||||||
|
if file_id is not None:
|
||||||
|
conditions.append(
|
||||||
|
models.FieldCondition(key="file_id", match=models.MatchValue(value=str(file_id)))
|
||||||
|
)
|
||||||
|
if not include_inactive:
|
||||||
|
conditions.append(
|
||||||
|
models.FieldCondition(key="is_active", match=models.MatchValue(value=True))
|
||||||
|
)
|
||||||
|
return conditions
|
||||||
|
|
||||||
|
|
||||||
|
class QdrantPointRepository:
|
||||||
|
"""A `PointRepository` (see `src/application/ports/point_repository.py`)."""
|
||||||
|
|
||||||
|
def __init__(self, client: AsyncQdrantClient, *, collection: str) -> None:
|
||||||
|
self._client = client
|
||||||
|
self._collection = collection
|
||||||
|
|
||||||
|
async def get(
|
||||||
|
self, *, tenant_id: uuid.UUID, point_id: uuid.UUID, with_vectors: bool = False
|
||||||
|
) -> Point | None:
|
||||||
|
records, _ = await self._client.scroll(
|
||||||
|
collection_name=self._collection,
|
||||||
|
scroll_filter=models.Filter(
|
||||||
|
must=[
|
||||||
|
*_conditions(tenant_id=tenant_id, include_inactive=True),
|
||||||
|
models.HasIdCondition(has_id=[str(point_id)]),
|
||||||
|
]
|
||||||
|
),
|
||||||
|
limit=1,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=with_vectors,
|
||||||
|
)
|
||||||
|
if not records:
|
||||||
|
return None
|
||||||
|
return _to_point(records[0], with_vectors=with_vectors)
|
||||||
|
|
||||||
|
async def get_many(
|
||||||
|
self, *, tenant_id: uuid.UUID, point_ids: Sequence[uuid.UUID]
|
||||||
|
) -> tuple[Point, ...]:
|
||||||
|
if not point_ids:
|
||||||
|
return ()
|
||||||
|
records, _ = await self._client.scroll(
|
||||||
|
collection_name=self._collection,
|
||||||
|
scroll_filter=models.Filter(
|
||||||
|
must=[
|
||||||
|
*_conditions(tenant_id=tenant_id, include_inactive=True),
|
||||||
|
models.HasIdCondition(has_id=[str(point_id) for point_id in point_ids]),
|
||||||
|
]
|
||||||
|
),
|
||||||
|
limit=len(point_ids),
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
)
|
||||||
|
return tuple(_to_point(record, with_vectors=False) for record in records)
|
||||||
|
|
||||||
|
async def list_by_file(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointPage:
|
||||||
|
conditions = _conditions(
|
||||||
|
tenant_id=tenant_id, file_id=file_id, include_inactive=include_inactive
|
||||||
|
)
|
||||||
|
if cursor is not None:
|
||||||
|
conditions.append(
|
||||||
|
models.FieldCondition(key="order_id", range=models.Range(gt=float(cursor)))
|
||||||
|
)
|
||||||
|
|
||||||
|
records, _ = await self._client.scroll(
|
||||||
|
collection_name=self._collection,
|
||||||
|
scroll_filter=models.Filter(must=conditions),
|
||||||
|
order_by=models.OrderBy(key="order_id", direction=models.Direction.ASC),
|
||||||
|
limit=limit,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
)
|
||||||
|
points = tuple(_to_point(record, with_vectors=False) for record in records)
|
||||||
|
# A short page means the listing is exhausted. A full page might be, but
|
||||||
|
# claiming so would need an extra round trip; handing back a cursor that
|
||||||
|
# yields an empty final page is the cheaper honest answer.
|
||||||
|
next_cursor = repr(points[-1].order_id) if len(points) == limit else None
|
||||||
|
return PointPage(points=points, next_cursor=next_cursor)
|
||||||
|
|
||||||
|
async def count(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> int:
|
||||||
|
result = await self._client.count(
|
||||||
|
collection_name=self._collection,
|
||||||
|
count_filter=models.Filter(
|
||||||
|
must=_conditions(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
exact=True,
|
||||||
|
)
|
||||||
|
return result.count
|
||||||
|
|
||||||
|
async def keyword_search(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
query: str,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None = None,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointPage:
|
||||||
|
conditions = _conditions(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
conditions.append(models.FieldCondition(key="content", match=models.MatchText(text=query)))
|
||||||
|
|
||||||
|
# No `order_by` here, so Qdrant does return a page offset: the full-text
|
||||||
|
# index filters rather than scores, and imposing `order_id` ordering
|
||||||
|
# across files would be meaningless (`order_id` is per-file).
|
||||||
|
records, next_offset = await self._client.scroll(
|
||||||
|
collection_name=self._collection,
|
||||||
|
scroll_filter=models.Filter(must=conditions),
|
||||||
|
limit=limit,
|
||||||
|
offset=cursor,
|
||||||
|
with_payload=True,
|
||||||
|
with_vectors=False,
|
||||||
|
)
|
||||||
|
return PointPage(
|
||||||
|
points=tuple(_to_point(record, with_vectors=False) for record in records),
|
||||||
|
next_cursor=str(next_offset) if next_offset is not None else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def apply_patches(self, *, tenant_id: uuid.UUID, patches: Sequence[PayloadPatch]) -> None:
|
||||||
|
if not patches:
|
||||||
|
return
|
||||||
|
await self._client.batch_update_points(
|
||||||
|
collection_name=self._collection,
|
||||||
|
update_operations=[
|
||||||
|
models.SetPayloadOperation(
|
||||||
|
set_payload=models.SetPayload(
|
||||||
|
payload=patch.payload,
|
||||||
|
filter=models.Filter(must=self._patch_conditions(tenant_id, patch)),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for patch in patches
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _patch_conditions(tenant_id: uuid.UUID, patch: PayloadPatch) -> list[models.Condition]:
|
||||||
|
"""Address one point by id, under this tenant, at an expected version.
|
||||||
|
|
||||||
|
`include_inactive=True`: soft-deleting and relinking both have to reach
|
||||||
|
points the default read filter hides, and a patch is addressed by an
|
||||||
|
explicit id rather than discovered by a listing.
|
||||||
|
"""
|
||||||
|
conditions: list[models.Condition] = [
|
||||||
|
*_conditions(tenant_id=tenant_id, include_inactive=True),
|
||||||
|
models.HasIdCondition(has_id=[str(patch.point_id)]),
|
||||||
|
]
|
||||||
|
if patch.expected_version is not None:
|
||||||
|
conditions.append(
|
||||||
|
models.FieldCondition(
|
||||||
|
key="version", match=models.MatchValue(value=patch.expected_version)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return conditions
|
||||||
97
src/infrastructure/qdrant/points.py
Normal file
97
src/infrastructure/qdrant/points.py
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
"""Qdrant adapter for the `PointStorage` port (ADR-0001, ADR-0002, ADR-0015).
|
||||||
|
|
||||||
|
The `qdrant_client` SDK appears here and nowhere in `application/`. This module
|
||||||
|
translates the SDK-free `ChunkPoint` into `PointStruct`s and builds every
|
||||||
|
filter — routers and application services never construct Qdrant filters.
|
||||||
|
|
||||||
|
`AsyncQdrantClient` is genuinely async, so unlike the `minio` adapter nothing
|
||||||
|
here needs a thread offload.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from qdrant_client import AsyncQdrantClient, models
|
||||||
|
|
||||||
|
from src.application.points.models import ChunkPoint
|
||||||
|
from src.infrastructure.qdrant.collection import SPARSE_VECTOR
|
||||||
|
|
||||||
|
|
||||||
|
def _tenant_file_filter(
|
||||||
|
tenant_id: uuid.UUID, file_id: uuid.UUID, *, from_chunk_index: int
|
||||||
|
) -> models.Filter:
|
||||||
|
"""Points of one file, at or past `from_chunk_index`, within one tenant.
|
||||||
|
|
||||||
|
`tenant_id` is always a condition, never optional: a `file_id` alone is not
|
||||||
|
authority to mutate anything (ADR-0002's isolation rule applies to every
|
||||||
|
code path, not just reads).
|
||||||
|
"""
|
||||||
|
return models.Filter(
|
||||||
|
must=[
|
||||||
|
models.FieldCondition(key="tenant_id", match=models.MatchValue(value=str(tenant_id))),
|
||||||
|
models.FieldCondition(key="file_id", match=models.MatchValue(value=str(file_id))),
|
||||||
|
models.FieldCondition(key="chunk_index", range=models.Range(gte=from_chunk_index)),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class QdrantPointStorage:
|
||||||
|
"""A `PointStorage` (see `src/application/ports/point_storage.py`)."""
|
||||||
|
|
||||||
|
def __init__(self, client: AsyncQdrantClient, *, collection: str) -> None:
|
||||||
|
self._client = client
|
||||||
|
self._collection = collection
|
||||||
|
|
||||||
|
async def upsert_points(self, points: Sequence[ChunkPoint]) -> None:
|
||||||
|
if not points:
|
||||||
|
return
|
||||||
|
await self._client.upsert(
|
||||||
|
collection_name=self._collection,
|
||||||
|
points=[
|
||||||
|
models.PointStruct(
|
||||||
|
id=str(point.point_id),
|
||||||
|
vector={
|
||||||
|
**point.dense,
|
||||||
|
SPARSE_VECTOR: models.SparseVector(
|
||||||
|
indices=point.sparse.indices, values=point.sparse.values
|
||||||
|
),
|
||||||
|
},
|
||||||
|
payload=point.payload,
|
||||||
|
)
|
||||||
|
for point in points
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
async def deactivate_points_from_index(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
from_chunk_index: int,
|
||||||
|
deleted_at: datetime,
|
||||||
|
updated_by: str,
|
||||||
|
) -> int:
|
||||||
|
"""Soft-delete via `set_payload` — the points stay for audit (ADR-0002).
|
||||||
|
|
||||||
|
Counts first so the caller can report how many points the sweep
|
||||||
|
touched; `set_payload` itself reports only an operation status.
|
||||||
|
"""
|
||||||
|
point_filter = _tenant_file_filter(tenant_id, file_id, from_chunk_index=from_chunk_index)
|
||||||
|
stale = await self._client.count(
|
||||||
|
collection_name=self._collection, count_filter=point_filter, exact=True
|
||||||
|
)
|
||||||
|
if stale.count == 0:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
await self._client.set_payload(
|
||||||
|
collection_name=self._collection,
|
||||||
|
payload={
|
||||||
|
"is_active": False,
|
||||||
|
"deleted_at": deleted_at.isoformat(),
|
||||||
|
"updated_at": deleted_at.isoformat(),
|
||||||
|
"updated_by": updated_by,
|
||||||
|
},
|
||||||
|
points=models.FilterSelector(filter=point_filter),
|
||||||
|
)
|
||||||
|
return stale.count
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
||||||
|
from src.api.errors import register_exception_handlers
|
||||||
|
from src.api.middleware import RequestIdMiddleware
|
||||||
from src.api.router import router as v1_router
|
from src.api.router import router as v1_router
|
||||||
from src.api.routers.health import router as health_router
|
from src.api.routers.health import router as health_router
|
||||||
from src.bootstrap.lifespan import create_lifespan
|
from src.bootstrap.lifespan import create_lifespan
|
||||||
@@ -8,6 +10,8 @@ from src.config import Settings
|
|||||||
|
|
||||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||||
app = FastAPI(lifespan=create_lifespan(settings))
|
app = FastAPI(lifespan=create_lifespan(settings))
|
||||||
|
app.add_middleware(RequestIdMiddleware)
|
||||||
|
register_exception_handlers(app)
|
||||||
app.include_router(health_router)
|
app.include_router(health_router)
|
||||||
app.include_router(v1_router, prefix="/v1")
|
app.include_router(v1_router, prefix="/v1")
|
||||||
return app
|
return app
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator, Iterator
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
|
import structlog
|
||||||
from asgi_lifespan import LifespanManager
|
from asgi_lifespan import LifespanManager
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from httpx import ASGITransport, AsyncClient
|
from httpx import ASGITransport, AsyncClient
|
||||||
@@ -9,15 +10,50 @@ from httpx import ASGITransport, AsyncClient
|
|||||||
from src.config import Settings
|
from src.config import Settings
|
||||||
from src.main import create_app
|
from src.main import create_app
|
||||||
|
|
||||||
|
# Disposable Postgres/MinIO/Qdrant containers (ADR-0016). Registered here
|
||||||
|
# because `pytest_plugins` is only honoured in the root conftest, and both
|
||||||
|
# `tests/integration/*/` and `tests/e2e/` need the same containers. The
|
||||||
|
# fixtures are session-scoped but lazy, so unit runs still need no Docker.
|
||||||
|
pytest_plugins = ["tests.support.containers"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_structlog_after_test() -> Iterator[None]:
|
||||||
|
"""Undo any real `configure_logging()` call before the next test runs.
|
||||||
|
|
||||||
|
Any test that exercises the app's lifespan (directly, or via the `client`/
|
||||||
|
`api_client` fixtures below and in `test_domains_api.py`) calls the real
|
||||||
|
`configure_logging()`, which mutates *global* structlog/stdlib state --
|
||||||
|
including `cache_logger_on_first_use=True`. Left in place, that setting
|
||||||
|
silently breaks `structlog.testing.capture_logs()` in unrelated tests
|
||||||
|
later in the same pytest process: a module-level
|
||||||
|
`logger = structlog.get_logger(__name__)` cached under the real config no
|
||||||
|
longer routes through `capture_logs()`'s temporary processor swap, so
|
||||||
|
assertions on captured events see nothing (ADR-0016: isolate per test --
|
||||||
|
this generalizes to global config mutations, not just data).
|
||||||
|
"""
|
||||||
|
yield
|
||||||
|
structlog.reset_defaults()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def settings() -> Settings:
|
def settings() -> Settings:
|
||||||
|
# Every external dependency points at a closed port so unit tests never
|
||||||
|
# reach real infrastructure. For the embedders this matters twice over:
|
||||||
|
# the real defaults are a colleague's Ollama box and OpenAI's paid API,
|
||||||
|
# and ADR-0016 forbids routine runs calling either. Connection-refused is
|
||||||
|
# immediate, so the lifespan's fail-soft warm-up costs nothing here --
|
||||||
|
# and these tests passing at all is what proves it is fail-soft.
|
||||||
return Settings(
|
return Settings(
|
||||||
postgres={"host": "127.0.0.1", "port": 1},
|
postgres={"host": "127.0.0.1", "port": 1},
|
||||||
minio={"endpoint": "127.0.0.1:1"},
|
minio={"endpoint": "127.0.0.1:1"},
|
||||||
ingestion={"timeout_seconds": 1.0},
|
ingestion={"timeout_seconds": 1.0},
|
||||||
qdrant={"url": "http://127.0.0.1:1"},
|
qdrant={"url": "http://127.0.0.1:1"},
|
||||||
app={"readiness_check_timeout_seconds": 0.5},
|
app={"readiness_check_timeout_seconds": 0.5},
|
||||||
|
embedding={
|
||||||
|
"nomic": {"base_url": "http://127.0.0.1:1/v1", "timeout_seconds": 0.5},
|
||||||
|
"openai": {"base_url": "http://127.0.0.1:1/v1", "timeout_seconds": 0.5},
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -26,6 +62,24 @@ def app(settings: Settings) -> FastAPI:
|
|||||||
return create_app(settings)
|
return create_app(settings)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _no_real_logging_configuration(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""Prevent the app lifespan from calling the real `configure_logging()`.
|
||||||
|
|
||||||
|
It sets `cache_logger_on_first_use=True` (ADR-0011), which permanently
|
||||||
|
monkeypatches the `.bind` method on whichever module-level
|
||||||
|
`logger = structlog.get_logger(__name__)` instance is used first --
|
||||||
|
`structlog.reset_defaults()` only resets *global* config, not that
|
||||||
|
per-instance mutation, so real configuration leaking into one test would
|
||||||
|
silently break `structlog.testing.capture_logs()` in every test that runs
|
||||||
|
afterward in the same process (ADR-0016: isolate per test). Tests that
|
||||||
|
spin up the full app via `LifespanManager` (`client`, `api_client`) are
|
||||||
|
testing HTTP behavior, not logging output, so they don't need it for
|
||||||
|
real.
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr("src.bootstrap.lifespan.configure_logging", lambda *a, **k: None)
|
||||||
|
|
||||||
|
|
||||||
@pytest_asyncio.fixture
|
@pytest_asyncio.fixture
|
||||||
async def client(app: FastAPI) -> AsyncIterator[AsyncClient]:
|
async def client(app: FastAPI) -> AsyncIterator[AsyncClient]:
|
||||||
async with (
|
async with (
|
||||||
|
|||||||
244
tests/e2e/conftest.py
Normal file
244
tests/e2e/conftest.py
Normal file
@@ -0,0 +1,244 @@
|
|||||||
|
"""The vertical slice against real Postgres, MinIO, and Qdrant (ADR-0016).
|
||||||
|
|
||||||
|
These are the ADR-0016 end-to-end tests: "a small vertical-slice acceptance
|
||||||
|
contract through the real application composition". They run on
|
||||||
|
Testcontainers, in the default `uv run pytest` run, because ADR-0016 makes
|
||||||
|
Testcontainers "the standard automated integration-test resource mechanism"
|
||||||
|
and reserves Docker Compose for manual validation and the separate serialized
|
||||||
|
smoke test of the *running web process* (`tests/e2e/test_compose_smoke.py`).
|
||||||
|
|
||||||
|
What is real here: routing, auth, scopes, the domain allowlist, the ADR-0008
|
||||||
|
error envelope, the two-transaction upload, MinIO object writes, BM25 sparse
|
||||||
|
embedding, and Qdrant upserts into a per-test collection created by the
|
||||||
|
production `ensure_chunks_collection`.
|
||||||
|
|
||||||
|
What is faked, and only this: the two dense embedders. They are paid/remote
|
||||||
|
network calls, and ADR-0016 forbids routine runs from touching a live
|
||||||
|
provider. Their fakes return the pinned 768/3072 dimensions, because the real
|
||||||
|
collection rejects anything else.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import AsyncIterator, Callable, Coroutine
|
||||||
|
from contextlib import AsyncExitStack
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
from asgi_lifespan import LifespanManager
|
||||||
|
from httpx import ASGITransport, AsyncClient
|
||||||
|
from qdrant_client import AsyncQdrantClient, models
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.bootstrap.dependencies import get_dense_embedders, get_sessionmaker
|
||||||
|
from src.config import MinioSettings, PostgresSettings, QdrantSettings, Settings
|
||||||
|
from src.infrastructure.postgres.database import create_sessionmaker
|
||||||
|
from src.infrastructure.qdrant.collection import (
|
||||||
|
DENSE_NOMIC_DIMENSIONS,
|
||||||
|
DENSE_OPENAI_DIMENSIONS,
|
||||||
|
ensure_chunks_collection,
|
||||||
|
)
|
||||||
|
from src.main import create_app
|
||||||
|
from tests.fakes import FakeDenseEmbedder
|
||||||
|
from tests.support.factories import create_api_key, create_tenant, create_tenant_domain
|
||||||
|
|
||||||
|
CSV_BYTES = b"question,answer\nhow do I file a claim,call the branch\nwhat is covered,see policy\n"
|
||||||
|
|
||||||
|
# Past validation's OOXML magic-byte check, so the failure lands in the DOCX
|
||||||
|
# parser rather than in upload validation -- the branch this exercises.
|
||||||
|
CORRUPT_DOCX_BYTES = b"PK\x03\x04" + b"not really a docx" * 8
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class E2EStack:
|
||||||
|
"""One configured app instance plus the handles a test asserts against."""
|
||||||
|
|
||||||
|
client: AsyncClient
|
||||||
|
collection: str
|
||||||
|
dense_embedders: list[FakeDenseEmbedder]
|
||||||
|
|
||||||
|
|
||||||
|
StackFactory = Callable[..., Coroutine[Any, Any, E2EStack]]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Principal:
|
||||||
|
"""A provisioned tenant with a bearer token and a registered domain.
|
||||||
|
|
||||||
|
Plain values, not the ORM `Tenant`: tests re-read rows the app committed on
|
||||||
|
a shared connection, and any expiry/refresh of a held ORM instance would
|
||||||
|
lazy-load outside a greenlet context (`MissingGreenlet`).
|
||||||
|
"""
|
||||||
|
|
||||||
|
tenant_id: uuid.UUID
|
||||||
|
tenant_slug: str
|
||||||
|
token: str
|
||||||
|
domain: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def headers(self) -> dict[str, str]:
|
||||||
|
return {"Authorization": f"Bearer {self.token}"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def e2e_settings(
|
||||||
|
postgres_settings: PostgresSettings,
|
||||||
|
minio_settings: MinioSettings,
|
||||||
|
qdrant_settings: QdrantSettings,
|
||||||
|
) -> Settings:
|
||||||
|
"""Settings wired to this test's containers.
|
||||||
|
|
||||||
|
Every nested settings object is passed explicitly. `src/config.py` cascades
|
||||||
|
`.env` into each nested class, so omitting one would let a developer's real
|
||||||
|
`EMBEDDING_NOMIC_BASE_URL` (a colleague's Ollama box) into a routine test
|
||||||
|
run. The embedder URLs below point at a closed port for the same reason:
|
||||||
|
the lifespan warms the *real* embedders before the fakes are substituted,
|
||||||
|
and connection-refused is both instant and free.
|
||||||
|
"""
|
||||||
|
return Settings(
|
||||||
|
postgres=postgres_settings,
|
||||||
|
minio=minio_settings,
|
||||||
|
qdrant=qdrant_settings,
|
||||||
|
embedding={
|
||||||
|
"nomic": {"base_url": "http://127.0.0.1:1/v1", "timeout_seconds": 0.5},
|
||||||
|
"openai": {"base_url": "http://127.0.0.1:1/v1", "timeout_seconds": 0.5},
|
||||||
|
},
|
||||||
|
app={"readiness_check_timeout_seconds": 2.0},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture(loop_scope="session")
|
||||||
|
async def make_stack(
|
||||||
|
e2e_settings: Settings,
|
||||||
|
db_sessionmaker: async_sessionmaker[AsyncSession],
|
||||||
|
qdrant_client: AsyncQdrantClient,
|
||||||
|
) -> AsyncIterator[StackFactory]:
|
||||||
|
"""Build an app under test, optionally with tightened ingestion bounds.
|
||||||
|
|
||||||
|
A factory rather than a fixture because the capacity and timeout tests need
|
||||||
|
their own `INGESTION_*` values, and those are read at app construction.
|
||||||
|
"""
|
||||||
|
async with AsyncExitStack() as exit_stack:
|
||||||
|
|
||||||
|
async def _make(
|
||||||
|
*,
|
||||||
|
ingestion: dict[str, object] | None = None,
|
||||||
|
collection: str | None = None,
|
||||||
|
dense_delay_seconds: float = 0.0,
|
||||||
|
bootstrap_collection: bool = True,
|
||||||
|
sessionmaker: async_sessionmaker[AsyncSession] | None = None,
|
||||||
|
) -> E2EStack:
|
||||||
|
resolved_collection = collection or e2e_settings.qdrant.collection
|
||||||
|
if bootstrap_collection:
|
||||||
|
await ensure_chunks_collection(qdrant_client, collection=resolved_collection)
|
||||||
|
|
||||||
|
settings = e2e_settings.model_copy(
|
||||||
|
update={
|
||||||
|
"qdrant": e2e_settings.qdrant.model_copy(
|
||||||
|
update={"collection": resolved_collection}
|
||||||
|
),
|
||||||
|
"ingestion": e2e_settings.ingestion.model_copy(update=ingestion or {}),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
dense = [
|
||||||
|
FakeDenseEmbedder(
|
||||||
|
name="dense_nomic",
|
||||||
|
dimensions=DENSE_NOMIC_DIMENSIONS,
|
||||||
|
value=0.1,
|
||||||
|
delay_seconds=dense_delay_seconds,
|
||||||
|
),
|
||||||
|
FakeDenseEmbedder(
|
||||||
|
name="dense_openai",
|
||||||
|
dimensions=DENSE_OPENAI_DIMENSIONS,
|
||||||
|
value=0.2,
|
||||||
|
delay_seconds=dense_delay_seconds,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
app = create_app(settings)
|
||||||
|
# The session factory is the test's, so factory-created tenants are
|
||||||
|
# visible to the app and every write rolls back at teardown. Object
|
||||||
|
# storage, point storage, and the sparse embedder stay real.
|
||||||
|
resolved_sessionmaker = sessionmaker or db_sessionmaker
|
||||||
|
app.dependency_overrides[get_sessionmaker] = lambda: resolved_sessionmaker
|
||||||
|
app.dependency_overrides[get_dense_embedders] = lambda: dense
|
||||||
|
|
||||||
|
manager = await exit_stack.enter_async_context(LifespanManager(app))
|
||||||
|
client = await exit_stack.enter_async_context(
|
||||||
|
AsyncClient(
|
||||||
|
transport=ASGITransport(app=manager.app),
|
||||||
|
base_url="http://test",
|
||||||
|
timeout=30.0,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return E2EStack(client=client, collection=resolved_collection, dense_embedders=dense)
|
||||||
|
|
||||||
|
yield _make
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture(loop_scope="session")
|
||||||
|
async def committed_sessionmaker(
|
||||||
|
postgres_engine: AsyncEngine,
|
||||||
|
) -> async_sessionmaker[AsyncSession]:
|
||||||
|
"""A session factory whose writes really commit, one connection each.
|
||||||
|
|
||||||
|
The default `db_sessionmaker` pins every session to a single connection
|
||||||
|
inside one rolled-back transaction, which is what isolates a test's writes
|
||||||
|
-- but savepoints on a shared connection cannot interleave, so any test
|
||||||
|
that issues genuinely *concurrent* requests deadlocks or fails with
|
||||||
|
`InvalidSavepointSpecificationError`. Those tests use this instead and rely
|
||||||
|
on their uuid-keyed tenant for isolation (ADR-0016 allows either).
|
||||||
|
"""
|
||||||
|
return create_sessionmaker(postgres_engine)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture(loop_scope="session")
|
||||||
|
async def principal(db_session: AsyncSession) -> Principal:
|
||||||
|
"""A tenant with an upload-scoped key and one registered domain.
|
||||||
|
|
||||||
|
Registering the domain is not optional: `POST /v1/files` rejects an
|
||||||
|
unregistered one with `400 unknown_domain` before anything else runs
|
||||||
|
(ADR-0009).
|
||||||
|
"""
|
||||||
|
return await provision_principal(db_session)
|
||||||
|
|
||||||
|
|
||||||
|
async def provision_principal(
|
||||||
|
session: AsyncSession, *, domain: str = "fire", scopes: list[str] | None = None
|
||||||
|
) -> Principal:
|
||||||
|
tenant = await create_tenant(session)
|
||||||
|
_, token = await create_api_key(session, tenant=tenant, scopes=scopes or ["files:write"])
|
||||||
|
await create_tenant_domain(session, tenant=tenant, domain=domain)
|
||||||
|
await session.commit()
|
||||||
|
return Principal(tenant_id=tenant.id, tenant_slug=tenant.slug, token=token, domain=domain)
|
||||||
|
|
||||||
|
|
||||||
|
def upload_payload(
|
||||||
|
*, domain: str, filename: str = "faq.csv", data: bytes = CSV_BYTES
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {"files": {"file": (filename, data, "text/csv")}, "data": {"domain": domain}}
|
||||||
|
|
||||||
|
|
||||||
|
async def count_active_points(
|
||||||
|
client: AsyncQdrantClient, collection: str, *, tenant_id: uuid.UUID
|
||||||
|
) -> int:
|
||||||
|
"""Active points for one tenant, read through the raw client.
|
||||||
|
|
||||||
|
`PointStorage` is deliberately write-only (point reads are plan 002's
|
||||||
|
`/v1/points`), so the verification read here uses the SDK directly.
|
||||||
|
"""
|
||||||
|
result = await client.count(
|
||||||
|
collection_name=collection,
|
||||||
|
count_filter=models.Filter(
|
||||||
|
must=[
|
||||||
|
models.FieldCondition(
|
||||||
|
key="tenant_id", match=models.MatchValue(value=str(tenant_id))
|
||||||
|
),
|
||||||
|
models.FieldCondition(key="is_active", match=models.MatchValue(value=True)),
|
||||||
|
]
|
||||||
|
),
|
||||||
|
exact=True,
|
||||||
|
)
|
||||||
|
return result.count
|
||||||
143
tests/e2e/test_compose_smoke.py
Normal file
143
tests/e2e/test_compose_smoke.py
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
"""Operational smoke test against the *running web process* (ADR-0016).
|
||||||
|
|
||||||
|
Different in kind from every other test in this repo. Everything else drives
|
||||||
|
the app in-process through `ASGITransport`; this drives a real uvicorn process
|
||||||
|
over a real socket, backed by the Docker Compose stack, after the two
|
||||||
|
deployment steps (`alembic upgrade head`, `src.cli.qdrant_bootstrap`) have run
|
||||||
|
for real. ADR-0016 reserves Compose for exactly this: "manual local validation
|
||||||
|
and a later, serialized operational smoke test of the running web process",
|
||||||
|
run "as a serialized pre-release or scheduled gate".
|
||||||
|
|
||||||
|
It is therefore **skipped unless `SMOKE_BASE_URL` is set**, so a plain
|
||||||
|
`uv run pytest` never touches Docker Compose. Run it through `scripts/smoke.sh`,
|
||||||
|
which brings the stack up, provisions a tenant, starts the process, and exports
|
||||||
|
the environment below.
|
||||||
|
|
||||||
|
Because the app runs in its own process, this is also the only place in the
|
||||||
|
suite where the real `configure_logging()` is in effect --
|
||||||
|
`tests/conftest.py::_no_real_logging_configuration` no-ops it everywhere else
|
||||||
|
to keep global structlog state out of other tests. So the ADR-0011 log sink is
|
||||||
|
asserted here and nowhere else.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
from collections.abc import AsyncIterator
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
from httpx import AsyncClient
|
||||||
|
from qdrant_client import AsyncQdrantClient, models
|
||||||
|
|
||||||
|
BASE_URL = os.environ.get("SMOKE_BASE_URL")
|
||||||
|
API_KEY = os.environ.get("SMOKE_API_KEY", "")
|
||||||
|
DOMAIN = os.environ.get("SMOKE_DOMAIN", "smoke")
|
||||||
|
LOG_PATH = os.environ.get("SMOKE_LOG_PATH", "")
|
||||||
|
QDRANT_URL = os.environ.get("SMOKE_QDRANT_URL", "http://127.0.0.1:6343")
|
||||||
|
QDRANT_COLLECTION = os.environ.get("SMOKE_QDRANT_COLLECTION", "chunks")
|
||||||
|
|
||||||
|
pytestmark = [
|
||||||
|
pytest.mark.e2e,
|
||||||
|
pytest.mark.slow,
|
||||||
|
pytest.mark.asyncio,
|
||||||
|
pytest.mark.skipif(
|
||||||
|
not BASE_URL,
|
||||||
|
reason="SMOKE_BASE_URL is unset; run this through scripts/smoke.sh",
|
||||||
|
),
|
||||||
|
# Well past the global 10s budget: this drives a real process over a
|
||||||
|
# socket, and the upload does real embedding work end to end.
|
||||||
|
pytest.mark.timeout(120),
|
||||||
|
]
|
||||||
|
|
||||||
|
# Unique per run so a re-run against the same persistent Compose volumes is a
|
||||||
|
# new file rather than an idempotent duplicate-hash hit (ADR-0017).
|
||||||
|
_CSV = f"question,answer\nsmoke run {uuid.uuid4().hex},indexed\n".encode()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture
|
||||||
|
async def client() -> AsyncIterator[AsyncClient]:
|
||||||
|
async with AsyncClient(base_url=BASE_URL or "", timeout=120.0) as async_client:
|
||||||
|
yield async_client
|
||||||
|
|
||||||
|
|
||||||
|
async def test_running_process_reports_healthy_and_ready() -> None:
|
||||||
|
"""Readiness must be green *after* the deployment steps, not before --
|
||||||
|
an unbootstrapped Qdrant is deliberately unready.
|
||||||
|
"""
|
||||||
|
async with AsyncClient(base_url=BASE_URL or "", timeout=30.0) as client:
|
||||||
|
healthz = await client.get("/healthz")
|
||||||
|
readyz = await client.get("/readyz")
|
||||||
|
|
||||||
|
assert healthz.status_code == 200
|
||||||
|
assert healthz.json() == {"status": "ok"}
|
||||||
|
assert readyz.status_code == 200, readyz.text
|
||||||
|
assert readyz.json() == {"postgres": True, "minio": True, "qdrant": True}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_through_the_running_process_indexes_retrievable_points(
|
||||||
|
client: AsyncClient,
|
||||||
|
) -> None:
|
||||||
|
"""The whole slice against the deployed shape: HTTP in, Qdrant points out."""
|
||||||
|
upload = await client.post(
|
||||||
|
"/v1/files",
|
||||||
|
files={"file": ("smoke.csv", _CSV, "text/csv")},
|
||||||
|
data={"domain": DOMAIN},
|
||||||
|
headers={"Authorization": f"Bearer {API_KEY}"},
|
||||||
|
)
|
||||||
|
assert upload.status_code == 201, upload.text
|
||||||
|
body = upload.json()
|
||||||
|
assert body["status"] == "succeeded"
|
||||||
|
assert body["chunks_indexed"] > 0
|
||||||
|
|
||||||
|
status = await client.get(
|
||||||
|
f"/v1/files/{body['file_id']}", headers={"Authorization": f"Bearer {API_KEY}"}
|
||||||
|
)
|
||||||
|
assert status.status_code == 200
|
||||||
|
assert status.json()["ingestion_status"] == "succeeded"
|
||||||
|
|
||||||
|
qdrant = AsyncQdrantClient(url=QDRANT_URL)
|
||||||
|
try:
|
||||||
|
count = await qdrant.count(
|
||||||
|
collection_name=QDRANT_COLLECTION,
|
||||||
|
count_filter=models.Filter(
|
||||||
|
must=[
|
||||||
|
models.FieldCondition(
|
||||||
|
key="file_id", match=models.MatchValue(value=body["file_id"])
|
||||||
|
),
|
||||||
|
models.FieldCondition(key="is_active", match=models.MatchValue(value=True)),
|
||||||
|
]
|
||||||
|
),
|
||||||
|
exact=True,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
await qdrant.close()
|
||||||
|
|
||||||
|
assert count.count == body["chunks_indexed"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_running_process_writes_structured_ingestion_logs() -> None:
|
||||||
|
"""The real ADR-0011 sink, which only a separate process exercises.
|
||||||
|
|
||||||
|
An upload that succeeds while emitting nothing parseable is an upload no
|
||||||
|
operator can investigate -- and the runbook's failure procedure is written
|
||||||
|
against these exact event names.
|
||||||
|
"""
|
||||||
|
if not LOG_PATH:
|
||||||
|
pytest.skip("SMOKE_LOG_PATH is unset; scripts/smoke.sh normally provides it")
|
||||||
|
|
||||||
|
lines = Path(LOG_PATH).read_text("utf-8").splitlines()
|
||||||
|
events = [json.loads(line) for line in lines if line.startswith("{")]
|
||||||
|
|
||||||
|
by_name = {event.get("event") for event in events}
|
||||||
|
assert "ingestion.job.started" in by_name
|
||||||
|
assert "ingestion.job.completed" in by_name
|
||||||
|
|
||||||
|
completed = next(event for event in events if event.get("event") == "ingestion.job.completed")
|
||||||
|
assert completed["tenant_id"]
|
||||||
|
assert completed["ingestion_job_id"]
|
||||||
|
assert completed["points_upserted"] > 0
|
||||||
|
# Static process context bound once at startup (ADR-0011).
|
||||||
|
assert completed["env"]
|
||||||
|
assert completed["service_version"]
|
||||||
329
tests/e2e/test_ingestion_slice.py
Normal file
329
tests/e2e/test_ingestion_slice.py
Normal file
@@ -0,0 +1,329 @@
|
|||||||
|
"""The ingestion vertical slice end to end (plan 001, Phase 6).
|
||||||
|
|
||||||
|
Every test here asserts a reliability invariant ADR-0016 lists as a reusable
|
||||||
|
contract, not a happy path: the bound maps to its status code, the failure
|
||||||
|
still writes a terminal job row, the retry converges, and the tenant filter
|
||||||
|
holds. Failures additionally assert the ADR-0008 error envelope, because the
|
||||||
|
status code alone is not the contract a client integrates against.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from httpx import Response
|
||||||
|
from qdrant_client import AsyncQdrantClient
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from src.infrastructure.postgres.models.ingestion_job import IngestionJob
|
||||||
|
from tests.e2e.conftest import (
|
||||||
|
CORRUPT_DOCX_BYTES,
|
||||||
|
Principal,
|
||||||
|
StackFactory,
|
||||||
|
count_active_points,
|
||||||
|
provision_principal,
|
||||||
|
upload_payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = [
|
||||||
|
pytest.mark.e2e,
|
||||||
|
pytest.mark.postgres,
|
||||||
|
pytest.mark.minio,
|
||||||
|
pytest.mark.qdrant,
|
||||||
|
pytest.mark.asyncio(loop_scope="session"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def _latest_job(session: AsyncSession, *, tenant_id: uuid.UUID) -> IngestionJob:
|
||||||
|
"""The tenant's most recent job row, read fresh.
|
||||||
|
|
||||||
|
`populate_existing` matters: this session shares a connection with the
|
||||||
|
app's sessions, so without it the identity map can answer from a row
|
||||||
|
loaded before the request under test committed its update.
|
||||||
|
"""
|
||||||
|
result = await session.execute(
|
||||||
|
select(IngestionJob)
|
||||||
|
.where(IngestionJob.tenant_id == tenant_id)
|
||||||
|
.order_by(IngestionJob.created_at.desc(), IngestionJob.id)
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
)
|
||||||
|
return result.scalars().first() or pytest.fail("no ingestion job was written")
|
||||||
|
|
||||||
|
|
||||||
|
def _envelope(response: Response) -> dict[str, object]:
|
||||||
|
"""The ADR-0008 error body. Asserting the code here, not just the HTTP
|
||||||
|
status, is the difference between testing the contract and testing the
|
||||||
|
number."""
|
||||||
|
error: dict[str, object] = response.json()["error"]
|
||||||
|
assert error["request_id"], "every error envelope carries its correlation id"
|
||||||
|
return error
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_succeeds_and_indexes_points_under_the_tenant_filter(
|
||||||
|
make_stack: StackFactory, principal: Principal, qdrant_client: AsyncQdrantClient
|
||||||
|
) -> None:
|
||||||
|
stack = await make_stack()
|
||||||
|
|
||||||
|
response = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 201
|
||||||
|
body = response.json()
|
||||||
|
assert body["status"] == "succeeded"
|
||||||
|
assert body["chunks_indexed"] > 0
|
||||||
|
indexed = await count_active_points(
|
||||||
|
qdrant_client, stack.collection, tenant_id=principal.tenant_id
|
||||||
|
)
|
||||||
|
assert indexed == body["chunks_indexed"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_file_reports_the_terminal_status_after_upload(
|
||||||
|
make_stack: StackFactory, principal: Principal
|
||||||
|
) -> None:
|
||||||
|
stack = await make_stack()
|
||||||
|
created = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
status = await stack.client.get(
|
||||||
|
f"/v1/files/{created.json()['file_id']}", headers=principal.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert status.status_code == 200
|
||||||
|
assert status.json()["ingestion_status"] == "succeeded"
|
||||||
|
assert status.json()["domain"] == principal.domain
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_duplicate_content_returns_the_existing_job_without_reingesting(
|
||||||
|
make_stack: StackFactory, principal: Principal, qdrant_client: AsyncQdrantClient
|
||||||
|
) -> None:
|
||||||
|
stack = await make_stack()
|
||||||
|
first = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
points_after_first = await count_active_points(
|
||||||
|
qdrant_client, stack.collection, tenant_id=principal.tenant_id
|
||||||
|
)
|
||||||
|
|
||||||
|
second = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert first.status_code == 201
|
||||||
|
# 200, not 201: nothing was created the second time (ADR-0017 idempotency).
|
||||||
|
assert second.status_code == 200
|
||||||
|
assert second.json()["file_id"] == first.json()["file_id"]
|
||||||
|
assert second.json()["ingestion_job_id"] == first.json()["ingestion_job_id"]
|
||||||
|
assert (
|
||||||
|
await count_active_points(qdrant_client, stack.collection, tenant_id=principal.tenant_id)
|
||||||
|
== points_after_first
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_retried_after_a_failed_job_succeeds_without_duplicate_points(
|
||||||
|
make_stack: StackFactory,
|
||||||
|
principal: Principal,
|
||||||
|
db_session: AsyncSession,
|
||||||
|
qdrant_client: AsyncQdrantClient,
|
||||||
|
) -> None:
|
||||||
|
"""ADR-0017, "Re-running an ingestion stays safe": a failed attempt is
|
||||||
|
retried by re-uploading the same bytes, and deterministic point ids make
|
||||||
|
the retry converge instead of duplicating.
|
||||||
|
"""
|
||||||
|
stack = await make_stack()
|
||||||
|
stack.dense_embedders[0].fail_next = True
|
||||||
|
|
||||||
|
failed = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
failed_job = await _latest_job(db_session, tenant_id=principal.tenant_id)
|
||||||
|
retried = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert failed.status_code == 502
|
||||||
|
assert failed_job.status == "failed"
|
||||||
|
assert failed_job.error_code == "embedding_failed"
|
||||||
|
assert retried.status_code == 201
|
||||||
|
assert retried.json()["status"] == "succeeded"
|
||||||
|
# Same source file, new attempt -- a terminal job is never reused or
|
||||||
|
# transitioned back to running.
|
||||||
|
assert retried.json()["ingestion_job_id"] != str(failed_job.id)
|
||||||
|
indexed = await count_active_points(
|
||||||
|
qdrant_client, stack.collection, tenant_id=principal.tenant_id
|
||||||
|
)
|
||||||
|
assert indexed == retried.json()["chunks_indexed"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_file_for_another_tenants_file_returns_404_not_403(
|
||||||
|
make_stack: StackFactory, principal: Principal, db_session: AsyncSession
|
||||||
|
) -> None:
|
||||||
|
"""404, never 403: a 403 would confirm the file exists to a stranger."""
|
||||||
|
stack = await make_stack()
|
||||||
|
created = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
other = await provision_principal(db_session)
|
||||||
|
|
||||||
|
response = await stack.client.get(
|
||||||
|
f"/v1/files/{created.json()['file_id']}", headers=other.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 404
|
||||||
|
assert _envelope(response)["code"] == "not_found"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_to_an_unregistered_domain_is_rejected_before_any_job_is_written(
|
||||||
|
make_stack: StackFactory, principal: Principal, db_session: AsyncSession
|
||||||
|
) -> None:
|
||||||
|
stack = await make_stack()
|
||||||
|
|
||||||
|
response = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain="never-registered"), headers=principal.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 400
|
||||||
|
assert _envelope(response)["code"] == "unknown_domain"
|
||||||
|
jobs = await db_session.execute(
|
||||||
|
select(IngestionJob).where(IngestionJob.tenant_id == principal.tenant_id)
|
||||||
|
)
|
||||||
|
assert jobs.scalars().all() == []
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_at_capacity_is_rejected_with_503_and_retry_after(
|
||||||
|
make_stack: StackFactory, committed_sessionmaker: async_sessionmaker[AsyncSession]
|
||||||
|
) -> None:
|
||||||
|
"""ADR-0017 rejects rather than queues, so the second concurrent upload
|
||||||
|
must fail fast with a backoff hint instead of waiting for a slot.
|
||||||
|
|
||||||
|
This is the one test that needs two requests genuinely in flight at once,
|
||||||
|
so it runs on `committed_sessionmaker` -- see that fixture for why the
|
||||||
|
shared-connection default cannot serve concurrent sessions.
|
||||||
|
"""
|
||||||
|
async with committed_sessionmaker() as session:
|
||||||
|
principal = await provision_principal(session)
|
||||||
|
stack = await make_stack(
|
||||||
|
ingestion={"max_concurrency": 1},
|
||||||
|
dense_delay_seconds=0.4,
|
||||||
|
sessionmaker=committed_sessionmaker,
|
||||||
|
)
|
||||||
|
|
||||||
|
first, second = await asyncio.gather(
|
||||||
|
stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
),
|
||||||
|
stack.client.post(
|
||||||
|
"/v1/files",
|
||||||
|
**upload_payload(domain=principal.domain, filename="other.csv", data=b"a,b\n1,2\n"),
|
||||||
|
headers=principal.headers,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
statuses = sorted([first.status_code, second.status_code])
|
||||||
|
assert statuses == [201, 503]
|
||||||
|
rejected = first if first.status_code == 503 else second
|
||||||
|
assert _envelope(rejected)["code"] == "ingestion_at_capacity"
|
||||||
|
assert rejected.headers["Retry-After"] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_exceeding_the_timeout_returns_504_and_writes_a_terminal_job(
|
||||||
|
make_stack: StackFactory, principal: Principal, db_session: AsyncSession
|
||||||
|
) -> None:
|
||||||
|
"""A timeout must never leave a job stuck in `running` (ADR-0016)."""
|
||||||
|
stack = await make_stack(ingestion={"timeout_seconds": 0.05}, dense_delay_seconds=0.5)
|
||||||
|
|
||||||
|
response = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 504
|
||||||
|
assert _envelope(response)["code"] == "ingestion_timeout"
|
||||||
|
job = await _latest_job(db_session, tenant_id=principal.tenant_id)
|
||||||
|
assert job.status == "failed"
|
||||||
|
assert job.error_code == "timeout"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_of_an_unparseable_document_returns_400_and_a_failed_job(
|
||||||
|
make_stack: StackFactory, principal: Principal, db_session: AsyncSession
|
||||||
|
) -> None:
|
||||||
|
stack = await make_stack()
|
||||||
|
|
||||||
|
response = await stack.client.post(
|
||||||
|
"/v1/files",
|
||||||
|
**upload_payload(domain=principal.domain, filename="broken.docx", data=CORRUPT_DOCX_BYTES),
|
||||||
|
headers=principal.headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 400
|
||||||
|
assert _envelope(response)["code"] == "validation_error"
|
||||||
|
job = await _latest_job(db_session, tenant_id=principal.tenant_id)
|
||||||
|
assert job.status == "failed"
|
||||||
|
assert job.error_code == "parse_failed"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_returns_502_when_qdrant_indexing_fails(
|
||||||
|
make_stack: StackFactory, principal: Principal, db_session: AsyncSession
|
||||||
|
) -> None:
|
||||||
|
"""A real Qdrant error, not a fake: the collection the deployment step
|
||||||
|
should have created is missing, which is the failure an operator actually
|
||||||
|
hits when `qdrant_bootstrap` was skipped.
|
||||||
|
"""
|
||||||
|
stack = await make_stack(
|
||||||
|
collection=f"never_bootstrapped_{uuid.uuid4().hex}", bootstrap_collection=False
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=principal.domain), headers=principal.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 502
|
||||||
|
assert _envelope(response)["code"] == "index_error"
|
||||||
|
job = await _latest_job(db_session, tenant_id=principal.tenant_id)
|
||||||
|
assert job.status == "failed"
|
||||||
|
assert job.error_code == "index_failed"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_upload_without_the_files_write_scope_is_forbidden(
|
||||||
|
make_stack: StackFactory, db_session: AsyncSession
|
||||||
|
) -> None:
|
||||||
|
stack = await make_stack()
|
||||||
|
reader = await provision_principal(db_session, scopes=["domains:read"])
|
||||||
|
|
||||||
|
response = await stack.client.post(
|
||||||
|
"/v1/files", **upload_payload(domain=reader.domain), headers=reader.headers
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 403
|
||||||
|
assert _envelope(response)["code"] == "missing_scope"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_readyz_reports_every_dependency_ready_against_real_services(
|
||||||
|
make_stack: StackFactory,
|
||||||
|
) -> None:
|
||||||
|
"""`/readyz` is dependency readiness, `/healthz` is process health."""
|
||||||
|
stack = await make_stack()
|
||||||
|
|
||||||
|
ready = await stack.client.get("/readyz")
|
||||||
|
healthy = await stack.client.get("/healthz")
|
||||||
|
|
||||||
|
assert healthy.status_code == 200
|
||||||
|
assert ready.status_code == 200
|
||||||
|
assert ready.json() == {"postgres": True, "minio": True, "qdrant": True}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_readyz_is_unready_when_the_chunks_collection_is_missing(
|
||||||
|
make_stack: StackFactory,
|
||||||
|
) -> None:
|
||||||
|
"""A reachable but unbootstrapped Qdrant is deliberately not ready --
|
||||||
|
uploads to it would 502 (see the indexing test above).
|
||||||
|
"""
|
||||||
|
stack = await make_stack(
|
||||||
|
collection=f"never_bootstrapped_{uuid.uuid4().hex}", bootstrap_collection=False
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await stack.client.get("/readyz")
|
||||||
|
|
||||||
|
assert response.status_code == 503
|
||||||
|
assert response.json()["qdrant"] is False
|
||||||
304
tests/fakes.py
304
tests/fakes.py
@@ -1,5 +1,301 @@
|
|||||||
"""Hand-written fakes for narrow application-owned ports.
|
"""Hand-written fakes for narrow application-owned ports (ADR-0016)."""
|
||||||
|
|
||||||
No application ports exist yet (Phase 1 only wires infrastructure client
|
import asyncio
|
||||||
lifecycle). Fakes are added here as ports are introduced in later phases.
|
import uuid
|
||||||
"""
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from src.application.ingestion.models import SparseVector
|
||||||
|
from src.application.points.models import ChunkPoint
|
||||||
|
from src.application.points.point import Point
|
||||||
|
from src.application.ports.point_repository import PayloadPatch, PointPage
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FakeObjectStorage:
|
||||||
|
"""In-memory `ObjectStorage`. `fail_next` simulates one upload failure."""
|
||||||
|
|
||||||
|
objects: dict[str, bytes] = field(default_factory=dict)
|
||||||
|
fail_next: bool = False
|
||||||
|
|
||||||
|
async def put_object(self, *, key: str, data: bytes, content_type: str) -> None:
|
||||||
|
if self.fail_next:
|
||||||
|
self.fail_next = False
|
||||||
|
raise OSError("simulated object storage failure")
|
||||||
|
self.objects[key] = data
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FakeDenseEmbedder:
|
||||||
|
"""A scripted `DenseEmbedder`. Returns a fixed-dimension zero vector per
|
||||||
|
text by default; `fail_next` simulates one batch failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
dimensions: int = 4
|
||||||
|
value: float = 0.0
|
||||||
|
"""Component value of every returned vector. Non-zero where a real Qdrant
|
||||||
|
has to score the result, since a zero vector has no direction to compare."""
|
||||||
|
model_version: str = "fake-dense-v1"
|
||||||
|
calls: list[list[str]] = field(default_factory=list)
|
||||||
|
fail_next: bool = False
|
||||||
|
delay_seconds: float = 0.0
|
||||||
|
"""Simulates a slow provider call, e.g. to exercise timeout handling."""
|
||||||
|
|
||||||
|
async def embed_batch(self, texts: Sequence[str]) -> list[list[float]]:
|
||||||
|
self.calls.append(list(texts))
|
||||||
|
if self.delay_seconds:
|
||||||
|
await asyncio.sleep(self.delay_seconds)
|
||||||
|
if self.fail_next:
|
||||||
|
self.fail_next = False
|
||||||
|
raise RuntimeError("simulated embedder failure")
|
||||||
|
return [[self.value] * self.dimensions for _ in texts]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FakeSparseEmbedder:
|
||||||
|
"""A scripted `SparseEmbedder`. Returns an empty sparse vector per text."""
|
||||||
|
|
||||||
|
name: str = "sparse"
|
||||||
|
model_version: str = "fake-sparse-v1"
|
||||||
|
calls: list[list[str]] = field(default_factory=list)
|
||||||
|
fail_next: bool = False
|
||||||
|
|
||||||
|
def embed_batch(self, texts: Sequence[str], *, query: bool = False) -> list[SparseVector]:
|
||||||
|
self.calls.append(list(texts))
|
||||||
|
if self.fail_next:
|
||||||
|
self.fail_next = False
|
||||||
|
raise RuntimeError("simulated embedder failure")
|
||||||
|
return [SparseVector(indices=[], values=[]) for _ in texts]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FakePointStorage:
|
||||||
|
"""An in-memory `PointStorage`.
|
||||||
|
|
||||||
|
`points` is keyed by point id, so a re-upsert of the same deterministic id
|
||||||
|
overwrites rather than accumulating — the property a test asserting "a
|
||||||
|
retry produces no duplicate points" needs the fake to actually model.
|
||||||
|
|
||||||
|
`fail_on_batch` fails the Nth (0-based) upsert batch, which is how a test
|
||||||
|
checks that the soft-delete sweep never runs after a partial failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
points: dict[str, ChunkPoint] = field(default_factory=dict)
|
||||||
|
upsert_batches: list[int] = field(default_factory=list)
|
||||||
|
deactivate_calls: list[dict[str, object]] = field(default_factory=list)
|
||||||
|
fail_on_batch: int | None = None
|
||||||
|
fail_deactivate: bool = False
|
||||||
|
max_in_flight: int = 0
|
||||||
|
_in_flight: int = 0
|
||||||
|
|
||||||
|
async def upsert_points(self, points: Sequence[ChunkPoint]) -> None:
|
||||||
|
self._in_flight += 1
|
||||||
|
self.max_in_flight = max(self.max_in_flight, self._in_flight)
|
||||||
|
try:
|
||||||
|
# Yield so concurrent batches actually overlap; without this the
|
||||||
|
# in-flight ceiling is trivially 1 and the bound goes untested.
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
index = len(self.upsert_batches)
|
||||||
|
self.upsert_batches.append(len(points))
|
||||||
|
if self.fail_on_batch is not None and index == self.fail_on_batch:
|
||||||
|
raise RuntimeError("simulated point storage failure")
|
||||||
|
for point in points:
|
||||||
|
self.points[str(point.point_id)] = point
|
||||||
|
finally:
|
||||||
|
self._in_flight -= 1
|
||||||
|
|
||||||
|
async def deactivate_points_from_index(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
from_chunk_index: int,
|
||||||
|
deleted_at: datetime,
|
||||||
|
updated_by: str,
|
||||||
|
) -> int:
|
||||||
|
self.deactivate_calls.append(
|
||||||
|
{
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"file_id": file_id,
|
||||||
|
"from_chunk_index": from_chunk_index,
|
||||||
|
"updated_by": updated_by,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if self.fail_deactivate:
|
||||||
|
raise RuntimeError("simulated deactivate failure")
|
||||||
|
|
||||||
|
def is_stale(point: ChunkPoint) -> bool:
|
||||||
|
chunk_index = point.payload.get("chunk_index")
|
||||||
|
return (
|
||||||
|
point.payload.get("file_id") == str(file_id)
|
||||||
|
and point.payload.get("tenant_id") == str(tenant_id)
|
||||||
|
and point.payload.get("is_active") is True
|
||||||
|
and isinstance(chunk_index, int)
|
||||||
|
and chunk_index >= from_chunk_index
|
||||||
|
)
|
||||||
|
|
||||||
|
stale = [point for point in self.points.values() if is_stale(point)]
|
||||||
|
for point in stale:
|
||||||
|
point.payload["is_active"] = False
|
||||||
|
point.payload["deleted_at"] = deleted_at.isoformat()
|
||||||
|
return len(stale)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FakePointRepository:
|
||||||
|
"""An in-memory `PointRepository`.
|
||||||
|
|
||||||
|
Stores `Point` models keyed by point id. The filter, ordering, and version
|
||||||
|
semantics below are reimplemented in Python rather than stubbed, because
|
||||||
|
every service this fake stands behind is being tested *for* those semantics:
|
||||||
|
a stub that returned points in insertion order would pass a reorder test
|
||||||
|
that real Qdrant fails.
|
||||||
|
|
||||||
|
Three behaviours are modelled deliberately:
|
||||||
|
|
||||||
|
- `is_active` is filtered out unless `include_inactive`, matching ADR-0002's
|
||||||
|
implied read filter.
|
||||||
|
- `list_by_file` sorts by `order_id` and paginates by value, the same cursor
|
||||||
|
mechanism the Qdrant adapter uses, so a test can catch a cursor that skips
|
||||||
|
or repeats.
|
||||||
|
- `apply_patches` honours `expected_version` by silently matching nothing on
|
||||||
|
a mismatch — the no-op-not-error behaviour Qdrant's filtered `set_payload`
|
||||||
|
actually has, which is what makes the service's post-check necessary.
|
||||||
|
"""
|
||||||
|
|
||||||
|
points: dict[str, Point] = field(default_factory=dict)
|
||||||
|
patch_batches: list[list[PayloadPatch]] = field(default_factory=list)
|
||||||
|
fail_next: bool = False
|
||||||
|
|
||||||
|
def add(self, point: Point) -> None:
|
||||||
|
"""Seed a point. Test-only helper, not part of the port."""
|
||||||
|
self.points[str(point.point_id)] = point
|
||||||
|
|
||||||
|
def _visible(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> list[Point]:
|
||||||
|
return [
|
||||||
|
point
|
||||||
|
for point in self.points.values()
|
||||||
|
if point.tenant_id == tenant_id
|
||||||
|
and (domain is None or point.domain == domain)
|
||||||
|
and (file_id is None or point.file_id == file_id)
|
||||||
|
and (include_inactive or point.is_active)
|
||||||
|
]
|
||||||
|
|
||||||
|
async def get(
|
||||||
|
self, *, tenant_id: uuid.UUID, point_id: uuid.UUID, with_vectors: bool = False
|
||||||
|
) -> Point | None:
|
||||||
|
point = self.points.get(str(point_id))
|
||||||
|
if point is None or point.tenant_id != tenant_id:
|
||||||
|
return None
|
||||||
|
return point if with_vectors else point.model_copy(update={"vectors": None})
|
||||||
|
|
||||||
|
async def get_many(
|
||||||
|
self, *, tenant_id: uuid.UUID, point_ids: Sequence[uuid.UUID]
|
||||||
|
) -> tuple[Point, ...]:
|
||||||
|
wanted = {str(point_id) for point_id in point_ids}
|
||||||
|
return tuple(
|
||||||
|
point
|
||||||
|
for key, point in self.points.items()
|
||||||
|
if key in wanted and point.tenant_id == tenant_id
|
||||||
|
)
|
||||||
|
|
||||||
|
async def list_by_file(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
file_id: uuid.UUID,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointPage:
|
||||||
|
ordered = sorted(
|
||||||
|
self._visible(tenant_id=tenant_id, file_id=file_id, include_inactive=include_inactive),
|
||||||
|
key=lambda point: point.order_id,
|
||||||
|
)
|
||||||
|
if cursor is not None:
|
||||||
|
ordered = [point for point in ordered if point.order_id > float(cursor)]
|
||||||
|
|
||||||
|
page = tuple(ordered[:limit])
|
||||||
|
next_cursor = repr(page[-1].order_id) if len(page) == limit else None
|
||||||
|
return PointPage(points=page, next_cursor=next_cursor)
|
||||||
|
|
||||||
|
async def count(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> int:
|
||||||
|
return len(
|
||||||
|
self._visible(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def keyword_search(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
tenant_id: uuid.UUID,
|
||||||
|
query: str,
|
||||||
|
limit: int,
|
||||||
|
cursor: str | None = None,
|
||||||
|
domain: str | None = None,
|
||||||
|
file_id: uuid.UUID | None = None,
|
||||||
|
include_inactive: bool = False,
|
||||||
|
) -> PointPage:
|
||||||
|
# Whole-token containment, lowercased: an approximation of Qdrant's
|
||||||
|
# full-text index good enough to tell "matched" from "did not", which is
|
||||||
|
# all the service layer branches on. Ranking is not modelled because the
|
||||||
|
# real index does not rank either (ADR-0002).
|
||||||
|
terms = query.lower().split()
|
||||||
|
matches = [
|
||||||
|
point
|
||||||
|
for point in self._visible(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
domain=domain,
|
||||||
|
file_id=file_id,
|
||||||
|
include_inactive=include_inactive,
|
||||||
|
)
|
||||||
|
if all(term in point.content.lower().split() for term in terms)
|
||||||
|
]
|
||||||
|
start = int(cursor) if cursor is not None else 0
|
||||||
|
page = tuple(matches[start : start + limit])
|
||||||
|
next_start = start + limit
|
||||||
|
return PointPage(
|
||||||
|
points=page, next_cursor=str(next_start) if next_start < len(matches) else None
|
||||||
|
)
|
||||||
|
|
||||||
|
async def apply_patches(self, *, tenant_id: uuid.UUID, patches: Sequence[PayloadPatch]) -> None:
|
||||||
|
self.patch_batches.append(list(patches))
|
||||||
|
if self.fail_next:
|
||||||
|
self.fail_next = False
|
||||||
|
raise RuntimeError("simulated point repository failure")
|
||||||
|
|
||||||
|
for patch in patches:
|
||||||
|
point = self.points.get(str(patch.point_id))
|
||||||
|
if point is None or point.tenant_id != tenant_id:
|
||||||
|
continue
|
||||||
|
if patch.expected_version is not None and point.version != patch.expected_version:
|
||||||
|
continue
|
||||||
|
# Re-validated rather than `model_copy`d, because a patch payload
|
||||||
|
# carries wire values (an ISO `deleted_at`, a stringified
|
||||||
|
# `previous_chunk_id`) exactly as it would reach Qdrant. Copying
|
||||||
|
# without validation would leave the fake holding a `str` where a
|
||||||
|
# read from real Qdrant returns a `datetime`, and a service bug that
|
||||||
|
# depends on the difference would pass here and fail in production.
|
||||||
|
updated = point.model_dump()
|
||||||
|
updated.update(patch.payload)
|
||||||
|
self.points[str(patch.point_id)] = Point.model_validate(updated)
|
||||||
|
|||||||
39
tests/integration/minio/test_storage.py
Normal file
39
tests/integration/minio/test_storage.py
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
import uuid
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from anyio import CapacityLimiter
|
||||||
|
|
||||||
|
from src.application.files.storage_keys import source_file_object_key
|
||||||
|
from src.config import MinioSettings
|
||||||
|
from src.infrastructure.minio.client import create_client
|
||||||
|
from src.infrastructure.minio.storage import MinioObjectStorage
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.integration, pytest.mark.minio, pytest.mark.asyncio]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_put_object_stores_bytes_under_server_derived_key(
|
||||||
|
minio_settings: MinioSettings,
|
||||||
|
) -> None:
|
||||||
|
client = create_client(minio_settings)
|
||||||
|
storage = MinioObjectStorage(client, bucket=minio_settings.bucket, limiter=CapacityLimiter(2))
|
||||||
|
key = source_file_object_key(uuid.uuid4(), uuid.uuid4())
|
||||||
|
data = b"tenant-scoped, id-addressed bytes"
|
||||||
|
|
||||||
|
await storage.put_object(key=key, data=data, content_type="text/csv")
|
||||||
|
|
||||||
|
stored = client.get_object(minio_settings.bucket, key).read()
|
||||||
|
assert stored == data
|
||||||
|
|
||||||
|
|
||||||
|
async def test_put_object_overwrites_existing_object_at_same_key(
|
||||||
|
minio_settings: MinioSettings,
|
||||||
|
) -> None:
|
||||||
|
client = create_client(minio_settings)
|
||||||
|
storage = MinioObjectStorage(client, bucket=minio_settings.bucket, limiter=CapacityLimiter(2))
|
||||||
|
key = source_file_object_key(uuid.uuid4(), uuid.uuid4())
|
||||||
|
|
||||||
|
await storage.put_object(key=key, data=b"first", content_type="text/csv")
|
||||||
|
await storage.put_object(key=key, data=b"second", content_type="text/csv")
|
||||||
|
|
||||||
|
stored = client.get_object(minio_settings.bucket, key).read()
|
||||||
|
assert stored == b"second"
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
from collections.abc import AsyncIterator, Iterator
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
import pytest_asyncio
|
|
||||||
from alembic.command import upgrade
|
|
||||||
from alembic.config import Config
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
|
||||||
from testcontainers.community.postgres import PostgresContainer
|
|
||||||
|
|
||||||
from src.config import PostgresSettings
|
|
||||||
from src.infrastructure.postgres.database import create_engine
|
|
||||||
|
|
||||||
|
|
||||||
def _alembic_config(database_url: str) -> Config:
|
|
||||||
config = Config("alembic.ini")
|
|
||||||
config.set_main_option("sqlalchemy.url", database_url)
|
|
||||||
return config
|
|
||||||
|
|
||||||
|
|
||||||
def _settings_from_url(url: str) -> PostgresSettings:
|
|
||||||
# testcontainers returns postgresql+asyncpg://user:pass@host:port/db ;
|
|
||||||
# PostgresSettings builds its own dsn from parts, so parse the parts back out.
|
|
||||||
without_scheme = url.split("://", 1)[1]
|
|
||||||
creds, hostpart = without_scheme.split("@", 1)
|
|
||||||
user, password = creds.split(":", 1)
|
|
||||||
hostport, db = hostpart.split("/", 1)
|
|
||||||
host, port = hostport.split(":", 1)
|
|
||||||
return PostgresSettings(host=host, port=int(port), user=user, password=password, db=db)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
|
||||||
def postgres_container() -> Iterator[PostgresContainer]:
|
|
||||||
with PostgresContainer("postgres:17", driver="asyncpg") as container:
|
|
||||||
yield container
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
|
||||||
def postgres_url(postgres_container: PostgresContainer) -> str:
|
|
||||||
return postgres_container.get_connection_url()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
|
||||||
def migrated_postgres_url(postgres_url: str) -> str:
|
|
||||||
"""The container's URL, after Alembic has created the schema on it once."""
|
|
||||||
upgrade(_alembic_config(postgres_url), "head")
|
|
||||||
return postgres_url
|
|
||||||
|
|
||||||
|
|
||||||
@pytest_asyncio.fixture(scope="session")
|
|
||||||
async def postgres_engine(migrated_postgres_url: str) -> AsyncIterator[AsyncEngine]:
|
|
||||||
engine = create_engine(_settings_from_url(migrated_postgres_url))
|
|
||||||
try:
|
|
||||||
yield engine
|
|
||||||
finally:
|
|
||||||
await engine.dispose()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest_asyncio.fixture
|
|
||||||
async def db_session(postgres_engine: AsyncEngine) -> AsyncIterator[AsyncSession]:
|
|
||||||
"""One session per test, bound to a rolled-back outer transaction.
|
|
||||||
|
|
||||||
Isolates each test's writes (ADR-0016: isolate data per test) without
|
|
||||||
needing a fresh container or unique keys per test.
|
|
||||||
"""
|
|
||||||
async with postgres_engine.connect() as connection:
|
|
||||||
outer_transaction = await connection.begin()
|
|
||||||
sessionmaker = async_sessionmaker(
|
|
||||||
bind=connection, expire_on_commit=False, join_transaction_mode="create_savepoint"
|
|
||||||
)
|
|
||||||
async with sessionmaker() as session:
|
|
||||||
yield session
|
|
||||||
await outer_transaction.rollback()
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user