5e0addcc55
Why: - The chunks collection needs four named vectors (dense_nomic, dense_openai, sparse, late_interaction) and payload indexes defined at creation time per ADR-0001; sparse/multivector fields cannot be added to an existing collection without recreating it, so schema drift here is expensive. - Creating it at FastAPI startup would mirror the DDL-at-boot anti-pattern ADR-0009 already rejects for Postgres and ADR-0012 rejects for LangGraph's setup(), so it is a deployment step instead. Changes: - src/infrastructure/qdrant/collection.py: ensure_chunks_collection(), idempotent and schema-verifying (raises on dimension/modifier mismatch rather than silently accepting a misconfigured collection). - src/cli/qdrant_bootstrap.py: the operator entry point (python -m src.cli.qdrant_bootstrap). - QdrantSettings gains collection/upsert_batch_size/upsert_concurrency. Impact: - Deployments must run the new bootstrap command before the first upload; see ADR-0001's new "Collection provisioning" section.
103 lines
3.8 KiB
Plaintext
103 lines
3.8 KiB
Plaintext
# Application local development environment.
|
|
#
|
|
# Usage:
|
|
# cp .env.example .env
|
|
# docker compose -f docker-compose.yml up -d
|
|
#
|
|
# All values below are non-secret local-development defaults matching
|
|
# docker-compose.yml. Do not commit .env.
|
|
|
|
# Application
|
|
APP_ENV=local
|
|
APP_READINESS_CHECK_TIMEOUT_SECONDS=2.0
|
|
|
|
# Logging
|
|
LOG_LEVEL=INFO
|
|
LOG_JSON_FORMAT=false
|
|
|
|
# Postgres (application database, separate from Langfuse's Postgres)
|
|
# Use 127.0.0.1 rather than localhost: some environments resolve localhost to
|
|
# the IPv6 loopback first, and Docker only publishes these ports on IPv4.
|
|
POSTGRES_HOST=127.0.0.1
|
|
POSTGRES_PORT=5433
|
|
POSTGRES_USER=chatbot
|
|
POSTGRES_PASSWORD=chatbot
|
|
POSTGRES_DB=chatbot
|
|
|
|
# MinIO (application bucket, separate from Langfuse's MinIO)
|
|
MINIO_ENDPOINT=127.0.0.1:9100
|
|
MINIO_ACCESS_KEY=chatbot
|
|
MINIO_SECRET_KEY=chatbot-secret
|
|
MINIO_SECURE=false
|
|
MINIO_BUCKET=chatbot-source-files
|
|
|
|
# Inline ingestion bounds (ADR-0017; no broker, no queue).
|
|
# INGESTION_TIMEOUT_SECONDS must stay below any proxy/client read timeout.
|
|
INGESTION_MAX_CONCURRENCY=4
|
|
INGESTION_THREAD_POOL_SIZE=8
|
|
INGESTION_TIMEOUT_SECONDS=120.0
|
|
INGESTION_MAX_UPLOAD_SIZE_MB=25
|
|
INGESTION_MAX_CHUNKS_PER_FILE=5000
|
|
INGESTION_EMBED_BATCH_SIZE=128
|
|
INGESTION_EMBED_CONCURRENCY=4
|
|
|
|
# Qdrant
|
|
QDRANT_URL=http://127.0.0.1:6343
|
|
QDRANT_API_KEY=
|
|
QDRANT_COLLECTION=chunks
|
|
QDRANT_UPSERT_BATCH_SIZE=128
|
|
QDRANT_UPSERT_CONCURRENCY=4
|
|
|
|
# Dense embedders (ADR-0001). Both speak an OpenAI-compatible /embeddings
|
|
# endpoint, so one adapter serves both. Models and endpoints are the ones the
|
|
# `emet` evaluation lab benchmarked as winners on the Farsi corpus.
|
|
#
|
|
# dense_nomic runs behind Ollama's OpenAI-compat shim, which accepts any
|
|
# non-empty API key. KEEP_ALIVE holds the model resident: a cold load of
|
|
# nomic-embed-text-v2-moe takes >150s, well past INGESTION_TIMEOUT_SECONDS,
|
|
# so an idle-then-upload would otherwise 504.
|
|
EMBEDDING_NOMIC_BASE_URL=http://192.168.10.10:11435/v1
|
|
EMBEDDING_NOMIC_MODEL=nomic-embed-text-v2-moe
|
|
EMBEDDING_NOMIC_API_KEY=sk-not-set
|
|
EMBEDDING_NOMIC_KEEP_ALIVE=30m
|
|
EMBEDDING_NOMIC_TIMEOUT_SECONDS=30.0
|
|
# Empty = emet parity. The model card specifies `search_document: ` (ADR-0004),
|
|
# but the benchmark ran without it and the prefix shifts the vector a lot
|
|
# (cosine 0.57 on identical text) — so if you set this, the query side must
|
|
# send `search_query: ` to match, or retrieval gets worse rather than better.
|
|
EMBEDDING_NOMIC_DOCUMENT_PREFIX=
|
|
|
|
# Leave DIMENSIONS empty for text-embedding-3-large's native 3072, which is
|
|
# what was benchmarked. Setting it truncates via Matryoshka and is a
|
|
# re-embedding migration, not a config tweak.
|
|
EMBEDDING_OPENAI_BASE_URL=https://api.openai.com/v1
|
|
EMBEDDING_OPENAI_MODEL=text-embedding-3-large
|
|
EMBEDDING_OPENAI_API_KEY=
|
|
EMBEDDING_OPENAI_DIMENSIONS=
|
|
EMBEDDING_OPENAI_DOCUMENT_PREFIX=
|
|
EMBEDDING_OPENAI_TIMEOUT_SECONDS=30.0
|
|
|
|
# Sparse BM25 (ADR-0001, ADR-0005): the benchmarked `bm25-fa-norm-stop`.
|
|
# k/b saturation is applied client-side; IDF comes from Qdrant's
|
|
# modifier="idf" on the sparse vector field. AVG_LEN is the average document
|
|
# length in analyzer tokens — emet's placeholder, worth recalibrating from
|
|
# real corpus statistics.
|
|
EMBEDDING_SPARSE_ANALYZER=fa_norm_stop
|
|
EMBEDDING_SPARSE_K=1.2
|
|
EMBEDDING_SPARSE_B=0.75
|
|
EMBEDDING_SPARSE_AVG_LEN=256.0
|
|
|
|
# Parsing and chunking (ADR-0018).
|
|
# max_chunk_tokens is nomic-embed-text-v2-moe's sequence length; text past it
|
|
# is silently truncated by the model, so the cap is enforced before embedding.
|
|
# chunk_size sits under it to leave room for the `search_document: ` prefix.
|
|
CHUNKING_STRATEGY=fixed_size
|
|
CHUNKING_CHUNK_SIZE=400
|
|
CHUNKING_CHUNK_OVERLAP=60
|
|
CHUNKING_MAX_CHUNK_TOKENS=512
|
|
CHUNKING_ENCODING_NAME=cl100k_base
|
|
|
|
# tiktoken downloads its vocabulary on first use; point this at a
|
|
# pre-populated directory for offline/air-gapped deployments.
|
|
# TIKTOKEN_CACHE_DIR=
|