Ali Zarinkolah
5cdfb70085
Adds src/application/ingestion/ -- Persian normalization, DOCX body walk with structural data/layout table classification, CSV/XLSX row rendering, and fixed-size token chunking (cl100k_base, 400/60/512) -- as pure functions per ADR-0015, tested against real production documents (asia_data_sample, kept out of the repo). ADR-0018 records where this diverges from ADR-0004 (fixed-size default, no invented headings/tree, structural table classification, header-provable labeling only). Plan 001's scope line is corrected from CSV-only to DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is updated to match current implementation state.
61 lines
1.8 KiB
Plaintext
61 lines
1.8 KiB
Plaintext
# Application local development environment.
|
|
#
|
|
# Usage:
|
|
# cp .env.example .env
|
|
# docker compose -f docker-compose.yml up -d
|
|
#
|
|
# All values below are non-secret local-development defaults matching
|
|
# docker-compose.yml. Do not commit .env.
|
|
|
|
# Application
|
|
APP_ENV=local
|
|
APP_MAX_UPLOAD_SIZE_MB=25
|
|
APP_READINESS_CHECK_TIMEOUT_SECONDS=2.0
|
|
|
|
# Logging
|
|
LOG_LEVEL=INFO
|
|
LOG_JSON_FORMAT=false
|
|
|
|
# Postgres (application database, separate from Langfuse's Postgres)
|
|
# Use 127.0.0.1 rather than localhost: some environments resolve localhost to
|
|
# the IPv6 loopback first, and Docker only publishes these ports on IPv4.
|
|
POSTGRES_HOST=127.0.0.1
|
|
POSTGRES_PORT=5433
|
|
POSTGRES_USER=chatbot
|
|
POSTGRES_PASSWORD=chatbot
|
|
POSTGRES_DB=chatbot
|
|
|
|
# MinIO (application bucket, separate from Langfuse's MinIO)
|
|
MINIO_ENDPOINT=127.0.0.1:9100
|
|
MINIO_ACCESS_KEY=chatbot
|
|
MINIO_SECRET_KEY=chatbot-secret
|
|
MINIO_SECURE=false
|
|
MINIO_BUCKET=chatbot-source-files
|
|
|
|
# Inline ingestion bounds (ADR-0017; no broker, no queue).
|
|
# INGESTION_TIMEOUT_SECONDS must stay below any proxy/client read timeout.
|
|
INGESTION_MAX_CONCURRENCY=4
|
|
INGESTION_THREAD_POOL_SIZE=8
|
|
INGESTION_TIMEOUT_SECONDS=120.0
|
|
INGESTION_MAX_CHUNKS_PER_FILE=5000
|
|
INGESTION_EMBED_BATCH_SIZE=128
|
|
INGESTION_EMBED_CONCURRENCY=4
|
|
|
|
# Qdrant
|
|
QDRANT_URL=http://127.0.0.1:6343
|
|
QDRANT_API_KEY=
|
|
|
|
# Parsing and chunking (ADR-0018).
|
|
# max_chunk_tokens is nomic-embed-text-v2-moe's sequence length; text past it
|
|
# is silently truncated by the model, so the cap is enforced before embedding.
|
|
# chunk_size sits under it to leave room for the `search_document: ` prefix.
|
|
CHUNKING_STRATEGY=fixed_size
|
|
CHUNKING_CHUNK_SIZE=400
|
|
CHUNKING_CHUNK_OVERLAP=60
|
|
CHUNKING_MAX_CHUNK_TOKENS=512
|
|
CHUNKING_ENCODING_NAME=cl100k_base
|
|
|
|
# tiktoken downloads its vocabulary on first use; point this at a
|
|
# pre-populated directory for offline/air-gapped deployments.
|
|
# TIKTOKEN_CACHE_DIR=
|