feat(ingestion): add DOCX/CSV/XLSX parsing and fixed-size chunking (ADR-0018)
Adds src/application/ingestion/ -- Persian normalization, DOCX body walk with structural data/layout table classification, CSV/XLSX row rendering, and fixed-size token chunking (cl100k_base, 400/60/512) -- as pure functions per ADR-0015, tested against real production documents (asia_data_sample, kept out of the repo). ADR-0018 records where this diverges from ADR-0004 (fixed-size default, no invented headings/tree, structural table classification, header-provable labeling only). Plan 001's scope line is corrected from CSV-only to DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is updated to match current implementation state.
This commit is contained in:
@@ -2,8 +2,10 @@ from collections.abc import AsyncIterator, Callable
|
||||
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
||||
|
||||
import structlog
|
||||
from anyio import to_thread
|
||||
from fastapi import FastAPI
|
||||
|
||||
from src.application.ingestion import get_encoder
|
||||
from src.bootstrap.dependencies import AppResources
|
||||
from src.config import Settings
|
||||
from src.infrastructure.minio.client import create_client as create_minio_client
|
||||
@@ -22,6 +24,15 @@ def create_lifespan(
|
||||
resolved_settings = settings or Settings()
|
||||
configure_logging(resolved_settings.logging)
|
||||
|
||||
# tiktoken fetches its vocabulary over the network on first use, so warm
|
||||
# it here: a missing vocabulary should fail the process at boot, not the
|
||||
# first upload. Blocking, hence the thread.
|
||||
await to_thread.run_sync(get_encoder, resolved_settings.chunking.encoding_name)
|
||||
logger.info(
|
||||
"lifespan.tokenizer.loaded",
|
||||
encoding=resolved_settings.chunking.encoding_name,
|
||||
)
|
||||
|
||||
db_engine = create_engine(resolved_settings.postgres)
|
||||
db_sessionmaker = create_sessionmaker(db_engine)
|
||||
logger.info("lifespan.postgres.engine.created")
|
||||
|
||||
Reference in New Issue
Block a user