From 5cdfb7008567a077580d5a2f6e8b957cc52f5d6d Mon Sep 17 00:00:00 2001 From: Ali Zarinkolah Date: Tue, 18 Aug 2026 10:22:17 +0330 Subject: [PATCH] feat(ingestion): add DOCX/CSV/XLSX parsing and fixed-size chunking (ADR-0018) Adds src/application/ingestion/ -- Persian normalization, DOCX body walk with structural data/layout table classification, CSV/XLSX row rendering, and fixed-size token chunking (cl100k_base, 400/60/512) -- as pure functions per ADR-0015, tested against real production documents (asia_data_sample, kept out of the repo). ADR-0018 records where this diverges from ADR-0004 (fixed-size default, no invented headings/tree, structural table classification, header-provable labeling only). Plan 001's scope line is corrected from CSV-only to DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is updated to match current implementation state. --- .env.example | 14 + CLAUDE.md | 19 +- docs/adr/0004-docx-csv-chunking-strategy.md | 17 + ...dsheet-parsing-with-fixed-size-chunking.md | 306 ++++++++++++++++++ docs/backlog.md | 109 +++++++ docs/plans/001-ingestion-vertical-slice.md | 31 +- pyproject.toml | 3 + src/application/__init__.py | 0 src/application/ingestion/__init__.py | 47 +++ src/application/ingestion/chunking.py | 129 ++++++++ src/application/ingestion/docx_parser.py | 165 ++++++++++ src/application/ingestion/errors.py | 41 +++ src/application/ingestion/models.py | 66 ++++ src/application/ingestion/normalization.py | 73 +++++ .../ingestion/spreadsheet_parser.py | 121 +++++++ src/application/ingestion/tabular.py | 167 ++++++++++ src/application/ingestion/tokenizer.py | 33 ++ src/bootstrap/lifespan.py | 11 + src/config.py | 35 +- tests/support/documents.json | 30 ++ tests/support/documents.py | 52 +++ tests/unit/application/test_chunking.py | 270 ++++++++++++++++ tests/unit/application/test_docx_parser.py | 225 +++++++++++++ tests/unit/application/test_normalization.py | 95 ++++++ .../application/test_spreadsheet_parser.py | 183 +++++++++++ uv.lock | 214 ++++++++++++ 26 files changed, 2438 insertions(+), 18 deletions(-) create mode 100644 docs/adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md create mode 100644 docs/backlog.md create mode 100644 src/application/__init__.py create mode 100644 src/application/ingestion/__init__.py create mode 100644 src/application/ingestion/chunking.py create mode 100644 src/application/ingestion/docx_parser.py create mode 100644 src/application/ingestion/errors.py create mode 100644 src/application/ingestion/models.py create mode 100644 src/application/ingestion/normalization.py create mode 100644 src/application/ingestion/spreadsheet_parser.py create mode 100644 src/application/ingestion/tabular.py create mode 100644 src/application/ingestion/tokenizer.py create mode 100644 tests/support/documents.json create mode 100644 tests/support/documents.py create mode 100644 tests/unit/application/test_chunking.py create mode 100644 tests/unit/application/test_docx_parser.py create mode 100644 tests/unit/application/test_normalization.py create mode 100644 tests/unit/application/test_spreadsheet_parser.py diff --git a/.env.example b/.env.example index 71d8677..d11cbe3 100644 --- a/.env.example +++ b/.env.example @@ -44,3 +44,17 @@ INGESTION_EMBED_CONCURRENCY=4 # Qdrant QDRANT_URL=http://127.0.0.1:6343 QDRANT_API_KEY= + +# Parsing and chunking (ADR-0018). +# max_chunk_tokens is nomic-embed-text-v2-moe's sequence length; text past it +# is silently truncated by the model, so the cap is enforced before embedding. +# chunk_size sits under it to leave room for the `search_document: ` prefix. +CHUNKING_STRATEGY=fixed_size +CHUNKING_CHUNK_SIZE=400 +CHUNKING_CHUNK_OVERLAP=60 +CHUNKING_MAX_CHUNK_TOKENS=512 +CHUNKING_ENCODING_NAME=cl100k_base + +# tiktoken downloads its vocabulary on first use; point this at a +# pre-populated directory for offline/air-gapped deployments. +# TIKTOKEN_CACHE_DIR= diff --git a/CLAUDE.md b/CLAUDE.md index 84b9e07..6326a2d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,12 +4,19 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project status -This repo is currently ADR-driven and mostly pre-implementation: `src/` contains -only an empty `main.py`/`config.py` scaffold and empty `api/routers`, -`api/dependencies`, `db`, and `schemas` directories. Architecture decisions live -in `docs/adr/` (17 ADRs plus the 0000 template; 0001–0004 are `Accepted`, -0014 is `Superseded by 0017`, and the rest — 0005–0013 and 0015–0017 — are -`Proposed`). Implementation plans live in `docs/plans/`: +This repo is ADR-driven and early in implementation. Working today: the FastAPI +app factory and lifespan wiring (`src/bootstrap/`), `/healthz` and `/readyz`, +structlog config, Postgres/MinIO/Qdrant clients (`src/infrastructure/`), five +SQLAlchemy models with one Alembic migration, and document parsing plus +fixed-size chunking (`src/application/ingestion/`). Not built yet: API-key auth, +`POST /v1/files` (so `/v1` currently exposes no routes), repositories, embedding +adapters, Qdrant collection bootstrap, and `src/agent/`. That maps to plan 001 +Phase 1 done, Phase 2 partly done, and the parsing half of Phase 5. + +Architecture decisions live in `docs/adr/` (18 ADRs plus the 0000 template; +0001–0004 are `Accepted` — 0004 amended by 0018; 0014 is `Superseded by 0017`; +the rest — 0005–0013 and 0015–0018 — are `Proposed`). Implementation plans live +in `docs/plans/`: `001-ingestion-vertical-slice.md` and `002-point-crud-and-keyword-search.md`. **Read the relevant ADR(s) before implementing anything** — the ADRs are the source of truth for diff --git a/docs/adr/0004-docx-csv-chunking-strategy.md b/docs/adr/0004-docx-csv-chunking-strategy.md index 79daa22..273201f 100644 --- a/docs/adr/0004-docx-csv-chunking-strategy.md +++ b/docs/adr/0004-docx-csv-chunking-strategy.md @@ -4,6 +4,23 @@ Accepted +> Amended by +> [ADR-0018](0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md): +> v1 ships **fixed-size** chunking rather than the semantic-aware default +> below, and defers `qa_pair` detection and image captioning. Docx tables do +> become `table_row` units as this ADR specifies, but only when they are data: +> a table holding a cell larger than one chunk, or a cell containing nested +> tables, is treated as page layout and its cells are chunked as prose. +> Row labels are applied only when row 0 is provably a header, and cells are +> joined unlabeled otherwise. No document tree is built, and no heading is +> inferred from text. `.doc` is rejected with `415` pending an out-of-process +> conversion service rather than shelling out to LibreOffice. ADR-0018 also +> adds a Persian normalization step at parse time and fixes the chunk-size +> numbers this ADR left open. The spreadsheet row-to-chunk rules, the +> embedding model, the task-prefix invariant, and the `content_type` value set +> below all apply unchanged — except that `.csv` is read with the standard +> library `csv` module rather than `pandas`. + ## Context ADR-0001 deferred two things to "when we start the docx/csv chunking work": diff --git a/docs/adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md b/docs/adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md new file mode 100644 index 0000000..3f7f0d7 --- /dev/null +++ b/docs/adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md @@ -0,0 +1,306 @@ +# 0018. DOCX and spreadsheet parsing with fixed-size chunking + +## Status + +Proposed + +## Context + +ADR-0004 specified the full parsing and chunking design: structural extraction +before chunking (table rows and Q&A pairs kept atomic, prose chunked +separately), semantic-aware chunking as the default, image captioning via a +vision API, and legacy `.doc` conversion through headless LibreOffice. Plan +001's first vertical slice needs a working parser now, and that full design is +substantially more work than the slice can absorb. This ADR records what v1 +actually ships and why it differs, so the code does not silently contradict an +Accepted ADR. + +Four forces shaped the decision: + +**A working extractor already exists.** The `chunking_strategies_evaluation` +repository — the harness the user built to compare chunking strategies against +this same Farsi corpus — contains a DOCX extractor that walks the document body +in reading order and handles the "prose lives inside table cells" pattern +common in Farsi documents exported from older Word versions. Roughly 200 lines +of it are production-quality; the rest is evaluation scaffolding (five +competing strategies, an LLM-as-judge benchmark, a dashboard, an HTML report +generator). Porting it is cheaper and better-tested against real documents than +writing a parser from scratch. + +**Semantic chunking does not fit the inline request.** ADR-0004 chose +semantic-aware chunking as the default on the strength of the user's own +offline accuracy comparison, in which fixed-size was the "viable, simpler +runner-up". That comparison measured retrieval accuracy, not ingestion cost. +Semantic boundary detection requires embedding every sentence *before* chunk +boundaries can be decided — a second network round-trip pass inside the request +budget ADR-0017 bounds with `INGESTION_TIMEOUT_SECONDS`. ADR-0017's own cost +table already assumes the cheaper strategy, listing "Chunk (fixed-size, +ADR-0004) | Blocking CPU, pure Python | Negligible". ADR-0004 and ADR-0017 are +therefore already in tension, and this ADR resolves it toward ADR-0017 for v1. +Fixed-size is acceptable specifically because ADR-0001 gives every chunk +`previous_chunk_id`/`next_chunk_id` pointers: a chunk boundary that cuts a +thought in half is recoverable by expanding to neighbors at retrieval time +(ADR-0003). + +**Nothing normalizes the text the dense embedders see.** ADR-0005 resolved the +sparse analyzer as `bm25-fa-norm-stop` — "normalization + stopword removal", +computed in our own BM25 pipeline outside Qdrant. That covers the sparse vector +only. Persian text authored on mixed Arabic/Persian keyboards contains both +`ک` (U+06A9) and `ك` (U+0643), both `ی` (U+06CC) and `ي` (U+064A); these are +distinct codepoints and therefore distinct tokens to `nomic-embed-text-v2-moe` +and `text-embedding-3-large` alike, so the same Persian word can embed two +different ways depending on which key the author pressed. Word documents in +this corpus reliably contain both forms. + +**The corpus is spreadsheets more than it is CSVs.** ADR-0004 inspected the +real sample and found the tabular files are entirely `.xlsx`; no `.csv` exists +in practice. Plan 001 and the `source_files.source_type` CHECK constraint both +name `csv`. The row-to-chunk mapping is identical either way — only the reader +differs — so v1 reads both rather than forcing a manual export step that would +silently drop the merged-cell values ADR-0004 warns about. + +## Decision + +### 1. Formats + +v1 ingests `.docx`, `.csv`, and `.xlsx`. + +`.doc` is rejected with `415 Unsupported Media Type`. ADR-0004 specified +conversion via `soffice --headless --convert-to docx`; that is a subprocess +with a multi-second startup cost running inside the inline request ADR-0017 +defines, and it adds a system binary to the container image. Conversion is +deferred to an out-of-process HTTP conversion service, tracked in the backlog. +`source_files.source_type` continues to allow `doc` so the row can be recorded +once conversion lands. + +### 2. Structural units before chunking + +Every source file is decomposed into ordered **structural units** before any +chunking runs, as ADR-0004 requires. There are two kinds: + +- `PARAGRAPH` — a run of flowing prose. Consecutive paragraphs accumulate into + one unit rather than one unit each, so the splitter sees flowing text instead + of a series of one-sentence fragments ("the whole remaining run of + paragraphs", per ADR-0004). +- `TABLE_ROW` — one row of a data table. Atomic; split only when a single row + exceeds the model's sequence length. + +Walk `doc.element.body` children in document order — not `doc.paragraphs` and +`doc.tables` separately — so tables interleaved with paragraphs keep their +position. This is ADR-0004's rule, unchanged. + +**No headings are invented.** A real `Heading N` Word style becomes a `#` +prefix on its paragraph; where a document declares none, none appear. The +evaluation repository upgrades paragraphs to headings by text pattern (`^بخش`, +`^\d+[-.]\d+`, leading `*`); those patterns are tuned to a Farsi regulatory +corpus, not an insurance one, and a wrongly-detected heading silently reshapes +the document in a way that is hard to notice downstream. They are not adopted. + +There is also **no document tree**. An earlier draft built a `Document > +Section > Article > Paragraph` hierarchy from heading styles. Not one document +in the sample corpus carries a single `Heading` style, so that tree was flat in +every real case, and nothing consumed it — chunking works from the unit list, +and ADR-0001's payload has no tree field. It is not built. + +### 3. Data tables against layout tables + +A docx table is either data or page furniture, and the two need opposite +treatment. The classification is **structural, never a reading of content**: a +data cell is by definition small enough to be a chunk, so a table containing a +cell that alone exceeds `chunk_size`, or a cell containing nested tables, is a +layout container. In the sample corpus this separates by two orders of +magnitude — 24 to 171 tokens for the largest cell of each data table, against +54,007 tokens for a cell holding an entire sub-document across 738 paragraphs +and 5 nested tables. + +- **Data table** → one `TABLE_ROW` unit per row, rendered by the same code that + renders spreadsheet rows. +- **Layout table** → its cells are prose, recursed into and folded into the + surrounding prose block. + +### 4. Table rows are labeled only when a header is provable + +Applied to docx tables and spreadsheets alike: + +- A header is **row 0 or nothing**. Never scan further down for a + header-shaped row. Scanning discarded every row above the match and then + labeled the rest from a data row, turning a 30-row compensation table into + chunks reading `80: 70`. +- Leading rows that are structurally a merged banner — fewer than two populated + cells, or one value repeated across the row — are skipped first. That is a + fact about the merge, not a guess about meaning. +- Row 0 is accepted as a header only when it is **inconsistent with the column + beneath it**: a text label above a numeric column, or a short label above much + longer values. This is the test `csv.Sniffer.has_header` uses; it is a + property of the table rather than a pattern borrowed from one document. +- When no header is provable, cells are joined with `" | "` — unlabeled, but + never mislabeled. Losing a label is recoverable at retrieval time; labeling + every row from a data row is not. +- A header merged vertically across two rows resolves to the same text in the + row below it; that duplicate is skipped rather than emitted as data. +- A cell merged across columns is reported once per grid position it spans; + those repeats are collapsed. + + + is a pipeline invariant + +### 5. Persian normalization is a pipeline invariant + +Every extracted text block is normalized before chunking, for all formats: + +- Arabic to Persian letter folding: `ك`→`ک`, `ي`→`ی`, `ى`→`ی`, `أ`/`إ`→`ا` +- `unicodedata.normalize("NFKC")` +- Removal of harakat (diacritics) and tatweel +- `¬` → space, then collapse runs of whitespace + +Digits and punctuation are **not** rewritten. Persian digits (`۱۲۳`) and +Persian punctuation (`؛`, `٬`) are left as authored, because chunk `content` is +what citations render back to the user and Western digits inside Persian prose +read as wrong. + +Normalization runs **per text block, before the markdown is assembled** — the +whitespace-collapse step maps `\n` to a space, so applying it to an assembled +document would flatten every heading and paragraph onto a single line. + +This complements rather than replaces ADR-0005's sparse-side normalization, +which additionally removes stopwords and is specific to the BM25 vector. It +also stabilizes the text that feeds `content_hash`. + +### 6. Spreadsheet handling + +ADR-0004's rules stand, under the header discipline of section 4: each cell is +rendered as `"{column_header}: {cell_value}"`, merged cell ranges are +forward-filled before rendering, and sheets with no non-empty data rows are +skipped. + +Forward-filling merges is not cosmetic. openpyxl stores a merged range's value +only in its top-left cell, so in the branch directory the province is present +on the first branch of each province and absent from every other one. Filling +the range makes each row-chunk self-contained — a branch carries its province +even though the source cell is blank. + +One correction: `.csv` is read with the standard library's `csv` module, not +`pandas` as ADR-0004 states. Adding pandas for delimiter handling and row +iteration is not warranted. `.xlsx` uses `openpyxl`, as ADR-0004 assumed. + +There is no sheet-shape sniffing. A two-column Q&A sheet and a branch-directory +sheet go through the same generic renderer; a Q&A row renders as +`question: …\nanswer: …` and a branch row as `branch_name: …\ncity: …`, both +self-describing without a schema heuristic that could misfire. + +### 7. Chunking + +The `fixed_size` strategy, over tokens counted with tiktoken `cl100k_base`: + +| Setting | Value | +|---|---| +| `chunk_size` | 400 tokens | +| `chunk_overlap` | 60 tokens | +| `max_chunk_tokens` | 512 (hard cap) | + +These numbers are recorded here because they exist in no ADR today — ADR-0001 +explicitly left "size/overlap are tunable config, not fixed by this ADR" open, +and ADR-0004 gives only the 512 ceiling. + +`cl100k_base` is a deliberate proxy. `text-embedding-3-large` has an 8191-token +window and never binds; `nomic-embed-text-v2-moe`'s 512-token sequence length +is the only real constraint. cl100k tokenizes Persian inefficiently while +nomic's multilingual tokenizer does not, so a cl100k count reliably +*over-estimates* the nomic count — measuring with cl100k and capping at 512 is +safe in the conservative direction, without shipping a second tokenizer and its +model download into the ingestion path. The 400/512 gap leaves headroom for the +mandatory `search_document: ` task prefix (ADR-0004) and any heading text +carried into a chunk. + +Spreadsheet rows are atomic and bypass the splitter. A row that exceeds +`max_chunk_tokens` falls through the fixed-size splitter in place, emitting +several ordered chunks, rather than being silently truncated by the embedding +model. + +### 8. `content_type` values emitted + +ADR-0004's four-value set is unchanged. v1 emits `paragraph` (DOCX prose and +flattened tables) and `table_row` (spreadsheet rows). `qa_pair` and +`image_caption` remain defined but are not produced. + +### 9. Deferred, not rejected + +Semantic-aware chunking; DOCX `table_row` and `qa_pair` structural detection; +embedded-image captioning; `.doc` conversion; Matryoshka-256 truncation. Each +remains ADR-0004's stated intent; this ADR only records that v1 does not ship +them. + +## Consequences + +### Positive + +- Plan 001's ingestion slice is unblocked with a parser already proven against + this specific Farsi corpus, rather than one written speculatively. +- Chunking stays pure, synchronous, and cheap — it fits ADR-0017's inline + request budget with no network round-trip, and runs safely under + `anyio.to_thread.run_sync`. +- Persian normalization closes a real defect that would otherwise degrade both + dense vectors silently, with no error and no obvious symptom. +- Concrete chunk-size numbers and their rationale are now recorded somewhere + other than a config default, so a later change is a visible decision. +- Q&A sheets, branch directories, and docx contact tables are all served by one + renderer with no schema sniffing, so a new sheet shape needs no new code. +- Refusing to label a table whose header is unprovable means the parser degrades + to unlabeled rows instead of producing confidently wrong ones, which is the + failure mode that is hard to notice downstream. + +### Negative + +- **v1 ships the strategy the user's own comparison ranked second.** Retrieval + accuracy is expected to be measurably lower than semantic chunking would + give. Neighbor expansion via ADR-0001's pointers is the mitigation, and it is + unproven at this scale. +- A table whose header cannot be proven — short text over short text, which is + genuinely ambiguous — produces unlabeled `" | "` rows. A reader or model can + still see the values but not which column each belongs to. +- The layout-table rule keys on `chunk_size`, so changing that setting silently + changes which tables are treated as data. The observed margin is two orders of + magnitude, so this is unlikely to flip in practice, but it is a coupling. +- A DOCX that alternates literal `سوال:`/`پاسخ:` paragraphs loses question/answer + atomicity; a boundary can fall between a question and its answer, because + `qa_pair` detection is deferred. +- cl100k is a proxy for nomic's tokenizer. The relationship is safe in the + conservative direction for Persian, but a document in another language could + in principle tokenize the other way; the 512 assertion is what catches it. +- Normalizing stored `content` means the text served in citations is not + byte-identical to the source document. Letter folding was chosen over full + normalization specifically to keep this difference invisible to a reader. +- `.doc` files are rejected outright rather than converted, so any legacy + document must be re-saved by hand until the conversion service lands. + +## Alternatives Considered + +- **Implement ADR-0004 in full now** (DOCX table-row detection with header + inference, Q&A pair heuristics, image captioning, semantic boundary + detection): rejected for v1 as roughly triple the work, none of which exists + in the evaluation repository to port, and which would block the first + ingestion slice on parser research. +- **Semantic chunking inside the inline request**: rejected — it adds a + per-sentence embedding pass to a request already bounded by + `INGESTION_TIMEOUT_SECONDS`, and ADR-0017 chose inline ingestion on the + assumption that chunking is negligible. Revisit when ingestion moves back off + the request path. +- **The `nomic-embed-text-v2-moe` tokenizer** for exact chunk sizing: rejected + — it requires `transformers`/`tokenizers` and a model file download in the + ingestion path to buy precision that the conservative cl100k over-estimate + already provides. +- **Character-based splitting** (no tokenizer at all): rejected — Persian + characters-per-token varies enough that a character budget cannot guarantee + the 512-token ceiling that actually matters. +- **Full Persian normalization** including digit unification (`۱۲۳`→`123`) and + punctuation mapping: rejected — it would improve lexical matching slightly + when a query uses the other digit form, at the cost of rendering Persian + citations with Western digits. +- **Storing raw and normalized text as separate payload fields**: rejected — + ADR-0001 fixes the payload field list, and doubling the stored text per point + is not justified when letter folding alone is visually lossless. +- **Row-chunking DOCX tables like spreadsheets**: rejected for v1 — the Farsi + `.doc` exports in this corpus use tables as page layout, with ordinary prose + inside cells, so treating every row as an atomic unit would shred paragraphs + mid-sentence. Revisit once real chunk output from the table-heavy documents + has been inspected. diff --git a/docs/backlog.md b/docs/backlog.md new file mode 100644 index 0000000..c4f8750 --- /dev/null +++ b/docs/backlog.md @@ -0,0 +1,109 @@ +# Backlog + +Ideas and open questions not yet ready to be an ADR decision or a plan phase. +Each entry is short: what the idea is, which ADR/plan it would eventually +touch, and what's still unresolved. When an entry is picked up, turn it into +an ADR amendment (or a new ADR) and delete it from here — this file is not a +permanent record, `docs/adr/` is. + +## Legacy `.doc` conversion + +Relates to: [ADR-0018](adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md). + +`.doc` is rejected with `415`. ADR-0004 specified `soffice --headless +--convert-to docx`, which ADR-0018 rejected as a multi-second subprocess inside +an inline request. Gotenberg is the obvious candidate since it is already in +use elsewhere — **but verify before committing to it**: Gotenberg's LibreOffice +route is built for converting *to PDF*, and `.doc` → `.docx` output may not be +supported on that endpoint. If it is not, the options are a dedicated +LibreOffice sidecar or asking uploaders to re-save. + +## Structural units ADR-0004 specifies but v1 does not emit + +Relates to: [ADR-0004](adr/0004-docx-csv-chunking-strategy.md), +[ADR-0018](adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md). + +- **`qa_pair`**: one sample document alternates literal `سوال:`/`پاسخ:` + paragraphs. v1 chunks it as prose, so a chunk boundary can fall between a + question and its answer. +- **`image_caption`**: two sample documents embed images with no alt text. + ADR-0004 routes these through a vision API at ingest; v1 drops them silently. + +Both need a decision on whether heuristic detection is worth the misfire risk — +the header-detection work showed that guessing structure is expensive when wrong. + +## Tables whose header cannot be proven + +Relates to: [ADR-0018](adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md). + +A table of short text over short text (`branch,city` with no numeric or long +column) is genuinely ambiguous, so v1 emits unlabeled `" | "` rows rather than +risk labeling every row from a data row. No file in the current corpus hits +this, but a future one will. + +The honest fix is not a better heuristic — it is to stop guessing: let the +upload declare whether a sheet has a header, since the uploader knows. That is +a `POST /v1/files` contract change, so it belongs with plan 001 Phase 3 rather +than in the parser. + +## Recalibrate chunk size against nomic's tokenizer + +Relates to: [ADR-0018](adr/0018-docx-and-spreadsheet-parsing-with-fixed-size-chunking.md). +**Revisit after retrieval quality is measurable** — deliberately deferred, not +forgotten. + +ADR-0018 counts tokens with tiktoken `cl100k_base` and caps chunks at 512. But +512 is `nomic-embed-text-v2-moe`'s limit, measured in *nomic's* tokenizer, not +OpenAI's. Those are different units, and on Persian they differ by a lot. + +Measured against a real production document (`bimeh_havades.docx`, 5,911 chars +of Farsi) via the Ollama server that already hosts the model: + +| Sample | cl100k tokens | nomic tokens | ratio | +|---|---|---|---| +| 300 chars | 217 | 80 | 2.71 | +| 600 chars | 426 | 165 | 2.58 | +| 1,200 chars | 846 | 303 | 2.79 | + +So **~2.7 cl100k tokens per nomic token** on Persian. The current +`chunk_size=400` is therefore about **148 nomic tokens — roughly 29% of the +512-token window**. Chunks land near 570 characters where ~1,500 would fit. + +Two things this measurement also established: + +- **Silent truncation is real, and now demonstrated.** Feeding 2,400 and 4,800 + characters both returned `prompt_eval_count` of exactly 512, with no error + and no warning. This is what ADR-0004 meant by "silently truncated by the + model, not an error", confirmed on our own hardware. +- **Measuring nomic tokens needs no new dependency.** Ollama's `/api/embed` + returns `prompt_eval_count`, so the real count is obtainable from the + embedding call we already have to make. Note the value saturates at 512, so + it cannot measure anything longer than the window — calibration samples must + stay under it. + +When picking this up, decide between: raising `chunk_size`/`max_chunk_tokens` +in cl100k terms using a calibration ratio (cheap, drifts if the corpus language +mix changes); counting with nomic's own tokenizer offline via HuggingFace +`tokenizers` and its `tokenizer.json` (exact, and lighter than ADR-0018 +assumed — the tokenizer file only, not the 475M-param model weights); or +keeping small chunks because neighbor expansion recovers the context anyway. + +Do not change this on the ratio alone. The reason to keep 400/60/512 for now is +that smaller chunks are not automatically worse for retrieval — measure +retrieval quality first, then decide. + +Also note `nomic-embed-text:latest` (v1.5) is on the same Ollama server with a +2,048-token context, but it is the English-focused model; v2-moe is the +multilingual one and the reason ADR-0004 chose it for Farsi. Do not switch to +v1.5 just to get a bigger window. + +## Get LLM usage/price from the OpenAI API + +Relates to: [ADR-0009](adr/0009-postgres-sqlalchemy-alembic-schema.md)'s +`llm_calls`/`llm_pricing` tables. + +Get token usage and price from the OpenAI API's response metadata, instead of +computing/tracking them ourselves. Need to check whether OpenAI actually +returns price, or only token counts — if only counts, we still need +`llm_pricing` for price and this only changes how `llm_calls` gets its +usage numbers. diff --git a/docs/plans/001-ingestion-vertical-slice.md b/docs/plans/001-ingestion-vertical-slice.md index 1b2b6e0..74f6a4c 100644 --- a/docs/plans/001-ingestion-vertical-slice.md +++ b/docs/plans/001-ingestion-vertical-slice.md @@ -3,7 +3,7 @@ ## Purpose This plan turns the accepted architectural direction in the ADRs into the first -working product slice: a tenant-scoped CSV upload is stored in MinIO, represented +working product slice: a tenant-scoped DOCX/XLSX/CSV upload is stored in MinIO, represented by durable Postgres records, parsed/chunked/embedded inline in the request (ADR-0017), and indexed as Qdrant points before the response returns. @@ -47,14 +47,19 @@ them. ### In scope -- `POST /v1/files` for authenticated tenant-scoped **CSV** upload. +- `POST /v1/files` for authenticated tenant-scoped **DOCX, XLSX, and CSV** upload. + `.doc` is rejected with `415` pending an out-of-process conversion service + (ADR-0018). An earlier revision of this plan scoped the slice to CSV only and + placed DOCX out of scope; the real corpus is DOCX and XLSX, so ADR-0018 + corrects that. - File validation, size limits, content hashing, and streaming upload to MinIO. - Alembic-managed Postgres schema for the minimal tenant/auth, source file, ingestion job, and job event records needed by this slice. - Inline ingestion in `POST /v1/files`, with batched/bounded-concurrent embedding, thread-offloaded parsing, and enforced size/timeout/capacity bounds. -- CSV parsing and deterministic chunk creation. +- DOCX/XLSX/CSV parsing into structural units and deterministic chunk creation + (ADR-0018, implemented in `src/application/ingestion/`). - Tenant-filtered Qdrant point upserts using deterministic point identifiers. - Job status/progress persistence and `GET /v1/files/{file_id}` status lookup. - Structured correlation logging at HTTP and ingestion-stage boundaries. @@ -63,7 +68,11 @@ them. ### Explicitly out of scope -- XLSX, DOCX, and legacy DOC ingestion. +- Legacy `.doc` ingestion — rejected with `415` until an out-of-process + conversion service exists (ADR-0018). DOCX and XLSX are **in** scope; they were + listed here before ADR-0018 corrected the scope line. +- `qa_pair` structural detection and embedded-image captioning (ADR-0004), + deferred by ADR-0018. - The conversational LangGraph API and SSE streaming. - Final reranker selection, GPU deployment, or unresolved model licensing from ADR-0005. @@ -196,7 +205,7 @@ reads/writes and valid job transitions. ### Phase 3: MinIO upload and durable job creation 1. Implement API-key authentication and `AuthContext` tenant derivation. -2. Implement `POST /v1/files` for CSV only, including streaming-size controls, +2. Implement `POST /v1/files` for DOCX, XLSX, and CSV, including streaming-size controls, file-type validation, SHA-256 calculation, and a private MinIO upload using an internal object key. 3. In one short Postgres transaction, persist `source_files` and create @@ -208,11 +217,11 @@ reads/writes and valid job transitions. response that does not expose raw storage credentials or internal artifacts. 6. Add cleanup/compensation handling for a MinIO upload that succeeds while the database transaction fails. -7. Add unit/API tests for trusted tenant derivation, CSV validation, idempotency, +7. Add unit/API tests for trusted tenant derivation, upload validation, idempotency, the terminal `201 Created` response, and tenant-scoped status. Add MinIO adapter integration tests for server-derived private object paths and compensation behavior. -**Exit criteria:** an authenticated CSV upload creates a private object and a +**Exit criteria:** an authenticated upload creates a private object and a `running` job row committed before any ingestion work; a tenant cannot retrieve another tenant's file status. @@ -241,14 +250,14 @@ code and a terminal job row. 1. Implement the ingestion service called by the route, using the application-lifetime database, MinIO, Qdrant, model, and logging clients. 2. Validate the persisted records before fetching the MinIO object. -3. Append progress events, parse CSV, create deterministic chunks, embed them, +3. Append progress events, parse the document, create deterministic chunks, embed them, and upsert tenant-scoped Qdrant points — without holding a Postgres session open across the work. 4. In a second short transaction, mark the job `succeeded` with counters or `failed` with a safe error summary, then return the terminal response. 5. Make a retried upload safe: no duplicate logical chunks, no incorrect counters, and no transition from a terminal state back to `running`. -6. Add unit tests for deterministic CSV chunks, point IDs, and terminal job +6. Add unit tests for deterministic chunks, point IDs, and terminal job transitions. Add Testcontainers Qdrant and Postgres integration tests for tenant-filtered upserts, terminal state persistence, retrying an upload, and parser/Qdrant failure handling. @@ -274,7 +283,7 @@ retrying the upload produces a correct final state without duplicate chunks. and the operations runbook. **Exit criteria:** a new developer can start the stack, apply migrations, upload a -CSV, observe the job through completion, and understand how to investigate or +a document, observe the job through completion, and understand how to investigate or retry a failure. ## Definition of done for the vertical slice @@ -283,7 +292,7 @@ The first slice is done when the following path works in local Compose and is covered by automated tests: ```text -POST /v1/files (authenticated CSV upload) +POST /v1/files (authenticated DOCX/XLSX/CSV upload) -> raw bytes stored privately in MinIO -> source file and running job committed in Postgres, connection released -> parse/chunk on threads, embed in bounded concurrent batches diff --git a/pyproject.toml b/pyproject.toml index c9feea3..e7e6e15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,10 +11,13 @@ dependencies = [ "fastapi[standard]==0.141.1", "langgraph>=1.2.10", "minio>=7.2.20", + "openpyxl>=3.1.5", "pydantic-settings>=2.15.0", + "python-docx>=1.2.0", "qdrant-client>=1.19.0", "sqlalchemy>=2.0.51", "structlog>=26.1.0", + "tiktoken>=0.13.0", ] [dependency-groups] diff --git a/src/application/__init__.py b/src/application/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/application/ingestion/__init__.py b/src/application/ingestion/__init__.py new file mode 100644 index 0000000..ffe0058 --- /dev/null +++ b/src/application/ingestion/__init__.py @@ -0,0 +1,47 @@ +"""Document parsing and fixed-size chunking (ADR-0004, ADR-0018). + +Everything here is pure and synchronous: no I/O, no ports, no SDK clients +(ADR-0015 reserves ports for external side effects). Parsing and chunking are +blocking CPU work, so callers run them through `anyio.to_thread.run_sync` with +the ingestion `CapacityLimiter` rather than on the event loop (ADR-0017). +""" + +from src.application.ingestion.chunking import chunk_document, chunk_id_for, split_by_tokens +from src.application.ingestion.docx_parser import parse_docx +from src.application.ingestion.errors import ( + ChunkLimitExceededError, + ChunkTooLargeError, + DocumentParseError, + IngestionError, + UnsupportedSourceTypeError, +) +from src.application.ingestion.models import ( + Chunk, + ContentType, + ParsedDocument, + StructuralUnit, +) +from src.application.ingestion.normalization import normalize_persian_text +from src.application.ingestion.spreadsheet_parser import parse_csv, parse_xlsx +from src.application.ingestion.tokenizer import count_tokens, get_encoder + +__all__ = [ + "Chunk", + "ChunkLimitExceededError", + "ChunkTooLargeError", + "ContentType", + "DocumentParseError", + "IngestionError", + "ParsedDocument", + "StructuralUnit", + "UnsupportedSourceTypeError", + "chunk_document", + "chunk_id_for", + "count_tokens", + "get_encoder", + "normalize_persian_text", + "parse_csv", + "parse_docx", + "parse_xlsx", + "split_by_tokens", +] diff --git a/src/application/ingestion/chunking.py b/src/application/ingestion/chunking.py new file mode 100644 index 0000000..e8674a9 --- /dev/null +++ b/src/application/ingestion/chunking.py @@ -0,0 +1,129 @@ +"""Fixed-size chunking with overlap (ADR-0018). + +DOCX markdown is split on token windows; spreadsheet rows are already atomic +and bypass the splitter, falling through it only when a single row exceeds the +embedding model's sequence length. +""" + +import uuid + +from src.application.ingestion.errors import ChunkTooLargeError, DocumentParseError +from src.application.ingestion.models import Chunk, ContentType, ParsedDocument +from src.application.ingestion.tokenizer import count_tokens, get_encoder +from src.config import ChunkingSettings + +# Fixed namespace so chunk ids stay stable across processes and releases. +CHUNK_ID_NAMESPACE = uuid.UUID("6f9619ff-8b86-d011-b42d-00c04fc964ff") + + +def chunk_id_for(file_id: uuid.UUID, chunk_index: int) -> uuid.UUID: + """Return the deterministic point id for a chunk (ADR-0001). + + Derived from `file_id` and the immutable ingestion ordinal, so re-ingesting + a file upserts its points instead of duplicating them. + """ + return uuid.uuid5(CHUNK_ID_NAMESPACE, f"{file_id}:{chunk_index}") + + +def split_by_tokens(text: str, *, chunk_size: int, overlap: int, encoding_name: str) -> list[str]: + """Split text into overlapping windows of at most `chunk_size` tokens.""" + if overlap >= chunk_size: + raise ValueError(f"overlap ({overlap}) must be smaller than chunk_size ({chunk_size})") + + encoder = get_encoder(encoding_name) + tokens = encoder.encode(text) + if len(tokens) <= chunk_size: + return [text] + + windows: list[str] = [] + start = 0 + while start < len(tokens): + end = min(start + chunk_size, len(tokens)) + windows.append(encoder.decode(tokens[start:end])) + if end >= len(tokens): + break + start = end - overlap + + return windows + + +def _split_oversized(text: str, settings: ChunkingSettings) -> list[str]: + """Split a row only if it exceeds the cap; otherwise keep it atomic.""" + if count_tokens(text, settings.encoding_name) <= settings.max_chunk_tokens: + return [text] + return split_by_tokens( + text, + chunk_size=settings.chunk_size, + overlap=settings.chunk_overlap, + encoding_name=settings.encoding_name, + ) + + +def _content_units( + parsed: ParsedDocument, settings: ChunkingSettings +) -> list[tuple[str, ContentType]]: + """Reduce a parsed document's structural units to ordered text pieces. + + Prose is cut into token windows; a table row is atomic and survives whole + unless it alone exceeds the model's sequence length (ADR-0004). + """ + if not parsed.units: + raise DocumentParseError("Parsed document has no structural units") + + pieces: list[tuple[str, ContentType]] = [] + for unit in parsed.units: + if unit.content_type is ContentType.PARAGRAPH: + windows = split_by_tokens( + unit.text, + chunk_size=settings.chunk_size, + overlap=settings.chunk_overlap, + encoding_name=settings.encoding_name, + ) + else: + windows = _split_oversized(unit.text, settings) + pieces.extend((window, unit.content_type) for window in windows) + + return pieces + + +def chunk_document( + parsed: ParsedDocument, + *, + file_id: uuid.UUID, + settings: ChunkingSettings, +) -> list[Chunk]: + """Turn a parsed document into ordered, neighbor-linked chunks.""" + units = [ + (text.strip(), content_type) for text, content_type in _content_units(parsed, settings) + ] + # Drop blanks *before* assigning indices: an index gap would break the + # previous/next chain that retrieval-time expansion walks. + units = [(text, content_type) for text, content_type in units if text] + + chunks: list[Chunk] = [] + for index, (text, content_type) in enumerate(units): + token_count = count_tokens(text, settings.encoding_name) + if token_count > settings.max_chunk_tokens: + raise ChunkTooLargeError( + f"chunk {index} is {token_count} tokens, over the " + f"{settings.max_chunk_tokens}-token cap" + ) + chunks.append( + Chunk( + chunk_id=chunk_id_for(file_id, index), + chunk_index=index, + order_id=float(index + 1), + content=text, + content_type=content_type, + token_count=token_count, + character_count=len(text), + ) + ) + + for position, chunk in enumerate(chunks): + if position > 0: + chunk.previous_chunk_id = chunks[position - 1].chunk_id + if position < len(chunks) - 1: + chunk.next_chunk_id = chunks[position + 1].chunk_id + + return chunks diff --git a/src/application/ingestion/docx_parser.py b/src/application/ingestion/docx_parser.py new file mode 100644 index 0000000..f47660e --- /dev/null +++ b/src/application/ingestion/docx_parser.py @@ -0,0 +1,165 @@ +"""DOCX parsing into ordered structural units (ADR-0004, ADR-0018). + +The body is walked in document order and decomposed into structural units +before any chunking runs: runs of flowing prose become `PARAGRAPH` units the +splitter cuts into token windows, and data-table rows become atomic +`TABLE_ROW` units. + +The one classification this makes is between a *data* table and a table used +as page layout, and it is made structurally rather than by inspecting content: +a data cell fits inside a chunk by definition, so a table holding a cell that +alone exceeds `chunk_size`, or a cell containing nested tables, is a layout +container whose cells are prose. +""" + +import io + +from docx import Document +from docx.document import Document as DocxDocument +from docx.oxml.table import CT_Tbl +from docx.oxml.text.paragraph import CT_P +from docx.table import Table, _Cell +from docx.text.paragraph import Paragraph + +from src.application.ingestion.errors import DocumentParseError +from src.application.ingestion.models import ( + ContentType, + ParsedDocument, + StructuralUnit, +) +from src.application.ingestion.normalization import normalize_persian_text +from src.application.ingestion.tabular import clean_rows, render_rows +from src.application.ingestion.tokenizer import count_tokens +from src.config import ChunkingSettings + +_NORMAL_STYLE = "Normal" + + +def heading_level_from_style(style_name: str) -> int | None: + """Return the heading level of a paragraph style, or None for body text. + + Word stores the styleId (`Heading1`), not the friendly name (`Heading 1`), + so both spellings must resolve. Only real Word styles count -- no heading + is ever inferred from the text itself (ADR-0018). + """ + normalized = style_name.strip().lower().replace(" ", "") + if not normalized.startswith("heading"): + return None + suffix = normalized.removeprefix("heading") + return int(suffix) if suffix.isdigit() else None + + +def _paragraph_style(paragraph: Paragraph) -> str: + style = paragraph.style.name if paragraph.style is not None else None + return style or _NORMAL_STYLE + + +def _is_layout_table(table: Table, settings: ChunkingSettings) -> bool: + """Whether a table is page layout rather than data. + + Structural, not a content heuristic: a data cell is small enough to be a + chunk, so a cell that alone overflows `chunk_size` -- or that nests another + table -- holds a document, not a field. In the sample corpus this separates + by two orders of magnitude (32-171 tokens for data tables against 54,007 + for a cell containing a whole sub-document). + """ + for row in table.rows: + for cell in row.cells: + if cell.tables: + return True + if count_tokens(cell.text, settings.encoding_name) > settings.chunk_size: + return True + return False + + +def _cell_prose(cell: _Cell, settings: ChunkingSettings) -> list[StructuralUnit]: + """Extract a layout cell's contents as units, recursing into nested tables.""" + units: list[StructuralUnit] = [] + for block in _iter_block_items(cell, settings): + units.append(block) + return units + + +def _table_units(table: Table, settings: ChunkingSettings) -> list[StructuralUnit]: + """Convert a table to structural units.""" + if _is_layout_table(table, settings): + units: list[StructuralUnit] = [] + # Walk the physical `w:tc` elements rather than `row.cells`, which + # repeats a merged cell once per grid position it spans. Identity + # tracking is not an option here: lxml builds element proxies on + # demand, so `id()` is neither stable nor unique across them. + for row in table.rows: + for tc in row._tr.tc_lst: + units.extend(_cell_prose(_Cell(tc, table), settings)) + return units + + rows = clean_rows([[cell.text for cell in row.cells] for row in table.rows]) + return [ + StructuralUnit(text=text, content_type=ContentType.TABLE_ROW) for text in render_rows(rows) + ] + + +def _iter_block_items( + container: DocxDocument | _Cell, settings: ChunkingSettings +) -> list[StructuralUnit]: + """Walk a body or cell in document order, emitting structural units. + + Consecutive paragraphs accumulate into one prose unit rather than becoming + one unit each: ADR-0004 treats "the whole remaining run of paragraphs" as a + single prose block, so the splitter sees flowing text instead of a series + of one-sentence fragments. + """ + element = container.element.body if isinstance(container, DocxDocument) else container._tc + + units: list[StructuralUnit] = [] + prose: list[str] = [] + + def flush() -> None: + if prose: + units.append( + StructuralUnit(text="\n\n".join(prose), content_type=ContentType.PARAGRAPH) + ) + prose.clear() + + for child in element: + if isinstance(child, CT_P): + paragraph = Paragraph(child, container) + # `Paragraph.text` includes hyperlink text, which a raw `w:r` walk + # silently drops. + text = normalize_persian_text(paragraph.text) + if not text: + continue + level = heading_level_from_style(_paragraph_style(paragraph)) + prose.append(f"{'#' * level} {text}" if level else text) + elif isinstance(child, CT_Tbl): + table_units = _table_units(Table(child, container), settings) + # A layout table is prose; keep it in the surrounding prose block + # instead of fragmenting the document around it. + if table_units and all( + unit.content_type is ContentType.PARAGRAPH for unit in table_units + ): + prose.extend(unit.text for unit in table_units) + else: + flush() + units.extend(table_units) + + flush() + return units + + +def parse_docx(data: bytes, settings: ChunkingSettings) -> ParsedDocument: + """Parse DOCX bytes into ordered structural units.""" + try: + doc = Document(io.BytesIO(data)) + except Exception as exc: + raise DocumentParseError(f"Could not open DOCX: {exc}") from exc + + units = _iter_block_items(doc, settings) + if not units: + raise DocumentParseError("Document contains no text content") + + return ParsedDocument( + units=units, + markdown="\n\n".join(unit.text for unit in units), + block_count=len(units), + ) diff --git a/src/application/ingestion/errors.py b/src/application/ingestion/errors.py new file mode 100644 index 0000000..2c50d1e --- /dev/null +++ b/src/application/ingestion/errors.py @@ -0,0 +1,41 @@ +"""Errors raised by the parsing and chunking pipeline (ADR-0018). + +These carry no HTTP knowledge — the API layer maps them to status codes +(ADR-0015: `application/` contains no FastAPI request objects). +""" + + +class IngestionError(Exception): + """Base class for ingestion failures.""" + + +class DocumentParseError(IngestionError): + """A source file could not be decoded, opened, or yielded no text. + + Maps to `400` per ADR-0017 ("unparseable file → 400"). + """ + + +class UnsupportedSourceTypeError(IngestionError): + """A source file's type is not ingestible in this version. + + Maps to `415`. `.doc` lands here until an out-of-process conversion + service exists (ADR-0018). + """ + + +class ChunkLimitExceededError(IngestionError): + """A document produced more chunks than `INGESTION_MAX_CHUNKS_PER_FILE`. + + Maps to `413` per ADR-0017. + """ + + +class ChunkTooLargeError(IngestionError): + """A chunk exceeded the embedding model's sequence length. + + This is an internal invariant violation, not a user error: the splitter is + supposed to make it impossible. It exists because the failure it guards + against is silent — `nomic-embed-text-v2-moe` truncates over-long input + without raising (ADR-0004). + """ diff --git a/src/application/ingestion/models.py b/src/application/ingestion/models.py new file mode 100644 index 0000000..4f6b34e --- /dev/null +++ b/src/application/ingestion/models.py @@ -0,0 +1,66 @@ +"""Domain models for parsing and chunking (ADR-0001, ADR-0004, ADR-0018).""" + +import uuid +from enum import StrEnum + +from pydantic import BaseModel, Field + + +class ContentType(StrEnum): + """ADR-0004's finalized `content_type` value set. + + v1 emits `PARAGRAPH` and `TABLE_ROW` only; `QA_PAIR` and `IMAGE_CAPTION` + are defined but not produced yet (ADR-0018). + """ + + PARAGRAPH = "paragraph" + TABLE_ROW = "table_row" + QA_PAIR = "qa_pair" + IMAGE_CAPTION = "image_caption" + + +class StructuralUnit(BaseModel): + """One structural unit of a document, in reading order (ADR-0004). + + A document is decomposed into these *before* any chunking runs, because + the two kinds are chunked differently: + + - `PARAGRAPH` is a run of flowing prose; the fixed-size splitter cuts it + into token windows. + - `TABLE_ROW` is already atomic; it becomes one chunk, and is split only + when a single row is too large for the embedding model. + """ + + text: str + content_type: ContentType + + +class ParsedDocument(BaseModel): + """The output of a parser, before chunking. + + `units` is the content, in document order. `markdown` is the same content + rendered as one string, for eyeballing a parse; nothing chunks from it. + """ + + units: list[StructuralUnit] = Field(default_factory=list) + markdown: str | None = None + block_count: int = 0 + + +class Chunk(BaseModel): + """One indexable unit of a document. + + `chunk_id` is a deterministic UUIDv5 of `file_id` and `chunk_index` + (ADR-0001), so re-ingesting a file upserts its points rather than + duplicating them. + """ + + chunk_id: uuid.UUID + chunk_index: int + order_id: float + content: str + content_type: ContentType + previous_chunk_id: uuid.UUID | None = None + next_chunk_id: uuid.UUID | None = None + token_count: int + character_count: int diff --git a/src/application/ingestion/normalization.py b/src/application/ingestion/normalization.py new file mode 100644 index 0000000..fab5c8b --- /dev/null +++ b/src/application/ingestion/normalization.py @@ -0,0 +1,73 @@ +"""Persian text normalization (ADR-0018). + +Applied to every extracted text block before chunking, for all source formats. + +The problem this solves is silent: Persian authored on mixed Arabic/Persian +keyboards contains both U+06A9 and U+0643 for "k", both U+06CC and U+064A for +"y". Those are distinct codepoints and therefore distinct tokens to every +embedding model, so the same word embeds two different ways depending on which +key the author pressed. + +Letter folding only -- digits and punctuation are left as authored, because +chunk content is what citations render back to the reader and Western digits +inside Persian prose read as wrong. +""" + +import re +import unicodedata + +# Both tables are written as codepoints rather than character literals. Arabic +# letterforms are visually indistinguishable from one another (and alef from a +# Latin "l") in a monospace editor -- which is the very confusion this module +# exists to resolve -- and literals would render right-to-left, visually +# reordering the source line. +# +# `str.translate` accepts an ordinal->ordinal mapping directly, and an ordinal +# mapped to None is deleted. + +_ARABIC_KAF = 0x0643 +_ARABIC_YEH = 0x064A +_ALEF_MAKSURA = 0x0649 +_ALEF_HAMZA_ABOVE = 0x0623 +_ALEF_HAMZA_BELOW = 0x0625 +_NOT_SIGN = 0x00AC + +_PERSIAN_KEHEH = 0x06A9 +_PERSIAN_YEH = 0x06CC +_ALEF = 0x0627 +_SPACE = 0x0020 + +_LETTER_FOLDING: dict[int, int] = { + _ARABIC_KAF: _PERSIAN_KEHEH, + _ARABIC_YEH: _PERSIAN_YEH, + _ALEF_MAKSURA: _PERSIAN_YEH, + _ALEF_HAMZA_ABOVE: _ALEF, + _ALEF_HAMZA_BELOW: _ALEF, + # A soft-hyphen artifact from documents exported by older Word versions. + _NOT_SIGN: _SPACE, +} + +_TATWEEL = 0x0640 +_SUPERSCRIPT_ALEF = 0x0670 +_HARAKAT = range(0x064B, 0x0660) + +# Applied after NFKC, which can itself decompose presentation forms into a +# base letter plus a combining mark. +_MARK_REMOVAL: dict[int, int | None] = dict.fromkeys(_HARAKAT) +_MARK_REMOVAL[_TATWEEL] = None +_MARK_REMOVAL[_SUPERSCRIPT_ALEF] = None + +_WHITESPACE = re.compile(r"\s+") + + +def normalize_persian_text(text: str) -> str: + """Fold Arabic letterforms to Persian and collapse whitespace. + + Call this per text block, **before** blocks are assembled into a document. + The whitespace collapse maps `\\n` to a space, so running it over assembled + markdown would flatten every heading and paragraph onto one line. + """ + text = text.translate(_LETTER_FOLDING) + text = unicodedata.normalize("NFKC", text) + text = text.translate(_MARK_REMOVAL) + return _WHITESPACE.sub(" ", text).strip() diff --git a/src/application/ingestion/spreadsheet_parser.py b/src/application/ingestion/spreadsheet_parser.py new file mode 100644 index 0000000..268d502 --- /dev/null +++ b/src/application/ingestion/spreadsheet_parser.py @@ -0,0 +1,121 @@ +"""CSV and XLSX parsing: row = chunk (ADR-0004, ADR-0018). + +Both formats reduce to rows and hand them to the shared renderer in +`tabular`, so a Q&A sheet and a branch directory go through one code path with +no shape detection. +""" + +import csv +import io + +import openpyxl +from openpyxl.worksheet.worksheet import Worksheet + +from src.application.ingestion.errors import DocumentParseError +from src.application.ingestion.models import ContentType, ParsedDocument, StructuralUnit +from src.application.ingestion.tabular import Row, clean_cell, clean_rows, render_rows + +# Farsi exports from older Excel are frequently cp1256 (Windows Arabic). +_ENCODINGS = ("utf-8-sig", "utf-8", "cp1256") + +_SNIFF_BYTES = 8192 + + +def _decode(data: bytes) -> str: + for encoding in _ENCODINGS: + try: + return data.decode(encoding) + except UnicodeDecodeError: + continue + raise DocumentParseError(f"Could not decode file as any of: {', '.join(_ENCODINGS)}") + + +def _to_units(rendered: list[str]) -> list[StructuralUnit]: + return [StructuralUnit(text=text, content_type=ContentType.TABLE_ROW) for text in rendered] + + +def parse_csv(data: bytes) -> ParsedDocument: + """Parse CSV bytes into one structural unit per row.""" + text = _decode(data) + + try: + dialect = csv.Sniffer().sniff(text[:_SNIFF_BYTES]) + reader = csv.reader(io.StringIO(text), dialect) + except csv.Error: + # A single-column file has no delimiter to find; that is not an error. + reader = csv.reader(io.StringIO(text)) + + rows = clean_rows(reader) + if not rows: + raise DocumentParseError("File contains no rows") + + units = _to_units(render_rows(rows)) + if not units: + raise DocumentParseError("File contains no data rows") + + return ParsedDocument(units=units, block_count=len(units)) + + +def _forward_fill_merges(worksheet: Worksheet) -> dict[tuple[int, int], str]: + """Map every cell of a merged range to the range's value. + + openpyxl stores a merged range's value only in its top-left cell; the rest + read as None. Without this a branch row inherits nothing from the province + cell merged above it and silently loses that field (ADR-0004). + + Iterate the range collection itself and read corners via `bounds`: `.ranges` + is a set subclass and `.min_row` and friends are descriptors, neither of + which resolves to an int for a type checker. + """ + filled: dict[tuple[int, int], str] = {} + + for merged in worksheet.merged_cells: + min_col, min_row, max_col, max_row = merged.bounds + value = clean_cell(worksheet.cell(row=min_row, column=min_col).value) + if not value: + continue + for row in range(min_row, max_row + 1): + for column in range(min_col, max_col + 1): + filled[(row, column)] = value + + return filled + + +def _sheet_rows(worksheet: Worksheet) -> list[Row]: + """Read a worksheet into normalized text rows, merges resolved.""" + merged = _forward_fill_merges(worksheet) + rows: list[Row] = [] + + for row_index, row in enumerate(worksheet.iter_rows(), start=1): + values = [ + merged.get((row_index, column_index), clean_cell(cell.value)) + for column_index, cell in enumerate(row, start=1) + ] + if any(values): + rows.append(values) + + return rows + + +def parse_xlsx(data: bytes) -> ParsedDocument: + """Parse XLSX bytes into one structural unit per row, across all sheets.""" + try: + workbook = openpyxl.load_workbook(io.BytesIO(data), data_only=True) + except Exception as exc: + raise DocumentParseError(f"Could not open XLSX: {exc}") from exc + + units: list[StructuralUnit] = [] + try: + for worksheet in workbook.worksheets: + rows = _sheet_rows(worksheet) + # The dead second sheet seen throughout the sample corpus. + if not rows: + continue + units.extend(_to_units(render_rows(rows))) + finally: + workbook.close() + + if not units: + raise DocumentParseError("Workbook contains no data rows") + + return ParsedDocument(units=units, block_count=len(units)) diff --git a/src/application/ingestion/tabular.py b/src/application/ingestion/tabular.py new file mode 100644 index 0000000..1bdceef --- /dev/null +++ b/src/application/ingestion/tabular.py @@ -0,0 +1,167 @@ +"""Shared row-to-chunk rendering for every tabular source (ADR-0004, ADR-0018). + +A table row is an atomic structural unit regardless of the container it +arrived in -- a docx table, an xlsx sheet, or a csv file -- so one renderer +serves all three and a Q&A sheet needs no special case against a branch +directory: + + q: ... ردیف: 1 + a: ... استان: اردبیل + شعبه: پارس آباد + +The header rules exist because real tables are not uniform. Of the tables in +the sample corpus, some carry a header row, one is a bare list of values with +no header at all, and one is page decoration. The guiding constraint is +therefore: **never invent structure that is not provably there, and never +discard a row.** A wrongly-detected header turns every chunk into nonsense +(`80: 70`), which is worse than an unlabeled row. +""" + +from collections.abc import Iterable, Sequence + +from src.application.ingestion.normalization import normalize_persian_text + +Row = Sequence[str] + +# A header cell is a label, not a sentence. +MAX_HEADER_CELL_LENGTH = 80 + +# How much shorter a header cell must be than the column beneath it before +# length alone is taken as evidence of a header (the `q`/`a` case, where +# one-character labels sit above paragraph-long answers). +_HEADER_LENGTH_RATIO = 3.0 + + +def clean_cell(value: object) -> str: + """Normalize a cell to text; None and blank cells become the empty string. + + `object` rather than a union: a spreadsheet cell holds whatever the + workbook stored -- str, int, float, bool, datetime, a formula error -- and + every one of them is handled the same way, by rendering it. + """ + if value is None: + return "" + return normalize_persian_text(str(value)) + + +def _is_numeric(text: str) -> bool: + return bool(text) and text.replace(",", "").replace(".", "").replace("-", "").isdigit() + + +def _looks_like_title_row(row: Row) -> bool: + """A merged title spanning the sheet resolves to one value, or repeats it.""" + populated = [cell for cell in row if cell] + return len(populated) < 2 or len(set(populated)) == 1 + + +def has_header(rows: Sequence[Row]) -> bool: + """Whether the first row labels the columns beneath it. + + Decided by comparing row 0 against the column below it, not by how row 0 + looks on its own: a label row is *inconsistent* with its data (text above + numbers, or a short label above long prose), while a data row is + consistent with the rows that follow. This is the test `csv.Sniffer` + uses, and it is a property of the table rather than a pattern borrowed + from one document. + """ + if len(rows) < 2: + return False + + candidate, data = rows[0], rows[1:] + if not all(cell for cell in candidate[: len(data[0])] if cell) and _looks_like_title_row( + candidate + ): + return False + if any(len(cell) > MAX_HEADER_CELL_LENGTH for cell in candidate): + return False + + for index, label in enumerate(candidate): + if not label: + continue + column = [row[index] for row in data if index < len(row) and row[index]] + if not column: + continue + + # A text label above a numeric column. + if not _is_numeric(label) and all(_is_numeric(value) for value in column): + return True + + # A short label above a column of much longer values. + mean_length = sum(len(value) for value in column) / len(column) + if mean_length > len(label) * _HEADER_LENGTH_RATIO: + return True + + return False + + +def strip_title_rows(rows: Sequence[Row]) -> Sequence[Row]: + """Drop leading merged-title and blank rows. + + Structural, not a content guess: these rows are the artifact of a merged + range spanning the sheet width, so they carry one value across many cells. + Only *leading* rows are dropped, so no data row is ever lost. + """ + start = 0 + while start < len(rows) and _looks_like_title_row(rows[start]): + start += 1 + return rows[start:] + + +def _column_name(header: Row, index: int) -> str: + """Return a header label, falling back positionally past the header width.""" + if index < len(header) and header[index]: + return header[index] + return f"column_{index + 1}" + + +def _dedupe_horizontal_merge(row: Row) -> list[str]: + """Collapse the repeats a horizontally merged cell produces. + + Both python-docx and openpyxl report a merged cell once per grid column it + spans, so an unlabeled row would otherwise repeat the same value. + """ + collapsed: list[str] = [] + for cell in row: + if cell and (not collapsed or collapsed[-1] != cell): + collapsed.append(cell) + return collapsed + + +def render_rows(rows: Sequence[Row]) -> list[str]: + """Render table rows as text, one string per row. + + With a provable header each row becomes `"{header}: {value}"` lines, which + makes it self-describing. Without one, cells are joined with `" | "` -- + unlabeled, but never mislabeled. + """ + rows = strip_title_rows(rows) + if not rows: + return [] + + if not has_header(rows): + return [text for row in rows if (text := " | ".join(_dedupe_horizontal_merge(row)))] + + header, data = rows[0], rows[1:] + # A header merged vertically across two rows resolves to the same text in + # the row below it; that duplicate is the header, not data. + while data and list(data[0]) == list(header): + data = data[1:] + + rendered: list[str] = [] + for row in data: + lines = [ + f"{_column_name(header, index)}: {value}" for index, value in enumerate(row) if value + ] + if lines: + rendered.append("\n".join(lines)) + return rendered + + +def clean_rows(rows: Iterable[Iterable[object]]) -> list[Row]: + """Normalize every cell and drop rows that are entirely empty.""" + cleaned: list[Row] = [] + for row in rows: + values = [clean_cell(cell) for cell in row] + if any(values): + cleaned.append(values) + return cleaned diff --git a/src/application/ingestion/tokenizer.py b/src/application/ingestion/tokenizer.py new file mode 100644 index 0000000..2f306be --- /dev/null +++ b/src/application/ingestion/tokenizer.py @@ -0,0 +1,33 @@ +"""Token counting for chunk sizing (ADR-0018). + +`cl100k_base` is a deliberate proxy for the embedding models' own tokenizers. +`text-embedding-3-large` has an 8191-token window and never binds; +`nomic-embed-text-v2-moe`'s 512-token sequence length is the only real +constraint. cl100k tokenizes Persian inefficiently while nomic's multilingual +tokenizer does not, so a cl100k count reliably over-estimates the nomic count -- +safe in the conservative direction, without shipping a second tokenizer and its +model download into the ingestion path. +""" + +from functools import lru_cache + +import tiktoken + + +@lru_cache(maxsize=4) +def get_encoder(encoding_name: str) -> tiktoken.Encoding: + """Return a cached tiktoken encoder. + + Deliberately not a module-level constant: `tiktoken` fetches the BPE + vocabulary over the network the first time an encoding is used, and + ADR-0012 forbids external resource setup as an import-time side effect. + The lifespan warms this at startup so a process fails fast at boot rather + than inside the first ingestion request. Set `TIKTOKEN_CACHE_DIR` to a + pre-populated directory for offline deployments. + """ + return tiktoken.get_encoding(encoding_name) + + +def count_tokens(text: str, encoding_name: str) -> int: + """Return the number of tokens `text` encodes to.""" + return len(get_encoder(encoding_name).encode(text)) diff --git a/src/bootstrap/lifespan.py b/src/bootstrap/lifespan.py index 7e239b0..03edcf7 100644 --- a/src/bootstrap/lifespan.py +++ b/src/bootstrap/lifespan.py @@ -2,8 +2,10 @@ from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager import structlog +from anyio import to_thread from fastapi import FastAPI +from src.application.ingestion import get_encoder from src.bootstrap.dependencies import AppResources from src.config import Settings from src.infrastructure.minio.client import create_client as create_minio_client @@ -22,6 +24,15 @@ def create_lifespan( resolved_settings = settings or Settings() configure_logging(resolved_settings.logging) + # tiktoken fetches its vocabulary over the network on first use, so warm + # it here: a missing vocabulary should fail the process at boot, not the + # first upload. Blocking, hence the thread. + await to_thread.run_sync(get_encoder, resolved_settings.chunking.encoding_name) + logger.info( + "lifespan.tokenizer.loaded", + encoding=resolved_settings.chunking.encoding_name, + ) + db_engine = create_engine(resolved_settings.postgres) db_sessionmaker = create_sessionmaker(db_engine) logger.info("lifespan.postgres.engine.created") diff --git a/src/config.py b/src/config.py index e4935ed..94c1e12 100644 --- a/src/config.py +++ b/src/config.py @@ -1,4 +1,4 @@ -from pydantic import Field +from pydantic import Field, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -43,6 +43,38 @@ class IngestionSettings(BaseSettings): embed_concurrency: int = 4 +class ChunkingSettings(BaseSettings): + """Parsing and chunking parameters (ADR-0018). + + `max_chunk_tokens` is `nomic-embed-text-v2-moe`'s sequence length. Text past + it is silently truncated by the model rather than rejected, so the cap is + enforced here instead. `chunk_size` sits well under it to leave room for the + `search_document: ` task prefix and any heading text carried into a chunk. + """ + + model_config = SettingsConfigDict(env_prefix="CHUNKING_", extra="ignore") + + strategy: str = "fixed_size" + chunk_size: int = 400 + chunk_overlap: int = 60 + max_chunk_tokens: int = 512 + encoding_name: str = "cl100k_base" + + @model_validator(mode="after") + def _validate_sizes(self) -> "ChunkingSettings": + if self.chunk_overlap >= self.chunk_size: + raise ValueError( + f"chunk_overlap ({self.chunk_overlap}) must be smaller than " + f"chunk_size ({self.chunk_size}); otherwise splitting never advances" + ) + if self.chunk_size > self.max_chunk_tokens: + raise ValueError( + f"chunk_size ({self.chunk_size}) must not exceed " + f"max_chunk_tokens ({self.max_chunk_tokens})" + ) + return self + + class QdrantSettings(BaseSettings): model_config = SettingsConfigDict(env_prefix="QDRANT_", extra="ignore") @@ -71,6 +103,7 @@ class Settings(BaseSettings): postgres: PostgresSettings = Field(default_factory=PostgresSettings) minio: MinioSettings = Field(default_factory=MinioSettings) ingestion: IngestionSettings = Field(default_factory=IngestionSettings) + chunking: ChunkingSettings = Field(default_factory=ChunkingSettings) qdrant: QdrantSettings = Field(default_factory=QdrantSettings) app: AppLimitSettings = Field(default_factory=AppLimitSettings) logging: LoggingSettings = Field(default_factory=LoggingSettings) diff --git a/tests/support/documents.json b/tests/support/documents.json new file mode 100644 index 0000000..5dd9d06 --- /dev/null +++ b/tests/support/documents.json @@ -0,0 +1,30 @@ +{ + "prose_docx": { + "filename": "bimeh_havades.docx", + "why": "38 paragraphs, all Normal, no tables: the plain-prose path, and the proof that no heading is invented where the document declares none." + }, + "list_docx": { + "filename": "طرح ها-عمروحوادث-14050220.docx", + "why": "109 paragraphs mixing Normal and List Paragraph: prose carrying list styling." + }, + "table_docx": { + "filename": "شماره تماس معاونت و مدیریتها14050309.docx", + "why": "One 35x5 contact table, no body prose. A data table with a real header row and a vertically merged first column." + }, + "mixed_docx": { + "filename": "شرایط عمومی بیمه حوادث اشخاص14041205.docx", + "why": "Prose interleaved with two tables: a 30x3 list of injury compensations with NO header row, and an 11x2 table that has one." + }, + "layout_docx": { + "filename": "چت بات-مهندسی.docx", + "why": "A 3x2 layout table whose single merged cell holds an entire 77k-character sub-document across 738 paragraphs and 5 nested tables." + }, + "qa_xlsx": { + "filename": "fire14050319.xlsx", + "why": "Two-column Q&A sheet (q/a) plus a dead second sheet." + }, + "branches_xlsx": { + "filename": "مشخصات شعب 28 اردیبهشت 1405.xlsx", + "why": "Branch directory: a merged title row above a two-row-tall merged header, and a vertically merged province column spanning each province's branches." + } +} diff --git a/tests/support/documents.py b/tests/support/documents.py new file mode 100644 index 0000000..c5e6654 --- /dev/null +++ b/tests/support/documents.py @@ -0,0 +1,52 @@ +"""Load real production documents as test fixtures. + +Real files rather than generated ones: a fixture built with the same +understanding of the format that produced the parser cannot catch a wrong +mental model -- it is wrong in the same way, so the test passes and production +breaks. Every table shape this parser handles was found by reading these +files, not by reasoning about what DOCX can contain. + +The documents live outside the repository so no customer-facing file enters +git history. Point `TEST_DOCUMENTS_DIR` at the directory holding them; tests +that need one skip when it is absent. + +These skips cover a missing *external input*, never a failing assertion -- but +a machine without the directory does run a smaller suite than CI should. + +Filenames live in `documents.json` rather than in this module: they are mostly +Persian, and Arabic letterforms in Python source are flagged as ambiguous with +Latin lookalikes (RUF001). They are data, so they belong in a data file. +""" + +import json +import os +from pathlib import Path + +import pytest + +DOCUMENTS_DIR = Path( + os.environ.get("TEST_DOCUMENTS_DIR", "~/Documents/asia_data_sample") +).expanduser() + +_MANIFEST = json.loads((Path(__file__).parent / "documents.json").read_text("utf-8")) + +# Each entry is the only sample exercising some structure; `why` in the +# manifest records what would go untested without it. +PROSE_DOCX = _MANIFEST["prose_docx"]["filename"] +LIST_DOCX = _MANIFEST["list_docx"]["filename"] +TABLE_DOCX = _MANIFEST["table_docx"]["filename"] +MIXED_DOCX = _MANIFEST["mixed_docx"]["filename"] +LAYOUT_DOCX = _MANIFEST["layout_docx"]["filename"] +QA_XLSX = _MANIFEST["qa_xlsx"]["filename"] +BRANCHES_XLSX = _MANIFEST["branches_xlsx"]["filename"] + + +def load_document(filename: str) -> bytes: + """Return a real document's bytes, skipping the test when it is unavailable.""" + path = DOCUMENTS_DIR / filename + if not path.is_file(): + pytest.skip( + f"{filename} not found in {DOCUMENTS_DIR}; " + f"set TEST_DOCUMENTS_DIR to the directory holding the sample documents" + ) + return path.read_bytes() diff --git a/tests/unit/application/test_chunking.py b/tests/unit/application/test_chunking.py new file mode 100644 index 0000000..5c5a504 --- /dev/null +++ b/tests/unit/application/test_chunking.py @@ -0,0 +1,270 @@ +"""Fixed-size chunking, chunk identity, and neighbor linking (ADR-0001, ADR-0018).""" + +import uuid + +import pytest +from pydantic import ValidationError + +from src.application.ingestion import ( + ContentType, + ParsedDocument, + StructuralUnit, + chunk_document, + chunk_id_for, + count_tokens, + split_by_tokens, +) +from src.config import ChunkingSettings + +pytestmark = pytest.mark.unit + +ENCODING = "cl100k_base" + +FILE_ID = uuid.UUID("11111111-1111-1111-1111-111111111111") +OTHER_FILE_ID = uuid.UUID("22222222-2222-2222-2222-222222222222") + + +@pytest.fixture +def settings() -> ChunkingSettings: + return ChunkingSettings() + + +def _long_text(tokens: int) -> str: + """Text of roughly `tokens` cl100k tokens: one common word per token.""" + return " ".join(["word"] * tokens) + + +def _rows(rows: list[str]) -> ParsedDocument: + """A parsed spreadsheet: every row an atomic table_row unit.""" + return ParsedDocument( + units=[StructuralUnit(text=row, content_type=ContentType.TABLE_ROW) for row in rows], + block_count=len(rows), + ) + + +def _prose(text: str) -> ParsedDocument: + """A parsed document consisting of one flowing prose block.""" + return ParsedDocument( + units=[StructuralUnit(text=text, content_type=ContentType.PARAGRAPH)], + block_count=1, + ) + + +# ── split_by_tokens ─────────────────────────────────────────────── + + +def test_split_by_tokens_text_under_limit_returns_single_chunk() -> None: + text = _long_text(50) + + assert split_by_tokens(text, chunk_size=400, overlap=60, encoding_name=ENCODING) == [text] + + +def test_split_by_tokens_long_text_splits_into_multiple_windows() -> None: + windows = split_by_tokens(_long_text(1000), chunk_size=400, overlap=60, encoding_name=ENCODING) + + assert len(windows) > 1 + assert all(count_tokens(window, ENCODING) <= 400 for window in windows) + + +def test_split_by_tokens_consecutive_windows_share_overlapping_tokens() -> None: + windows = split_by_tokens( + " ".join(str(number) for number in range(1000)), + chunk_size=100, + overlap=20, + encoding_name=ENCODING, + ) + + # The tail of one window must reappear at the head of the next, or a + # sentence cut at the boundary is unrecoverable. + tail_tokens = windows[0].split()[-5:] + assert " ".join(tail_tokens) in windows[1] + + +def test_split_by_tokens_overlap_at_least_chunk_size_raises() -> None: + """Guards an infinite loop: `start = end - overlap` never advances.""" + with pytest.raises(ValueError, match="smaller than chunk_size"): + split_by_tokens("text", chunk_size=100, overlap=100, encoding_name=ENCODING) + + +# ── chunk identity ──────────────────────────────────────────────── + + +def test_chunk_id_for_is_deterministic_across_calls() -> None: + assert chunk_id_for(FILE_ID, 7) == chunk_id_for(FILE_ID, 7) + + +def test_chunk_id_for_differs_across_files_and_indices() -> None: + assert chunk_id_for(FILE_ID, 0) != chunk_id_for(OTHER_FILE_ID, 0) + assert chunk_id_for(FILE_ID, 0) != chunk_id_for(FILE_ID, 1) + + +def test_chunk_document_identical_content_in_different_files_gets_different_ids( + settings: ChunkingSettings, +) -> None: + """The evaluation repo derived ids from the filename with non-ASCII stripped, + so every Farsi filename collapsed to underscores and collided.""" + parsed = _rows(["a: 1"]) + + first = chunk_document(parsed, file_id=FILE_ID, settings=settings) + second = chunk_document(parsed, file_id=OTHER_FILE_ID, settings=settings) + + assert first[0].chunk_id != second[0].chunk_id + + +def test_chunk_document_reingesting_same_file_yields_same_ids( + settings: ChunkingSettings, +) -> None: + """Deterministic ids are what make a re-upload upsert instead of duplicate.""" + parsed = _rows(["a: 1", "a: 2"]) + + first = chunk_document(parsed, file_id=FILE_ID, settings=settings) + second = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert [chunk.chunk_id for chunk in first] == [chunk.chunk_id for chunk in second] + + +# ── ordering and neighbor links ─────────────────────────────────── + + +def test_chunk_document_assigns_contiguous_indices_and_one_based_order( + settings: ChunkingSettings, +) -> None: + parsed = _rows(["a: 1", "a: 2", "a: 3"]) + + chunks = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert [chunk.chunk_index for chunk in chunks] == [0, 1, 2] + assert [chunk.order_id for chunk in chunks] == [1.0, 2.0, 3.0] + + +def test_chunk_document_blank_rows_do_not_leave_index_gaps( + settings: ChunkingSettings, +) -> None: + """An index gap breaks the previous/next chain retrieval expansion walks.""" + parsed = _rows(["a: 1", " ", "a: 2"]) + + chunks = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert [chunk.chunk_index for chunk in chunks] == [0, 1] + + +def test_chunk_document_links_neighbors_with_null_at_both_ends( + settings: ChunkingSettings, +) -> None: + parsed = _rows(["a: 1", "a: 2", "a: 3"]) + + chunks = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert chunks[0].previous_chunk_id is None + assert chunks[-1].next_chunk_id is None + for position, chunk in enumerate(chunks[:-1]): + assert chunk.next_chunk_id == chunks[position + 1].chunk_id + assert chunks[position + 1].previous_chunk_id == chunk.chunk_id + + +def test_chunk_document_single_chunk_has_no_neighbors(settings: ChunkingSettings) -> None: + parsed = _rows(["a: 1"]) + + (chunk,) = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert chunk.previous_chunk_id is None + assert chunk.next_chunk_id is None + + +# ── content types and the token cap ─────────────────────────────── + + +def test_chunk_document_spreadsheet_row_becomes_one_atomic_chunk( + settings: ChunkingSettings, +) -> None: + rows = ["branch: one\ncity: two", "branch: three\ncity: four"] + parsed = _rows(rows) + + chunks = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert [chunk.content for chunk in chunks] == rows + assert all(chunk.content_type is ContentType.TABLE_ROW for chunk in chunks) + + +def test_chunk_document_oversized_row_splits_rather_than_truncating( + settings: ChunkingSettings, +) -> None: + """Silent truncation is the failure mode: the model drops the tail, not raises.""" + oversized = _long_text(settings.max_chunk_tokens * 2) + parsed = _rows([oversized]) + + chunks = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert len(chunks) > 1 + assert all(chunk.token_count <= settings.max_chunk_tokens for chunk in chunks) + + +def test_chunk_document_markdown_produces_paragraph_chunks( + settings: ChunkingSettings, +) -> None: + parsed = _prose("# Heading\n\nBody text.") + + chunks = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert all(chunk.content_type is ContentType.PARAGRAPH for chunk in chunks) + + +def test_chunk_document_no_chunk_exceeds_the_model_sequence_length( + settings: ChunkingSettings, +) -> None: + parsed = _prose(_long_text(5000)) + + chunks = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert chunks + assert all(chunk.token_count <= settings.max_chunk_tokens for chunk in chunks) + + +def test_chunk_document_persian_text_never_exceeds_the_cap( + settings: ChunkingSettings, +) -> None: + """Persian is the case where the cap can actually be breached. + + Slicing a token list can cut a multi-byte character in half; decoding then + re-encoding that window does not always round-trip to the same token count. + `ChunkTooLargeError` is the guard for exactly this, and it must not fire. + """ + persian_word = "".join(chr(code) for code in (0x0628, 0x06CC, 0x0645, 0x0647)) + parsed = _prose(" ".join([persian_word] * 4000)) + + chunks = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert len(chunks) > 1 + assert all(chunk.token_count <= settings.max_chunk_tokens for chunk in chunks) + + +def test_chunk_document_records_token_and_character_counts( + settings: ChunkingSettings, +) -> None: + parsed = _rows(["branch: one"]) + + (chunk,) = chunk_document(parsed, file_id=FILE_ID, settings=settings) + + assert chunk.character_count == len("branch: one") + assert chunk.token_count == count_tokens("branch: one", ENCODING) + + +# ── settings validation ─────────────────────────────────────────── + + +def test_chunking_settings_overlap_not_smaller_than_size_is_rejected() -> None: + with pytest.raises(ValidationError, match="must be smaller than"): + ChunkingSettings(chunk_size=100, chunk_overlap=100) + + +def test_chunking_settings_size_over_model_limit_is_rejected() -> None: + with pytest.raises(ValidationError, match="must not exceed"): + ChunkingSettings(chunk_size=600, max_chunk_tokens=512) + + +def test_chunking_settings_defaults_match_adr_0018() -> None: + settings = ChunkingSettings() + + assert (settings.chunk_size, settings.chunk_overlap) == (400, 60) + assert settings.max_chunk_tokens == 512 + assert settings.strategy == "fixed_size" diff --git a/tests/unit/application/test_docx_parser.py b/tests/unit/application/test_docx_parser.py new file mode 100644 index 0000000..a9f7df4 --- /dev/null +++ b/tests/unit/application/test_docx_parser.py @@ -0,0 +1,225 @@ +"""DOCX parsing against the real sample corpus (ADR-0004, ADR-0018). + +Every assertion here was written after reading the document it covers, so the +numbers are observations rather than predictions. +""" + +import pytest + +from src.application.ingestion import ContentType, DocumentParseError, parse_docx +from src.application.ingestion.docx_parser import heading_level_from_style +from src.config import ChunkingSettings +from tests.support.documents import ( + LAYOUT_DOCX, + LIST_DOCX, + MIXED_DOCX, + PROSE_DOCX, + TABLE_DOCX, + load_document, +) + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def settings() -> ChunkingSettings: + return ChunkingSettings() + + +def _units(filename: str, settings: ChunkingSettings): + return parse_docx(load_document(filename), settings).units + + +def _types(units) -> set[ContentType]: + return {unit.content_type for unit in units} + + +# ── heading styles ──────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("style", "expected"), + [ + ("Heading1", 1), # the styleId Word actually stores + ("Heading 1", 1), # the friendly name python-docx reports + ("heading 2", 2), + ("HEADING3", 3), + ("Normal", None), + ("List Paragraph", None), + ("Heading", None), + ("Headingfoo", None), + ], +) +def test_heading_level_from_style_maps_word_styles_only(style: str, expected: int | None) -> None: + assert heading_level_from_style(style) == expected + + +def test_parse_docx_invents_no_headings_when_document_declares_none( + settings: ChunkingSettings, +) -> None: + """Every document in the corpus lacks Heading styles, so this is the norm. + + The evaluation repo upgraded paragraphs to headings by text pattern; a + wrongly-detected heading silently reshapes the tree, so ADR-0018 drops it. + """ + parsed = parse_docx(load_document(PROSE_DOCX), settings) + + assert "#" not in (parsed.markdown or "") + + +# ── prose documents ─────────────────────────────────────────────── + + +def test_parse_docx_prose_document_becomes_one_flowing_block( + settings: ChunkingSettings, +) -> None: + """Consecutive paragraphs accumulate into one prose unit (ADR-0004). + + One unit per paragraph would hand the splitter 36 fragments instead of + flowing text. + """ + units = _units(PROSE_DOCX, settings) + + assert len(units) == 1 + assert units[0].content_type is ContentType.PARAGRAPH + assert units[0].text.count("\n\n") == 35 + + +def test_parse_docx_list_styled_paragraphs_stay_prose(settings: ChunkingSettings) -> None: + units = _units(LIST_DOCX, settings) + + assert _types(units) == {ContentType.PARAGRAPH} + + +def test_parse_docx_normalizes_persian_letterforms(settings: ChunkingSettings) -> None: + parsed = parse_docx(load_document(PROSE_DOCX), settings) + + # No Arabic kaf or yeh survives normalization. + assert chr(0x0643) not in (parsed.markdown or "") + assert chr(0x064A) not in (parsed.markdown or "") + + +def test_parse_docx_preserves_paragraph_breaks_in_markdown( + settings: ChunkingSettings, +) -> None: + """Normalization collapses whitespace, so it must run per block. + + Applied to the assembled document it would flatten every paragraph onto a + single line -- the easiest way to get this wrong. + """ + parsed = parse_docx(load_document(PROSE_DOCX), settings) + + assert "\n\n" in (parsed.markdown or "") + + +# ── data tables ─────────────────────────────────────────────────── + + +def test_parse_docx_data_table_becomes_one_unit_per_row(settings: ChunkingSettings) -> None: + """A 35-row contact table: one header row plus 34 data rows.""" + units = _units(TABLE_DOCX, settings) + + assert len(units) == 34 + assert _types(units) == {ContentType.TABLE_ROW} + + +def test_parse_docx_data_table_rows_are_labeled_with_headers( + settings: ChunkingSettings, +) -> None: + units = _units(TABLE_DOCX, settings) + + assert "\n" in units[0].text + assert all(":" in line for line in units[0].text.split("\n")) + + +def test_parse_docx_vertically_merged_cell_repeats_on_every_row( + settings: ChunkingSettings, +) -> None: + """The group column is merged down several rows; each row must carry it.""" + units = _units(TABLE_DOCX, settings) + + labels = [line.split(":")[0] for unit in units for line in unit.text.split("\n")] + group_label = units[1].text.split(":")[0] + + assert labels.count(group_label) > 1 + + +def test_parse_docx_empty_cell_is_omitted_not_rendered_as_bare_label( + settings: ChunkingSettings, +) -> None: + """The first data row has no group; it should have one fewer field.""" + units = _units(TABLE_DOCX, settings) + + assert len(units[0].text.split("\n")) < len(units[1].text.split("\n")) + + +# ── tables without a header ─────────────────────────────────────── + + +def test_parse_docx_headerless_table_keeps_every_row(settings: ChunkingSettings) -> None: + """A 30-row compensation list plus an 11-row table, all rows preserved. + + Regression: scanning forward for a header row discarded the rows above the + match and labeled the rest from a data row (`80: 70`). + """ + units = _units(MIXED_DOCX, settings) + rows = [unit for unit in units if unit.content_type is ContentType.TABLE_ROW] + + assert len(rows) == 41 + + +def test_parse_docx_headerless_table_rows_are_unlabeled(settings: ChunkingSettings) -> None: + """No header is provable, so cells are joined rather than mislabeled.""" + units = _units(MIXED_DOCX, settings) + first_row = next(unit for unit in units if unit.content_type is ContentType.TABLE_ROW) + + assert " | " in first_row.text + assert first_row.text.startswith("1 | ") + + +def test_parse_docx_mixed_document_interleaves_prose_and_rows( + settings: ChunkingSettings, +) -> None: + units = _units(MIXED_DOCX, settings) + + assert _types(units) == {ContentType.PARAGRAPH, ContentType.TABLE_ROW} + + +# ── layout tables ───────────────────────────────────────────────── + + +def test_parse_docx_extracts_document_held_inside_a_table_cell( + settings: ChunkingSettings, +) -> None: + """One merged cell holds 77k characters across 738 paragraphs. + + Regression: a merged cell reported once per spanned grid position was + deduplicated by `id()`, which lxml reuses across element proxies, so the + whole sub-document vanished and the file yielded 70 tokens. + """ + units = _units(LAYOUT_DOCX, settings) + + assert sum(len(unit.text) for unit in units) > 70_000 + + +def test_parse_docx_layout_table_cells_become_prose_not_rows( + settings: ChunkingSettings, +) -> None: + """A cell that alone exceeds chunk_size holds a document, not a field.""" + units = _units(LAYOUT_DOCX, settings) + + assert ContentType.PARAGRAPH in _types(units) + + +# ── failure modes ───────────────────────────────────────────────── + + +def test_parse_docx_unopenable_bytes_raises_parse_error(settings: ChunkingSettings) -> None: + with pytest.raises(DocumentParseError, match="Could not open DOCX"): + parse_docx(b"not a docx at all", settings) + + +def test_parse_docx_legacy_doc_binary_raises_parse_error(settings: ChunkingSettings) -> None: + """`.doc` is rejected rather than converted (ADR-0018); python-docx cannot open it.""" + with pytest.raises(DocumentParseError): + parse_docx(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 512, settings) diff --git a/tests/unit/application/test_normalization.py b/tests/unit/application/test_normalization.py new file mode 100644 index 0000000..47b1fdc --- /dev/null +++ b/tests/unit/application/test_normalization.py @@ -0,0 +1,95 @@ +"""Persian normalization (ADR-0018). + +Characters are written as escapes rather than literals: Arabic letterforms are +visually indistinguishable in a monospace editor, and literals would render +right-to-left and visually reorder each assertion. +""" + +import pytest + +from src.application.ingestion import normalize_persian_text + +pytestmark = pytest.mark.unit + +ARABIC_KAF = chr(0x0643) +PERSIAN_KEHEH = chr(0x06A9) +ARABIC_YEH = chr(0x064A) +ALEF_MAKSURA = chr(0x0649) +PERSIAN_YEH = chr(0x06CC) +ALEF_HAMZA_ABOVE = chr(0x0623) +ALEF_HAMZA_BELOW = chr(0x0625) +ALEF = chr(0x0627) +ALEF_FINAL_FORM = chr(0xFE8E) + +FATHA = chr(0x064E) +SHADDA = chr(0x0651) +TATWEEL = chr(0x0640) +SUPERSCRIPT_ALEF = chr(0x0670) + +PERSIAN_DIGITS = "".join(chr(0x06F1 + offset) for offset in range(3)) +ARABIC_SEMICOLON = chr(0x061B) +ARABIC_THOUSANDS_SEPARATOR = chr(0x066C) + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + (ARABIC_KAF, PERSIAN_KEHEH), + (ARABIC_YEH, PERSIAN_YEH), + (ALEF_MAKSURA, PERSIAN_YEH), + (ALEF_HAMZA_ABOVE, ALEF), + (ALEF_HAMZA_BELOW, ALEF), + ], +) +def test_normalize_arabic_letterform_folds_to_persian(source: str, expected: str) -> None: + assert normalize_persian_text(source) == expected + + +def test_normalize_mixed_keyboard_spellings_converge_to_one_form() -> None: + """The defect this module exists for: one word, two spellings, two embeddings.""" + arabic_spelling = f"{ARABIC_KAF}{ARABIC_YEH}" + persian_spelling = f"{PERSIAN_KEHEH}{PERSIAN_YEH}" + + assert normalize_persian_text(arabic_spelling) == normalize_persian_text(persian_spelling) + + +@pytest.mark.parametrize("mark", [FATHA, SHADDA, TATWEEL, SUPERSCRIPT_ALEF]) +def test_normalize_diacritic_is_removed(mark: str) -> None: + assert normalize_persian_text(f"{ALEF}{mark}{ALEF}") == f"{ALEF}{ALEF}" + + +def test_normalize_not_sign_becomes_space() -> None: + assert normalize_persian_text(f"{ALEF}¬{ALEF}") == f"{ALEF} {ALEF}" + + +def test_normalize_applies_nfkc_compatibility_forms() -> None: + # U+FE8E is the final presentation form of alef; NFKC maps it to the base + # letter, so a document pasted from a PDF renderer matches a typed one. + assert normalize_persian_text(ALEF_FINAL_FORM) == ALEF + + +def test_normalize_collapses_whitespace_runs_and_strips() -> None: + assert normalize_persian_text(" a \t\n b ") == "a b" + + +@pytest.mark.parametrize( + "preserved", + [PERSIAN_DIGITS, ARABIC_SEMICOLON, ARABIC_THOUSANDS_SEPARATOR], +) +def test_normalize_digits_and_punctuation_survive_unchanged(preserved: str) -> None: + """ADR-0018 deliberately folds letters only. + + Rewriting Persian digits to Western ones would render citations wrong to a + Persian reader, so this guards against a later 'helpful' addition. + """ + assert normalize_persian_text(preserved) == preserved + + +def test_normalize_already_normalized_text_is_unchanged() -> None: + normalized = normalize_persian_text(f"{ARABIC_KAF}{FATHA} {ALEF_HAMZA_ABOVE}") + + assert normalize_persian_text(normalized) == normalized + + +def test_normalize_empty_string_returns_empty() -> None: + assert normalize_persian_text("") == "" diff --git a/tests/unit/application/test_spreadsheet_parser.py b/tests/unit/application/test_spreadsheet_parser.py new file mode 100644 index 0000000..69d6aa0 --- /dev/null +++ b/tests/unit/application/test_spreadsheet_parser.py @@ -0,0 +1,183 @@ +"""CSV and XLSX parsing against the real sample corpus (ADR-0004, ADR-0018).""" + +import pytest + +from src.application.ingestion import ContentType, DocumentParseError, parse_csv, parse_xlsx +from tests.support.documents import BRANCHES_XLSX, QA_XLSX, load_document + +pytestmark = pytest.mark.unit + + +def _field(text: str, label: str) -> str | None: + for line in text.split("\n"): + name, _, value = line.partition(":") + if name == label: + return value.strip() + return None + + +# ── the two real sheet shapes ───────────────────────────────────── + + +def test_parse_xlsx_qa_sheet_renders_question_and_answer_labels() -> None: + """A two-column q/a sheet needs no special case; the generic renderer serves it.""" + parsed = parse_xlsx(load_document(QA_XLSX)) + + assert parsed.units[0].content_type is ContentType.TABLE_ROW + assert _field(parsed.units[0].text, "q") + assert _field(parsed.units[0].text, "a") + + +def test_parse_xlsx_branch_sheet_renders_every_column_with_its_header() -> None: + """The same renderer, a completely different schema, no shape detection.""" + parsed = parse_xlsx(load_document(BRANCHES_XLSX)) + + first = parsed.units[0].text + assert len(first.split("\n")) > 5 + assert all(":" in line for line in first.split("\n")) + + +def test_parse_xlsx_dead_second_sheet_is_skipped() -> None: + """The q/a workbook carries an empty Sheet2 seen throughout the corpus.""" + parsed = parse_xlsx(load_document(QA_XLSX)) + + assert parsed.block_count == 150 + + +# ── merged cells ────────────────────────────────────────────────── + + +def test_parse_xlsx_merged_title_row_is_not_treated_as_header() -> None: + """A merged banner spans the sheet; the real header sits below it.""" + parsed = parse_xlsx(load_document(BRANCHES_XLSX)) + + assert _field(parsed.units[0].text, "استان") is not None + + +def test_parse_xlsx_two_row_tall_merged_header_is_not_emitted_as_data() -> None: + """Regression: forward fill duplicates the header into the row beneath it. + + Without skipping that duplicate the first chunk was the header labeling + itself (`استان: استان`). + """ + parsed = parse_xlsx(load_document(BRANCHES_XLSX)) + + assert _field(parsed.units[0].text, "استان") != "استان" + + +def test_parse_xlsx_vertically_merged_column_fills_every_row_in_its_range() -> None: + """openpyxl stores a merged range's value only in its top-left cell. + + The province is merged across each of its branches, so without forward + filling every branch but the first loses the field entirely. + """ + parsed = parse_xlsx(load_document(BRANCHES_XLSX)) + + provinces = [_field(unit.text, "استان") for unit in parsed.units] + + assert all(province for province in provinces) + assert len(set(provinces)) == 31 + + +def test_parse_xlsx_branch_rows_are_self_contained() -> None: + """Each branch is its own chunk carrying its province, not just its name.""" + parsed = parse_xlsx(load_document(BRANCHES_XLSX)) + + second = parsed.units[1].text + + assert _field(second, "استان") == _field(parsed.units[0].text, "استان") + assert _field(second, "شعبه") != _field(parsed.units[0].text, "شعبه") + + +# ── csv ─────────────────────────────────────────────────────────── + + +def test_parse_csv_renders_rows_with_headers() -> None: + data = b"branch,city,phone\nvali asr,tehran,021\nazadi,karaj,026\n" + + parsed = parse_csv(data) + + assert parsed.block_count == 2 + assert _field(parsed.units[0].text, "branch") == "vali asr" + assert _field(parsed.units[1].text, "city") == "karaj" + + +def test_parse_csv_sniffs_semicolon_delimiter() -> None: + data = b"branch;city;phone\nvali asr;tehran;021\nazadi;karaj;026\n" + + parsed = parse_csv(data) + + assert _field(parsed.units[0].text, "city") == "tehran" + + +def test_parse_csv_unprovable_header_falls_back_to_joined_cells() -> None: + """Short text over short text is genuinely ambiguous, so nothing is labeled. + + The cost of this rule is unlabeled rows; the cost of guessing is labeling + every row from a data row, which is how a headerless compensation table + became `80: 70`. Losing a label is recoverable, mislabeling is not. + """ + parsed = parse_csv(b"branch,city\nvali asr,tehran\nazadi,karaj\n") + + assert parsed.units[0].text == "branch | city" + assert parsed.block_count == 3 + + +def test_parse_csv_empty_cells_are_omitted() -> None: + data = b"branch,city,phone\nvali asr,,021\n" + + parsed = parse_csv(data) + + assert _field(parsed.units[0].text, "city") is None + assert _field(parsed.units[0].text, "phone") == "021" + + +def test_parse_csv_blank_rows_are_skipped() -> None: + data = b"branch,city,phone\nvali asr,tehran,021\n\n,,\nazadi,karaj,026\n" + + parsed = parse_csv(data) + + assert parsed.block_count == 2 + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig", "cp1256"]) +def test_parse_csv_decodes_farsi_in_each_supported_encoding(encoding: str) -> None: + """cp1256 is common for Farsi exported by older Excel.""" + tehran = "تهران" + parsed = parse_csv(f"branch,city,phone\nvali asr,{tehran},021\n".encode(encoding)) + + assert _field(parsed.units[0].text, "city") == tehran + + +def test_parse_csv_normalizes_persian_letterforms() -> None: + arabic_yeh, persian_yeh = chr(0x064A), chr(0x06CC) + + parsed = parse_csv(f"branch,city,phone\nvali asr,{arabic_yeh},021\n".encode()) + + assert _field(parsed.units[0].text, "city") == persian_yeh + + +# ── failure modes ───────────────────────────────────────────────── + + +def test_parse_csv_cp1256_fallback_always_decodes_rather_than_raising() -> None: + """cp1256 is single-byte, so it maps every byte and never fails. + + That makes the "could not decode" branch unreachable in practice: bytes + that are not valid UTF-8 come back as mojibake instead of an error. The + guard stays as defence, but this documents the real behaviour so nobody + relies on a decode failure to reject a bad upload. + """ + parsed = parse_csv(b"branch,city,phone\nvali asr,\xff\xfe,021\n") + + assert parsed.block_count == 1 + + +def test_parse_csv_empty_file_raises_parse_error() -> None: + with pytest.raises(DocumentParseError, match="no rows"): + parse_csv(b"") + + +def test_parse_xlsx_unopenable_bytes_raise_parse_error() -> None: + with pytest.raises(DocumentParseError, match="Could not open XLSX"): + parse_xlsx(b"not a workbook") diff --git a/uv.lock b/uv.lock index bebfb42..3a1c4ef 100644 --- a/uv.lock +++ b/uv.lock @@ -426,6 +426,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "fastapi" version = "0.141.1" @@ -898,6 +907,68 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/e6/e264e87ce79467aa0cc6c5e945e74050d7880394da325b3559cfdb41a186/langsmith-0.10.17-py3-none-any.whl", hash = "sha256:2a1242a0500147ed846ecbd5fe2f41c21b1652f6833b1af0f209231006f67e5e", size = 734484, upload-time = "2026-08-07T17:23:02.598Z" }, ] +[[package]] +name = "lxml" +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/eb/7e6f37c5584ccbb2ff267f56fd0339016938c1c8684cfefab9b33ffc2f36/lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736", size = 8559780, upload-time = "2026-05-18T19:17:57.661Z" }, + { url = "https://files.pythonhosted.org/packages/a1/36/587c2521cf23a2cd6c9c22108aa7528f683a1f195ed7ccd23a4b1786ad36/lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9", size = 4618006, upload-time = "2026-05-18T19:18:04.452Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" }, + { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" }, + { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" }, + { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" }, + { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" }, + { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" }, + { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f2/1a2b9f1b7a49d45495369be7ef9ad05b262930f2eab3e3145706fca8083f/lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca", size = 3596903, upload-time = "2026-05-18T19:17:29.863Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/f4ffb024f238eec2131aaa09f3278fb6129cf892741bf68e1fc1afb8c100/lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660", size = 3995869, upload-time = "2026-05-18T19:18:02.596Z" }, + { url = "https://files.pythonhosted.org/packages/d1/53/70eb8c5c6037f27448f1e3c54ebede9545a801ae63f0a7254afca4fe8e45/lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc", size = 3658490, upload-time = "2026-05-19T19:22:53.846Z" }, + { url = "https://files.pythonhosted.org/packages/13/e2/2e325795566de01d0d7c3bb57d3c370616b2d07b01214e84eec5d3b10963/lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0", size = 8577146, upload-time = "2026-05-18T19:18:17.765Z" }, + { url = "https://files.pythonhosted.org/packages/93/cf/5630b5e4be7d2e6bee8efe83865c925221103cf0221303b104ce134b01e2/lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840", size = 4623866, upload-time = "2026-05-18T19:18:30.669Z" }, + { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" }, + { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" }, + { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" }, + { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" }, + { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" }, + { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" }, + { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" }, + { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" }, + { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" }, + { url = "https://files.pythonhosted.org/packages/d6/85/be36fb1425b30db3c3f9df75fe86343ebffb79e6320bd7f588e25bfeac39/lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603", size = 3657202, upload-time = "2026-05-18T19:17:39.509Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ce/3cf9a827342269f54d405a6202397de63f07c69cbd6ce7d183a3f0cba1e9/lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137", size = 4064497, upload-time = "2026-05-18T19:18:14.662Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3e/1a957bde8f0760039e627f94699f82caa782c9d838d86c3d28245ee67212/lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf", size = 3741991, upload-time = "2026-05-19T19:22:59.111Z" }, + { url = "https://files.pythonhosted.org/packages/78/b2/00ed55b3a2efa4658fb795c38d1090ec9b3e8a6c3683d4441fa517f09c3b/lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee", size = 8827545, upload-time = "2026-05-18T19:18:41.193Z" }, + { url = "https://files.pythonhosted.org/packages/c0/73/74573db19baa618d5f266f2407898b087ff6927115b00b71e5fc1b700847/lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c", size = 4735736, upload-time = "2026-05-18T19:18:46.761Z" }, + { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" }, + { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" }, + { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" }, + { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" }, + { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" }, + { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" }, + { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" }, + { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/ce/68/b30e913340c380ddac9580c6e6230991fc37240ec4f64704833e4f3e2769/lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1", size = 3897345, upload-time = "2026-05-18T19:17:33.562Z" }, + { url = "https://files.pythonhosted.org/packages/3c/4e/9eb2af5335545f9fbcd7af57bcf87c6025d31eaa31b14ec184a6c8675328/lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e", size = 4393350, upload-time = "2026-05-18T19:18:10.076Z" }, + { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" }, +] + [[package]] name = "mako" version = "1.4.1" @@ -1010,10 +1081,13 @@ dependencies = [ { name = "fastapi", extra = ["standard"] }, { name = "langgraph" }, { name = "minio" }, + { name = "openpyxl" }, { name = "pydantic-settings" }, + { name = "python-docx" }, { name = "qdrant-client" }, { name = "sqlalchemy" }, { name = "structlog" }, + { name = "tiktoken" }, ] [package.dev-dependencies] @@ -1037,10 +1111,13 @@ requires-dist = [ { name = "fastapi", extras = ["standard"], specifier = "==0.141.1" }, { name = "langgraph", specifier = ">=1.2.10" }, { name = "minio", specifier = ">=7.2.20" }, + { name = "openpyxl", specifier = ">=3.1.5" }, { name = "pydantic-settings", specifier = ">=2.15.0" }, + { name = "python-docx", specifier = ">=1.2.0" }, { name = "qdrant-client", specifier = ">=1.19.0" }, { name = "sqlalchemy", specifier = ">=2.0.51" }, { name = "structlog", specifier = ">=26.1.0" }, + { name = "tiktoken", specifier = ">=0.13.0" }, ] [package.metadata.requires-dev] @@ -1118,6 +1195,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + [[package]] name = "orjson" version = "3.11.9" @@ -1436,6 +1525,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "python-docx" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/f7/eddfe33871520adab45aaa1a71f0402a2252050c14c7e3009446c8f4701c/python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce", size = 5723256, upload-time = "2025-06-16T20:46:27.921Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/00/1e03a4989fa5795da308cd774f05b704ace555a70f9bf9d3be057b680bcf/python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7", size = 252987, upload-time = "2025-06-16T20:46:22.506Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2" @@ -1524,6 +1626,78 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/3c/480c61cc8d5a3e76bb44e86231f408c93643e5498beadbbeb381ab55d02b/qdrant_client-1.19.0-py3-none-any.whl", hash = "sha256:13602a2b3478a95ecdf42f97b93d7f703b63a3361cd912a04495a33a5ac14121", size = 396157, upload-time = "2026-08-04T14:32:55.734Z" }, ] +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1802,6 +1976,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/7e/424aac8b355597835deb333e757a0e94b5ccf38ad00f07fe6ed1f4e17c88/testcontainers-4.15.0-py3-none-any.whl", hash = "sha256:8796c14e76604031ad39cf0ed3b8e9806283a1fbf5270965c2b1c594caa31b74", size = 160771, upload-time = "2026-07-24T23:08:00.13Z" }, ] +[[package]] +name = "tiktoken" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/83/b096c859c2a47c11731bf2f5885f4028b809dfe2396582883eed9cae372f/tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154", size = 1034228, upload-time = "2026-05-15T04:50:40.988Z" }, + { url = "https://files.pythonhosted.org/packages/53/61/c68e123b6d753e3fc2751e9b18e732c9d8bf1e1926762e736eee935d931c/tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545", size = 982978, upload-time = "2026-05-15T04:50:42.195Z" }, + { url = "https://files.pythonhosted.org/packages/ef/8b/96cc178cc584e65d363134500f297790b06cd48cdeb1e8fcf7bbe60f4715/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2", size = 1116355, upload-time = "2026-05-15T04:50:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/86/f5/bab735d2c72ea55404b295d02d092644eb5f7cc6205e34d35eb9abfb9ab2/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf", size = 1135772, upload-time = "2026-05-15T04:50:44.782Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b9/6de04ebdf904edfaad87788011b3735087a0c9ea671b9027e1e4e965e8c8/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486", size = 1182415, upload-time = "2026-05-15T04:50:46.422Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9c/470a05f3b1caf038f44880e334d47ab674e0c80d514c66b375d14d5afa10/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615", size = 1239879, upload-time = "2026-05-15T04:50:48.052Z" }, + { url = "https://files.pythonhosted.org/packages/42/a6/c1936d16055436cb32e6c6128d68629622e00f4768562f55653752d34768/tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7", size = 874829, upload-time = "2026-05-15T04:50:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/d6/07/acb5992c3772b5a36284f742cfb7a5895aa4471d1848ac31464ad50d7fdf/tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67", size = 1033600, upload-time = "2026-05-15T04:50:50.4Z" }, + { url = "https://files.pythonhosted.org/packages/14/e9/742e9aec30f59b9f161f7ff7cd072e02ea836c9e1c0854a8076dfcd40d5c/tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a", size = 982516, upload-time = "2026-05-15T04:50:52.03Z" }, + { url = "https://files.pythonhosted.org/packages/72/74/ca1541b053e7648254d2e4b42a253e1bb4359f2c91a0a8d49228c794e1a0/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d", size = 1115518, upload-time = "2026-05-15T04:50:53.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/e3/93825eaf5a4a504795b787e5d5dea07fbeb3dabf97aa7b450be8bde59c89/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce", size = 1136867, upload-time = "2026-05-15T04:50:55.191Z" }, + { url = "https://files.pythonhosted.org/packages/8c/46/002b68de6827091d5ae90b048f326e8aad8d953520950e5ce1508879414f/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2", size = 1181826, upload-time = "2026-05-15T04:50:56.296Z" }, + { url = "https://files.pythonhosted.org/packages/db/c6/d393e3185a276505182f7abd93fe714f3c444a2be9180798fa052347504e/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f", size = 1239489, upload-time = "2026-05-15T04:50:57.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/4d/bc07d1f1635d4897a202acc0ae11c2886eaa7325c359ba4741b47bf8e225/tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec", size = 873820, upload-time = "2026-05-15T04:50:59.528Z" }, + { url = "https://files.pythonhosted.org/packages/8c/93/0dd6adca026a616c3a92974566b43381eea4b475ce1f36c062b8271a9ac5/tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471", size = 1034977, upload-time = "2026-05-15T04:51:00.957Z" }, + { url = "https://files.pythonhosted.org/packages/d9/77/5ec6e6bc5b30bed6d93f7f2162d8f6b32437b3ba27cb527cfe004f6109c9/tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd", size = 983635, upload-time = "2026-05-15T04:51:02.629Z" }, + { url = "https://files.pythonhosted.org/packages/94/b0/c8ae9aff00d625c50659b4513e707a0462c4bf5d4d6cc1b802103225c02e/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881", size = 1116036, upload-time = "2026-05-15T04:51:04.082Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ac/6a5dddd1d0a6018ecb389bd0353e6b4a515eb4d2286611bd0ace1937b9e1/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24", size = 1135544, upload-time = "2026-05-15T04:51:05.229Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b8/585032b4384b2f7dcdaddcb52865c83a701a420d09e3c2b4a2be1c450c57/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273", size = 1182217, upload-time = "2026-05-15T04:51:06.517Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b6/993ff1ded3958215fd341a847b8e5ffeb5de473f435296870d314fc91ac4/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51", size = 1239404, upload-time = "2026-05-15T04:51:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3d/fef7e06e3b33e7538db0ced734cf9fe23b6832d2ac4990c119c377aec55e/tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58", size = 918686, upload-time = "2026-05-15T04:51:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/c1/82/a7fc44582bc32ab00de988a2299bf77c077f59068b233109e34b7d6ca7e6/tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b", size = 1034454, upload-time = "2026-05-15T04:51:10.035Z" }, + { url = "https://files.pythonhosted.org/packages/37/d0/24d8a890c14f432a05cea669c17bebeaa99f96a7c79523b590f564246411/tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448", size = 982976, upload-time = "2026-05-15T04:51:11.23Z" }, + { url = "https://files.pythonhosted.org/packages/49/b7/2ab43f62788a9266187a9bfc1d3af99ad83e5eaa25fbef168a69cd5ad14f/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a", size = 1115526, upload-time = "2026-05-15T04:51:12.608Z" }, + { url = "https://files.pythonhosted.org/packages/64/39/1494321ed323ce7a14d88e3cd6cb9058625977df1c6961ddc492bd10a9f3/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad", size = 1136466, upload-time = "2026-05-15T04:51:13.926Z" }, + { url = "https://files.pythonhosted.org/packages/96/d9/dfd086aa2d918c563a140720e0ce296cada1634efd2783d5cf51e05f984e/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e", size = 1181863, upload-time = "2026-05-15T04:51:15.025Z" }, + { url = "https://files.pythonhosted.org/packages/2f/68/a18b4f307086954fdae32714cb4f85562e34f9d34ab206e61f1816aa6018/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424", size = 1239218, upload-time = "2026-05-15T04:51:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, +] + [[package]] name = "ty" version = "0.0.69"