Ali Zarinkolah
5cdfb70085
Adds src/application/ingestion/ -- Persian normalization, DOCX body walk with structural data/layout table classification, CSV/XLSX row rendering, and fixed-size token chunking (cl100k_base, 400/60/512) -- as pure functions per ADR-0015, tested against real production documents (asia_data_sample, kept out of the repo). ADR-0018 records where this diverges from ADR-0004 (fixed-size default, no invented headings/tree, structural table classification, header-provable labeling only). Plan 001's scope line is corrected from CSV-only to DOCX/XLSX/CSV, and CLAUDE.md's stale project-status paragraph is updated to match current implementation state.
130 lines
4.6 KiB
Python
130 lines
4.6 KiB
Python
"""Fixed-size chunking with overlap (ADR-0018).
|
|
|
|
DOCX markdown is split on token windows; spreadsheet rows are already atomic
|
|
and bypass the splitter, falling through it only when a single row exceeds the
|
|
embedding model's sequence length.
|
|
"""
|
|
|
|
import uuid
|
|
|
|
from src.application.ingestion.errors import ChunkTooLargeError, DocumentParseError
|
|
from src.application.ingestion.models import Chunk, ContentType, ParsedDocument
|
|
from src.application.ingestion.tokenizer import count_tokens, get_encoder
|
|
from src.config import ChunkingSettings
|
|
|
|
# Fixed namespace so chunk ids stay stable across processes and releases.
|
|
CHUNK_ID_NAMESPACE = uuid.UUID("6f9619ff-8b86-d011-b42d-00c04fc964ff")
|
|
|
|
|
|
def chunk_id_for(file_id: uuid.UUID, chunk_index: int) -> uuid.UUID:
|
|
"""Return the deterministic point id for a chunk (ADR-0001).
|
|
|
|
Derived from `file_id` and the immutable ingestion ordinal, so re-ingesting
|
|
a file upserts its points instead of duplicating them.
|
|
"""
|
|
return uuid.uuid5(CHUNK_ID_NAMESPACE, f"{file_id}:{chunk_index}")
|
|
|
|
|
|
def split_by_tokens(text: str, *, chunk_size: int, overlap: int, encoding_name: str) -> list[str]:
|
|
"""Split text into overlapping windows of at most `chunk_size` tokens."""
|
|
if overlap >= chunk_size:
|
|
raise ValueError(f"overlap ({overlap}) must be smaller than chunk_size ({chunk_size})")
|
|
|
|
encoder = get_encoder(encoding_name)
|
|
tokens = encoder.encode(text)
|
|
if len(tokens) <= chunk_size:
|
|
return [text]
|
|
|
|
windows: list[str] = []
|
|
start = 0
|
|
while start < len(tokens):
|
|
end = min(start + chunk_size, len(tokens))
|
|
windows.append(encoder.decode(tokens[start:end]))
|
|
if end >= len(tokens):
|
|
break
|
|
start = end - overlap
|
|
|
|
return windows
|
|
|
|
|
|
def _split_oversized(text: str, settings: ChunkingSettings) -> list[str]:
|
|
"""Split a row only if it exceeds the cap; otherwise keep it atomic."""
|
|
if count_tokens(text, settings.encoding_name) <= settings.max_chunk_tokens:
|
|
return [text]
|
|
return split_by_tokens(
|
|
text,
|
|
chunk_size=settings.chunk_size,
|
|
overlap=settings.chunk_overlap,
|
|
encoding_name=settings.encoding_name,
|
|
)
|
|
|
|
|
|
def _content_units(
|
|
parsed: ParsedDocument, settings: ChunkingSettings
|
|
) -> list[tuple[str, ContentType]]:
|
|
"""Reduce a parsed document's structural units to ordered text pieces.
|
|
|
|
Prose is cut into token windows; a table row is atomic and survives whole
|
|
unless it alone exceeds the model's sequence length (ADR-0004).
|
|
"""
|
|
if not parsed.units:
|
|
raise DocumentParseError("Parsed document has no structural units")
|
|
|
|
pieces: list[tuple[str, ContentType]] = []
|
|
for unit in parsed.units:
|
|
if unit.content_type is ContentType.PARAGRAPH:
|
|
windows = split_by_tokens(
|
|
unit.text,
|
|
chunk_size=settings.chunk_size,
|
|
overlap=settings.chunk_overlap,
|
|
encoding_name=settings.encoding_name,
|
|
)
|
|
else:
|
|
windows = _split_oversized(unit.text, settings)
|
|
pieces.extend((window, unit.content_type) for window in windows)
|
|
|
|
return pieces
|
|
|
|
|
|
def chunk_document(
|
|
parsed: ParsedDocument,
|
|
*,
|
|
file_id: uuid.UUID,
|
|
settings: ChunkingSettings,
|
|
) -> list[Chunk]:
|
|
"""Turn a parsed document into ordered, neighbor-linked chunks."""
|
|
units = [
|
|
(text.strip(), content_type) for text, content_type in _content_units(parsed, settings)
|
|
]
|
|
# Drop blanks *before* assigning indices: an index gap would break the
|
|
# previous/next chain that retrieval-time expansion walks.
|
|
units = [(text, content_type) for text, content_type in units if text]
|
|
|
|
chunks: list[Chunk] = []
|
|
for index, (text, content_type) in enumerate(units):
|
|
token_count = count_tokens(text, settings.encoding_name)
|
|
if token_count > settings.max_chunk_tokens:
|
|
raise ChunkTooLargeError(
|
|
f"chunk {index} is {token_count} tokens, over the "
|
|
f"{settings.max_chunk_tokens}-token cap"
|
|
)
|
|
chunks.append(
|
|
Chunk(
|
|
chunk_id=chunk_id_for(file_id, index),
|
|
chunk_index=index,
|
|
order_id=float(index + 1),
|
|
content=text,
|
|
content_type=content_type,
|
|
token_count=token_count,
|
|
character_count=len(text),
|
|
)
|
|
)
|
|
|
|
for position, chunk in enumerate(chunks):
|
|
if position > 0:
|
|
chunk.previous_chunk_id = chunks[position - 1].chunk_id
|
|
if position < len(chunks) - 1:
|
|
chunk.next_chunk_id = chunks[position + 1].chunk_id
|
|
|
|
return chunks
|