"""Fixed-size chunking with overlap (ADR-0018). DOCX markdown is split on token windows; spreadsheet rows are already atomic and bypass the splitter, falling through it only when a single row exceeds the embedding model's sequence length. """ import uuid from src.application.ingestion.errors import ChunkTooLargeError, DocumentParseError from src.application.ingestion.models import Chunk, ContentType, ParsedDocument from src.application.ingestion.tokenizer import count_tokens, get_encoder from src.config import ChunkingSettings # Fixed namespace so chunk ids stay stable across processes and releases. CHUNK_ID_NAMESPACE = uuid.UUID("6f9619ff-8b86-d011-b42d-00c04fc964ff") def chunk_id_for(file_id: uuid.UUID, chunk_index: int) -> uuid.UUID: """Return the deterministic point id for a chunk (ADR-0001). Derived from `file_id` and the immutable ingestion ordinal, so re-ingesting a file upserts its points instead of duplicating them. """ return uuid.uuid5(CHUNK_ID_NAMESPACE, f"{file_id}:{chunk_index}") def split_by_tokens(text: str, *, chunk_size: int, overlap: int, encoding_name: str) -> list[str]: """Split text into overlapping windows of at most `chunk_size` tokens.""" if overlap >= chunk_size: raise ValueError(f"overlap ({overlap}) must be smaller than chunk_size ({chunk_size})") encoder = get_encoder(encoding_name) tokens = encoder.encode(text) if len(tokens) <= chunk_size: return [text] windows: list[str] = [] start = 0 while start < len(tokens): end = min(start + chunk_size, len(tokens)) windows.append(encoder.decode(tokens[start:end])) if end >= len(tokens): break start = end - overlap return windows def _split_oversized(text: str, settings: ChunkingSettings) -> list[str]: """Split a row only if it exceeds the cap; otherwise keep it atomic.""" if count_tokens(text, settings.encoding_name) <= settings.max_chunk_tokens: return [text] return split_by_tokens( text, chunk_size=settings.chunk_size, overlap=settings.chunk_overlap, encoding_name=settings.encoding_name, ) def _content_units( parsed: ParsedDocument, settings: ChunkingSettings ) -> list[tuple[str, ContentType]]: """Reduce a parsed document's structural units to ordered text pieces. Prose is cut into token windows; a table row is atomic and survives whole unless it alone exceeds the model's sequence length (ADR-0004). """ if not parsed.units: raise DocumentParseError("Parsed document has no structural units") pieces: list[tuple[str, ContentType]] = [] for unit in parsed.units: if unit.content_type is ContentType.PARAGRAPH: windows = split_by_tokens( unit.text, chunk_size=settings.chunk_size, overlap=settings.chunk_overlap, encoding_name=settings.encoding_name, ) else: windows = _split_oversized(unit.text, settings) pieces.extend((window, unit.content_type) for window in windows) return pieces def chunk_document( parsed: ParsedDocument, *, file_id: uuid.UUID, settings: ChunkingSettings, ) -> list[Chunk]: """Turn a parsed document into ordered, neighbor-linked chunks.""" units = [ (text.strip(), content_type) for text, content_type in _content_units(parsed, settings) ] # Drop blanks *before* assigning indices: an index gap would break the # previous/next chain that retrieval-time expansion walks. units = [(text, content_type) for text, content_type in units if text] chunks: list[Chunk] = [] for index, (text, content_type) in enumerate(units): token_count = count_tokens(text, settings.encoding_name) if token_count > settings.max_chunk_tokens: raise ChunkTooLargeError( f"chunk {index} is {token_count} tokens, over the " f"{settings.max_chunk_tokens}-token cap" ) chunks.append( Chunk( chunk_id=chunk_id_for(file_id, index), chunk_index=index, order_id=float(index + 1), content=text, content_type=content_type, token_count=token_count, character_count=len(text), ) ) for position, chunk in enumerate(chunks): if position > 0: chunk.previous_chunk_id = chunks[position - 1].chunk_id if position < len(chunks) - 1: chunk.next_chunk_id = chunks[position + 1].chunk_id return chunks