mba/chunking_strategies_evaluation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(models): add text field to ChunkMetadata for Qdrant storage

Browse Source 
Why:
- Qdrant was storing only metadata, not chunk text
- Queries returned empty text in retrieved chunks
- Scores were low (~0.25) because embeddings were on empty strings

Changes:
- Added 'text' field to ChunkMetadata model
- Updated chunk_to_metadata() to copy text
This commit is contained in:
Mahdi Bazrafshan
2026-07-26 12:05:52 +03:30
parent 292ae17cfa
commit b75fde8185
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/core/models.py
View File

@@ -21,7 +21,6 @@ class Chunk(BaseModel):
Strategy-specific fields (parent_id, enriched_content) are nullable Strategy-specific fields (parent_id, enriched_content) are nullable
when not applicable to the strategy that produced the chunk. when not applicable to the strategy that produced the chunk.
""" """
document_name: str document_name: str
chunk_id: str chunk_id: str
strategy_name: StrategyName strategy_name: StrategyName
@@ -35,11 +34,11 @@ class Chunk(BaseModel):
class ChunkMetadata(BaseModel): class ChunkMetadata(BaseModel):
"""Payload stored alongside every chunk vector in Qdrant.""" """Payload stored alongside every chunk vector in Qdrant."""
document_name: str document_name: str
chunk_id: str chunk_id: str
strategy_name: StrategyName strategy_name: StrategyName
chunk_index: int chunk_index: int
text: str
token_count: int token_count: int
character_count: int character_count: int
parent_id: Optional[str] = None parent_id: Optional[str] = None
@@ -52,6 +51,7 @@ def chunk_to_metadata(chunk: Chunk) -> ChunkMetadata:
chunk_id=chunk.chunk_id, chunk_id=chunk.chunk_id,
strategy_name=chunk.strategy_name, strategy_name=chunk.strategy_name,
chunk_index=chunk.chunk_index, chunk_index=chunk.chunk_index,
text=chunk.text,
token_count=chunk.token_count, token_count=chunk.token_count,
character_count=chunk.character_count, character_count=chunk.character_count,
parent_id=chunk.parent_id, parent_id=chunk.parent_id,