feat(documents): snapshot boundary and corpus models on process
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -16,6 +16,8 @@ class DocumentResponse(BaseModel):
|
||||
paragraph_count: int = 0
|
||||
chunk_counts: dict[str, int] = Field(default_factory=dict)
|
||||
created_at: str
|
||||
last_corpus_embedding_model_id: Optional[str] = None
|
||||
last_boundary_embedding_model_id: Optional[str] = None
|
||||
|
||||
|
||||
class DocumentDetailResponse(DocumentResponse):
|
||||
@@ -37,6 +39,14 @@ class ProcessRequest(BaseModel):
|
||||
description="Which chunking strategies to run (defaults to all 5)",
|
||||
min_length=1,
|
||||
)
|
||||
boundary_model_id: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Boundary Embedding Model id (semantic cuts); default = Admin Boundary",
|
||||
)
|
||||
corpus_model_id: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Corpus Embedding Model id (storage); default = Admin Corpus",
|
||||
)
|
||||
|
||||
|
||||
class StrategyResult(BaseModel):
|
||||
@@ -52,6 +62,8 @@ class ProcessResponse(BaseModel):
|
||||
document_id: str
|
||||
strategies_completed: list[StrategyResult]
|
||||
strategies_failed: list[StrategyResult]
|
||||
corpus_embedding_model_id: Optional[str] = None
|
||||
boundary_embedding_model_id: Optional[str] = None
|
||||
|
||||
|
||||
class DeleteResponse(BaseModel):
|
||||
|
||||
@@ -101,7 +101,12 @@ def process_document(
|
||||
# Import here to avoid circular imports at module level
|
||||
from src.chunking.service import run_strategies
|
||||
|
||||
completed_raw, failed_raw = run_strategies(doc_id, request.strategies)
|
||||
completed_raw, failed_raw = run_strategies(
|
||||
doc_id,
|
||||
request.strategies,
|
||||
boundary_model_id=request.boundary_model_id,
|
||||
corpus_model_id=request.corpus_model_id,
|
||||
)
|
||||
|
||||
completed = [
|
||||
StrategyResult(
|
||||
@@ -120,17 +125,28 @@ def process_document(
|
||||
for r in failed_raw
|
||||
]
|
||||
|
||||
corpus_id = None
|
||||
boundary_id = None
|
||||
for r in completed_raw:
|
||||
corpus_id = r.get("corpus_embedding_model_id") or corpus_id
|
||||
if r.get("boundary_embedding_model_id"):
|
||||
boundary_id = r["boundary_embedding_model_id"]
|
||||
|
||||
return ProcessResponse(
|
||||
document_id=doc_id,
|
||||
strategies_completed=completed,
|
||||
strategies_failed=failed,
|
||||
corpus_embedding_model_id=corpus_id,
|
||||
boundary_embedding_model_id=boundary_id,
|
||||
)
|
||||
|
||||
|
||||
# ── Delete ─────────────────────────────────────────────────────────
|
||||
|
||||
def delete_document(doc_id: str) -> bool:
|
||||
"""Delete a document and all its Qdrant vectors."""
|
||||
"""Delete a document and all its Qdrant vectors across Model Corpora."""
|
||||
from src.chunking.embedding_models import list_models
|
||||
|
||||
doc = db.get_document(doc_id)
|
||||
if doc is None:
|
||||
return False
|
||||
@@ -139,7 +155,20 @@ def delete_document(doc_id: str) -> bool:
|
||||
if count > 0:
|
||||
try:
|
||||
strategy = StrategyName(strategy_name)
|
||||
qdr.delete_document_chunks(strategy, doc["filename"])
|
||||
for model in list_models():
|
||||
try:
|
||||
qdr.delete_document_chunks(
|
||||
strategy,
|
||||
doc["filename"],
|
||||
model_id=model.id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to delete Qdrant vectors for %s/%s: %s",
|
||||
strategy_name,
|
||||
model.id,
|
||||
exc,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to delete Qdrant vectors for %s: %s", strategy_name, exc)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user