Mahdi Bazrafshan
56b8d9401a
Why: - Compare is the wrong surface for two-stage family selection over the 10-doc set. Changes: - Add the Decision Tab; raise GET /experiments default/max so the board can load the grid client-side. Impact: - Operators pick fixed_size ±N vs semantic@Boundary from existing Experiments. Co-authored-by: Cursor <cursoragent@cursor.com>
394 lines
15 KiB
Python
394 lines
15 KiB
Python
"""Query and Benchmark API routes.
|
|
|
|
Endpoints:
|
|
POST /queries Ask a question against a strategy
|
|
GET /queries/{id} Retrieve a past query
|
|
POST /benchmarks Run a benchmark (or dry run)
|
|
GET /benchmarks/{id} Retrieve experiment results
|
|
GET /experiments List all experiments
|
|
"""
|
|
|
|
from fastapi import APIRouter, Query
|
|
from fastapi.responses import HTMLResponse
|
|
|
|
from src.core.exceptions import BenchmarkError, QueryError
|
|
from src.core.models import StrategyName
|
|
from src.benchmarking import benchmark_service, query_service
|
|
from src.benchmarking.models import (
|
|
BenchmarkRequest,
|
|
BenchmarkResponse,
|
|
CostEstimate,
|
|
ExperimentDetailResponse,
|
|
QueryRequest,
|
|
QueryResponse,
|
|
QueryDetailResponse,
|
|
RetrievedChunk,
|
|
StrategyMetrics,
|
|
)
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
# ── Query Endpoints ───────────────────────────────────────────────
|
|
|
|
@router.post("/queries", response_model=QueryResponse, status_code=201)
|
|
async def create_query(request: QueryRequest):
|
|
"""Ask a question against a document using a specific chunking strategy.
|
|
|
|
Pipeline: embed question → vector search → generate answer → store result.
|
|
"""
|
|
result = query_service.run_query(
|
|
document_id=request.document_id,
|
|
strategy_name=request.strategy,
|
|
question=request.question,
|
|
top_k=request.top_k,
|
|
neighbor_prev=request.neighbor_prev,
|
|
neighbor_next=request.neighbor_next,
|
|
corpus_model_id=request.corpus_model_id,
|
|
)
|
|
|
|
return QueryResponse(
|
|
query_id=result["query_id"],
|
|
document_id=result["document_id"],
|
|
strategy=result["strategy"],
|
|
question=result["question"],
|
|
answer=result["answer"],
|
|
retrieved_chunks=[
|
|
RetrievedChunk(**chunk) for chunk in result["retrieved_chunks"]
|
|
],
|
|
expansion_tree=result.get("expansion_tree") or [],
|
|
neighbor_prev=result.get("neighbor_prev", 0),
|
|
neighbor_next=result.get("neighbor_next", 0),
|
|
latency_breakdown=result["latency_breakdown"],
|
|
token_usage=result["token_usage"],
|
|
created_at=result["created_at"],
|
|
)
|
|
|
|
|
|
@router.get("/queries/{query_id}", response_model=QueryDetailResponse)
|
|
async def get_query(query_id: str):
|
|
"""Retrieve a past query by ID."""
|
|
result = query_service.get_query(query_id)
|
|
if result is None:
|
|
raise QueryError(f"Query not found: {query_id}")
|
|
|
|
return QueryDetailResponse(
|
|
id=result["id"],
|
|
document_id=result["document_id"],
|
|
strategy_name=result["strategy_name"],
|
|
question=result["question"],
|
|
answer=result["answer"],
|
|
retrieved_chunks=result["retrieved_chunks"],
|
|
expansion_tree=result.get("expansion_tree") or [],
|
|
latency_breakdown=result["latency_breakdown"],
|
|
token_usage=result["token_usage"],
|
|
created_at=result["created_at"],
|
|
)
|
|
|
|
|
|
# ── Benchmark Endpoints ──────────────────────────────────────────
|
|
|
|
@router.post("/benchmarks", response_model=BenchmarkResponse, status_code=201)
|
|
async def create_benchmark(request: BenchmarkRequest):
|
|
"""Run a benchmark comparing multiple strategies on multiple questions.
|
|
|
|
Can run in dry_run mode to get cost estimate without executing.
|
|
"""
|
|
# Load questions - prefer questions_file over inline questions
|
|
if request.questions_file:
|
|
questions = benchmark_service.load_questions(request.questions_file)
|
|
elif request.questions and len(request.questions) > 0:
|
|
# Validate that questions have required fields
|
|
valid_questions = [
|
|
q for q in request.questions
|
|
if isinstance(q, dict) and "question" in q
|
|
]
|
|
if not valid_questions:
|
|
raise BenchmarkError("Invalid questions: each question must have a 'question' field")
|
|
questions = valid_questions
|
|
else:
|
|
raise BenchmarkError("Either 'questions' or 'questions_file' must be provided")
|
|
|
|
if not questions:
|
|
raise BenchmarkError("No questions to benchmark")
|
|
|
|
# Dry run - return cost estimate only
|
|
if request.dry_run:
|
|
estimate = benchmark_service.estimate_cost(
|
|
num_questions=len(questions),
|
|
num_strategies=len(request.strategies),
|
|
)
|
|
# Return as BenchmarkResponse with minimal data
|
|
return BenchmarkResponse(
|
|
experiment_id="dry_run",
|
|
document_id=request.document_id,
|
|
strategies_used=[s.value for s in request.strategies],
|
|
questions_count=len(questions),
|
|
aggregate_metrics={},
|
|
best_strategy="N/A (dry run)",
|
|
total_latency_seconds=0,
|
|
estimated_cost_usd=estimate["estimated_cost_usd"],
|
|
created_at="N/A",
|
|
)
|
|
|
|
# Run full benchmark
|
|
result = benchmark_service.run_benchmark(
|
|
document_id=request.document_id,
|
|
strategies=request.strategies,
|
|
questions=questions,
|
|
top_k=request.top_k,
|
|
neighbor_prev=request.neighbor_prev,
|
|
neighbor_next=request.neighbor_next,
|
|
corpus_model_id=request.corpus_model_id,
|
|
)
|
|
|
|
return BenchmarkResponse(
|
|
experiment_id=result["experiment_id"],
|
|
document_id=result["document_id"],
|
|
strategies_used=result["strategies_used"],
|
|
questions_count=result["questions_count"],
|
|
aggregate_metrics={
|
|
k: StrategyMetrics(**v) for k, v in result["aggregate_metrics"].items()
|
|
},
|
|
best_strategy=result["best_strategy"],
|
|
total_latency_seconds=result["total_latency_seconds"],
|
|
estimated_cost_usd=result["estimated_cost_usd"],
|
|
created_at=result["created_at"],
|
|
)
|
|
|
|
|
|
@router.get("/benchmarks/{experiment_id}", response_model=ExperimentDetailResponse)
|
|
async def get_benchmark(experiment_id: str):
|
|
"""Retrieve experiment results by ID."""
|
|
result = benchmark_service.get_experiment(experiment_id)
|
|
if result is None:
|
|
raise BenchmarkError(f"Experiment not found: {experiment_id}")
|
|
|
|
# Resolve document filename
|
|
from src.storage import sqlite as db
|
|
doc = db.get_document(result.get("document_id", ""))
|
|
doc_filename = doc.get("filename", "Unknown") if doc else "Deleted"
|
|
|
|
return ExperimentDetailResponse(
|
|
id=result["id"],
|
|
document_id=result["document_id"],
|
|
document_filename=doc_filename,
|
|
benchmark_config=result.get("benchmark_config", {}),
|
|
questions=result.get("questions", []),
|
|
per_question=result.get("per_question", []),
|
|
aggregate_metrics={
|
|
k: StrategyMetrics(**v) for k, v in result.get("aggregate_metrics", {}).items()
|
|
},
|
|
strategies_used=result.get("strategies_used", []),
|
|
created_at=result["created_at"],
|
|
)
|
|
|
|
|
|
@router.get("/experiments")
|
|
async def list_experiments(
|
|
document_id: str | None = None,
|
|
offset: int = Query(0, ge=0),
|
|
limit: int = Query(200, ge=1, le=500),
|
|
):
|
|
"""List all experiments, optionally filtered by document."""
|
|
result = benchmark_service.list_experiments(
|
|
document_id=document_id, offset=offset, limit=limit
|
|
)
|
|
# Enrich with document filenames and best_strategy
|
|
from src.storage import sqlite as db
|
|
for item in result.get("items", []):
|
|
doc = db.get_document(item.get("document_id", ""))
|
|
item["document_filename"] = doc.get("filename", "Unknown") if doc else "Deleted"
|
|
questions = item.get("questions") or []
|
|
item["questions_count"] = (
|
|
item.get("benchmark_config", {}).get("num_questions")
|
|
or (len(questions) if isinstance(questions, list) else 0)
|
|
)
|
|
# Calculate best_strategy from aggregate_metrics
|
|
aggs = item.get("aggregate_metrics", {})
|
|
best_strat, best_score = "N/A", -1
|
|
for strat, m in aggs.items():
|
|
score = (m.get("avg_context_relevance", 0) * 0.3 +
|
|
m.get("avg_answer_similarity", 0) * 0.4 +
|
|
m.get("avg_faithfulness", 0) * 0.3)
|
|
adjusted = score * (1 - m.get("hallucination_rate", 0))
|
|
if adjusted > best_score:
|
|
best_score = adjusted
|
|
best_strat = strat
|
|
item["best_strategy"] = best_strat
|
|
# Surface embedding on list even if only in benchmark_config
|
|
if not item.get("embedding_model_id"):
|
|
cfg = item.get("benchmark_config") or {}
|
|
item["embedding_model_id"] = cfg.get("embedding_model_id")
|
|
item["embedding_provider"] = cfg.get("embedding_provider")
|
|
return result
|
|
|
|
|
|
@router.delete("/experiments/{experiment_id}")
|
|
async def delete_experiment(experiment_id: str):
|
|
"""Delete an experiment by ID."""
|
|
from src.storage import sqlite as db
|
|
deleted = db.delete_experiment(experiment_id)
|
|
if not deleted:
|
|
raise BenchmarkError(f"Experiment not found: {experiment_id}")
|
|
return {"deleted": True, "experiment_id": experiment_id}
|
|
|
|
|
|
@router.get("/benchmarks/{experiment_id}/report", response_class=HTMLResponse)
|
|
async def get_report(experiment_id: str, view: str = "managerial"):
|
|
"""Generate and return an HTML report for an experiment.
|
|
|
|
Views:
|
|
- managerial: Decision-focused, high-level insights (default)
|
|
- technical: Full observability with detailed data
|
|
"""
|
|
from src.benchmarking.report import generate_report
|
|
|
|
result = benchmark_service.get_experiment(experiment_id)
|
|
if result is None:
|
|
raise BenchmarkError(f"Experiment not found: {experiment_id}")
|
|
|
|
# Validate view parameter
|
|
if view not in ("managerial", "technical"):
|
|
view = "managerial"
|
|
|
|
# Generate HTML report
|
|
html = generate_report(result, view)
|
|
return HTMLResponse(content=html)
|
|
|
|
|
|
# ── HTML Report Generator ─────────────────────────────────────────
|
|
|
|
def _generate_html_report(experiment: dict) -> str:
|
|
"""Generate an HTML report from experiment data."""
|
|
config = experiment.get("benchmark_config", {})
|
|
aggregate = experiment.get("aggregate_metrics", {})
|
|
per_question = experiment.get("per_question", [])
|
|
strategies = experiment.get("strategies_used", [])
|
|
|
|
# Build strategy rows for summary table
|
|
strategy_rows = ""
|
|
for strategy_name in strategies:
|
|
metrics = aggregate.get(strategy_name, {})
|
|
strategy_rows += f"""
|
|
<tr>
|
|
<td><strong>{strategy_name}</strong></td>
|
|
<td>{metrics.get('avg_context_relevance', 0):.1f}</td>
|
|
<td>{metrics.get('avg_answer_similarity', 0):.1f}</td>
|
|
<td>{metrics.get('avg_faithfulness', 0):.1f}</td>
|
|
<td>{metrics.get('hallucination_rate', 0)*100:.1f}%</td>
|
|
<td>{metrics.get('total_questions', 0)}</td>
|
|
</tr>"""
|
|
|
|
# Build per-question rows
|
|
question_rows = ""
|
|
for qr in per_question:
|
|
q_text = qr.get("question", "")[:80]
|
|
q_id = qr.get("question_id", "")
|
|
for strategy_name in strategies:
|
|
strat_result = qr.get("strategies", {}).get(strategy_name, {})
|
|
scores = strat_result.get("scores", {})
|
|
answer_preview = strat_result.get("answer", "")[:100]
|
|
question_rows += f"""
|
|
<tr>
|
|
<td>{q_id}</td>
|
|
<td>{q_text}...</td>
|
|
<td>{strategy_name}</td>
|
|
<td>{scores.get('context_relevance', 'N/A')}</td>
|
|
<td>{scores.get('answer_similarity', 'N/A')}</td>
|
|
<td>{scores.get('faithfulness', 'N/A')}</td>
|
|
<td>{'✓' if not scores.get('hallucination', False) else '✗'}</td>
|
|
<td>{answer_preview}...</td>
|
|
</tr>"""
|
|
|
|
html = f"""<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Benchmark Report - {experiment.get('id', 'Unknown')}</title>
|
|
<style>
|
|
body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 40px; background: #f5f5f5; }}
|
|
.container {{ max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }}
|
|
h1 {{ color: #333; border-bottom: 2px solid #4CAF50; padding-bottom: 10px; }}
|
|
h2 {{ color: #555; margin-top: 30px; }}
|
|
.summary {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin: 20px 0; }}
|
|
.summary-card {{ background: #f9f9f9; padding: 20px; border-radius: 8px; text-align: center; }}
|
|
.summary-card h3 {{ margin: 0; color: #666; font-size: 14px; }}
|
|
.summary-card .value {{ font-size: 24px; font-weight: bold; color: #4CAF50; }}
|
|
table {{ width: 100%; border-collapse: collapse; margin: 20px 0; }}
|
|
th, td {{ padding: 12px; text-align: left; border-bottom: 1px solid #ddd; }}
|
|
th {{ background: #4CAF50; color: white; }}
|
|
tr:hover {{ background: #f5f5f5; }}
|
|
.best {{ background: #e8f5e9; font-weight: bold; }}
|
|
.footer {{ margin-top: 30px; padding-top: 20px; border-top: 1px solid #ddd; color: #666; font-size: 12px; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<h1>📊 Benchmark Report</h1>
|
|
|
|
<div class="summary">
|
|
<div class="summary-card">
|
|
<h3>Experiment ID</h3>
|
|
<div class="value" style="font-size: 14px;">{experiment.get('id', 'N/A')[:16]}...</div>
|
|
</div>
|
|
<div class="summary-card">
|
|
<h3>Document</h3>
|
|
<div class="value" style="font-size: 14px;">{experiment.get('document_id', 'N/A')[:16]}...</div>
|
|
</div>
|
|
<div class="summary-card">
|
|
<h3>Questions</h3>
|
|
<div class="value">{config.get('num_questions', len(per_question))}</div>
|
|
</div>
|
|
<div class="summary-card">
|
|
<h3>Strategies</h3>
|
|
<div class="value">{len(strategies)}</div>
|
|
</div>
|
|
</div>
|
|
|
|
<h2>📈 Summary by Strategy</h2>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Strategy</th>
|
|
<th>Context Relevance</th>
|
|
<th>Answer Similarity</th>
|
|
<th>Faithfulness</th>
|
|
<th>Hallucination Rate</th>
|
|
<th>Questions</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{strategy_rows}
|
|
</tbody>
|
|
</table>
|
|
|
|
<h2>📝 Detailed Results</h2>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Q-ID</th>
|
|
<th>Question</th>
|
|
<th>Strategy</th>
|
|
<th>Context</th>
|
|
<th>Similarity</th>
|
|
<th>Faithfulness</th>
|
|
<th>No Halluc.</th>
|
|
<th>Answer Preview</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{question_rows}
|
|
</tbody>
|
|
</table>
|
|
|
|
<div class="footer">
|
|
<p>Generated by RAG Chunking Benchmarker | {experiment.get('created_at', 'N/A')}</p>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>"""
|
|
|
|
return html
|