feat(benchmarking): add benchmark models and routes with view parameter
Why:
- Need request/response models for benchmark endpoints
- Need routes for creating and retrieving benchmarks
- Need view parameter for managerial vs technical report views
Changes:
- models.py: Added BenchmarkRequest, BenchmarkResponse, StrategyMetrics, ExperimentDetailResponse
- routes.py: Added POST /benchmarks, GET /benchmarks/{id}, GET /experiments, view parameter for reports
This commit is contained in:
@@ -51,3 +51,77 @@ class QueryDetailResponse(BaseModel):
|
||||
latency_breakdown: dict[str, float]
|
||||
token_usage: dict[str, int]
|
||||
created_at: str
|
||||
|
||||
|
||||
# ── Benchmark Request/Response ────────────────────────────────────
|
||||
|
||||
class BenchmarkRequest(BaseModel):
|
||||
"""Body for POST /benchmarks."""
|
||||
document_id: str = Field(description="Document ID to benchmark against")
|
||||
strategies: list[StrategyName] = Field(
|
||||
default=[
|
||||
StrategyName.RECURSIVE,
|
||||
StrategyName.FIXED_SIZE,
|
||||
StrategyName.SEMANTIC,
|
||||
StrategyName.CONTEXTUAL_RETRIEVAL,
|
||||
StrategyName.SEMANTIC_PARENT_CHILD,
|
||||
],
|
||||
description="Strategies to benchmark (defaults to all 5)",
|
||||
min_length=1,
|
||||
)
|
||||
questions: list[dict] = Field(
|
||||
default_factory=list,
|
||||
description="List of question objects (overrides questions_file if provided)",
|
||||
)
|
||||
questions_file: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Path to questions JSON file (relative to project root)",
|
||||
)
|
||||
top_k: int = Field(default=5, description="Number of chunks to retrieve", ge=1, le=20)
|
||||
dry_run: bool = Field(default=False, description="Only return cost estimate, don't run benchmark")
|
||||
|
||||
|
||||
class CostEstimate(BaseModel):
|
||||
"""Cost estimation for a benchmark run."""
|
||||
num_questions: int
|
||||
num_strategies: int
|
||||
total_queries: int
|
||||
total_evaluations: int
|
||||
estimated_tokens: dict[str, int]
|
||||
estimated_cost_usd: float
|
||||
cost_breakdown: dict[str, float]
|
||||
|
||||
|
||||
class StrategyMetrics(BaseModel):
|
||||
"""Aggregated metrics for a single strategy."""
|
||||
avg_context_relevance: float
|
||||
avg_answer_similarity: float
|
||||
avg_faithfulness: float
|
||||
hallucination_rate: float
|
||||
total_questions: int
|
||||
failed_questions: int
|
||||
|
||||
|
||||
class BenchmarkResponse(BaseModel):
|
||||
"""Returned after running a benchmark."""
|
||||
experiment_id: str
|
||||
document_id: str
|
||||
strategies_used: list[str]
|
||||
questions_count: int
|
||||
aggregate_metrics: dict[str, StrategyMetrics]
|
||||
best_strategy: str
|
||||
total_latency_seconds: float
|
||||
estimated_cost_usd: float
|
||||
created_at: str
|
||||
|
||||
|
||||
class ExperimentDetailResponse(BaseModel):
|
||||
"""Returned when retrieving an experiment."""
|
||||
id: str
|
||||
document_id: str
|
||||
benchmark_config: dict
|
||||
questions: list[dict]
|
||||
per_question: list[dict]
|
||||
aggregate_metrics: dict[str, StrategyMetrics]
|
||||
strategies_used: list[str]
|
||||
created_at: str
|
||||
|
||||
@@ -1,25 +1,36 @@
|
||||
"""Query API routes.
|
||||
"""Query and Benchmark API routes.
|
||||
|
||||
Endpoints:
|
||||
POST /queries Ask a question against a strategy
|
||||
GET /queries/{id} Retrieve a past query
|
||||
POST /benchmarks Run a benchmark (or dry run)
|
||||
GET /benchmarks/{id} Retrieve experiment results
|
||||
GET /experiments List all experiments
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from src.core.exceptions import QueryError
|
||||
from src.core.exceptions import BenchmarkError, QueryError
|
||||
from src.core.models import StrategyName
|
||||
from src.benchmarking import query_service
|
||||
from src.benchmarking import benchmark_service, query_service
|
||||
from src.benchmarking.models import (
|
||||
BenchmarkRequest,
|
||||
BenchmarkResponse,
|
||||
CostEstimate,
|
||||
ExperimentDetailResponse,
|
||||
QueryRequest,
|
||||
QueryResponse,
|
||||
QueryDetailResponse,
|
||||
RetrievedChunk,
|
||||
StrategyMetrics,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Query Endpoints ───────────────────────────────────────────────
|
||||
|
||||
@router.post("/queries", response_model=QueryResponse, status_code=201)
|
||||
async def create_query(request: QueryRequest):
|
||||
"""Ask a question against a document using a specific chunking strategy.
|
||||
@@ -66,3 +77,257 @@ async def get_query(query_id: str):
|
||||
token_usage=result["token_usage"],
|
||||
created_at=result["created_at"],
|
||||
)
|
||||
|
||||
|
||||
# ── Benchmark Endpoints ──────────────────────────────────────────
|
||||
|
||||
@router.post("/benchmarks", response_model=BenchmarkResponse, status_code=201)
|
||||
async def create_benchmark(request: BenchmarkRequest):
|
||||
"""Run a benchmark comparing multiple strategies on multiple questions.
|
||||
|
||||
Can run in dry_run mode to get cost estimate without executing.
|
||||
"""
|
||||
# Load questions - prefer questions_file over inline questions
|
||||
if request.questions_file:
|
||||
questions = benchmark_service.load_questions(request.questions_file)
|
||||
elif request.questions and len(request.questions) > 0:
|
||||
# Validate that questions have required fields
|
||||
valid_questions = [
|
||||
q for q in request.questions
|
||||
if isinstance(q, dict) and "question" in q
|
||||
]
|
||||
if not valid_questions:
|
||||
raise BenchmarkError("Invalid questions: each question must have a 'question' field")
|
||||
questions = valid_questions
|
||||
else:
|
||||
raise BenchmarkError("Either 'questions' or 'questions_file' must be provided")
|
||||
|
||||
if not questions:
|
||||
raise BenchmarkError("No questions to benchmark")
|
||||
|
||||
# Dry run - return cost estimate only
|
||||
if request.dry_run:
|
||||
estimate = benchmark_service.estimate_cost(
|
||||
num_questions=len(questions),
|
||||
num_strategies=len(request.strategies),
|
||||
)
|
||||
# Return as BenchmarkResponse with minimal data
|
||||
return BenchmarkResponse(
|
||||
experiment_id="dry_run",
|
||||
document_id=request.document_id,
|
||||
strategies_used=[s.value for s in request.strategies],
|
||||
questions_count=len(questions),
|
||||
aggregate_metrics={},
|
||||
best_strategy="N/A (dry run)",
|
||||
total_latency_seconds=0,
|
||||
estimated_cost_usd=estimate["estimated_cost_usd"],
|
||||
created_at="N/A",
|
||||
)
|
||||
|
||||
# Run full benchmark
|
||||
result = benchmark_service.run_benchmark(
|
||||
document_id=request.document_id,
|
||||
strategies=request.strategies,
|
||||
questions=questions,
|
||||
top_k=request.top_k,
|
||||
)
|
||||
|
||||
return BenchmarkResponse(
|
||||
experiment_id=result["experiment_id"],
|
||||
document_id=result["document_id"],
|
||||
strategies_used=result["strategies_used"],
|
||||
questions_count=result["questions_count"],
|
||||
aggregate_metrics={
|
||||
k: StrategyMetrics(**v) for k, v in result["aggregate_metrics"].items()
|
||||
},
|
||||
best_strategy=result["best_strategy"],
|
||||
total_latency_seconds=result["total_latency_seconds"],
|
||||
estimated_cost_usd=result["estimated_cost_usd"],
|
||||
created_at=result["created_at"],
|
||||
)
|
||||
|
||||
|
||||
@router.get("/benchmarks/{experiment_id}", response_model=ExperimentDetailResponse)
|
||||
async def get_benchmark(experiment_id: str):
|
||||
"""Retrieve experiment results by ID."""
|
||||
result = benchmark_service.get_experiment(experiment_id)
|
||||
if result is None:
|
||||
raise BenchmarkError(f"Experiment not found: {experiment_id}")
|
||||
|
||||
return ExperimentDetailResponse(
|
||||
id=result["id"],
|
||||
document_id=result["document_id"],
|
||||
benchmark_config=result.get("benchmark_config", {}),
|
||||
questions=result.get("questions", []),
|
||||
per_question=result.get("per_question", []),
|
||||
aggregate_metrics={
|
||||
k: StrategyMetrics(**v) for k, v in result.get("aggregate_metrics", {}).items()
|
||||
},
|
||||
strategies_used=result.get("strategies_used", []),
|
||||
created_at=result["created_at"],
|
||||
)
|
||||
|
||||
|
||||
@router.get("/experiments")
|
||||
async def list_experiments(document_id: str | None = None):
|
||||
"""List all experiments, optionally filtered by document."""
|
||||
return benchmark_service.list_experiments(document_id=document_id)
|
||||
|
||||
|
||||
@router.get("/benchmarks/{experiment_id}/report", response_class=HTMLResponse)
|
||||
async def get_report(experiment_id: str, view: str = "managerial"):
|
||||
"""Generate and return an HTML report for an experiment.
|
||||
|
||||
Views:
|
||||
- managerial: Decision-focused, high-level insights (default)
|
||||
- technical: Full observability with detailed data
|
||||
"""
|
||||
from src.benchmarking.report import generate_report
|
||||
|
||||
result = benchmark_service.get_experiment(experiment_id)
|
||||
if result is None:
|
||||
raise BenchmarkError(f"Experiment not found: {experiment_id}")
|
||||
|
||||
# Validate view parameter
|
||||
if view not in ("managerial", "technical"):
|
||||
view = "managerial"
|
||||
|
||||
# Generate HTML report
|
||||
html = generate_report(result, view)
|
||||
return HTMLResponse(content=html)
|
||||
|
||||
|
||||
# ── HTML Report Generator ─────────────────────────────────────────
|
||||
|
||||
def _generate_html_report(experiment: dict) -> str:
|
||||
"""Generate an HTML report from experiment data."""
|
||||
config = experiment.get("benchmark_config", {})
|
||||
aggregate = experiment.get("aggregate_metrics", {})
|
||||
per_question = experiment.get("per_question", [])
|
||||
strategies = experiment.get("strategies_used", [])
|
||||
|
||||
# Build strategy rows for summary table
|
||||
strategy_rows = ""
|
||||
for strategy_name in strategies:
|
||||
metrics = aggregate.get(strategy_name, {})
|
||||
strategy_rows += f"""
|
||||
<tr>
|
||||
<td><strong>{strategy_name}</strong></td>
|
||||
<td>{metrics.get('avg_context_relevance', 0):.1f}</td>
|
||||
<td>{metrics.get('avg_answer_similarity', 0):.1f}</td>
|
||||
<td>{metrics.get('avg_faithfulness', 0):.1f}</td>
|
||||
<td>{metrics.get('hallucination_rate', 0)*100:.1f}%</td>
|
||||
<td>{metrics.get('total_questions', 0)}</td>
|
||||
</tr>"""
|
||||
|
||||
# Build per-question rows
|
||||
question_rows = ""
|
||||
for qr in per_question:
|
||||
q_text = qr.get("question", "")[:80]
|
||||
q_id = qr.get("question_id", "")
|
||||
for strategy_name in strategies:
|
||||
strat_result = qr.get("strategies", {}).get(strategy_name, {})
|
||||
scores = strat_result.get("scores", {})
|
||||
answer_preview = strat_result.get("answer", "")[:100]
|
||||
question_rows += f"""
|
||||
<tr>
|
||||
<td>{q_id}</td>
|
||||
<td>{q_text}...</td>
|
||||
<td>{strategy_name}</td>
|
||||
<td>{scores.get('context_relevance', 'N/A')}</td>
|
||||
<td>{scores.get('answer_similarity', 'N/A')}</td>
|
||||
<td>{scores.get('faithfulness', 'N/A')}</td>
|
||||
<td>{'✓' if not scores.get('hallucination', False) else '✗'}</td>
|
||||
<td>{answer_preview}...</td>
|
||||
</tr>"""
|
||||
|
||||
html = f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Benchmark Report - {experiment.get('id', 'Unknown')}</title>
|
||||
<style>
|
||||
body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 40px; background: #f5f5f5; }}
|
||||
.container {{ max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }}
|
||||
h1 {{ color: #333; border-bottom: 2px solid #4CAF50; padding-bottom: 10px; }}
|
||||
h2 {{ color: #555; margin-top: 30px; }}
|
||||
.summary {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin: 20px 0; }}
|
||||
.summary-card {{ background: #f9f9f9; padding: 20px; border-radius: 8px; text-align: center; }}
|
||||
.summary-card h3 {{ margin: 0; color: #666; font-size: 14px; }}
|
||||
.summary-card .value {{ font-size: 24px; font-weight: bold; color: #4CAF50; }}
|
||||
table {{ width: 100%; border-collapse: collapse; margin: 20px 0; }}
|
||||
th, td {{ padding: 12px; text-align: left; border-bottom: 1px solid #ddd; }}
|
||||
th {{ background: #4CAF50; color: white; }}
|
||||
tr:hover {{ background: #f5f5f5; }}
|
||||
.best {{ background: #e8f5e9; font-weight: bold; }}
|
||||
.footer {{ margin-top: 30px; padding-top: 20px; border-top: 1px solid #ddd; color: #666; font-size: 12px; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>📊 Benchmark Report</h1>
|
||||
|
||||
<div class="summary">
|
||||
<div class="summary-card">
|
||||
<h3>Experiment ID</h3>
|
||||
<div class="value" style="font-size: 14px;">{experiment.get('id', 'N/A')[:16]}...</div>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<h3>Document</h3>
|
||||
<div class="value" style="font-size: 14px;">{experiment.get('document_id', 'N/A')[:16]}...</div>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<h3>Questions</h3>
|
||||
<div class="value">{config.get('num_questions', len(per_question))}</div>
|
||||
</div>
|
||||
<div class="summary-card">
|
||||
<h3>Strategies</h3>
|
||||
<div class="value">{len(strategies)}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h2>📈 Summary by Strategy</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Strategy</th>
|
||||
<th>Context Relevance</th>
|
||||
<th>Answer Similarity</th>
|
||||
<th>Faithfulness</th>
|
||||
<th>Hallucination Rate</th>
|
||||
<th>Questions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{strategy_rows}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>📝 Detailed Results</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Q-ID</th>
|
||||
<th>Question</th>
|
||||
<th>Strategy</th>
|
||||
<th>Context</th>
|
||||
<th>Similarity</th>
|
||||
<th>Faithfulness</th>
|
||||
<th>No Halluc.</th>
|
||||
<th>Answer Preview</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{question_rows}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div class="footer">
|
||||
<p>Generated by RAG Chunking Benchmarker | {experiment.get('created_at', 'N/A')}</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
return html
|
||||
|
||||
Reference in New Issue
Block a user