diff --git a/src/benchmarking/models.py b/src/benchmarking/models.py
index 20d7bb4..8fb77c3 100644
--- a/src/benchmarking/models.py
+++ b/src/benchmarking/models.py
@@ -51,3 +51,77 @@ class QueryDetailResponse(BaseModel):
latency_breakdown: dict[str, float]
token_usage: dict[str, int]
created_at: str
+
+
+# ── Benchmark Request/Response ────────────────────────────────────
+
+class BenchmarkRequest(BaseModel):
+ """Body for POST /benchmarks."""
+ document_id: str = Field(description="Document ID to benchmark against")
+ strategies: list[StrategyName] = Field(
+ default=[
+ StrategyName.RECURSIVE,
+ StrategyName.FIXED_SIZE,
+ StrategyName.SEMANTIC,
+ StrategyName.CONTEXTUAL_RETRIEVAL,
+ StrategyName.SEMANTIC_PARENT_CHILD,
+ ],
+ description="Strategies to benchmark (defaults to all 5)",
+ min_length=1,
+ )
+ questions: list[dict] = Field(
+ default_factory=list,
+ description="List of question objects (overrides questions_file if provided)",
+ )
+ questions_file: Optional[str] = Field(
+ default=None,
+ description="Path to questions JSON file (relative to project root)",
+ )
+ top_k: int = Field(default=5, description="Number of chunks to retrieve", ge=1, le=20)
+ dry_run: bool = Field(default=False, description="Only return cost estimate, don't run benchmark")
+
+
+class CostEstimate(BaseModel):
+ """Cost estimation for a benchmark run."""
+ num_questions: int
+ num_strategies: int
+ total_queries: int
+ total_evaluations: int
+ estimated_tokens: dict[str, int]
+ estimated_cost_usd: float
+ cost_breakdown: dict[str, float]
+
+
+class StrategyMetrics(BaseModel):
+ """Aggregated metrics for a single strategy."""
+ avg_context_relevance: float
+ avg_answer_similarity: float
+ avg_faithfulness: float
+ hallucination_rate: float
+ total_questions: int
+ failed_questions: int
+
+
+class BenchmarkResponse(BaseModel):
+ """Returned after running a benchmark."""
+ experiment_id: str
+ document_id: str
+ strategies_used: list[str]
+ questions_count: int
+ aggregate_metrics: dict[str, StrategyMetrics]
+ best_strategy: str
+ total_latency_seconds: float
+ estimated_cost_usd: float
+ created_at: str
+
+
+class ExperimentDetailResponse(BaseModel):
+ """Returned when retrieving an experiment."""
+ id: str
+ document_id: str
+ benchmark_config: dict
+ questions: list[dict]
+ per_question: list[dict]
+ aggregate_metrics: dict[str, StrategyMetrics]
+ strategies_used: list[str]
+ created_at: str
diff --git a/src/benchmarking/routes.py b/src/benchmarking/routes.py
index fbed4cf..1b40e3f 100644
--- a/src/benchmarking/routes.py
+++ b/src/benchmarking/routes.py
@@ -1,25 +1,36 @@
-"""Query API routes.
+"""Query and Benchmark API routes.
Endpoints:
POST /queries Ask a question against a strategy
GET /queries/{id} Retrieve a past query
+ POST /benchmarks Run a benchmark (or dry run)
+ GET /benchmarks/{id} Retrieve experiment results
+ GET /experiments List all experiments
"""
from fastapi import APIRouter
+from fastapi.responses import HTMLResponse
-from src.core.exceptions import QueryError
+from src.core.exceptions import BenchmarkError, QueryError
from src.core.models import StrategyName
-from src.benchmarking import query_service
+from src.benchmarking import benchmark_service, query_service
from src.benchmarking.models import (
+ BenchmarkRequest,
+ BenchmarkResponse,
+ CostEstimate,
+ ExperimentDetailResponse,
QueryRequest,
QueryResponse,
QueryDetailResponse,
RetrievedChunk,
+ StrategyMetrics,
)
router = APIRouter()
+# ── Query Endpoints ───────────────────────────────────────────────
+
@router.post("/queries", response_model=QueryResponse, status_code=201)
async def create_query(request: QueryRequest):
"""Ask a question against a document using a specific chunking strategy.
@@ -66,3 +77,257 @@ async def get_query(query_id: str):
token_usage=result["token_usage"],
created_at=result["created_at"],
)
+
+
+# ── Benchmark Endpoints ──────────────────────────────────────────
+
+@router.post("/benchmarks", response_model=BenchmarkResponse, status_code=201)
+async def create_benchmark(request: BenchmarkRequest):
+ """Run a benchmark comparing multiple strategies on multiple questions.
+
+ Can run in dry_run mode to get cost estimate without executing.
+ """
+ # Load questions - prefer questions_file over inline questions
+ if request.questions_file:
+ questions = benchmark_service.load_questions(request.questions_file)
+ elif request.questions and len(request.questions) > 0:
+ # Validate that questions have required fields
+ valid_questions = [
+ q for q in request.questions
+ if isinstance(q, dict) and "question" in q
+ ]
+ if not valid_questions:
+ raise BenchmarkError("Invalid questions: each question must have a 'question' field")
+ questions = valid_questions
+ else:
+ raise BenchmarkError("Either 'questions' or 'questions_file' must be provided")
+
+ if not questions:
+ raise BenchmarkError("No questions to benchmark")
+
+ # Dry run - return cost estimate only
+ if request.dry_run:
+ estimate = benchmark_service.estimate_cost(
+ num_questions=len(questions),
+ num_strategies=len(request.strategies),
+ )
+ # Return as BenchmarkResponse with minimal data
+ return BenchmarkResponse(
+ experiment_id="dry_run",
+ document_id=request.document_id,
+ strategies_used=[s.value for s in request.strategies],
+ questions_count=len(questions),
+ aggregate_metrics={},
+ best_strategy="N/A (dry run)",
+ total_latency_seconds=0,
+ estimated_cost_usd=estimate["estimated_cost_usd"],
+ created_at="N/A",
+ )
+
+ # Run full benchmark
+ result = benchmark_service.run_benchmark(
+ document_id=request.document_id,
+ strategies=request.strategies,
+ questions=questions,
+ top_k=request.top_k,
+ )
+
+ return BenchmarkResponse(
+ experiment_id=result["experiment_id"],
+ document_id=result["document_id"],
+ strategies_used=result["strategies_used"],
+ questions_count=result["questions_count"],
+ aggregate_metrics={
+ k: StrategyMetrics(**v) for k, v in result["aggregate_metrics"].items()
+ },
+ best_strategy=result["best_strategy"],
+ total_latency_seconds=result["total_latency_seconds"],
+ estimated_cost_usd=result["estimated_cost_usd"],
+ created_at=result["created_at"],
+ )
+
+
+@router.get("/benchmarks/{experiment_id}", response_model=ExperimentDetailResponse)
+async def get_benchmark(experiment_id: str):
+ """Retrieve experiment results by ID."""
+ result = benchmark_service.get_experiment(experiment_id)
+ if result is None:
+ raise BenchmarkError(f"Experiment not found: {experiment_id}")
+
+ return ExperimentDetailResponse(
+ id=result["id"],
+ document_id=result["document_id"],
+ benchmark_config=result.get("benchmark_config", {}),
+ questions=result.get("questions", []),
+ per_question=result.get("per_question", []),
+ aggregate_metrics={
+ k: StrategyMetrics(**v) for k, v in result.get("aggregate_metrics", {}).items()
+ },
+ strategies_used=result.get("strategies_used", []),
+ created_at=result["created_at"],
+ )
+
+
+@router.get("/experiments")
+async def list_experiments(document_id: str | None = None):
+ """List all experiments, optionally filtered by document."""
+ return benchmark_service.list_experiments(document_id=document_id)
+
+
+@router.get("/benchmarks/{experiment_id}/report", response_class=HTMLResponse)
+async def get_report(experiment_id: str, view: str = "managerial"):
+ """Generate and return an HTML report for an experiment.
+
+ Views:
+ - managerial: Decision-focused, high-level insights (default)
+ - technical: Full observability with detailed data
+ """
+ from src.benchmarking.report import generate_report
+
+ result = benchmark_service.get_experiment(experiment_id)
+ if result is None:
+ raise BenchmarkError(f"Experiment not found: {experiment_id}")
+
+ # Validate view parameter
+ if view not in ("managerial", "technical"):
+ view = "managerial"
+
+ # Generate HTML report
+ html = generate_report(result, view)
+ return HTMLResponse(content=html)
+
+
+# ── HTML Report Generator ─────────────────────────────────────────
+
+def _generate_html_report(experiment: dict) -> str:
+ """Generate an HTML report from experiment data."""
+ config = experiment.get("benchmark_config", {})
+ aggregate = experiment.get("aggregate_metrics", {})
+ per_question = experiment.get("per_question", [])
+ strategies = experiment.get("strategies_used", [])
+
+ # Build strategy rows for summary table
+ strategy_rows = ""
+ for strategy_name in strategies:
+ metrics = aggregate.get(strategy_name, {})
+ strategy_rows += f"""
+
+ | {strategy_name} |
+ {metrics.get('avg_context_relevance', 0):.1f} |
+ {metrics.get('avg_answer_similarity', 0):.1f} |
+ {metrics.get('avg_faithfulness', 0):.1f} |
+ {metrics.get('hallucination_rate', 0)*100:.1f}% |
+ {metrics.get('total_questions', 0)} |
+
"""
+
+ # Build per-question rows
+ question_rows = ""
+ for qr in per_question:
+ q_text = qr.get("question", "")[:80]
+ q_id = qr.get("question_id", "")
+ for strategy_name in strategies:
+ strat_result = qr.get("strategies", {}).get(strategy_name, {})
+ scores = strat_result.get("scores", {})
+ answer_preview = strat_result.get("answer", "")[:100]
+ question_rows += f"""
+
+ | {q_id} |
+ {q_text}... |
+ {strategy_name} |
+ {scores.get('context_relevance', 'N/A')} |
+ {scores.get('answer_similarity', 'N/A')} |
+ {scores.get('faithfulness', 'N/A')} |
+ {'✓' if not scores.get('hallucination', False) else '✗'} |
+ {answer_preview}... |
+
"""
+
+ html = f"""
+
+
+
+
+ Benchmark Report - {experiment.get('id', 'Unknown')}
+
+
+
+
+
📊 Benchmark Report
+
+
+
+
Experiment ID
+
{experiment.get('id', 'N/A')[:16]}...
+
+
+
Document
+
{experiment.get('document_id', 'N/A')[:16]}...
+
+
+
Questions
+
{config.get('num_questions', len(per_question))}
+
+
+
Strategies
+
{len(strategies)}
+
+
+
+
📈 Summary by Strategy
+
+
+
+ | Strategy |
+ Context Relevance |
+ Answer Similarity |
+ Faithfulness |
+ Hallucination Rate |
+ Questions |
+
+
+
+ {strategy_rows}
+
+
+
+
📝 Detailed Results
+
+
+
+ | Q-ID |
+ Question |
+ Strategy |
+ Context |
+ Similarity |
+ Faithfulness |
+ No Halluc. |
+ Answer Preview |
+
+
+
+ {question_rows}
+
+
+
+
+