feat(benchmarking): add benchmark models and routes with view parameter

Why:
- Need request/response models for benchmark endpoints
- Need routes for creating and retrieving benchmarks
- Need view parameter for managerial vs technical report views

Changes:
- models.py: Added BenchmarkRequest, BenchmarkResponse, StrategyMetrics, ExperimentDetailResponse
- routes.py: Added POST /benchmarks, GET /benchmarks/{id}, GET /experiments, view parameter for reports

src/benchmarking/models.py

@@ -51,3 +51,77 @@ class QueryDetailResponse(BaseModel):
     latency_breakdown: dict[str, float]
     token_usage: dict[str, int]
     created_at: str
+
+
+# ── Benchmark Request/Response ────────────────────────────────────
+
+class BenchmarkRequest(BaseModel):
+    """Body for POST /benchmarks."""
+    document_id: str = Field(description="Document ID to benchmark against")
+    strategies: list[StrategyName] = Field(
+        default=[
+            StrategyName.RECURSIVE,
+            StrategyName.FIXED_SIZE,
+            StrategyName.SEMANTIC,
+            StrategyName.CONTEXTUAL_RETRIEVAL,
+            StrategyName.SEMANTIC_PARENT_CHILD,
+        ],
+        description="Strategies to benchmark (defaults to all 5)",
+        min_length=1,
+    )
+    questions: list[dict] = Field(
+        default_factory=list,
+        description="List of question objects (overrides questions_file if provided)",
+    )
+    questions_file: Optional[str] = Field(
+        default=None,
+        description="Path to questions JSON file (relative to project root)",
+    )
+    top_k: int = Field(default=5, description="Number of chunks to retrieve", ge=1, le=20)
+    dry_run: bool = Field(default=False, description="Only return cost estimate, don't run benchmark")
+
+
+class CostEstimate(BaseModel):
+    """Cost estimation for a benchmark run."""
+    num_questions: int
+    num_strategies: int
+    total_queries: int
+    total_evaluations: int
+    estimated_tokens: dict[str, int]
+    estimated_cost_usd: float
+    cost_breakdown: dict[str, float]
+
+
+class StrategyMetrics(BaseModel):
+    """Aggregated metrics for a single strategy."""
+    avg_context_relevance: float
+    avg_answer_similarity: float
+    avg_faithfulness: float
+    hallucination_rate: float
+    total_questions: int
+    failed_questions: int
+
+
+class BenchmarkResponse(BaseModel):
+    """Returned after running a benchmark."""
+    experiment_id: str
+    document_id: str
+    strategies_used: list[str]
+    questions_count: int
+    aggregate_metrics: dict[str, StrategyMetrics]
+    best_strategy: str
+    total_latency_seconds: float
+    estimated_cost_usd: float
+    created_at: str
+
+
+class ExperimentDetailResponse(BaseModel):
+    """Returned when retrieving an experiment."""
+    id: str
+    document_id: str
+    benchmark_config: dict
+    questions: list[dict]
+    per_question: list[dict]
+    aggregate_metrics: dict[str, StrategyMetrics]
+    strategies_used: list[str]
+    created_at: str

src/benchmarking/routes.py

@@ -1,25 +1,36 @@
-"""Query API routes.
+"""Query and Benchmark API routes.
 
 Endpoints:
   POST   /queries              Ask a question against a strategy
   GET    /queries/{id}         Retrieve a past query
+  POST   /benchmarks           Run a benchmark (or dry run)
+  GET    /benchmarks/{id}      Retrieve experiment results
+  GET    /experiments          List all experiments
 """
 
 from fastapi import APIRouter
+from fastapi.responses import HTMLResponse
 
-from src.core.exceptions import QueryError
+from src.core.exceptions import BenchmarkError, QueryError
 from src.core.models import StrategyName
-from src.benchmarking import query_service
+from src.benchmarking import benchmark_service, query_service
 from src.benchmarking.models import (
+    BenchmarkRequest,
+    BenchmarkResponse,
+    CostEstimate,
+    ExperimentDetailResponse,
     QueryRequest,
     QueryResponse,
     QueryDetailResponse,
     RetrievedChunk,
+    StrategyMetrics,
 )
 
 router = APIRouter()
 
 
+# ── Query Endpoints ───────────────────────────────────────────────
+
 @router.post("/queries", response_model=QueryResponse, status_code=201)
 async def create_query(request: QueryRequest):
     """Ask a question against a document using a specific chunking strategy.
@@ -66,3 +77,257 @@ async def get_query(query_id: str):
         token_usage=result["token_usage"],
         created_at=result["created_at"],
     )
+
+
+# ── Benchmark Endpoints ──────────────────────────────────────────
+
+@router.post("/benchmarks", response_model=BenchmarkResponse, status_code=201)
+async def create_benchmark(request: BenchmarkRequest):
+    """Run a benchmark comparing multiple strategies on multiple questions.
+
+    Can run in dry_run mode to get cost estimate without executing.
+    """
+    # Load questions - prefer questions_file over inline questions
+    if request.questions_file:
+        questions = benchmark_service.load_questions(request.questions_file)
+    elif request.questions and len(request.questions) > 0:
+        # Validate that questions have required fields
+        valid_questions = [
+            q for q in request.questions
+            if isinstance(q, dict) and "question" in q
+        ]
+        if not valid_questions:
+            raise BenchmarkError("Invalid questions: each question must have a 'question' field")
+        questions = valid_questions
+    else:
+        raise BenchmarkError("Either 'questions' or 'questions_file' must be provided")
+
+    if not questions:
+        raise BenchmarkError("No questions to benchmark")
+
+    # Dry run - return cost estimate only
+    if request.dry_run:
+        estimate = benchmark_service.estimate_cost(
+            num_questions=len(questions),
+            num_strategies=len(request.strategies),
+        )
+        # Return as BenchmarkResponse with minimal data
+        return BenchmarkResponse(
+            experiment_id="dry_run",
+            document_id=request.document_id,
+            strategies_used=[s.value for s in request.strategies],
+            questions_count=len(questions),
+            aggregate_metrics={},
+            best_strategy="N/A (dry run)",
+            total_latency_seconds=0,
+            estimated_cost_usd=estimate["estimated_cost_usd"],
+            created_at="N/A",
+        )
+
+    # Run full benchmark
+    result = benchmark_service.run_benchmark(
+        document_id=request.document_id,
+        strategies=request.strategies,
+        questions=questions,
+        top_k=request.top_k,
+    )
+
+    return BenchmarkResponse(
+        experiment_id=result["experiment_id"],
+        document_id=result["document_id"],
+        strategies_used=result["strategies_used"],
+        questions_count=result["questions_count"],
+        aggregate_metrics={
+            k: StrategyMetrics(**v) for k, v in result["aggregate_metrics"].items()
+        },
+        best_strategy=result["best_strategy"],
+        total_latency_seconds=result["total_latency_seconds"],
+        estimated_cost_usd=result["estimated_cost_usd"],
+        created_at=result["created_at"],
+    )
+
+
+@router.get("/benchmarks/{experiment_id}", response_model=ExperimentDetailResponse)
+async def get_benchmark(experiment_id: str):
+    """Retrieve experiment results by ID."""
+    result = benchmark_service.get_experiment(experiment_id)
+    if result is None:
+        raise BenchmarkError(f"Experiment not found: {experiment_id}")
+
+    return ExperimentDetailResponse(
+        id=result["id"],
+        document_id=result["document_id"],
+        benchmark_config=result.get("benchmark_config", {}),
+        questions=result.get("questions", []),
+        per_question=result.get("per_question", []),
+        aggregate_metrics={
+            k: StrategyMetrics(**v) for k, v in result.get("aggregate_metrics", {}).items()
+        },
+        strategies_used=result.get("strategies_used", []),
+        created_at=result["created_at"],
+    )
+
+
+@router.get("/experiments")
+async def list_experiments(document_id: str | None = None):
+    """List all experiments, optionally filtered by document."""
+    return benchmark_service.list_experiments(document_id=document_id)
+
+
+@router.get("/benchmarks/{experiment_id}/report", response_class=HTMLResponse)
+async def get_report(experiment_id: str, view: str = "managerial"):
+    """Generate and return an HTML report for an experiment.
+
+    Views:
+    - managerial: Decision-focused, high-level insights (default)
+    - technical: Full observability with detailed data
+    """
+    from src.benchmarking.report import generate_report
+
+    result = benchmark_service.get_experiment(experiment_id)
+    if result is None:
+        raise BenchmarkError(f"Experiment not found: {experiment_id}")
+
+    # Validate view parameter
+    if view not in ("managerial", "technical"):
+        view = "managerial"
+
+    # Generate HTML report
+    html = generate_report(result, view)
+    return HTMLResponse(content=html)
+
+
+# ── HTML Report Generator ─────────────────────────────────────────
+
+def _generate_html_report(experiment: dict) -> str:
+    """Generate an HTML report from experiment data."""
+    config = experiment.get("benchmark_config", {})
+    aggregate = experiment.get("aggregate_metrics", {})
+    per_question = experiment.get("per_question", [])
+    strategies = experiment.get("strategies_used", [])
+
+    # Build strategy rows for summary table
+    strategy_rows = ""
+    for strategy_name in strategies:
+        metrics = aggregate.get(strategy_name, {})
+        strategy_rows += f"""
+        <tr>
+            <td><strong>{strategy_name}</strong></td>
+            <td>{metrics.get('avg_context_relevance', 0):.1f}</td>
+            <td>{metrics.get('avg_answer_similarity', 0):.1f}</td>
+            <td>{metrics.get('avg_faithfulness', 0):.1f}</td>
+            <td>{metrics.get('hallucination_rate', 0)*100:.1f}%</td>
+            <td>{metrics.get('total_questions', 0)}</td>
+        </tr>"""
+
+    # Build per-question rows
+    question_rows = ""
+    for qr in per_question:
+        q_text = qr.get("question", "")[:80]
+        q_id = qr.get("question_id", "")
+        for strategy_name in strategies:
+            strat_result = qr.get("strategies", {}).get(strategy_name, {})
+            scores = strat_result.get("scores", {})
+            answer_preview = strat_result.get("answer", "")[:100]
+            question_rows += f"""
+            <tr>
+                <td>{q_id}</td>
+                <td>{q_text}...</td>
+                <td>{strategy_name}</td>
+                <td>{scores.get('context_relevance', 'N/A')}</td>
+                <td>{scores.get('answer_similarity', 'N/A')}</td>
+                <td>{scores.get('faithfulness', 'N/A')}</td>
+                <td>{'✓' if not scores.get('hallucination', False) else '✗'}</td>
+                <td>{answer_preview}...</td>
+            </tr>"""
+
+    html = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Benchmark Report - {experiment.get('id', 'Unknown')}</title>
+    <style>
+        body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 40px; background: #f5f5f5; }}
+        .container {{ max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 8px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }}
+        h1 {{ color: #333; border-bottom: 2px solid #4CAF50; padding-bottom: 10px; }}
+        h2 {{ color: #555; margin-top: 30px; }}
+        .summary {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin: 20px 0; }}
+        .summary-card {{ background: #f9f9f9; padding: 20px; border-radius: 8px; text-align: center; }}
+        .summary-card h3 {{ margin: 0; color: #666; font-size: 14px; }}
+        .summary-card .value {{ font-size: 24px; font-weight: bold; color: #4CAF50; }}
+        table {{ width: 100%; border-collapse: collapse; margin: 20px 0; }}
+        th, td {{ padding: 12px; text-align: left; border-bottom: 1px solid #ddd; }}
+        th {{ background: #4CAF50; color: white; }}
+        tr:hover {{ background: #f5f5f5; }}
+        .best {{ background: #e8f5e9; font-weight: bold; }}
+        .footer {{ margin-top: 30px; padding-top: 20px; border-top: 1px solid #ddd; color: #666; font-size: 12px; }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>📊 Benchmark Report</h1>
+
+        <div class="summary">
+            <div class="summary-card">
+                <h3>Experiment ID</h3>
+                <div class="value" style="font-size: 14px;">{experiment.get('id', 'N/A')[:16]}...</div>
+            </div>
+            <div class="summary-card">
+                <h3>Document</h3>
+                <div class="value" style="font-size: 14px;">{experiment.get('document_id', 'N/A')[:16]}...</div>
+            </div>
+            <div class="summary-card">
+                <h3>Questions</h3>
+                <div class="value">{config.get('num_questions', len(per_question))}</div>
+            </div>
+            <div class="summary-card">
+                <h3>Strategies</h3>
+                <div class="value">{len(strategies)}</div>
+            </div>
+        </div>
+
+        <h2>📈 Summary by Strategy</h2>
+        <table>
+            <thead>
+                <tr>
+                    <th>Strategy</th>
+                    <th>Context Relevance</th>
+                    <th>Answer Similarity</th>
+                    <th>Faithfulness</th>
+                    <th>Hallucination Rate</th>
+                    <th>Questions</th>
+                </tr>
+            </thead>
+            <tbody>
+                {strategy_rows}
+            </tbody>
+        </table>
+
+        <h2>📝 Detailed Results</h2>
+        <table>
+            <thead>
+                <tr>
+                    <th>Q-ID</th>
+                    <th>Question</th>
+                    <th>Strategy</th>
+                    <th>Context</th>
+                    <th>Similarity</th>
+                    <th>Faithfulness</th>
+                    <th>No Halluc.</th>
+                    <th>Answer Preview</th>
+                </tr>
+            </thead>
+            <tbody>
+                {question_rows}
+            </tbody>
+        </table>
+
+        <div class="footer">
+            <p>Generated by RAG Chunking Benchmarker | {experiment.get('created_at', 'N/A')}</p>
+        </div>
+    </div>
+</body>
+</html>"""
+
+    return html