feat(benchmarking): add evaluation, benchmark service, and report
Why: - Need LLM-as-Judge evaluation for automated scoring - Need benchmark orchestration to run questions × strategies - Need HTML report generation with two views (managerial/technical) Changes: - evaluation.py: LLM-as-Judge scoring on 4 metrics (context, similarity, faithfulness, hallucination) - benchmark_service.py: Orchestration with per-strategy failure isolation - report.py: Dual-view HTML reports with dark mode, charts, and strategy cards
This commit is contained in:
815
src/benchmarking/report.py
Normal file
815
src/benchmarking/report.py
Normal file
@@ -0,0 +1,815 @@
|
||||
"""Enhanced HTML report generator with two views:
|
||||
1. Managerial: Decision-focused, high-level insights
|
||||
2. Technical: Full observability with detailed data
|
||||
|
||||
Design: Dark mode, amber/teal accents, Inter + JetBrains Mono.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
# ── Design Tokens ─────────────────────────────────────────────────
|
||||
|
||||
COLORS = {
|
||||
"bg": "#0F1419",
|
||||
"surface": "#1A2332",
|
||||
"surface-hover": "#243044",
|
||||
"border": "#2D3A4A",
|
||||
"text": "#E8EDF2",
|
||||
"text-muted": "#7A8BA0",
|
||||
"accent": "#F59E0B",
|
||||
"accent-dim": "#D97706",
|
||||
"teal": "#14B8A6",
|
||||
"rose": "#F43F5E",
|
||||
"violet": "#8B5CF6",
|
||||
"emerald": "#10B981",
|
||||
}
|
||||
|
||||
CHART_COLORS = ["#F59E0B", "#14B8A6", "#8B5CF6", "#F43F5E", "#3B82F6"]
|
||||
|
||||
FONT_DISPLAY = "'Inter', -apple-system, sans-serif"
|
||||
FONT_DATA = "'JetBrains Mono', 'Fira Code', monospace"
|
||||
|
||||
|
||||
# ── Main Entry Points ─────────────────────────────────────────────
|
||||
|
||||
def generate_report(experiment: dict, view: str = "managerial") -> str:
|
||||
"""Generate report for specified view.
|
||||
|
||||
Args:
|
||||
experiment: Experiment data from SQLite
|
||||
view: "managerial" or "technical"
|
||||
"""
|
||||
if view == "technical":
|
||||
return generate_technical_report(experiment)
|
||||
return generate_managerial_report(experiment)
|
||||
|
||||
|
||||
def generate_managerial_report(experiment: dict) -> str:
|
||||
"""Managerial view: Decision-focused, high-level insights."""
|
||||
config = experiment.get("benchmark_config", {})
|
||||
aggregate = experiment.get("aggregate_metrics", {})
|
||||
per_question = experiment.get("per_question", [])
|
||||
strategies = experiment.get("strategies_used", [])
|
||||
rankings = _calculate_rankings(aggregate, strategies)
|
||||
|
||||
# Calculate cost from token usage in per-question data
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
for qr in per_question:
|
||||
for strategy in strategies:
|
||||
strat = qr.get("strategies", {}).get(strategy, {})
|
||||
usage = strat.get("token_usage", {})
|
||||
total_prompt_tokens += usage.get("prompt_tokens", 0)
|
||||
total_completion_tokens += usage.get("completion_tokens", 0)
|
||||
|
||||
estimated_cost = (total_prompt_tokens * 0.15 + total_completion_tokens * 0.60) / 1_000_000
|
||||
|
||||
html = _base_html(experiment, "Managerial View", f"""
|
||||
<!-- Header -->
|
||||
<header class="header animate">
|
||||
<div class="eyebrow">Benchmark Results</div>
|
||||
<h1>Strategy Comparison</h1>
|
||||
<div class="meta">
|
||||
{config.get('num_questions', 0)} questions ·
|
||||
{len(strategies)} strategies ·
|
||||
${estimated_cost:.4f} cost
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- KPIs -->
|
||||
<div class="kpi-row animate delay-1">
|
||||
<div class="kpi">
|
||||
<div class="label">Best Strategy</div>
|
||||
<div class="value accent">{rankings[0][0] if rankings else 'N/A'}</div>
|
||||
</div>
|
||||
<div class="kpi">
|
||||
<div class="label">Overall Score</div>
|
||||
<div class="value teal">{rankings[0][1]:.2f}</div>
|
||||
</div>
|
||||
<div class="kpi">
|
||||
<div class="label">Questions</div>
|
||||
<div class="value">{config.get('num_questions', 0)}</div>
|
||||
</div>
|
||||
<div class="kpi">
|
||||
<div class="label">Cost</div>
|
||||
<div class="value">${estimated_cost:.4f}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Winner -->
|
||||
<div class="winner animate delay-2">
|
||||
<div class="medal">🏆</div>
|
||||
<div class="content">
|
||||
<h2>Recommendation: {rankings[0][0] if rankings else 'N/A'}</h2>
|
||||
<p>{_get_recommendation_text(rankings, aggregate)}</p>
|
||||
</div>
|
||||
<div class="score">{rankings[0][1]:.2f}</div>
|
||||
</div>
|
||||
|
||||
<!-- Strategy Rankings -->
|
||||
<div class="section-header animate delay-3">
|
||||
<h2>How Strategies Compare</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="strategy-grid animate delay-3">
|
||||
{_build_strategy_cards(rankings, aggregate)}
|
||||
</div>
|
||||
|
||||
<!-- Charts -->
|
||||
<div class="section-header animate delay-4">
|
||||
<h2>Visual Comparison</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="charts-row animate delay-4">
|
||||
<div class="chart-panel">
|
||||
<h3>Performance Radar</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="radarChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
<div class="chart-panel">
|
||||
<h3>Score Comparison</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="barChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Decision Insights -->
|
||||
<div class="section-header">
|
||||
<h2>Decision Guide</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="insights">
|
||||
{_generate_decision_insights(rankings, aggregate)}
|
||||
</div>
|
||||
|
||||
<!-- Quick Links -->
|
||||
<div class="section-header">
|
||||
<h2>Details</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="quick-links">
|
||||
<a href="/benchmarks/{experiment.get('id', '')}/report?view=technical" class="link-card">
|
||||
<span class="link-icon">🔍</span>
|
||||
<span class="link-text">View Full Technical Report</span>
|
||||
<span class="link-arrow">→</span>
|
||||
</a>
|
||||
</div>
|
||||
""")
|
||||
|
||||
return html
|
||||
|
||||
|
||||
def generate_technical_report(experiment: dict) -> str:
|
||||
"""Technical view: Full observability with detailed data."""
|
||||
config = experiment.get("benchmark_config", {})
|
||||
aggregate = experiment.get("aggregate_metrics", {})
|
||||
per_question = experiment.get("per_question", [])
|
||||
strategies = experiment.get("strategies_used", [])
|
||||
rankings = _calculate_rankings(aggregate, strategies)
|
||||
|
||||
# Calculate metrics from available data
|
||||
total_queries = len(per_question) * len(strategies)
|
||||
total_questions = config.get("num_questions", len(per_question))
|
||||
top_k = config.get("top_k", 5)
|
||||
|
||||
# Calculate token usage from per-question data
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
for qr in per_question:
|
||||
for strategy in strategies:
|
||||
strat = qr.get("strategies", {}).get(strategy, {})
|
||||
usage = strat.get("token_usage", {})
|
||||
total_prompt_tokens += usage.get("prompt_tokens", 0)
|
||||
total_completion_tokens += usage.get("completion_tokens", 0)
|
||||
|
||||
total_tokens = total_prompt_tokens + total_completion_tokens
|
||||
|
||||
# Estimate cost based on token usage (GPT-4o-mini pricing)
|
||||
# Input: $0.15/1M tokens, Output: $0.60/1M tokens
|
||||
estimated_cost = (total_prompt_tokens * 0.15 + total_completion_tokens * 0.60) / 1_000_000
|
||||
|
||||
# Estimate latency (rough: ~0.5s per query + ~1s per evaluation)
|
||||
estimated_latency = total_queries * 1.5
|
||||
|
||||
html = _base_html(experiment, "Technical View", f"""
|
||||
<!-- Header -->
|
||||
<header class="header animate">
|
||||
<div class="eyebrow">Technical Report</div>
|
||||
<h1>Full Observability</h1>
|
||||
<div class="meta">
|
||||
Experiment: {experiment.get('id', 'N/A')[:16]}... ·
|
||||
{total_questions} questions ·
|
||||
{len(strategies)} strategies
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- KPIs -->
|
||||
<div class="kpi-row animate delay-1">
|
||||
<div class="kpi">
|
||||
<div class="label">Total Queries</div>
|
||||
<div class="value accent">{total_queries}</div>
|
||||
</div>
|
||||
<div class="kpi">
|
||||
<div class="label">Total Tokens</div>
|
||||
<div class="value">{total_tokens:,}</div>
|
||||
</div>
|
||||
<div class="kpi">
|
||||
<div class="label">Cost</div>
|
||||
<div class="value teal">${estimated_cost:.4f}</div>
|
||||
</div>
|
||||
<div class="kpi">
|
||||
<div class="label">Tokens per Query</div>
|
||||
<div class="value">{total_tokens // max(total_queries, 1):,}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Aggregate Metrics Table -->
|
||||
<div class="section-header animate delay-2">
|
||||
<h2>Aggregate Metrics</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="table-wrapper animate delay-2">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Strategy</th>
|
||||
<th>Context Relevance</th>
|
||||
<th>Answer Similarity</th>
|
||||
<th>Faithfulness</th>
|
||||
<th>Hallucination</th>
|
||||
<th>Questions</th>
|
||||
<th>Failed</th>
|
||||
<th>Overall</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{_build_technical_aggregate_rows(rankings, aggregate)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Charts -->
|
||||
<div class="section-header animate delay-3">
|
||||
<h2>Visual Analysis</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="charts-row animate delay-3">
|
||||
<div class="chart-panel">
|
||||
<h3>Multi-Metric Radar</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="radarChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
<div class="chart-panel">
|
||||
<h3>Score Distribution</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="barChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Per-Question Results -->
|
||||
<div class="section-header">
|
||||
<h2>Per-Question Results</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="table-wrapper">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Question</th>
|
||||
<th>Category</th>
|
||||
<th>Difficulty</th>
|
||||
{_build_strategy_headers(strategies)}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{_build_per_question_rows(per_question, strategies)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Detailed Scores -->
|
||||
<div class="section-header">
|
||||
<h2>Detailed Scores</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="table-wrapper">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Q-ID</th>
|
||||
<th>Strategy</th>
|
||||
<th>Context</th>
|
||||
<th>Similarity</th>
|
||||
<th>Faithfulness</th>
|
||||
<th>Hallucination</th>
|
||||
<th>Answer Preview</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{_build_detailed_rows(per_question, strategies)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Token Usage -->
|
||||
<div class="section-header">
|
||||
<h2>Token Usage</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="table-wrapper">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Q-ID</th>
|
||||
<th>Strategy</th>
|
||||
<th>Prompt Tokens</th>
|
||||
<th>Completion Tokens</th>
|
||||
<th>Total Tokens</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{_build_token_rows(per_question, strategies)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Quick Links -->
|
||||
<div class="section-header">
|
||||
<h2>Navigation</h2>
|
||||
<div class="line"></div>
|
||||
</div>
|
||||
<div class="quick-links">
|
||||
<a href="/benchmarks/{experiment.get('id', '')}/report?view=managerial" class="link-card">
|
||||
<span class="link-icon">📊</span>
|
||||
<span class="link-text">Switch to Managerial View</span>
|
||||
<span class="link-arrow">→</span>
|
||||
</a>
|
||||
</div>
|
||||
""")
|
||||
|
||||
return html
|
||||
|
||||
|
||||
# ── Base HTML Template ────────────────────────────────────────────
|
||||
|
||||
def _base_html(experiment: dict, title: str, content: str) -> str:
|
||||
"""Base HTML wrapper with styles and scripts."""
|
||||
strategies = experiment.get("strategies_used", [])
|
||||
aggregate = experiment.get("aggregate_metrics", {})
|
||||
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{title}</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
||||
<style>
|
||||
*, *::before, *::after {{ margin: 0; padding: 0; box-sizing: border-box; }}
|
||||
|
||||
:root {{
|
||||
--bg: {COLORS['bg']};
|
||||
--surface: {COLORS['surface']};
|
||||
--surface-hover: {COLORS['surface-hover']};
|
||||
--border: {COLORS['border']};
|
||||
--text: {COLORS['text']};
|
||||
--text-muted: {COLORS['text-muted']};
|
||||
--accent: {COLORS['accent']};
|
||||
--accent-dim: {COLORS['accent-dim']};
|
||||
--teal: {COLORS['teal']};
|
||||
--rose: {COLORS['rose']};
|
||||
--violet: {COLORS['violet']};
|
||||
--emerald: {COLORS['emerald']};
|
||||
}}
|
||||
|
||||
html {{ scroll-behavior: smooth; }}
|
||||
|
||||
body {{
|
||||
font-family: {FONT_DISPLAY};
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
line-height: 1.6;
|
||||
min-height: 100vh;
|
||||
}}
|
||||
|
||||
.page {{ max-width: 1280px; margin: 0 auto; padding: 48px 32px; }}
|
||||
|
||||
@keyframes fadeUp {{
|
||||
from {{ opacity: 0; transform: translateY(20px); }}
|
||||
to {{ opacity: 1; transform: translateY(0); }}
|
||||
}}
|
||||
.animate {{ animation: fadeUp 0.6s ease-out forwards; opacity: 0; }}
|
||||
.delay-1 {{ animation-delay: 0.1s; }}
|
||||
.delay-2 {{ animation-delay: 0.2s; }}
|
||||
.delay-3 {{ animation-delay: 0.3s; }}
|
||||
.delay-4 {{ animation-delay: 0.4s; }}
|
||||
|
||||
.header {{ margin-bottom: 48px; padding-bottom: 32px; border-bottom: 1px solid var(--border); }}
|
||||
.header .eyebrow {{ font-family: {FONT_DATA}; font-size: 12px; color: var(--accent); text-transform: uppercase; letter-spacing: 2px; margin-bottom: 12px; }}
|
||||
.header h1 {{ font-size: 36px; font-weight: 700; letter-spacing: -0.5px; margin-bottom: 8px; }}
|
||||
.header .meta {{ font-size: 14px; color: var(--text-muted); font-family: {FONT_DATA}; }}
|
||||
|
||||
.kpi-row {{ display: grid; grid-template-columns: repeat(4, 1fr); gap: 24px; margin-bottom: 48px; }}
|
||||
.kpi {{ padding: 24px; background: var(--surface); border: 1px solid var(--border); border-radius: 12px; transition: border-color 0.2s ease, transform 0.2s ease; }}
|
||||
.kpi:hover {{ border-color: var(--accent); transform: translateY(-2px); }}
|
||||
.kpi .label {{ font-size: 12px; color: var(--text-muted); text-transform: uppercase; letter-spacing: 1px; margin-bottom: 8px; }}
|
||||
.kpi .value {{ font-family: {FONT_DATA}; font-size: 32px; font-weight: 600; color: var(--text); }}
|
||||
.kpi .value.accent {{ color: var(--accent); }}
|
||||
.kpi .value.teal {{ color: var(--teal); }}
|
||||
|
||||
.winner {{ display: flex; align-items: center; gap: 24px; padding: 32px; background: linear-gradient(135deg, var(--surface) 0%, var(--surface-hover) 100%); border: 1px solid var(--accent); border-radius: 16px; margin-bottom: 48px; }}
|
||||
.winner .medal {{ width: 64px; height: 64px; background: var(--accent); border-radius: 50%; display: flex; align-items: center; justify-content: center; font-size: 28px; flex-shrink: 0; }}
|
||||
.winner .content h2 {{ font-size: 24px; font-weight: 600; margin-bottom: 4px; }}
|
||||
.winner .content p {{ color: var(--text-muted); font-size: 14px; max-width: 600px; }}
|
||||
.winner .score {{ margin-left: auto; font-family: {FONT_DATA}; font-size: 48px; font-weight: 700; color: var(--accent); }}
|
||||
|
||||
.section-header {{ display: flex; align-items: center; gap: 12px; margin-bottom: 24px; }}
|
||||
.section-header h2 {{ font-size: 20px; font-weight: 600; }}
|
||||
.section-header .line {{ flex: 1; height: 1px; background: var(--border); }}
|
||||
|
||||
.strategy-grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 16px; margin-bottom: 48px; }}
|
||||
.strategy-card {{ background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 24px; position: relative; overflow: hidden; transition: border-color 0.2s ease; }}
|
||||
.strategy-card:hover {{ border-color: var(--text-muted); }}
|
||||
.strategy-card.rank-1 {{ border-color: var(--accent); background: linear-gradient(180deg, rgba(245,158,11,0.08) 0%, var(--surface) 100%); }}
|
||||
.strategy-card .rank-badge {{ position: absolute; top: 16px; right: 16px; width: 28px; height: 28px; border-radius: 50%; display: flex; align-items: center; justify-content: center; font-family: {FONT_DATA}; font-size: 12px; font-weight: 600; }}
|
||||
.strategy-card.rank-1 .rank-badge {{ background: var(--accent); color: var(--bg); }}
|
||||
.strategy-card.rank-2 .rank-badge {{ background: var(--text-muted); color: var(--bg); }}
|
||||
.strategy-card.rank-3 .rank-badge {{ background: var(--accent-dim); color: var(--bg); }}
|
||||
.strategy-card .name {{ font-size: 14px; font-weight: 600; margin-bottom: 16px; padding-right: 40px; }}
|
||||
.strategy-card .metric {{ margin-bottom: 12px; }}
|
||||
.strategy-card .metric .label {{ font-size: 11px; color: var(--text-muted); margin-bottom: 4px; }}
|
||||
.strategy-card .metric .bar-bg {{ height: 6px; background: var(--border); border-radius: 3px; overflow: hidden; }}
|
||||
.strategy-card .metric .bar-fill {{ height: 100%; border-radius: 3px; transition: width 0.8s ease-out; }}
|
||||
.strategy-card .metric .bar-fill.teal {{ background: var(--teal); }}
|
||||
.strategy-card .metric .bar-fill.accent {{ background: var(--accent); }}
|
||||
.strategy-card .metric .bar-fill.violet {{ background: var(--violet); }}
|
||||
.strategy-card .metric .bar-fill.rose {{ background: var(--rose); }}
|
||||
.strategy-card .metric .value {{ font-family: {FONT_DATA}; font-size: 12px; color: var(--text); margin-top: 4px; }}
|
||||
|
||||
.charts-row {{ display: grid; grid-template-columns: 1fr 1fr; gap: 24px; margin-bottom: 48px; }}
|
||||
.chart-panel {{ background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 24px; }}
|
||||
.chart-panel h3 {{ font-size: 14px; font-weight: 500; color: var(--text-muted); margin-bottom: 16px; }}
|
||||
.chart-container {{ position: relative; height: 280px; }}
|
||||
|
||||
.insights {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 16px; margin-bottom: 48px; }}
|
||||
.insight {{ padding: 20px; border-radius: 12px; border-left: 3px solid; }}
|
||||
.insight.positive {{ background: rgba(16,185,129,0.1); border-color: var(--emerald); }}
|
||||
.insight.warning {{ background: rgba(244,63,94,0.1); border-color: var(--rose); }}
|
||||
.insight.info {{ background: rgba(139,92,246,0.1); border-color: var(--violet); }}
|
||||
.insight .title {{ font-size: 13px; font-weight: 600; margin-bottom: 4px; }}
|
||||
.insight .desc {{ font-size: 13px; color: var(--text-muted); }}
|
||||
|
||||
.table-wrapper {{ background: var(--surface); border: 1px solid var(--border); border-radius: 12px; overflow: hidden; margin-bottom: 48px; }}
|
||||
.table-wrapper table {{ width: 100%; }}
|
||||
.table-wrapper th {{ background: var(--surface-hover); padding: 12px 16px; text-align: left; font-size: 11px; font-weight: 600; color: var(--text-muted); text-transform: uppercase; letter-spacing: 0.5px; border-bottom: 1px solid var(--border); }}
|
||||
.table-wrapper td {{ padding: 12px 16px; font-size: 13px; border-bottom: 1px solid var(--border); }}
|
||||
.table-wrapper tr:last-child td {{ border-bottom: none; }}
|
||||
.table-wrapper tr:hover td {{ background: var(--surface-hover); }}
|
||||
|
||||
.pill {{ display: inline-block; padding: 2px 8px; border-radius: 6px; font-family: {FONT_DATA}; font-size: 12px; font-weight: 500; }}
|
||||
.pill.high {{ background: rgba(16,185,129,0.15); color: var(--emerald); }}
|
||||
.pill.mid {{ background: rgba(245,158,11,0.15); color: var(--accent); }}
|
||||
.pill.low {{ background: rgba(244,63,94,0.15); color: var(--rose); }}
|
||||
|
||||
.quick-links {{ margin-bottom: 48px; }}
|
||||
.link-card {{ display: flex; align-items: center; gap: 16px; padding: 20px 24px; background: var(--surface); border: 1px solid var(--border); border-radius: 12px; text-decoration: none; color: var(--text); transition: border-color 0.2s ease, transform 0.2s ease; }}
|
||||
.link-card:hover {{ border-color: var(--accent); transform: translateX(4px); }}
|
||||
.link-icon {{ font-size: 24px; }}
|
||||
.link-text {{ flex: 1; font-weight: 500; }}
|
||||
.link-arrow {{ color: var(--accent); font-size: 18px; }}
|
||||
|
||||
.footer {{ padding-top: 32px; border-top: 1px solid var(--border); font-size: 12px; color: var(--text-muted); text-align: center; }}
|
||||
|
||||
@media (max-width: 768px) {{
|
||||
.page {{ padding: 24px 16px; }}
|
||||
.kpi-row {{ grid-template-columns: repeat(2, 1fr); }}
|
||||
.charts-row {{ grid-template-columns: 1fr; }}
|
||||
.winner {{ flex-direction: column; text-align: center; }}
|
||||
.winner .score {{ margin-left: 0; margin-top: 16px; }}
|
||||
}}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {{
|
||||
.animate {{ animation: none; opacity: 1; }}
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="page">
|
||||
{content}
|
||||
|
||||
<footer class="footer">
|
||||
Generated by RAG Chunking Benchmarker · {experiment.get('created_at', 'N/A')}
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const colors = {json.dumps(CHART_COLORS)};
|
||||
const strategies = {json.dumps(strategies)};
|
||||
const aggregate = {json.dumps(aggregate)};
|
||||
|
||||
// Radar
|
||||
new Chart(document.getElementById('radarChart'), {{
|
||||
type: 'radar',
|
||||
data: {{
|
||||
labels: ['Context', 'Similarity', 'Faithfulness', 'Consistency'],
|
||||
datasets: strategies.map((s, i) => ({{
|
||||
label: s,
|
||||
data: [
|
||||
aggregate[s]?.avg_context_relevance || 0,
|
||||
aggregate[s]?.avg_answer_similarity || 0,
|
||||
aggregate[s]?.avg_faithfulness || 0,
|
||||
(1 - (aggregate[s]?.hallucination_rate || 0)) * 10
|
||||
],
|
||||
borderColor: colors[i % colors.length],
|
||||
backgroundColor: colors[i % colors.length] + '20',
|
||||
pointBackgroundColor: colors[i % colors.length],
|
||||
borderWidth: 2
|
||||
}}))
|
||||
}},
|
||||
options: {{
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {{ legend: {{ display: false }} }},
|
||||
scales: {{
|
||||
r: {{
|
||||
beginAtZero: true,
|
||||
max: 10,
|
||||
grid: {{ color: '{COLORS["border"]}' }},
|
||||
angleLines: {{ color: '{COLORS["border"]}' }},
|
||||
pointLabels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'JetBrains Mono'", size: 11 }} }},
|
||||
ticks: {{ display: false }}
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
}});
|
||||
|
||||
// Bar
|
||||
new Chart(document.getElementById('barChart'), {{
|
||||
type: 'bar',
|
||||
data: {{
|
||||
labels: strategies.map(s => s.length > 12 ? s.substring(0, 12) + '...' : s),
|
||||
datasets: [
|
||||
{{ label: 'Context', data: strategies.map(s => aggregate[s]?.avg_context_relevance || 0), backgroundColor: colors[0] }},
|
||||
{{ label: 'Similarity', data: strategies.map(s => aggregate[s]?.avg_answer_similarity || 0), backgroundColor: colors[1] }},
|
||||
{{ label: 'Faithfulness', data: strategies.map(s => aggregate[s]?.avg_faithfulness || 0), backgroundColor: colors[2] }}
|
||||
]
|
||||
}},
|
||||
options: {{
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {{ legend: {{ position: 'bottom', labels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'Inter'" }} }} }} }},
|
||||
scales: {{
|
||||
x: {{ grid: {{ display: false }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }},
|
||||
y: {{ beginAtZero: true, max: 10, grid: {{ color: '{COLORS["border"]}' }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }}
|
||||
}}
|
||||
}}
|
||||
}});
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
# ── Helper Functions ──────────────────────────────────────────────
|
||||
|
||||
def _calculate_rankings(aggregate: dict, strategies: list) -> list:
|
||||
"""Calculate overall scores and rank strategies."""
|
||||
rankings = []
|
||||
for strategy in strategies:
|
||||
m = aggregate.get(strategy, {})
|
||||
overall = (
|
||||
m.get("avg_context_relevance", 0) * 0.3
|
||||
+ m.get("avg_answer_similarity", 0) * 0.4
|
||||
+ m.get("avg_faithfulness", 0) * 0.3
|
||||
) * (1 - m.get("hallucination_rate", 0))
|
||||
rankings.append((strategy, round(overall, 2)))
|
||||
rankings.sort(key=lambda x: x[1], reverse=True)
|
||||
return rankings
|
||||
|
||||
|
||||
def _get_recommendation_text(rankings: list, aggregate: dict) -> str:
|
||||
"""Generate recommendation text for the winner."""
|
||||
if not rankings:
|
||||
return "No data available."
|
||||
|
||||
best = rankings[0][0]
|
||||
m = aggregate.get(best, {})
|
||||
|
||||
parts = []
|
||||
if m.get("avg_context_relevance", 0) >= 8:
|
||||
parts.append("strong context retrieval")
|
||||
if m.get("avg_faithfulness", 0) >= 8:
|
||||
parts.append("high faithfulness")
|
||||
if m.get("hallucination_rate", 0) < 0.1:
|
||||
parts.append("low hallucination")
|
||||
|
||||
if parts:
|
||||
return f"{best} excels in {', '.join(parts)}, making it the most reliable choice for your use case."
|
||||
return f"{best} achieved the highest overall score across all evaluation metrics."
|
||||
|
||||
|
||||
def _build_strategy_cards(rankings: list, aggregate: dict) -> str:
|
||||
"""Build strategy comparison cards."""
|
||||
cards = ""
|
||||
for rank, (strategy, overall) in enumerate(rankings, 1):
|
||||
m = aggregate.get(strategy, {})
|
||||
rank_class = f"rank-{rank}" if rank <= 3 else ""
|
||||
|
||||
cards += f"""
|
||||
<div class="strategy-card {rank_class}">
|
||||
<div class="rank-badge">{rank}</div>
|
||||
<div class="name">{strategy}</div>
|
||||
{_metric_bar("Context", m.get("avg_context_relevance", 0), "teal")}
|
||||
{_metric_bar("Similarity", m.get("avg_answer_similarity", 0), "accent")}
|
||||
{_metric_bar("Faithfulness", m.get("avg_faithfulness", 0), "violet")}
|
||||
{_metric_bar("No Hallucination", (1 - m.get("hallucination_rate", 0)) * 10, "teal" if m.get("hallucination_rate", 0) <= 0.1 else "rose")}
|
||||
</div>"""
|
||||
|
||||
return cards
|
||||
|
||||
|
||||
def _metric_bar(label: str, value: float, color_class: str) -> str:
|
||||
"""Build a single metric bar."""
|
||||
width = min(value * 10, 100)
|
||||
return f"""
|
||||
<div class="metric">
|
||||
<div class="label">{label}</div>
|
||||
<div class="bar-bg"><div class="bar-fill {color_class}" style="width: {width}%;"></div></div>
|
||||
<div class="value">{value:.1f}</div>
|
||||
</div>"""
|
||||
|
||||
|
||||
def _generate_decision_insights(rankings: list, aggregate: dict) -> str:
|
||||
"""Generate decision-focused insights."""
|
||||
insights = []
|
||||
|
||||
if rankings:
|
||||
best = rankings[0][0]
|
||||
worst = rankings[-1][0]
|
||||
gap = (1 - rankings[-1][1] / rankings[0][1]) * 100 if rankings[0][1] > 0 else 0
|
||||
|
||||
insights.append(f"""
|
||||
<div class="insight positive">
|
||||
<div class="title">Recommended Strategy</div>
|
||||
<div class="desc">{best} is the best choice with {rankings[0][1]:.2f} overall score.</div>
|
||||
</div>""")
|
||||
|
||||
if gap > 20:
|
||||
insights.append(f"""
|
||||
<div class="insight warning">
|
||||
<div class="title">Avoid</div>
|
||||
<div class="desc">{worst} underperforms by {gap:.0f}%. Use only if specific constraints require it.</div>
|
||||
</div>""")
|
||||
|
||||
# Cost vs quality
|
||||
best_m = aggregate.get(best, {})
|
||||
if best_m.get("avg_context_relevance", 0) >= 8 and best_m.get("hallucination_rate", 0) < 0.1:
|
||||
insights.append(f"""
|
||||
<div class="insight positive">
|
||||
<div class="title">Quality Assessment</div>
|
||||
<div class="desc">{best} delivers high-quality answers with reliable context retrieval.</div>
|
||||
</div>""")
|
||||
else:
|
||||
insights.append(f"""
|
||||
<div class="insight info">
|
||||
<div class="title">Quality Note</div>
|
||||
<div class="desc">Consider tuning parameters or adding more context for better results.</div>
|
||||
</div>""")
|
||||
|
||||
return "\n".join(insights)
|
||||
|
||||
|
||||
def _build_technical_aggregate_rows(rankings: list, aggregate: dict) -> str:
|
||||
"""Build technical aggregate table rows."""
|
||||
rows = ""
|
||||
for rank, (strategy, overall) in enumerate(rankings, 1):
|
||||
m = aggregate.get(strategy, {})
|
||||
rows += f"""
|
||||
<tr>
|
||||
<td><strong>{strategy}</strong></td>
|
||||
<td>{_pill(m.get('avg_context_relevance', 0))}</td>
|
||||
<td>{_pill(m.get('avg_answer_similarity', 0))}</td>
|
||||
<td>{_pill(m.get('avg_faithfulness', 0))}</td>
|
||||
<td>{_hall_pill(m.get('hallucination_rate', 0))}</td>
|
||||
<td>{m.get('total_questions', 0)}</td>
|
||||
<td>{m.get('failed_questions', 0)}</td>
|
||||
<td><strong style="font-family: {FONT_DATA};">{overall:.2f}</strong></td>
|
||||
</tr>"""
|
||||
return rows
|
||||
|
||||
|
||||
def _build_strategy_headers(strategies: list) -> str:
|
||||
"""Build table headers."""
|
||||
return "".join(f'<th>{s[:12]}</th>' for s in strategies)
|
||||
|
||||
|
||||
def _build_per_question_rows(per_question: list, strategies: list) -> str:
|
||||
"""Build per-question rows."""
|
||||
rows = ""
|
||||
for qr in per_question:
|
||||
q_id = qr.get("question_id", "")
|
||||
q_text = qr.get("question", "")[:40]
|
||||
category = qr.get("category", "")
|
||||
difficulty = qr.get("difficulty", "")
|
||||
|
||||
cells = ""
|
||||
for strategy in strategies:
|
||||
strat = qr.get("strategies", {}).get(strategy, {})
|
||||
scores = strat.get("scores", {})
|
||||
sim = scores.get("answer_similarity", 0)
|
||||
cells += f"<td>{_pill(sim)}</td>"
|
||||
|
||||
rows += f"""
|
||||
<tr>
|
||||
<td style="font-family: {FONT_DATA}; color: var(--text-muted);">{q_id}</td>
|
||||
<td>{q_text}...</td>
|
||||
<td style="color: var(--text-muted);">{category}</td>
|
||||
<td><span class="pill {'high' if difficulty == 'easy' else 'mid' if difficulty == 'medium' else 'low'}">{difficulty}</span></td>
|
||||
{cells}
|
||||
</tr>"""
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _build_detailed_rows(per_question: list, strategies: list) -> str:
|
||||
"""Build detailed score rows."""
|
||||
rows = ""
|
||||
for qr in per_question:
|
||||
q_id = qr.get("question_id", "")
|
||||
for strategy in strategies:
|
||||
strat = qr.get("strategies", {}).get(strategy, {})
|
||||
scores = strat.get("scores", {})
|
||||
answer = strat.get("answer", "")[:60]
|
||||
|
||||
rows += f"""
|
||||
<tr>
|
||||
<td style="font-family: {FONT_DATA}; font-size: 12px;">{q_id}</td>
|
||||
<td>{strategy}</td>
|
||||
<td>{_pill(scores.get('context_relevance', 0))}</td>
|
||||
<td>{_pill(scores.get('answer_similarity', 0))}</td>
|
||||
<td>{_pill(scores.get('faithfulness', 0))}</td>
|
||||
<td>{'✓' if not scores.get('hallucination', False) else '✗'}</td>
|
||||
<td style="font-size: 13px; color: var(--text-muted);">{answer}...</td>
|
||||
</tr>"""
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _build_token_rows(per_question: list, strategies: list) -> str:
|
||||
"""Build token usage rows."""
|
||||
rows = ""
|
||||
for qr in per_question:
|
||||
q_id = qr.get("question_id", "")
|
||||
for strategy in strategies:
|
||||
strat = qr.get("strategies", {}).get(strategy, {})
|
||||
usage = strat.get("token_usage", {})
|
||||
|
||||
rows += f"""
|
||||
<tr>
|
||||
<td style="font-family: {FONT_DATA}; font-size: 12px;">{q_id}</td>
|
||||
<td>{strategy}</td>
|
||||
<td>{usage.get('prompt_tokens', 0)}</td>
|
||||
<td>{usage.get('completion_tokens', 0)}</td>
|
||||
<td><strong>{usage.get('total_tokens', 0)}</strong></td>
|
||||
</tr>"""
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _pill(score: float) -> str:
|
||||
"""Create a score pill."""
|
||||
if score >= 8:
|
||||
css = "high"
|
||||
elif score >= 6:
|
||||
css = "mid"
|
||||
else:
|
||||
css = "low"
|
||||
return f'<span class="pill {css}">{score:.1f}</span>'
|
||||
|
||||
|
||||
def _hall_pill(rate: float) -> str:
|
||||
"""Create hallucination rate pill."""
|
||||
if rate <= 0.1:
|
||||
css = "high"
|
||||
elif rate <= 0.2:
|
||||
css = "mid"
|
||||
else:
|
||||
css = "low"
|
||||
return f'<span class="pill {css}">{rate*100:.0f}%</span>'
|
||||
|
||||
|
||||
# ── Legacy compatibility ──────────────────────────────────────────
|
||||
|
||||
def generate_enhanced_report(experiment: dict) -> str:
|
||||
"""Backward-compatible entry point (defaults to managerial)."""
|
||||
return generate_managerial_report(experiment)
|
||||
Reference in New Issue
Block a user