Mahdi Bazrafshan
ef8bab5962
Why: - Need LLM-as-Judge evaluation for automated scoring - Need benchmark orchestration to run questions × strategies - Need HTML report generation with two views (managerial/technical) Changes: - evaluation.py: LLM-as-Judge scoring on 4 metrics (context, similarity, faithfulness, hallucination) - benchmark_service.py: Orchestration with per-strategy failure isolation - report.py: Dual-view HTML reports with dark mode, charts, and strategy cards
816 lines
34 KiB
Python
816 lines
34 KiB
Python
"""Enhanced HTML report generator with two views:
|
|
1. Managerial: Decision-focused, high-level insights
|
|
2. Technical: Full observability with detailed data
|
|
|
|
Design: Dark mode, amber/teal accents, Inter + JetBrains Mono.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
|
|
# ── Design Tokens ─────────────────────────────────────────────────
|
|
|
|
COLORS = {
|
|
"bg": "#0F1419",
|
|
"surface": "#1A2332",
|
|
"surface-hover": "#243044",
|
|
"border": "#2D3A4A",
|
|
"text": "#E8EDF2",
|
|
"text-muted": "#7A8BA0",
|
|
"accent": "#F59E0B",
|
|
"accent-dim": "#D97706",
|
|
"teal": "#14B8A6",
|
|
"rose": "#F43F5E",
|
|
"violet": "#8B5CF6",
|
|
"emerald": "#10B981",
|
|
}
|
|
|
|
CHART_COLORS = ["#F59E0B", "#14B8A6", "#8B5CF6", "#F43F5E", "#3B82F6"]
|
|
|
|
FONT_DISPLAY = "'Inter', -apple-system, sans-serif"
|
|
FONT_DATA = "'JetBrains Mono', 'Fira Code', monospace"
|
|
|
|
|
|
# ── Main Entry Points ─────────────────────────────────────────────
|
|
|
|
def generate_report(experiment: dict, view: str = "managerial") -> str:
|
|
"""Generate report for specified view.
|
|
|
|
Args:
|
|
experiment: Experiment data from SQLite
|
|
view: "managerial" or "technical"
|
|
"""
|
|
if view == "technical":
|
|
return generate_technical_report(experiment)
|
|
return generate_managerial_report(experiment)
|
|
|
|
|
|
def generate_managerial_report(experiment: dict) -> str:
|
|
"""Managerial view: Decision-focused, high-level insights."""
|
|
config = experiment.get("benchmark_config", {})
|
|
aggregate = experiment.get("aggregate_metrics", {})
|
|
per_question = experiment.get("per_question", [])
|
|
strategies = experiment.get("strategies_used", [])
|
|
rankings = _calculate_rankings(aggregate, strategies)
|
|
|
|
# Calculate cost from token usage in per-question data
|
|
total_prompt_tokens = 0
|
|
total_completion_tokens = 0
|
|
for qr in per_question:
|
|
for strategy in strategies:
|
|
strat = qr.get("strategies", {}).get(strategy, {})
|
|
usage = strat.get("token_usage", {})
|
|
total_prompt_tokens += usage.get("prompt_tokens", 0)
|
|
total_completion_tokens += usage.get("completion_tokens", 0)
|
|
|
|
estimated_cost = (total_prompt_tokens * 0.15 + total_completion_tokens * 0.60) / 1_000_000
|
|
|
|
html = _base_html(experiment, "Managerial View", f"""
|
|
<!-- Header -->
|
|
<header class="header animate">
|
|
<div class="eyebrow">Benchmark Results</div>
|
|
<h1>Strategy Comparison</h1>
|
|
<div class="meta">
|
|
{config.get('num_questions', 0)} questions ·
|
|
{len(strategies)} strategies ·
|
|
${estimated_cost:.4f} cost
|
|
</div>
|
|
</header>
|
|
|
|
<!-- KPIs -->
|
|
<div class="kpi-row animate delay-1">
|
|
<div class="kpi">
|
|
<div class="label">Best Strategy</div>
|
|
<div class="value accent">{rankings[0][0] if rankings else 'N/A'}</div>
|
|
</div>
|
|
<div class="kpi">
|
|
<div class="label">Overall Score</div>
|
|
<div class="value teal">{rankings[0][1]:.2f}</div>
|
|
</div>
|
|
<div class="kpi">
|
|
<div class="label">Questions</div>
|
|
<div class="value">{config.get('num_questions', 0)}</div>
|
|
</div>
|
|
<div class="kpi">
|
|
<div class="label">Cost</div>
|
|
<div class="value">${estimated_cost:.4f}</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Winner -->
|
|
<div class="winner animate delay-2">
|
|
<div class="medal">🏆</div>
|
|
<div class="content">
|
|
<h2>Recommendation: {rankings[0][0] if rankings else 'N/A'}</h2>
|
|
<p>{_get_recommendation_text(rankings, aggregate)}</p>
|
|
</div>
|
|
<div class="score">{rankings[0][1]:.2f}</div>
|
|
</div>
|
|
|
|
<!-- Strategy Rankings -->
|
|
<div class="section-header animate delay-3">
|
|
<h2>How Strategies Compare</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="strategy-grid animate delay-3">
|
|
{_build_strategy_cards(rankings, aggregate)}
|
|
</div>
|
|
|
|
<!-- Charts -->
|
|
<div class="section-header animate delay-4">
|
|
<h2>Visual Comparison</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="charts-row animate delay-4">
|
|
<div class="chart-panel">
|
|
<h3>Performance Radar</h3>
|
|
<div class="chart-container">
|
|
<canvas id="radarChart"></canvas>
|
|
</div>
|
|
</div>
|
|
<div class="chart-panel">
|
|
<h3>Score Comparison</h3>
|
|
<div class="chart-container">
|
|
<canvas id="barChart"></canvas>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Decision Insights -->
|
|
<div class="section-header">
|
|
<h2>Decision Guide</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="insights">
|
|
{_generate_decision_insights(rankings, aggregate)}
|
|
</div>
|
|
|
|
<!-- Quick Links -->
|
|
<div class="section-header">
|
|
<h2>Details</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="quick-links">
|
|
<a href="/benchmarks/{experiment.get('id', '')}/report?view=technical" class="link-card">
|
|
<span class="link-icon">🔍</span>
|
|
<span class="link-text">View Full Technical Report</span>
|
|
<span class="link-arrow">→</span>
|
|
</a>
|
|
</div>
|
|
""")
|
|
|
|
return html
|
|
|
|
|
|
def generate_technical_report(experiment: dict) -> str:
|
|
"""Technical view: Full observability with detailed data."""
|
|
config = experiment.get("benchmark_config", {})
|
|
aggregate = experiment.get("aggregate_metrics", {})
|
|
per_question = experiment.get("per_question", [])
|
|
strategies = experiment.get("strategies_used", [])
|
|
rankings = _calculate_rankings(aggregate, strategies)
|
|
|
|
# Calculate metrics from available data
|
|
total_queries = len(per_question) * len(strategies)
|
|
total_questions = config.get("num_questions", len(per_question))
|
|
top_k = config.get("top_k", 5)
|
|
|
|
# Calculate token usage from per-question data
|
|
total_prompt_tokens = 0
|
|
total_completion_tokens = 0
|
|
for qr in per_question:
|
|
for strategy in strategies:
|
|
strat = qr.get("strategies", {}).get(strategy, {})
|
|
usage = strat.get("token_usage", {})
|
|
total_prompt_tokens += usage.get("prompt_tokens", 0)
|
|
total_completion_tokens += usage.get("completion_tokens", 0)
|
|
|
|
total_tokens = total_prompt_tokens + total_completion_tokens
|
|
|
|
# Estimate cost based on token usage (GPT-4o-mini pricing)
|
|
# Input: $0.15/1M tokens, Output: $0.60/1M tokens
|
|
estimated_cost = (total_prompt_tokens * 0.15 + total_completion_tokens * 0.60) / 1_000_000
|
|
|
|
# Estimate latency (rough: ~0.5s per query + ~1s per evaluation)
|
|
estimated_latency = total_queries * 1.5
|
|
|
|
html = _base_html(experiment, "Technical View", f"""
|
|
<!-- Header -->
|
|
<header class="header animate">
|
|
<div class="eyebrow">Technical Report</div>
|
|
<h1>Full Observability</h1>
|
|
<div class="meta">
|
|
Experiment: {experiment.get('id', 'N/A')[:16]}... ·
|
|
{total_questions} questions ·
|
|
{len(strategies)} strategies
|
|
</div>
|
|
</header>
|
|
|
|
<!-- KPIs -->
|
|
<div class="kpi-row animate delay-1">
|
|
<div class="kpi">
|
|
<div class="label">Total Queries</div>
|
|
<div class="value accent">{total_queries}</div>
|
|
</div>
|
|
<div class="kpi">
|
|
<div class="label">Total Tokens</div>
|
|
<div class="value">{total_tokens:,}</div>
|
|
</div>
|
|
<div class="kpi">
|
|
<div class="label">Cost</div>
|
|
<div class="value teal">${estimated_cost:.4f}</div>
|
|
</div>
|
|
<div class="kpi">
|
|
<div class="label">Tokens per Query</div>
|
|
<div class="value">{total_tokens // max(total_queries, 1):,}</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Aggregate Metrics Table -->
|
|
<div class="section-header animate delay-2">
|
|
<h2>Aggregate Metrics</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="table-wrapper animate delay-2">
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Strategy</th>
|
|
<th>Context Relevance</th>
|
|
<th>Answer Similarity</th>
|
|
<th>Faithfulness</th>
|
|
<th>Hallucination</th>
|
|
<th>Questions</th>
|
|
<th>Failed</th>
|
|
<th>Overall</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{_build_technical_aggregate_rows(rankings, aggregate)}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<!-- Charts -->
|
|
<div class="section-header animate delay-3">
|
|
<h2>Visual Analysis</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="charts-row animate delay-3">
|
|
<div class="chart-panel">
|
|
<h3>Multi-Metric Radar</h3>
|
|
<div class="chart-container">
|
|
<canvas id="radarChart"></canvas>
|
|
</div>
|
|
</div>
|
|
<div class="chart-panel">
|
|
<h3>Score Distribution</h3>
|
|
<div class="chart-container">
|
|
<canvas id="barChart"></canvas>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Per-Question Results -->
|
|
<div class="section-header">
|
|
<h2>Per-Question Results</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="table-wrapper">
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>ID</th>
|
|
<th>Question</th>
|
|
<th>Category</th>
|
|
<th>Difficulty</th>
|
|
{_build_strategy_headers(strategies)}
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{_build_per_question_rows(per_question, strategies)}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<!-- Detailed Scores -->
|
|
<div class="section-header">
|
|
<h2>Detailed Scores</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="table-wrapper">
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Q-ID</th>
|
|
<th>Strategy</th>
|
|
<th>Context</th>
|
|
<th>Similarity</th>
|
|
<th>Faithfulness</th>
|
|
<th>Hallucination</th>
|
|
<th>Answer Preview</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{_build_detailed_rows(per_question, strategies)}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<!-- Token Usage -->
|
|
<div class="section-header">
|
|
<h2>Token Usage</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="table-wrapper">
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>Q-ID</th>
|
|
<th>Strategy</th>
|
|
<th>Prompt Tokens</th>
|
|
<th>Completion Tokens</th>
|
|
<th>Total Tokens</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{_build_token_rows(per_question, strategies)}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<!-- Quick Links -->
|
|
<div class="section-header">
|
|
<h2>Navigation</h2>
|
|
<div class="line"></div>
|
|
</div>
|
|
<div class="quick-links">
|
|
<a href="/benchmarks/{experiment.get('id', '')}/report?view=managerial" class="link-card">
|
|
<span class="link-icon">📊</span>
|
|
<span class="link-text">Switch to Managerial View</span>
|
|
<span class="link-arrow">→</span>
|
|
</a>
|
|
</div>
|
|
""")
|
|
|
|
return html
|
|
|
|
|
|
# ── Base HTML Template ────────────────────────────────────────────
|
|
|
|
def _base_html(experiment: dict, title: str, content: str) -> str:
|
|
"""Base HTML wrapper with styles and scripts."""
|
|
strategies = experiment.get("strategies_used", [])
|
|
aggregate = experiment.get("aggregate_metrics", {})
|
|
|
|
return f"""<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>{title}</title>
|
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
|
|
<style>
|
|
*, *::before, *::after {{ margin: 0; padding: 0; box-sizing: border-box; }}
|
|
|
|
:root {{
|
|
--bg: {COLORS['bg']};
|
|
--surface: {COLORS['surface']};
|
|
--surface-hover: {COLORS['surface-hover']};
|
|
--border: {COLORS['border']};
|
|
--text: {COLORS['text']};
|
|
--text-muted: {COLORS['text-muted']};
|
|
--accent: {COLORS['accent']};
|
|
--accent-dim: {COLORS['accent-dim']};
|
|
--teal: {COLORS['teal']};
|
|
--rose: {COLORS['rose']};
|
|
--violet: {COLORS['violet']};
|
|
--emerald: {COLORS['emerald']};
|
|
}}
|
|
|
|
html {{ scroll-behavior: smooth; }}
|
|
|
|
body {{
|
|
font-family: {FONT_DISPLAY};
|
|
background: var(--bg);
|
|
color: var(--text);
|
|
line-height: 1.6;
|
|
min-height: 100vh;
|
|
}}
|
|
|
|
.page {{ max-width: 1280px; margin: 0 auto; padding: 48px 32px; }}
|
|
|
|
@keyframes fadeUp {{
|
|
from {{ opacity: 0; transform: translateY(20px); }}
|
|
to {{ opacity: 1; transform: translateY(0); }}
|
|
}}
|
|
.animate {{ animation: fadeUp 0.6s ease-out forwards; opacity: 0; }}
|
|
.delay-1 {{ animation-delay: 0.1s; }}
|
|
.delay-2 {{ animation-delay: 0.2s; }}
|
|
.delay-3 {{ animation-delay: 0.3s; }}
|
|
.delay-4 {{ animation-delay: 0.4s; }}
|
|
|
|
.header {{ margin-bottom: 48px; padding-bottom: 32px; border-bottom: 1px solid var(--border); }}
|
|
.header .eyebrow {{ font-family: {FONT_DATA}; font-size: 12px; color: var(--accent); text-transform: uppercase; letter-spacing: 2px; margin-bottom: 12px; }}
|
|
.header h1 {{ font-size: 36px; font-weight: 700; letter-spacing: -0.5px; margin-bottom: 8px; }}
|
|
.header .meta {{ font-size: 14px; color: var(--text-muted); font-family: {FONT_DATA}; }}
|
|
|
|
.kpi-row {{ display: grid; grid-template-columns: repeat(4, 1fr); gap: 24px; margin-bottom: 48px; }}
|
|
.kpi {{ padding: 24px; background: var(--surface); border: 1px solid var(--border); border-radius: 12px; transition: border-color 0.2s ease, transform 0.2s ease; }}
|
|
.kpi:hover {{ border-color: var(--accent); transform: translateY(-2px); }}
|
|
.kpi .label {{ font-size: 12px; color: var(--text-muted); text-transform: uppercase; letter-spacing: 1px; margin-bottom: 8px; }}
|
|
.kpi .value {{ font-family: {FONT_DATA}; font-size: 32px; font-weight: 600; color: var(--text); }}
|
|
.kpi .value.accent {{ color: var(--accent); }}
|
|
.kpi .value.teal {{ color: var(--teal); }}
|
|
|
|
.winner {{ display: flex; align-items: center; gap: 24px; padding: 32px; background: linear-gradient(135deg, var(--surface) 0%, var(--surface-hover) 100%); border: 1px solid var(--accent); border-radius: 16px; margin-bottom: 48px; }}
|
|
.winner .medal {{ width: 64px; height: 64px; background: var(--accent); border-radius: 50%; display: flex; align-items: center; justify-content: center; font-size: 28px; flex-shrink: 0; }}
|
|
.winner .content h2 {{ font-size: 24px; font-weight: 600; margin-bottom: 4px; }}
|
|
.winner .content p {{ color: var(--text-muted); font-size: 14px; max-width: 600px; }}
|
|
.winner .score {{ margin-left: auto; font-family: {FONT_DATA}; font-size: 48px; font-weight: 700; color: var(--accent); }}
|
|
|
|
.section-header {{ display: flex; align-items: center; gap: 12px; margin-bottom: 24px; }}
|
|
.section-header h2 {{ font-size: 20px; font-weight: 600; }}
|
|
.section-header .line {{ flex: 1; height: 1px; background: var(--border); }}
|
|
|
|
.strategy-grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 16px; margin-bottom: 48px; }}
|
|
.strategy-card {{ background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 24px; position: relative; overflow: hidden; transition: border-color 0.2s ease; }}
|
|
.strategy-card:hover {{ border-color: var(--text-muted); }}
|
|
.strategy-card.rank-1 {{ border-color: var(--accent); background: linear-gradient(180deg, rgba(245,158,11,0.08) 0%, var(--surface) 100%); }}
|
|
.strategy-card .rank-badge {{ position: absolute; top: 16px; right: 16px; width: 28px; height: 28px; border-radius: 50%; display: flex; align-items: center; justify-content: center; font-family: {FONT_DATA}; font-size: 12px; font-weight: 600; }}
|
|
.strategy-card.rank-1 .rank-badge {{ background: var(--accent); color: var(--bg); }}
|
|
.strategy-card.rank-2 .rank-badge {{ background: var(--text-muted); color: var(--bg); }}
|
|
.strategy-card.rank-3 .rank-badge {{ background: var(--accent-dim); color: var(--bg); }}
|
|
.strategy-card .name {{ font-size: 14px; font-weight: 600; margin-bottom: 16px; padding-right: 40px; }}
|
|
.strategy-card .metric {{ margin-bottom: 12px; }}
|
|
.strategy-card .metric .label {{ font-size: 11px; color: var(--text-muted); margin-bottom: 4px; }}
|
|
.strategy-card .metric .bar-bg {{ height: 6px; background: var(--border); border-radius: 3px; overflow: hidden; }}
|
|
.strategy-card .metric .bar-fill {{ height: 100%; border-radius: 3px; transition: width 0.8s ease-out; }}
|
|
.strategy-card .metric .bar-fill.teal {{ background: var(--teal); }}
|
|
.strategy-card .metric .bar-fill.accent {{ background: var(--accent); }}
|
|
.strategy-card .metric .bar-fill.violet {{ background: var(--violet); }}
|
|
.strategy-card .metric .bar-fill.rose {{ background: var(--rose); }}
|
|
.strategy-card .metric .value {{ font-family: {FONT_DATA}; font-size: 12px; color: var(--text); margin-top: 4px; }}
|
|
|
|
.charts-row {{ display: grid; grid-template-columns: 1fr 1fr; gap: 24px; margin-bottom: 48px; }}
|
|
.chart-panel {{ background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 24px; }}
|
|
.chart-panel h3 {{ font-size: 14px; font-weight: 500; color: var(--text-muted); margin-bottom: 16px; }}
|
|
.chart-container {{ position: relative; height: 280px; }}
|
|
|
|
.insights {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 16px; margin-bottom: 48px; }}
|
|
.insight {{ padding: 20px; border-radius: 12px; border-left: 3px solid; }}
|
|
.insight.positive {{ background: rgba(16,185,129,0.1); border-color: var(--emerald); }}
|
|
.insight.warning {{ background: rgba(244,63,94,0.1); border-color: var(--rose); }}
|
|
.insight.info {{ background: rgba(139,92,246,0.1); border-color: var(--violet); }}
|
|
.insight .title {{ font-size: 13px; font-weight: 600; margin-bottom: 4px; }}
|
|
.insight .desc {{ font-size: 13px; color: var(--text-muted); }}
|
|
|
|
.table-wrapper {{ background: var(--surface); border: 1px solid var(--border); border-radius: 12px; overflow: hidden; margin-bottom: 48px; }}
|
|
.table-wrapper table {{ width: 100%; }}
|
|
.table-wrapper th {{ background: var(--surface-hover); padding: 12px 16px; text-align: left; font-size: 11px; font-weight: 600; color: var(--text-muted); text-transform: uppercase; letter-spacing: 0.5px; border-bottom: 1px solid var(--border); }}
|
|
.table-wrapper td {{ padding: 12px 16px; font-size: 13px; border-bottom: 1px solid var(--border); }}
|
|
.table-wrapper tr:last-child td {{ border-bottom: none; }}
|
|
.table-wrapper tr:hover td {{ background: var(--surface-hover); }}
|
|
|
|
.pill {{ display: inline-block; padding: 2px 8px; border-radius: 6px; font-family: {FONT_DATA}; font-size: 12px; font-weight: 500; }}
|
|
.pill.high {{ background: rgba(16,185,129,0.15); color: var(--emerald); }}
|
|
.pill.mid {{ background: rgba(245,158,11,0.15); color: var(--accent); }}
|
|
.pill.low {{ background: rgba(244,63,94,0.15); color: var(--rose); }}
|
|
|
|
.quick-links {{ margin-bottom: 48px; }}
|
|
.link-card {{ display: flex; align-items: center; gap: 16px; padding: 20px 24px; background: var(--surface); border: 1px solid var(--border); border-radius: 12px; text-decoration: none; color: var(--text); transition: border-color 0.2s ease, transform 0.2s ease; }}
|
|
.link-card:hover {{ border-color: var(--accent); transform: translateX(4px); }}
|
|
.link-icon {{ font-size: 24px; }}
|
|
.link-text {{ flex: 1; font-weight: 500; }}
|
|
.link-arrow {{ color: var(--accent); font-size: 18px; }}
|
|
|
|
.footer {{ padding-top: 32px; border-top: 1px solid var(--border); font-size: 12px; color: var(--text-muted); text-align: center; }}
|
|
|
|
@media (max-width: 768px) {{
|
|
.page {{ padding: 24px 16px; }}
|
|
.kpi-row {{ grid-template-columns: repeat(2, 1fr); }}
|
|
.charts-row {{ grid-template-columns: 1fr; }}
|
|
.winner {{ flex-direction: column; text-align: center; }}
|
|
.winner .score {{ margin-left: 0; margin-top: 16px; }}
|
|
}}
|
|
|
|
@media (prefers-reduced-motion: reduce) {{
|
|
.animate {{ animation: none; opacity: 1; }}
|
|
}}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="page">
|
|
{content}
|
|
|
|
<footer class="footer">
|
|
Generated by RAG Chunking Benchmarker · {experiment.get('created_at', 'N/A')}
|
|
</footer>
|
|
</div>
|
|
|
|
<script>
|
|
const colors = {json.dumps(CHART_COLORS)};
|
|
const strategies = {json.dumps(strategies)};
|
|
const aggregate = {json.dumps(aggregate)};
|
|
|
|
// Radar
|
|
new Chart(document.getElementById('radarChart'), {{
|
|
type: 'radar',
|
|
data: {{
|
|
labels: ['Context', 'Similarity', 'Faithfulness', 'Consistency'],
|
|
datasets: strategies.map((s, i) => ({{
|
|
label: s,
|
|
data: [
|
|
aggregate[s]?.avg_context_relevance || 0,
|
|
aggregate[s]?.avg_answer_similarity || 0,
|
|
aggregate[s]?.avg_faithfulness || 0,
|
|
(1 - (aggregate[s]?.hallucination_rate || 0)) * 10
|
|
],
|
|
borderColor: colors[i % colors.length],
|
|
backgroundColor: colors[i % colors.length] + '20',
|
|
pointBackgroundColor: colors[i % colors.length],
|
|
borderWidth: 2
|
|
}}))
|
|
}},
|
|
options: {{
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
plugins: {{ legend: {{ display: false }} }},
|
|
scales: {{
|
|
r: {{
|
|
beginAtZero: true,
|
|
max: 10,
|
|
grid: {{ color: '{COLORS["border"]}' }},
|
|
angleLines: {{ color: '{COLORS["border"]}' }},
|
|
pointLabels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'JetBrains Mono'", size: 11 }} }},
|
|
ticks: {{ display: false }}
|
|
}}
|
|
}}
|
|
}}
|
|
}});
|
|
|
|
// Bar
|
|
new Chart(document.getElementById('barChart'), {{
|
|
type: 'bar',
|
|
data: {{
|
|
labels: strategies.map(s => s.length > 12 ? s.substring(0, 12) + '...' : s),
|
|
datasets: [
|
|
{{ label: 'Context', data: strategies.map(s => aggregate[s]?.avg_context_relevance || 0), backgroundColor: colors[0] }},
|
|
{{ label: 'Similarity', data: strategies.map(s => aggregate[s]?.avg_answer_similarity || 0), backgroundColor: colors[1] }},
|
|
{{ label: 'Faithfulness', data: strategies.map(s => aggregate[s]?.avg_faithfulness || 0), backgroundColor: colors[2] }}
|
|
]
|
|
}},
|
|
options: {{
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
plugins: {{ legend: {{ position: 'bottom', labels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'Inter'" }} }} }} }},
|
|
scales: {{
|
|
x: {{ grid: {{ display: false }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }},
|
|
y: {{ beginAtZero: true, max: 10, grid: {{ color: '{COLORS["border"]}' }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }}
|
|
}}
|
|
}}
|
|
}});
|
|
</script>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
# ── Helper Functions ──────────────────────────────────────────────
|
|
|
|
def _calculate_rankings(aggregate: dict, strategies: list) -> list:
|
|
"""Calculate overall scores and rank strategies."""
|
|
rankings = []
|
|
for strategy in strategies:
|
|
m = aggregate.get(strategy, {})
|
|
overall = (
|
|
m.get("avg_context_relevance", 0) * 0.3
|
|
+ m.get("avg_answer_similarity", 0) * 0.4
|
|
+ m.get("avg_faithfulness", 0) * 0.3
|
|
) * (1 - m.get("hallucination_rate", 0))
|
|
rankings.append((strategy, round(overall, 2)))
|
|
rankings.sort(key=lambda x: x[1], reverse=True)
|
|
return rankings
|
|
|
|
|
|
def _get_recommendation_text(rankings: list, aggregate: dict) -> str:
|
|
"""Generate recommendation text for the winner."""
|
|
if not rankings:
|
|
return "No data available."
|
|
|
|
best = rankings[0][0]
|
|
m = aggregate.get(best, {})
|
|
|
|
parts = []
|
|
if m.get("avg_context_relevance", 0) >= 8:
|
|
parts.append("strong context retrieval")
|
|
if m.get("avg_faithfulness", 0) >= 8:
|
|
parts.append("high faithfulness")
|
|
if m.get("hallucination_rate", 0) < 0.1:
|
|
parts.append("low hallucination")
|
|
|
|
if parts:
|
|
return f"{best} excels in {', '.join(parts)}, making it the most reliable choice for your use case."
|
|
return f"{best} achieved the highest overall score across all evaluation metrics."
|
|
|
|
|
|
def _build_strategy_cards(rankings: list, aggregate: dict) -> str:
|
|
"""Build strategy comparison cards."""
|
|
cards = ""
|
|
for rank, (strategy, overall) in enumerate(rankings, 1):
|
|
m = aggregate.get(strategy, {})
|
|
rank_class = f"rank-{rank}" if rank <= 3 else ""
|
|
|
|
cards += f"""
|
|
<div class="strategy-card {rank_class}">
|
|
<div class="rank-badge">{rank}</div>
|
|
<div class="name">{strategy}</div>
|
|
{_metric_bar("Context", m.get("avg_context_relevance", 0), "teal")}
|
|
{_metric_bar("Similarity", m.get("avg_answer_similarity", 0), "accent")}
|
|
{_metric_bar("Faithfulness", m.get("avg_faithfulness", 0), "violet")}
|
|
{_metric_bar("No Hallucination", (1 - m.get("hallucination_rate", 0)) * 10, "teal" if m.get("hallucination_rate", 0) <= 0.1 else "rose")}
|
|
</div>"""
|
|
|
|
return cards
|
|
|
|
|
|
def _metric_bar(label: str, value: float, color_class: str) -> str:
|
|
"""Build a single metric bar."""
|
|
width = min(value * 10, 100)
|
|
return f"""
|
|
<div class="metric">
|
|
<div class="label">{label}</div>
|
|
<div class="bar-bg"><div class="bar-fill {color_class}" style="width: {width}%;"></div></div>
|
|
<div class="value">{value:.1f}</div>
|
|
</div>"""
|
|
|
|
|
|
def _generate_decision_insights(rankings: list, aggregate: dict) -> str:
|
|
"""Generate decision-focused insights."""
|
|
insights = []
|
|
|
|
if rankings:
|
|
best = rankings[0][0]
|
|
worst = rankings[-1][0]
|
|
gap = (1 - rankings[-1][1] / rankings[0][1]) * 100 if rankings[0][1] > 0 else 0
|
|
|
|
insights.append(f"""
|
|
<div class="insight positive">
|
|
<div class="title">Recommended Strategy</div>
|
|
<div class="desc">{best} is the best choice with {rankings[0][1]:.2f} overall score.</div>
|
|
</div>""")
|
|
|
|
if gap > 20:
|
|
insights.append(f"""
|
|
<div class="insight warning">
|
|
<div class="title">Avoid</div>
|
|
<div class="desc">{worst} underperforms by {gap:.0f}%. Use only if specific constraints require it.</div>
|
|
</div>""")
|
|
|
|
# Cost vs quality
|
|
best_m = aggregate.get(best, {})
|
|
if best_m.get("avg_context_relevance", 0) >= 8 and best_m.get("hallucination_rate", 0) < 0.1:
|
|
insights.append(f"""
|
|
<div class="insight positive">
|
|
<div class="title">Quality Assessment</div>
|
|
<div class="desc">{best} delivers high-quality answers with reliable context retrieval.</div>
|
|
</div>""")
|
|
else:
|
|
insights.append(f"""
|
|
<div class="insight info">
|
|
<div class="title">Quality Note</div>
|
|
<div class="desc">Consider tuning parameters or adding more context for better results.</div>
|
|
</div>""")
|
|
|
|
return "\n".join(insights)
|
|
|
|
|
|
def _build_technical_aggregate_rows(rankings: list, aggregate: dict) -> str:
|
|
"""Build technical aggregate table rows."""
|
|
rows = ""
|
|
for rank, (strategy, overall) in enumerate(rankings, 1):
|
|
m = aggregate.get(strategy, {})
|
|
rows += f"""
|
|
<tr>
|
|
<td><strong>{strategy}</strong></td>
|
|
<td>{_pill(m.get('avg_context_relevance', 0))}</td>
|
|
<td>{_pill(m.get('avg_answer_similarity', 0))}</td>
|
|
<td>{_pill(m.get('avg_faithfulness', 0))}</td>
|
|
<td>{_hall_pill(m.get('hallucination_rate', 0))}</td>
|
|
<td>{m.get('total_questions', 0)}</td>
|
|
<td>{m.get('failed_questions', 0)}</td>
|
|
<td><strong style="font-family: {FONT_DATA};">{overall:.2f}</strong></td>
|
|
</tr>"""
|
|
return rows
|
|
|
|
|
|
def _build_strategy_headers(strategies: list) -> str:
|
|
"""Build table headers."""
|
|
return "".join(f'<th>{s[:12]}</th>' for s in strategies)
|
|
|
|
|
|
def _build_per_question_rows(per_question: list, strategies: list) -> str:
|
|
"""Build per-question rows."""
|
|
rows = ""
|
|
for qr in per_question:
|
|
q_id = qr.get("question_id", "")
|
|
q_text = qr.get("question", "")[:40]
|
|
category = qr.get("category", "")
|
|
difficulty = qr.get("difficulty", "")
|
|
|
|
cells = ""
|
|
for strategy in strategies:
|
|
strat = qr.get("strategies", {}).get(strategy, {})
|
|
scores = strat.get("scores", {})
|
|
sim = scores.get("answer_similarity", 0)
|
|
cells += f"<td>{_pill(sim)}</td>"
|
|
|
|
rows += f"""
|
|
<tr>
|
|
<td style="font-family: {FONT_DATA}; color: var(--text-muted);">{q_id}</td>
|
|
<td>{q_text}...</td>
|
|
<td style="color: var(--text-muted);">{category}</td>
|
|
<td><span class="pill {'high' if difficulty == 'easy' else 'mid' if difficulty == 'medium' else 'low'}">{difficulty}</span></td>
|
|
{cells}
|
|
</tr>"""
|
|
|
|
return rows
|
|
|
|
|
|
def _build_detailed_rows(per_question: list, strategies: list) -> str:
|
|
"""Build detailed score rows."""
|
|
rows = ""
|
|
for qr in per_question:
|
|
q_id = qr.get("question_id", "")
|
|
for strategy in strategies:
|
|
strat = qr.get("strategies", {}).get(strategy, {})
|
|
scores = strat.get("scores", {})
|
|
answer = strat.get("answer", "")[:60]
|
|
|
|
rows += f"""
|
|
<tr>
|
|
<td style="font-family: {FONT_DATA}; font-size: 12px;">{q_id}</td>
|
|
<td>{strategy}</td>
|
|
<td>{_pill(scores.get('context_relevance', 0))}</td>
|
|
<td>{_pill(scores.get('answer_similarity', 0))}</td>
|
|
<td>{_pill(scores.get('faithfulness', 0))}</td>
|
|
<td>{'✓' if not scores.get('hallucination', False) else '✗'}</td>
|
|
<td style="font-size: 13px; color: var(--text-muted);">{answer}...</td>
|
|
</tr>"""
|
|
|
|
return rows
|
|
|
|
|
|
def _build_token_rows(per_question: list, strategies: list) -> str:
|
|
"""Build token usage rows."""
|
|
rows = ""
|
|
for qr in per_question:
|
|
q_id = qr.get("question_id", "")
|
|
for strategy in strategies:
|
|
strat = qr.get("strategies", {}).get(strategy, {})
|
|
usage = strat.get("token_usage", {})
|
|
|
|
rows += f"""
|
|
<tr>
|
|
<td style="font-family: {FONT_DATA}; font-size: 12px;">{q_id}</td>
|
|
<td>{strategy}</td>
|
|
<td>{usage.get('prompt_tokens', 0)}</td>
|
|
<td>{usage.get('completion_tokens', 0)}</td>
|
|
<td><strong>{usage.get('total_tokens', 0)}</strong></td>
|
|
</tr>"""
|
|
|
|
return rows
|
|
|
|
|
|
def _pill(score: float) -> str:
|
|
"""Create a score pill."""
|
|
if score >= 8:
|
|
css = "high"
|
|
elif score >= 6:
|
|
css = "mid"
|
|
else:
|
|
css = "low"
|
|
return f'<span class="pill {css}">{score:.1f}</span>'
|
|
|
|
|
|
def _hall_pill(rate: float) -> str:
|
|
"""Create hallucination rate pill."""
|
|
if rate <= 0.1:
|
|
css = "high"
|
|
elif rate <= 0.2:
|
|
css = "mid"
|
|
else:
|
|
css = "low"
|
|
return f'<span class="pill {css}">{rate*100:.0f}%</span>'
|
|
|
|
|
|
# ── Legacy compatibility ──────────────────────────────────────────
|
|
|
|
def generate_enhanced_report(experiment: dict) -> str:
|
|
"""Backward-compatible entry point (defaults to managerial)."""
|
|
return generate_managerial_report(experiment)
|