"""Enhanced HTML report generator with two views:
1. Managerial: Decision-focused, high-level insights
2. Technical: Full observability with detailed data
Design: Dark mode, amber/teal accents, Inter + JetBrains Mono.
"""
from __future__ import annotations
import json
from typing import Any
# ── Design Tokens ─────────────────────────────────────────────────
COLORS = {
"bg": "#0F1419",
"surface": "#1A2332",
"surface-hover": "#243044",
"border": "#2D3A4A",
"text": "#E8EDF2",
"text-muted": "#7A8BA0",
"accent": "#F59E0B",
"accent-dim": "#D97706",
"teal": "#14B8A6",
"rose": "#F43F5E",
"violet": "#8B5CF6",
"emerald": "#10B981",
}
CHART_COLORS = ["#F59E0B", "#14B8A6", "#8B5CF6", "#F43F5E", "#3B82F6"]
FONT_DISPLAY = "'Inter', -apple-system, sans-serif"
FONT_DATA = "'JetBrains Mono', 'Fira Code', monospace"
# ── Main Entry Points ─────────────────────────────────────────────
def generate_report(experiment: dict, view: str = "managerial") -> str:
"""Generate report for specified view.
Args:
experiment: Experiment data from SQLite
view: "managerial" or "technical"
"""
if view == "technical":
return generate_technical_report(experiment)
return generate_managerial_report(experiment)
def generate_managerial_report(experiment: dict) -> str:
"""Managerial view: Decision-focused, high-level insights."""
config = experiment.get("benchmark_config", {})
aggregate = experiment.get("aggregate_metrics", {})
per_question = experiment.get("per_question", [])
strategies = experiment.get("strategies_used", [])
rankings = _calculate_rankings(aggregate, strategies)
# Calculate cost from token usage in per-question data
total_prompt_tokens = 0
total_completion_tokens = 0
for qr in per_question:
for strategy in strategies:
strat = qr.get("strategies", {}).get(strategy, {})
usage = strat.get("token_usage", {})
total_prompt_tokens += usage.get("prompt_tokens", 0)
total_completion_tokens += usage.get("completion_tokens", 0)
estimated_cost = (total_prompt_tokens * 0.15 + total_completion_tokens * 0.60) / 1_000_000
html = _base_html(experiment, "Managerial View", f"""
Best Strategy
{rankings[0][0] if rankings else 'N/A'}
Overall Score
{rankings[0][1]:.2f}
Questions
{config.get('num_questions', 0)}
Cost
${estimated_cost:.4f}
🏆
Recommendation: {rankings[0][0] if rankings else 'N/A'}
{_get_recommendation_text(rankings, aggregate)}
{rankings[0][1]:.2f}
{_build_strategy_cards(rankings, aggregate)}
{_generate_decision_insights(rankings, aggregate)}
""")
return html
def generate_technical_report(experiment: dict) -> str:
"""Technical view: Full observability with detailed data."""
config = experiment.get("benchmark_config", {})
aggregate = experiment.get("aggregate_metrics", {})
per_question = experiment.get("per_question", [])
strategies = experiment.get("strategies_used", [])
rankings = _calculate_rankings(aggregate, strategies)
# Calculate metrics from available data
total_queries = len(per_question) * len(strategies)
total_questions = config.get("num_questions", len(per_question))
top_k = config.get("top_k", 5)
# Calculate token usage from per-question data
total_prompt_tokens = 0
total_completion_tokens = 0
for qr in per_question:
for strategy in strategies:
strat = qr.get("strategies", {}).get(strategy, {})
usage = strat.get("token_usage", {})
total_prompt_tokens += usage.get("prompt_tokens", 0)
total_completion_tokens += usage.get("completion_tokens", 0)
total_tokens = total_prompt_tokens + total_completion_tokens
# Estimate cost based on token usage (GPT-4o-mini pricing)
# Input: $0.15/1M tokens, Output: $0.60/1M tokens
estimated_cost = (total_prompt_tokens * 0.15 + total_completion_tokens * 0.60) / 1_000_000
# Estimate latency (rough: ~0.5s per query + ~1s per evaluation)
estimated_latency = total_queries * 1.5
html = _base_html(experiment, "Technical View", f"""
Total Queries
{total_queries}
Total Tokens
{total_tokens:,}
Cost
${estimated_cost:.4f}
Tokens per Query
{total_tokens // max(total_queries, 1):,}
| Strategy |
Context Relevance |
Answer Similarity |
Faithfulness |
Hallucination |
Questions |
Failed |
Overall |
{_build_technical_aggregate_rows(rankings, aggregate)}
| ID |
Question |
Category |
Difficulty |
{_build_strategy_headers(strategies)}
{_build_per_question_rows(per_question, strategies)}
| Q-ID |
Strategy |
Context |
Similarity |
Faithfulness |
Hallucination |
Answer Preview |
{_build_detailed_rows(per_question, strategies)}
| Q-ID |
Strategy |
Prompt Tokens |
Completion Tokens |
Total Tokens |
{_build_token_rows(per_question, strategies)}
""")
return html
# ── Base HTML Template ────────────────────────────────────────────
def _base_html(experiment: dict, title: str, content: str) -> str:
"""Base HTML wrapper with styles and scripts."""
strategies = experiment.get("strategies_used", [])
aggregate = experiment.get("aggregate_metrics", {})
return f"""
{title}
{content}
"""
# ── Helper Functions ──────────────────────────────────────────────
def _calculate_rankings(aggregate: dict, strategies: list) -> list:
"""Calculate overall scores and rank strategies."""
rankings = []
for strategy in strategies:
m = aggregate.get(strategy, {})
overall = (
m.get("avg_context_relevance", 0) * 0.3
+ m.get("avg_answer_similarity", 0) * 0.4
+ m.get("avg_faithfulness", 0) * 0.3
) * (1 - m.get("hallucination_rate", 0))
rankings.append((strategy, round(overall, 2)))
rankings.sort(key=lambda x: x[1], reverse=True)
return rankings
def _get_recommendation_text(rankings: list, aggregate: dict) -> str:
"""Generate recommendation text for the winner."""
if not rankings:
return "No data available."
best = rankings[0][0]
m = aggregate.get(best, {})
parts = []
if m.get("avg_context_relevance", 0) >= 8:
parts.append("strong context retrieval")
if m.get("avg_faithfulness", 0) >= 8:
parts.append("high faithfulness")
if m.get("hallucination_rate", 0) < 0.1:
parts.append("low hallucination")
if parts:
return f"{best} excels in {', '.join(parts)}, making it the most reliable choice for your use case."
return f"{best} achieved the highest overall score across all evaluation metrics."
def _build_strategy_cards(rankings: list, aggregate: dict) -> str:
"""Build strategy comparison cards."""
cards = ""
for rank, (strategy, overall) in enumerate(rankings, 1):
m = aggregate.get(strategy, {})
rank_class = f"rank-{rank}" if rank <= 3 else ""
cards += f"""
{rank}
{strategy}
{_metric_bar("Context", m.get("avg_context_relevance", 0), "teal")}
{_metric_bar("Similarity", m.get("avg_answer_similarity", 0), "accent")}
{_metric_bar("Faithfulness", m.get("avg_faithfulness", 0), "violet")}
{_metric_bar("No Hallucination", (1 - m.get("hallucination_rate", 0)) * 10, "teal" if m.get("hallucination_rate", 0) <= 0.1 else "rose")}
"""
return cards
def _metric_bar(label: str, value: float, color_class: str) -> str:
"""Build a single metric bar."""
width = min(value * 10, 100)
return f"""
"""
def _generate_decision_insights(rankings: list, aggregate: dict) -> str:
"""Generate decision-focused insights."""
insights = []
if rankings:
best = rankings[0][0]
worst = rankings[-1][0]
gap = (1 - rankings[-1][1] / rankings[0][1]) * 100 if rankings[0][1] > 0 else 0
insights.append(f"""
Recommended Strategy
{best} is the best choice with {rankings[0][1]:.2f} overall score.
""")
if gap > 20:
insights.append(f"""
Avoid
{worst} underperforms by {gap:.0f}%. Use only if specific constraints require it.
""")
# Cost vs quality
best_m = aggregate.get(best, {})
if best_m.get("avg_context_relevance", 0) >= 8 and best_m.get("hallucination_rate", 0) < 0.1:
insights.append(f"""
Quality Assessment
{best} delivers high-quality answers with reliable context retrieval.
""")
else:
insights.append(f"""
Quality Note
Consider tuning parameters or adding more context for better results.
""")
return "\n".join(insights)
def _build_technical_aggregate_rows(rankings: list, aggregate: dict) -> str:
"""Build technical aggregate table rows."""
rows = ""
for rank, (strategy, overall) in enumerate(rankings, 1):
m = aggregate.get(strategy, {})
rows += f"""
| {strategy} |
{_pill(m.get('avg_context_relevance', 0))} |
{_pill(m.get('avg_answer_similarity', 0))} |
{_pill(m.get('avg_faithfulness', 0))} |
{_hall_pill(m.get('hallucination_rate', 0))} |
{m.get('total_questions', 0)} |
{m.get('failed_questions', 0)} |
{overall:.2f} |
"""
return rows
def _build_strategy_headers(strategies: list) -> str:
"""Build table headers."""
return "".join(f'{s[:12]} | ' for s in strategies)
def _build_per_question_rows(per_question: list, strategies: list) -> str:
"""Build per-question rows."""
rows = ""
for qr in per_question:
q_id = qr.get("question_id", "")
q_text = qr.get("question", "")[:40]
category = qr.get("category", "")
difficulty = qr.get("difficulty", "")
cells = ""
for strategy in strategies:
strat = qr.get("strategies", {}).get(strategy, {})
scores = strat.get("scores", {})
sim = scores.get("answer_similarity", 0)
cells += f"{_pill(sim)} | "
rows += f"""
| {q_id} |
{q_text}... |
{category} |
{difficulty} |
{cells}
"""
return rows
def _build_detailed_rows(per_question: list, strategies: list) -> str:
"""Build detailed score rows."""
rows = ""
for qr in per_question:
q_id = qr.get("question_id", "")
for strategy in strategies:
strat = qr.get("strategies", {}).get(strategy, {})
scores = strat.get("scores", {})
answer = strat.get("answer", "")[:60]
rows += f"""
| {q_id} |
{strategy} |
{_pill(scores.get('context_relevance', 0))} |
{_pill(scores.get('answer_similarity', 0))} |
{_pill(scores.get('faithfulness', 0))} |
{'✓' if not scores.get('hallucination', False) else '✗'} |
{answer}... |
"""
return rows
def _build_token_rows(per_question: list, strategies: list) -> str:
"""Build token usage rows."""
rows = ""
for qr in per_question:
q_id = qr.get("question_id", "")
for strategy in strategies:
strat = qr.get("strategies", {}).get(strategy, {})
usage = strat.get("token_usage", {})
rows += f"""
| {q_id} |
{strategy} |
{usage.get('prompt_tokens', 0)} |
{usage.get('completion_tokens', 0)} |
{usage.get('total_tokens', 0)} |
"""
return rows
def _pill(score: float) -> str:
"""Create a score pill."""
if score >= 8:
css = "high"
elif score >= 6:
css = "mid"
else:
css = "low"
return f'{score:.1f}'
def _hall_pill(rate: float) -> str:
"""Create hallucination rate pill."""
if rate <= 0.1:
css = "high"
elif rate <= 0.2:
css = "mid"
else:
css = "low"
return f'{rate*100:.0f}%'
# ── Legacy compatibility ──────────────────────────────────────────
def generate_enhanced_report(experiment: dict) -> str:
"""Backward-compatible entry point (defaults to managerial)."""
return generate_managerial_report(experiment)