diff --git a/src/benchmarking/report.py b/src/benchmarking/report.py index 83985e0..92782be 100644 --- a/src/benchmarking/report.py +++ b/src/benchmarking/report.py @@ -7,6 +7,7 @@ Design: Dark mode, amber/teal accents, Inter + JetBrains Mono. from __future__ import annotations +import html import json from typing import Any @@ -34,8 +35,104 @@ FONT_DISPLAY = "'Inter', -apple-system, sans-serif" FONT_DATA = "'JetBrains Mono', 'Fira Code', monospace" +# ── Compare Answers Modal (JS) ──────────────────────────────────── + +_MODAL_JS = """ + // ── Compare Answers Modal ───────────────────────────────── + function escapeHtml(str) { + if (str == null) return ''; + return String(str) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + } + + function scoreChip(label, val) { + val = Number(val) || 0; + const cls = val >= 8 ? 'high' : (val >= 6 ? 'mid' : 'low'); + return '' + label + ' ' + val.toFixed(1) + ''; + } + + function showCompare(idx) { + const q = perQuestion[idx]; + if (!q) return; + + document.getElementById('modalQId').textContent = 'Question ' + (q.question_id || (idx + 1)); + document.getElementById('modalQuestion').textContent = q.question || ''; + + const expected = q.expected_answer || ''; + document.getElementById('modalExpected').innerHTML = expected + ? '
Expected Answer (Ground Truth)

' + escapeHtml(expected) + '

' + : ''; + + const grid = document.getElementById('modalAnswers'); + grid.innerHTML = strategies.map(s => { + const strat = (q.strategies || {})[s] || {}; + const scores = strat.scores || {}; + const hasError = !!strat.error; + const hasAnswer = !!(strat.answer && String(strat.answer).trim()); + const answer = hasAnswer + ? strat.answer + : (hasError ? ('Error: ' + strat.error) : 'No answer recorded.'); + + const chips = []; + if (scores.context_relevance != null) chips.push(scoreChip('Context', scores.context_relevance)); + if (scores.answer_similarity != null) chips.push(scoreChip('Sim', scores.answer_similarity)); + if (scores.faithfulness != null) chips.push(scoreChip('Faith', scores.faithfulness)); + if (scores.hallucination != null) { + chips.push(scores.hallucination + ? 'Hallucinated' + : 'No Hallucination'); + } + if (strat.latency && strat.latency.query_seconds != null) { + chips.push('' + Number(strat.latency.query_seconds).toFixed(2) + 's'); + } + const usage = strat.token_usage || {}; + if (usage.total_tokens != null) { + chips.push('' + Number(usage.total_tokens).toLocaleString() + ' tokens'); + } + + return '
' + + '
' + escapeHtml(s) + + '' + (hasError ? 'FAILED' : 'ANSWER') + '
' + + '
' + chips.join('') + '
' + + '
' + escapeHtml(answer) + '
' + + '
'; + }).join(''); + + document.getElementById('compareModal').classList.add('open'); + document.body.style.overflow = 'hidden'; + } + + function closeCompare() { + document.getElementById('compareModal').classList.remove('open'); + document.body.style.overflow = ''; + } + + document.getElementById('compareModal').addEventListener('click', function (e) { + if (e.target === this) closeCompare(); + }); + + document.addEventListener('keydown', function (e) { + if (e.key === 'Escape') closeCompare(); + }); +""" + + # ── Main Entry Points ───────────────────────────────────────────── +def _source_format_label(experiment: dict) -> str: + """Human-readable source format badge from document filename.""" + filename = (experiment.get("document_filename") or "").lower() + if filename.endswith(".pdf"): + return "Text PDF" + if filename.endswith(".docx") or filename.endswith(".doc"): + return "DOCX" + return "" + + def generate_report(experiment: dict, view: str = "managerial") -> str: """Generate report for specified view. @@ -67,6 +164,8 @@ def generate_managerial_report(experiment: dict) -> str: total_completion_tokens += usage.get("completion_tokens", 0) estimated_cost = (total_prompt_tokens * 0.15 + total_completion_tokens * 0.60) / 1_000_000 + source_label = _source_format_label(experiment) + source_meta = f" · Source: {source_label}" if source_label else "" html = _base_html(experiment, "Managerial View", f""" @@ -76,7 +175,7 @@ def generate_managerial_report(experiment: dict) -> str:
{config.get('num_questions', 0)} questions · {len(strategies)} strategies · - ${estimated_cost:.4f} cost + ${estimated_cost:.4f} cost{source_meta}
@@ -196,6 +295,8 @@ def generate_technical_report(experiment: dict) -> str: # Estimate latency (rough: ~0.5s per query + ~1s per evaluation) estimated_latency = total_queries * 1.5 + source_label = _source_format_label(experiment) + source_meta = f" · Source: {source_label}" if source_label else "" html = _base_html(experiment, "Technical View", f""" @@ -205,7 +306,7 @@ def generate_technical_report(experiment: dict) -> str:
Experiment: {experiment.get('id', 'N/A')[:16]}... · {total_questions} questions · - {len(strategies)} strategies + {len(strategies)} strategies{source_meta}
@@ -279,6 +380,7 @@ def generate_technical_report(experiment: dict) -> str:

Per-Question Results

+
🔍 Click any question to compare full answers from all strategies side by side.
@@ -489,6 +591,34 @@ def _base_html(experiment: dict, title: str, content: str) -> str: .link-text {{ flex: 1; font-weight: 500; }} .link-arrow {{ color: var(--accent); font-size: 18px; }} + .question-link {{ color: var(--accent); cursor: pointer; text-decoration: none; border-bottom: 1px dashed var(--accent); }} + .question-link:hover {{ color: var(--accent-dim); border-bottom-style: solid; }} + + .hint {{ font-size: 12px; color: var(--text-muted); font-family: {FONT_DATA}; margin-bottom: 16px; }} + + .modal-overlay {{ position: fixed; inset: 0; background: rgba(4, 8, 12, 0.72); display: none; align-items: center; justify-content: center; z-index: 1000; padding: 24px; backdrop-filter: blur(4px); }} + .modal-overlay.open {{ display: flex; }} + .modal {{ background: var(--surface); border: 1px solid var(--border); border-radius: 16px; max-width: 1100px; width: 100%; max-height: 85vh; display: flex; flex-direction: column; box-shadow: 0 24px 48px rgba(0, 0, 0, 0.5); }} + .modal-header {{ padding: 24px 28px; border-bottom: 1px solid var(--border); display: flex; align-items: flex-start; gap: 16px; }} + .modal-header .eyebrow {{ margin-bottom: 8px; }} + .modal-header h3 {{ font-size: 18px; font-weight: 600; line-height: 1.4; }} + .modal-close {{ margin-left: auto; background: var(--surface-hover); border: 1px solid var(--border); color: var(--text-muted); width: 36px; height: 36px; border-radius: 8px; font-size: 20px; line-height: 1; cursor: pointer; flex-shrink: 0; transition: all 0.2s ease; }} + .modal-close:hover {{ color: var(--text); border-color: var(--accent); }} + .modal-body {{ padding: 24px 28px; overflow-y: auto; }} + .expected-block {{ background: rgba(139, 92, 246, 0.08); border-left: 3px solid var(--violet); border-radius: 8px; padding: 16px 20px; margin-bottom: 20px; }} + .expected-block strong {{ font-size: 12px; color: var(--violet); text-transform: uppercase; letter-spacing: 1px; display: block; margin-bottom: 6px; }} + .expected-block p {{ font-size: 13px; color: var(--text); margin: 0; }} + .answer-grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 16px; }} + .answer-card {{ background: var(--surface-hover); border: 1px solid var(--border); border-radius: 12px; padding: 20px; display: flex; flex-direction: column; gap: 12px; }} + .answer-card .strategy-name {{ font-size: 13px; font-weight: 600; color: var(--accent); display: flex; align-items: center; justify-content: space-between; gap: 8px; }} + .answer-card .answer-text {{ font-size: 13px; line-height: 1.7; color: var(--text); white-space: pre-wrap; word-break: break-word; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 14px 16px; max-height: 320px; overflow-y: auto; }} + .answer-card .answer-text.empty {{ color: var(--text-muted); font-style: italic; }} + .score-row {{ display: flex; flex-wrap: wrap; gap: 8px; }} + .score-chip {{ font-family: {FONT_DATA}; font-size: 11px; padding: 3px 8px; border-radius: 6px; background: var(--surface); border: 1px solid var(--border); }} + .score-chip.high {{ color: var(--emerald); border-color: rgba(16, 185, 129, 0.4); }} + .score-chip.mid {{ color: var(--accent); border-color: rgba(245, 158, 11, 0.4); }} + .score-chip.low {{ color: var(--rose); border-color: rgba(244, 63, 94, 0.4); }} + .footer {{ padding-top: 32px; border-top: 1px solid var(--border); font-size: 12px; color: var(--text-muted); text-align: center; }} @media (max-width: 768px) {{ @@ -508,6 +638,27 @@ def _base_html(experiment: dict, title: str, content: str) -> str:
{content} + + +
Generated by RAG Chunking Benchmarker · {experiment.get('created_at', 'N/A')}
@@ -517,64 +668,73 @@ def _base_html(experiment: dict, title: str, content: str) -> str: const colors = {json.dumps(CHART_COLORS)}; const strategies = {json.dumps(strategies)}; const aggregate = {json.dumps(aggregate)}; + const perQuestion = {json.dumps(experiment.get("per_question", []), ensure_ascii=False).replace(" ({{ - label: s, - data: [ - aggregate[s]?.avg_context_relevance || 0, - aggregate[s]?.avg_answer_similarity || 0, - aggregate[s]?.avg_faithfulness || 0, - (1 - (aggregate[s]?.hallucination_rate || 0)) * 10 - ], - borderColor: colors[i % colors.length], - backgroundColor: colors[i % colors.length] + '20', - pointBackgroundColor: colors[i % colors.length], - borderWidth: 2 - }})) - }}, - options: {{ - responsive: true, - maintainAspectRatio: false, - plugins: {{ legend: {{ display: false }} }}, - scales: {{ - r: {{ - beginAtZero: true, - max: 10, - grid: {{ color: '{COLORS["border"]}' }}, - angleLines: {{ color: '{COLORS["border"]}' }}, - pointLabels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'JetBrains Mono'", size: 11 }} }}, - ticks: {{ display: false }} + const radarEl = document.getElementById('radarChart'); + if (radarEl) {{ + new Chart(radarEl, {{ + type: 'radar', + data: {{ + labels: ['Context', 'Similarity', 'Faithfulness', 'Consistency'], + datasets: strategies.map((s, i) => ({{ + label: s, + data: [ + aggregate[s]?.avg_context_relevance || 0, + aggregate[s]?.avg_answer_similarity || 0, + aggregate[s]?.avg_faithfulness || 0, + (1 - (aggregate[s]?.hallucination_rate || 0)) * 10 + ], + borderColor: colors[i % colors.length], + backgroundColor: colors[i % colors.length] + '20', + pointBackgroundColor: colors[i % colors.length], + borderWidth: 2 + }})) + }}, + options: {{ + responsive: true, + maintainAspectRatio: false, + plugins: {{ legend: {{ display: false }} }}, + scales: {{ + r: {{ + beginAtZero: true, + max: 10, + grid: {{ color: '{COLORS["border"]}' }}, + angleLines: {{ color: '{COLORS["border"]}' }}, + pointLabels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'JetBrains Mono'", size: 11 }} }}, + ticks: {{ display: false }} + }} }} }} - }} - }}); + }}); + }} // Bar - new Chart(document.getElementById('barChart'), {{ - type: 'bar', - data: {{ - labels: strategies.map(s => s.length > 12 ? s.substring(0, 12) + '...' : s), - datasets: [ - {{ label: 'Context', data: strategies.map(s => aggregate[s]?.avg_context_relevance || 0), backgroundColor: colors[0] }}, - {{ label: 'Similarity', data: strategies.map(s => aggregate[s]?.avg_answer_similarity || 0), backgroundColor: colors[1] }}, - {{ label: 'Faithfulness', data: strategies.map(s => aggregate[s]?.avg_faithfulness || 0), backgroundColor: colors[2] }} - ] - }}, - options: {{ - responsive: true, - maintainAspectRatio: false, - plugins: {{ legend: {{ position: 'bottom', labels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'Inter'" }} }} }} }}, - scales: {{ - x: {{ grid: {{ display: false }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }}, - y: {{ beginAtZero: true, max: 10, grid: {{ color: '{COLORS["border"]}' }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }} + const barEl = document.getElementById('barChart'); + if (barEl) {{ + new Chart(barEl, {{ + type: 'bar', + data: {{ + labels: strategies.map(s => s.length > 12 ? s.substring(0, 12) + '...' : s), + datasets: [ + {{ label: 'Context', data: strategies.map(s => aggregate[s]?.avg_context_relevance || 0), backgroundColor: colors[0] }}, + {{ label: 'Similarity', data: strategies.map(s => aggregate[s]?.avg_answer_similarity || 0), backgroundColor: colors[1] }}, + {{ label: 'Faithfulness', data: strategies.map(s => aggregate[s]?.avg_faithfulness || 0), backgroundColor: colors[2] }} + ] + }}, + options: {{ + responsive: true, + maintainAspectRatio: false, + plugins: {{ legend: {{ position: 'bottom', labels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'Inter'" }} }} }} }}, + scales: {{ + x: {{ grid: {{ display: false }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }}, + y: {{ beginAtZero: true, max: 10, grid: {{ color: '{COLORS["border"]}' }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }} + }} }} - }} - }}); + }}); + }} """ @@ -739,13 +899,16 @@ def _build_strategy_headers(strategies: list) -> str: def _build_per_question_rows(per_question: list, strategies: list) -> str: - """Build per-question rows.""" + """Build per-question rows with clickable questions opening the compare modal.""" rows = "" - for qr in per_question: - q_id = qr.get("question_id", "") - q_text = qr.get("question", "")[:40] - category = qr.get("category", "") - difficulty = qr.get("difficulty", "") + for idx, qr in enumerate(per_question): + q_id = html.escape(str(qr.get("question_id", ""))) + full_q = str(qr.get("question", "") or "") + q_preview = full_q[:60] + ("..." if len(full_q) > 60 else "") + q_text = html.escape(q_preview) + category = html.escape(str(qr.get("category", "") or "")) + difficulty = str(qr.get("difficulty", "") or "") + difficulty_safe = html.escape(difficulty) cells = "" for strategy in strategies: @@ -755,11 +918,11 @@ def _build_per_question_rows(per_question: list, strategies: list) -> str: cells += f"
" rows += f""" - + - + - + {cells} """ @@ -770,21 +933,24 @@ def _build_detailed_rows(per_question: list, strategies: list) -> str: """Build detailed score rows.""" rows = "" for qr in per_question: - q_id = qr.get("question_id", "") + q_id = html.escape(str(qr.get("question_id", ""))) for strategy in strategies: strat = qr.get("strategies", {}).get(strategy, {}) scores = strat.get("scores", {}) - answer = strat.get("answer", "")[:60] + full_answer = str(strat.get("answer", "") or "") + answer_preview = full_answer[:60] + ("..." if len(full_answer) > 60 else "") + answer = html.escape(answer_preview) + strategy_safe = html.escape(str(strategy)) rows += f""" - + - + """ return rows
{_pill(sim)}
{q_id}{q_text}...{q_text} {category}{difficulty}{difficulty_safe}
{q_id}{strategy}{strategy_safe} {_pill(scores.get('context_relevance', 0))} {_pill(scores.get('answer_similarity', 0))} {_pill(scores.get('faithfulness', 0))} {'✓' if not scores.get('hallucination', False) else '✗'}{answer}...{answer}