feat(reports): add answer compare modal and source format badge

Why:
- Make per-question strategy answers inspectable and show DOCX vs Text PDF source.

Changes:
- Side-by-side compare modal with HTML escaping; header source badge from filename.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/benchmarking/report.py

@@ -7,6 +7,7 @@ Design: Dark mode, amber/teal accents, Inter + JetBrains Mono.
 
 from __future__ import annotations
 
+import html
 import json
 from typing import Any
 
@@ -34,8 +35,104 @@ FONT_DISPLAY = "'Inter', -apple-system, sans-serif"
 FONT_DATA = "'JetBrains Mono', 'Fira Code', monospace"
 
 
+# ── Compare Answers Modal (JS) ────────────────────────────────────
+
+_MODAL_JS = """
+        // ── Compare Answers Modal ─────────────────────────────────
+        function escapeHtml(str) {
+            if (str == null) return '';
+            return String(str)
+                .replace(/&/g, '&')
+                .replace(/</g, '&lt;')
+                .replace(/>/g, '&gt;')
+                .replace(/"/g, '&quot;')
+                .replace(/'/g, '&#39;');
+        }
+
+        function scoreChip(label, val) {
+            val = Number(val) || 0;
+            const cls = val >= 8 ? 'high' : (val >= 6 ? 'mid' : 'low');
+            return '<span class="score-chip ' + cls + '">' + label + ' ' + val.toFixed(1) + '</span>';
+        }
+
+        function showCompare(idx) {
+            const q = perQuestion[idx];
+            if (!q) return;
+
+            document.getElementById('modalQId').textContent = 'Question ' + (q.question_id || (idx + 1));
+            document.getElementById('modalQuestion').textContent = q.question || '';
+
+            const expected = q.expected_answer || '';
+            document.getElementById('modalExpected').innerHTML = expected
+                ? '<div class="expected-block"><strong>Expected Answer (Ground Truth)</strong><p>' + escapeHtml(expected) + '</p></div>'
+                : '';
+
+            const grid = document.getElementById('modalAnswers');
+            grid.innerHTML = strategies.map(s => {
+                const strat = (q.strategies || {})[s] || {};
+                const scores = strat.scores || {};
+                const hasError = !!strat.error;
+                const hasAnswer = !!(strat.answer && String(strat.answer).trim());
+                const answer = hasAnswer
+                    ? strat.answer
+                    : (hasError ? ('Error: ' + strat.error) : 'No answer recorded.');
+
+                const chips = [];
+                if (scores.context_relevance != null) chips.push(scoreChip('Context', scores.context_relevance));
+                if (scores.answer_similarity != null) chips.push(scoreChip('Sim', scores.answer_similarity));
+                if (scores.faithfulness != null) chips.push(scoreChip('Faith', scores.faithfulness));
+                if (scores.hallucination != null) {
+                    chips.push(scores.hallucination
+                        ? '<span class="score-chip low">Hallucinated</span>'
+                        : '<span class="score-chip high">No Hallucination</span>');
+                }
+                if (strat.latency && strat.latency.query_seconds != null) {
+                    chips.push('<span class="score-chip">' + Number(strat.latency.query_seconds).toFixed(2) + 's</span>');
+                }
+                const usage = strat.token_usage || {};
+                if (usage.total_tokens != null) {
+                    chips.push('<span class="score-chip">' + Number(usage.total_tokens).toLocaleString() + ' tokens</span>');
+                }
+
+                return '<div class="answer-card">'
+                    + '<div class="strategy-name">' + escapeHtml(s)
+                    + '<span class="badge pill ' + (hasError ? 'low' : 'mid') + '">' + (hasError ? 'FAILED' : 'ANSWER') + '</span></div>'
+                    + '<div class="score-row">' + chips.join('') + '</div>'
+                    + '<div class="answer-text' + (hasAnswer ? '' : ' empty') + '">' + escapeHtml(answer) + '</div>'
+                    + '</div>';
+            }).join('');
+
+            document.getElementById('compareModal').classList.add('open');
+            document.body.style.overflow = 'hidden';
+        }
+
+        function closeCompare() {
+            document.getElementById('compareModal').classList.remove('open');
+            document.body.style.overflow = '';
+        }
+
+        document.getElementById('compareModal').addEventListener('click', function (e) {
+            if (e.target === this) closeCompare();
+        });
+
+        document.addEventListener('keydown', function (e) {
+            if (e.key === 'Escape') closeCompare();
+        });
+"""
+
+
 # ── Main Entry Points ─────────────────────────────────────────────
 
+def _source_format_label(experiment: dict) -> str:
+    """Human-readable source format badge from document filename."""
+    filename = (experiment.get("document_filename") or "").lower()
+    if filename.endswith(".pdf"):
+        return "Text PDF"
+    if filename.endswith(".docx") or filename.endswith(".doc"):
+        return "DOCX"
+    return ""
+
+
 def generate_report(experiment: dict, view: str = "managerial") -> str:
     """Generate report for specified view.
 
@@ -67,6 +164,8 @@ def generate_managerial_report(experiment: dict) -> str:
             total_completion_tokens += usage.get("completion_tokens", 0)
 
     estimated_cost = (total_prompt_tokens * 0.15 + total_completion_tokens * 0.60) / 1_000_000
+    source_label = _source_format_label(experiment)
+    source_meta = f" &middot; Source: {source_label}" if source_label else ""
 
     html = _base_html(experiment, "Managerial View", f"""
         <!-- Header -->
@@ -76,7 +175,7 @@ def generate_managerial_report(experiment: dict) -> str:
             <div class="meta">
                 {config.get('num_questions', 0)} questions &middot;
                 {len(strategies)} strategies &middot;
-                ${estimated_cost:.4f} cost
+                ${estimated_cost:.4f} cost{source_meta}
             </div>
         </header>
 
@@ -196,6 +295,8 @@ def generate_technical_report(experiment: dict) -> str:
 
     # Estimate latency (rough: ~0.5s per query + ~1s per evaluation)
     estimated_latency = total_queries * 1.5
+    source_label = _source_format_label(experiment)
+    source_meta = f" &middot; Source: {source_label}" if source_label else ""
 
     html = _base_html(experiment, "Technical View", f"""
         <!-- Header -->
@@ -205,7 +306,7 @@ def generate_technical_report(experiment: dict) -> str:
             <div class="meta">
                 Experiment: {experiment.get('id', 'N/A')[:16]}... &middot;
                 {total_questions} questions &middot;
-                {len(strategies)} strategies
+                {len(strategies)} strategies{source_meta}
             </div>
         </header>
 
@@ -279,6 +380,7 @@ def generate_technical_report(experiment: dict) -> str:
             <h2>Per-Question Results</h2>
             <div class="line"></div>
         </div>
+        <div class="hint">&#128269; Click any question to compare full answers from all strategies side by side.</div>
         <div class="table-wrapper">
             <table>
                 <thead>
@@ -489,6 +591,34 @@ def _base_html(experiment: dict, title: str, content: str) -> str:
         .link-text {{ flex: 1; font-weight: 500; }}
         .link-arrow {{ color: var(--accent); font-size: 18px; }}
 
+        .question-link {{ color: var(--accent); cursor: pointer; text-decoration: none; border-bottom: 1px dashed var(--accent); }}
+        .question-link:hover {{ color: var(--accent-dim); border-bottom-style: solid; }}
+
+        .hint {{ font-size: 12px; color: var(--text-muted); font-family: {FONT_DATA}; margin-bottom: 16px; }}
+
+        .modal-overlay {{ position: fixed; inset: 0; background: rgba(4, 8, 12, 0.72); display: none; align-items: center; justify-content: center; z-index: 1000; padding: 24px; backdrop-filter: blur(4px); }}
+        .modal-overlay.open {{ display: flex; }}
+        .modal {{ background: var(--surface); border: 1px solid var(--border); border-radius: 16px; max-width: 1100px; width: 100%; max-height: 85vh; display: flex; flex-direction: column; box-shadow: 0 24px 48px rgba(0, 0, 0, 0.5); }}
+        .modal-header {{ padding: 24px 28px; border-bottom: 1px solid var(--border); display: flex; align-items: flex-start; gap: 16px; }}
+        .modal-header .eyebrow {{ margin-bottom: 8px; }}
+        .modal-header h3 {{ font-size: 18px; font-weight: 600; line-height: 1.4; }}
+        .modal-close {{ margin-left: auto; background: var(--surface-hover); border: 1px solid var(--border); color: var(--text-muted); width: 36px; height: 36px; border-radius: 8px; font-size: 20px; line-height: 1; cursor: pointer; flex-shrink: 0; transition: all 0.2s ease; }}
+        .modal-close:hover {{ color: var(--text); border-color: var(--accent); }}
+        .modal-body {{ padding: 24px 28px; overflow-y: auto; }}
+        .expected-block {{ background: rgba(139, 92, 246, 0.08); border-left: 3px solid var(--violet); border-radius: 8px; padding: 16px 20px; margin-bottom: 20px; }}
+        .expected-block strong {{ font-size: 12px; color: var(--violet); text-transform: uppercase; letter-spacing: 1px; display: block; margin-bottom: 6px; }}
+        .expected-block p {{ font-size: 13px; color: var(--text); margin: 0; }}
+        .answer-grid {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 16px; }}
+        .answer-card {{ background: var(--surface-hover); border: 1px solid var(--border); border-radius: 12px; padding: 20px; display: flex; flex-direction: column; gap: 12px; }}
+        .answer-card .strategy-name {{ font-size: 13px; font-weight: 600; color: var(--accent); display: flex; align-items: center; justify-content: space-between; gap: 8px; }}
+        .answer-card .answer-text {{ font-size: 13px; line-height: 1.7; color: var(--text); white-space: pre-wrap; word-break: break-word; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 14px 16px; max-height: 320px; overflow-y: auto; }}
+        .answer-card .answer-text.empty {{ color: var(--text-muted); font-style: italic; }}
+        .score-row {{ display: flex; flex-wrap: wrap; gap: 8px; }}
+        .score-chip {{ font-family: {FONT_DATA}; font-size: 11px; padding: 3px 8px; border-radius: 6px; background: var(--surface); border: 1px solid var(--border); }}
+        .score-chip.high {{ color: var(--emerald); border-color: rgba(16, 185, 129, 0.4); }}
+        .score-chip.mid {{ color: var(--accent); border-color: rgba(245, 158, 11, 0.4); }}
+        .score-chip.low {{ color: var(--rose); border-color: rgba(244, 63, 94, 0.4); }}
+
         .footer {{ padding-top: 32px; border-top: 1px solid var(--border); font-size: 12px; color: var(--text-muted); text-align: center; }}
 
         @media (max-width: 768px) {{
@@ -508,6 +638,27 @@ def _base_html(experiment: dict, title: str, content: str) -> str:
     <div class="page">
         {content}
 
+        <!-- Compare Answers Modal -->
+        <div class="modal-overlay" id="compareModal">
+            <div class="modal">
+                <div class="modal-header">
+                    <div>
+                        <div class="eyebrow" id="modalQId">Question</div>
+                        <h3 id="modalQuestion"></h3>
+                    </div>
+                    <button class="modal-close" onclick="closeCompare()" aria-label="Close">&times;</button>
+                </div>
+                <div class="modal-body">
+                    <div id="modalExpected"></div>
+                    <div class="section-header">
+                        <h2>Answers by Strategy</h2>
+                        <div class="line"></div>
+                    </div>
+                    <div class="answer-grid" id="modalAnswers"></div>
+                </div>
+            </div>
+        </div>
+
         <footer class="footer">
             Generated by RAG Chunking Benchmarker &middot; {experiment.get('created_at', 'N/A')}
         </footer>
@@ -517,64 +668,73 @@ def _base_html(experiment: dict, title: str, content: str) -> str:
         const colors = {json.dumps(CHART_COLORS)};
         const strategies = {json.dumps(strategies)};
         const aggregate = {json.dumps(aggregate)};
+        const perQuestion = {json.dumps(experiment.get("per_question", []), ensure_ascii=False).replace("</", "<\\/")};
+
+        {_MODAL_JS}
 
         // Radar
-        new Chart(document.getElementById('radarChart'), {{
+        const radarEl = document.getElementById('radarChart');
-            type: 'radar',
+        if (radarEl) {{
-            data: {{
+            new Chart(radarEl, {{
-                labels: ['Context', 'Similarity', 'Faithfulness', 'Consistency'],
+                type: 'radar',
-                datasets: strategies.map((s, i) => ({{
+                data: {{
-                    label: s,
+                    labels: ['Context', 'Similarity', 'Faithfulness', 'Consistency'],
-                    data: [
+                    datasets: strategies.map((s, i) => ({{
-                        aggregate[s]?.avg_context_relevance || 0,
+                        label: s,
-                        aggregate[s]?.avg_answer_similarity || 0,
+                        data: [
-                        aggregate[s]?.avg_faithfulness || 0,
+                            aggregate[s]?.avg_context_relevance || 0,
-                        (1 - (aggregate[s]?.hallucination_rate || 0)) * 10
+                            aggregate[s]?.avg_answer_similarity || 0,
-                    ],
+                            aggregate[s]?.avg_faithfulness || 0,
-                    borderColor: colors[i % colors.length],
+                            (1 - (aggregate[s]?.hallucination_rate || 0)) * 10
-                    backgroundColor: colors[i % colors.length] + '20',
+                        ],
-                    pointBackgroundColor: colors[i % colors.length],
+                        borderColor: colors[i % colors.length],
-                    borderWidth: 2
+                        backgroundColor: colors[i % colors.length] + '20',
-                }}))
+                        pointBackgroundColor: colors[i % colors.length],
-            }},
+                        borderWidth: 2
-            options: {{
+                    }}))
-                responsive: true,
+                }},
-                maintainAspectRatio: false,
+                options: {{
-                plugins: {{ legend: {{ display: false }} }},
+                    responsive: true,
-                scales: {{
+                    maintainAspectRatio: false,
-                    r: {{
+                    plugins: {{ legend: {{ display: false }} }},
-                        beginAtZero: true,
+                    scales: {{
-                        max: 10,
+                        r: {{
-                        grid: {{ color: '{COLORS["border"]}' }},
+                            beginAtZero: true,
-                        angleLines: {{ color: '{COLORS["border"]}' }},
+                            max: 10,
-                        pointLabels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'JetBrains Mono'", size: 11 }} }},
+                            grid: {{ color: '{COLORS["border"]}' }},
-                        ticks: {{ display: false }}
+                            angleLines: {{ color: '{COLORS["border"]}' }},
+                            pointLabels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'JetBrains Mono'", size: 11 }} }},
+                            ticks: {{ display: false }}
+                        }}
                     }}
                 }}
-            }}
+            }});
-        }});
+        }}
 
         // Bar
-        new Chart(document.getElementById('barChart'), {{
+        const barEl = document.getElementById('barChart');
-            type: 'bar',
+        if (barEl) {{
-            data: {{
+            new Chart(barEl, {{
-                labels: strategies.map(s => s.length > 12 ? s.substring(0, 12) + '...' : s),
+                type: 'bar',
-                datasets: [
+                data: {{
-                    {{ label: 'Context', data: strategies.map(s => aggregate[s]?.avg_context_relevance || 0), backgroundColor: colors[0] }},
+                    labels: strategies.map(s => s.length > 12 ? s.substring(0, 12) + '...' : s),
-                    {{ label: 'Similarity', data: strategies.map(s => aggregate[s]?.avg_answer_similarity || 0), backgroundColor: colors[1] }},
+                    datasets: [
-                    {{ label: 'Faithfulness', data: strategies.map(s => aggregate[s]?.avg_faithfulness || 0), backgroundColor: colors[2] }}
+                        {{ label: 'Context', data: strategies.map(s => aggregate[s]?.avg_context_relevance || 0), backgroundColor: colors[0] }},
-                ]
+                        {{ label: 'Similarity', data: strategies.map(s => aggregate[s]?.avg_answer_similarity || 0), backgroundColor: colors[1] }},
-            }},
+                        {{ label: 'Faithfulness', data: strategies.map(s => aggregate[s]?.avg_faithfulness || 0), backgroundColor: colors[2] }}
-            options: {{
+                    ]
-                responsive: true,
+                }},
-                maintainAspectRatio: false,
+                options: {{
-                plugins: {{ legend: {{ position: 'bottom', labels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'Inter'" }} }} }} }},
+                    responsive: true,
-                scales: {{
+                    maintainAspectRatio: false,
-                    x: {{ grid: {{ display: false }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }},
+                    plugins: {{ legend: {{ position: 'bottom', labels: {{ color: '{COLORS["text-muted"]}', font: {{ family: "'Inter'" }} }} }} }},
-                    y: {{ beginAtZero: true, max: 10, grid: {{ color: '{COLORS["border"]}' }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }}
+                    scales: {{
+                        x: {{ grid: {{ display: false }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }},
+                        y: {{ beginAtZero: true, max: 10, grid: {{ color: '{COLORS["border"]}' }}, ticks: {{ color: '{COLORS["text-muted"]}' }} }}
+                    }}
                 }}
-            }}
+            }});
-        }});
+        }}
     </script>
 </body>
 </html>"""
@@ -739,13 +899,16 @@ def _build_strategy_headers(strategies: list) -> str:
 
 
 def _build_per_question_rows(per_question: list, strategies: list) -> str:
-    """Build per-question rows."""
+    """Build per-question rows with clickable questions opening the compare modal."""
     rows = ""
-    for qr in per_question:
+    for idx, qr in enumerate(per_question):
-        q_id = qr.get("question_id", "")
+        q_id = html.escape(str(qr.get("question_id", "")))
-        q_text = qr.get("question", "")[:40]
+        full_q = str(qr.get("question", "") or "")
-        category = qr.get("category", "")
+        q_preview = full_q[:60] + ("..." if len(full_q) > 60 else "")
-        difficulty = qr.get("difficulty", "")
+        q_text = html.escape(q_preview)
+        category = html.escape(str(qr.get("category", "") or ""))
+        difficulty = str(qr.get("difficulty", "") or "")
+        difficulty_safe = html.escape(difficulty)
 
         cells = ""
         for strategy in strategies:
@@ -755,11 +918,11 @@ def _build_per_question_rows(per_question: list, strategies: list) -> str:
             cells += f"<td>{_pill(sim)}</td>"
 
         rows += f"""
-        <tr>
+        <tr style="cursor: pointer;" onclick="showCompare({idx})" title="Click to compare all strategy answers">
             <td style="font-family: {FONT_DATA}; color: var(--text-muted);">{q_id}</td>
-            <td>{q_text}...</td>
+            <td><span class="question-link">{q_text}</span></td>
             <td style="color: var(--text-muted);">{category}</td>
-            <td><span class="pill {'high' if difficulty == 'easy' else 'mid' if difficulty == 'medium' else 'low'}">{difficulty}</span></td>
+            <td><span class="pill {'high' if difficulty == 'easy' else 'mid' if difficulty == 'medium' else 'low'}">{difficulty_safe}</span></td>
             {cells}
         </tr>"""
 
@@ -770,21 +933,24 @@ def _build_detailed_rows(per_question: list, strategies: list) -> str:
     """Build detailed score rows."""
     rows = ""
     for qr in per_question:
-        q_id = qr.get("question_id", "")
+        q_id = html.escape(str(qr.get("question_id", "")))
         for strategy in strategies:
             strat = qr.get("strategies", {}).get(strategy, {})
             scores = strat.get("scores", {})
-            answer = strat.get("answer", "")[:60]
+            full_answer = str(strat.get("answer", "") or "")
+            answer_preview = full_answer[:60] + ("..." if len(full_answer) > 60 else "")
+            answer = html.escape(answer_preview)
+            strategy_safe = html.escape(str(strategy))
 
             rows += f"""
             <tr>
                 <td style="font-family: {FONT_DATA}; font-size: 12px;">{q_id}</td>
-                <td>{strategy}</td>
+                <td>{strategy_safe}</td>
                 <td>{_pill(scores.get('context_relevance', 0))}</td>
                 <td>{_pill(scores.get('answer_similarity', 0))}</td>
                 <td>{_pill(scores.get('faithfulness', 0))}</td>
                 <td>{'&#10003;' if not scores.get('hallucination', False) else '&#10007;'}</td>
-                <td style="font-size: 13px; color: var(--text-muted);">{answer}...</td>
+                <td style="font-size: 13px; color: var(--text-muted);">{answer}</td>
             </tr>"""
 
     return rows