diff --git a/src/benchmarking/benchmark_service.py b/src/benchmarking/benchmark_service.py index fb8e8b0..920dd8e 100644 --- a/src/benchmarking/benchmark_service.py +++ b/src/benchmarking/benchmark_service.py @@ -399,6 +399,11 @@ def get_experiment(experiment_id: str) -> dict[str, Any] | None: return db.get_experiment(experiment_id) -def list_experiments(document_id: str | None = None) -> dict[str, Any]: +def list_experiments( + document_id: str | None = None, + *, + offset: int = 0, + limit: int = 200, +) -> dict[str, Any]: """List experiments, optionally filtered by document.""" - return db.list_experiments(document_id=document_id) + return db.list_experiments(document_id=document_id, offset=offset, limit=limit) diff --git a/src/benchmarking/routes.py b/src/benchmarking/routes.py index 13b2fd3..87df44a 100644 --- a/src/benchmarking/routes.py +++ b/src/benchmarking/routes.py @@ -8,7 +8,7 @@ Endpoints: GET /experiments List all experiments """ -from fastapi import APIRouter +from fastapi import APIRouter, Query from fastapi.responses import HTMLResponse from src.core.exceptions import BenchmarkError, QueryError @@ -185,9 +185,15 @@ async def get_benchmark(experiment_id: str): @router.get("/experiments") -async def list_experiments(document_id: str | None = None): +async def list_experiments( + document_id: str | None = None, + offset: int = Query(0, ge=0), + limit: int = Query(200, ge=1, le=500), +): """List all experiments, optionally filtered by document.""" - result = benchmark_service.list_experiments(document_id=document_id) + result = benchmark_service.list_experiments( + document_id=document_id, offset=offset, limit=limit + ) # Enrich with document filenames and best_strategy from src.storage import sqlite as db for item in result.get("items", []): diff --git a/src/static/index.html b/src/static/index.html index ba68a80..bd57e89 100644 --- a/src/static/index.html +++ b/src/static/index.html @@ -434,6 +434,128 @@ .cmp-exp-2 { background: rgba(99,102,241,0.15); color: #818cf8; } .cmp-exp-3 { background: rgba(244,63,94,0.15); color: #fb7185; } .cmp-divider { height: 1px; background: var(--border); margin: 20px 0; } + /* ── Decision Board ─────────────────────────────────────── */ + .decision-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } + @media (max-width: 960px) { .decision-grid { grid-template-columns: 1fr; } } + .decision-cand { + border: 1px solid var(--border); border-radius: var(--radius); + padding: 12px 14px; margin-bottom: 8px; cursor: pointer; + background: var(--bg-base); transition: border-color 0.15s, background 0.15s; + } + .decision-cand:hover { border-color: #3a3a42; } + .decision-cand.selected { + border-color: var(--accent); background: rgba(234,179,8,0.08); + } + .decision-cand .cand-title { font-weight: 600; color: var(--text-heading); margin-bottom: 6px; } + .decision-metrics { display: flex; flex-wrap: wrap; gap: 8px 14px; font-size: 12px; color: var(--text-muted); } + .decision-metrics strong { color: var(--text-body); font-variant-numeric: tabular-nums; } + .decision-legend { + display: flex; flex-wrap: wrap; gap: 10px 16px; align-items: center; + margin-bottom: 14px; padding: 12px 14px; + background: rgba(15, 20, 25, 0.55); + border: 1px solid rgba(255,255,255,0.06); + border-radius: 10px; + font-size: 12px; color: var(--text-muted); + } + .decision-legend-item { display: inline-flex; align-items: center; gap: 7px; } + .decision-legend-swatch { + width: 22px; height: 14px; border-radius: 4px; + border: 1px solid rgba(255,255,255,0.08); + box-shadow: inset 3px 0 0 var(--swatch-accent, transparent); + flex-shrink: 0; + } + .decision-matrix-wrap { + overflow-x: auto; + border: 1px solid rgba(255,255,255,0.06); + border-radius: 12px; + background: rgba(15, 20, 25, 0.35); + } + .decision-matrix { + width: 100%; border-collapse: separate; border-spacing: 0; + font-size: 13px; margin: 0; + } + .decision-matrix th, + .decision-matrix td { + padding: 10px 12px; + border-bottom: 1px solid rgba(255,255,255,0.05); + border-right: 1px solid rgba(255,255,255,0.04); + text-align: center; + vertical-align: middle; + } + .decision-matrix th:last-child, + .decision-matrix td:last-child { border-right: none; } + .decision-matrix tbody tr:last-child th, + .decision-matrix tbody tr:last-child td { border-bottom: none; } + .decision-matrix thead th { + position: sticky; top: 0; z-index: 2; + background: rgba(21, 32, 51, 0.96); + backdrop-filter: blur(8px); + color: #94A3B8; + font-size: 11px; font-weight: 600; + letter-spacing: 0.03em; + white-space: nowrap; + } + .decision-matrix .sticky-col { + position: sticky; left: 0; z-index: 1; + text-align: left; font-weight: 500; white-space: nowrap; + background: #1A2332; color: var(--text-heading); + min-width: 160px; max-width: 220px; + } + .decision-matrix thead .sticky-col { z-index: 3; background: rgba(21, 32, 51, 0.96); } + .decision-matrix tr.mean-row td, + .decision-matrix tr.mean-row .sticky-col { + border-top: 1px solid rgba(255,255,255,0.1); + background: rgba(15, 20, 25, 0.65); + font-weight: 700; + } + .decision-matrix tr.wins-row td, + .decision-matrix tr.wins-row .sticky-col { + background: rgba(15, 20, 25, 0.45); + color: var(--text-muted); + font-weight: 600; + } + .decision-matrix th.col-duel { + color: #FBBF24; + } + .decision-heat { + font-variant-numeric: tabular-nums; + font-size: 12.5px; font-weight: 600; + min-width: 76px; + letter-spacing: 0.01em; + transition: filter 0.15s ease; + } + .decision-heat:hover { filter: brightness(1.12); } + .decision-heat .best-mark, + .decision-legend .best-mark, + .best-mark { + display: inline-block; + font-size: 9px; font-weight: 700; line-height: 1; + margin-right: 5px; padding: 2px 5px; + border-radius: 999px; vertical-align: middle; + letter-spacing: 0.04em; + } + .decision-heat .best-mark.fs, + .decision-legend .best-mark.fs, + .best-mark.fs { + background: rgba(56, 189, 248, 0.15); + color: #7DD3FC; + border: 1px solid rgba(56, 189, 248, 0.28); + } + .decision-heat .best-mark.sem, + .decision-legend .best-mark.sem, + .best-mark.sem { + background: rgba(167, 139, 250, 0.15); + color: #C4B5FD; + border: 1px solid rgba(167, 139, 250, 0.28); + } + .decision-duel { + display: grid; grid-template-columns: 1fr auto 1fr; gap: 16px; align-items: stretch; + } + @media (max-width: 800px) { .decision-duel { grid-template-columns: 1fr; } } + .decision-vs { + display: flex; align-items: center; justify-content: center; + font-weight: 700; color: var(--text-muted); font-size: 18px; + } @@ -554,7 +676,7 @@ function HomeTab() { useEffect(() => { Promise.all([ api('/documents').catch(() => ({ total: 0 })), - api('/experiments').catch(() => []), + api('/experiments?limit=500').catch(() => []), ]).then(([docs, exps]) => { setStats({ docs: docs.total || 0, experiments: (exps?.items || []).length }); setLoading(false); @@ -589,6 +711,7 @@ function HomeTab() { React.createElement('li', null, 'Process it with chunking strategies'), React.createElement('li', null, 'Ask a question in ', React.createElement('b', null, 'Query'), ' (or inside PDF Workspace)'), React.createElement('li', null, 'Run a full benchmark in ', React.createElement('b', null, 'Benchmarks'), ' / PDF Workspace'), + React.createElement('li', null, 'Pick a final Strategy on the ', React.createElement('b', null, 'Decision'), ' board'), React.createElement('li', null, 'Check system health in the ', React.createElement('b', null, 'Admin'), ' tab') ) ) @@ -1660,7 +1783,7 @@ function BenchmarksTab({ documents, strategies, addToast, questionsFile, formatF const [inspectLoadingId, setInspectLoadingId] = useState(null); const fetchExperiments = useCallback(() => { - api('/experiments').then(d => setExperiments(d?.items || (Array.isArray(d) ? d : []))).catch(() => {}); + api('/experiments?limit=500').then(d => setExperiments(d?.items || (Array.isArray(d) ? d : []))).catch(() => {}); }, []); useEffect(() => { fetchExperiments(); }, [fetchExperiments]); @@ -2267,6 +2390,648 @@ function PdfWorkspaceTab({ documents, setDocuments, strategies, addToast, questi ); } +// -- Tab: Decision Board (ADR-0026) -------------------------- +const DECISION_DOC_FILENAMES = [ + 'bazresi.docx', + 'customer1.docx', + 'fire.docx', + 'general-havades-individuals.doc', + 'havades.docx', + 'life-time-individual.docx', + 'moavenin.docx', + 'Refah.docx', + 'website.docx', + 'lifetime-compensation.docx', +]; + +function decisionComposite(m) { + if (!m) return null; + return ((m.avg_context_relevance || 0) * 0.3 + + (m.avg_answer_similarity || 0) * 0.4 + + (m.avg_faithfulness || 0) * 0.3) * (1 - (m.hallucination_rate || 0)); +} + +function decisionFmt(v, digits) { + if (v == null || Number.isNaN(v)) return '—'; + return Number(v).toFixed(digits == null ? 2 : digits); +} + +function decisionPct(v) { + if (v == null || Number.isNaN(v)) return '—'; + return `${(Number(v) * 100).toFixed(0)}%`; +} + +function decisionHeatBin(score) { + if (score == null || Number.isNaN(score)) { + return { + key: 'missing', label: '—', + bg: 'transparent', fg: 'var(--text-muted)', accent: 'transparent', + }; + } + const s = Number(score); + // Modern dark-dashboard scale: translucent wash + luminous text + left accent rail + // (no solid primary blocks — avoids Windows-98 / crayon look) + if (s < 8.0) { + return { + key: 'lt80', label: '<8.0', + bg: 'rgba(244, 63, 94, 0.14)', fg: '#FB7185', accent: '#F43F5E', + }; + } + if (s < 8.5) { + return { + key: '80_85', label: '8.0–8.5', + bg: 'rgba(251, 146, 60, 0.13)', fg: '#FB923C', accent: '#F97316', + }; + } + if (s < 9.0) { + return { + key: '85_90', label: '8.5–9.0', + bg: 'rgba(250, 204, 21, 0.14)', fg: '#FDE047', accent: '#EAB308', + }; + } + if (s < 9.5) { + return { + key: '90_95', label: '9.0–9.5', + bg: 'rgba(45, 212, 191, 0.13)', fg: '#2DD4BF', accent: '#14B8A6', + }; + } + return { + key: 'ge95', label: '≥9.5', + bg: 'rgba(52, 211, 153, 0.16)', fg: '#34D399', accent: '#10B981', + }; +} + +function decisionHeatStyle(score) { + const bin = decisionHeatBin(score); + const style = { background: bin.bg, color: bin.fg }; + if (bin.accent && bin.accent !== 'transparent') { + style.boxShadow = `inset 3px 0 0 ${bin.accent}`; + } + return style; +} + +/** Per-row (or mean-row) ids of best fixed_size and best semantic Candidate. */ +function decisionFamilyBestIds(candidates, scoreOf) { + let bestFs = null, bestFsScore = -Infinity; + let bestSem = null, bestSemScore = -Infinity; + (candidates || []).forEach(c => { + const s = scoreOf(c); + if (s == null || Number.isNaN(s)) return; + if (c.family === 'fixed_size' && s > bestFsScore) { + bestFsScore = s; + bestFs = c.id; + } + if (c.family === 'semantic' && s > bestSemScore) { + bestSemScore = s; + bestSem = c.id; + } + }); + return { fixed: bestFs, semantic: bestSem }; +} + +const DECISION_HEAT_LEGEND = [ + decisionHeatBin(null), + decisionHeatBin(7.9), + decisionHeatBin(8.2), + decisionHeatBin(8.7), + decisionHeatBin(9.2), + decisionHeatBin(9.6), +]; + +function DecisionHeatLegend() { + return React.createElement('div', { className: 'decision-legend', role: 'list', 'aria-label': 'Composite score color guide' }, + React.createElement('span', { style: { fontWeight: 600, color: 'var(--text-body)' } }, 'Score guide'), + DECISION_HEAT_LEGEND.map(bin => + React.createElement('span', { key: bin.key, className: 'decision-legend-item', role: 'listitem' }, + React.createElement('span', { + className: 'decision-legend-swatch', + style: { + background: bin.bg === 'transparent' ? 'rgba(255,255,255,0.04)' : bin.bg, + boxShadow: bin.accent && bin.accent !== 'transparent' ? `inset 3px 0 0 ${bin.accent}` : undefined, + }, + 'aria-hidden': true, + }), + bin.label + ) + ), + React.createElement('span', { + style: { width: 1, height: 14, background: 'rgba(255,255,255,0.1)', margin: '0 4px' }, + 'aria-hidden': true, + }), + React.createElement('span', { className: 'decision-legend-item' }, + React.createElement('span', { className: 'best-mark fs' }, 'F'), + 'best fixed_size' + ), + React.createElement('span', { className: 'decision-legend-item' }, + React.createElement('span', { className: 'best-mark sem' }, 'S'), + 'best semantic' + ) + ); +} + +function meanOf(nums) { + const vals = nums.filter(v => v != null && !Number.isNaN(v)); + if (!vals.length) return null; + return vals.reduce((a, b) => a + b, 0) / vals.length; +} + +function buildDecisionBoard(experiments, corpusId, excludedIds) { + const excluded = new Set(excludedIds || []); + const pool = (experiments || []).filter(e => { + if (excluded.has(e.id)) return false; + const fn = e.document_filename || ''; + if (!DECISION_DOC_FILENAMES.includes(fn)) return false; + if (corpusIdOf(e) !== corpusId) return false; + const strats = e.strategies_used || []; + return strats.length === 1; + }); + + // Newest first assumed from API; keep first hit per cell + const cellMap = {}; // key: `${fn}||${candId}` -> exp + const fsLevels = new Set([0, 1, 2, 3]); + const semBounds = new Set(); + + pool.forEach(e => { + const strat = (e.strategies_used || [])[0]; + const fn = e.document_filename; + if (strat === 'fixed_size') { + const { prev, next } = neighborCounts(e); + if (prev !== next || !fsLevels.has(prev)) return; + const candId = `fixed_size:±${prev}`; + const key = `${fn}||${candId}`; + if (!cellMap[key]) cellMap[key] = e; + } else if (strat === 'semantic') { + const b = boundaryIdOf(e); + if (!b) return; + semBounds.add(b); + const candId = `semantic:${b}`; + const key = `${fn}||${candId}`; + if (!cellMap[key]) cellMap[key] = e; + } + }); + + const fixedCands = [0, 1, 2, 3].map(n => ({ + id: `fixed_size:±${n}`, + family: 'fixed_size', + label: `fixed_size ±${n}/${n}`, + short: `±${n}`, + level: n, + })); + const semanticCands = [...semBounds].sort().map(b => ({ + id: `semantic:${b}`, + family: 'semantic', + label: `semantic @ ${b}`, + short: b, + boundary: b, + })); + + function metricsFor(exp, family) { + if (!exp) return null; + return (exp.aggregate_metrics || {})[family] || null; + } + + function summarize(cands) { + return cands.map(c => { + const perDoc = {}; + const composites = []; + const metricBags = { + avg_context_relevance: [], + avg_answer_similarity: [], + avg_faithfulness: [], + hallucination_rate: [], + }; + let filled = 0; + DECISION_DOC_FILENAMES.forEach(fn => { + const exp = cellMap[`${fn}||${c.id}`]; + const m = metricsFor(exp, c.family); + const score = decisionComposite(m); + perDoc[fn] = { exp, metrics: m, composite: score }; + if (score != null) { + filled += 1; + composites.push(score); + Object.keys(metricBags).forEach(k => { + if (m && m[k] != null) metricBags[k].push(m[k]); + }); + } + }); + return { + ...c, + perDoc, + filled, + totalDocs: DECISION_DOC_FILENAMES.length, + meanComposite: meanOf(composites), + meanMetrics: { + avg_context_relevance: meanOf(metricBags.avg_context_relevance), + avg_answer_similarity: meanOf(metricBags.avg_answer_similarity), + avg_faithfulness: meanOf(metricBags.avg_faithfulness), + hallucination_rate: meanOf(metricBags.hallucination_rate), + }, + wins: 0, + }; + }); + } + + const fixedSummaries = summarize(fixedCands); + const semanticSummaries = summarize(semanticCands); + + function assignWins(summaries) { + DECISION_DOC_FILENAMES.forEach(fn => { + let best = null; + let bestScore = -Infinity; + summaries.forEach(s => { + const sc = s.perDoc[fn]?.composite; + if (sc == null) return; + if (sc > bestScore) { + bestScore = sc; + best = s; + } + }); + if (best) best.wins += 1; + }); + } + assignWins(fixedSummaries); + assignWins(semanticSummaries); + + function autoPick(summaries) { + if (!summaries.length) return null; + return [...summaries].sort((a, b) => { + const ma = a.meanComposite == null ? -1 : a.meanComposite; + const mb = b.meanComposite == null ? -1 : b.meanComposite; + if (mb !== ma) return mb - ma; + if (b.wins !== a.wins) return b.wins - a.wins; + return a.id.localeCompare(b.id); + })[0]; + } + + return { + cellMap, + fixedSummaries, + semanticSummaries, + autoFixed: autoPick(fixedSummaries), + autoSemantic: autoPick(semanticSummaries), + allCandidates: [...fixedSummaries, ...semanticSummaries], + }; +} + +function DecisionTab({ addToast }) { + const [experiments, setExperiments] = useState([]); + const [loading, setLoading] = useState(true); + const [corpusId, setCorpusId] = useState('text-embedding-3-large'); + const [embModels, setEmbModels] = useState([]); + const [excludedIds, setExcludedIds] = useState([]); + const [overrideFixed, setOverrideFixed] = useState(null); + const [overrideSemantic, setOverrideSemantic] = useState(null); + + const refresh = useCallback(() => { + setLoading(true); + Promise.all([ + api('/experiments?limit=500'), + api('/admin/embedding-models'), + ]).then(([ex, emb]) => { + setExperiments(ex?.items || []); + setEmbModels(emb?.models || []); + if (emb?.corpus_id) { + setCorpusId(prev => prev || emb.corpus_id); + } + }).catch(() => addToast('Failed to load Decision Board data', 'error')) + .finally(() => setLoading(false)); + }, [addToast]); + + useEffect(() => { refresh(); }, [refresh]); + + const board = React.useMemo( + () => buildDecisionBoard(experiments, corpusId, excludedIds), + [experiments, corpusId, excludedIds] + ); + + const winnerFixed = (overrideFixed && board.fixedSummaries.find(c => c.id === overrideFixed)) + || board.autoFixed; + const winnerSemantic = (overrideSemantic && board.semanticSummaries.find(c => c.id === overrideSemantic)) + || board.autoSemantic; + + const excludeId = (id) => { + if (!id) return; + setExcludedIds(prev => prev.includes(id) ? prev : [...prev, id]); + }; + const unexcludeId = (id) => setExcludedIds(prev => prev.filter(x => x !== id)); + + const fsFilled = board.fixedSummaries.reduce((a, c) => a + c.filled, 0); + const fsTotal = board.fixedSummaries.length * DECISION_DOC_FILENAMES.length; + const semFilled = board.semanticSummaries.reduce((a, c) => a + c.filled, 0); + const semTotal = board.semanticSummaries.length * DECISION_DOC_FILENAMES.length; + + // Stage-2 head-to-head + let duel = null; + if (winnerFixed && winnerSemantic) { + let fsWins = 0, semWins = 0, ties = 0; + const perDoc = DECISION_DOC_FILENAMES.map(fn => { + const a = winnerFixed.perDoc[fn]?.composite; + const b = winnerSemantic.perDoc[fn]?.composite; + let winner = '—'; + if (a != null && b != null) { + if (a > b) { fsWins += 1; winner = 'fixed'; } + else if (b > a) { semWins += 1; winner = 'semantic'; } + else { ties += 1; winner = 'tie'; } + } + return { fn, a, b, winner }; + }); + const recommend = (winnerFixed.meanComposite || 0) >= (winnerSemantic.meanComposite || 0) + ? winnerFixed : winnerSemantic; + // Prefer win-count if mean close? stick to mean primary with wins as display + duel = { fsWins, semWins, ties, perDoc, recommend }; + } + + const renderCandCard = (c, selected, onSelect, familyAutoId) => + React.createElement('div', { + key: c.id, + className: `decision-cand${selected ? ' selected' : ''}`, + onClick: () => onSelect(selected ? null : (c.id === familyAutoId ? null : c.id)), + title: 'Click to override family winner; click selected again to return to auto', + }, + React.createElement('div', { className: 'cand-title', style: { display: 'flex', justifyContent: 'space-between', gap: 8 } }, + React.createElement('span', null, + React.createElement('input', { + type: 'radio', + checked: !!selected, + readOnly: true, + style: { marginRight: 8 }, + }), + c.label, + c.id === familyAutoId + ? React.createElement('span', { className: 'badge badge-accent', style: { marginLeft: 8 } }, 'auto') + : null + ), + React.createElement('span', { style: { color: 'var(--accent)', fontVariantNumeric: 'tabular-nums' } }, + decisionFmt(c.meanComposite)) + ), + React.createElement('div', { className: 'decision-metrics' }, + React.createElement('span', null, 'Wins ', React.createElement('strong', null, `${c.wins}/${c.totalDocs}`)), + React.createElement('span', null, 'Coverage ', React.createElement('strong', null, `${c.filled}/${c.totalDocs}`)), + React.createElement('span', null, 'Context ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_context_relevance))), + React.createElement('span', null, 'Similarity ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_answer_similarity))), + React.createElement('span', null, 'Faithfulness ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_faithfulness))), + React.createElement('span', null, 'Halluc. ', React.createElement('strong', null, decisionPct(c.meanMetrics.hallucination_rate))) + ) + ); + + const duelPanel = (c, side) => { + if (!c) { + return React.createElement('div', { className: 'card', style: { margin: 0 } }, + React.createElement('div', { className: 'text-muted' }, `No ${side} winner yet`)); + } + const isRec = duel && duel.recommend && duel.recommend.id === c.id; + return React.createElement('div', { + className: 'card', + style: { + margin: 0, + borderColor: isRec ? 'var(--accent)' : undefined, + boxShadow: isRec ? '0 0 0 1px rgba(234,179,8,0.35)' : undefined, + }, + }, + React.createElement('div', { className: 'flex-between mb-2' }, + React.createElement('div', { className: 'card-title mb-0' }, c.label), + isRec ? React.createElement('span', { className: 'badge badge-success' }, 'Recommended') : null + ), + React.createElement('div', { style: { fontSize: 28, fontWeight: 700, color: 'var(--accent)', marginBottom: 8 } }, + decisionFmt(c.meanComposite)), + React.createElement('div', { className: 'decision-metrics', style: { marginBottom: 8 } }, + React.createElement('span', null, 'Doc wins vs other ', React.createElement('strong', null, + side === 'fixed' ? (duel ? duel.fsWins : '—') : (duel ? duel.semWins : '—'))), + React.createElement('span', null, 'Family wins ', React.createElement('strong', null, `${c.wins}/${c.totalDocs}`)), + React.createElement('span', null, 'Coverage ', React.createElement('strong', null, `${c.filled}/${c.totalDocs}`)) + ), + React.createElement('table', { className: 'cmp-table' }, + React.createElement('tbody', null, + [['Context Relevance', c.meanMetrics.avg_context_relevance, false], + ['Answer Similarity', c.meanMetrics.avg_answer_similarity, false], + ['Faithfulness', c.meanMetrics.avg_faithfulness, false], + ['Hallucination Rate', c.meanMetrics.hallucination_rate, true]].map(([label, val, isPct]) => + React.createElement('tr', { key: label }, + React.createElement('td', null, label), + React.createElement('td', { style: { fontWeight: 600 } }, isPct ? decisionPct(val) : decisionFmt(val)) + ) + ) + ) + ) + ); + }; + + return React.createElement('div', null, + React.createElement('div', { className: 'flex-between mb-2' }, + React.createElement('h1', { style: { color: 'var(--text-heading)', margin: 0, fontSize: '22px' } }, 'Decision Board'), + React.createElement('button', { className: 'btn btn-secondary btn-sm', onClick: refresh, disabled: loading }, + loading ? React.createElement('span', { className: 'spinner' }) : '↻ Refresh') + ), + React.createElement('p', { className: 'text-sm text-muted', style: { marginBottom: 16, maxWidth: 720 } }, + 'Two-stage final selection: pick the best fixed_size Neighbor level and best semantic Boundary, then compare those winners across the 10-doc evaluation set. Mean composite ranks stage 1; win-counts are shown; click a Candidate to override.' + ), + + // Header controls + React.createElement('div', { className: 'card mb-4' }, + React.createElement('div', { className: 'row mb-0', style: { alignItems: 'flex-end' } }, + React.createElement(EmbeddingModelSelect, { + label: 'Corpus filter', + value: corpusId, + onChange: (v) => { setCorpusId(v); setOverrideFixed(null); setOverrideSemantic(null); }, + models: embModels, + role: 'corpus', + title: 'Only single-strategy Experiments under this Corpus Embedding Model', + style: { maxWidth: 360 }, + }), + React.createElement('div', { className: 'col' }, + React.createElement('div', { className: 'text-sm text-muted' }, 'Coverage'), + React.createElement('div', { style: { fontWeight: 600, color: 'var(--text-heading)', marginTop: 6 } }, + `fixed_size ${fsFilled}/${fsTotal || 40} · semantic ${semFilled}/${semTotal || '—'}`) + ) + ), + excludedIds.length > 0 && React.createElement('div', { style: { marginTop: 12 } }, + React.createElement('div', { className: 'text-sm text-muted mb-1' }, 'Excluded Experiments (next-newest fills the cell)'), + React.createElement('div', { style: { display: 'flex', flexWrap: 'wrap', gap: 6 } }, + excludedIds.map(id => + React.createElement('button', { + key: id, + className: 'btn btn-secondary btn-sm', + onClick: () => unexcludeId(id), + title: 'Click to restore', + }, `✕ ${id.substring(0, 8)}…`) + ) + ) + ) + ), + + loading && React.createElement('div', { style: { textAlign: 'center', padding: 24 } }, + React.createElement('span', { className: 'spinner' })), + + !loading && React.createElement(React.Fragment, null, + // Stage 1 + React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Stage 1 — Best Candidate per family'), + React.createElement('div', { className: 'decision-grid mb-4' }, + React.createElement('div', { className: 'card', style: { margin: 0 } }, + React.createElement('div', { className: 'card-title' }, 'fixed_size (Neighbor Expansion)'), + React.createElement('div', { className: 'text-sm text-muted mb-2' }, + 'Auto winner: ', board.autoFixed ? board.autoFixed.label : '—', + overrideFixed ? ' · override active' : ''), + board.fixedSummaries.map(c => renderCandCard( + c, + winnerFixed && winnerFixed.id === c.id, + setOverrideFixed, + board.autoFixed?.id + )) + ), + React.createElement('div', { className: 'card', style: { margin: 0 } }, + React.createElement('div', { className: 'card-title' }, 'semantic (Boundary Embedding Model)'), + React.createElement('div', { className: 'text-sm text-muted mb-2' }, + 'Auto winner: ', board.autoSemantic ? board.autoSemantic.label : '—', + overrideSemantic ? ' · override active' : ''), + board.semanticSummaries.length === 0 + ? React.createElement('div', { className: 'empty-state' }, 'No semantic single-strategy Experiments for this Corpus') + : board.semanticSummaries.map(c => renderCandCard( + c, + winnerSemantic && winnerSemantic.id === c.id, + setOverrideSemantic, + board.autoSemantic?.id + )) + ) + ), + + // Stage 2 + React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Stage 2 — Family showdown'), + React.createElement('div', { className: 'decision-duel mb-4' }, + duelPanel(winnerFixed, 'fixed'), + React.createElement('div', { className: 'decision-vs' }, 'vs'), + duelPanel(winnerSemantic, 'semantic') + ), + duel && React.createElement('div', { className: 'card mb-4' }, + React.createElement('div', { className: 'card-title' }, 'Head-to-head by document'), + React.createElement('div', { className: 'text-sm text-muted mb-2' }, + `fixed_size wins ${duel.fsWins} · semantic wins ${duel.semWins} · ties ${duel.ties}`), + React.createElement('table', { className: 'cmp-table' }, + React.createElement('thead', null, + React.createElement('tr', null, + React.createElement('th', null, 'Document'), + React.createElement('th', null, winnerFixed?.short || 'fixed'), + React.createElement('th', null, winnerSemantic?.short || 'semantic'), + React.createElement('th', null, 'Winner') + ) + ), + React.createElement('tbody', null, + duel.perDoc.map(row => + React.createElement('tr', { key: row.fn }, + React.createElement('td', null, row.fn), + React.createElement('td', { style: { textAlign: 'center', fontVariantNumeric: 'tabular-nums' } }, decisionFmt(row.a)), + React.createElement('td', { style: { textAlign: 'center', fontVariantNumeric: 'tabular-nums' } }, decisionFmt(row.b)), + React.createElement('td', null, + row.winner === 'fixed' ? React.createElement('span', { className: 'badge badge-accent' }, 'fixed_size') + : row.winner === 'semantic' ? React.createElement('span', { className: 'badge badge-success' }, 'semantic') + : row.winner === 'tie' ? React.createElement('span', { className: 'badge' }, 'tie') + : React.createElement('span', { className: 'text-muted' }, '—') + ) + ) + ) + ) + ) + ), + + // Matrix + React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Per-document matrix'), + React.createElement('div', { className: 'card', style: { paddingBottom: 14 } }, + React.createElement(DecisionHeatLegend), + React.createElement('div', { className: 'decision-matrix-wrap' }, + React.createElement('table', { className: 'decision-matrix' }, + React.createElement('thead', null, + React.createElement('tr', null, + React.createElement('th', { className: 'sticky-col' }, 'Document'), + board.allCandidates.map(c => + React.createElement('th', { + key: c.id, + className: (winnerFixed && c.id === winnerFixed.id) || (winnerSemantic && c.id === winnerSemantic.id) + ? 'col-duel' : undefined, + title: c.label, + }, c.family === 'fixed_size' ? c.short : (c.short || '').substring(0, 14)) + ) + ) + ), + React.createElement('tbody', null, + DECISION_DOC_FILENAMES.map(fn => { + const rowBest = decisionFamilyBestIds(board.allCandidates, c => c.perDoc[fn]?.composite); + return React.createElement('tr', { key: fn }, + React.createElement('td', { className: 'sticky-col' }, fn), + board.allCandidates.map(c => { + const cell = c.perDoc[fn]; + const score = cell?.composite; + const exp = cell?.exp; + const mark = c.id === rowBest.fixed ? 'fs' : (c.id === rowBest.semantic ? 'sem' : null); + return React.createElement('td', { + key: c.id, + className: 'decision-heat', + style: { ...decisionHeatStyle(score), cursor: exp ? 'pointer' : 'default' }, + title: exp + ? `${c.label} · ${fn}\ncomposite=${decisionFmt(score)}${mark ? `\nbest ${mark === 'fs' ? 'fixed_size' : 'semantic'} in row` : ''}\nid=${exp.id}\nClick: report · Shift+click: exclude` + : `${c.label} · ${fn}: missing`, + onClick: (e) => { + if (!exp) return; + if (e.shiftKey) { + excludeId(exp.id); + addToast(`Excluded ${exp.id.substring(0, 8)}…`, 'success'); + return; + } + window.open(`/benchmarks/${exp.id}/report`, '_blank'); + }, + }, + mark + ? React.createElement('span', { + className: `best-mark ${mark}`, + 'aria-label': mark === 'fs' ? 'Best fixed_size in row' : 'Best semantic in row', + }, mark === 'fs' ? 'F' : 'S') + : null, + decisionFmt(score) + ); + }) + ); + }), + (() => { + const meanBest = decisionFamilyBestIds(board.allCandidates, c => c.meanComposite); + return React.createElement('tr', { key: '_mean', className: 'mean-row' }, + React.createElement('td', { className: 'sticky-col' }, 'Mean'), + board.allCandidates.map(c => { + const mark = c.id === meanBest.fixed ? 'fs' : (c.id === meanBest.semantic ? 'sem' : null); + return React.createElement('td', { + key: c.id, + className: 'decision-heat', + style: decisionHeatStyle(c.meanComposite), + }, + mark + ? React.createElement('span', { + className: `best-mark ${mark}`, + 'aria-label': mark === 'fs' ? 'Best fixed_size mean' : 'Best semantic mean', + }, mark === 'fs' ? 'F' : 'S') + : null, + decisionFmt(c.meanComposite) + ); + }) + ); + })(), + React.createElement('tr', { key: '_wins', className: 'wins-row' }, + React.createElement('td', { className: 'sticky-col' }, 'Wins'), + board.allCandidates.map(c => + React.createElement('td', { key: c.id }, `${c.wins}/${c.totalDocs}`) + ) + ) + ) + ) + ), + React.createElement('div', { className: 'text-sm text-muted', style: { marginTop: 12 } }, + 'Click a cell to open its Experiment report. Shift+click to exclude. ', + React.createElement('span', { className: 'best-mark fs' }, 'F'), + ' / ', + React.createElement('span', { className: 'best-mark sem' }, 'S'), + ' mark the best score in each family per row.' + ) + ) + ) + ); +} + // -- Tab: Admin --------------------------------------------- function AdminTab({ addToast, documents, strategies, setActiveTab, setQuestionsFile }) { // Collapse state for each section @@ -2804,6 +3569,7 @@ function App() { { id: 'pdf', label: 'PDF' }, { id: 'query', label: 'Query' }, { id: 'benchmarks', label: 'Benchmarks' }, + { id: 'decision', label: 'Decision' }, { id: 'admin', label: 'Admin' }, ]; @@ -2818,6 +3584,7 @@ function App() { documents, strategies, addToast, formatFilter: 'word' }); case 'benchmarks': return React.createElement(BenchmarksTab, { documents, strategies, addToast, questionsFile, formatFilter: 'word' }); + case 'decision': return React.createElement(DecisionTab, { addToast }); case 'admin': return React.createElement(AdminTab, { addToast, documents, strategies, setActiveTab, setQuestionsFile }); default: return null; diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index 200738f..1216447 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -408,9 +408,11 @@ def get_experiment(experiment_id: str) -> dict[str, Any] | None: def list_experiments( - *, document_id: str | None = None, offset: int = 0, limit: int = 50 + *, document_id: str | None = None, offset: int = 0, limit: int = 200 ) -> dict[str, Any]: """List experiments, optionally filtered by document.""" + limit = max(1, min(int(limit), 500)) + offset = max(0, int(offset)) conn = _connect() try: if document_id: