feat(dashboard): add decision board for strategy selection

Why:
- Compare is the wrong surface for two-stage family selection over the 10-doc set.

Changes:
- Add the Decision Tab; raise GET /experiments default/max so the board can load the grid client-side.

Impact:
- Operators pick fixed_size ±N vs semantic@Boundary from existing Experiments.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-17 14:22:21 +03:30
parent e2c7fdc059
commit 56b8d9401a
4 changed files with 788 additions and 8 deletions

View File

@@ -399,6 +399,11 @@ def get_experiment(experiment_id: str) -> dict[str, Any] | None:
return db.get_experiment(experiment_id) return db.get_experiment(experiment_id)
def list_experiments(document_id: str | None = None) -> dict[str, Any]: def list_experiments(
document_id: str | None = None,
*,
offset: int = 0,
limit: int = 200,
) -> dict[str, Any]:
"""List experiments, optionally filtered by document.""" """List experiments, optionally filtered by document."""
return db.list_experiments(document_id=document_id) return db.list_experiments(document_id=document_id, offset=offset, limit=limit)

View File

@@ -8,7 +8,7 @@ Endpoints:
GET /experiments List all experiments GET /experiments List all experiments
""" """
from fastapi import APIRouter from fastapi import APIRouter, Query
from fastapi.responses import HTMLResponse from fastapi.responses import HTMLResponse
from src.core.exceptions import BenchmarkError, QueryError from src.core.exceptions import BenchmarkError, QueryError
@@ -185,9 +185,15 @@ async def get_benchmark(experiment_id: str):
@router.get("/experiments") @router.get("/experiments")
async def list_experiments(document_id: str | None = None): async def list_experiments(
document_id: str | None = None,
offset: int = Query(0, ge=0),
limit: int = Query(200, ge=1, le=500),
):
"""List all experiments, optionally filtered by document.""" """List all experiments, optionally filtered by document."""
result = benchmark_service.list_experiments(document_id=document_id) result = benchmark_service.list_experiments(
document_id=document_id, offset=offset, limit=limit
)
# Enrich with document filenames and best_strategy # Enrich with document filenames and best_strategy
from src.storage import sqlite as db from src.storage import sqlite as db
for item in result.get("items", []): for item in result.get("items", []):

View File

@@ -434,6 +434,128 @@
.cmp-exp-2 { background: rgba(99,102,241,0.15); color: #818cf8; } .cmp-exp-2 { background: rgba(99,102,241,0.15); color: #818cf8; }
.cmp-exp-3 { background: rgba(244,63,94,0.15); color: #fb7185; } .cmp-exp-3 { background: rgba(244,63,94,0.15); color: #fb7185; }
.cmp-divider { height: 1px; background: var(--border); margin: 20px 0; } .cmp-divider { height: 1px; background: var(--border); margin: 20px 0; }
/* ── Decision Board ─────────────────────────────────────── */
.decision-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
@media (max-width: 960px) { .decision-grid { grid-template-columns: 1fr; } }
.decision-cand {
border: 1px solid var(--border); border-radius: var(--radius);
padding: 12px 14px; margin-bottom: 8px; cursor: pointer;
background: var(--bg-base); transition: border-color 0.15s, background 0.15s;
}
.decision-cand:hover { border-color: #3a3a42; }
.decision-cand.selected {
border-color: var(--accent); background: rgba(234,179,8,0.08);
}
.decision-cand .cand-title { font-weight: 600; color: var(--text-heading); margin-bottom: 6px; }
.decision-metrics { display: flex; flex-wrap: wrap; gap: 8px 14px; font-size: 12px; color: var(--text-muted); }
.decision-metrics strong { color: var(--text-body); font-variant-numeric: tabular-nums; }
.decision-legend {
display: flex; flex-wrap: wrap; gap: 10px 16px; align-items: center;
margin-bottom: 14px; padding: 12px 14px;
background: rgba(15, 20, 25, 0.55);
border: 1px solid rgba(255,255,255,0.06);
border-radius: 10px;
font-size: 12px; color: var(--text-muted);
}
.decision-legend-item { display: inline-flex; align-items: center; gap: 7px; }
.decision-legend-swatch {
width: 22px; height: 14px; border-radius: 4px;
border: 1px solid rgba(255,255,255,0.08);
box-shadow: inset 3px 0 0 var(--swatch-accent, transparent);
flex-shrink: 0;
}
.decision-matrix-wrap {
overflow-x: auto;
border: 1px solid rgba(255,255,255,0.06);
border-radius: 12px;
background: rgba(15, 20, 25, 0.35);
}
.decision-matrix {
width: 100%; border-collapse: separate; border-spacing: 0;
font-size: 13px; margin: 0;
}
.decision-matrix th,
.decision-matrix td {
padding: 10px 12px;
border-bottom: 1px solid rgba(255,255,255,0.05);
border-right: 1px solid rgba(255,255,255,0.04);
text-align: center;
vertical-align: middle;
}
.decision-matrix th:last-child,
.decision-matrix td:last-child { border-right: none; }
.decision-matrix tbody tr:last-child th,
.decision-matrix tbody tr:last-child td { border-bottom: none; }
.decision-matrix thead th {
position: sticky; top: 0; z-index: 2;
background: rgba(21, 32, 51, 0.96);
backdrop-filter: blur(8px);
color: #94A3B8;
font-size: 11px; font-weight: 600;
letter-spacing: 0.03em;
white-space: nowrap;
}
.decision-matrix .sticky-col {
position: sticky; left: 0; z-index: 1;
text-align: left; font-weight: 500; white-space: nowrap;
background: #1A2332; color: var(--text-heading);
min-width: 160px; max-width: 220px;
}
.decision-matrix thead .sticky-col { z-index: 3; background: rgba(21, 32, 51, 0.96); }
.decision-matrix tr.mean-row td,
.decision-matrix tr.mean-row .sticky-col {
border-top: 1px solid rgba(255,255,255,0.1);
background: rgba(15, 20, 25, 0.65);
font-weight: 700;
}
.decision-matrix tr.wins-row td,
.decision-matrix tr.wins-row .sticky-col {
background: rgba(15, 20, 25, 0.45);
color: var(--text-muted);
font-weight: 600;
}
.decision-matrix th.col-duel {
color: #FBBF24;
}
.decision-heat {
font-variant-numeric: tabular-nums;
font-size: 12.5px; font-weight: 600;
min-width: 76px;
letter-spacing: 0.01em;
transition: filter 0.15s ease;
}
.decision-heat:hover { filter: brightness(1.12); }
.decision-heat .best-mark,
.decision-legend .best-mark,
.best-mark {
display: inline-block;
font-size: 9px; font-weight: 700; line-height: 1;
margin-right: 5px; padding: 2px 5px;
border-radius: 999px; vertical-align: middle;
letter-spacing: 0.04em;
}
.decision-heat .best-mark.fs,
.decision-legend .best-mark.fs,
.best-mark.fs {
background: rgba(56, 189, 248, 0.15);
color: #7DD3FC;
border: 1px solid rgba(56, 189, 248, 0.28);
}
.decision-heat .best-mark.sem,
.decision-legend .best-mark.sem,
.best-mark.sem {
background: rgba(167, 139, 250, 0.15);
color: #C4B5FD;
border: 1px solid rgba(167, 139, 250, 0.28);
}
.decision-duel {
display: grid; grid-template-columns: 1fr auto 1fr; gap: 16px; align-items: stretch;
}
@media (max-width: 800px) { .decision-duel { grid-template-columns: 1fr; } }
.decision-vs {
display: flex; align-items: center; justify-content: center;
font-weight: 700; color: var(--text-muted); font-size: 18px;
}
</style> </style>
</head> </head>
<body> <body>
@@ -554,7 +676,7 @@ function HomeTab() {
useEffect(() => { useEffect(() => {
Promise.all([ Promise.all([
api('/documents').catch(() => ({ total: 0 })), api('/documents').catch(() => ({ total: 0 })),
api('/experiments').catch(() => []), api('/experiments?limit=500').catch(() => []),
]).then(([docs, exps]) => { ]).then(([docs, exps]) => {
setStats({ docs: docs.total || 0, experiments: (exps?.items || []).length }); setStats({ docs: docs.total || 0, experiments: (exps?.items || []).length });
setLoading(false); setLoading(false);
@@ -589,6 +711,7 @@ function HomeTab() {
React.createElement('li', null, 'Process it with chunking strategies'), React.createElement('li', null, 'Process it with chunking strategies'),
React.createElement('li', null, 'Ask a question in ', React.createElement('b', null, 'Query'), ' (or inside PDF Workspace)'), React.createElement('li', null, 'Ask a question in ', React.createElement('b', null, 'Query'), ' (or inside PDF Workspace)'),
React.createElement('li', null, 'Run a full benchmark in ', React.createElement('b', null, 'Benchmarks'), ' / PDF Workspace'), React.createElement('li', null, 'Run a full benchmark in ', React.createElement('b', null, 'Benchmarks'), ' / PDF Workspace'),
React.createElement('li', null, 'Pick a final Strategy on the ', React.createElement('b', null, 'Decision'), ' board'),
React.createElement('li', null, 'Check system health in the ', React.createElement('b', null, 'Admin'), ' tab') React.createElement('li', null, 'Check system health in the ', React.createElement('b', null, 'Admin'), ' tab')
) )
) )
@@ -1660,7 +1783,7 @@ function BenchmarksTab({ documents, strategies, addToast, questionsFile, formatF
const [inspectLoadingId, setInspectLoadingId] = useState(null); const [inspectLoadingId, setInspectLoadingId] = useState(null);
const fetchExperiments = useCallback(() => { const fetchExperiments = useCallback(() => {
api('/experiments').then(d => setExperiments(d?.items || (Array.isArray(d) ? d : []))).catch(() => {}); api('/experiments?limit=500').then(d => setExperiments(d?.items || (Array.isArray(d) ? d : []))).catch(() => {});
}, []); }, []);
useEffect(() => { fetchExperiments(); }, [fetchExperiments]); useEffect(() => { fetchExperiments(); }, [fetchExperiments]);
@@ -2267,6 +2390,648 @@ function PdfWorkspaceTab({ documents, setDocuments, strategies, addToast, questi
); );
} }
// -- Tab: Decision Board (ADR-0026) --------------------------
const DECISION_DOC_FILENAMES = [
'bazresi.docx',
'customer1.docx',
'fire.docx',
'general-havades-individuals.doc',
'havades.docx',
'life-time-individual.docx',
'moavenin.docx',
'Refah.docx',
'website.docx',
'lifetime-compensation.docx',
];
function decisionComposite(m) {
if (!m) return null;
return ((m.avg_context_relevance || 0) * 0.3
+ (m.avg_answer_similarity || 0) * 0.4
+ (m.avg_faithfulness || 0) * 0.3) * (1 - (m.hallucination_rate || 0));
}
function decisionFmt(v, digits) {
if (v == null || Number.isNaN(v)) return '—';
return Number(v).toFixed(digits == null ? 2 : digits);
}
function decisionPct(v) {
if (v == null || Number.isNaN(v)) return '—';
return `${(Number(v) * 100).toFixed(0)}%`;
}
function decisionHeatBin(score) {
if (score == null || Number.isNaN(score)) {
return {
key: 'missing', label: '—',
bg: 'transparent', fg: 'var(--text-muted)', accent: 'transparent',
};
}
const s = Number(score);
// Modern dark-dashboard scale: translucent wash + luminous text + left accent rail
// (no solid primary blocks — avoids Windows-98 / crayon look)
if (s < 8.0) {
return {
key: 'lt80', label: '<8.0',
bg: 'rgba(244, 63, 94, 0.14)', fg: '#FB7185', accent: '#F43F5E',
};
}
if (s < 8.5) {
return {
key: '80_85', label: '8.0–8.5',
bg: 'rgba(251, 146, 60, 0.13)', fg: '#FB923C', accent: '#F97316',
};
}
if (s < 9.0) {
return {
key: '85_90', label: '8.5–9.0',
bg: 'rgba(250, 204, 21, 0.14)', fg: '#FDE047', accent: '#EAB308',
};
}
if (s < 9.5) {
return {
key: '90_95', label: '9.0–9.5',
bg: 'rgba(45, 212, 191, 0.13)', fg: '#2DD4BF', accent: '#14B8A6',
};
}
return {
key: 'ge95', label: '≥9.5',
bg: 'rgba(52, 211, 153, 0.16)', fg: '#34D399', accent: '#10B981',
};
}
function decisionHeatStyle(score) {
const bin = decisionHeatBin(score);
const style = { background: bin.bg, color: bin.fg };
if (bin.accent && bin.accent !== 'transparent') {
style.boxShadow = `inset 3px 0 0 ${bin.accent}`;
}
return style;
}
/** Per-row (or mean-row) ids of best fixed_size and best semantic Candidate. */
function decisionFamilyBestIds(candidates, scoreOf) {
let bestFs = null, bestFsScore = -Infinity;
let bestSem = null, bestSemScore = -Infinity;
(candidates || []).forEach(c => {
const s = scoreOf(c);
if (s == null || Number.isNaN(s)) return;
if (c.family === 'fixed_size' && s > bestFsScore) {
bestFsScore = s;
bestFs = c.id;
}
if (c.family === 'semantic' && s > bestSemScore) {
bestSemScore = s;
bestSem = c.id;
}
});
return { fixed: bestFs, semantic: bestSem };
}
const DECISION_HEAT_LEGEND = [
decisionHeatBin(null),
decisionHeatBin(7.9),
decisionHeatBin(8.2),
decisionHeatBin(8.7),
decisionHeatBin(9.2),
decisionHeatBin(9.6),
];
function DecisionHeatLegend() {
return React.createElement('div', { className: 'decision-legend', role: 'list', 'aria-label': 'Composite score color guide' },
React.createElement('span', { style: { fontWeight: 600, color: 'var(--text-body)' } }, 'Score guide'),
DECISION_HEAT_LEGEND.map(bin =>
React.createElement('span', { key: bin.key, className: 'decision-legend-item', role: 'listitem' },
React.createElement('span', {
className: 'decision-legend-swatch',
style: {
background: bin.bg === 'transparent' ? 'rgba(255,255,255,0.04)' : bin.bg,
boxShadow: bin.accent && bin.accent !== 'transparent' ? `inset 3px 0 0 ${bin.accent}` : undefined,
},
'aria-hidden': true,
}),
bin.label
)
),
React.createElement('span', {
style: { width: 1, height: 14, background: 'rgba(255,255,255,0.1)', margin: '0 4px' },
'aria-hidden': true,
}),
React.createElement('span', { className: 'decision-legend-item' },
React.createElement('span', { className: 'best-mark fs' }, 'F'),
'best fixed_size'
),
React.createElement('span', { className: 'decision-legend-item' },
React.createElement('span', { className: 'best-mark sem' }, 'S'),
'best semantic'
)
);
}
function meanOf(nums) {
const vals = nums.filter(v => v != null && !Number.isNaN(v));
if (!vals.length) return null;
return vals.reduce((a, b) => a + b, 0) / vals.length;
}
function buildDecisionBoard(experiments, corpusId, excludedIds) {
const excluded = new Set(excludedIds || []);
const pool = (experiments || []).filter(e => {
if (excluded.has(e.id)) return false;
const fn = e.document_filename || '';
if (!DECISION_DOC_FILENAMES.includes(fn)) return false;
if (corpusIdOf(e) !== corpusId) return false;
const strats = e.strategies_used || [];
return strats.length === 1;
});
// Newest first assumed from API; keep first hit per cell
const cellMap = {}; // key: `${fn}||${candId}` -> exp
const fsLevels = new Set([0, 1, 2, 3]);
const semBounds = new Set();
pool.forEach(e => {
const strat = (e.strategies_used || [])[0];
const fn = e.document_filename;
if (strat === 'fixed_size') {
const { prev, next } = neighborCounts(e);
if (prev !== next || !fsLevels.has(prev)) return;
const candId = `fixed_size:±${prev}`;
const key = `${fn}||${candId}`;
if (!cellMap[key]) cellMap[key] = e;
} else if (strat === 'semantic') {
const b = boundaryIdOf(e);
if (!b) return;
semBounds.add(b);
const candId = `semantic:${b}`;
const key = `${fn}||${candId}`;
if (!cellMap[key]) cellMap[key] = e;
}
});
const fixedCands = [0, 1, 2, 3].map(n => ({
id: `fixed_size:±${n}`,
family: 'fixed_size',
label: `fixed_size ±${n}/${n}`,
short: `±${n}`,
level: n,
}));
const semanticCands = [...semBounds].sort().map(b => ({
id: `semantic:${b}`,
family: 'semantic',
label: `semantic @ ${b}`,
short: b,
boundary: b,
}));
function metricsFor(exp, family) {
if (!exp) return null;
return (exp.aggregate_metrics || {})[family] || null;
}
function summarize(cands) {
return cands.map(c => {
const perDoc = {};
const composites = [];
const metricBags = {
avg_context_relevance: [],
avg_answer_similarity: [],
avg_faithfulness: [],
hallucination_rate: [],
};
let filled = 0;
DECISION_DOC_FILENAMES.forEach(fn => {
const exp = cellMap[`${fn}||${c.id}`];
const m = metricsFor(exp, c.family);
const score = decisionComposite(m);
perDoc[fn] = { exp, metrics: m, composite: score };
if (score != null) {
filled += 1;
composites.push(score);
Object.keys(metricBags).forEach(k => {
if (m && m[k] != null) metricBags[k].push(m[k]);
});
}
});
return {
...c,
perDoc,
filled,
totalDocs: DECISION_DOC_FILENAMES.length,
meanComposite: meanOf(composites),
meanMetrics: {
avg_context_relevance: meanOf(metricBags.avg_context_relevance),
avg_answer_similarity: meanOf(metricBags.avg_answer_similarity),
avg_faithfulness: meanOf(metricBags.avg_faithfulness),
hallucination_rate: meanOf(metricBags.hallucination_rate),
},
wins: 0,
};
});
}
const fixedSummaries = summarize(fixedCands);
const semanticSummaries = summarize(semanticCands);
function assignWins(summaries) {
DECISION_DOC_FILENAMES.forEach(fn => {
let best = null;
let bestScore = -Infinity;
summaries.forEach(s => {
const sc = s.perDoc[fn]?.composite;
if (sc == null) return;
if (sc > bestScore) {
bestScore = sc;
best = s;
}
});
if (best) best.wins += 1;
});
}
assignWins(fixedSummaries);
assignWins(semanticSummaries);
function autoPick(summaries) {
if (!summaries.length) return null;
return [...summaries].sort((a, b) => {
const ma = a.meanComposite == null ? -1 : a.meanComposite;
const mb = b.meanComposite == null ? -1 : b.meanComposite;
if (mb !== ma) return mb - ma;
if (b.wins !== a.wins) return b.wins - a.wins;
return a.id.localeCompare(b.id);
})[0];
}
return {
cellMap,
fixedSummaries,
semanticSummaries,
autoFixed: autoPick(fixedSummaries),
autoSemantic: autoPick(semanticSummaries),
allCandidates: [...fixedSummaries, ...semanticSummaries],
};
}
function DecisionTab({ addToast }) {
const [experiments, setExperiments] = useState([]);
const [loading, setLoading] = useState(true);
const [corpusId, setCorpusId] = useState('text-embedding-3-large');
const [embModels, setEmbModels] = useState([]);
const [excludedIds, setExcludedIds] = useState([]);
const [overrideFixed, setOverrideFixed] = useState(null);
const [overrideSemantic, setOverrideSemantic] = useState(null);
const refresh = useCallback(() => {
setLoading(true);
Promise.all([
api('/experiments?limit=500'),
api('/admin/embedding-models'),
]).then(([ex, emb]) => {
setExperiments(ex?.items || []);
setEmbModels(emb?.models || []);
if (emb?.corpus_id) {
setCorpusId(prev => prev || emb.corpus_id);
}
}).catch(() => addToast('Failed to load Decision Board data', 'error'))
.finally(() => setLoading(false));
}, [addToast]);
useEffect(() => { refresh(); }, [refresh]);
const board = React.useMemo(
() => buildDecisionBoard(experiments, corpusId, excludedIds),
[experiments, corpusId, excludedIds]
);
const winnerFixed = (overrideFixed && board.fixedSummaries.find(c => c.id === overrideFixed))
|| board.autoFixed;
const winnerSemantic = (overrideSemantic && board.semanticSummaries.find(c => c.id === overrideSemantic))
|| board.autoSemantic;
const excludeId = (id) => {
if (!id) return;
setExcludedIds(prev => prev.includes(id) ? prev : [...prev, id]);
};
const unexcludeId = (id) => setExcludedIds(prev => prev.filter(x => x !== id));
const fsFilled = board.fixedSummaries.reduce((a, c) => a + c.filled, 0);
const fsTotal = board.fixedSummaries.length * DECISION_DOC_FILENAMES.length;
const semFilled = board.semanticSummaries.reduce((a, c) => a + c.filled, 0);
const semTotal = board.semanticSummaries.length * DECISION_DOC_FILENAMES.length;
// Stage-2 head-to-head
let duel = null;
if (winnerFixed && winnerSemantic) {
let fsWins = 0, semWins = 0, ties = 0;
const perDoc = DECISION_DOC_FILENAMES.map(fn => {
const a = winnerFixed.perDoc[fn]?.composite;
const b = winnerSemantic.perDoc[fn]?.composite;
let winner = '—';
if (a != null && b != null) {
if (a > b) { fsWins += 1; winner = 'fixed'; }
else if (b > a) { semWins += 1; winner = 'semantic'; }
else { ties += 1; winner = 'tie'; }
}
return { fn, a, b, winner };
});
const recommend = (winnerFixed.meanComposite || 0) >= (winnerSemantic.meanComposite || 0)
? winnerFixed : winnerSemantic;
// Prefer win-count if mean close? stick to mean primary with wins as display
duel = { fsWins, semWins, ties, perDoc, recommend };
}
const renderCandCard = (c, selected, onSelect, familyAutoId) =>
React.createElement('div', {
key: c.id,
className: `decision-cand${selected ? ' selected' : ''}`,
onClick: () => onSelect(selected ? null : (c.id === familyAutoId ? null : c.id)),
title: 'Click to override family winner; click selected again to return to auto',
},
React.createElement('div', { className: 'cand-title', style: { display: 'flex', justifyContent: 'space-between', gap: 8 } },
React.createElement('span', null,
React.createElement('input', {
type: 'radio',
checked: !!selected,
readOnly: true,
style: { marginRight: 8 },
}),
c.label,
c.id === familyAutoId
? React.createElement('span', { className: 'badge badge-accent', style: { marginLeft: 8 } }, 'auto')
: null
),
React.createElement('span', { style: { color: 'var(--accent)', fontVariantNumeric: 'tabular-nums' } },
decisionFmt(c.meanComposite))
),
React.createElement('div', { className: 'decision-metrics' },
React.createElement('span', null, 'Wins ', React.createElement('strong', null, `${c.wins}/${c.totalDocs}`)),
React.createElement('span', null, 'Coverage ', React.createElement('strong', null, `${c.filled}/${c.totalDocs}`)),
React.createElement('span', null, 'Context ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_context_relevance))),
React.createElement('span', null, 'Similarity ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_answer_similarity))),
React.createElement('span', null, 'Faithfulness ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_faithfulness))),
React.createElement('span', null, 'Halluc. ', React.createElement('strong', null, decisionPct(c.meanMetrics.hallucination_rate)))
)
);
const duelPanel = (c, side) => {
if (!c) {
return React.createElement('div', { className: 'card', style: { margin: 0 } },
React.createElement('div', { className: 'text-muted' }, `No ${side} winner yet`));
}
const isRec = duel && duel.recommend && duel.recommend.id === c.id;
return React.createElement('div', {
className: 'card',
style: {
margin: 0,
borderColor: isRec ? 'var(--accent)' : undefined,
boxShadow: isRec ? '0 0 0 1px rgba(234,179,8,0.35)' : undefined,
},
},
React.createElement('div', { className: 'flex-between mb-2' },
React.createElement('div', { className: 'card-title mb-0' }, c.label),
isRec ? React.createElement('span', { className: 'badge badge-success' }, 'Recommended') : null
),
React.createElement('div', { style: { fontSize: 28, fontWeight: 700, color: 'var(--accent)', marginBottom: 8 } },
decisionFmt(c.meanComposite)),
React.createElement('div', { className: 'decision-metrics', style: { marginBottom: 8 } },
React.createElement('span', null, 'Doc wins vs other ', React.createElement('strong', null,
side === 'fixed' ? (duel ? duel.fsWins : '—') : (duel ? duel.semWins : '—'))),
React.createElement('span', null, 'Family wins ', React.createElement('strong', null, `${c.wins}/${c.totalDocs}`)),
React.createElement('span', null, 'Coverage ', React.createElement('strong', null, `${c.filled}/${c.totalDocs}`))
),
React.createElement('table', { className: 'cmp-table' },
React.createElement('tbody', null,
[['Context Relevance', c.meanMetrics.avg_context_relevance, false],
['Answer Similarity', c.meanMetrics.avg_answer_similarity, false],
['Faithfulness', c.meanMetrics.avg_faithfulness, false],
['Hallucination Rate', c.meanMetrics.hallucination_rate, true]].map(([label, val, isPct]) =>
React.createElement('tr', { key: label },
React.createElement('td', null, label),
React.createElement('td', { style: { fontWeight: 600 } }, isPct ? decisionPct(val) : decisionFmt(val))
)
)
)
)
);
};
return React.createElement('div', null,
React.createElement('div', { className: 'flex-between mb-2' },
React.createElement('h1', { style: { color: 'var(--text-heading)', margin: 0, fontSize: '22px' } }, 'Decision Board'),
React.createElement('button', { className: 'btn btn-secondary btn-sm', onClick: refresh, disabled: loading },
loading ? React.createElement('span', { className: 'spinner' }) : '↻ Refresh')
),
React.createElement('p', { className: 'text-sm text-muted', style: { marginBottom: 16, maxWidth: 720 } },
'Two-stage final selection: pick the best fixed_size Neighbor level and best semantic Boundary, then compare those winners across the 10-doc evaluation set. Mean composite ranks stage 1; win-counts are shown; click a Candidate to override.'
),
// Header controls
React.createElement('div', { className: 'card mb-4' },
React.createElement('div', { className: 'row mb-0', style: { alignItems: 'flex-end' } },
React.createElement(EmbeddingModelSelect, {
label: 'Corpus filter',
value: corpusId,
onChange: (v) => { setCorpusId(v); setOverrideFixed(null); setOverrideSemantic(null); },
models: embModels,
role: 'corpus',
title: 'Only single-strategy Experiments under this Corpus Embedding Model',
style: { maxWidth: 360 },
}),
React.createElement('div', { className: 'col' },
React.createElement('div', { className: 'text-sm text-muted' }, 'Coverage'),
React.createElement('div', { style: { fontWeight: 600, color: 'var(--text-heading)', marginTop: 6 } },
`fixed_size ${fsFilled}/${fsTotal || 40} · semantic ${semFilled}/${semTotal || '—'}`)
)
),
excludedIds.length > 0 && React.createElement('div', { style: { marginTop: 12 } },
React.createElement('div', { className: 'text-sm text-muted mb-1' }, 'Excluded Experiments (next-newest fills the cell)'),
React.createElement('div', { style: { display: 'flex', flexWrap: 'wrap', gap: 6 } },
excludedIds.map(id =>
React.createElement('button', {
key: id,
className: 'btn btn-secondary btn-sm',
onClick: () => unexcludeId(id),
title: 'Click to restore',
}, `✕ ${id.substring(0, 8)}…`)
)
)
)
),
loading && React.createElement('div', { style: { textAlign: 'center', padding: 24 } },
React.createElement('span', { className: 'spinner' })),
!loading && React.createElement(React.Fragment, null,
// Stage 1
React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Stage 1 — Best Candidate per family'),
React.createElement('div', { className: 'decision-grid mb-4' },
React.createElement('div', { className: 'card', style: { margin: 0 } },
React.createElement('div', { className: 'card-title' }, 'fixed_size (Neighbor Expansion)'),
React.createElement('div', { className: 'text-sm text-muted mb-2' },
'Auto winner: ', board.autoFixed ? board.autoFixed.label : '—',
overrideFixed ? ' · override active' : ''),
board.fixedSummaries.map(c => renderCandCard(
c,
winnerFixed && winnerFixed.id === c.id,
setOverrideFixed,
board.autoFixed?.id
))
),
React.createElement('div', { className: 'card', style: { margin: 0 } },
React.createElement('div', { className: 'card-title' }, 'semantic (Boundary Embedding Model)'),
React.createElement('div', { className: 'text-sm text-muted mb-2' },
'Auto winner: ', board.autoSemantic ? board.autoSemantic.label : '—',
overrideSemantic ? ' · override active' : ''),
board.semanticSummaries.length === 0
? React.createElement('div', { className: 'empty-state' }, 'No semantic single-strategy Experiments for this Corpus')
: board.semanticSummaries.map(c => renderCandCard(
c,
winnerSemantic && winnerSemantic.id === c.id,
setOverrideSemantic,
board.autoSemantic?.id
))
)
),
// Stage 2
React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Stage 2 — Family showdown'),
React.createElement('div', { className: 'decision-duel mb-4' },
duelPanel(winnerFixed, 'fixed'),
React.createElement('div', { className: 'decision-vs' }, 'vs'),
duelPanel(winnerSemantic, 'semantic')
),
duel && React.createElement('div', { className: 'card mb-4' },
React.createElement('div', { className: 'card-title' }, 'Head-to-head by document'),
React.createElement('div', { className: 'text-sm text-muted mb-2' },
`fixed_size wins ${duel.fsWins} · semantic wins ${duel.semWins} · ties ${duel.ties}`),
React.createElement('table', { className: 'cmp-table' },
React.createElement('thead', null,
React.createElement('tr', null,
React.createElement('th', null, 'Document'),
React.createElement('th', null, winnerFixed?.short || 'fixed'),
React.createElement('th', null, winnerSemantic?.short || 'semantic'),
React.createElement('th', null, 'Winner')
)
),
React.createElement('tbody', null,
duel.perDoc.map(row =>
React.createElement('tr', { key: row.fn },
React.createElement('td', null, row.fn),
React.createElement('td', { style: { textAlign: 'center', fontVariantNumeric: 'tabular-nums' } }, decisionFmt(row.a)),
React.createElement('td', { style: { textAlign: 'center', fontVariantNumeric: 'tabular-nums' } }, decisionFmt(row.b)),
React.createElement('td', null,
row.winner === 'fixed' ? React.createElement('span', { className: 'badge badge-accent' }, 'fixed_size')
: row.winner === 'semantic' ? React.createElement('span', { className: 'badge badge-success' }, 'semantic')
: row.winner === 'tie' ? React.createElement('span', { className: 'badge' }, 'tie')
: React.createElement('span', { className: 'text-muted' }, '—')
)
)
)
)
)
),
// Matrix
React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Per-document matrix'),
React.createElement('div', { className: 'card', style: { paddingBottom: 14 } },
React.createElement(DecisionHeatLegend),
React.createElement('div', { className: 'decision-matrix-wrap' },
React.createElement('table', { className: 'decision-matrix' },
React.createElement('thead', null,
React.createElement('tr', null,
React.createElement('th', { className: 'sticky-col' }, 'Document'),
board.allCandidates.map(c =>
React.createElement('th', {
key: c.id,
className: (winnerFixed && c.id === winnerFixed.id) || (winnerSemantic && c.id === winnerSemantic.id)
? 'col-duel' : undefined,
title: c.label,
}, c.family === 'fixed_size' ? c.short : (c.short || '').substring(0, 14))
)
)
),
React.createElement('tbody', null,
DECISION_DOC_FILENAMES.map(fn => {
const rowBest = decisionFamilyBestIds(board.allCandidates, c => c.perDoc[fn]?.composite);
return React.createElement('tr', { key: fn },
React.createElement('td', { className: 'sticky-col' }, fn),
board.allCandidates.map(c => {
const cell = c.perDoc[fn];
const score = cell?.composite;
const exp = cell?.exp;
const mark = c.id === rowBest.fixed ? 'fs' : (c.id === rowBest.semantic ? 'sem' : null);
return React.createElement('td', {
key: c.id,
className: 'decision-heat',
style: { ...decisionHeatStyle(score), cursor: exp ? 'pointer' : 'default' },
title: exp
? `${c.label} · ${fn}\ncomposite=${decisionFmt(score)}${mark ? `\nbest ${mark === 'fs' ? 'fixed_size' : 'semantic'} in row` : ''}\nid=${exp.id}\nClick: report · Shift+click: exclude`
: `${c.label} · ${fn}: missing`,
onClick: (e) => {
if (!exp) return;
if (e.shiftKey) {
excludeId(exp.id);
addToast(`Excluded ${exp.id.substring(0, 8)}…`, 'success');
return;
}
window.open(`/benchmarks/${exp.id}/report`, '_blank');
},
},
mark
? React.createElement('span', {
className: `best-mark ${mark}`,
'aria-label': mark === 'fs' ? 'Best fixed_size in row' : 'Best semantic in row',
}, mark === 'fs' ? 'F' : 'S')
: null,
decisionFmt(score)
);
})
);
}),
(() => {
const meanBest = decisionFamilyBestIds(board.allCandidates, c => c.meanComposite);
return React.createElement('tr', { key: '_mean', className: 'mean-row' },
React.createElement('td', { className: 'sticky-col' }, 'Mean'),
board.allCandidates.map(c => {
const mark = c.id === meanBest.fixed ? 'fs' : (c.id === meanBest.semantic ? 'sem' : null);
return React.createElement('td', {
key: c.id,
className: 'decision-heat',
style: decisionHeatStyle(c.meanComposite),
},
mark
? React.createElement('span', {
className: `best-mark ${mark}`,
'aria-label': mark === 'fs' ? 'Best fixed_size mean' : 'Best semantic mean',
}, mark === 'fs' ? 'F' : 'S')
: null,
decisionFmt(c.meanComposite)
);
})
);
})(),
React.createElement('tr', { key: '_wins', className: 'wins-row' },
React.createElement('td', { className: 'sticky-col' }, 'Wins'),
board.allCandidates.map(c =>
React.createElement('td', { key: c.id }, `${c.wins}/${c.totalDocs}`)
)
)
)
)
),
React.createElement('div', { className: 'text-sm text-muted', style: { marginTop: 12 } },
'Click a cell to open its Experiment report. Shift+click to exclude. ',
React.createElement('span', { className: 'best-mark fs' }, 'F'),
' / ',
React.createElement('span', { className: 'best-mark sem' }, 'S'),
' mark the best score in each family per row.'
)
)
)
);
}
// -- Tab: Admin --------------------------------------------- // -- Tab: Admin ---------------------------------------------
function AdminTab({ addToast, documents, strategies, setActiveTab, setQuestionsFile }) { function AdminTab({ addToast, documents, strategies, setActiveTab, setQuestionsFile }) {
// Collapse state for each section // Collapse state for each section
@@ -2804,6 +3569,7 @@ function App() {
{ id: 'pdf', label: 'PDF' }, { id: 'pdf', label: 'PDF' },
{ id: 'query', label: 'Query' }, { id: 'query', label: 'Query' },
{ id: 'benchmarks', label: 'Benchmarks' }, { id: 'benchmarks', label: 'Benchmarks' },
{ id: 'decision', label: 'Decision' },
{ id: 'admin', label: 'Admin' }, { id: 'admin', label: 'Admin' },
]; ];
@@ -2818,6 +3584,7 @@ function App() {
documents, strategies, addToast, formatFilter: 'word' }); documents, strategies, addToast, formatFilter: 'word' });
case 'benchmarks': return React.createElement(BenchmarksTab, { case 'benchmarks': return React.createElement(BenchmarksTab, {
documents, strategies, addToast, questionsFile, formatFilter: 'word' }); documents, strategies, addToast, questionsFile, formatFilter: 'word' });
case 'decision': return React.createElement(DecisionTab, { addToast });
case 'admin': return React.createElement(AdminTab, { case 'admin': return React.createElement(AdminTab, {
addToast, documents, strategies, setActiveTab, setQuestionsFile }); addToast, documents, strategies, setActiveTab, setQuestionsFile });
default: return null; default: return null;

View File

@@ -408,9 +408,11 @@ def get_experiment(experiment_id: str) -> dict[str, Any] | None:
def list_experiments( def list_experiments(
*, document_id: str | None = None, offset: int = 0, limit: int = 50 *, document_id: str | None = None, offset: int = 0, limit: int = 200
) -> dict[str, Any]: ) -> dict[str, Any]:
"""List experiments, optionally filtered by document.""" """List experiments, optionally filtered by document."""
limit = max(1, min(int(limit), 500))
offset = max(0, int(offset))
conn = _connect() conn = _connect()
try: try:
if document_id: if document_id: