feat(dashboard): add decision board for strategy selection
Why: - Compare is the wrong surface for two-stage family selection over the 10-doc set. Changes: - Add the Decision Tab; raise GET /experiments default/max so the board can load the grid client-side. Impact: - Operators pick fixed_size ±N vs semantic@Boundary from existing Experiments. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -434,6 +434,128 @@
|
||||
.cmp-exp-2 { background: rgba(99,102,241,0.15); color: #818cf8; }
|
||||
.cmp-exp-3 { background: rgba(244,63,94,0.15); color: #fb7185; }
|
||||
.cmp-divider { height: 1px; background: var(--border); margin: 20px 0; }
|
||||
/* ── Decision Board ─────────────────────────────────────── */
|
||||
.decision-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
|
||||
@media (max-width: 960px) { .decision-grid { grid-template-columns: 1fr; } }
|
||||
.decision-cand {
|
||||
border: 1px solid var(--border); border-radius: var(--radius);
|
||||
padding: 12px 14px; margin-bottom: 8px; cursor: pointer;
|
||||
background: var(--bg-base); transition: border-color 0.15s, background 0.15s;
|
||||
}
|
||||
.decision-cand:hover { border-color: #3a3a42; }
|
||||
.decision-cand.selected {
|
||||
border-color: var(--accent); background: rgba(234,179,8,0.08);
|
||||
}
|
||||
.decision-cand .cand-title { font-weight: 600; color: var(--text-heading); margin-bottom: 6px; }
|
||||
.decision-metrics { display: flex; flex-wrap: wrap; gap: 8px 14px; font-size: 12px; color: var(--text-muted); }
|
||||
.decision-metrics strong { color: var(--text-body); font-variant-numeric: tabular-nums; }
|
||||
.decision-legend {
|
||||
display: flex; flex-wrap: wrap; gap: 10px 16px; align-items: center;
|
||||
margin-bottom: 14px; padding: 12px 14px;
|
||||
background: rgba(15, 20, 25, 0.55);
|
||||
border: 1px solid rgba(255,255,255,0.06);
|
||||
border-radius: 10px;
|
||||
font-size: 12px; color: var(--text-muted);
|
||||
}
|
||||
.decision-legend-item { display: inline-flex; align-items: center; gap: 7px; }
|
||||
.decision-legend-swatch {
|
||||
width: 22px; height: 14px; border-radius: 4px;
|
||||
border: 1px solid rgba(255,255,255,0.08);
|
||||
box-shadow: inset 3px 0 0 var(--swatch-accent, transparent);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.decision-matrix-wrap {
|
||||
overflow-x: auto;
|
||||
border: 1px solid rgba(255,255,255,0.06);
|
||||
border-radius: 12px;
|
||||
background: rgba(15, 20, 25, 0.35);
|
||||
}
|
||||
.decision-matrix {
|
||||
width: 100%; border-collapse: separate; border-spacing: 0;
|
||||
font-size: 13px; margin: 0;
|
||||
}
|
||||
.decision-matrix th,
|
||||
.decision-matrix td {
|
||||
padding: 10px 12px;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.05);
|
||||
border-right: 1px solid rgba(255,255,255,0.04);
|
||||
text-align: center;
|
||||
vertical-align: middle;
|
||||
}
|
||||
.decision-matrix th:last-child,
|
||||
.decision-matrix td:last-child { border-right: none; }
|
||||
.decision-matrix tbody tr:last-child th,
|
||||
.decision-matrix tbody tr:last-child td { border-bottom: none; }
|
||||
.decision-matrix thead th {
|
||||
position: sticky; top: 0; z-index: 2;
|
||||
background: rgba(21, 32, 51, 0.96);
|
||||
backdrop-filter: blur(8px);
|
||||
color: #94A3B8;
|
||||
font-size: 11px; font-weight: 600;
|
||||
letter-spacing: 0.03em;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.decision-matrix .sticky-col {
|
||||
position: sticky; left: 0; z-index: 1;
|
||||
text-align: left; font-weight: 500; white-space: nowrap;
|
||||
background: #1A2332; color: var(--text-heading);
|
||||
min-width: 160px; max-width: 220px;
|
||||
}
|
||||
.decision-matrix thead .sticky-col { z-index: 3; background: rgba(21, 32, 51, 0.96); }
|
||||
.decision-matrix tr.mean-row td,
|
||||
.decision-matrix tr.mean-row .sticky-col {
|
||||
border-top: 1px solid rgba(255,255,255,0.1);
|
||||
background: rgba(15, 20, 25, 0.65);
|
||||
font-weight: 700;
|
||||
}
|
||||
.decision-matrix tr.wins-row td,
|
||||
.decision-matrix tr.wins-row .sticky-col {
|
||||
background: rgba(15, 20, 25, 0.45);
|
||||
color: var(--text-muted);
|
||||
font-weight: 600;
|
||||
}
|
||||
.decision-matrix th.col-duel {
|
||||
color: #FBBF24;
|
||||
}
|
||||
.decision-heat {
|
||||
font-variant-numeric: tabular-nums;
|
||||
font-size: 12.5px; font-weight: 600;
|
||||
min-width: 76px;
|
||||
letter-spacing: 0.01em;
|
||||
transition: filter 0.15s ease;
|
||||
}
|
||||
.decision-heat:hover { filter: brightness(1.12); }
|
||||
.decision-heat .best-mark,
|
||||
.decision-legend .best-mark,
|
||||
.best-mark {
|
||||
display: inline-block;
|
||||
font-size: 9px; font-weight: 700; line-height: 1;
|
||||
margin-right: 5px; padding: 2px 5px;
|
||||
border-radius: 999px; vertical-align: middle;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.decision-heat .best-mark.fs,
|
||||
.decision-legend .best-mark.fs,
|
||||
.best-mark.fs {
|
||||
background: rgba(56, 189, 248, 0.15);
|
||||
color: #7DD3FC;
|
||||
border: 1px solid rgba(56, 189, 248, 0.28);
|
||||
}
|
||||
.decision-heat .best-mark.sem,
|
||||
.decision-legend .best-mark.sem,
|
||||
.best-mark.sem {
|
||||
background: rgba(167, 139, 250, 0.15);
|
||||
color: #C4B5FD;
|
||||
border: 1px solid rgba(167, 139, 250, 0.28);
|
||||
}
|
||||
.decision-duel {
|
||||
display: grid; grid-template-columns: 1fr auto 1fr; gap: 16px; align-items: stretch;
|
||||
}
|
||||
@media (max-width: 800px) { .decision-duel { grid-template-columns: 1fr; } }
|
||||
.decision-vs {
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
font-weight: 700; color: var(--text-muted); font-size: 18px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -554,7 +676,7 @@ function HomeTab() {
|
||||
useEffect(() => {
|
||||
Promise.all([
|
||||
api('/documents').catch(() => ({ total: 0 })),
|
||||
api('/experiments').catch(() => []),
|
||||
api('/experiments?limit=500').catch(() => []),
|
||||
]).then(([docs, exps]) => {
|
||||
setStats({ docs: docs.total || 0, experiments: (exps?.items || []).length });
|
||||
setLoading(false);
|
||||
@@ -589,6 +711,7 @@ function HomeTab() {
|
||||
React.createElement('li', null, 'Process it with chunking strategies'),
|
||||
React.createElement('li', null, 'Ask a question in ', React.createElement('b', null, 'Query'), ' (or inside PDF Workspace)'),
|
||||
React.createElement('li', null, 'Run a full benchmark in ', React.createElement('b', null, 'Benchmarks'), ' / PDF Workspace'),
|
||||
React.createElement('li', null, 'Pick a final Strategy on the ', React.createElement('b', null, 'Decision'), ' board'),
|
||||
React.createElement('li', null, 'Check system health in the ', React.createElement('b', null, 'Admin'), ' tab')
|
||||
)
|
||||
)
|
||||
@@ -1660,7 +1783,7 @@ function BenchmarksTab({ documents, strategies, addToast, questionsFile, formatF
|
||||
const [inspectLoadingId, setInspectLoadingId] = useState(null);
|
||||
|
||||
const fetchExperiments = useCallback(() => {
|
||||
api('/experiments').then(d => setExperiments(d?.items || (Array.isArray(d) ? d : []))).catch(() => {});
|
||||
api('/experiments?limit=500').then(d => setExperiments(d?.items || (Array.isArray(d) ? d : []))).catch(() => {});
|
||||
}, []);
|
||||
|
||||
useEffect(() => { fetchExperiments(); }, [fetchExperiments]);
|
||||
@@ -2267,6 +2390,648 @@ function PdfWorkspaceTab({ documents, setDocuments, strategies, addToast, questi
|
||||
);
|
||||
}
|
||||
|
||||
// -- Tab: Decision Board (ADR-0026) --------------------------
|
||||
const DECISION_DOC_FILENAMES = [
|
||||
'bazresi.docx',
|
||||
'customer1.docx',
|
||||
'fire.docx',
|
||||
'general-havades-individuals.doc',
|
||||
'havades.docx',
|
||||
'life-time-individual.docx',
|
||||
'moavenin.docx',
|
||||
'Refah.docx',
|
||||
'website.docx',
|
||||
'lifetime-compensation.docx',
|
||||
];
|
||||
|
||||
function decisionComposite(m) {
|
||||
if (!m) return null;
|
||||
return ((m.avg_context_relevance || 0) * 0.3
|
||||
+ (m.avg_answer_similarity || 0) * 0.4
|
||||
+ (m.avg_faithfulness || 0) * 0.3) * (1 - (m.hallucination_rate || 0));
|
||||
}
|
||||
|
||||
function decisionFmt(v, digits) {
|
||||
if (v == null || Number.isNaN(v)) return '—';
|
||||
return Number(v).toFixed(digits == null ? 2 : digits);
|
||||
}
|
||||
|
||||
function decisionPct(v) {
|
||||
if (v == null || Number.isNaN(v)) return '—';
|
||||
return `${(Number(v) * 100).toFixed(0)}%`;
|
||||
}
|
||||
|
||||
function decisionHeatBin(score) {
|
||||
if (score == null || Number.isNaN(score)) {
|
||||
return {
|
||||
key: 'missing', label: '—',
|
||||
bg: 'transparent', fg: 'var(--text-muted)', accent: 'transparent',
|
||||
};
|
||||
}
|
||||
const s = Number(score);
|
||||
// Modern dark-dashboard scale: translucent wash + luminous text + left accent rail
|
||||
// (no solid primary blocks — avoids Windows-98 / crayon look)
|
||||
if (s < 8.0) {
|
||||
return {
|
||||
key: 'lt80', label: '<8.0',
|
||||
bg: 'rgba(244, 63, 94, 0.14)', fg: '#FB7185', accent: '#F43F5E',
|
||||
};
|
||||
}
|
||||
if (s < 8.5) {
|
||||
return {
|
||||
key: '80_85', label: '8.0–8.5',
|
||||
bg: 'rgba(251, 146, 60, 0.13)', fg: '#FB923C', accent: '#F97316',
|
||||
};
|
||||
}
|
||||
if (s < 9.0) {
|
||||
return {
|
||||
key: '85_90', label: '8.5–9.0',
|
||||
bg: 'rgba(250, 204, 21, 0.14)', fg: '#FDE047', accent: '#EAB308',
|
||||
};
|
||||
}
|
||||
if (s < 9.5) {
|
||||
return {
|
||||
key: '90_95', label: '9.0–9.5',
|
||||
bg: 'rgba(45, 212, 191, 0.13)', fg: '#2DD4BF', accent: '#14B8A6',
|
||||
};
|
||||
}
|
||||
return {
|
||||
key: 'ge95', label: '≥9.5',
|
||||
bg: 'rgba(52, 211, 153, 0.16)', fg: '#34D399', accent: '#10B981',
|
||||
};
|
||||
}
|
||||
|
||||
function decisionHeatStyle(score) {
|
||||
const bin = decisionHeatBin(score);
|
||||
const style = { background: bin.bg, color: bin.fg };
|
||||
if (bin.accent && bin.accent !== 'transparent') {
|
||||
style.boxShadow = `inset 3px 0 0 ${bin.accent}`;
|
||||
}
|
||||
return style;
|
||||
}
|
||||
|
||||
/** Per-row (or mean-row) ids of best fixed_size and best semantic Candidate. */
|
||||
function decisionFamilyBestIds(candidates, scoreOf) {
|
||||
let bestFs = null, bestFsScore = -Infinity;
|
||||
let bestSem = null, bestSemScore = -Infinity;
|
||||
(candidates || []).forEach(c => {
|
||||
const s = scoreOf(c);
|
||||
if (s == null || Number.isNaN(s)) return;
|
||||
if (c.family === 'fixed_size' && s > bestFsScore) {
|
||||
bestFsScore = s;
|
||||
bestFs = c.id;
|
||||
}
|
||||
if (c.family === 'semantic' && s > bestSemScore) {
|
||||
bestSemScore = s;
|
||||
bestSem = c.id;
|
||||
}
|
||||
});
|
||||
return { fixed: bestFs, semantic: bestSem };
|
||||
}
|
||||
|
||||
const DECISION_HEAT_LEGEND = [
|
||||
decisionHeatBin(null),
|
||||
decisionHeatBin(7.9),
|
||||
decisionHeatBin(8.2),
|
||||
decisionHeatBin(8.7),
|
||||
decisionHeatBin(9.2),
|
||||
decisionHeatBin(9.6),
|
||||
];
|
||||
|
||||
function DecisionHeatLegend() {
|
||||
return React.createElement('div', { className: 'decision-legend', role: 'list', 'aria-label': 'Composite score color guide' },
|
||||
React.createElement('span', { style: { fontWeight: 600, color: 'var(--text-body)' } }, 'Score guide'),
|
||||
DECISION_HEAT_LEGEND.map(bin =>
|
||||
React.createElement('span', { key: bin.key, className: 'decision-legend-item', role: 'listitem' },
|
||||
React.createElement('span', {
|
||||
className: 'decision-legend-swatch',
|
||||
style: {
|
||||
background: bin.bg === 'transparent' ? 'rgba(255,255,255,0.04)' : bin.bg,
|
||||
boxShadow: bin.accent && bin.accent !== 'transparent' ? `inset 3px 0 0 ${bin.accent}` : undefined,
|
||||
},
|
||||
'aria-hidden': true,
|
||||
}),
|
||||
bin.label
|
||||
)
|
||||
),
|
||||
React.createElement('span', {
|
||||
style: { width: 1, height: 14, background: 'rgba(255,255,255,0.1)', margin: '0 4px' },
|
||||
'aria-hidden': true,
|
||||
}),
|
||||
React.createElement('span', { className: 'decision-legend-item' },
|
||||
React.createElement('span', { className: 'best-mark fs' }, 'F'),
|
||||
'best fixed_size'
|
||||
),
|
||||
React.createElement('span', { className: 'decision-legend-item' },
|
||||
React.createElement('span', { className: 'best-mark sem' }, 'S'),
|
||||
'best semantic'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function meanOf(nums) {
|
||||
const vals = nums.filter(v => v != null && !Number.isNaN(v));
|
||||
if (!vals.length) return null;
|
||||
return vals.reduce((a, b) => a + b, 0) / vals.length;
|
||||
}
|
||||
|
||||
function buildDecisionBoard(experiments, corpusId, excludedIds) {
|
||||
const excluded = new Set(excludedIds || []);
|
||||
const pool = (experiments || []).filter(e => {
|
||||
if (excluded.has(e.id)) return false;
|
||||
const fn = e.document_filename || '';
|
||||
if (!DECISION_DOC_FILENAMES.includes(fn)) return false;
|
||||
if (corpusIdOf(e) !== corpusId) return false;
|
||||
const strats = e.strategies_used || [];
|
||||
return strats.length === 1;
|
||||
});
|
||||
|
||||
// Newest first assumed from API; keep first hit per cell
|
||||
const cellMap = {}; // key: `${fn}||${candId}` -> exp
|
||||
const fsLevels = new Set([0, 1, 2, 3]);
|
||||
const semBounds = new Set();
|
||||
|
||||
pool.forEach(e => {
|
||||
const strat = (e.strategies_used || [])[0];
|
||||
const fn = e.document_filename;
|
||||
if (strat === 'fixed_size') {
|
||||
const { prev, next } = neighborCounts(e);
|
||||
if (prev !== next || !fsLevels.has(prev)) return;
|
||||
const candId = `fixed_size:±${prev}`;
|
||||
const key = `${fn}||${candId}`;
|
||||
if (!cellMap[key]) cellMap[key] = e;
|
||||
} else if (strat === 'semantic') {
|
||||
const b = boundaryIdOf(e);
|
||||
if (!b) return;
|
||||
semBounds.add(b);
|
||||
const candId = `semantic:${b}`;
|
||||
const key = `${fn}||${candId}`;
|
||||
if (!cellMap[key]) cellMap[key] = e;
|
||||
}
|
||||
});
|
||||
|
||||
const fixedCands = [0, 1, 2, 3].map(n => ({
|
||||
id: `fixed_size:±${n}`,
|
||||
family: 'fixed_size',
|
||||
label: `fixed_size ±${n}/${n}`,
|
||||
short: `±${n}`,
|
||||
level: n,
|
||||
}));
|
||||
const semanticCands = [...semBounds].sort().map(b => ({
|
||||
id: `semantic:${b}`,
|
||||
family: 'semantic',
|
||||
label: `semantic @ ${b}`,
|
||||
short: b,
|
||||
boundary: b,
|
||||
}));
|
||||
|
||||
function metricsFor(exp, family) {
|
||||
if (!exp) return null;
|
||||
return (exp.aggregate_metrics || {})[family] || null;
|
||||
}
|
||||
|
||||
function summarize(cands) {
|
||||
return cands.map(c => {
|
||||
const perDoc = {};
|
||||
const composites = [];
|
||||
const metricBags = {
|
||||
avg_context_relevance: [],
|
||||
avg_answer_similarity: [],
|
||||
avg_faithfulness: [],
|
||||
hallucination_rate: [],
|
||||
};
|
||||
let filled = 0;
|
||||
DECISION_DOC_FILENAMES.forEach(fn => {
|
||||
const exp = cellMap[`${fn}||${c.id}`];
|
||||
const m = metricsFor(exp, c.family);
|
||||
const score = decisionComposite(m);
|
||||
perDoc[fn] = { exp, metrics: m, composite: score };
|
||||
if (score != null) {
|
||||
filled += 1;
|
||||
composites.push(score);
|
||||
Object.keys(metricBags).forEach(k => {
|
||||
if (m && m[k] != null) metricBags[k].push(m[k]);
|
||||
});
|
||||
}
|
||||
});
|
||||
return {
|
||||
...c,
|
||||
perDoc,
|
||||
filled,
|
||||
totalDocs: DECISION_DOC_FILENAMES.length,
|
||||
meanComposite: meanOf(composites),
|
||||
meanMetrics: {
|
||||
avg_context_relevance: meanOf(metricBags.avg_context_relevance),
|
||||
avg_answer_similarity: meanOf(metricBags.avg_answer_similarity),
|
||||
avg_faithfulness: meanOf(metricBags.avg_faithfulness),
|
||||
hallucination_rate: meanOf(metricBags.hallucination_rate),
|
||||
},
|
||||
wins: 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
const fixedSummaries = summarize(fixedCands);
|
||||
const semanticSummaries = summarize(semanticCands);
|
||||
|
||||
function assignWins(summaries) {
|
||||
DECISION_DOC_FILENAMES.forEach(fn => {
|
||||
let best = null;
|
||||
let bestScore = -Infinity;
|
||||
summaries.forEach(s => {
|
||||
const sc = s.perDoc[fn]?.composite;
|
||||
if (sc == null) return;
|
||||
if (sc > bestScore) {
|
||||
bestScore = sc;
|
||||
best = s;
|
||||
}
|
||||
});
|
||||
if (best) best.wins += 1;
|
||||
});
|
||||
}
|
||||
assignWins(fixedSummaries);
|
||||
assignWins(semanticSummaries);
|
||||
|
||||
function autoPick(summaries) {
|
||||
if (!summaries.length) return null;
|
||||
return [...summaries].sort((a, b) => {
|
||||
const ma = a.meanComposite == null ? -1 : a.meanComposite;
|
||||
const mb = b.meanComposite == null ? -1 : b.meanComposite;
|
||||
if (mb !== ma) return mb - ma;
|
||||
if (b.wins !== a.wins) return b.wins - a.wins;
|
||||
return a.id.localeCompare(b.id);
|
||||
})[0];
|
||||
}
|
||||
|
||||
return {
|
||||
cellMap,
|
||||
fixedSummaries,
|
||||
semanticSummaries,
|
||||
autoFixed: autoPick(fixedSummaries),
|
||||
autoSemantic: autoPick(semanticSummaries),
|
||||
allCandidates: [...fixedSummaries, ...semanticSummaries],
|
||||
};
|
||||
}
|
||||
|
||||
function DecisionTab({ addToast }) {
|
||||
const [experiments, setExperiments] = useState([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [corpusId, setCorpusId] = useState('text-embedding-3-large');
|
||||
const [embModels, setEmbModels] = useState([]);
|
||||
const [excludedIds, setExcludedIds] = useState([]);
|
||||
const [overrideFixed, setOverrideFixed] = useState(null);
|
||||
const [overrideSemantic, setOverrideSemantic] = useState(null);
|
||||
|
||||
const refresh = useCallback(() => {
|
||||
setLoading(true);
|
||||
Promise.all([
|
||||
api('/experiments?limit=500'),
|
||||
api('/admin/embedding-models'),
|
||||
]).then(([ex, emb]) => {
|
||||
setExperiments(ex?.items || []);
|
||||
setEmbModels(emb?.models || []);
|
||||
if (emb?.corpus_id) {
|
||||
setCorpusId(prev => prev || emb.corpus_id);
|
||||
}
|
||||
}).catch(() => addToast('Failed to load Decision Board data', 'error'))
|
||||
.finally(() => setLoading(false));
|
||||
}, [addToast]);
|
||||
|
||||
useEffect(() => { refresh(); }, [refresh]);
|
||||
|
||||
const board = React.useMemo(
|
||||
() => buildDecisionBoard(experiments, corpusId, excludedIds),
|
||||
[experiments, corpusId, excludedIds]
|
||||
);
|
||||
|
||||
const winnerFixed = (overrideFixed && board.fixedSummaries.find(c => c.id === overrideFixed))
|
||||
|| board.autoFixed;
|
||||
const winnerSemantic = (overrideSemantic && board.semanticSummaries.find(c => c.id === overrideSemantic))
|
||||
|| board.autoSemantic;
|
||||
|
||||
const excludeId = (id) => {
|
||||
if (!id) return;
|
||||
setExcludedIds(prev => prev.includes(id) ? prev : [...prev, id]);
|
||||
};
|
||||
const unexcludeId = (id) => setExcludedIds(prev => prev.filter(x => x !== id));
|
||||
|
||||
const fsFilled = board.fixedSummaries.reduce((a, c) => a + c.filled, 0);
|
||||
const fsTotal = board.fixedSummaries.length * DECISION_DOC_FILENAMES.length;
|
||||
const semFilled = board.semanticSummaries.reduce((a, c) => a + c.filled, 0);
|
||||
const semTotal = board.semanticSummaries.length * DECISION_DOC_FILENAMES.length;
|
||||
|
||||
// Stage-2 head-to-head
|
||||
let duel = null;
|
||||
if (winnerFixed && winnerSemantic) {
|
||||
let fsWins = 0, semWins = 0, ties = 0;
|
||||
const perDoc = DECISION_DOC_FILENAMES.map(fn => {
|
||||
const a = winnerFixed.perDoc[fn]?.composite;
|
||||
const b = winnerSemantic.perDoc[fn]?.composite;
|
||||
let winner = '—';
|
||||
if (a != null && b != null) {
|
||||
if (a > b) { fsWins += 1; winner = 'fixed'; }
|
||||
else if (b > a) { semWins += 1; winner = 'semantic'; }
|
||||
else { ties += 1; winner = 'tie'; }
|
||||
}
|
||||
return { fn, a, b, winner };
|
||||
});
|
||||
const recommend = (winnerFixed.meanComposite || 0) >= (winnerSemantic.meanComposite || 0)
|
||||
? winnerFixed : winnerSemantic;
|
||||
// Prefer win-count if mean close? stick to mean primary with wins as display
|
||||
duel = { fsWins, semWins, ties, perDoc, recommend };
|
||||
}
|
||||
|
||||
const renderCandCard = (c, selected, onSelect, familyAutoId) =>
|
||||
React.createElement('div', {
|
||||
key: c.id,
|
||||
className: `decision-cand${selected ? ' selected' : ''}`,
|
||||
onClick: () => onSelect(selected ? null : (c.id === familyAutoId ? null : c.id)),
|
||||
title: 'Click to override family winner; click selected again to return to auto',
|
||||
},
|
||||
React.createElement('div', { className: 'cand-title', style: { display: 'flex', justifyContent: 'space-between', gap: 8 } },
|
||||
React.createElement('span', null,
|
||||
React.createElement('input', {
|
||||
type: 'radio',
|
||||
checked: !!selected,
|
||||
readOnly: true,
|
||||
style: { marginRight: 8 },
|
||||
}),
|
||||
c.label,
|
||||
c.id === familyAutoId
|
||||
? React.createElement('span', { className: 'badge badge-accent', style: { marginLeft: 8 } }, 'auto')
|
||||
: null
|
||||
),
|
||||
React.createElement('span', { style: { color: 'var(--accent)', fontVariantNumeric: 'tabular-nums' } },
|
||||
decisionFmt(c.meanComposite))
|
||||
),
|
||||
React.createElement('div', { className: 'decision-metrics' },
|
||||
React.createElement('span', null, 'Wins ', React.createElement('strong', null, `${c.wins}/${c.totalDocs}`)),
|
||||
React.createElement('span', null, 'Coverage ', React.createElement('strong', null, `${c.filled}/${c.totalDocs}`)),
|
||||
React.createElement('span', null, 'Context ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_context_relevance))),
|
||||
React.createElement('span', null, 'Similarity ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_answer_similarity))),
|
||||
React.createElement('span', null, 'Faithfulness ', React.createElement('strong', null, decisionFmt(c.meanMetrics.avg_faithfulness))),
|
||||
React.createElement('span', null, 'Halluc. ', React.createElement('strong', null, decisionPct(c.meanMetrics.hallucination_rate)))
|
||||
)
|
||||
);
|
||||
|
||||
const duelPanel = (c, side) => {
|
||||
if (!c) {
|
||||
return React.createElement('div', { className: 'card', style: { margin: 0 } },
|
||||
React.createElement('div', { className: 'text-muted' }, `No ${side} winner yet`));
|
||||
}
|
||||
const isRec = duel && duel.recommend && duel.recommend.id === c.id;
|
||||
return React.createElement('div', {
|
||||
className: 'card',
|
||||
style: {
|
||||
margin: 0,
|
||||
borderColor: isRec ? 'var(--accent)' : undefined,
|
||||
boxShadow: isRec ? '0 0 0 1px rgba(234,179,8,0.35)' : undefined,
|
||||
},
|
||||
},
|
||||
React.createElement('div', { className: 'flex-between mb-2' },
|
||||
React.createElement('div', { className: 'card-title mb-0' }, c.label),
|
||||
isRec ? React.createElement('span', { className: 'badge badge-success' }, 'Recommended') : null
|
||||
),
|
||||
React.createElement('div', { style: { fontSize: 28, fontWeight: 700, color: 'var(--accent)', marginBottom: 8 } },
|
||||
decisionFmt(c.meanComposite)),
|
||||
React.createElement('div', { className: 'decision-metrics', style: { marginBottom: 8 } },
|
||||
React.createElement('span', null, 'Doc wins vs other ', React.createElement('strong', null,
|
||||
side === 'fixed' ? (duel ? duel.fsWins : '—') : (duel ? duel.semWins : '—'))),
|
||||
React.createElement('span', null, 'Family wins ', React.createElement('strong', null, `${c.wins}/${c.totalDocs}`)),
|
||||
React.createElement('span', null, 'Coverage ', React.createElement('strong', null, `${c.filled}/${c.totalDocs}`))
|
||||
),
|
||||
React.createElement('table', { className: 'cmp-table' },
|
||||
React.createElement('tbody', null,
|
||||
[['Context Relevance', c.meanMetrics.avg_context_relevance, false],
|
||||
['Answer Similarity', c.meanMetrics.avg_answer_similarity, false],
|
||||
['Faithfulness', c.meanMetrics.avg_faithfulness, false],
|
||||
['Hallucination Rate', c.meanMetrics.hallucination_rate, true]].map(([label, val, isPct]) =>
|
||||
React.createElement('tr', { key: label },
|
||||
React.createElement('td', null, label),
|
||||
React.createElement('td', { style: { fontWeight: 600 } }, isPct ? decisionPct(val) : decisionFmt(val))
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
return React.createElement('div', null,
|
||||
React.createElement('div', { className: 'flex-between mb-2' },
|
||||
React.createElement('h1', { style: { color: 'var(--text-heading)', margin: 0, fontSize: '22px' } }, 'Decision Board'),
|
||||
React.createElement('button', { className: 'btn btn-secondary btn-sm', onClick: refresh, disabled: loading },
|
||||
loading ? React.createElement('span', { className: 'spinner' }) : '↻ Refresh')
|
||||
),
|
||||
React.createElement('p', { className: 'text-sm text-muted', style: { marginBottom: 16, maxWidth: 720 } },
|
||||
'Two-stage final selection: pick the best fixed_size Neighbor level and best semantic Boundary, then compare those winners across the 10-doc evaluation set. Mean composite ranks stage 1; win-counts are shown; click a Candidate to override.'
|
||||
),
|
||||
|
||||
// Header controls
|
||||
React.createElement('div', { className: 'card mb-4' },
|
||||
React.createElement('div', { className: 'row mb-0', style: { alignItems: 'flex-end' } },
|
||||
React.createElement(EmbeddingModelSelect, {
|
||||
label: 'Corpus filter',
|
||||
value: corpusId,
|
||||
onChange: (v) => { setCorpusId(v); setOverrideFixed(null); setOverrideSemantic(null); },
|
||||
models: embModels,
|
||||
role: 'corpus',
|
||||
title: 'Only single-strategy Experiments under this Corpus Embedding Model',
|
||||
style: { maxWidth: 360 },
|
||||
}),
|
||||
React.createElement('div', { className: 'col' },
|
||||
React.createElement('div', { className: 'text-sm text-muted' }, 'Coverage'),
|
||||
React.createElement('div', { style: { fontWeight: 600, color: 'var(--text-heading)', marginTop: 6 } },
|
||||
`fixed_size ${fsFilled}/${fsTotal || 40} · semantic ${semFilled}/${semTotal || '—'}`)
|
||||
)
|
||||
),
|
||||
excludedIds.length > 0 && React.createElement('div', { style: { marginTop: 12 } },
|
||||
React.createElement('div', { className: 'text-sm text-muted mb-1' }, 'Excluded Experiments (next-newest fills the cell)'),
|
||||
React.createElement('div', { style: { display: 'flex', flexWrap: 'wrap', gap: 6 } },
|
||||
excludedIds.map(id =>
|
||||
React.createElement('button', {
|
||||
key: id,
|
||||
className: 'btn btn-secondary btn-sm',
|
||||
onClick: () => unexcludeId(id),
|
||||
title: 'Click to restore',
|
||||
}, `✕ ${id.substring(0, 8)}…`)
|
||||
)
|
||||
)
|
||||
)
|
||||
),
|
||||
|
||||
loading && React.createElement('div', { style: { textAlign: 'center', padding: 24 } },
|
||||
React.createElement('span', { className: 'spinner' })),
|
||||
|
||||
!loading && React.createElement(React.Fragment, null,
|
||||
// Stage 1
|
||||
React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Stage 1 — Best Candidate per family'),
|
||||
React.createElement('div', { className: 'decision-grid mb-4' },
|
||||
React.createElement('div', { className: 'card', style: { margin: 0 } },
|
||||
React.createElement('div', { className: 'card-title' }, 'fixed_size (Neighbor Expansion)'),
|
||||
React.createElement('div', { className: 'text-sm text-muted mb-2' },
|
||||
'Auto winner: ', board.autoFixed ? board.autoFixed.label : '—',
|
||||
overrideFixed ? ' · override active' : ''),
|
||||
board.fixedSummaries.map(c => renderCandCard(
|
||||
c,
|
||||
winnerFixed && winnerFixed.id === c.id,
|
||||
setOverrideFixed,
|
||||
board.autoFixed?.id
|
||||
))
|
||||
),
|
||||
React.createElement('div', { className: 'card', style: { margin: 0 } },
|
||||
React.createElement('div', { className: 'card-title' }, 'semantic (Boundary Embedding Model)'),
|
||||
React.createElement('div', { className: 'text-sm text-muted mb-2' },
|
||||
'Auto winner: ', board.autoSemantic ? board.autoSemantic.label : '—',
|
||||
overrideSemantic ? ' · override active' : ''),
|
||||
board.semanticSummaries.length === 0
|
||||
? React.createElement('div', { className: 'empty-state' }, 'No semantic single-strategy Experiments for this Corpus')
|
||||
: board.semanticSummaries.map(c => renderCandCard(
|
||||
c,
|
||||
winnerSemantic && winnerSemantic.id === c.id,
|
||||
setOverrideSemantic,
|
||||
board.autoSemantic?.id
|
||||
))
|
||||
)
|
||||
),
|
||||
|
||||
// Stage 2
|
||||
React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Stage 2 — Family showdown'),
|
||||
React.createElement('div', { className: 'decision-duel mb-4' },
|
||||
duelPanel(winnerFixed, 'fixed'),
|
||||
React.createElement('div', { className: 'decision-vs' }, 'vs'),
|
||||
duelPanel(winnerSemantic, 'semantic')
|
||||
),
|
||||
duel && React.createElement('div', { className: 'card mb-4' },
|
||||
React.createElement('div', { className: 'card-title' }, 'Head-to-head by document'),
|
||||
React.createElement('div', { className: 'text-sm text-muted mb-2' },
|
||||
`fixed_size wins ${duel.fsWins} · semantic wins ${duel.semWins} · ties ${duel.ties}`),
|
||||
React.createElement('table', { className: 'cmp-table' },
|
||||
React.createElement('thead', null,
|
||||
React.createElement('tr', null,
|
||||
React.createElement('th', null, 'Document'),
|
||||
React.createElement('th', null, winnerFixed?.short || 'fixed'),
|
||||
React.createElement('th', null, winnerSemantic?.short || 'semantic'),
|
||||
React.createElement('th', null, 'Winner')
|
||||
)
|
||||
),
|
||||
React.createElement('tbody', null,
|
||||
duel.perDoc.map(row =>
|
||||
React.createElement('tr', { key: row.fn },
|
||||
React.createElement('td', null, row.fn),
|
||||
React.createElement('td', { style: { textAlign: 'center', fontVariantNumeric: 'tabular-nums' } }, decisionFmt(row.a)),
|
||||
React.createElement('td', { style: { textAlign: 'center', fontVariantNumeric: 'tabular-nums' } }, decisionFmt(row.b)),
|
||||
React.createElement('td', null,
|
||||
row.winner === 'fixed' ? React.createElement('span', { className: 'badge badge-accent' }, 'fixed_size')
|
||||
: row.winner === 'semantic' ? React.createElement('span', { className: 'badge badge-success' }, 'semantic')
|
||||
: row.winner === 'tie' ? React.createElement('span', { className: 'badge' }, 'tie')
|
||||
: React.createElement('span', { className: 'text-muted' }, '—')
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
),
|
||||
|
||||
// Matrix
|
||||
React.createElement('h2', { style: { fontSize: 16, color: 'var(--text-heading)', marginBottom: 10 } }, 'Per-document matrix'),
|
||||
React.createElement('div', { className: 'card', style: { paddingBottom: 14 } },
|
||||
React.createElement(DecisionHeatLegend),
|
||||
React.createElement('div', { className: 'decision-matrix-wrap' },
|
||||
React.createElement('table', { className: 'decision-matrix' },
|
||||
React.createElement('thead', null,
|
||||
React.createElement('tr', null,
|
||||
React.createElement('th', { className: 'sticky-col' }, 'Document'),
|
||||
board.allCandidates.map(c =>
|
||||
React.createElement('th', {
|
||||
key: c.id,
|
||||
className: (winnerFixed && c.id === winnerFixed.id) || (winnerSemantic && c.id === winnerSemantic.id)
|
||||
? 'col-duel' : undefined,
|
||||
title: c.label,
|
||||
}, c.family === 'fixed_size' ? c.short : (c.short || '').substring(0, 14))
|
||||
)
|
||||
)
|
||||
),
|
||||
React.createElement('tbody', null,
|
||||
DECISION_DOC_FILENAMES.map(fn => {
|
||||
const rowBest = decisionFamilyBestIds(board.allCandidates, c => c.perDoc[fn]?.composite);
|
||||
return React.createElement('tr', { key: fn },
|
||||
React.createElement('td', { className: 'sticky-col' }, fn),
|
||||
board.allCandidates.map(c => {
|
||||
const cell = c.perDoc[fn];
|
||||
const score = cell?.composite;
|
||||
const exp = cell?.exp;
|
||||
const mark = c.id === rowBest.fixed ? 'fs' : (c.id === rowBest.semantic ? 'sem' : null);
|
||||
return React.createElement('td', {
|
||||
key: c.id,
|
||||
className: 'decision-heat',
|
||||
style: { ...decisionHeatStyle(score), cursor: exp ? 'pointer' : 'default' },
|
||||
title: exp
|
||||
? `${c.label} · ${fn}\ncomposite=${decisionFmt(score)}${mark ? `\nbest ${mark === 'fs' ? 'fixed_size' : 'semantic'} in row` : ''}\nid=${exp.id}\nClick: report · Shift+click: exclude`
|
||||
: `${c.label} · ${fn}: missing`,
|
||||
onClick: (e) => {
|
||||
if (!exp) return;
|
||||
if (e.shiftKey) {
|
||||
excludeId(exp.id);
|
||||
addToast(`Excluded ${exp.id.substring(0, 8)}…`, 'success');
|
||||
return;
|
||||
}
|
||||
window.open(`/benchmarks/${exp.id}/report`, '_blank');
|
||||
},
|
||||
},
|
||||
mark
|
||||
? React.createElement('span', {
|
||||
className: `best-mark ${mark}`,
|
||||
'aria-label': mark === 'fs' ? 'Best fixed_size in row' : 'Best semantic in row',
|
||||
}, mark === 'fs' ? 'F' : 'S')
|
||||
: null,
|
||||
decisionFmt(score)
|
||||
);
|
||||
})
|
||||
);
|
||||
}),
|
||||
(() => {
|
||||
const meanBest = decisionFamilyBestIds(board.allCandidates, c => c.meanComposite);
|
||||
return React.createElement('tr', { key: '_mean', className: 'mean-row' },
|
||||
React.createElement('td', { className: 'sticky-col' }, 'Mean'),
|
||||
board.allCandidates.map(c => {
|
||||
const mark = c.id === meanBest.fixed ? 'fs' : (c.id === meanBest.semantic ? 'sem' : null);
|
||||
return React.createElement('td', {
|
||||
key: c.id,
|
||||
className: 'decision-heat',
|
||||
style: decisionHeatStyle(c.meanComposite),
|
||||
},
|
||||
mark
|
||||
? React.createElement('span', {
|
||||
className: `best-mark ${mark}`,
|
||||
'aria-label': mark === 'fs' ? 'Best fixed_size mean' : 'Best semantic mean',
|
||||
}, mark === 'fs' ? 'F' : 'S')
|
||||
: null,
|
||||
decisionFmt(c.meanComposite)
|
||||
);
|
||||
})
|
||||
);
|
||||
})(),
|
||||
React.createElement('tr', { key: '_wins', className: 'wins-row' },
|
||||
React.createElement('td', { className: 'sticky-col' }, 'Wins'),
|
||||
board.allCandidates.map(c =>
|
||||
React.createElement('td', { key: c.id }, `${c.wins}/${c.totalDocs}`)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
),
|
||||
React.createElement('div', { className: 'text-sm text-muted', style: { marginTop: 12 } },
|
||||
'Click a cell to open its Experiment report. Shift+click to exclude. ',
|
||||
React.createElement('span', { className: 'best-mark fs' }, 'F'),
|
||||
' / ',
|
||||
React.createElement('span', { className: 'best-mark sem' }, 'S'),
|
||||
' mark the best score in each family per row.'
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// -- Tab: Admin ---------------------------------------------
|
||||
function AdminTab({ addToast, documents, strategies, setActiveTab, setQuestionsFile }) {
|
||||
// Collapse state for each section
|
||||
@@ -2804,6 +3569,7 @@ function App() {
|
||||
{ id: 'pdf', label: 'PDF' },
|
||||
{ id: 'query', label: 'Query' },
|
||||
{ id: 'benchmarks', label: 'Benchmarks' },
|
||||
{ id: 'decision', label: 'Decision' },
|
||||
{ id: 'admin', label: 'Admin' },
|
||||
];
|
||||
|
||||
@@ -2818,6 +3584,7 @@ function App() {
|
||||
documents, strategies, addToast, formatFilter: 'word' });
|
||||
case 'benchmarks': return React.createElement(BenchmarksTab, {
|
||||
documents, strategies, addToast, questionsFile, formatFilter: 'word' });
|
||||
case 'decision': return React.createElement(DecisionTab, { addToast });
|
||||
case 'admin': return React.createElement(AdminTab, {
|
||||
addToast, documents, strategies, setActiveTab, setQuestionsFile });
|
||||
default: return null;
|
||||
|
||||
Reference in New Issue
Block a user