docs: record fixed_size as the winning strategy family
Why: - Close the benchmark with a written family decision backed by Experiment composites. Changes: - Add the decision memo and SVG charts generated from SQLite. Impact: - Documents fixed_size over semantic under text-embedding-3-large; ±N remains follow-up. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
458
docs/assets/decision/generate_charts.py
Normal file
458
docs/assets/decision/generate_charts.py
Normal file
@@ -0,0 +1,458 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate SVG charts for the final chunking-strategy decision report.
|
||||
|
||||
Reads Experiment aggregates from SQLite (Decision Board rules) and writes
|
||||
SVG files next to this script. Re-run after new Experiments if needed:
|
||||
|
||||
.venv/bin/python docs/assets/decision/generate_charts.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
DB = ROOT / "data" / "chunking_benchmark.db"
|
||||
OUT = Path(__file__).resolve().parent
|
||||
|
||||
DECISION_DOCS = [
|
||||
"bazresi.docx",
|
||||
"customer1.docx",
|
||||
"fire.docx",
|
||||
"general-havades-individuals.doc",
|
||||
"havades.docx",
|
||||
"life-time-individual.docx",
|
||||
"moavenin.docx",
|
||||
"Refah.docx",
|
||||
"website.docx",
|
||||
"lifetime-compensation.docx",
|
||||
]
|
||||
SHORT = {
|
||||
"bazresi.docx": "bazresi",
|
||||
"customer1.docx": "customer1",
|
||||
"fire.docx": "fire",
|
||||
"general-havades-individuals.doc": "havades-ind",
|
||||
"havades.docx": "havades",
|
||||
"life-time-individual.docx": "lifetime-ind",
|
||||
"moavenin.docx": "moavenin",
|
||||
"Refah.docx": "Refah",
|
||||
"website.docx": "website",
|
||||
"lifetime-compensation.docx": "lifetime-comp",
|
||||
}
|
||||
CORPUS = "text-embedding-3-large"
|
||||
|
||||
TEAL_FILL = "#14b8a6"
|
||||
AMBER_FILL = "#f59e0b"
|
||||
INK = "#111827"
|
||||
MUTED = "#6b7280"
|
||||
GRID = "#e5e7eb"
|
||||
BG = "#ffffff"
|
||||
WIN = "#047857"
|
||||
|
||||
|
||||
def esc(s: str) -> str:
|
||||
return (
|
||||
str(s)
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace('"', """)
|
||||
)
|
||||
|
||||
|
||||
def composite(m: dict | None) -> float | None:
|
||||
if not m:
|
||||
return None
|
||||
return (
|
||||
(m.get("avg_context_relevance") or 0) * 0.3
|
||||
+ (m.get("avg_answer_similarity") or 0) * 0.4
|
||||
+ (m.get("avg_faithfulness") or 0) * 0.3
|
||||
) * (1 - (m.get("hallucination_rate") or 0))
|
||||
|
||||
|
||||
def load_cells() -> dict[tuple[str, str], dict]:
|
||||
conn = sqlite3.connect(DB)
|
||||
conn.row_factory = sqlite3.Row
|
||||
docs = {r["id"]: r["filename"] for r in conn.execute("SELECT id, filename FROM documents")}
|
||||
rows = []
|
||||
for e in conn.execute(
|
||||
"""
|
||||
SELECT id, document_id, strategies_used, aggregate_metrics, benchmark_config,
|
||||
embedding_model_id, boundary_embedding_model_id, created_at, questions
|
||||
FROM experiments ORDER BY created_at DESC
|
||||
"""
|
||||
):
|
||||
fn = docs.get(e["document_id"])
|
||||
strats = json.loads(e["strategies_used"] or "[]")
|
||||
agg = json.loads(e["aggregate_metrics"] or "{}")
|
||||
cfg = json.loads(e["benchmark_config"] or "{}")
|
||||
corpus = e["embedding_model_id"] or cfg.get("corpus_embedding_model_id")
|
||||
prev = int(cfg.get("neighbor_prev") or 0)
|
||||
nxt = int(cfg.get("neighbor_next") or 0)
|
||||
bound = e["boundary_embedding_model_id"] or cfg.get("boundary_embedding_model_id")
|
||||
if fn not in DECISION_DOCS or corpus != CORPUS or len(strats) != 1:
|
||||
continue
|
||||
strat = strats[0]
|
||||
if strat == "fixed_size":
|
||||
if prev != nxt or prev not in (0, 1, 2, 3):
|
||||
continue
|
||||
cid = f"fixed_size:±{prev}"
|
||||
family = "fixed_size"
|
||||
elif strat == "semantic":
|
||||
if not bound:
|
||||
continue
|
||||
cid = f"semantic:{bound}"
|
||||
family = "semantic"
|
||||
else:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"fn": fn,
|
||||
"cid": cid,
|
||||
"family": family,
|
||||
"agg": agg,
|
||||
"metrics": agg.get(family) or {},
|
||||
"id": e["id"],
|
||||
}
|
||||
)
|
||||
conn.close()
|
||||
cells: dict[tuple[str, str], dict] = {}
|
||||
for r in rows:
|
||||
key = (r["fn"], r["cid"])
|
||||
if key not in cells:
|
||||
cells[key] = r
|
||||
return cells
|
||||
|
||||
|
||||
def mean(xs: list[float]) -> float | None:
|
||||
return sum(xs) / len(xs) if xs else None
|
||||
|
||||
|
||||
def svg_wrap(w: int, h: int, body: str, title: str) -> str:
|
||||
return f'''<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}" role="img" aria-label="{esc(title)}">
|
||||
<title>{esc(title)}</title>
|
||||
<rect width="{w}" height="{h}" fill="{BG}"/>
|
||||
{body}
|
||||
</svg>
|
||||
'''
|
||||
|
||||
|
||||
def chart_stage1(cells: dict) -> None:
|
||||
cands = [
|
||||
("fixed_size:±3", "fixed_size ±3", TEAL_FILL, True),
|
||||
("fixed_size:±2", "fixed_size ±2", TEAL_FILL, False),
|
||||
("fixed_size:±1", "fixed_size ±1", TEAL_FILL, False),
|
||||
("semantic:text-embedding-3-large", "semantic @ large", AMBER_FILL, False),
|
||||
("fixed_size:±0", "fixed_size ±0", "#99f6e4", False),
|
||||
("semantic:nomic-embed-text-v2-moe", "semantic @ nomic", "#fcd34d", False),
|
||||
]
|
||||
values = []
|
||||
for cid, label, color, winner in cands:
|
||||
scores = []
|
||||
for fn in DECISION_DOCS:
|
||||
r = cells.get((fn, cid))
|
||||
if r:
|
||||
sc = composite(r["metrics"])
|
||||
if sc is not None:
|
||||
scores.append(sc)
|
||||
values.append((cid, label, color, winner, mean(scores) or 0))
|
||||
|
||||
w, h = 820, 420
|
||||
left, right, top, bottom = 210, 40, 56, 48
|
||||
plot_w = w - left - right
|
||||
plot_h = h - top - bottom
|
||||
vmin, vmax = 8.4, 9.2
|
||||
bar_h = plot_h / len(values) * 0.62
|
||||
gap = plot_h / len(values)
|
||||
|
||||
parts = [
|
||||
f'<text x="24" y="32" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 1 — mean composite (10 documents)</text>',
|
||||
f'<text x="24" y="50" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Higher is better. Official ranking used by Decision Board. Winner: fixed_size ±3.</text>',
|
||||
]
|
||||
# grid
|
||||
for tick in [8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2]:
|
||||
x = left + (tick - vmin) / (vmax - vmin) * plot_w
|
||||
parts.append(f'<line x1="{x:.1f}" y1="{top}" x2="{x:.1f}" y2="{h - bottom}" stroke="{GRID}" stroke-width="1"/>')
|
||||
parts.append(
|
||||
f'<text x="{x:.1f}" y="{h - 18}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">{tick:.1f}</text>'
|
||||
)
|
||||
|
||||
for i, (_, label, color, winner, val) in enumerate(values):
|
||||
y = top + i * gap + (gap - bar_h) / 2
|
||||
bw = (val - vmin) / (vmax - vmin) * plot_w
|
||||
stroke = WIN if winner else "none"
|
||||
sw = 2 if winner else 0
|
||||
parts.append(
|
||||
f'<rect x="{left}" y="{y:.1f}" width="{max(bw, 0):.1f}" height="{bar_h:.1f}" rx="4" fill="{color}" stroke="{stroke}" stroke-width="{sw}"/>'
|
||||
)
|
||||
weight = "700" if winner else "500"
|
||||
parts.append(
|
||||
f'<text x="{left - 10}" y="{y + bar_h * 0.68:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="{weight}" fill="{INK}">{esc(label)}</text>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="{left + bw + 8:.1f}" y="{y + bar_h * 0.68:.1f}" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="{INK}">{val:.3f}</text>'
|
||||
)
|
||||
|
||||
(OUT / "stage1-mean-composite.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 1 mean composite"), encoding="utf-8")
|
||||
|
||||
|
||||
def chart_stage2(cells: dict) -> None:
|
||||
fs_id = "fixed_size:±3"
|
||||
sem_id = "semantic:text-embedding-3-large"
|
||||
pairs = []
|
||||
for fn in DECISION_DOCS:
|
||||
a = composite(cells[(fn, fs_id)]["metrics"])
|
||||
b = composite(cells[(fn, sem_id)]["metrics"])
|
||||
pairs.append((SHORT[fn], a, b))
|
||||
|
||||
w, h = 920, 460
|
||||
left, right, top, bottom = 52, 24, 64, 88
|
||||
plot_w = w - left - right
|
||||
plot_h = h - top - bottom
|
||||
n = len(pairs)
|
||||
slot = plot_w / n
|
||||
bar_w = slot * 0.32
|
||||
vmin, vmax = 5.8, 10.0
|
||||
|
||||
def y_of(v: float) -> float:
|
||||
return top + (1 - (v - vmin) / (vmax - vmin)) * plot_h
|
||||
|
||||
parts = [
|
||||
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 2 — per document (fixed_size ±3 vs semantic @ large)</text>',
|
||||
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Mean composite still favors fixed_size. Semantic wins 6 of 10 docs, but by smaller margins except website/customer1.</text>',
|
||||
f'<rect x="620" y="14" width="12" height="12" rx="2" fill="{TEAL_FILL}"/>',
|
||||
f'<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">fixed_size ±3</text>',
|
||||
f'<rect x="760" y="14" width="12" height="12" rx="2" fill="{AMBER_FILL}"/>',
|
||||
f'<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">semantic @ large</text>',
|
||||
]
|
||||
for tick in [6, 7, 8, 9, 10]:
|
||||
y = y_of(tick)
|
||||
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{w - right}" y2="{y:.1f}" stroke="{GRID}" stroke-width="1"/>')
|
||||
parts.append(
|
||||
f'<text x="{left - 8}" y="{y + 4:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">{tick}</text>'
|
||||
)
|
||||
|
||||
for i, (label, a, b) in enumerate(pairs):
|
||||
cx = left + i * slot + slot / 2
|
||||
xa = cx - bar_w - 3
|
||||
xb = cx + 3
|
||||
ha = plot_h - (y_of(a) - top)
|
||||
hb = plot_h - (y_of(b) - top)
|
||||
parts.append(f'<rect x="{xa:.1f}" y="{y_of(a):.1f}" width="{bar_w:.1f}" height="{ha:.1f}" rx="3" fill="{TEAL_FILL}"/>')
|
||||
parts.append(f'<rect x="{xb:.1f}" y="{y_of(b):.1f}" width="{bar_w:.1f}" height="{hb:.1f}" rx="3" fill="{AMBER_FILL}"/>')
|
||||
parts.append(
|
||||
f'<text x="{cx:.1f}" y="{h - 36}" text-anchor="end" transform="rotate(-32 {cx:.1f} {h - 36})" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{INK}">{esc(label)}</text>'
|
||||
)
|
||||
|
||||
(OUT / "stage2-per-document.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 per document"), encoding="utf-8")
|
||||
|
||||
|
||||
def chart_delta(cells: dict) -> None:
|
||||
fs_id = "fixed_size:±3"
|
||||
sem_id = "semantic:text-embedding-3-large"
|
||||
deltas = []
|
||||
for fn in DECISION_DOCS:
|
||||
a = composite(cells[(fn, fs_id)]["metrics"])
|
||||
b = composite(cells[(fn, sem_id)]["metrics"])
|
||||
deltas.append((SHORT[fn], a - b))
|
||||
deltas.sort(key=lambda x: x[1], reverse=True)
|
||||
|
||||
w, h = 820, 440
|
||||
left, right, top, bottom = 120, 56, 56, 36
|
||||
plot_w = w - left - right
|
||||
plot_h = h - top - bottom
|
||||
n = len(deltas)
|
||||
gap = plot_h / n
|
||||
bar_h = gap * 0.62
|
||||
max_abs = max(abs(d) for _, d in deltas)
|
||||
# fire is 2.58, others < 1.2 — use 2.8
|
||||
max_abs = 2.8
|
||||
zero_x = left + plot_w / 2
|
||||
|
||||
parts = [
|
||||
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Margin: fixed_size ±3 minus semantic @ large</text>',
|
||||
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Teal = fixed_size wins the document. Amber = semantic wins. Mean ranking is driven by large teal bars (especially fire).</text>',
|
||||
f'<line x1="{zero_x:.1f}" y1="{top}" x2="{zero_x:.1f}" y2="{h - bottom}" stroke="{INK}" stroke-width="1.2"/>',
|
||||
f'<text x="{left}" y="{h - 12}" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">semantic better</text>',
|
||||
f'<text x="{w - right}" y="{h - 12}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">fixed_size better</text>',
|
||||
]
|
||||
for i, (label, d) in enumerate(deltas):
|
||||
y = top + i * gap + (gap - bar_h) / 2
|
||||
bw = abs(d) / max_abs * (plot_w / 2)
|
||||
if d >= 0:
|
||||
x = zero_x
|
||||
color = TEAL_FILL
|
||||
tx = x + bw + 6
|
||||
anchor = "start"
|
||||
else:
|
||||
x = zero_x - bw
|
||||
color = AMBER_FILL
|
||||
tx = x - 6
|
||||
anchor = "end"
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bw:.1f}" height="{bar_h:.1f}" rx="3" fill="{color}"/>')
|
||||
parts.append(
|
||||
f'<text x="{left - 8}" y="{y + bar_h * 0.7:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">{esc(label)}</text>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="{tx:.1f}" y="{y + bar_h * 0.7:.1f}" text-anchor="{anchor}" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="{INK}">{d:+.2f}</text>'
|
||||
)
|
||||
|
||||
(OUT / "stage2-margins.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 margins"), encoding="utf-8")
|
||||
|
||||
|
||||
def chart_heatmap(cells: dict) -> None:
|
||||
cands = [
|
||||
("fixed_size:±0", "±0"),
|
||||
("fixed_size:±1", "±1"),
|
||||
("fixed_size:±2", "±2"),
|
||||
("fixed_size:±3", "±3"),
|
||||
("semantic:text-embedding-3-large", "sem@large"),
|
||||
("semantic:nomic-embed-text-v2-moe", "sem@nomic"),
|
||||
]
|
||||
scores: list[list[float | None]] = []
|
||||
for fn in DECISION_DOCS:
|
||||
row = []
|
||||
for cid, _ in cands:
|
||||
r = cells.get((fn, cid))
|
||||
row.append(composite(r["metrics"]) if r else None)
|
||||
scores.append(row)
|
||||
|
||||
cell_w, cell_h = 78, 32
|
||||
left, top = 128, 72
|
||||
w = left + cell_w * len(cands) + 24
|
||||
h = top + cell_h * len(DECISION_DOCS) + 36
|
||||
|
||||
def color_for(s: float | None) -> tuple[str, str]:
|
||||
if s is None:
|
||||
return "#f3f4f6", MUTED
|
||||
# 6.2 .. 9.8
|
||||
t = max(0.0, min(1.0, (s - 7.0) / (9.8 - 7.0)))
|
||||
# pale rose -> amber -> teal
|
||||
if t < 0.5:
|
||||
u = t * 2
|
||||
r = int(251 + (20 - 251) * 0)
|
||||
# interpolate rose 251,113,133 -> amber 245,158,11
|
||||
rr = int(251 + (245 - 251) * u)
|
||||
gg = int(113 + (158 - 113) * u)
|
||||
bb = int(133 + (11 - 133) * u)
|
||||
else:
|
||||
u = (t - 0.5) * 2
|
||||
rr = int(245 + (13 - 245) * u)
|
||||
gg = int(158 + (148 - 158) * u)
|
||||
bb = int(11 + (136 - 11) * u)
|
||||
bg = f"#{rr:02x}{gg:02x}{bb:02x}"
|
||||
fg = "#111827" if t < 0.72 else "#f9fafb"
|
||||
return bg, fg
|
||||
|
||||
parts = [
|
||||
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Candidate heatmap — composite by document</text>',
|
||||
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set.</text>',
|
||||
]
|
||||
for j, (_, lab) in enumerate(cands):
|
||||
x = left + j * cell_w + cell_w / 2
|
||||
parts.append(
|
||||
f'<text x="{x:.1f}" y="{top - 10}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{esc(lab)}</text>'
|
||||
)
|
||||
for i, fn in enumerate(DECISION_DOCS):
|
||||
y = top + i * cell_h
|
||||
parts.append(
|
||||
f'<text x="{left - 8}" y="{y + cell_h * 0.65:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{INK}">{esc(SHORT[fn])}</text>'
|
||||
)
|
||||
for j, val in enumerate(scores[i]):
|
||||
x = left + j * cell_w
|
||||
bg, fg = color_for(val)
|
||||
parts.append(f'<rect x="{x}" y="{y}" width="{cell_w - 3}" height="{cell_h - 3}" rx="4" fill="{bg}"/>')
|
||||
txt = "—" if val is None else f"{val:.2f}"
|
||||
parts.append(
|
||||
f'<text x="{x + (cell_w - 3) / 2:.1f}" y="{y + cell_h * 0.62:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{fg}">{txt}</text>'
|
||||
)
|
||||
|
||||
(OUT / "heatmap-candidates.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Candidate heatmap"), encoding="utf-8")
|
||||
|
||||
|
||||
def chart_metrics(cells: dict) -> None:
|
||||
"""Grouped bars: four judge metrics for the two stage-2 winners."""
|
||||
def bag(cid: str, family: str) -> dict[str, float]:
|
||||
keys = [
|
||||
"avg_context_relevance",
|
||||
"avg_answer_similarity",
|
||||
"avg_faithfulness",
|
||||
"hallucination_rate",
|
||||
]
|
||||
acc = {k: [] for k in keys}
|
||||
for fn in DECISION_DOCS:
|
||||
m = cells[(fn, cid)]["metrics"]
|
||||
for k in keys:
|
||||
acc[k].append(m[k])
|
||||
return {k: sum(v) / len(v) for k, v in acc.items()}
|
||||
|
||||
fs = bag("fixed_size:±3", "fixed_size")
|
||||
sem = bag("semantic:text-embedding-3-large", "semantic")
|
||||
labels = [
|
||||
("Context relevance", fs["avg_context_relevance"], sem["avg_context_relevance"], False),
|
||||
("Answer similarity", fs["avg_answer_similarity"], sem["avg_answer_similarity"], False),
|
||||
("Faithfulness", fs["avg_faithfulness"], sem["avg_faithfulness"], False),
|
||||
("Hallucination % (lower better)", fs["hallucination_rate"] * 100, sem["hallucination_rate"] * 100, True),
|
||||
]
|
||||
|
||||
w, h = 820, 380
|
||||
left, right, top, bottom = 52, 24, 64, 48
|
||||
plot_w = w - left - right
|
||||
plot_h = h - top - bottom
|
||||
n = len(labels)
|
||||
slot = plot_w / n
|
||||
bar_w = slot * 0.28
|
||||
|
||||
parts = [
|
||||
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 2 winners — mean judge metrics</text>',
|
||||
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">fixed_size ±3 leads on all four metrics, including lower hallucination.</text>',
|
||||
f'<rect x="620" y="14" width="12" height="12" rx="2" fill="{TEAL_FILL}"/>',
|
||||
f'<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">fixed_size ±3</text>',
|
||||
f'<rect x="760" y="14" width="12" height="12" rx="2" fill="{AMBER_FILL}"/>',
|
||||
f'<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">semantic @ large</text>',
|
||||
]
|
||||
|
||||
for i, (lab, a, b, is_pct) in enumerate(labels):
|
||||
vmax = 10 if not is_pct else max(a, b) * 1.35
|
||||
cx = left + i * slot + slot / 2
|
||||
xa = cx - bar_w - 3
|
||||
xb = cx + 3
|
||||
|
||||
def bar(x: float, val: float, color: str) -> str:
|
||||
bh = val / vmax * plot_h
|
||||
y = top + plot_h - bh
|
||||
return f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_w:.1f}" height="{bh:.1f}" rx="3" fill="{color}"/>'
|
||||
|
||||
parts.append(bar(xa, a, TEAL_FILL))
|
||||
parts.append(bar(xb, b, AMBER_FILL))
|
||||
fmt = (lambda v: f"{v:.1f}%") if is_pct else (lambda v: f"{v:.2f}")
|
||||
ya = top + plot_h - a / vmax * plot_h - 6
|
||||
yb = top + plot_h - b / vmax * plot_h - 6
|
||||
parts.append(
|
||||
f'<text x="{xa + bar_w / 2:.1f}" y="{ya:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{fmt(a)}</text>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="{xb + bar_w / 2:.1f}" y="{yb:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{fmt(b)}</text>'
|
||||
)
|
||||
parts.append(
|
||||
f'<text x="{cx:.1f}" y="{h - 16}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">{esc(lab)}</text>'
|
||||
)
|
||||
|
||||
(OUT / "stage2-metrics.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 mean metrics"), encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cells = load_cells()
|
||||
chart_stage1(cells)
|
||||
chart_stage2(cells)
|
||||
chart_delta(cells)
|
||||
chart_heatmap(cells)
|
||||
chart_metrics(cells)
|
||||
print(f"Wrote SVGs in {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
143
docs/assets/decision/heatmap-candidates.svg
Normal file
143
docs/assets/decision/heatmap-candidates.svg
Normal file
@@ -0,0 +1,143 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="620" height="428" viewBox="0 0 620 428" role="img" aria-label="Candidate heatmap">
|
||||
<title>Candidate heatmap</title>
|
||||
<rect width="620" height="428" fill="#ffffff"/>
|
||||
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Candidate heatmap — composite by document</text>
|
||||
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set.</text>
|
||||
<text x="167.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±0</text>
|
||||
<text x="245.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±1</text>
|
||||
<text x="323.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±2</text>
|
||||
<text x="401.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±3</text>
|
||||
<text x="479.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">sem@large</text>
|
||||
<text x="557.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">sem@nomic</text>
|
||||
<text x="120" y="92.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">bazresi</text>
|
||||
<rect x="128" y="72" width="75" height="29" rx="4" fill="#e19d15"/>
|
||||
<text x="165.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.52</text>
|
||||
<rect x="206" y="72" width="75" height="29" rx="4" fill="#58975f"/>
|
||||
<text x="243.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.34</text>
|
||||
<rect x="284" y="72" width="75" height="29" rx="4" fill="#58975f"/>
|
||||
<text x="321.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.34</text>
|
||||
<rect x="362" y="72" width="75" height="29" rx="4" fill="#549761"/>
|
||||
<text x="399.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.37</text>
|
||||
<rect x="440" y="72" width="75" height="29" rx="4" fill="#4e9664"/>
|
||||
<text x="477.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.40</text>
|
||||
<rect x="518" y="72" width="75" height="29" rx="4" fill="#7c984b"/>
|
||||
<text x="555.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.12</text>
|
||||
<text x="120" y="124.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">customer1</text>
|
||||
<rect x="128" y="104" width="75" height="29" rx="4" fill="#9f9a39"/>
|
||||
<text x="165.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.92</text>
|
||||
<rect x="206" y="104" width="75" height="29" rx="4" fill="#96993e"/>
|
||||
<text x="243.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.97</text>
|
||||
<rect x="284" y="104" width="75" height="29" rx="4" fill="#97993d"/>
|
||||
<text x="321.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.96</text>
|
||||
<rect x="362" y="104" width="75" height="29" rx="4" fill="#ec9d0f"/>
|
||||
<text x="399.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.45</text>
|
||||
<rect x="440" y="104" width="75" height="29" rx="4" fill="#96993e"/>
|
||||
<text x="477.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.97</text>
|
||||
<rect x="518" y="104" width="75" height="29" rx="4" fill="#f88255"/>
|
||||
<text x="555.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.54</text>
|
||||
<text x="120" y="156.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">fire</text>
|
||||
<rect x="128" y="136" width="75" height="29" rx="4" fill="#f69426"/>
|
||||
<text x="165.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.09</text>
|
||||
<rect x="206" y="136" width="75" height="29" rx="4" fill="#f59c10"/>
|
||||
<text x="243.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.34</text>
|
||||
<rect x="284" y="136" width="75" height="29" rx="4" fill="#f68f33"/>
|
||||
<text x="321.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.93</text>
|
||||
<rect x="362" y="136" width="75" height="29" rx="4" fill="#b19b2f"/>
|
||||
<text x="399.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.81</text>
|
||||
<rect x="440" y="136" width="75" height="29" rx="4" fill="#fb7185"/>
|
||||
<text x="477.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">6.23</text>
|
||||
<rect x="518" y="136" width="75" height="29" rx="4" fill="#fa7674"/>
|
||||
<text x="555.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.19</text>
|
||||
<text x="120" y="188.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades-ind</text>
|
||||
<rect x="128" y="168" width="75" height="29" rx="4" fill="#f78b3c"/>
|
||||
<text x="165.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.83</text>
|
||||
<rect x="206" y="168" width="75" height="29" rx="4" fill="#c99c22"/>
|
||||
<text x="243.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.66</text>
|
||||
<rect x="284" y="168" width="75" height="29" rx="4" fill="#7e984a"/>
|
||||
<text x="321.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.12</text>
|
||||
<rect x="362" y="168" width="75" height="29" rx="4" fill="#6f9852"/>
|
||||
<text x="399.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.21</text>
|
||||
<rect x="440" y="168" width="75" height="29" rx="4" fill="#d99c19"/>
|
||||
<text x="477.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.56</text>
|
||||
<rect x="518" y="168" width="75" height="29" rx="4" fill="#e09d16"/>
|
||||
<text x="555.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.53</text>
|
||||
<text x="120" y="220.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades</text>
|
||||
<rect x="128" y="200" width="75" height="29" rx="4" fill="#f59b12"/>
|
||||
<text x="165.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.31</text>
|
||||
<rect x="206" y="200" width="75" height="29" rx="4" fill="#ef9d0d"/>
|
||||
<text x="243.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.43</text>
|
||||
<rect x="284" y="200" width="75" height="29" rx="4" fill="#f49d0b"/>
|
||||
<text x="321.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.40</text>
|
||||
<rect x="362" y="200" width="75" height="29" rx="4" fill="#e79d12"/>
|
||||
<text x="399.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.48</text>
|
||||
<rect x="440" y="200" width="75" height="29" rx="4" fill="#f69030"/>
|
||||
<text x="477.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.97</text>
|
||||
<rect x="518" y="200" width="75" height="29" rx="4" fill="#f29d0c"/>
|
||||
<text x="555.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.42</text>
|
||||
<text x="120" y="252.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-ind</text>
|
||||
<rect x="128" y="232" width="75" height="29" rx="4" fill="#6d9854"/>
|
||||
<text x="165.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.22</text>
|
||||
<rect x="206" y="232" width="75" height="29" rx="4" fill="#4b9666"/>
|
||||
<text x="243.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.42</text>
|
||||
<rect x="284" y="232" width="75" height="29" rx="4" fill="#489667"/>
|
||||
<text x="321.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.44</text>
|
||||
<rect x="362" y="232" width="75" height="29" rx="4" fill="#479668"/>
|
||||
<text x="399.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.45</text>
|
||||
<rect x="440" y="232" width="75" height="29" rx="4" fill="#3d966d"/>
|
||||
<text x="477.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.50</text>
|
||||
<rect x="518" y="232" width="75" height="29" rx="4" fill="#4e9664"/>
|
||||
<text x="555.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.41</text>
|
||||
<text x="120" y="284.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">moavenin</text>
|
||||
<rect x="128" y="264" width="75" height="29" rx="4" fill="#879945"/>
|
||||
<text x="165.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.06</text>
|
||||
<rect x="206" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||||
<text x="243.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||||
<rect x="284" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||||
<text x="321.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||||
<rect x="362" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||||
<text x="399.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||||
<rect x="440" y="264" width="75" height="29" rx="4" fill="#d09c1e"/>
|
||||
<text x="477.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.62</text>
|
||||
<rect x="518" y="264" width="75" height="29" rx="4" fill="#f59a14"/>
|
||||
<text x="555.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.29</text>
|
||||
<text x="120" y="316.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">Refah</text>
|
||||
<rect x="128" y="296" width="75" height="29" rx="4" fill="#459669"/>
|
||||
<text x="165.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.46</text>
|
||||
<rect x="206" y="296" width="75" height="29" rx="4" fill="#1c947f"/>
|
||||
<text x="243.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.71</text>
|
||||
<rect x="284" y="296" width="75" height="29" rx="4" fill="#1e947e"/>
|
||||
<text x="321.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.70</text>
|
||||
<rect x="362" y="296" width="75" height="29" rx="4" fill="#20947d"/>
|
||||
<text x="399.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.68</text>
|
||||
<rect x="440" y="296" width="75" height="29" rx="4" fill="#0d9488"/>
|
||||
<text x="477.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.81</text>
|
||||
<rect x="518" y="296" width="75" height="29" rx="4" fill="#349572"/>
|
||||
<text x="555.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.56</text>
|
||||
<text x="120" y="348.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">website</text>
|
||||
<rect x="128" y="328" width="75" height="29" rx="4" fill="#f8864b"/>
|
||||
<text x="165.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.67</text>
|
||||
<rect x="206" y="328" width="75" height="29" rx="4" fill="#f78e35"/>
|
||||
<text x="243.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.91</text>
|
||||
<rect x="284" y="328" width="75" height="29" rx="4" fill="#e29d15"/>
|
||||
<text x="321.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.51</text>
|
||||
<rect x="362" y="328" width="75" height="29" rx="4" fill="#ea9d10"/>
|
||||
<text x="399.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.47</text>
|
||||
<rect x="440" y="328" width="75" height="29" rx="4" fill="#879945"/>
|
||||
<text x="477.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.06</text>
|
||||
<rect x="518" y="328" width="75" height="29" rx="4" fill="#e59d13"/>
|
||||
<text x="555.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.49</text>
|
||||
<text x="120" y="380.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-comp</text>
|
||||
<rect x="128" y="360" width="75" height="29" rx="4" fill="#93993f"/>
|
||||
<text x="165.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.99</text>
|
||||
<rect x="206" y="360" width="75" height="29" rx="4" fill="#bd9b28"/>
|
||||
<text x="243.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.73</text>
|
||||
<rect x="284" y="360" width="75" height="29" rx="4" fill="#bd9b28"/>
|
||||
<text x="321.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.73</text>
|
||||
<rect x="362" y="360" width="75" height="29" rx="4" fill="#b99b2b"/>
|
||||
<text x="399.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.76</text>
|
||||
<rect x="440" y="360" width="75" height="29" rx="4" fill="#7c984c"/>
|
||||
<text x="477.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.13</text>
|
||||
<rect x="518" y="360" width="75" height="29" rx="4" fill="#98993d"/>
|
||||
<text x="555.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.96</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 16 KiB |
41
docs/assets/decision/stage1-mean-composite.svg
Normal file
41
docs/assets/decision/stage1-mean-composite.svg
Normal file
@@ -0,0 +1,41 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="820" height="420" viewBox="0 0 820 420" role="img" aria-label="Stage 1 mean composite">
|
||||
<title>Stage 1 mean composite</title>
|
||||
<rect width="820" height="420" fill="#ffffff"/>
|
||||
<text x="24" y="32" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Stage 1 — mean composite (10 documents)</text>
|
||||
<text x="24" y="50" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Higher is better. Official ranking used by Decision Board. Winner: fixed_size ±3.</text>
|
||||
<line x1="281.2" y1="56" x2="281.2" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="281.2" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.5</text>
|
||||
<line x1="352.5" y1="56" x2="352.5" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="352.5" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.6</text>
|
||||
<line x1="423.7" y1="56" x2="423.7" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="423.7" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.7</text>
|
||||
<line x1="495.0" y1="56" x2="495.0" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="495.0" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.8</text>
|
||||
<line x1="566.3" y1="56" x2="566.3" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="566.3" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.9</text>
|
||||
<line x1="637.5" y1="56" x2="637.5" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="637.5" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">9.0</text>
|
||||
<line x1="708.8" y1="56" x2="708.8" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="708.8" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">9.1</text>
|
||||
<line x1="780.0" y1="56" x2="780.0" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="780.0" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">9.2</text>
|
||||
<rect x="210" y="66.0" width="456.2" height="32.7" rx="4" fill="#14b8a6" stroke="#047857" stroke-width="2"/>
|
||||
<text x="200" y="88.2" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="700" fill="#111827">fixed_size ±3</text>
|
||||
<text x="674.2" y="88.2" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">9.040</text>
|
||||
<rect x="210" y="118.7" width="418.4" height="32.7" rx="4" fill="#14b8a6" stroke="none" stroke-width="0"/>
|
||||
<text x="200" y="140.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">fixed_size ±2</text>
|
||||
<text x="636.4" y="140.9" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.987</text>
|
||||
<rect x="210" y="171.3" width="374.8" height="32.7" rx="4" fill="#14b8a6" stroke="none" stroke-width="0"/>
|
||||
<text x="200" y="193.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">fixed_size ±1</text>
|
||||
<text x="592.8" y="193.5" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.926</text>
|
||||
<rect x="210" y="224.0" width="232.8" height="32.7" rx="4" fill="#f59e0b" stroke="none" stroke-width="0"/>
|
||||
<text x="200" y="246.2" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">semantic @ large</text>
|
||||
<text x="450.8" y="246.2" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.727</text>
|
||||
<rect x="210" y="276.7" width="146.7" height="32.7" rx="4" fill="#99f6e4" stroke="none" stroke-width="0"/>
|
||||
<text x="200" y="298.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">fixed_size ±0</text>
|
||||
<text x="364.7" y="298.9" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.606</text>
|
||||
<rect x="210" y="329.3" width="107.4" height="32.7" rx="4" fill="#fcd34d" stroke="none" stroke-width="0"/>
|
||||
<text x="200" y="351.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">semantic @ nomic</text>
|
||||
<text x="325.4" y="351.5" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.551</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.5 KiB |
40
docs/assets/decision/stage2-margins.svg
Normal file
40
docs/assets/decision/stage2-margins.svg
Normal file
@@ -0,0 +1,40 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="820" height="440" viewBox="0 0 820 440" role="img" aria-label="Stage 2 margins">
|
||||
<title>Stage 2 margins</title>
|
||||
<rect width="820" height="440" fill="#ffffff"/>
|
||||
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Margin: fixed_size ±3 minus semantic @ large</text>
|
||||
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Teal = fixed_size wins the document. Amber = semantic wins. Mean ranking is driven by large teal bars (especially fire).</text>
|
||||
<line x1="442.0" y1="56" x2="442.0" y2="404" stroke="#111827" stroke-width="1.2"/>
|
||||
<text x="120" y="428" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">semantic better</text>
|
||||
<text x="764" y="428" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">fixed_size better</text>
|
||||
<rect x="442.0" y="62.6" width="296.9" height="21.6" rx="3" fill="#14b8a6"/>
|
||||
<text x="112" y="77.7" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">fire</text>
|
||||
<text x="744.9" y="77.7" text-anchor="start" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">+2.58</text>
|
||||
<rect x="442.0" y="97.4" width="128.1" height="21.6" rx="3" fill="#14b8a6"/>
|
||||
<text x="112" y="112.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">moavenin</text>
|
||||
<text x="576.1" y="112.5" text-anchor="start" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">+1.11</text>
|
||||
<rect x="442.0" y="132.2" width="73.7" height="21.6" rx="3" fill="#14b8a6"/>
|
||||
<text x="112" y="147.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">havades-ind</text>
|
||||
<text x="521.7" y="147.3" text-anchor="start" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">+0.64</text>
|
||||
<rect x="442.0" y="167.0" width="59.1" height="21.6" rx="3" fill="#14b8a6"/>
|
||||
<text x="112" y="182.1" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">havades</text>
|
||||
<text x="507.1" y="182.1" text-anchor="start" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">+0.51</text>
|
||||
<rect x="437.5" y="201.8" width="4.5" height="21.6" rx="3" fill="#f59e0b"/>
|
||||
<text x="112" y="216.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">bazresi</text>
|
||||
<text x="431.5" y="216.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.04</text>
|
||||
<rect x="435.1" y="236.6" width="6.9" height="21.6" rx="3" fill="#f59e0b"/>
|
||||
<text x="112" y="251.7" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">lifetime-ind</text>
|
||||
<text x="429.1" y="251.7" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.06</text>
|
||||
<rect x="426.5" y="271.4" width="15.5" height="21.6" rx="3" fill="#f59e0b"/>
|
||||
<text x="112" y="286.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Refah</text>
|
||||
<text x="420.5" y="286.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.13</text>
|
||||
<rect x="399.7" y="306.2" width="42.3" height="21.6" rx="3" fill="#f59e0b"/>
|
||||
<text x="112" y="321.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">lifetime-comp</text>
|
||||
<text x="393.7" y="321.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.37</text>
|
||||
<rect x="382.0" y="341.0" width="60.0" height="21.6" rx="3" fill="#f59e0b"/>
|
||||
<text x="112" y="356.1" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">customer1</text>
|
||||
<text x="376.0" y="356.1" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.52</text>
|
||||
<rect x="373.9" y="375.8" width="68.1" height="21.6" rx="3" fill="#f59e0b"/>
|
||||
<text x="112" y="390.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">website</text>
|
||||
<text x="367.9" y="390.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.59</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.4 KiB |
31
docs/assets/decision/stage2-metrics.svg
Normal file
31
docs/assets/decision/stage2-metrics.svg
Normal file
@@ -0,0 +1,31 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="820" height="380" viewBox="0 0 820 380" role="img" aria-label="Stage 2 mean metrics">
|
||||
<title>Stage 2 mean metrics</title>
|
||||
<rect width="820" height="380" fill="#ffffff"/>
|
||||
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Stage 2 winners — mean judge metrics</text>
|
||||
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">fixed_size ±3 leads on all four metrics, including lower hallucination.</text>
|
||||
<rect x="620" y="14" width="12" height="12" rx="2" fill="#14b8a6"/>
|
||||
<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">fixed_size ±3</text>
|
||||
<rect x="760" y="14" width="12" height="12" rx="2" fill="#f59e0b"/>
|
||||
<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">semantic @ large</text>
|
||||
<rect x="89.9" y="80.0" width="52.1" height="252.0" rx="3" fill="#14b8a6"/>
|
||||
<rect x="148.0" y="80.7" width="52.1" height="251.3" rx="3" fill="#f59e0b"/>
|
||||
<text x="116.0" y="74.0" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">9.40</text>
|
||||
<text x="174.0" y="74.7" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">9.38</text>
|
||||
<text x="145.0" y="364" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Context relevance</text>
|
||||
<rect x="275.9" y="92.1" width="52.1" height="239.9" rx="3" fill="#14b8a6"/>
|
||||
<rect x="334.0" y="95.3" width="52.1" height="236.7" rx="3" fill="#f59e0b"/>
|
||||
<text x="302.0" y="86.1" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.95</text>
|
||||
<text x="360.0" y="89.3" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.83</text>
|
||||
<text x="331.0" y="364" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Answer similarity</text>
|
||||
<rect x="461.9" y="78.2" width="52.1" height="253.8" rx="3" fill="#14b8a6"/>
|
||||
<rect x="520.0" y="82.7" width="52.1" height="249.3" rx="3" fill="#f59e0b"/>
|
||||
<text x="488.0" y="72.2" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">9.47</text>
|
||||
<text x="546.0" y="76.7" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">9.30</text>
|
||||
<text x="517.0" y="364" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Faithfulness</text>
|
||||
<rect x="647.9" y="241.0" width="52.1" height="91.0" rx="3" fill="#14b8a6"/>
|
||||
<rect x="706.0" y="133.5" width="52.1" height="198.5" rx="3" fill="#f59e0b"/>
|
||||
<text x="674.0" y="235.0" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">2.2%</text>
|
||||
<text x="732.0" y="127.5" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">4.8%</text>
|
||||
<text x="703.0" y="364" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Hallucination % (lower better)</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.3 KiB |
51
docs/assets/decision/stage2-per-document.svg
Normal file
51
docs/assets/decision/stage2-per-document.svg
Normal file
@@ -0,0 +1,51 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="920" height="460" viewBox="0 0 920 460" role="img" aria-label="Stage 2 per document">
|
||||
<title>Stage 2 per document</title>
|
||||
<rect width="920" height="460" fill="#ffffff"/>
|
||||
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Stage 2 — per document (fixed_size ±3 vs semantic @ large)</text>
|
||||
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Mean composite still favors fixed_size. Semantic wins 6 of 10 docs, but by smaller margins except website/customer1.</text>
|
||||
<rect x="620" y="14" width="12" height="12" rx="2" fill="#14b8a6"/>
|
||||
<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">fixed_size ±3</text>
|
||||
<rect x="760" y="14" width="12" height="12" rx="2" fill="#f59e0b"/>
|
||||
<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">semantic @ large</text>
|
||||
<line x1="52" y1="357.3" x2="896" y2="357.3" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="44" y="361.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">6</text>
|
||||
<line x1="52" y1="284.0" x2="896" y2="284.0" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="44" y="288.0" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">7</text>
|
||||
<line x1="52" y1="210.7" x2="896" y2="210.7" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="44" y="214.7" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8</text>
|
||||
<line x1="52" y1="137.3" x2="896" y2="137.3" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="44" y="141.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">9</text>
|
||||
<line x1="52" y1="64.0" x2="896" y2="64.0" stroke="#e5e7eb" stroke-width="1"/>
|
||||
<text x="44" y="68.0" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">10</text>
|
||||
<rect x="64.2" y="110.5" width="27.0" height="261.5" rx="3" fill="#14b8a6"/>
|
||||
<rect x="97.2" y="107.6" width="27.0" height="264.4" rx="3" fill="#f59e0b"/>
|
||||
<text x="94.2" y="424" text-anchor="end" transform="rotate(-32 94.2 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">bazresi</text>
|
||||
<rect x="148.6" y="177.6" width="27.0" height="194.4" rx="3" fill="#14b8a6"/>
|
||||
<rect x="181.6" y="139.3" width="27.0" height="232.7" rx="3" fill="#f59e0b"/>
|
||||
<text x="178.6" y="424" text-anchor="end" transform="rotate(-32 178.6 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">customer1</text>
|
||||
<rect x="233.0" y="151.3" width="27.0" height="220.7" rx="3" fill="#14b8a6"/>
|
||||
<rect x="266.0" y="340.6" width="27.0" height="31.4" rx="3" fill="#f59e0b"/>
|
||||
<text x="263.0" y="424" text-anchor="end" transform="rotate(-32 263.0 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">fire</text>
|
||||
<rect x="317.4" y="122.3" width="27.0" height="249.7" rx="3" fill="#14b8a6"/>
|
||||
<rect x="350.4" y="169.3" width="27.0" height="202.7" rx="3" fill="#f59e0b"/>
|
||||
<text x="347.4" y="424" text-anchor="end" transform="rotate(-32 347.4 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades-ind</text>
|
||||
<rect x="401.8" y="175.2" width="27.0" height="196.8" rx="3" fill="#14b8a6"/>
|
||||
<rect x="434.8" y="212.9" width="27.0" height="159.1" rx="3" fill="#f59e0b"/>
|
||||
<text x="431.8" y="424" text-anchor="end" transform="rotate(-32 431.8 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades</text>
|
||||
<rect x="486.2" y="104.7" width="27.0" height="267.3" rx="3" fill="#14b8a6"/>
|
||||
<rect x="519.2" y="100.3" width="27.0" height="271.7" rx="3" fill="#f59e0b"/>
|
||||
<text x="516.2" y="424" text-anchor="end" transform="rotate(-32 516.2 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-ind</text>
|
||||
<rect x="570.6" y="83.4" width="27.0" height="288.6" rx="3" fill="#14b8a6"/>
|
||||
<rect x="603.6" y="165.1" width="27.0" height="206.9" rx="3" fill="#f59e0b"/>
|
||||
<text x="600.6" y="424" text-anchor="end" transform="rotate(-32 600.6 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">moavenin</text>
|
||||
<rect x="655.0" y="87.5" width="27.0" height="284.5" rx="3" fill="#14b8a6"/>
|
||||
<rect x="688.0" y="77.6" width="27.0" height="294.4" rx="3" fill="#f59e0b"/>
|
||||
<text x="685.0" y="424" text-anchor="end" transform="rotate(-32 685.0 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">Refah</text>
|
||||
<rect x="739.4" y="176.5" width="27.0" height="195.5" rx="3" fill="#14b8a6"/>
|
||||
<rect x="772.4" y="133.1" width="27.0" height="238.9" rx="3" fill="#f59e0b"/>
|
||||
<text x="769.4" y="424" text-anchor="end" transform="rotate(-32 769.4 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">website</text>
|
||||
<rect x="823.8" y="154.9" width="27.0" height="217.1" rx="3" fill="#14b8a6"/>
|
||||
<rect x="856.8" y="127.9" width="27.0" height="244.1" rx="3" fill="#f59e0b"/>
|
||||
<text x="853.8" y="424" text-anchor="end" transform="rotate(-32 853.8 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-comp</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.1 KiB |
Reference in New Issue
Block a user