docs: record fixed_size as the winning strategy family
Why: - Close the benchmark with a written family decision backed by Experiment composites. Changes: - Add the decision memo and SVG charts generated from SQLite. Impact: - Documents fixed_size over semantic under text-embedding-3-large; ±N remains follow-up. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
458
docs/assets/decision/generate_charts.py
Normal file
458
docs/assets/decision/generate_charts.py
Normal file
@@ -0,0 +1,458 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate SVG charts for the final chunking-strategy decision report.
|
||||||
|
|
||||||
|
Reads Experiment aggregates from SQLite (Decision Board rules) and writes
|
||||||
|
SVG files next to this script. Re-run after new Experiments if needed:
|
||||||
|
|
||||||
|
.venv/bin/python docs/assets/decision/generate_charts.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
DB = ROOT / "data" / "chunking_benchmark.db"
|
||||||
|
OUT = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
DECISION_DOCS = [
|
||||||
|
"bazresi.docx",
|
||||||
|
"customer1.docx",
|
||||||
|
"fire.docx",
|
||||||
|
"general-havades-individuals.doc",
|
||||||
|
"havades.docx",
|
||||||
|
"life-time-individual.docx",
|
||||||
|
"moavenin.docx",
|
||||||
|
"Refah.docx",
|
||||||
|
"website.docx",
|
||||||
|
"lifetime-compensation.docx",
|
||||||
|
]
|
||||||
|
SHORT = {
|
||||||
|
"bazresi.docx": "bazresi",
|
||||||
|
"customer1.docx": "customer1",
|
||||||
|
"fire.docx": "fire",
|
||||||
|
"general-havades-individuals.doc": "havades-ind",
|
||||||
|
"havades.docx": "havades",
|
||||||
|
"life-time-individual.docx": "lifetime-ind",
|
||||||
|
"moavenin.docx": "moavenin",
|
||||||
|
"Refah.docx": "Refah",
|
||||||
|
"website.docx": "website",
|
||||||
|
"lifetime-compensation.docx": "lifetime-comp",
|
||||||
|
}
|
||||||
|
CORPUS = "text-embedding-3-large"
|
||||||
|
|
||||||
|
TEAL_FILL = "#14b8a6"
|
||||||
|
AMBER_FILL = "#f59e0b"
|
||||||
|
INK = "#111827"
|
||||||
|
MUTED = "#6b7280"
|
||||||
|
GRID = "#e5e7eb"
|
||||||
|
BG = "#ffffff"
|
||||||
|
WIN = "#047857"
|
||||||
|
|
||||||
|
|
||||||
|
def esc(s: str) -> str:
|
||||||
|
return (
|
||||||
|
str(s)
|
||||||
|
.replace("&", "&")
|
||||||
|
.replace("<", "<")
|
||||||
|
.replace(">", ">")
|
||||||
|
.replace('"', """)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def composite(m: dict | None) -> float | None:
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
return (
|
||||||
|
(m.get("avg_context_relevance") or 0) * 0.3
|
||||||
|
+ (m.get("avg_answer_similarity") or 0) * 0.4
|
||||||
|
+ (m.get("avg_faithfulness") or 0) * 0.3
|
||||||
|
) * (1 - (m.get("hallucination_rate") or 0))
|
||||||
|
|
||||||
|
|
||||||
|
def load_cells() -> dict[tuple[str, str], dict]:
|
||||||
|
conn = sqlite3.connect(DB)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
docs = {r["id"]: r["filename"] for r in conn.execute("SELECT id, filename FROM documents")}
|
||||||
|
rows = []
|
||||||
|
for e in conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT id, document_id, strategies_used, aggregate_metrics, benchmark_config,
|
||||||
|
embedding_model_id, boundary_embedding_model_id, created_at, questions
|
||||||
|
FROM experiments ORDER BY created_at DESC
|
||||||
|
"""
|
||||||
|
):
|
||||||
|
fn = docs.get(e["document_id"])
|
||||||
|
strats = json.loads(e["strategies_used"] or "[]")
|
||||||
|
agg = json.loads(e["aggregate_metrics"] or "{}")
|
||||||
|
cfg = json.loads(e["benchmark_config"] or "{}")
|
||||||
|
corpus = e["embedding_model_id"] or cfg.get("corpus_embedding_model_id")
|
||||||
|
prev = int(cfg.get("neighbor_prev") or 0)
|
||||||
|
nxt = int(cfg.get("neighbor_next") or 0)
|
||||||
|
bound = e["boundary_embedding_model_id"] or cfg.get("boundary_embedding_model_id")
|
||||||
|
if fn not in DECISION_DOCS or corpus != CORPUS or len(strats) != 1:
|
||||||
|
continue
|
||||||
|
strat = strats[0]
|
||||||
|
if strat == "fixed_size":
|
||||||
|
if prev != nxt or prev not in (0, 1, 2, 3):
|
||||||
|
continue
|
||||||
|
cid = f"fixed_size:±{prev}"
|
||||||
|
family = "fixed_size"
|
||||||
|
elif strat == "semantic":
|
||||||
|
if not bound:
|
||||||
|
continue
|
||||||
|
cid = f"semantic:{bound}"
|
||||||
|
family = "semantic"
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"fn": fn,
|
||||||
|
"cid": cid,
|
||||||
|
"family": family,
|
||||||
|
"agg": agg,
|
||||||
|
"metrics": agg.get(family) or {},
|
||||||
|
"id": e["id"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
conn.close()
|
||||||
|
cells: dict[tuple[str, str], dict] = {}
|
||||||
|
for r in rows:
|
||||||
|
key = (r["fn"], r["cid"])
|
||||||
|
if key not in cells:
|
||||||
|
cells[key] = r
|
||||||
|
return cells
|
||||||
|
|
||||||
|
|
||||||
|
def mean(xs: list[float]) -> float | None:
|
||||||
|
return sum(xs) / len(xs) if xs else None
|
||||||
|
|
||||||
|
|
||||||
|
def svg_wrap(w: int, h: int, body: str, title: str) -> str:
|
||||||
|
return f'''<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}" role="img" aria-label="{esc(title)}">
|
||||||
|
<title>{esc(title)}</title>
|
||||||
|
<rect width="{w}" height="{h}" fill="{BG}"/>
|
||||||
|
{body}
|
||||||
|
</svg>
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def chart_stage1(cells: dict) -> None:
|
||||||
|
cands = [
|
||||||
|
("fixed_size:±3", "fixed_size ±3", TEAL_FILL, True),
|
||||||
|
("fixed_size:±2", "fixed_size ±2", TEAL_FILL, False),
|
||||||
|
("fixed_size:±1", "fixed_size ±1", TEAL_FILL, False),
|
||||||
|
("semantic:text-embedding-3-large", "semantic @ large", AMBER_FILL, False),
|
||||||
|
("fixed_size:±0", "fixed_size ±0", "#99f6e4", False),
|
||||||
|
("semantic:nomic-embed-text-v2-moe", "semantic @ nomic", "#fcd34d", False),
|
||||||
|
]
|
||||||
|
values = []
|
||||||
|
for cid, label, color, winner in cands:
|
||||||
|
scores = []
|
||||||
|
for fn in DECISION_DOCS:
|
||||||
|
r = cells.get((fn, cid))
|
||||||
|
if r:
|
||||||
|
sc = composite(r["metrics"])
|
||||||
|
if sc is not None:
|
||||||
|
scores.append(sc)
|
||||||
|
values.append((cid, label, color, winner, mean(scores) or 0))
|
||||||
|
|
||||||
|
w, h = 820, 420
|
||||||
|
left, right, top, bottom = 210, 40, 56, 48
|
||||||
|
plot_w = w - left - right
|
||||||
|
plot_h = h - top - bottom
|
||||||
|
vmin, vmax = 8.4, 9.2
|
||||||
|
bar_h = plot_h / len(values) * 0.62
|
||||||
|
gap = plot_h / len(values)
|
||||||
|
|
||||||
|
parts = [
|
||||||
|
f'<text x="24" y="32" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 1 — mean composite (10 documents)</text>',
|
||||||
|
f'<text x="24" y="50" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Higher is better. Official ranking used by Decision Board. Winner: fixed_size ±3.</text>',
|
||||||
|
]
|
||||||
|
# grid
|
||||||
|
for tick in [8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2]:
|
||||||
|
x = left + (tick - vmin) / (vmax - vmin) * plot_w
|
||||||
|
parts.append(f'<line x1="{x:.1f}" y1="{top}" x2="{x:.1f}" y2="{h - bottom}" stroke="{GRID}" stroke-width="1"/>')
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{x:.1f}" y="{h - 18}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">{tick:.1f}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
for i, (_, label, color, winner, val) in enumerate(values):
|
||||||
|
y = top + i * gap + (gap - bar_h) / 2
|
||||||
|
bw = (val - vmin) / (vmax - vmin) * plot_w
|
||||||
|
stroke = WIN if winner else "none"
|
||||||
|
sw = 2 if winner else 0
|
||||||
|
parts.append(
|
||||||
|
f'<rect x="{left}" y="{y:.1f}" width="{max(bw, 0):.1f}" height="{bar_h:.1f}" rx="4" fill="{color}" stroke="{stroke}" stroke-width="{sw}"/>'
|
||||||
|
)
|
||||||
|
weight = "700" if winner else "500"
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{left - 10}" y="{y + bar_h * 0.68:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="{weight}" fill="{INK}">{esc(label)}</text>'
|
||||||
|
)
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{left + bw + 8:.1f}" y="{y + bar_h * 0.68:.1f}" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="{INK}">{val:.3f}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
(OUT / "stage1-mean-composite.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 1 mean composite"), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def chart_stage2(cells: dict) -> None:
|
||||||
|
fs_id = "fixed_size:±3"
|
||||||
|
sem_id = "semantic:text-embedding-3-large"
|
||||||
|
pairs = []
|
||||||
|
for fn in DECISION_DOCS:
|
||||||
|
a = composite(cells[(fn, fs_id)]["metrics"])
|
||||||
|
b = composite(cells[(fn, sem_id)]["metrics"])
|
||||||
|
pairs.append((SHORT[fn], a, b))
|
||||||
|
|
||||||
|
w, h = 920, 460
|
||||||
|
left, right, top, bottom = 52, 24, 64, 88
|
||||||
|
plot_w = w - left - right
|
||||||
|
plot_h = h - top - bottom
|
||||||
|
n = len(pairs)
|
||||||
|
slot = plot_w / n
|
||||||
|
bar_w = slot * 0.32
|
||||||
|
vmin, vmax = 5.8, 10.0
|
||||||
|
|
||||||
|
def y_of(v: float) -> float:
|
||||||
|
return top + (1 - (v - vmin) / (vmax - vmin)) * plot_h
|
||||||
|
|
||||||
|
parts = [
|
||||||
|
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 2 — per document (fixed_size ±3 vs semantic @ large)</text>',
|
||||||
|
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Mean composite still favors fixed_size. Semantic wins 6 of 10 docs, but by smaller margins except website/customer1.</text>',
|
||||||
|
f'<rect x="620" y="14" width="12" height="12" rx="2" fill="{TEAL_FILL}"/>',
|
||||||
|
f'<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">fixed_size ±3</text>',
|
||||||
|
f'<rect x="760" y="14" width="12" height="12" rx="2" fill="{AMBER_FILL}"/>',
|
||||||
|
f'<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">semantic @ large</text>',
|
||||||
|
]
|
||||||
|
for tick in [6, 7, 8, 9, 10]:
|
||||||
|
y = y_of(tick)
|
||||||
|
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{w - right}" y2="{y:.1f}" stroke="{GRID}" stroke-width="1"/>')
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{left - 8}" y="{y + 4:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">{tick}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
for i, (label, a, b) in enumerate(pairs):
|
||||||
|
cx = left + i * slot + slot / 2
|
||||||
|
xa = cx - bar_w - 3
|
||||||
|
xb = cx + 3
|
||||||
|
ha = plot_h - (y_of(a) - top)
|
||||||
|
hb = plot_h - (y_of(b) - top)
|
||||||
|
parts.append(f'<rect x="{xa:.1f}" y="{y_of(a):.1f}" width="{bar_w:.1f}" height="{ha:.1f}" rx="3" fill="{TEAL_FILL}"/>')
|
||||||
|
parts.append(f'<rect x="{xb:.1f}" y="{y_of(b):.1f}" width="{bar_w:.1f}" height="{hb:.1f}" rx="3" fill="{AMBER_FILL}"/>')
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{cx:.1f}" y="{h - 36}" text-anchor="end" transform="rotate(-32 {cx:.1f} {h - 36})" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{INK}">{esc(label)}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
(OUT / "stage2-per-document.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 per document"), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def chart_delta(cells: dict) -> None:
|
||||||
|
fs_id = "fixed_size:±3"
|
||||||
|
sem_id = "semantic:text-embedding-3-large"
|
||||||
|
deltas = []
|
||||||
|
for fn in DECISION_DOCS:
|
||||||
|
a = composite(cells[(fn, fs_id)]["metrics"])
|
||||||
|
b = composite(cells[(fn, sem_id)]["metrics"])
|
||||||
|
deltas.append((SHORT[fn], a - b))
|
||||||
|
deltas.sort(key=lambda x: x[1], reverse=True)
|
||||||
|
|
||||||
|
w, h = 820, 440
|
||||||
|
left, right, top, bottom = 120, 56, 56, 36
|
||||||
|
plot_w = w - left - right
|
||||||
|
plot_h = h - top - bottom
|
||||||
|
n = len(deltas)
|
||||||
|
gap = plot_h / n
|
||||||
|
bar_h = gap * 0.62
|
||||||
|
max_abs = max(abs(d) for _, d in deltas)
|
||||||
|
# fire is 2.58, others < 1.2 — use 2.8
|
||||||
|
max_abs = 2.8
|
||||||
|
zero_x = left + plot_w / 2
|
||||||
|
|
||||||
|
parts = [
|
||||||
|
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Margin: fixed_size ±3 minus semantic @ large</text>',
|
||||||
|
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Teal = fixed_size wins the document. Amber = semantic wins. Mean ranking is driven by large teal bars (especially fire).</text>',
|
||||||
|
f'<line x1="{zero_x:.1f}" y1="{top}" x2="{zero_x:.1f}" y2="{h - bottom}" stroke="{INK}" stroke-width="1.2"/>',
|
||||||
|
f'<text x="{left}" y="{h - 12}" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">semantic better</text>',
|
||||||
|
f'<text x="{w - right}" y="{h - 12}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">fixed_size better</text>',
|
||||||
|
]
|
||||||
|
for i, (label, d) in enumerate(deltas):
|
||||||
|
y = top + i * gap + (gap - bar_h) / 2
|
||||||
|
bw = abs(d) / max_abs * (plot_w / 2)
|
||||||
|
if d >= 0:
|
||||||
|
x = zero_x
|
||||||
|
color = TEAL_FILL
|
||||||
|
tx = x + bw + 6
|
||||||
|
anchor = "start"
|
||||||
|
else:
|
||||||
|
x = zero_x - bw
|
||||||
|
color = AMBER_FILL
|
||||||
|
tx = x - 6
|
||||||
|
anchor = "end"
|
||||||
|
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bw:.1f}" height="{bar_h:.1f}" rx="3" fill="{color}"/>')
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{left - 8}" y="{y + bar_h * 0.7:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">{esc(label)}</text>'
|
||||||
|
)
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{tx:.1f}" y="{y + bar_h * 0.7:.1f}" text-anchor="{anchor}" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="{INK}">{d:+.2f}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
(OUT / "stage2-margins.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 margins"), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def chart_heatmap(cells: dict) -> None:
|
||||||
|
cands = [
|
||||||
|
("fixed_size:±0", "±0"),
|
||||||
|
("fixed_size:±1", "±1"),
|
||||||
|
("fixed_size:±2", "±2"),
|
||||||
|
("fixed_size:±3", "±3"),
|
||||||
|
("semantic:text-embedding-3-large", "sem@large"),
|
||||||
|
("semantic:nomic-embed-text-v2-moe", "sem@nomic"),
|
||||||
|
]
|
||||||
|
scores: list[list[float | None]] = []
|
||||||
|
for fn in DECISION_DOCS:
|
||||||
|
row = []
|
||||||
|
for cid, _ in cands:
|
||||||
|
r = cells.get((fn, cid))
|
||||||
|
row.append(composite(r["metrics"]) if r else None)
|
||||||
|
scores.append(row)
|
||||||
|
|
||||||
|
cell_w, cell_h = 78, 32
|
||||||
|
left, top = 128, 72
|
||||||
|
w = left + cell_w * len(cands) + 24
|
||||||
|
h = top + cell_h * len(DECISION_DOCS) + 36
|
||||||
|
|
||||||
|
def color_for(s: float | None) -> tuple[str, str]:
|
||||||
|
if s is None:
|
||||||
|
return "#f3f4f6", MUTED
|
||||||
|
# 6.2 .. 9.8
|
||||||
|
t = max(0.0, min(1.0, (s - 7.0) / (9.8 - 7.0)))
|
||||||
|
# pale rose -> amber -> teal
|
||||||
|
if t < 0.5:
|
||||||
|
u = t * 2
|
||||||
|
r = int(251 + (20 - 251) * 0)
|
||||||
|
# interpolate rose 251,113,133 -> amber 245,158,11
|
||||||
|
rr = int(251 + (245 - 251) * u)
|
||||||
|
gg = int(113 + (158 - 113) * u)
|
||||||
|
bb = int(133 + (11 - 133) * u)
|
||||||
|
else:
|
||||||
|
u = (t - 0.5) * 2
|
||||||
|
rr = int(245 + (13 - 245) * u)
|
||||||
|
gg = int(158 + (148 - 158) * u)
|
||||||
|
bb = int(11 + (136 - 11) * u)
|
||||||
|
bg = f"#{rr:02x}{gg:02x}{bb:02x}"
|
||||||
|
fg = "#111827" if t < 0.72 else "#f9fafb"
|
||||||
|
return bg, fg
|
||||||
|
|
||||||
|
parts = [
|
||||||
|
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Candidate heatmap — composite by document</text>',
|
||||||
|
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set.</text>',
|
||||||
|
]
|
||||||
|
for j, (_, lab) in enumerate(cands):
|
||||||
|
x = left + j * cell_w + cell_w / 2
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{x:.1f}" y="{top - 10}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{esc(lab)}</text>'
|
||||||
|
)
|
||||||
|
for i, fn in enumerate(DECISION_DOCS):
|
||||||
|
y = top + i * cell_h
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{left - 8}" y="{y + cell_h * 0.65:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{INK}">{esc(SHORT[fn])}</text>'
|
||||||
|
)
|
||||||
|
for j, val in enumerate(scores[i]):
|
||||||
|
x = left + j * cell_w
|
||||||
|
bg, fg = color_for(val)
|
||||||
|
parts.append(f'<rect x="{x}" y="{y}" width="{cell_w - 3}" height="{cell_h - 3}" rx="4" fill="{bg}"/>')
|
||||||
|
txt = "—" if val is None else f"{val:.2f}"
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{x + (cell_w - 3) / 2:.1f}" y="{y + cell_h * 0.62:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{fg}">{txt}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
(OUT / "heatmap-candidates.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Candidate heatmap"), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def chart_metrics(cells: dict) -> None:
|
||||||
|
"""Grouped bars: four judge metrics for the two stage-2 winners."""
|
||||||
|
def bag(cid: str, family: str) -> dict[str, float]:
|
||||||
|
keys = [
|
||||||
|
"avg_context_relevance",
|
||||||
|
"avg_answer_similarity",
|
||||||
|
"avg_faithfulness",
|
||||||
|
"hallucination_rate",
|
||||||
|
]
|
||||||
|
acc = {k: [] for k in keys}
|
||||||
|
for fn in DECISION_DOCS:
|
||||||
|
m = cells[(fn, cid)]["metrics"]
|
||||||
|
for k in keys:
|
||||||
|
acc[k].append(m[k])
|
||||||
|
return {k: sum(v) / len(v) for k, v in acc.items()}
|
||||||
|
|
||||||
|
fs = bag("fixed_size:±3", "fixed_size")
|
||||||
|
sem = bag("semantic:text-embedding-3-large", "semantic")
|
||||||
|
labels = [
|
||||||
|
("Context relevance", fs["avg_context_relevance"], sem["avg_context_relevance"], False),
|
||||||
|
("Answer similarity", fs["avg_answer_similarity"], sem["avg_answer_similarity"], False),
|
||||||
|
("Faithfulness", fs["avg_faithfulness"], sem["avg_faithfulness"], False),
|
||||||
|
("Hallucination % (lower better)", fs["hallucination_rate"] * 100, sem["hallucination_rate"] * 100, True),
|
||||||
|
]
|
||||||
|
|
||||||
|
w, h = 820, 380
|
||||||
|
left, right, top, bottom = 52, 24, 64, 48
|
||||||
|
plot_w = w - left - right
|
||||||
|
plot_h = h - top - bottom
|
||||||
|
n = len(labels)
|
||||||
|
slot = plot_w / n
|
||||||
|
bar_w = slot * 0.28
|
||||||
|
|
||||||
|
parts = [
|
||||||
|
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 2 winners — mean judge metrics</text>',
|
||||||
|
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">fixed_size ±3 leads on all four metrics, including lower hallucination.</text>',
|
||||||
|
f'<rect x="620" y="14" width="12" height="12" rx="2" fill="{TEAL_FILL}"/>',
|
||||||
|
f'<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">fixed_size ±3</text>',
|
||||||
|
f'<rect x="760" y="14" width="12" height="12" rx="2" fill="{AMBER_FILL}"/>',
|
||||||
|
f'<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">semantic @ large</text>',
|
||||||
|
]
|
||||||
|
|
||||||
|
for i, (lab, a, b, is_pct) in enumerate(labels):
|
||||||
|
vmax = 10 if not is_pct else max(a, b) * 1.35
|
||||||
|
cx = left + i * slot + slot / 2
|
||||||
|
xa = cx - bar_w - 3
|
||||||
|
xb = cx + 3
|
||||||
|
|
||||||
|
def bar(x: float, val: float, color: str) -> str:
|
||||||
|
bh = val / vmax * plot_h
|
||||||
|
y = top + plot_h - bh
|
||||||
|
return f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_w:.1f}" height="{bh:.1f}" rx="3" fill="{color}"/>'
|
||||||
|
|
||||||
|
parts.append(bar(xa, a, TEAL_FILL))
|
||||||
|
parts.append(bar(xb, b, AMBER_FILL))
|
||||||
|
fmt = (lambda v: f"{v:.1f}%") if is_pct else (lambda v: f"{v:.2f}")
|
||||||
|
ya = top + plot_h - a / vmax * plot_h - 6
|
||||||
|
yb = top + plot_h - b / vmax * plot_h - 6
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{xa + bar_w / 2:.1f}" y="{ya:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{fmt(a)}</text>'
|
||||||
|
)
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{xb + bar_w / 2:.1f}" y="{yb:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{fmt(b)}</text>'
|
||||||
|
)
|
||||||
|
parts.append(
|
||||||
|
f'<text x="{cx:.1f}" y="{h - 16}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">{esc(lab)}</text>'
|
||||||
|
)
|
||||||
|
|
||||||
|
(OUT / "stage2-metrics.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 mean metrics"), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
cells = load_cells()
|
||||||
|
chart_stage1(cells)
|
||||||
|
chart_stage2(cells)
|
||||||
|
chart_delta(cells)
|
||||||
|
chart_heatmap(cells)
|
||||||
|
chart_metrics(cells)
|
||||||
|
print(f"Wrote SVGs in {OUT}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
143
docs/assets/decision/heatmap-candidates.svg
Normal file
143
docs/assets/decision/heatmap-candidates.svg
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="620" height="428" viewBox="0 0 620 428" role="img" aria-label="Candidate heatmap">
|
||||||
|
<title>Candidate heatmap</title>
|
||||||
|
<rect width="620" height="428" fill="#ffffff"/>
|
||||||
|
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Candidate heatmap — composite by document</text>
|
||||||
|
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set.</text>
|
||||||
|
<text x="167.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±0</text>
|
||||||
|
<text x="245.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±1</text>
|
||||||
|
<text x="323.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±2</text>
|
||||||
|
<text x="401.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±3</text>
|
||||||
|
<text x="479.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">sem@large</text>
|
||||||
|
<text x="557.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">sem@nomic</text>
|
||||||
|
<text x="120" y="92.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">bazresi</text>
|
||||||
|
<rect x="128" y="72" width="75" height="29" rx="4" fill="#e19d15"/>
|
||||||
|
<text x="165.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.52</text>
|
||||||
|
<rect x="206" y="72" width="75" height="29" rx="4" fill="#58975f"/>
|
||||||
|
<text x="243.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.34</text>
|
||||||
|
<rect x="284" y="72" width="75" height="29" rx="4" fill="#58975f"/>
|
||||||
|
<text x="321.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.34</text>
|
||||||
|
<rect x="362" y="72" width="75" height="29" rx="4" fill="#549761"/>
|
||||||
|
<text x="399.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.37</text>
|
||||||
|
<rect x="440" y="72" width="75" height="29" rx="4" fill="#4e9664"/>
|
||||||
|
<text x="477.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.40</text>
|
||||||
|
<rect x="518" y="72" width="75" height="29" rx="4" fill="#7c984b"/>
|
||||||
|
<text x="555.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.12</text>
|
||||||
|
<text x="120" y="124.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">customer1</text>
|
||||||
|
<rect x="128" y="104" width="75" height="29" rx="4" fill="#9f9a39"/>
|
||||||
|
<text x="165.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.92</text>
|
||||||
|
<rect x="206" y="104" width="75" height="29" rx="4" fill="#96993e"/>
|
||||||
|
<text x="243.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.97</text>
|
||||||
|
<rect x="284" y="104" width="75" height="29" rx="4" fill="#97993d"/>
|
||||||
|
<text x="321.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.96</text>
|
||||||
|
<rect x="362" y="104" width="75" height="29" rx="4" fill="#ec9d0f"/>
|
||||||
|
<text x="399.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.45</text>
|
||||||
|
<rect x="440" y="104" width="75" height="29" rx="4" fill="#96993e"/>
|
||||||
|
<text x="477.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.97</text>
|
||||||
|
<rect x="518" y="104" width="75" height="29" rx="4" fill="#f88255"/>
|
||||||
|
<text x="555.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.54</text>
|
||||||
|
<text x="120" y="156.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">fire</text>
|
||||||
|
<rect x="128" y="136" width="75" height="29" rx="4" fill="#f69426"/>
|
||||||
|
<text x="165.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.09</text>
|
||||||
|
<rect x="206" y="136" width="75" height="29" rx="4" fill="#f59c10"/>
|
||||||
|
<text x="243.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.34</text>
|
||||||
|
<rect x="284" y="136" width="75" height="29" rx="4" fill="#f68f33"/>
|
||||||
|
<text x="321.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.93</text>
|
||||||
|
<rect x="362" y="136" width="75" height="29" rx="4" fill="#b19b2f"/>
|
||||||
|
<text x="399.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.81</text>
|
||||||
|
<rect x="440" y="136" width="75" height="29" rx="4" fill="#fb7185"/>
|
||||||
|
<text x="477.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">6.23</text>
|
||||||
|
<rect x="518" y="136" width="75" height="29" rx="4" fill="#fa7674"/>
|
||||||
|
<text x="555.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.19</text>
|
||||||
|
<text x="120" y="188.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades-ind</text>
|
||||||
|
<rect x="128" y="168" width="75" height="29" rx="4" fill="#f78b3c"/>
|
||||||
|
<text x="165.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.83</text>
|
||||||
|
<rect x="206" y="168" width="75" height="29" rx="4" fill="#c99c22"/>
|
||||||
|
<text x="243.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.66</text>
|
||||||
|
<rect x="284" y="168" width="75" height="29" rx="4" fill="#7e984a"/>
|
||||||
|
<text x="321.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.12</text>
|
||||||
|
<rect x="362" y="168" width="75" height="29" rx="4" fill="#6f9852"/>
|
||||||
|
<text x="399.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.21</text>
|
||||||
|
<rect x="440" y="168" width="75" height="29" rx="4" fill="#d99c19"/>
|
||||||
|
<text x="477.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.56</text>
|
||||||
|
<rect x="518" y="168" width="75" height="29" rx="4" fill="#e09d16"/>
|
||||||
|
<text x="555.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.53</text>
|
||||||
|
<text x="120" y="220.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades</text>
|
||||||
|
<rect x="128" y="200" width="75" height="29" rx="4" fill="#f59b12"/>
|
||||||
|
<text x="165.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.31</text>
|
||||||
|
<rect x="206" y="200" width="75" height="29" rx="4" fill="#ef9d0d"/>
|
||||||
|
<text x="243.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.43</text>
|
||||||
|
<rect x="284" y="200" width="75" height="29" rx="4" fill="#f49d0b"/>
|
||||||
|
<text x="321.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.40</text>
|
||||||
|
<rect x="362" y="200" width="75" height="29" rx="4" fill="#e79d12"/>
|
||||||
|
<text x="399.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.48</text>
|
||||||
|
<rect x="440" y="200" width="75" height="29" rx="4" fill="#f69030"/>
|
||||||
|
<text x="477.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.97</text>
|
||||||
|
<rect x="518" y="200" width="75" height="29" rx="4" fill="#f29d0c"/>
|
||||||
|
<text x="555.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.42</text>
|
||||||
|
<text x="120" y="252.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-ind</text>
|
||||||
|
<rect x="128" y="232" width="75" height="29" rx="4" fill="#6d9854"/>
|
||||||
|
<text x="165.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.22</text>
|
||||||
|
<rect x="206" y="232" width="75" height="29" rx="4" fill="#4b9666"/>
|
||||||
|
<text x="243.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.42</text>
|
||||||
|
<rect x="284" y="232" width="75" height="29" rx="4" fill="#489667"/>
|
||||||
|
<text x="321.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.44</text>
|
||||||
|
<rect x="362" y="232" width="75" height="29" rx="4" fill="#479668"/>
|
||||||
|
<text x="399.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.45</text>
|
||||||
|
<rect x="440" y="232" width="75" height="29" rx="4" fill="#3d966d"/>
|
||||||
|
<text x="477.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.50</text>
|
||||||
|
<rect x="518" y="232" width="75" height="29" rx="4" fill="#4e9664"/>
|
||||||
|
<text x="555.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.41</text>
|
||||||
|
<text x="120" y="284.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">moavenin</text>
|
||||||
|
<rect x="128" y="264" width="75" height="29" rx="4" fill="#879945"/>
|
||||||
|
<text x="165.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.06</text>
|
||||||
|
<rect x="206" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||||||
|
<text x="243.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||||||
|
<rect x="284" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||||||
|
<text x="321.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||||||
|
<rect x="362" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||||||
|
<text x="399.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||||||
|
<rect x="440" y="264" width="75" height="29" rx="4" fill="#d09c1e"/>
|
||||||
|
<text x="477.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.62</text>
|
||||||
|
<rect x="518" y="264" width="75" height="29" rx="4" fill="#f59a14"/>
|
||||||
|
<text x="555.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.29</text>
|
||||||
|
<text x="120" y="316.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">Refah</text>
|
||||||
|
<rect x="128" y="296" width="75" height="29" rx="4" fill="#459669"/>
|
||||||
|
<text x="165.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.46</text>
|
||||||
|
<rect x="206" y="296" width="75" height="29" rx="4" fill="#1c947f"/>
|
||||||
|
<text x="243.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.71</text>
|
||||||
|
<rect x="284" y="296" width="75" height="29" rx="4" fill="#1e947e"/>
|
||||||
|
<text x="321.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.70</text>
|
||||||
|
<rect x="362" y="296" width="75" height="29" rx="4" fill="#20947d"/>
|
||||||
|
<text x="399.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.68</text>
|
||||||
|
<rect x="440" y="296" width="75" height="29" rx="4" fill="#0d9488"/>
|
||||||
|
<text x="477.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.81</text>
|
||||||
|
<rect x="518" y="296" width="75" height="29" rx="4" fill="#349572"/>
|
||||||
|
<text x="555.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.56</text>
|
||||||
|
<text x="120" y="348.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">website</text>
|
||||||
|
<rect x="128" y="328" width="75" height="29" rx="4" fill="#f8864b"/>
|
||||||
|
<text x="165.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.67</text>
|
||||||
|
<rect x="206" y="328" width="75" height="29" rx="4" fill="#f78e35"/>
|
||||||
|
<text x="243.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.91</text>
|
||||||
|
<rect x="284" y="328" width="75" height="29" rx="4" fill="#e29d15"/>
|
||||||
|
<text x="321.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.51</text>
|
||||||
|
<rect x="362" y="328" width="75" height="29" rx="4" fill="#ea9d10"/>
|
||||||
|
<text x="399.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.47</text>
|
||||||
|
<rect x="440" y="328" width="75" height="29" rx="4" fill="#879945"/>
|
||||||
|
<text x="477.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.06</text>
|
||||||
|
<rect x="518" y="328" width="75" height="29" rx="4" fill="#e59d13"/>
|
||||||
|
<text x="555.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.49</text>
|
||||||
|
<text x="120" y="380.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-comp</text>
|
||||||
|
<rect x="128" y="360" width="75" height="29" rx="4" fill="#93993f"/>
|
||||||
|
<text x="165.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.99</text>
|
||||||
|
<rect x="206" y="360" width="75" height="29" rx="4" fill="#bd9b28"/>
|
||||||
|
<text x="243.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.73</text>
|
||||||
|
<rect x="284" y="360" width="75" height="29" rx="4" fill="#bd9b28"/>
|
||||||
|
<text x="321.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.73</text>
|
||||||
|
<rect x="362" y="360" width="75" height="29" rx="4" fill="#b99b2b"/>
|
||||||
|
<text x="399.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.76</text>
|
||||||
|
<rect x="440" y="360" width="75" height="29" rx="4" fill="#7c984c"/>
|
||||||
|
<text x="477.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.13</text>
|
||||||
|
<rect x="518" y="360" width="75" height="29" rx="4" fill="#98993d"/>
|
||||||
|
<text x="555.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.96</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 16 KiB |
41
docs/assets/decision/stage1-mean-composite.svg
Normal file
41
docs/assets/decision/stage1-mean-composite.svg
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="820" height="420" viewBox="0 0 820 420" role="img" aria-label="Stage 1 mean composite">
|
||||||
|
<title>Stage 1 mean composite</title>
|
||||||
|
<rect width="820" height="420" fill="#ffffff"/>
|
||||||
|
<text x="24" y="32" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Stage 1 — mean composite (10 documents)</text>
|
||||||
|
<text x="24" y="50" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Higher is better. Official ranking used by Decision Board. Winner: fixed_size ±3.</text>
|
||||||
|
<line x1="281.2" y1="56" x2="281.2" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="281.2" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.5</text>
|
||||||
|
<line x1="352.5" y1="56" x2="352.5" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="352.5" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.6</text>
|
||||||
|
<line x1="423.7" y1="56" x2="423.7" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="423.7" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.7</text>
|
||||||
|
<line x1="495.0" y1="56" x2="495.0" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="495.0" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.8</text>
|
||||||
|
<line x1="566.3" y1="56" x2="566.3" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="566.3" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8.9</text>
|
||||||
|
<line x1="637.5" y1="56" x2="637.5" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="637.5" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">9.0</text>
|
||||||
|
<line x1="708.8" y1="56" x2="708.8" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="708.8" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">9.1</text>
|
||||||
|
<line x1="780.0" y1="56" x2="780.0" y2="372" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="780.0" y="402" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">9.2</text>
|
||||||
|
<rect x="210" y="66.0" width="456.2" height="32.7" rx="4" fill="#14b8a6" stroke="#047857" stroke-width="2"/>
|
||||||
|
<text x="200" y="88.2" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="700" fill="#111827">fixed_size ±3</text>
|
||||||
|
<text x="674.2" y="88.2" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">9.040</text>
|
||||||
|
<rect x="210" y="118.7" width="418.4" height="32.7" rx="4" fill="#14b8a6" stroke="none" stroke-width="0"/>
|
||||||
|
<text x="200" y="140.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">fixed_size ±2</text>
|
||||||
|
<text x="636.4" y="140.9" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.987</text>
|
||||||
|
<rect x="210" y="171.3" width="374.8" height="32.7" rx="4" fill="#14b8a6" stroke="none" stroke-width="0"/>
|
||||||
|
<text x="200" y="193.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">fixed_size ±1</text>
|
||||||
|
<text x="592.8" y="193.5" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.926</text>
|
||||||
|
<rect x="210" y="224.0" width="232.8" height="32.7" rx="4" fill="#f59e0b" stroke="none" stroke-width="0"/>
|
||||||
|
<text x="200" y="246.2" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">semantic @ large</text>
|
||||||
|
<text x="450.8" y="246.2" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.727</text>
|
||||||
|
<rect x="210" y="276.7" width="146.7" height="32.7" rx="4" fill="#99f6e4" stroke="none" stroke-width="0"/>
|
||||||
|
<text x="200" y="298.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">fixed_size ±0</text>
|
||||||
|
<text x="364.7" y="298.9" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.606</text>
|
||||||
|
<rect x="210" y="329.3" width="107.4" height="32.7" rx="4" fill="#fcd34d" stroke="none" stroke-width="0"/>
|
||||||
|
<text x="200" y="351.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="500" fill="#111827">semantic @ nomic</text>
|
||||||
|
<text x="325.4" y="351.5" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="#111827">8.551</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 4.5 KiB |
40
docs/assets/decision/stage2-margins.svg
Normal file
40
docs/assets/decision/stage2-margins.svg
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="820" height="440" viewBox="0 0 820 440" role="img" aria-label="Stage 2 margins">
|
||||||
|
<title>Stage 2 margins</title>
|
||||||
|
<rect width="820" height="440" fill="#ffffff"/>
|
||||||
|
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Margin: fixed_size ±3 minus semantic @ large</text>
|
||||||
|
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Teal = fixed_size wins the document. Amber = semantic wins. Mean ranking is driven by large teal bars (especially fire).</text>
|
||||||
|
<line x1="442.0" y1="56" x2="442.0" y2="404" stroke="#111827" stroke-width="1.2"/>
|
||||||
|
<text x="120" y="428" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">semantic better</text>
|
||||||
|
<text x="764" y="428" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">fixed_size better</text>
|
||||||
|
<rect x="442.0" y="62.6" width="296.9" height="21.6" rx="3" fill="#14b8a6"/>
|
||||||
|
<text x="112" y="77.7" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">fire</text>
|
||||||
|
<text x="744.9" y="77.7" text-anchor="start" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">+2.58</text>
|
||||||
|
<rect x="442.0" y="97.4" width="128.1" height="21.6" rx="3" fill="#14b8a6"/>
|
||||||
|
<text x="112" y="112.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">moavenin</text>
|
||||||
|
<text x="576.1" y="112.5" text-anchor="start" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">+1.11</text>
|
||||||
|
<rect x="442.0" y="132.2" width="73.7" height="21.6" rx="3" fill="#14b8a6"/>
|
||||||
|
<text x="112" y="147.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">havades-ind</text>
|
||||||
|
<text x="521.7" y="147.3" text-anchor="start" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">+0.64</text>
|
||||||
|
<rect x="442.0" y="167.0" width="59.1" height="21.6" rx="3" fill="#14b8a6"/>
|
||||||
|
<text x="112" y="182.1" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">havades</text>
|
||||||
|
<text x="507.1" y="182.1" text-anchor="start" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">+0.51</text>
|
||||||
|
<rect x="437.5" y="201.8" width="4.5" height="21.6" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="112" y="216.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">bazresi</text>
|
||||||
|
<text x="431.5" y="216.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.04</text>
|
||||||
|
<rect x="435.1" y="236.6" width="6.9" height="21.6" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="112" y="251.7" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">lifetime-ind</text>
|
||||||
|
<text x="429.1" y="251.7" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.06</text>
|
||||||
|
<rect x="426.5" y="271.4" width="15.5" height="21.6" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="112" y="286.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Refah</text>
|
||||||
|
<text x="420.5" y="286.5" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.13</text>
|
||||||
|
<rect x="399.7" y="306.2" width="42.3" height="21.6" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="112" y="321.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">lifetime-comp</text>
|
||||||
|
<text x="393.7" y="321.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.37</text>
|
||||||
|
<rect x="382.0" y="341.0" width="60.0" height="21.6" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="112" y="356.1" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">customer1</text>
|
||||||
|
<text x="376.0" y="356.1" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.52</text>
|
||||||
|
<rect x="373.9" y="375.8" width="68.1" height="21.6" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="112" y="390.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">website</text>
|
||||||
|
<text x="367.9" y="390.9" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="#111827">-0.59</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 4.4 KiB |
31
docs/assets/decision/stage2-metrics.svg
Normal file
31
docs/assets/decision/stage2-metrics.svg
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="820" height="380" viewBox="0 0 820 380" role="img" aria-label="Stage 2 mean metrics">
|
||||||
|
<title>Stage 2 mean metrics</title>
|
||||||
|
<rect width="820" height="380" fill="#ffffff"/>
|
||||||
|
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Stage 2 winners — mean judge metrics</text>
|
||||||
|
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">fixed_size ±3 leads on all four metrics, including lower hallucination.</text>
|
||||||
|
<rect x="620" y="14" width="12" height="12" rx="2" fill="#14b8a6"/>
|
||||||
|
<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">fixed_size ±3</text>
|
||||||
|
<rect x="760" y="14" width="12" height="12" rx="2" fill="#f59e0b"/>
|
||||||
|
<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">semantic @ large</text>
|
||||||
|
<rect x="89.9" y="80.0" width="52.1" height="252.0" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="148.0" y="80.7" width="52.1" height="251.3" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="116.0" y="74.0" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">9.40</text>
|
||||||
|
<text x="174.0" y="74.7" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">9.38</text>
|
||||||
|
<text x="145.0" y="364" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Context relevance</text>
|
||||||
|
<rect x="275.9" y="92.1" width="52.1" height="239.9" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="334.0" y="95.3" width="52.1" height="236.7" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="302.0" y="86.1" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.95</text>
|
||||||
|
<text x="360.0" y="89.3" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.83</text>
|
||||||
|
<text x="331.0" y="364" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Answer similarity</text>
|
||||||
|
<rect x="461.9" y="78.2" width="52.1" height="253.8" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="520.0" y="82.7" width="52.1" height="249.3" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="488.0" y="72.2" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">9.47</text>
|
||||||
|
<text x="546.0" y="76.7" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">9.30</text>
|
||||||
|
<text x="517.0" y="364" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Faithfulness</text>
|
||||||
|
<rect x="647.9" y="241.0" width="52.1" height="91.0" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="706.0" y="133.5" width="52.1" height="198.5" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="674.0" y="235.0" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">2.2%</text>
|
||||||
|
<text x="732.0" y="127.5" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">4.8%</text>
|
||||||
|
<text x="703.0" y="364" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">Hallucination % (lower better)</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 3.3 KiB |
51
docs/assets/decision/stage2-per-document.svg
Normal file
51
docs/assets/decision/stage2-per-document.svg
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="920" height="460" viewBox="0 0 920 460" role="img" aria-label="Stage 2 per document">
|
||||||
|
<title>Stage 2 per document</title>
|
||||||
|
<rect width="920" height="460" fill="#ffffff"/>
|
||||||
|
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Stage 2 — per document (fixed_size ±3 vs semantic @ large)</text>
|
||||||
|
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Mean composite still favors fixed_size. Semantic wins 6 of 10 docs, but by smaller margins except website/customer1.</text>
|
||||||
|
<rect x="620" y="14" width="12" height="12" rx="2" fill="#14b8a6"/>
|
||||||
|
<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">fixed_size ±3</text>
|
||||||
|
<rect x="760" y="14" width="12" height="12" rx="2" fill="#f59e0b"/>
|
||||||
|
<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#111827">semantic @ large</text>
|
||||||
|
<line x1="52" y1="357.3" x2="896" y2="357.3" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="44" y="361.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">6</text>
|
||||||
|
<line x1="52" y1="284.0" x2="896" y2="284.0" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="44" y="288.0" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">7</text>
|
||||||
|
<line x1="52" y1="210.7" x2="896" y2="210.7" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="44" y="214.7" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">8</text>
|
||||||
|
<line x1="52" y1="137.3" x2="896" y2="137.3" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="44" y="141.3" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">9</text>
|
||||||
|
<line x1="52" y1="64.0" x2="896" y2="64.0" stroke="#e5e7eb" stroke-width="1"/>
|
||||||
|
<text x="44" y="68.0" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#6b7280">10</text>
|
||||||
|
<rect x="64.2" y="110.5" width="27.0" height="261.5" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="97.2" y="107.6" width="27.0" height="264.4" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="94.2" y="424" text-anchor="end" transform="rotate(-32 94.2 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">bazresi</text>
|
||||||
|
<rect x="148.6" y="177.6" width="27.0" height="194.4" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="181.6" y="139.3" width="27.0" height="232.7" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="178.6" y="424" text-anchor="end" transform="rotate(-32 178.6 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">customer1</text>
|
||||||
|
<rect x="233.0" y="151.3" width="27.0" height="220.7" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="266.0" y="340.6" width="27.0" height="31.4" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="263.0" y="424" text-anchor="end" transform="rotate(-32 263.0 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">fire</text>
|
||||||
|
<rect x="317.4" y="122.3" width="27.0" height="249.7" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="350.4" y="169.3" width="27.0" height="202.7" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="347.4" y="424" text-anchor="end" transform="rotate(-32 347.4 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades-ind</text>
|
||||||
|
<rect x="401.8" y="175.2" width="27.0" height="196.8" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="434.8" y="212.9" width="27.0" height="159.1" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="431.8" y="424" text-anchor="end" transform="rotate(-32 431.8 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades</text>
|
||||||
|
<rect x="486.2" y="104.7" width="27.0" height="267.3" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="519.2" y="100.3" width="27.0" height="271.7" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="516.2" y="424" text-anchor="end" transform="rotate(-32 516.2 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-ind</text>
|
||||||
|
<rect x="570.6" y="83.4" width="27.0" height="288.6" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="603.6" y="165.1" width="27.0" height="206.9" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="600.6" y="424" text-anchor="end" transform="rotate(-32 600.6 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">moavenin</text>
|
||||||
|
<rect x="655.0" y="87.5" width="27.0" height="284.5" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="688.0" y="77.6" width="27.0" height="294.4" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="685.0" y="424" text-anchor="end" transform="rotate(-32 685.0 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">Refah</text>
|
||||||
|
<rect x="739.4" y="176.5" width="27.0" height="195.5" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="772.4" y="133.1" width="27.0" height="238.9" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="769.4" y="424" text-anchor="end" transform="rotate(-32 769.4 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">website</text>
|
||||||
|
<rect x="823.8" y="154.9" width="27.0" height="217.1" rx="3" fill="#14b8a6"/>
|
||||||
|
<rect x="856.8" y="127.9" width="27.0" height="244.1" rx="3" fill="#f59e0b"/>
|
||||||
|
<text x="853.8" y="424" text-anchor="end" transform="rotate(-32 853.8 424)" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-comp</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 5.1 KiB |
149
docs/final-chunking-strategy-decision.md
Normal file
149
docs/final-chunking-strategy-decision.md
Normal file
@@ -0,0 +1,149 @@
|
|||||||
|
# Chunking Strategy — Final Decision
|
||||||
|
|
||||||
|
**Date:** 17 August 2026
|
||||||
|
**Decision:** Adopt **`fixed_size`** as the chunking Strategy family.
|
||||||
|
**Not decided here:** Neighbor Expansion level **±N** (follow-up).
|
||||||
|
**Corpus Embedding Model:** `text-embedding-3-large`
|
||||||
|
**Evaluation set:** 10 Word documents (Decision Board / neighbor-sweep universe)
|
||||||
|
**Questions:** 201 per Candidate (nomic semantic: 195)
|
||||||
|
|
||||||
|
**One-line close:** **`fixed_size` wins** on official mean composite (**9.040** vs **8.727**). Pick **±N later**, but **not ±0**.
|
||||||
|
|
||||||
|
Charts below are generated from `data/chunking_benchmark.db` with the same composite as the Decision Board:
|
||||||
|
|
||||||
|
```text
|
||||||
|
(0.3 × context relevance + 0.4 × answer similarity + 0.3 × faithfulness)
|
||||||
|
× (1 − hallucination rate)
|
||||||
|
```
|
||||||
|
|
||||||
|
Regenerate figures after new Experiments:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/bin/python docs/assets/decision/generate_charts.py
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Why `fixed_size` is the winner
|
||||||
|
|
||||||
|
Decision Board ranks by **mean composite**, not by “how many documents won.” Stage 1 auto-picks **`fixed_size ±3`** vs **`semantic @ text-embedding-3-large`**. Stage 2 mean composite is **+0.313** for `fixed_size`.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
| Candidate | Mean composite | Context | Similarity | Faithfulness | Hallucination | Docs |
|
||||||
|
|-----------|----------------|---------|------------|--------------|---------------|------|
|
||||||
|
| **fixed_size ±3** | **9.040** | 9.40 | 8.95 | 9.47 | 2.2% | 10/10 |
|
||||||
|
| fixed_size ±2 | 8.987 | 9.43 | 8.93 | 9.44 | 2.7% | 10/10 |
|
||||||
|
| fixed_size ±1 | 8.926 | 9.38 | 8.89 | 9.42 | 3.0% | 10/10 |
|
||||||
|
| **semantic @ text-embedding-3-large** | **8.727** | 9.38 | 8.83 | 9.30 | 4.8% | 10/10 |
|
||||||
|
| fixed_size ±0 | 8.606 | 9.18 | 8.79 | 9.25 | 4.9% | 10/10 |
|
||||||
|
| semantic @ nomic-embed-text-v2-moe | 8.551 | 9.19 | 8.65 | 9.14 | 4.7% | 10/10 |
|
||||||
|
|
||||||
|
`fixed_size ±1`, `±2`, and `±3` all beat the best semantic Candidate on **mean**. **`±0` (no Neighbor Expansion) does not** (8.606 vs 8.727). Shipping `fixed_size` without expansion would weaken this family decision.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Judge metrics (stage 2 winners)
|
||||||
|
|
||||||
|
`fixed_size ±3` leads on every judge metric, including **lower hallucination**.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Per-document showdown
|
||||||
|
|
||||||
|
Mean ranking still favors `fixed_size`. **Head-to-head at ±3: semantic 6 / `fixed_size` 4.** Semantic’s six wins are mostly **small**; `fixed_size`’s four wins are **larger**, especially **fire**.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
| Document | Questions | fixed_size ±3 | semantic @ large | Winner |
|
||||||
|
|----------|-----------|---------------|------------------|--------|
|
||||||
|
| fire.docx | 20 | **8.810** | 6.228 | **fixed_size** (+2.58) |
|
||||||
|
| moavenin.docx | 20 | **9.735** | 8.621 | **fixed_size** (+1.11) |
|
||||||
|
| general-havades-individuals.doc | 20 | **9.205** | 8.564 | **fixed_size** (+0.64) |
|
||||||
|
| havades.docx | 20 | **8.483** | 7.969 | **fixed_size** (+0.51) |
|
||||||
|
| bazresi.docx | 16 | 9.366 | **9.405** | semantic (−0.04) |
|
||||||
|
| life-time-individual.docx | 20 | 9.445 | **9.505** | semantic (−0.06) |
|
||||||
|
| Refah.docx | 20 | 9.680 | **9.815** | semantic (−0.14) |
|
||||||
|
| lifetime-compensation.docx | 15 | 8.761 | **9.129** | semantic (−0.37) |
|
||||||
|
| customer1.docx | 20 | 8.451 | **8.973** | semantic (−0.52) |
|
||||||
|
| website.docx | 30 | 8.466 | **9.058** | semantic (−0.59) |
|
||||||
|
|
||||||
|
Win-count would pick semantic. **Official product ranking (mean composite) picks `fixed_size`.** This memo follows the Decision Board rule (ADR-0026).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Full Candidate heatmap
|
||||||
|
|
||||||
|
Every cell is a newest single-strategy Experiment under `text-embedding-3-large`. Semantic @ large **collapses on fire**; `fixed_size ±1…±3` stay high across the set.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Best `fixed_size` ±N **by document** (does not change the family call):
|
||||||
|
|
||||||
|
| ±N | Documents where it is the best fixed_size cell |
|
||||||
|
|----|------------------------------------------------|
|
||||||
|
| ±3 | bazresi, fire, general-havades-individuals, havades, life-time-individual (5) |
|
||||||
|
| ±1 | customer1, moavenin, Refah (3) |
|
||||||
|
| ±2 | website (1) |
|
||||||
|
| ±0 | lifetime-compensation (1) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Method
|
||||||
|
|
||||||
|
- Corpus Embedding Model locked to **`text-embedding-3-large`**
|
||||||
|
- Newest **single-strategy** Experiment per document × Candidate (Decision Board cells)
|
||||||
|
- Retrieval `top_k = 5`
|
||||||
|
- Stage 1: best `fixed_size` ±N vs best `semantic@Boundary`
|
||||||
|
- Stage 2: mean composite + per-document breakdown
|
||||||
|
- Experiments dated **9–10 August 2026**
|
||||||
|
- `logs/neighbor_sweep.log` recorded mid-run connection errors on some units; **SQLite now has a full 10×4 `fixed_size` grid** — treat the database as source of truth
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Excluded strategies
|
||||||
|
|
||||||
|
No comparable **10-doc, single-strategy, `text-embedding-3-large`** Experiments exist for the three Strategies below. They are not Decision Board Candidates (ADR-0026).
|
||||||
|
|
||||||
|
| Strategy | Why it is not the winner |
|
||||||
|
|----------|--------------------------|
|
||||||
|
| **recursive** | Not on the Decision Board grid. Informal PDF / `text-embedding-3-small` multi-strategy runs (not comparable to this close-out) were mixed; recursive **does not get Neighbor Expansion** (expansion is `fixed_size` only). |
|
||||||
|
| **contextual_retrieval** | Extra LLM call per chunk at process time. Same informal PDF/`small` runs scored **below** recursive and `fixed_size`. |
|
||||||
|
| **semantic_parent_child** | Boundary-detection cost plus parent fetch at query. Informal PDF/`small` composites were **much worse** (~1.8–2.8). Fail-hard if Boundary embeds mismatch (ADR-0020). |
|
||||||
|
|
||||||
|
One leftover five-strategy Experiment on `customer1` under large is **invalid** (four Strategies scored 0 / errors) and was ignored.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Binding decision
|
||||||
|
|
||||||
|
**Use `fixed_size` for chunking** under **`text-embedding-3-large`.**
|
||||||
|
|
||||||
|
**Do not use `semantic` as the default Strategy** on this evaluation universe.
|
||||||
|
|
||||||
|
**Follow-up:** choose Neighbor Expansion **±N** among **±1, ±2, ±3**. Stage 1 auto-pick is **±3**. **±0 is not recommended** if this family decision is to remain valid.
|
||||||
|
|
||||||
|
Production config change is **out of scope** for this record.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Evidence (stage 2 cells)
|
||||||
|
|
||||||
|
| Document | `fixed_size ±3` experiment id | `semantic @ large` experiment id |
|
||||||
|
|----------|-------------------------------|----------------------------------|
|
||||||
|
| bazresi.docx | `6fe08750ee324130b3d46d8b3bc95280` | `6263c4f9e63347cea9eb19f4d36309c0` |
|
||||||
|
| customer1.docx | `8c4d888d367b4c0299b2d19dd9654f80` | `0c4572e80902408796f0dd11776e8e55` |
|
||||||
|
| fire.docx | `230a3cf57ae74b679ecde83cfeb9b6f5` | `414162efc314406db082ba8537c2423f` |
|
||||||
|
| general-havades-individuals.doc | `7ad892cfcd6143109f77a7808c8dfbd9` | `14fa4f872373446c8b042fc8e84e6983` |
|
||||||
|
| havades.docx | `04bd45d836a64c3da2e68bb747c0c9b8` | `232238b4d3304243b82b02326ef64617` |
|
||||||
|
| life-time-individual.docx | `d682706f54824469b235c099bf5a3d3c` | `bc7d9f1fbe3540c097a5f8cc1a1c44bb` |
|
||||||
|
| moavenin.docx | `63f81cf0a632418aab2f93884518ed71` | `4e01960b5db14e81975a4fbc86c4e10c` |
|
||||||
|
| Refah.docx | `6304dba39f3f4c05935195c588a36a64` | `b5c630c45615451a93d97d80dee361df` |
|
||||||
|
| website.docx | `fb9d47750ca74761a3c82c57aa51030b` | `53256897cfb74dd7a6508e750ff1ba63` |
|
||||||
|
| lifetime-compensation.docx | `99736ffbc6da455b8b3ca3a0fb8e5ef7` | `e51c2b57e7364dfa97e5429f615d6e4e` |
|
||||||
|
|
||||||
|
HTML reports: `GET /benchmarks/{id}/report`. Dashboard: Decision Tab, Corpus = `text-embedding-3-large`.
|
||||||
Reference in New Issue
Block a user