Files
chunking_strategies_evaluation/docs/assets/decision/generate_charts.py
Mahdi Bazrafshan 640691b8ab docs: record fixed_size as the winning strategy family
Why:
- Close the benchmark with a written family decision backed by Experiment composites.

Changes:
- Add the decision memo and SVG charts generated from SQLite.

Impact:
- Documents fixed_size over semantic under text-embedding-3-large; ±N remains follow-up.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-17 14:22:38 +03:30

459 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Generate SVG charts for the final chunking-strategy decision report.
Reads Experiment aggregates from SQLite (Decision Board rules) and writes
SVG files next to this script. Re-run after new Experiments if needed:
.venv/bin/python docs/assets/decision/generate_charts.py
"""
from __future__ import annotations
import json
import sqlite3
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
DB = ROOT / "data" / "chunking_benchmark.db"
OUT = Path(__file__).resolve().parent
DECISION_DOCS = [
"bazresi.docx",
"customer1.docx",
"fire.docx",
"general-havades-individuals.doc",
"havades.docx",
"life-time-individual.docx",
"moavenin.docx",
"Refah.docx",
"website.docx",
"lifetime-compensation.docx",
]
SHORT = {
"bazresi.docx": "bazresi",
"customer1.docx": "customer1",
"fire.docx": "fire",
"general-havades-individuals.doc": "havades-ind",
"havades.docx": "havades",
"life-time-individual.docx": "lifetime-ind",
"moavenin.docx": "moavenin",
"Refah.docx": "Refah",
"website.docx": "website",
"lifetime-compensation.docx": "lifetime-comp",
}
CORPUS = "text-embedding-3-large"
TEAL_FILL = "#14b8a6"
AMBER_FILL = "#f59e0b"
INK = "#111827"
MUTED = "#6b7280"
GRID = "#e5e7eb"
BG = "#ffffff"
WIN = "#047857"
def esc(s: str) -> str:
return (
str(s)
.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
)
def composite(m: dict | None) -> float | None:
if not m:
return None
return (
(m.get("avg_context_relevance") or 0) * 0.3
+ (m.get("avg_answer_similarity") or 0) * 0.4
+ (m.get("avg_faithfulness") or 0) * 0.3
) * (1 - (m.get("hallucination_rate") or 0))
def load_cells() -> dict[tuple[str, str], dict]:
conn = sqlite3.connect(DB)
conn.row_factory = sqlite3.Row
docs = {r["id"]: r["filename"] for r in conn.execute("SELECT id, filename FROM documents")}
rows = []
for e in conn.execute(
"""
SELECT id, document_id, strategies_used, aggregate_metrics, benchmark_config,
embedding_model_id, boundary_embedding_model_id, created_at, questions
FROM experiments ORDER BY created_at DESC
"""
):
fn = docs.get(e["document_id"])
strats = json.loads(e["strategies_used"] or "[]")
agg = json.loads(e["aggregate_metrics"] or "{}")
cfg = json.loads(e["benchmark_config"] or "{}")
corpus = e["embedding_model_id"] or cfg.get("corpus_embedding_model_id")
prev = int(cfg.get("neighbor_prev") or 0)
nxt = int(cfg.get("neighbor_next") or 0)
bound = e["boundary_embedding_model_id"] or cfg.get("boundary_embedding_model_id")
if fn not in DECISION_DOCS or corpus != CORPUS or len(strats) != 1:
continue
strat = strats[0]
if strat == "fixed_size":
if prev != nxt or prev not in (0, 1, 2, 3):
continue
cid = f"fixed_size:±{prev}"
family = "fixed_size"
elif strat == "semantic":
if not bound:
continue
cid = f"semantic:{bound}"
family = "semantic"
else:
continue
rows.append(
{
"fn": fn,
"cid": cid,
"family": family,
"agg": agg,
"metrics": agg.get(family) or {},
"id": e["id"],
}
)
conn.close()
cells: dict[tuple[str, str], dict] = {}
for r in rows:
key = (r["fn"], r["cid"])
if key not in cells:
cells[key] = r
return cells
def mean(xs: list[float]) -> float | None:
return sum(xs) / len(xs) if xs else None
def svg_wrap(w: int, h: int, body: str, title: str) -> str:
return f'''<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}" role="img" aria-label="{esc(title)}">
<title>{esc(title)}</title>
<rect width="{w}" height="{h}" fill="{BG}"/>
{body}
</svg>
'''
def chart_stage1(cells: dict) -> None:
cands = [
("fixed_size:±3", "fixed_size ±3", TEAL_FILL, True),
("fixed_size:±2", "fixed_size ±2", TEAL_FILL, False),
("fixed_size:±1", "fixed_size ±1", TEAL_FILL, False),
("semantic:text-embedding-3-large", "semantic @ large", AMBER_FILL, False),
("fixed_size:±0", "fixed_size ±0", "#99f6e4", False),
("semantic:nomic-embed-text-v2-moe", "semantic @ nomic", "#fcd34d", False),
]
values = []
for cid, label, color, winner in cands:
scores = []
for fn in DECISION_DOCS:
r = cells.get((fn, cid))
if r:
sc = composite(r["metrics"])
if sc is not None:
scores.append(sc)
values.append((cid, label, color, winner, mean(scores) or 0))
w, h = 820, 420
left, right, top, bottom = 210, 40, 56, 48
plot_w = w - left - right
plot_h = h - top - bottom
vmin, vmax = 8.4, 9.2
bar_h = plot_h / len(values) * 0.62
gap = plot_h / len(values)
parts = [
f'<text x="24" y="32" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 1 — mean composite (10 documents)</text>',
f'<text x="24" y="50" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Higher is better. Official ranking used by Decision Board. Winner: fixed_size ±3.</text>',
]
# grid
for tick in [8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2]:
x = left + (tick - vmin) / (vmax - vmin) * plot_w
parts.append(f'<line x1="{x:.1f}" y1="{top}" x2="{x:.1f}" y2="{h - bottom}" stroke="{GRID}" stroke-width="1"/>')
parts.append(
f'<text x="{x:.1f}" y="{h - 18}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">{tick:.1f}</text>'
)
for i, (_, label, color, winner, val) in enumerate(values):
y = top + i * gap + (gap - bar_h) / 2
bw = (val - vmin) / (vmax - vmin) * plot_w
stroke = WIN if winner else "none"
sw = 2 if winner else 0
parts.append(
f'<rect x="{left}" y="{y:.1f}" width="{max(bw, 0):.1f}" height="{bar_h:.1f}" rx="4" fill="{color}" stroke="{stroke}" stroke-width="{sw}"/>'
)
weight = "700" if winner else "500"
parts.append(
f'<text x="{left - 10}" y="{y + bar_h * 0.68:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="{weight}" fill="{INK}">{esc(label)}</text>'
)
parts.append(
f'<text x="{left + bw + 8:.1f}" y="{y + bar_h * 0.68:.1f}" font-family="Inter, system-ui, sans-serif" font-size="13" font-weight="600" fill="{INK}">{val:.3f}</text>'
)
(OUT / "stage1-mean-composite.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 1 mean composite"), encoding="utf-8")
def chart_stage2(cells: dict) -> None:
fs_id = "fixed_size:±3"
sem_id = "semantic:text-embedding-3-large"
pairs = []
for fn in DECISION_DOCS:
a = composite(cells[(fn, fs_id)]["metrics"])
b = composite(cells[(fn, sem_id)]["metrics"])
pairs.append((SHORT[fn], a, b))
w, h = 920, 460
left, right, top, bottom = 52, 24, 64, 88
plot_w = w - left - right
plot_h = h - top - bottom
n = len(pairs)
slot = plot_w / n
bar_w = slot * 0.32
vmin, vmax = 5.8, 10.0
def y_of(v: float) -> float:
return top + (1 - (v - vmin) / (vmax - vmin)) * plot_h
parts = [
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 2 — per document (fixed_size ±3 vs semantic @ large)</text>',
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Mean composite still favors fixed_size. Semantic wins 6 of 10 docs, but by smaller margins except website/customer1.</text>',
f'<rect x="620" y="14" width="12" height="12" rx="2" fill="{TEAL_FILL}"/>',
f'<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">fixed_size ±3</text>',
f'<rect x="760" y="14" width="12" height="12" rx="2" fill="{AMBER_FILL}"/>',
f'<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">semantic @ large</text>',
]
for tick in [6, 7, 8, 9, 10]:
y = y_of(tick)
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{w - right}" y2="{y:.1f}" stroke="{GRID}" stroke-width="1"/>')
parts.append(
f'<text x="{left - 8}" y="{y + 4:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">{tick}</text>'
)
for i, (label, a, b) in enumerate(pairs):
cx = left + i * slot + slot / 2
xa = cx - bar_w - 3
xb = cx + 3
ha = plot_h - (y_of(a) - top)
hb = plot_h - (y_of(b) - top)
parts.append(f'<rect x="{xa:.1f}" y="{y_of(a):.1f}" width="{bar_w:.1f}" height="{ha:.1f}" rx="3" fill="{TEAL_FILL}"/>')
parts.append(f'<rect x="{xb:.1f}" y="{y_of(b):.1f}" width="{bar_w:.1f}" height="{hb:.1f}" rx="3" fill="{AMBER_FILL}"/>')
parts.append(
f'<text x="{cx:.1f}" y="{h - 36}" text-anchor="end" transform="rotate(-32 {cx:.1f} {h - 36})" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{INK}">{esc(label)}</text>'
)
(OUT / "stage2-per-document.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 per document"), encoding="utf-8")
def chart_delta(cells: dict) -> None:
fs_id = "fixed_size:±3"
sem_id = "semantic:text-embedding-3-large"
deltas = []
for fn in DECISION_DOCS:
a = composite(cells[(fn, fs_id)]["metrics"])
b = composite(cells[(fn, sem_id)]["metrics"])
deltas.append((SHORT[fn], a - b))
deltas.sort(key=lambda x: x[1], reverse=True)
w, h = 820, 440
left, right, top, bottom = 120, 56, 56, 36
plot_w = w - left - right
plot_h = h - top - bottom
n = len(deltas)
gap = plot_h / n
bar_h = gap * 0.62
max_abs = max(abs(d) for _, d in deltas)
# fire is 2.58, others < 1.2 — use 2.8
max_abs = 2.8
zero_x = left + plot_w / 2
parts = [
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Margin: fixed_size ±3 minus semantic @ large</text>',
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Teal = fixed_size wins the document. Amber = semantic wins. Mean ranking is driven by large teal bars (especially fire).</text>',
f'<line x1="{zero_x:.1f}" y1="{top}" x2="{zero_x:.1f}" y2="{h - bottom}" stroke="{INK}" stroke-width="1.2"/>',
f'<text x="{left}" y="{h - 12}" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">semantic better</text>',
f'<text x="{w - right}" y="{h - 12}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{MUTED}">fixed_size better</text>',
]
for i, (label, d) in enumerate(deltas):
y = top + i * gap + (gap - bar_h) / 2
bw = abs(d) / max_abs * (plot_w / 2)
if d >= 0:
x = zero_x
color = TEAL_FILL
tx = x + bw + 6
anchor = "start"
else:
x = zero_x - bw
color = AMBER_FILL
tx = x - 6
anchor = "end"
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bw:.1f}" height="{bar_h:.1f}" rx="3" fill="{color}"/>')
parts.append(
f'<text x="{left - 8}" y="{y + bar_h * 0.7:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">{esc(label)}</text>'
)
parts.append(
f'<text x="{tx:.1f}" y="{y + bar_h * 0.7:.1f}" text-anchor="{anchor}" font-family="Inter, system-ui, sans-serif" font-size="12" font-weight="600" fill="{INK}">{d:+.2f}</text>'
)
(OUT / "stage2-margins.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 margins"), encoding="utf-8")
def chart_heatmap(cells: dict) -> None:
cands = [
("fixed_size:±0", "±0"),
("fixed_size:±1", "±1"),
("fixed_size:±2", "±2"),
("fixed_size:±3", "±3"),
("semantic:text-embedding-3-large", "sem@large"),
("semantic:nomic-embed-text-v2-moe", "sem@nomic"),
]
scores: list[list[float | None]] = []
for fn in DECISION_DOCS:
row = []
for cid, _ in cands:
r = cells.get((fn, cid))
row.append(composite(r["metrics"]) if r else None)
scores.append(row)
cell_w, cell_h = 78, 32
left, top = 128, 72
w = left + cell_w * len(cands) + 24
h = top + cell_h * len(DECISION_DOCS) + 36
def color_for(s: float | None) -> tuple[str, str]:
if s is None:
return "#f3f4f6", MUTED
# 6.2 .. 9.8
t = max(0.0, min(1.0, (s - 7.0) / (9.8 - 7.0)))
# pale rose -> amber -> teal
if t < 0.5:
u = t * 2
r = int(251 + (20 - 251) * 0)
# interpolate rose 251,113,133 -> amber 245,158,11
rr = int(251 + (245 - 251) * u)
gg = int(113 + (158 - 113) * u)
bb = int(133 + (11 - 133) * u)
else:
u = (t - 0.5) * 2
rr = int(245 + (13 - 245) * u)
gg = int(158 + (148 - 158) * u)
bb = int(11 + (136 - 11) * u)
bg = f"#{rr:02x}{gg:02x}{bb:02x}"
fg = "#111827" if t < 0.72 else "#f9fafb"
return bg, fg
parts = [
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Candidate heatmap — composite by document</text>',
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set.</text>',
]
for j, (_, lab) in enumerate(cands):
x = left + j * cell_w + cell_w / 2
parts.append(
f'<text x="{x:.1f}" y="{top - 10}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{esc(lab)}</text>'
)
for i, fn in enumerate(DECISION_DOCS):
y = top + i * cell_h
parts.append(
f'<text x="{left - 8}" y="{y + cell_h * 0.65:.1f}" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="{INK}">{esc(SHORT[fn])}</text>'
)
for j, val in enumerate(scores[i]):
x = left + j * cell_w
bg, fg = color_for(val)
parts.append(f'<rect x="{x}" y="{y}" width="{cell_w - 3}" height="{cell_h - 3}" rx="4" fill="{bg}"/>')
txt = "—" if val is None else f"{val:.2f}"
parts.append(
f'<text x="{x + (cell_w - 3) / 2:.1f}" y="{y + cell_h * 0.62:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{fg}">{txt}</text>'
)
(OUT / "heatmap-candidates.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Candidate heatmap"), encoding="utf-8")
def chart_metrics(cells: dict) -> None:
"""Grouped bars: four judge metrics for the two stage-2 winners."""
def bag(cid: str, family: str) -> dict[str, float]:
keys = [
"avg_context_relevance",
"avg_answer_similarity",
"avg_faithfulness",
"hallucination_rate",
]
acc = {k: [] for k in keys}
for fn in DECISION_DOCS:
m = cells[(fn, cid)]["metrics"]
for k in keys:
acc[k].append(m[k])
return {k: sum(v) / len(v) for k, v in acc.items()}
fs = bag("fixed_size:±3", "fixed_size")
sem = bag("semantic:text-embedding-3-large", "semantic")
labels = [
("Context relevance", fs["avg_context_relevance"], sem["avg_context_relevance"], False),
("Answer similarity", fs["avg_answer_similarity"], sem["avg_answer_similarity"], False),
("Faithfulness", fs["avg_faithfulness"], sem["avg_faithfulness"], False),
("Hallucination % (lower better)", fs["hallucination_rate"] * 100, sem["hallucination_rate"] * 100, True),
]
w, h = 820, 380
left, right, top, bottom = 52, 24, 64, 48
plot_w = w - left - right
plot_h = h - top - bottom
n = len(labels)
slot = plot_w / n
bar_w = slot * 0.28
parts = [
f'<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="{INK}">Stage 2 winners — mean judge metrics</text>',
f'<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{MUTED}">fixed_size ±3 leads on all four metrics, including lower hallucination.</text>',
f'<rect x="620" y="14" width="12" height="12" rx="2" fill="{TEAL_FILL}"/>',
f'<text x="638" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">fixed_size ±3</text>',
f'<rect x="760" y="14" width="12" height="12" rx="2" fill="{AMBER_FILL}"/>',
f'<text x="778" y="25" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">semantic @ large</text>',
]
for i, (lab, a, b, is_pct) in enumerate(labels):
vmax = 10 if not is_pct else max(a, b) * 1.35
cx = left + i * slot + slot / 2
xa = cx - bar_w - 3
xb = cx + 3
def bar(x: float, val: float, color: str) -> str:
bh = val / vmax * plot_h
y = top + plot_h - bh
return f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_w:.1f}" height="{bh:.1f}" rx="3" fill="{color}"/>'
parts.append(bar(xa, a, TEAL_FILL))
parts.append(bar(xb, b, AMBER_FILL))
fmt = (lambda v: f"{v:.1f}%") if is_pct else (lambda v: f"{v:.2f}")
ya = top + plot_h - a / vmax * plot_h - 6
yb = top + plot_h - b / vmax * plot_h - 6
parts.append(
f'<text x="{xa + bar_w / 2:.1f}" y="{ya:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{fmt(a)}</text>'
)
parts.append(
f'<text x="{xb + bar_w / 2:.1f}" y="{yb:.1f}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="{INK}">{fmt(b)}</text>'
)
parts.append(
f'<text x="{cx:.1f}" y="{h - 16}" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="12" fill="{INK}">{esc(lab)}</text>'
)
(OUT / "stage2-metrics.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 mean metrics"), encoding="utf-8")
def main() -> None:
cells = load_cells()
chart_stage1(cells)
chart_stage2(cells)
chart_delta(cells)
chart_heatmap(cells)
chart_metrics(cells)
print(f"Wrote SVGs in {OUT}")
if __name__ == "__main__":
main()