#!/usr/bin/env python3
"""Generate SVG charts for the final chunking-strategy decision report.
Reads Experiment aggregates from SQLite (Decision Board rules) and writes
SVG files next to this script. Re-run after new Experiments if needed:
.venv/bin/python docs/assets/decision/generate_charts.py
"""
from __future__ import annotations
import json
import sqlite3
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
DB = ROOT / "data" / "chunking_benchmark.db"
OUT = Path(__file__).resolve().parent
DECISION_DOCS = [
"bazresi.docx",
"customer1.docx",
"fire.docx",
"general-havades-individuals.doc",
"havades.docx",
"life-time-individual.docx",
"moavenin.docx",
"Refah.docx",
"website.docx",
"lifetime-compensation.docx",
]
SHORT = {
"bazresi.docx": "bazresi",
"customer1.docx": "customer1",
"fire.docx": "fire",
"general-havades-individuals.doc": "havades-ind",
"havades.docx": "havades",
"life-time-individual.docx": "lifetime-ind",
"moavenin.docx": "moavenin",
"Refah.docx": "Refah",
"website.docx": "website",
"lifetime-compensation.docx": "lifetime-comp",
}
CORPUS = "text-embedding-3-large"
TEAL_FILL = "#14b8a6"
AMBER_FILL = "#f59e0b"
INK = "#111827"
MUTED = "#6b7280"
GRID = "#e5e7eb"
BG = "#ffffff"
WIN = "#047857"
def esc(s: str) -> str:
return (
str(s)
.replace("&", "&")
.replace("<", "<")
.replace(">", ">")
.replace('"', """)
)
def composite(m: dict | None) -> float | None:
if not m:
return None
return (
(m.get("avg_context_relevance") or 0) * 0.3
+ (m.get("avg_answer_similarity") or 0) * 0.4
+ (m.get("avg_faithfulness") or 0) * 0.3
) * (1 - (m.get("hallucination_rate") or 0))
def load_cells() -> dict[tuple[str, str], dict]:
conn = sqlite3.connect(DB)
conn.row_factory = sqlite3.Row
docs = {r["id"]: r["filename"] for r in conn.execute("SELECT id, filename FROM documents")}
rows = []
for e in conn.execute(
"""
SELECT id, document_id, strategies_used, aggregate_metrics, benchmark_config,
embedding_model_id, boundary_embedding_model_id, created_at, questions
FROM experiments ORDER BY created_at DESC
"""
):
fn = docs.get(e["document_id"])
strats = json.loads(e["strategies_used"] or "[]")
agg = json.loads(e["aggregate_metrics"] or "{}")
cfg = json.loads(e["benchmark_config"] or "{}")
corpus = e["embedding_model_id"] or cfg.get("corpus_embedding_model_id")
prev = int(cfg.get("neighbor_prev") or 0)
nxt = int(cfg.get("neighbor_next") or 0)
bound = e["boundary_embedding_model_id"] or cfg.get("boundary_embedding_model_id")
if fn not in DECISION_DOCS or corpus != CORPUS or len(strats) != 1:
continue
strat = strats[0]
if strat == "fixed_size":
if prev != nxt or prev not in (0, 1, 2, 3):
continue
cid = f"fixed_size:±{prev}"
family = "fixed_size"
elif strat == "semantic":
if not bound:
continue
cid = f"semantic:{bound}"
family = "semantic"
else:
continue
rows.append(
{
"fn": fn,
"cid": cid,
"family": family,
"agg": agg,
"metrics": agg.get(family) or {},
"id": e["id"],
}
)
conn.close()
cells: dict[tuple[str, str], dict] = {}
for r in rows:
key = (r["fn"], r["cid"])
if key not in cells:
cells[key] = r
return cells
def mean(xs: list[float]) -> float | None:
return sum(xs) / len(xs) if xs else None
def svg_wrap(w: int, h: int, body: str, title: str) -> str:
return f'''
'''
def chart_stage1(cells: dict) -> None:
cands = [
("fixed_size:±3", "fixed_size ±3", TEAL_FILL, True),
("fixed_size:±2", "fixed_size ±2", TEAL_FILL, False),
("fixed_size:±1", "fixed_size ±1", TEAL_FILL, False),
("semantic:text-embedding-3-large", "semantic @ large", AMBER_FILL, False),
("fixed_size:±0", "fixed_size ±0", "#99f6e4", False),
("semantic:nomic-embed-text-v2-moe", "semantic @ nomic", "#fcd34d", False),
]
values = []
for cid, label, color, winner in cands:
scores = []
for fn in DECISION_DOCS:
r = cells.get((fn, cid))
if r:
sc = composite(r["metrics"])
if sc is not None:
scores.append(sc)
values.append((cid, label, color, winner, mean(scores) or 0))
w, h = 820, 420
left, right, top, bottom = 210, 40, 56, 48
plot_w = w - left - right
plot_h = h - top - bottom
vmin, vmax = 8.4, 9.2
bar_h = plot_h / len(values) * 0.62
gap = plot_h / len(values)
parts = [
f'Stage 1 — mean composite (10 documents)',
f'Higher is better. Official ranking used by Decision Board. Winner: fixed_size ±3.',
]
# grid
for tick in [8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2]:
x = left + (tick - vmin) / (vmax - vmin) * plot_w
parts.append(f'')
parts.append(
f'{tick:.1f}'
)
for i, (_, label, color, winner, val) in enumerate(values):
y = top + i * gap + (gap - bar_h) / 2
bw = (val - vmin) / (vmax - vmin) * plot_w
stroke = WIN if winner else "none"
sw = 2 if winner else 0
parts.append(
f''
)
weight = "700" if winner else "500"
parts.append(
f'{esc(label)}'
)
parts.append(
f'{val:.3f}'
)
(OUT / "stage1-mean-composite.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 1 mean composite"), encoding="utf-8")
def chart_stage2(cells: dict) -> None:
fs_id = "fixed_size:±3"
sem_id = "semantic:text-embedding-3-large"
pairs = []
for fn in DECISION_DOCS:
a = composite(cells[(fn, fs_id)]["metrics"])
b = composite(cells[(fn, sem_id)]["metrics"])
pairs.append((SHORT[fn], a, b))
w, h = 920, 460
left, right, top, bottom = 52, 24, 64, 88
plot_w = w - left - right
plot_h = h - top - bottom
n = len(pairs)
slot = plot_w / n
bar_w = slot * 0.32
vmin, vmax = 5.8, 10.0
def y_of(v: float) -> float:
return top + (1 - (v - vmin) / (vmax - vmin)) * plot_h
parts = [
f'Stage 2 — per document (fixed_size ±3 vs semantic @ large)',
f'Mean composite still favors fixed_size. Semantic wins 6 of 10 docs, but by smaller margins except website/customer1.',
f'',
f'fixed_size ±3',
f'',
f'semantic @ large',
]
for tick in [6, 7, 8, 9, 10]:
y = y_of(tick)
parts.append(f'')
parts.append(
f'{tick}'
)
for i, (label, a, b) in enumerate(pairs):
cx = left + i * slot + slot / 2
xa = cx - bar_w - 3
xb = cx + 3
ha = plot_h - (y_of(a) - top)
hb = plot_h - (y_of(b) - top)
parts.append(f'')
parts.append(f'')
parts.append(
f'{esc(label)}'
)
(OUT / "stage2-per-document.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 per document"), encoding="utf-8")
def chart_delta(cells: dict) -> None:
fs_id = "fixed_size:±3"
sem_id = "semantic:text-embedding-3-large"
deltas = []
for fn in DECISION_DOCS:
a = composite(cells[(fn, fs_id)]["metrics"])
b = composite(cells[(fn, sem_id)]["metrics"])
deltas.append((SHORT[fn], a - b))
deltas.sort(key=lambda x: x[1], reverse=True)
w, h = 820, 440
left, right, top, bottom = 120, 56, 56, 36
plot_w = w - left - right
plot_h = h - top - bottom
n = len(deltas)
gap = plot_h / n
bar_h = gap * 0.62
max_abs = max(abs(d) for _, d in deltas)
# fire is 2.58, others < 1.2 — use 2.8
max_abs = 2.8
zero_x = left + plot_w / 2
parts = [
f'Margin: fixed_size ±3 minus semantic @ large',
f'Teal = fixed_size wins the document. Amber = semantic wins. Mean ranking is driven by large teal bars (especially fire).',
f'',
f'semantic better',
f'fixed_size better',
]
for i, (label, d) in enumerate(deltas):
y = top + i * gap + (gap - bar_h) / 2
bw = abs(d) / max_abs * (plot_w / 2)
if d >= 0:
x = zero_x
color = TEAL_FILL
tx = x + bw + 6
anchor = "start"
else:
x = zero_x - bw
color = AMBER_FILL
tx = x - 6
anchor = "end"
parts.append(f'')
parts.append(
f'{esc(label)}'
)
parts.append(
f'{d:+.2f}'
)
(OUT / "stage2-margins.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 margins"), encoding="utf-8")
def chart_heatmap(cells: dict) -> None:
cands = [
("fixed_size:±0", "±0"),
("fixed_size:±1", "±1"),
("fixed_size:±2", "±2"),
("fixed_size:±3", "±3"),
("semantic:text-embedding-3-large", "sem@large"),
("semantic:nomic-embed-text-v2-moe", "sem@nomic"),
]
scores: list[list[float | None]] = []
for fn in DECISION_DOCS:
row = []
for cid, _ in cands:
r = cells.get((fn, cid))
row.append(composite(r["metrics"]) if r else None)
scores.append(row)
cell_w, cell_h = 78, 32
left, top = 128, 72
w = left + cell_w * len(cands) + 24
h = top + cell_h * len(DECISION_DOCS) + 36
def color_for(s: float | None) -> tuple[str, str]:
if s is None:
return "#f3f4f6", MUTED
# 6.2 .. 9.8
t = max(0.0, min(1.0, (s - 7.0) / (9.8 - 7.0)))
# pale rose -> amber -> teal
if t < 0.5:
u = t * 2
r = int(251 + (20 - 251) * 0)
# interpolate rose 251,113,133 -> amber 245,158,11
rr = int(251 + (245 - 251) * u)
gg = int(113 + (158 - 113) * u)
bb = int(133 + (11 - 133) * u)
else:
u = (t - 0.5) * 2
rr = int(245 + (13 - 245) * u)
gg = int(158 + (148 - 158) * u)
bb = int(11 + (136 - 11) * u)
bg = f"#{rr:02x}{gg:02x}{bb:02x}"
fg = "#111827" if t < 0.72 else "#f9fafb"
return bg, fg
parts = [
f'Candidate heatmap — composite by document',
f'Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set.',
]
for j, (_, lab) in enumerate(cands):
x = left + j * cell_w + cell_w / 2
parts.append(
f'{esc(lab)}'
)
for i, fn in enumerate(DECISION_DOCS):
y = top + i * cell_h
parts.append(
f'{esc(SHORT[fn])}'
)
for j, val in enumerate(scores[i]):
x = left + j * cell_w
bg, fg = color_for(val)
parts.append(f'')
txt = "—" if val is None else f"{val:.2f}"
parts.append(
f'{txt}'
)
(OUT / "heatmap-candidates.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Candidate heatmap"), encoding="utf-8")
def chart_metrics(cells: dict) -> None:
"""Grouped bars: four judge metrics for the two stage-2 winners."""
def bag(cid: str, family: str) -> dict[str, float]:
keys = [
"avg_context_relevance",
"avg_answer_similarity",
"avg_faithfulness",
"hallucination_rate",
]
acc = {k: [] for k in keys}
for fn in DECISION_DOCS:
m = cells[(fn, cid)]["metrics"]
for k in keys:
acc[k].append(m[k])
return {k: sum(v) / len(v) for k, v in acc.items()}
fs = bag("fixed_size:±3", "fixed_size")
sem = bag("semantic:text-embedding-3-large", "semantic")
labels = [
("Context relevance", fs["avg_context_relevance"], sem["avg_context_relevance"], False),
("Answer similarity", fs["avg_answer_similarity"], sem["avg_answer_similarity"], False),
("Faithfulness", fs["avg_faithfulness"], sem["avg_faithfulness"], False),
("Hallucination % (lower better)", fs["hallucination_rate"] * 100, sem["hallucination_rate"] * 100, True),
]
w, h = 820, 380
left, right, top, bottom = 52, 24, 64, 48
plot_w = w - left - right
plot_h = h - top - bottom
n = len(labels)
slot = plot_w / n
bar_w = slot * 0.28
parts = [
f'Stage 2 winners — mean judge metrics',
f'fixed_size ±3 leads on all four metrics, including lower hallucination.',
f'',
f'fixed_size ±3',
f'',
f'semantic @ large',
]
for i, (lab, a, b, is_pct) in enumerate(labels):
vmax = 10 if not is_pct else max(a, b) * 1.35
cx = left + i * slot + slot / 2
xa = cx - bar_w - 3
xb = cx + 3
def bar(x: float, val: float, color: str) -> str:
bh = val / vmax * plot_h
y = top + plot_h - bh
return f''
parts.append(bar(xa, a, TEAL_FILL))
parts.append(bar(xb, b, AMBER_FILL))
fmt = (lambda v: f"{v:.1f}%") if is_pct else (lambda v: f"{v:.2f}")
ya = top + plot_h - a / vmax * plot_h - 6
yb = top + plot_h - b / vmax * plot_h - 6
parts.append(
f'{fmt(a)}'
)
parts.append(
f'{fmt(b)}'
)
parts.append(
f'{esc(lab)}'
)
(OUT / "stage2-metrics.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 mean metrics"), encoding="utf-8")
def main() -> None:
cells = load_cells()
chart_stage1(cells)
chart_stage2(cells)
chart_delta(cells)
chart_heatmap(cells)
chart_metrics(cells)
print(f"Wrote SVGs in {OUT}")
if __name__ == "__main__":
main()