From 640691b8ab3d5eb5b1876c75ccb1831b2d81455e Mon Sep 17 00:00:00 2001 From: Mahdi Bazrafshan Date: Mon, 17 Aug 2026 14:22:38 +0330 Subject: [PATCH] docs: record fixed_size as the winning strategy family MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: - Close the benchmark with a written family decision backed by Experiment composites. Changes: - Add the decision memo and SVG charts generated from SQLite. Impact: - Documents fixed_size over semantic under text-embedding-3-large; ±N remains follow-up. Co-authored-by: Cursor --- docs/assets/decision/generate_charts.py | 458 ++++++++++++++++++ docs/assets/decision/heatmap-candidates.svg | 143 ++++++ .../assets/decision/stage1-mean-composite.svg | 41 ++ docs/assets/decision/stage2-margins.svg | 40 ++ docs/assets/decision/stage2-metrics.svg | 31 ++ docs/assets/decision/stage2-per-document.svg | 51 ++ docs/final-chunking-strategy-decision.md | 149 ++++++ 7 files changed, 913 insertions(+) create mode 100644 docs/assets/decision/generate_charts.py create mode 100644 docs/assets/decision/heatmap-candidates.svg create mode 100644 docs/assets/decision/stage1-mean-composite.svg create mode 100644 docs/assets/decision/stage2-margins.svg create mode 100644 docs/assets/decision/stage2-metrics.svg create mode 100644 docs/assets/decision/stage2-per-document.svg create mode 100644 docs/final-chunking-strategy-decision.md diff --git a/docs/assets/decision/generate_charts.py b/docs/assets/decision/generate_charts.py new file mode 100644 index 0000000..029ba17 --- /dev/null +++ b/docs/assets/decision/generate_charts.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Generate SVG charts for the final chunking-strategy decision report. + +Reads Experiment aggregates from SQLite (Decision Board rules) and writes +SVG files next to this script. Re-run after new Experiments if needed: + + .venv/bin/python docs/assets/decision/generate_charts.py +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +DB = ROOT / "data" / "chunking_benchmark.db" +OUT = Path(__file__).resolve().parent + +DECISION_DOCS = [ + "bazresi.docx", + "customer1.docx", + "fire.docx", + "general-havades-individuals.doc", + "havades.docx", + "life-time-individual.docx", + "moavenin.docx", + "Refah.docx", + "website.docx", + "lifetime-compensation.docx", +] +SHORT = { + "bazresi.docx": "bazresi", + "customer1.docx": "customer1", + "fire.docx": "fire", + "general-havades-individuals.doc": "havades-ind", + "havades.docx": "havades", + "life-time-individual.docx": "lifetime-ind", + "moavenin.docx": "moavenin", + "Refah.docx": "Refah", + "website.docx": "website", + "lifetime-compensation.docx": "lifetime-comp", +} +CORPUS = "text-embedding-3-large" + +TEAL_FILL = "#14b8a6" +AMBER_FILL = "#f59e0b" +INK = "#111827" +MUTED = "#6b7280" +GRID = "#e5e7eb" +BG = "#ffffff" +WIN = "#047857" + + +def esc(s: str) -> str: + return ( + str(s) + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + ) + + +def composite(m: dict | None) -> float | None: + if not m: + return None + return ( + (m.get("avg_context_relevance") or 0) * 0.3 + + (m.get("avg_answer_similarity") or 0) * 0.4 + + (m.get("avg_faithfulness") or 0) * 0.3 + ) * (1 - (m.get("hallucination_rate") or 0)) + + +def load_cells() -> dict[tuple[str, str], dict]: + conn = sqlite3.connect(DB) + conn.row_factory = sqlite3.Row + docs = {r["id"]: r["filename"] for r in conn.execute("SELECT id, filename FROM documents")} + rows = [] + for e in conn.execute( + """ + SELECT id, document_id, strategies_used, aggregate_metrics, benchmark_config, + embedding_model_id, boundary_embedding_model_id, created_at, questions + FROM experiments ORDER BY created_at DESC + """ + ): + fn = docs.get(e["document_id"]) + strats = json.loads(e["strategies_used"] or "[]") + agg = json.loads(e["aggregate_metrics"] or "{}") + cfg = json.loads(e["benchmark_config"] or "{}") + corpus = e["embedding_model_id"] or cfg.get("corpus_embedding_model_id") + prev = int(cfg.get("neighbor_prev") or 0) + nxt = int(cfg.get("neighbor_next") or 0) + bound = e["boundary_embedding_model_id"] or cfg.get("boundary_embedding_model_id") + if fn not in DECISION_DOCS or corpus != CORPUS or len(strats) != 1: + continue + strat = strats[0] + if strat == "fixed_size": + if prev != nxt or prev not in (0, 1, 2, 3): + continue + cid = f"fixed_size:±{prev}" + family = "fixed_size" + elif strat == "semantic": + if not bound: + continue + cid = f"semantic:{bound}" + family = "semantic" + else: + continue + rows.append( + { + "fn": fn, + "cid": cid, + "family": family, + "agg": agg, + "metrics": agg.get(family) or {}, + "id": e["id"], + } + ) + conn.close() + cells: dict[tuple[str, str], dict] = {} + for r in rows: + key = (r["fn"], r["cid"]) + if key not in cells: + cells[key] = r + return cells + + +def mean(xs: list[float]) -> float | None: + return sum(xs) / len(xs) if xs else None + + +def svg_wrap(w: int, h: int, body: str, title: str) -> str: + return f''' + + {esc(title)} + + {body} + +''' + + +def chart_stage1(cells: dict) -> None: + cands = [ + ("fixed_size:±3", "fixed_size ±3", TEAL_FILL, True), + ("fixed_size:±2", "fixed_size ±2", TEAL_FILL, False), + ("fixed_size:±1", "fixed_size ±1", TEAL_FILL, False), + ("semantic:text-embedding-3-large", "semantic @ large", AMBER_FILL, False), + ("fixed_size:±0", "fixed_size ±0", "#99f6e4", False), + ("semantic:nomic-embed-text-v2-moe", "semantic @ nomic", "#fcd34d", False), + ] + values = [] + for cid, label, color, winner in cands: + scores = [] + for fn in DECISION_DOCS: + r = cells.get((fn, cid)) + if r: + sc = composite(r["metrics"]) + if sc is not None: + scores.append(sc) + values.append((cid, label, color, winner, mean(scores) or 0)) + + w, h = 820, 420 + left, right, top, bottom = 210, 40, 56, 48 + plot_w = w - left - right + plot_h = h - top - bottom + vmin, vmax = 8.4, 9.2 + bar_h = plot_h / len(values) * 0.62 + gap = plot_h / len(values) + + parts = [ + f'Stage 1 — mean composite (10 documents)', + f'Higher is better. Official ranking used by Decision Board. Winner: fixed_size ±3.', + ] + # grid + for tick in [8.5, 8.6, 8.7, 8.8, 8.9, 9.0, 9.1, 9.2]: + x = left + (tick - vmin) / (vmax - vmin) * plot_w + parts.append(f'') + parts.append( + f'{tick:.1f}' + ) + + for i, (_, label, color, winner, val) in enumerate(values): + y = top + i * gap + (gap - bar_h) / 2 + bw = (val - vmin) / (vmax - vmin) * plot_w + stroke = WIN if winner else "none" + sw = 2 if winner else 0 + parts.append( + f'' + ) + weight = "700" if winner else "500" + parts.append( + f'{esc(label)}' + ) + parts.append( + f'{val:.3f}' + ) + + (OUT / "stage1-mean-composite.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 1 mean composite"), encoding="utf-8") + + +def chart_stage2(cells: dict) -> None: + fs_id = "fixed_size:±3" + sem_id = "semantic:text-embedding-3-large" + pairs = [] + for fn in DECISION_DOCS: + a = composite(cells[(fn, fs_id)]["metrics"]) + b = composite(cells[(fn, sem_id)]["metrics"]) + pairs.append((SHORT[fn], a, b)) + + w, h = 920, 460 + left, right, top, bottom = 52, 24, 64, 88 + plot_w = w - left - right + plot_h = h - top - bottom + n = len(pairs) + slot = plot_w / n + bar_w = slot * 0.32 + vmin, vmax = 5.8, 10.0 + + def y_of(v: float) -> float: + return top + (1 - (v - vmin) / (vmax - vmin)) * plot_h + + parts = [ + f'Stage 2 — per document (fixed_size ±3 vs semantic @ large)', + f'Mean composite still favors fixed_size. Semantic wins 6 of 10 docs, but by smaller margins except website/customer1.', + f'', + f'fixed_size ±3', + f'', + f'semantic @ large', + ] + for tick in [6, 7, 8, 9, 10]: + y = y_of(tick) + parts.append(f'') + parts.append( + f'{tick}' + ) + + for i, (label, a, b) in enumerate(pairs): + cx = left + i * slot + slot / 2 + xa = cx - bar_w - 3 + xb = cx + 3 + ha = plot_h - (y_of(a) - top) + hb = plot_h - (y_of(b) - top) + parts.append(f'') + parts.append(f'') + parts.append( + f'{esc(label)}' + ) + + (OUT / "stage2-per-document.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 per document"), encoding="utf-8") + + +def chart_delta(cells: dict) -> None: + fs_id = "fixed_size:±3" + sem_id = "semantic:text-embedding-3-large" + deltas = [] + for fn in DECISION_DOCS: + a = composite(cells[(fn, fs_id)]["metrics"]) + b = composite(cells[(fn, sem_id)]["metrics"]) + deltas.append((SHORT[fn], a - b)) + deltas.sort(key=lambda x: x[1], reverse=True) + + w, h = 820, 440 + left, right, top, bottom = 120, 56, 56, 36 + plot_w = w - left - right + plot_h = h - top - bottom + n = len(deltas) + gap = plot_h / n + bar_h = gap * 0.62 + max_abs = max(abs(d) for _, d in deltas) + # fire is 2.58, others < 1.2 — use 2.8 + max_abs = 2.8 + zero_x = left + plot_w / 2 + + parts = [ + f'Margin: fixed_size ±3 minus semantic @ large', + f'Teal = fixed_size wins the document. Amber = semantic wins. Mean ranking is driven by large teal bars (especially fire).', + f'', + f'semantic better', + f'fixed_size better', + ] + for i, (label, d) in enumerate(deltas): + y = top + i * gap + (gap - bar_h) / 2 + bw = abs(d) / max_abs * (plot_w / 2) + if d >= 0: + x = zero_x + color = TEAL_FILL + tx = x + bw + 6 + anchor = "start" + else: + x = zero_x - bw + color = AMBER_FILL + tx = x - 6 + anchor = "end" + parts.append(f'') + parts.append( + f'{esc(label)}' + ) + parts.append( + f'{d:+.2f}' + ) + + (OUT / "stage2-margins.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 margins"), encoding="utf-8") + + +def chart_heatmap(cells: dict) -> None: + cands = [ + ("fixed_size:±0", "±0"), + ("fixed_size:±1", "±1"), + ("fixed_size:±2", "±2"), + ("fixed_size:±3", "±3"), + ("semantic:text-embedding-3-large", "sem@large"), + ("semantic:nomic-embed-text-v2-moe", "sem@nomic"), + ] + scores: list[list[float | None]] = [] + for fn in DECISION_DOCS: + row = [] + for cid, _ in cands: + r = cells.get((fn, cid)) + row.append(composite(r["metrics"]) if r else None) + scores.append(row) + + cell_w, cell_h = 78, 32 + left, top = 128, 72 + w = left + cell_w * len(cands) + 24 + h = top + cell_h * len(DECISION_DOCS) + 36 + + def color_for(s: float | None) -> tuple[str, str]: + if s is None: + return "#f3f4f6", MUTED + # 6.2 .. 9.8 + t = max(0.0, min(1.0, (s - 7.0) / (9.8 - 7.0))) + # pale rose -> amber -> teal + if t < 0.5: + u = t * 2 + r = int(251 + (20 - 251) * 0) + # interpolate rose 251,113,133 -> amber 245,158,11 + rr = int(251 + (245 - 251) * u) + gg = int(113 + (158 - 113) * u) + bb = int(133 + (11 - 133) * u) + else: + u = (t - 0.5) * 2 + rr = int(245 + (13 - 245) * u) + gg = int(158 + (148 - 158) * u) + bb = int(11 + (136 - 11) * u) + bg = f"#{rr:02x}{gg:02x}{bb:02x}" + fg = "#111827" if t < 0.72 else "#f9fafb" + return bg, fg + + parts = [ + f'Candidate heatmap — composite by document', + f'Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set.', + ] + for j, (_, lab) in enumerate(cands): + x = left + j * cell_w + cell_w / 2 + parts.append( + f'{esc(lab)}' + ) + for i, fn in enumerate(DECISION_DOCS): + y = top + i * cell_h + parts.append( + f'{esc(SHORT[fn])}' + ) + for j, val in enumerate(scores[i]): + x = left + j * cell_w + bg, fg = color_for(val) + parts.append(f'') + txt = "—" if val is None else f"{val:.2f}" + parts.append( + f'{txt}' + ) + + (OUT / "heatmap-candidates.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Candidate heatmap"), encoding="utf-8") + + +def chart_metrics(cells: dict) -> None: + """Grouped bars: four judge metrics for the two stage-2 winners.""" + def bag(cid: str, family: str) -> dict[str, float]: + keys = [ + "avg_context_relevance", + "avg_answer_similarity", + "avg_faithfulness", + "hallucination_rate", + ] + acc = {k: [] for k in keys} + for fn in DECISION_DOCS: + m = cells[(fn, cid)]["metrics"] + for k in keys: + acc[k].append(m[k]) + return {k: sum(v) / len(v) for k, v in acc.items()} + + fs = bag("fixed_size:±3", "fixed_size") + sem = bag("semantic:text-embedding-3-large", "semantic") + labels = [ + ("Context relevance", fs["avg_context_relevance"], sem["avg_context_relevance"], False), + ("Answer similarity", fs["avg_answer_similarity"], sem["avg_answer_similarity"], False), + ("Faithfulness", fs["avg_faithfulness"], sem["avg_faithfulness"], False), + ("Hallucination % (lower better)", fs["hallucination_rate"] * 100, sem["hallucination_rate"] * 100, True), + ] + + w, h = 820, 380 + left, right, top, bottom = 52, 24, 64, 48 + plot_w = w - left - right + plot_h = h - top - bottom + n = len(labels) + slot = plot_w / n + bar_w = slot * 0.28 + + parts = [ + f'Stage 2 winners — mean judge metrics', + f'fixed_size ±3 leads on all four metrics, including lower hallucination.', + f'', + f'fixed_size ±3', + f'', + f'semantic @ large', + ] + + for i, (lab, a, b, is_pct) in enumerate(labels): + vmax = 10 if not is_pct else max(a, b) * 1.35 + cx = left + i * slot + slot / 2 + xa = cx - bar_w - 3 + xb = cx + 3 + + def bar(x: float, val: float, color: str) -> str: + bh = val / vmax * plot_h + y = top + plot_h - bh + return f'' + + parts.append(bar(xa, a, TEAL_FILL)) + parts.append(bar(xb, b, AMBER_FILL)) + fmt = (lambda v: f"{v:.1f}%") if is_pct else (lambda v: f"{v:.2f}") + ya = top + plot_h - a / vmax * plot_h - 6 + yb = top + plot_h - b / vmax * plot_h - 6 + parts.append( + f'{fmt(a)}' + ) + parts.append( + f'{fmt(b)}' + ) + parts.append( + f'{esc(lab)}' + ) + + (OUT / "stage2-metrics.svg").write_text(svg_wrap(w, h, "\n".join(parts), "Stage 2 mean metrics"), encoding="utf-8") + + +def main() -> None: + cells = load_cells() + chart_stage1(cells) + chart_stage2(cells) + chart_delta(cells) + chart_heatmap(cells) + chart_metrics(cells) + print(f"Wrote SVGs in {OUT}") + + +if __name__ == "__main__": + main() diff --git a/docs/assets/decision/heatmap-candidates.svg b/docs/assets/decision/heatmap-candidates.svg new file mode 100644 index 0000000..a450401 --- /dev/null +++ b/docs/assets/decision/heatmap-candidates.svg @@ -0,0 +1,143 @@ + + + Candidate heatmap + + Candidate heatmap — composite by document +Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set. +±0 +±1 +±2 +±3 +sem@large +sem@nomic +bazresi + +8.52 + +9.34 + +9.34 + +9.37 + +9.40 + +9.12 +customer1 + +8.92 + +8.97 + +8.96 + +8.45 + +8.97 + +7.54 +fire + +8.09 + +8.34 + +7.93 + +8.81 + +6.23 + +7.19 +havades-ind + +7.83 + +8.66 + +9.12 + +9.21 + +8.56 + +8.53 +havades + +8.31 + +8.43 + +8.40 + +8.48 + +7.97 + +8.42 +lifetime-ind + +9.22 + +9.42 + +9.44 + +9.45 + +9.50 + +9.41 +moavenin + +9.06 + +9.73 + +9.73 + +9.73 + +8.62 + +8.29 +Refah + +9.46 + +9.71 + +9.70 + +9.68 + +9.81 + +9.56 +website + +7.67 + +7.91 + +8.51 + +8.47 + +9.06 + +8.49 +lifetime-comp + +8.99 + +8.73 + +8.73 + +8.76 + +9.13 + +8.96 + diff --git a/docs/assets/decision/stage1-mean-composite.svg b/docs/assets/decision/stage1-mean-composite.svg new file mode 100644 index 0000000..30db221 --- /dev/null +++ b/docs/assets/decision/stage1-mean-composite.svg @@ -0,0 +1,41 @@ + + + Stage 1 mean composite + + Stage 1 — mean composite (10 documents) +Higher is better. Official ranking used by Decision Board. Winner: fixed_size ±3. + +8.5 + +8.6 + +8.7 + +8.8 + +8.9 + +9.0 + +9.1 + +9.2 + +fixed_size ±3 +9.040 + +fixed_size ±2 +8.987 + +fixed_size ±1 +8.926 + +semantic @ large +8.727 + +fixed_size ±0 +8.606 + +semantic @ nomic +8.551 + diff --git a/docs/assets/decision/stage2-margins.svg b/docs/assets/decision/stage2-margins.svg new file mode 100644 index 0000000..bb54d67 --- /dev/null +++ b/docs/assets/decision/stage2-margins.svg @@ -0,0 +1,40 @@ + + + Stage 2 margins + + Margin: fixed_size ±3 minus semantic @ large +Teal = fixed_size wins the document. Amber = semantic wins. Mean ranking is driven by large teal bars (especially fire). + +semantic better +fixed_size better + +fire ++2.58 + +moavenin ++1.11 + +havades-ind ++0.64 + +havades ++0.51 + +bazresi +-0.04 + +lifetime-ind +-0.06 + +Refah +-0.13 + +lifetime-comp +-0.37 + +customer1 +-0.52 + +website +-0.59 + diff --git a/docs/assets/decision/stage2-metrics.svg b/docs/assets/decision/stage2-metrics.svg new file mode 100644 index 0000000..d476b42 --- /dev/null +++ b/docs/assets/decision/stage2-metrics.svg @@ -0,0 +1,31 @@ + + + Stage 2 mean metrics + + Stage 2 winners — mean judge metrics +fixed_size ±3 leads on all four metrics, including lower hallucination. + +fixed_size ±3 + +semantic @ large + + +9.40 +9.38 +Context relevance + + +8.95 +8.83 +Answer similarity + + +9.47 +9.30 +Faithfulness + + +2.2% +4.8% +Hallucination % (lower better) + diff --git a/docs/assets/decision/stage2-per-document.svg b/docs/assets/decision/stage2-per-document.svg new file mode 100644 index 0000000..34a218f --- /dev/null +++ b/docs/assets/decision/stage2-per-document.svg @@ -0,0 +1,51 @@ + + + Stage 2 per document + + Stage 2 — per document (fixed_size ±3 vs semantic @ large) +Mean composite still favors fixed_size. Semantic wins 6 of 10 docs, but by smaller margins except website/customer1. + +fixed_size ±3 + +semantic @ large + +6 + +7 + +8 + +9 + +10 + + +bazresi + + +customer1 + + +fire + + +havades-ind + + +havades + + +lifetime-ind + + +moavenin + + +Refah + + +website + + +lifetime-comp + diff --git a/docs/final-chunking-strategy-decision.md b/docs/final-chunking-strategy-decision.md new file mode 100644 index 0000000..5e03d69 --- /dev/null +++ b/docs/final-chunking-strategy-decision.md @@ -0,0 +1,149 @@ +# Chunking Strategy — Final Decision + +**Date:** 17 August 2026 +**Decision:** Adopt **`fixed_size`** as the chunking Strategy family. +**Not decided here:** Neighbor Expansion level **±N** (follow-up). +**Corpus Embedding Model:** `text-embedding-3-large` +**Evaluation set:** 10 Word documents (Decision Board / neighbor-sweep universe) +**Questions:** 201 per Candidate (nomic semantic: 195) + +**One-line close:** **`fixed_size` wins** on official mean composite (**9.040** vs **8.727**). Pick **±N later**, but **not ±0**. + +Charts below are generated from `data/chunking_benchmark.db` with the same composite as the Decision Board: + +```text +(0.3 × context relevance + 0.4 × answer similarity + 0.3 × faithfulness) + × (1 − hallucination rate) +``` + +Regenerate figures after new Experiments: + +```bash +.venv/bin/python docs/assets/decision/generate_charts.py +``` + +--- + +## 1. Why `fixed_size` is the winner + +Decision Board ranks by **mean composite**, not by “how many documents won.” Stage 1 auto-picks **`fixed_size ±3`** vs **`semantic @ text-embedding-3-large`**. Stage 2 mean composite is **+0.313** for `fixed_size`. + +![Stage 1 mean composite by Strategy Candidate](assets/decision/stage1-mean-composite.svg) + +| Candidate | Mean composite | Context | Similarity | Faithfulness | Hallucination | Docs | +|-----------|----------------|---------|------------|--------------|---------------|------| +| **fixed_size ±3** | **9.040** | 9.40 | 8.95 | 9.47 | 2.2% | 10/10 | +| fixed_size ±2 | 8.987 | 9.43 | 8.93 | 9.44 | 2.7% | 10/10 | +| fixed_size ±1 | 8.926 | 9.38 | 8.89 | 9.42 | 3.0% | 10/10 | +| **semantic @ text-embedding-3-large** | **8.727** | 9.38 | 8.83 | 9.30 | 4.8% | 10/10 | +| fixed_size ±0 | 8.606 | 9.18 | 8.79 | 9.25 | 4.9% | 10/10 | +| semantic @ nomic-embed-text-v2-moe | 8.551 | 9.19 | 8.65 | 9.14 | 4.7% | 10/10 | + +`fixed_size ±1`, `±2`, and `±3` all beat the best semantic Candidate on **mean**. **`±0` (no Neighbor Expansion) does not** (8.606 vs 8.727). Shipping `fixed_size` without expansion would weaken this family decision. + +--- + +## 2. Judge metrics (stage 2 winners) + +`fixed_size ±3` leads on every judge metric, including **lower hallucination**. + +![Mean judge metrics for stage 2 winners](assets/decision/stage2-metrics.svg) + +--- + +## 3. Per-document showdown + +Mean ranking still favors `fixed_size`. **Head-to-head at ±3: semantic 6 / `fixed_size` 4.** Semantic’s six wins are mostly **small**; `fixed_size`’s four wins are **larger**, especially **fire**. + +![Per-document composite: fixed_size ±3 vs semantic @ large](assets/decision/stage2-per-document.svg) + +![Signed margin per document (fixed_size − semantic)](assets/decision/stage2-margins.svg) + +| Document | Questions | fixed_size ±3 | semantic @ large | Winner | +|----------|-----------|---------------|------------------|--------| +| fire.docx | 20 | **8.810** | 6.228 | **fixed_size** (+2.58) | +| moavenin.docx | 20 | **9.735** | 8.621 | **fixed_size** (+1.11) | +| general-havades-individuals.doc | 20 | **9.205** | 8.564 | **fixed_size** (+0.64) | +| havades.docx | 20 | **8.483** | 7.969 | **fixed_size** (+0.51) | +| bazresi.docx | 16 | 9.366 | **9.405** | semantic (−0.04) | +| life-time-individual.docx | 20 | 9.445 | **9.505** | semantic (−0.06) | +| Refah.docx | 20 | 9.680 | **9.815** | semantic (−0.14) | +| lifetime-compensation.docx | 15 | 8.761 | **9.129** | semantic (−0.37) | +| customer1.docx | 20 | 8.451 | **8.973** | semantic (−0.52) | +| website.docx | 30 | 8.466 | **9.058** | semantic (−0.59) | + +Win-count would pick semantic. **Official product ranking (mean composite) picks `fixed_size`.** This memo follows the Decision Board rule (ADR-0026). + +--- + +## 4. Full Candidate heatmap + +Every cell is a newest single-strategy Experiment under `text-embedding-3-large`. Semantic @ large **collapses on fire**; `fixed_size ±1…±3` stay high across the set. + +![Heatmap of composite scores by document and Candidate](assets/decision/heatmap-candidates.svg) + +Best `fixed_size` ±N **by document** (does not change the family call): + +| ±N | Documents where it is the best fixed_size cell | +|----|------------------------------------------------| +| ±3 | bazresi, fire, general-havades-individuals, havades, life-time-individual (5) | +| ±1 | customer1, moavenin, Refah (3) | +| ±2 | website (1) | +| ±0 | lifetime-compensation (1) | + +--- + +## 5. Method + +- Corpus Embedding Model locked to **`text-embedding-3-large`** +- Newest **single-strategy** Experiment per document × Candidate (Decision Board cells) +- Retrieval `top_k = 5` +- Stage 1: best `fixed_size` ±N vs best `semantic@Boundary` +- Stage 2: mean composite + per-document breakdown +- Experiments dated **9–10 August 2026** +- `logs/neighbor_sweep.log` recorded mid-run connection errors on some units; **SQLite now has a full 10×4 `fixed_size` grid** — treat the database as source of truth + +--- + +## 6. Excluded strategies + +No comparable **10-doc, single-strategy, `text-embedding-3-large`** Experiments exist for the three Strategies below. They are not Decision Board Candidates (ADR-0026). + +| Strategy | Why it is not the winner | +|----------|--------------------------| +| **recursive** | Not on the Decision Board grid. Informal PDF / `text-embedding-3-small` multi-strategy runs (not comparable to this close-out) were mixed; recursive **does not get Neighbor Expansion** (expansion is `fixed_size` only). | +| **contextual_retrieval** | Extra LLM call per chunk at process time. Same informal PDF/`small` runs scored **below** recursive and `fixed_size`. | +| **semantic_parent_child** | Boundary-detection cost plus parent fetch at query. Informal PDF/`small` composites were **much worse** (~1.8–2.8). Fail-hard if Boundary embeds mismatch (ADR-0020). | + +One leftover five-strategy Experiment on `customer1` under large is **invalid** (four Strategies scored 0 / errors) and was ignored. + +--- + +## 7. Binding decision + +**Use `fixed_size` for chunking** under **`text-embedding-3-large`.** + +**Do not use `semantic` as the default Strategy** on this evaluation universe. + +**Follow-up:** choose Neighbor Expansion **±N** among **±1, ±2, ±3**. Stage 1 auto-pick is **±3**. **±0 is not recommended** if this family decision is to remain valid. + +Production config change is **out of scope** for this record. + +--- + +## 8. Evidence (stage 2 cells) + +| Document | `fixed_size ±3` experiment id | `semantic @ large` experiment id | +|----------|-------------------------------|----------------------------------| +| bazresi.docx | `6fe08750ee324130b3d46d8b3bc95280` | `6263c4f9e63347cea9eb19f4d36309c0` | +| customer1.docx | `8c4d888d367b4c0299b2d19dd9654f80` | `0c4572e80902408796f0dd11776e8e55` | +| fire.docx | `230a3cf57ae74b679ecde83cfeb9b6f5` | `414162efc314406db082ba8537c2423f` | +| general-havades-individuals.doc | `7ad892cfcd6143109f77a7808c8dfbd9` | `14fa4f872373446c8b042fc8e84e6983` | +| havades.docx | `04bd45d836a64c3da2e68bb747c0c9b8` | `232238b4d3304243b82b02326ef64617` | +| life-time-individual.docx | `d682706f54824469b235c099bf5a3d3c` | `bc7d9f1fbe3540c097a5f8cc1a1c44bb` | +| moavenin.docx | `63f81cf0a632418aab2f93884518ed71` | `4e01960b5db14e81975a4fbc86c4e10c` | +| Refah.docx | `6304dba39f3f4c05935195c588a36a64` | `b5c630c45615451a93d97d80dee361df` | +| website.docx | `fb9d47750ca74761a3c82c57aa51030b` | `53256897cfb74dd7a6508e750ff1ba63` | +| lifetime-compensation.docx | `99736ffbc6da455b8b3ca3a0fb8e5ef7` | `e51c2b57e7364dfa97e5429f615d6e4e` | + +HTML reports: `GET /benchmarks/{id}/report`. Dashboard: Decision Tab, Corpus = `text-embedding-3-large`.