Mahdi Bazrafshan
640691b8ab
Why: - Close the benchmark with a written family decision backed by Experiment composites. Changes: - Add the decision memo and SVG charts generated from SQLite. Impact: - Documents fixed_size over semantic under text-embedding-3-large; ±N remains follow-up. Co-authored-by: Cursor <cursoragent@cursor.com>
144 lines
16 KiB
XML
144 lines
16 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="620" height="428" viewBox="0 0 620 428" role="img" aria-label="Candidate heatmap">
|
||
<title>Candidate heatmap</title>
|
||
<rect width="620" height="428" fill="#ffffff"/>
|
||
<text x="24" y="28" font-family="Inter, system-ui, sans-serif" font-size="16" font-weight="700" fill="#111827">Candidate heatmap — composite by document</text>
|
||
<text x="24" y="48" font-family="Inter, system-ui, sans-serif" font-size="12" fill="#6b7280">Darker teal = stronger. Semantic @ large collapses on fire; ±1–±3 stay high across the set.</text>
|
||
<text x="167.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±0</text>
|
||
<text x="245.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±1</text>
|
||
<text x="323.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±2</text>
|
||
<text x="401.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">±3</text>
|
||
<text x="479.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">sem@large</text>
|
||
<text x="557.0" y="62" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">sem@nomic</text>
|
||
<text x="120" y="92.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">bazresi</text>
|
||
<rect x="128" y="72" width="75" height="29" rx="4" fill="#e19d15"/>
|
||
<text x="165.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.52</text>
|
||
<rect x="206" y="72" width="75" height="29" rx="4" fill="#58975f"/>
|
||
<text x="243.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.34</text>
|
||
<rect x="284" y="72" width="75" height="29" rx="4" fill="#58975f"/>
|
||
<text x="321.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.34</text>
|
||
<rect x="362" y="72" width="75" height="29" rx="4" fill="#549761"/>
|
||
<text x="399.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.37</text>
|
||
<rect x="440" y="72" width="75" height="29" rx="4" fill="#4e9664"/>
|
||
<text x="477.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.40</text>
|
||
<rect x="518" y="72" width="75" height="29" rx="4" fill="#7c984b"/>
|
||
<text x="555.5" y="91.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.12</text>
|
||
<text x="120" y="124.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">customer1</text>
|
||
<rect x="128" y="104" width="75" height="29" rx="4" fill="#9f9a39"/>
|
||
<text x="165.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.92</text>
|
||
<rect x="206" y="104" width="75" height="29" rx="4" fill="#96993e"/>
|
||
<text x="243.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.97</text>
|
||
<rect x="284" y="104" width="75" height="29" rx="4" fill="#97993d"/>
|
||
<text x="321.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.96</text>
|
||
<rect x="362" y="104" width="75" height="29" rx="4" fill="#ec9d0f"/>
|
||
<text x="399.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.45</text>
|
||
<rect x="440" y="104" width="75" height="29" rx="4" fill="#96993e"/>
|
||
<text x="477.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.97</text>
|
||
<rect x="518" y="104" width="75" height="29" rx="4" fill="#f88255"/>
|
||
<text x="555.5" y="123.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.54</text>
|
||
<text x="120" y="156.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">fire</text>
|
||
<rect x="128" y="136" width="75" height="29" rx="4" fill="#f69426"/>
|
||
<text x="165.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.09</text>
|
||
<rect x="206" y="136" width="75" height="29" rx="4" fill="#f59c10"/>
|
||
<text x="243.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.34</text>
|
||
<rect x="284" y="136" width="75" height="29" rx="4" fill="#f68f33"/>
|
||
<text x="321.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.93</text>
|
||
<rect x="362" y="136" width="75" height="29" rx="4" fill="#b19b2f"/>
|
||
<text x="399.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.81</text>
|
||
<rect x="440" y="136" width="75" height="29" rx="4" fill="#fb7185"/>
|
||
<text x="477.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">6.23</text>
|
||
<rect x="518" y="136" width="75" height="29" rx="4" fill="#fa7674"/>
|
||
<text x="555.5" y="155.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.19</text>
|
||
<text x="120" y="188.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades-ind</text>
|
||
<rect x="128" y="168" width="75" height="29" rx="4" fill="#f78b3c"/>
|
||
<text x="165.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.83</text>
|
||
<rect x="206" y="168" width="75" height="29" rx="4" fill="#c99c22"/>
|
||
<text x="243.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.66</text>
|
||
<rect x="284" y="168" width="75" height="29" rx="4" fill="#7e984a"/>
|
||
<text x="321.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.12</text>
|
||
<rect x="362" y="168" width="75" height="29" rx="4" fill="#6f9852"/>
|
||
<text x="399.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.21</text>
|
||
<rect x="440" y="168" width="75" height="29" rx="4" fill="#d99c19"/>
|
||
<text x="477.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.56</text>
|
||
<rect x="518" y="168" width="75" height="29" rx="4" fill="#e09d16"/>
|
||
<text x="555.5" y="187.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.53</text>
|
||
<text x="120" y="220.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">havades</text>
|
||
<rect x="128" y="200" width="75" height="29" rx="4" fill="#f59b12"/>
|
||
<text x="165.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.31</text>
|
||
<rect x="206" y="200" width="75" height="29" rx="4" fill="#ef9d0d"/>
|
||
<text x="243.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.43</text>
|
||
<rect x="284" y="200" width="75" height="29" rx="4" fill="#f49d0b"/>
|
||
<text x="321.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.40</text>
|
||
<rect x="362" y="200" width="75" height="29" rx="4" fill="#e79d12"/>
|
||
<text x="399.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.48</text>
|
||
<rect x="440" y="200" width="75" height="29" rx="4" fill="#f69030"/>
|
||
<text x="477.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.97</text>
|
||
<rect x="518" y="200" width="75" height="29" rx="4" fill="#f29d0c"/>
|
||
<text x="555.5" y="219.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.42</text>
|
||
<text x="120" y="252.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-ind</text>
|
||
<rect x="128" y="232" width="75" height="29" rx="4" fill="#6d9854"/>
|
||
<text x="165.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.22</text>
|
||
<rect x="206" y="232" width="75" height="29" rx="4" fill="#4b9666"/>
|
||
<text x="243.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.42</text>
|
||
<rect x="284" y="232" width="75" height="29" rx="4" fill="#489667"/>
|
||
<text x="321.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.44</text>
|
||
<rect x="362" y="232" width="75" height="29" rx="4" fill="#479668"/>
|
||
<text x="399.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.45</text>
|
||
<rect x="440" y="232" width="75" height="29" rx="4" fill="#3d966d"/>
|
||
<text x="477.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.50</text>
|
||
<rect x="518" y="232" width="75" height="29" rx="4" fill="#4e9664"/>
|
||
<text x="555.5" y="251.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.41</text>
|
||
<text x="120" y="284.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">moavenin</text>
|
||
<rect x="128" y="264" width="75" height="29" rx="4" fill="#879945"/>
|
||
<text x="165.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.06</text>
|
||
<rect x="206" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||
<text x="243.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||
<rect x="284" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||
<text x="321.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||
<rect x="362" y="264" width="75" height="29" rx="4" fill="#179482"/>
|
||
<text x="399.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.73</text>
|
||
<rect x="440" y="264" width="75" height="29" rx="4" fill="#d09c1e"/>
|
||
<text x="477.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.62</text>
|
||
<rect x="518" y="264" width="75" height="29" rx="4" fill="#f59a14"/>
|
||
<text x="555.5" y="283.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.29</text>
|
||
<text x="120" y="316.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">Refah</text>
|
||
<rect x="128" y="296" width="75" height="29" rx="4" fill="#459669"/>
|
||
<text x="165.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.46</text>
|
||
<rect x="206" y="296" width="75" height="29" rx="4" fill="#1c947f"/>
|
||
<text x="243.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.71</text>
|
||
<rect x="284" y="296" width="75" height="29" rx="4" fill="#1e947e"/>
|
||
<text x="321.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.70</text>
|
||
<rect x="362" y="296" width="75" height="29" rx="4" fill="#20947d"/>
|
||
<text x="399.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.68</text>
|
||
<rect x="440" y="296" width="75" height="29" rx="4" fill="#0d9488"/>
|
||
<text x="477.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.81</text>
|
||
<rect x="518" y="296" width="75" height="29" rx="4" fill="#349572"/>
|
||
<text x="555.5" y="315.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.56</text>
|
||
<text x="120" y="348.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">website</text>
|
||
<rect x="128" y="328" width="75" height="29" rx="4" fill="#f8864b"/>
|
||
<text x="165.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.67</text>
|
||
<rect x="206" y="328" width="75" height="29" rx="4" fill="#f78e35"/>
|
||
<text x="243.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">7.91</text>
|
||
<rect x="284" y="328" width="75" height="29" rx="4" fill="#e29d15"/>
|
||
<text x="321.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.51</text>
|
||
<rect x="362" y="328" width="75" height="29" rx="4" fill="#ea9d10"/>
|
||
<text x="399.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.47</text>
|
||
<rect x="440" y="328" width="75" height="29" rx="4" fill="#879945"/>
|
||
<text x="477.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.06</text>
|
||
<rect x="518" y="328" width="75" height="29" rx="4" fill="#e59d13"/>
|
||
<text x="555.5" y="347.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.49</text>
|
||
<text x="120" y="380.8" text-anchor="end" font-family="Inter, system-ui, sans-serif" font-size="11" fill="#111827">lifetime-comp</text>
|
||
<rect x="128" y="360" width="75" height="29" rx="4" fill="#93993f"/>
|
||
<text x="165.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.99</text>
|
||
<rect x="206" y="360" width="75" height="29" rx="4" fill="#bd9b28"/>
|
||
<text x="243.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.73</text>
|
||
<rect x="284" y="360" width="75" height="29" rx="4" fill="#bd9b28"/>
|
||
<text x="321.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.73</text>
|
||
<rect x="362" y="360" width="75" height="29" rx="4" fill="#b99b2b"/>
|
||
<text x="399.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.76</text>
|
||
<rect x="440" y="360" width="75" height="29" rx="4" fill="#7c984c"/>
|
||
<text x="477.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#f9fafb">9.13</text>
|
||
<rect x="518" y="360" width="75" height="29" rx="4" fill="#98993d"/>
|
||
<text x="555.5" y="379.8" text-anchor="middle" font-family="Inter, system-ui, sans-serif" font-size="11" font-weight="600" fill="#111827">8.96</text>
|
||
</svg>
|