fix(chunking): require semantic boundary detection

Why:
- Semantic strategies were silently falling back to fixed-count grouping when boundary embeds were missing.

Changes:
- Fail hard without aligned unit embeddings; orchestrator supplies Boundary embeds; Farsi-aware sentence split with line/paragraph fallback.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/chunking/base.py

@@ -55,13 +55,36 @@ def build_chunk(
 
 # ── Sentence splitting ────────────────────────────────────────────
 
-_SENTENCE_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9])")
+# After . ! ? or Persian/fullwidth ؟ ！, split on following whitespace.
+# Does NOT require a Latin capital next (that broke Farsi documents).
+# Periods inside numbers (12.5) are safe because there is no whitespace after.
+_SENTENCE_RE = re.compile(r"(?<=[.!?؟！])\s+")
 
 
 def split_sentences(text: str) -> list[str]:
-    """Split text into sentences using a simple regex heuristic."""
-    sentences = _SENTENCE_RE.split(text.strip())
-    return [s.strip() for s in sentences if s.strip()]
+    """Split text into sentence-like units for Semantic Boundary Detection.
+
+    Primary: punctuation-based splits (English + Farsi terminators).
+    Fallback: if that yields a single unit, use non-empty lines, then
+    blank-line paragraphs — so table/list docs still get multiple units.
+    """
+    text = text.strip()
+    if not text:
+        return []
+
+    sentences = [s.strip() for s in _SENTENCE_RE.split(text) if s.strip()]
+    if len(sentences) > 1:
+        return sentences
+
+    lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
+    if len(lines) > 1:
+        return lines
+
+    paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
+    if len(paragraphs) > 1:
+        return paragraphs
+
+    return sentences if sentences else [text]
 
 
 # ── Abstract base ─────────────────────────────────────────────────