"""Admin service — health checks, Qdrant management, chunk preview, questions CRUD, cost estimation.""" import json import logging import os from pathlib import Path from typing import Any from qdrant_client.models import VectorParams, Distance from src.core.dependencies import get_qdrant_client, get_openai_client from src.core.config import settings from src.core.models import StrategyName from src.storage import qdrant as qdrant_store from src.storage import sqlite as db logger = logging.getLogger(__name__) # Project root (two levels up from src/admin/) PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent QUESTIONS_DIR = PROJECT_ROOT / "files" # ── Health ────────────────────────────────────────────────── def get_health() -> dict[str, Any]: """Check server, Qdrant, SQLite, OpenAI, and Ollama status.""" from src.chunking.embedding import ( get_boundary_embedding_model, get_corpus_embedding_model, ) result: dict[str, Any] = {"status": "ok"} # Check Qdrant try: client = get_qdrant_client() collections = client.get_collections() result["qdrant_connected"] = True result["qdrant_collections"] = len(collections.collections) except Exception as exc: result["qdrant_connected"] = False result["qdrant_error"] = str(exc) logger.warning("Qdrant health check failed: %s", exc) # Check SQLite try: conn = db._connect() conn.execute("SELECT 1") conn.close() result["sqlite_ok"] = True except Exception as exc: result["sqlite_ok"] = False result["sqlite_error"] = str(exc) logger.warning("SQLite health check failed: %s", exc) # Check OpenAI try: get_openai_client() result["openai_configured"] = bool(settings.openai_api_key) except Exception: result["openai_configured"] = False try: corpus = get_corpus_embedding_model() boundary = get_boundary_embedding_model() result["corpus_embedding_model"] = { "id": corpus.id, "provider": corpus.provider.value, "model_name": corpus.model_name, "dimension": corpus.dimension, "display_name": corpus.display_name, } result["boundary_embedding_model"] = { "id": boundary.id, "provider": boundary.provider.value, "model_name": boundary.model_name, "dimension": boundary.dimension, "display_name": boundary.display_name, } # Legacy alias for older Dashboard code result["active_embedding_model"] = result["corpus_embedding_model"] except Exception as exc: result["corpus_embedding_model"] = None result["boundary_embedding_model"] = None result["active_embedding_model"] = None result["active_embedding_error"] = str(exc) result["ollama_base_url"] = settings.ollama_base_url try: import urllib.request url = settings.ollama_base_url.rstrip("/") + "/api/tags" with urllib.request.urlopen(url, timeout=2) as resp: result["ollama_reachable"] = resp.status == 200 except Exception as exc: result["ollama_reachable"] = False result["ollama_error"] = str(exc) return result # ── Embedding Models ──────────────────────────────────────── def list_embedding_models() -> dict[str, Any]: """List registry entries and Boundary/Corpus defaults.""" from src.chunking.embedding import ( get_boundary_embedding_model, get_corpus_embedding_model, ) from src.chunking.embedding_models import get_semantic_threshold, list_models corpus = get_corpus_embedding_model() boundary = get_boundary_embedding_model() models = [] for m in list_models(): models.append({ "id": m.id, "provider": m.provider.value, "model_name": m.model_name, "dimension": m.dimension, "display_name": m.display_name, "task_prefixes": m.task_prefixes, "default_semantic_threshold": m.default_semantic_threshold, "semantic_threshold": get_semantic_threshold(m.id), "is_corpus_default": m.id == corpus.id, "is_boundary_default": m.id == boundary.id, "is_active": m.id == corpus.id, # legacy }) return { "corpus_id": corpus.id, "boundary_id": boundary.id, "active_id": corpus.id, # legacy "models": models, "ollama_base_url": settings.ollama_base_url, } def set_active_embedding_model(model_id: str) -> dict[str, Any]: """Legacy: set Corpus default.""" return set_corpus_embedding_model(model_id) def set_corpus_embedding_model(model_id: str) -> dict[str, Any]: """Switch the Default Corpus Embedding Model.""" from src.chunking.embedding import set_corpus_embedding_model as set_corpus try: model = set_corpus(model_id) except KeyError as exc: return {"error": str(exc)} return { "corpus_id": model.id, "model": { "id": model.id, "provider": model.provider.value, "model_name": model.model_name, "dimension": model.dimension, "display_name": model.display_name, }, } def set_boundary_embedding_model(model_id: str) -> dict[str, Any]: """Switch the Default Boundary Embedding Model.""" from src.chunking.embedding import set_boundary_embedding_model as set_boundary try: model = set_boundary(model_id) except KeyError as exc: return {"error": str(exc)} return { "boundary_id": model.id, "model": { "id": model.id, "provider": model.provider.value, "model_name": model.model_name, "dimension": model.dimension, "display_name": model.display_name, }, } def update_semantic_threshold(model_id: str, threshold: float) -> dict[str, Any]: """Persist Admin override of semantic_threshold for a registry Embedding Model.""" from src.chunking.embedding_models import ( get_model, get_semantic_threshold, set_semantic_threshold, ) try: model = get_model(model_id) value = set_semantic_threshold(model_id, threshold) except KeyError as exc: return {"error": str(exc)} except ValueError as exc: return {"error": str(exc)} return { "id": model.id, "semantic_threshold": value, "default_semantic_threshold": model.default_semantic_threshold, "effective": get_semantic_threshold(model.id), } # ── Qdrant Collections ───────────────────────────────────── def list_qdrant_collections() -> dict[str, Any]: """List all Qdrant collections with point counts and Embedding Model labels.""" from src.chunking.embedding import get_active_embedding_model client = get_qdrant_client() collections_data = client.get_collections().collections active = get_active_embedding_model() result = [] for col in collections_data: meta = qdrant_store.parse_collection_meta(col.name) try: info = client.get_collection(collection_name=col.name) points = info.points_count or 0 except Exception as exc: points = None err = str(exc) else: err = None entry = { "name": col.name, "points_count": points, "strategy": meta.get("strategy"), "embedding_model_id": meta.get("embedding_model_id"), "is_legacy": meta.get("is_legacy", False), "is_active_corpus": meta.get("embedding_model_id") == active.id, } if err: entry["error"] = err result.append(entry) # Active Model Corpus first result.sort(key=lambda c: (not c.get("is_active_corpus", False), c["name"])) return { "collections": result, "active_embedding_model_id": active.id, } def create_qdrant_collection(collection_name: str) -> dict[str, Any]: """Create a new Qdrant collection using Active Embedding Model dimension.""" from src.chunking.embedding import get_active_embedding_model client = get_qdrant_client() active = get_active_embedding_model() existing = [c.name for c in client.get_collections().collections] if collection_name in existing: return {"created": False, "message": f"Collection '{collection_name}' already exists"} meta = qdrant_store.parse_collection_meta(collection_name) dimension = active.dimension if meta.get("embedding_model_id"): try: from src.chunking.embedding_models import get_model dimension = get_model(meta["embedding_model_id"]).dimension except KeyError: pass client.create_collection( collection_name=collection_name, vectors_config=VectorParams( size=dimension, distance=Distance.COSINE, ), ) logger.info("Created Qdrant collection: %s (dim=%d)", collection_name, dimension) return {"created": True, "collection": collection_name, "dimension": dimension} def delete_qdrant_collection(collection_name: str) -> dict[str, Any]: """Delete a Qdrant collection entirely.""" client = get_qdrant_client() client.delete_collection(collection_name=collection_name) logger.info("Deleted Qdrant collection: %s", collection_name) return {"deleted": True, "collection": collection_name} def wipe_qdrant_collection_points(collection_name: str) -> dict[str, Any]: """Delete all points in a collection but keep the collection.""" client = get_qdrant_client() from qdrant_client.models import PointIdsList info = client.get_collection(collection_name=collection_name) count = info.points_count or 0 if count == 0: return {"deleted": 0, "collection": collection_name} client.delete( collection_name=collection_name, points_selector=PointIdsList(points=list(range(count))), ) logger.info("Wiped %d points from %s", count, collection_name) return {"deleted": count, "collection": collection_name} # ── Chunk Preview ─────────────────────────────────────────── def preview_chunks(doc_id: str, strategy: str | None = None) -> dict[str, Any]: """Preview chunks for a document from the Active Embedding Model's corpus.""" from src.chunking.embedding import get_active_embedding_model client = get_qdrant_client() active = get_active_embedding_model() from qdrant_client.models import Filter, FieldCondition, MatchValue # Get document info from SQLite doc = db.get_document(doc_id) if doc is None: return {"error": f"Document not found: {doc_id}"} doc_name = doc.get("filename", "") # Determine which collections to search strategies_to_search = [] if strategy: strategies_to_search = [strategy] else: strategies_to_search = [s.value for s in StrategyName] results = {} for strat_name in strategies_to_search: col_name = qdrant_store.collection_name(strat_name, active.id) try: existing = [c.name for c in client.get_collections().collections] if col_name not in existing: results[strat_name] = {"chunks": [], "count": 0, "collection": col_name} continue scroll_filter = Filter( must=[FieldCondition(key="document_name", match=MatchValue(value=doc_name))] ) points, _ = client.scroll( collection_name=col_name, scroll_filter=scroll_filter, limit=10000, with_payload=True, with_vectors=False, ) chunks = [] for p in points: payload = p.payload or {} chunks.append({ "chunk_id": payload.get("chunk_id", str(p.id)), "chunk_index": payload.get("chunk_index"), "text": (payload.get("text") or "")[:500], "token_count": payload.get("token_count"), "character_count": payload.get("character_count"), "parent_id": payload.get("parent_id"), }) # Sort by chunk_index chunks.sort(key=lambda c: c.get("chunk_index") or 0) results[strat_name] = { "chunks": chunks, "count": len(chunks), "collection": col_name, } except Exception as exc: results[strat_name] = {"error": str(exc), "chunks": [], "count": 0} return { "document_id": doc_id, "filename": doc_name, "embedding_model_id": active.id, "strategies": results, } # ── Questions Dataset ─────────────────────────────────────── def list_question_files() -> dict[str, Any]: """List JSON files in the files/ directory.""" QUESTIONS_DIR.mkdir(parents=True, exist_ok=True) files = [] for f in sorted(QUESTIONS_DIR.glob("*.json")): try: with open(f, encoding="utf-8") as fh: data = json.load(fh) count = len(data.get("questions", [])) except Exception: count = -1 files.append({ "id": f.name, "name": f.name, "questions_count": count, "size_bytes": f.stat().st_size, }) return {"files": files} def upload_questions(filename: str, content: bytes) -> dict[str, Any]: """Save a questions JSON file to the files/ directory.""" QUESTIONS_DIR.mkdir(parents=True, exist_ok=True) # Validate JSON try: data = json.loads(content.decode("utf-8")) except (json.JSONDecodeError, UnicodeDecodeError) as exc: return {"error": f"Invalid JSON: {exc}"} if "questions" not in data and not isinstance(data, list): return {"error": "Invalid format: must have a 'questions' key or be a list"} # Ensure filename ends with .json if not filename.endswith(".json"): filename = filename + ".json" dest = QUESTIONS_DIR / filename dest.write_bytes(content) logger.info("Uploaded questions file: %s", dest) return {"uploaded": True, "id": filename, "questions_count": len(data.get("questions", []) if isinstance(data, dict) else data)} def get_questions(file_id: str) -> dict[str, Any]: """Read and return the content of a questions file.""" path = QUESTIONS_DIR / file_id if not path.exists(): return {"error": f"File not found: {file_id}"} with open(path, encoding="utf-8") as f: data = json.load(f) return {"id": file_id, "data": data} def delete_questions(file_id: str) -> dict[str, Any]: """Delete a questions JSON file.""" path = QUESTIONS_DIR / file_id if not path.exists(): return {"error": f"File not found: {file_id}"} path.unlink() logger.info("Deleted questions file: %s", path) return {"deleted": True, "id": file_id} # ── Cost Estimation ───────────────────────────────────────── def estimate_cost(num_questions: int, num_strategies: int) -> dict[str, Any]: """Estimate benchmark cost. Delegates to benchmark service.""" from src.benchmarking.benchmark_service import estimate_cost as bench_estimate return bench_estimate(num_questions, num_strategies)