feat(bootstrap,api,infra): scaffold app composition, health route, and postgres/minio/qdrant adapters
Why: - first runnable slice of ADR-0012's resource-lifetime rules and ADR-0015's package layout: app-lifetime clients built once in the lifespan, released via explicit dependencies. Changes: - Settings split into per-domain nested settings (postgres/minio/ingestion/qdrant/app/logging); FastAPI app wired with /healthz, /readyz and a /v1 router; Postgres/MinIO/Qdrant adapters and SQLAlchemy models for tenants, API keys, source files, ingestion jobs/events.
This commit is contained in:
0
src/infrastructure/__init__.py
Normal file
0
src/infrastructure/__init__.py
Normal file
0
src/infrastructure/minio/__init__.py
Normal file
0
src/infrastructure/minio/__init__.py
Normal file
23
src/infrastructure/minio/client.py
Normal file
23
src/infrastructure/minio/client.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import asyncio
|
||||
|
||||
from minio import Minio
|
||||
|
||||
from src.config import MinioSettings
|
||||
|
||||
|
||||
def create_client(settings: MinioSettings) -> Minio:
|
||||
return Minio(
|
||||
settings.endpoint,
|
||||
access_key=settings.access_key,
|
||||
secret_key=settings.secret_key,
|
||||
secure=settings.secure,
|
||||
)
|
||||
|
||||
|
||||
async def ping(client: Minio, timeout: float) -> bool:
|
||||
try:
|
||||
async with asyncio.timeout(timeout):
|
||||
await asyncio.to_thread(client.bucket_exists, "healthcheck-probe")
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
0
src/infrastructure/observability/__init__.py
Normal file
0
src/infrastructure/observability/__init__.py
Normal file
84
src/infrastructure/observability/logging.py
Normal file
84
src/infrastructure/observability/logging.py
Normal file
@@ -0,0 +1,84 @@
|
||||
import logging
|
||||
import logging.config
|
||||
import sys
|
||||
|
||||
import structlog
|
||||
|
||||
from src.config import LoggingSettings
|
||||
|
||||
|
||||
def configure_logging(settings: LoggingSettings) -> None:
|
||||
shared_processors = [
|
||||
structlog.contextvars.merge_contextvars,
|
||||
structlog.stdlib.add_log_level,
|
||||
structlog.stdlib.add_logger_name,
|
||||
structlog.processors.TimeStamper(fmt="iso", utc=True),
|
||||
structlog.processors.StackInfoRenderer(),
|
||||
]
|
||||
|
||||
structlog.configure(
|
||||
processors=[
|
||||
*shared_processors,
|
||||
structlog.processors.format_exc_info,
|
||||
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
|
||||
],
|
||||
logger_factory=structlog.stdlib.LoggerFactory(),
|
||||
wrapper_class=structlog.stdlib.BoundLogger,
|
||||
cache_logger_on_first_use=True,
|
||||
)
|
||||
|
||||
renderer = (
|
||||
structlog.processors.JSONRenderer()
|
||||
if settings.json_format
|
||||
else structlog.dev.ConsoleRenderer(colors=True)
|
||||
)
|
||||
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {
|
||||
"default": {
|
||||
"()": structlog.stdlib.ProcessorFormatter,
|
||||
"processors": [
|
||||
structlog.stdlib.ProcessorFormatter.remove_processors_meta,
|
||||
renderer,
|
||||
],
|
||||
"foreign_pre_chain": [
|
||||
structlog.stdlib.ExtraAdder(),
|
||||
*shared_processors,
|
||||
],
|
||||
},
|
||||
},
|
||||
"handlers": {
|
||||
"console": {
|
||||
"class": "logging.StreamHandler",
|
||||
"level": settings.level,
|
||||
"formatter": "default",
|
||||
"stream": sys.stdout,
|
||||
},
|
||||
},
|
||||
"loggers": {
|
||||
"": {
|
||||
"handlers": ["console"],
|
||||
"level": settings.level,
|
||||
"propagate": False,
|
||||
},
|
||||
"uvicorn": {
|
||||
"handlers": ["console"],
|
||||
"level": settings.level,
|
||||
"propagate": False,
|
||||
},
|
||||
"uvicorn.access": {
|
||||
"handlers": ["console"],
|
||||
"level": settings.level,
|
||||
"propagate": False,
|
||||
},
|
||||
"sqlalchemy.engine": {
|
||||
"handlers": ["console"],
|
||||
"level": "WARNING",
|
||||
"propagate": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
0
src/infrastructure/postgres/__init__.py
Normal file
0
src/infrastructure/postgres/__init__.py
Normal file
28
src/infrastructure/postgres/database.py
Normal file
28
src/infrastructure/postgres/database.py
Normal file
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
|
||||
from src.config import PostgresSettings
|
||||
|
||||
|
||||
def create_engine(settings: PostgresSettings) -> AsyncEngine:
|
||||
return create_async_engine(settings.dsn)
|
||||
|
||||
|
||||
def create_sessionmaker(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
async def ping(engine: AsyncEngine, timeout: float) -> bool:
|
||||
try:
|
||||
async with asyncio.timeout(timeout), engine.connect() as connection:
|
||||
await connection.execute(text("SELECT 1"))
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
15
src/infrastructure/postgres/models/__init__.py
Normal file
15
src/infrastructure/postgres/models/__init__.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from src.infrastructure.postgres.models.api_key import ApiKey
|
||||
from src.infrastructure.postgres.models.base import Base
|
||||
from src.infrastructure.postgres.models.ingestion_job import IngestionJob
|
||||
from src.infrastructure.postgres.models.ingestion_job_event import IngestionJobEvent
|
||||
from src.infrastructure.postgres.models.source_file import SourceFile
|
||||
from src.infrastructure.postgres.models.tenant import Tenant
|
||||
|
||||
__all__ = [
|
||||
"ApiKey",
|
||||
"Base",
|
||||
"IngestionJob",
|
||||
"IngestionJobEvent",
|
||||
"SourceFile",
|
||||
"Tenant",
|
||||
]
|
||||
41
src/infrastructure/postgres/models/api_key.py
Normal file
41
src/infrastructure/postgres/models/api_key.py
Normal file
@@ -0,0 +1,41 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, String, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from src.infrastructure.postgres.models.base import Base
|
||||
|
||||
API_KEY_ACTOR_TYPES = ("backend", "admin", "worker")
|
||||
API_KEY_STATUSES = ("active", "revoked", "expired")
|
||||
|
||||
|
||||
class ApiKey(Base):
|
||||
__tablename__ = "api_keys"
|
||||
__table_args__ = (
|
||||
CheckConstraint(f"actor_type IN {API_KEY_ACTOR_TYPES}", name="ck_api_keys_actor_type"),
|
||||
CheckConstraint(f"status IN {API_KEY_STATUSES}", name="ck_api_keys_status"),
|
||||
Index("ix_api_keys_tenant_id_status", "tenant_id", "status"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("tenants.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(200))
|
||||
key_prefix: Mapped[str] = mapped_column(String(32), unique=True, index=True)
|
||||
key_hash: Mapped[str] = mapped_column(String(255))
|
||||
scopes: Mapped[list[str]] = mapped_column(JSONB, default=list, server_default="[]")
|
||||
actor_type: Mapped[str] = mapped_column(String(20), default="backend", server_default="backend")
|
||||
status: Mapped[str] = mapped_column(String(20), default="active", server_default="active")
|
||||
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None)
|
||||
last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None)
|
||||
|
||||
created_by: Mapped[str | None] = mapped_column(String(200), default=None)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
5
src/infrastructure/postgres/models/base.py
Normal file
5
src/infrastructure/postgres/models/base.py
Normal file
@@ -0,0 +1,5 @@
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
59
src/infrastructure/postgres/models/ingestion_job.py
Normal file
59
src/infrastructure/postgres/models/ingestion_job.py
Normal file
@@ -0,0 +1,59 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, Integer, String, Text, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from src.infrastructure.postgres.models.base import Base
|
||||
|
||||
INGESTION_JOB_STATUSES = ("queued", "running", "succeeded", "failed", "cancelled")
|
||||
INGESTION_CHUNKING_STRATEGIES = ("semantic", "fixed_size")
|
||||
|
||||
|
||||
class IngestionJob(Base):
|
||||
__tablename__ = "ingestion_jobs"
|
||||
__table_args__ = (
|
||||
CheckConstraint(f"status IN {INGESTION_JOB_STATUSES}", name="ck_ingestion_jobs_status"),
|
||||
CheckConstraint(
|
||||
f"chunking_strategy IN {INGESTION_CHUNKING_STRATEGIES}",
|
||||
name="ck_ingestion_jobs_chunking_strategy",
|
||||
),
|
||||
Index("ix_ingestion_jobs_tenant_id_status_created_at", "tenant_id", "status", "created_at"),
|
||||
Index("ix_ingestion_jobs_source_file_id_created_at", "source_file_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("tenants.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
source_file_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("source_files.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
requested_by_api_key_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("api_keys.id", ondelete="SET NULL"), default=None
|
||||
)
|
||||
status: Mapped[str] = mapped_column(String(20), default="queued", server_default="queued")
|
||||
chunking_strategy: Mapped[str | None] = mapped_column(String(20), default=None)
|
||||
embedding_model_versions: Mapped[dict[str, object]] = mapped_column(
|
||||
JSONB, default=dict, server_default="{}"
|
||||
)
|
||||
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None)
|
||||
|
||||
points_created: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
points_updated: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
points_soft_deleted: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
points_skipped: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
error_code: Mapped[str | None] = mapped_column(String(100), default=None)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, default=None)
|
||||
metadata_: Mapped[dict[str, object]] = mapped_column(
|
||||
"metadata", JSONB, default=dict, server_default="{}"
|
||||
)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
47
src/infrastructure/postgres/models/ingestion_job_event.py
Normal file
47
src/infrastructure/postgres/models/ingestion_job_event.py
Normal file
@@ -0,0 +1,47 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, String, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from src.infrastructure.postgres.models.base import Base
|
||||
|
||||
INGESTION_JOB_EVENT_LEVELS = ("info", "warning", "error")
|
||||
INGESTION_JOB_EVENT_STAGES = (
|
||||
"received",
|
||||
"parsed",
|
||||
"chunked",
|
||||
"embedded",
|
||||
"upserted",
|
||||
"completed",
|
||||
)
|
||||
|
||||
|
||||
class IngestionJobEvent(Base):
|
||||
__tablename__ = "ingestion_job_events"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
f"level IN {INGESTION_JOB_EVENT_LEVELS}", name="ck_ingestion_job_events_level"
|
||||
),
|
||||
CheckConstraint(
|
||||
f"stage IN {INGESTION_JOB_EVENT_STAGES}", name="ck_ingestion_job_events_stage"
|
||||
),
|
||||
Index(
|
||||
"ix_ingestion_job_events_ingestion_job_id_created_at", "ingestion_job_id", "created_at"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("tenants.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
ingestion_job_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("ingestion_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
level: Mapped[str] = mapped_column(String(10))
|
||||
stage: Mapped[str] = mapped_column(String(20))
|
||||
message: Mapped[str] = mapped_column(String(1000))
|
||||
details: Mapped[dict[str, object]] = mapped_column(JSONB, default=dict, server_default="{}")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
41
src/infrastructure/postgres/models/source_file.py
Normal file
41
src/infrastructure/postgres/models/source_file.py
Normal file
@@ -0,0 +1,41 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from src.infrastructure.postgres.models.base import Base
|
||||
|
||||
SOURCE_FILE_TYPES = ("csv", "xlsx", "docx", "doc")
|
||||
SOURCE_FILE_STATUSES = ("active", "superseded", "soft_deleted", "purged")
|
||||
|
||||
|
||||
class SourceFile(Base):
|
||||
__tablename__ = "source_files"
|
||||
__table_args__ = (
|
||||
CheckConstraint(f"source_type IN {SOURCE_FILE_TYPES}", name="ck_source_files_source_type"),
|
||||
CheckConstraint(f"status IN {SOURCE_FILE_STATUSES}", name="ck_source_files_status"),
|
||||
Index("ix_source_files_tenant_id_domain_created_at", "tenant_id", "domain", "created_at"),
|
||||
Index("ix_source_files_tenant_id_content_sha256", "tenant_id", "content_sha256"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||
tenant_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("tenants.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
domain: Mapped[str] = mapped_column(String(80))
|
||||
source_filename: Mapped[str] = mapped_column(String(500))
|
||||
source_type: Mapped[str] = mapped_column(String(10))
|
||||
content_sha256: Mapped[str] = mapped_column(String(64))
|
||||
byte_size: Mapped[int] = mapped_column(BigInteger)
|
||||
storage_uri: Mapped[str | None] = mapped_column(String(1000), default=None)
|
||||
status: Mapped[str] = mapped_column(String(20), default="active", server_default="active")
|
||||
|
||||
created_by_api_key_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("api_keys.id", ondelete="SET NULL"), default=None
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None)
|
||||
27
src/infrastructure/postgres/models/tenant.py
Normal file
27
src/infrastructure/postgres/models/tenant.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, String, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from src.infrastructure.postgres.models.base import Base
|
||||
|
||||
TENANT_STATUSES = ("active", "suspended", "deleted")
|
||||
|
||||
|
||||
class Tenant(Base):
|
||||
__tablename__ = "tenants"
|
||||
__table_args__ = (CheckConstraint(f"status IN {TENANT_STATUSES}", name="ck_tenants_status"),)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
|
||||
slug: Mapped[str] = mapped_column(String(80), unique=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(200))
|
||||
status: Mapped[str] = mapped_column(String(20), default="active", server_default="active")
|
||||
settings: Mapped[dict[str, object]] = mapped_column(JSONB, default=dict, server_default="{}")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None)
|
||||
0
src/infrastructure/qdrant/__init__.py
Normal file
0
src/infrastructure/qdrant/__init__.py
Normal file
18
src/infrastructure/qdrant/client.py
Normal file
18
src/infrastructure/qdrant/client.py
Normal file
@@ -0,0 +1,18 @@
|
||||
import asyncio
|
||||
|
||||
from qdrant_client import AsyncQdrantClient
|
||||
|
||||
from src.config import QdrantSettings
|
||||
|
||||
|
||||
def create_client(settings: QdrantSettings) -> AsyncQdrantClient:
|
||||
return AsyncQdrantClient(url=settings.url, api_key=settings.api_key)
|
||||
|
||||
|
||||
async def ping(client: AsyncQdrantClient, timeout: float) -> bool:
|
||||
try:
|
||||
async with asyncio.timeout(timeout):
|
||||
await client.get_collections()
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
Reference in New Issue
Block a user