feat(files): add source-file upload with MinIO storage and Postgres repositories

Why:
- Implements plan 001 Phase 3's upload orchestration.

Changes:
- ObjectStorage port and MinIO adapter, thread-offloaded per ADR-0017.
- Tenant-scoped repositories for api_keys, source_files, ingestion_jobs.
- upload_source_file implementing the two-transaction shape with
  (tenant_id, domain, content_sha256) idempotency.

Impact:
- This phase stores bytes only -- chunks_indexed is always 0 until
  Phase 4/5 add parsing/embedding.
This commit is contained in:
2026-08-19 15:00:27 +03:30
parent e97ce6e5f3
commit c9cf7b368b
19 changed files with 758 additions and 4 deletions

View File

@@ -0,0 +1,33 @@
"""MinIO adapter for the `ObjectStorage` port (ADR-0013, ADR-0017).
The `minio` SDK is synchronous, so every call runs through
`anyio.to_thread.run_sync` bounded by the ingestion `CapacityLimiter` — the
same rule ADR-0017 applies to parsing/chunking. Calling the SDK directly from
`async def` would block every concurrent request in the process.
"""
import io
from functools import partial
from anyio import CapacityLimiter, to_thread
from minio import Minio
class MinioObjectStorage:
def __init__(self, client: Minio, *, bucket: str, limiter: CapacityLimiter) -> None:
self._client = client
self._bucket = bucket
self._limiter = limiter
async def put_object(self, *, key: str, data: bytes, content_type: str) -> None:
await to_thread.run_sync(
partial(
self._client.put_object,
self._bucket,
key,
io.BytesIO(data),
length=len(data),
content_type=content_type,
),
limiter=self._limiter,
)

View File

@@ -0,0 +1,17 @@
"""API-key lookups (ADR-0008, ADR-0009).
Plain functions over an `AsyncSession` the caller owns. No function here
commits, rolls back, or closes the session (ADR-0012). Secret comparison
happens in `src/application/auth`, not here — this module only fetches rows
by their non-secret `key_prefix`.
"""
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from src.infrastructure.postgres.models.api_key import ApiKey
async def get_by_prefix(session: AsyncSession, key_prefix: str) -> ApiKey | None:
result = await session.execute(select(ApiKey).where(ApiKey.key_prefix == key_prefix))
return result.scalar_one_or_none()

View File

@@ -0,0 +1,119 @@
"""`ingestion_jobs`/`ingestion_job_events` persistence (ADR-0009, ADR-0017).
Plain functions over an `AsyncSession` the caller owns. No function here
commits, rolls back, or closes the session (ADR-0012) — the two-transaction
shape in `src/application/files/upload.py` depends on that. Every read is
tenant-scoped by a required `tenant_id` argument.
"""
import uuid
from datetime import UTC, datetime
from sqlalchemy import desc, select
from sqlalchemy.ext.asyncio import AsyncSession
from src.infrastructure.postgres.models.ingestion_job import IngestionJob
from src.infrastructure.postgres.models.ingestion_job_event import IngestionJobEvent
async def get_by_id(
session: AsyncSession, *, tenant_id: uuid.UUID, ingestion_job_id: uuid.UUID
) -> IngestionJob | None:
result = await session.execute(
select(IngestionJob).where(
IngestionJob.id == ingestion_job_id, IngestionJob.tenant_id == tenant_id
)
)
return result.scalar_one_or_none()
async def get_latest_for_source_file(
session: AsyncSession, *, tenant_id: uuid.UUID, source_file_id: uuid.UUID
) -> IngestionJob | None:
result = await session.execute(
select(IngestionJob)
.where(
IngestionJob.tenant_id == tenant_id,
IngestionJob.source_file_id == source_file_id,
)
.order_by(desc(IngestionJob.created_at))
.limit(1)
)
return result.scalar_one_or_none()
def create_running(
session: AsyncSession,
*,
tenant_id: uuid.UUID,
source_file_id: uuid.UUID,
requested_by_api_key_id: uuid.UUID | None,
chunking_strategy: str,
) -> IngestionJob:
job = IngestionJob(
id=uuid.uuid4(),
tenant_id=tenant_id,
source_file_id=source_file_id,
requested_by_api_key_id=requested_by_api_key_id,
status="running",
chunking_strategy=chunking_strategy,
started_at=datetime.now(UTC),
)
session.add(job)
return job
async def mark_terminal(
session: AsyncSession,
*,
tenant_id: uuid.UUID,
ingestion_job_id: uuid.UUID,
status: str,
points_created: int = 0,
points_updated: int = 0,
points_soft_deleted: int = 0,
points_skipped: int = 0,
error_code: str | None = None,
error_message: str | None = None,
) -> IngestionJob | None:
"""Move a job from `running` to a terminal status.
Never reads/writes a job whose current status is already terminal — a
terminal job must not transition back to `running` or to a different
terminal status (ADR-0017).
"""
job = await get_by_id(session, tenant_id=tenant_id, ingestion_job_id=ingestion_job_id)
if job is None or job.status != "running":
return None
job.status = status
job.completed_at = datetime.now(UTC)
job.points_created = points_created
job.points_updated = points_updated
job.points_soft_deleted = points_soft_deleted
job.points_skipped = points_skipped
job.error_code = error_code
job.error_message = error_message
return job
def append_event(
session: AsyncSession,
*,
tenant_id: uuid.UUID,
ingestion_job_id: uuid.UUID,
level: str,
stage: str,
message: str,
details: dict[str, object] | None = None,
) -> IngestionJobEvent:
event = IngestionJobEvent(
id=uuid.uuid4(),
tenant_id=tenant_id,
ingestion_job_id=ingestion_job_id,
level=level,
stage=stage,
message=message,
details=details or {},
)
session.add(event)
return event

View File

@@ -0,0 +1,69 @@
"""`source_files` persistence (ADR-0009).
Plain functions over an `AsyncSession` the caller owns. No function here
commits, rolls back, or closes the session (ADR-0012). Every read is
tenant-scoped by a required `tenant_id` argument, so a missing filter is a
signature error rather than a cross-tenant leak.
"""
import uuid
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from src.infrastructure.postgres.models.source_file import SourceFile
async def get_by_id(
session: AsyncSession, *, tenant_id: uuid.UUID, source_file_id: uuid.UUID
) -> SourceFile | None:
result = await session.execute(
select(SourceFile).where(SourceFile.id == source_file_id, SourceFile.tenant_id == tenant_id)
)
return result.scalar_one_or_none()
async def find_active_by_content_hash(
session: AsyncSession, *, tenant_id: uuid.UUID, domain: str, content_sha256: str
) -> SourceFile | None:
result = await session.execute(
select(SourceFile).where(
SourceFile.tenant_id == tenant_id,
SourceFile.domain == domain,
SourceFile.content_sha256 == content_sha256,
SourceFile.status == "active",
)
)
return result.scalar_one_or_none()
def create(
session: AsyncSession,
*,
source_file_id: uuid.UUID,
tenant_id: uuid.UUID,
domain: str,
source_filename: str,
source_type: str,
content_sha256: str,
byte_size: int,
storage_uri: str,
created_by_api_key_id: uuid.UUID | None,
) -> SourceFile:
"""`source_file_id` is caller-generated: the upload service derives the
MinIO object key from it before this row exists, so the id has to be
chosen up front rather than assigned by the database.
"""
source_file = SourceFile(
id=source_file_id,
tenant_id=tenant_id,
domain=domain,
source_filename=source_filename,
source_type=source_type,
content_sha256=content_sha256,
byte_size=byte_size,
storage_uri=storage_uri,
created_by_api_key_id=created_by_api_key_id,
)
session.add(source_file)
return source_file

View File

@@ -0,0 +1,15 @@
"""Tenant lookups (ADR-0009).
Plain functions over an `AsyncSession` the caller owns. No function here
commits, rolls back, or closes the session (ADR-0012).
"""
import uuid
from sqlalchemy.ext.asyncio import AsyncSession
from src.infrastructure.postgres.models.tenant import Tenant
async def get_by_id(session: AsyncSession, tenant_id: uuid.UUID) -> Tenant | None:
return await session.get(Tenant, tenant_id)