feat(files): add source-file upload with MinIO storage and Postgres repositories
Why: - Implements plan 001 Phase 3's upload orchestration. Changes: - ObjectStorage port and MinIO adapter, thread-offloaded per ADR-0017. - Tenant-scoped repositories for api_keys, source_files, ingestion_jobs. - upload_source_file implementing the two-transaction shape with (tenant_id, domain, content_sha256) idempotency. Impact: - This phase stores bytes only -- chunks_indexed is always 0 until Phase 4/5 add parsing/embedding.
This commit is contained in:
33
src/infrastructure/minio/storage.py
Normal file
33
src/infrastructure/minio/storage.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""MinIO adapter for the `ObjectStorage` port (ADR-0013, ADR-0017).
|
||||
|
||||
The `minio` SDK is synchronous, so every call runs through
|
||||
`anyio.to_thread.run_sync` bounded by the ingestion `CapacityLimiter` — the
|
||||
same rule ADR-0017 applies to parsing/chunking. Calling the SDK directly from
|
||||
`async def` would block every concurrent request in the process.
|
||||
"""
|
||||
|
||||
import io
|
||||
from functools import partial
|
||||
|
||||
from anyio import CapacityLimiter, to_thread
|
||||
from minio import Minio
|
||||
|
||||
|
||||
class MinioObjectStorage:
|
||||
def __init__(self, client: Minio, *, bucket: str, limiter: CapacityLimiter) -> None:
|
||||
self._client = client
|
||||
self._bucket = bucket
|
||||
self._limiter = limiter
|
||||
|
||||
async def put_object(self, *, key: str, data: bytes, content_type: str) -> None:
|
||||
await to_thread.run_sync(
|
||||
partial(
|
||||
self._client.put_object,
|
||||
self._bucket,
|
||||
key,
|
||||
io.BytesIO(data),
|
||||
length=len(data),
|
||||
content_type=content_type,
|
||||
),
|
||||
limiter=self._limiter,
|
||||
)
|
||||
17
src/infrastructure/postgres/repositories/api_keys.py
Normal file
17
src/infrastructure/postgres/repositories/api_keys.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""API-key lookups (ADR-0008, ADR-0009).
|
||||
|
||||
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||
commits, rolls back, or closes the session (ADR-0012). Secret comparison
|
||||
happens in `src/application/auth`, not here — this module only fetches rows
|
||||
by their non-secret `key_prefix`.
|
||||
"""
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.infrastructure.postgres.models.api_key import ApiKey
|
||||
|
||||
|
||||
async def get_by_prefix(session: AsyncSession, key_prefix: str) -> ApiKey | None:
|
||||
result = await session.execute(select(ApiKey).where(ApiKey.key_prefix == key_prefix))
|
||||
return result.scalar_one_or_none()
|
||||
119
src/infrastructure/postgres/repositories/ingestion_jobs.py
Normal file
119
src/infrastructure/postgres/repositories/ingestion_jobs.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""`ingestion_jobs`/`ingestion_job_events` persistence (ADR-0009, ADR-0017).
|
||||
|
||||
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||
commits, rolls back, or closes the session (ADR-0012) — the two-transaction
|
||||
shape in `src/application/files/upload.py` depends on that. Every read is
|
||||
tenant-scoped by a required `tenant_id` argument.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import desc, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.infrastructure.postgres.models.ingestion_job import IngestionJob
|
||||
from src.infrastructure.postgres.models.ingestion_job_event import IngestionJobEvent
|
||||
|
||||
|
||||
async def get_by_id(
|
||||
session: AsyncSession, *, tenant_id: uuid.UUID, ingestion_job_id: uuid.UUID
|
||||
) -> IngestionJob | None:
|
||||
result = await session.execute(
|
||||
select(IngestionJob).where(
|
||||
IngestionJob.id == ingestion_job_id, IngestionJob.tenant_id == tenant_id
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_latest_for_source_file(
|
||||
session: AsyncSession, *, tenant_id: uuid.UUID, source_file_id: uuid.UUID
|
||||
) -> IngestionJob | None:
|
||||
result = await session.execute(
|
||||
select(IngestionJob)
|
||||
.where(
|
||||
IngestionJob.tenant_id == tenant_id,
|
||||
IngestionJob.source_file_id == source_file_id,
|
||||
)
|
||||
.order_by(desc(IngestionJob.created_at))
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def create_running(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
tenant_id: uuid.UUID,
|
||||
source_file_id: uuid.UUID,
|
||||
requested_by_api_key_id: uuid.UUID | None,
|
||||
chunking_strategy: str,
|
||||
) -> IngestionJob:
|
||||
job = IngestionJob(
|
||||
id=uuid.uuid4(),
|
||||
tenant_id=tenant_id,
|
||||
source_file_id=source_file_id,
|
||||
requested_by_api_key_id=requested_by_api_key_id,
|
||||
status="running",
|
||||
chunking_strategy=chunking_strategy,
|
||||
started_at=datetime.now(UTC),
|
||||
)
|
||||
session.add(job)
|
||||
return job
|
||||
|
||||
|
||||
async def mark_terminal(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
tenant_id: uuid.UUID,
|
||||
ingestion_job_id: uuid.UUID,
|
||||
status: str,
|
||||
points_created: int = 0,
|
||||
points_updated: int = 0,
|
||||
points_soft_deleted: int = 0,
|
||||
points_skipped: int = 0,
|
||||
error_code: str | None = None,
|
||||
error_message: str | None = None,
|
||||
) -> IngestionJob | None:
|
||||
"""Move a job from `running` to a terminal status.
|
||||
|
||||
Never reads/writes a job whose current status is already terminal — a
|
||||
terminal job must not transition back to `running` or to a different
|
||||
terminal status (ADR-0017).
|
||||
"""
|
||||
job = await get_by_id(session, tenant_id=tenant_id, ingestion_job_id=ingestion_job_id)
|
||||
if job is None or job.status != "running":
|
||||
return None
|
||||
job.status = status
|
||||
job.completed_at = datetime.now(UTC)
|
||||
job.points_created = points_created
|
||||
job.points_updated = points_updated
|
||||
job.points_soft_deleted = points_soft_deleted
|
||||
job.points_skipped = points_skipped
|
||||
job.error_code = error_code
|
||||
job.error_message = error_message
|
||||
return job
|
||||
|
||||
|
||||
def append_event(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
tenant_id: uuid.UUID,
|
||||
ingestion_job_id: uuid.UUID,
|
||||
level: str,
|
||||
stage: str,
|
||||
message: str,
|
||||
details: dict[str, object] | None = None,
|
||||
) -> IngestionJobEvent:
|
||||
event = IngestionJobEvent(
|
||||
id=uuid.uuid4(),
|
||||
tenant_id=tenant_id,
|
||||
ingestion_job_id=ingestion_job_id,
|
||||
level=level,
|
||||
stage=stage,
|
||||
message=message,
|
||||
details=details or {},
|
||||
)
|
||||
session.add(event)
|
||||
return event
|
||||
69
src/infrastructure/postgres/repositories/source_files.py
Normal file
69
src/infrastructure/postgres/repositories/source_files.py
Normal file
@@ -0,0 +1,69 @@
|
||||
"""`source_files` persistence (ADR-0009).
|
||||
|
||||
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||
commits, rolls back, or closes the session (ADR-0012). Every read is
|
||||
tenant-scoped by a required `tenant_id` argument, so a missing filter is a
|
||||
signature error rather than a cross-tenant leak.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.infrastructure.postgres.models.source_file import SourceFile
|
||||
|
||||
|
||||
async def get_by_id(
|
||||
session: AsyncSession, *, tenant_id: uuid.UUID, source_file_id: uuid.UUID
|
||||
) -> SourceFile | None:
|
||||
result = await session.execute(
|
||||
select(SourceFile).where(SourceFile.id == source_file_id, SourceFile.tenant_id == tenant_id)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def find_active_by_content_hash(
|
||||
session: AsyncSession, *, tenant_id: uuid.UUID, domain: str, content_sha256: str
|
||||
) -> SourceFile | None:
|
||||
result = await session.execute(
|
||||
select(SourceFile).where(
|
||||
SourceFile.tenant_id == tenant_id,
|
||||
SourceFile.domain == domain,
|
||||
SourceFile.content_sha256 == content_sha256,
|
||||
SourceFile.status == "active",
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
def create(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
source_file_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
domain: str,
|
||||
source_filename: str,
|
||||
source_type: str,
|
||||
content_sha256: str,
|
||||
byte_size: int,
|
||||
storage_uri: str,
|
||||
created_by_api_key_id: uuid.UUID | None,
|
||||
) -> SourceFile:
|
||||
"""`source_file_id` is caller-generated: the upload service derives the
|
||||
MinIO object key from it before this row exists, so the id has to be
|
||||
chosen up front rather than assigned by the database.
|
||||
"""
|
||||
source_file = SourceFile(
|
||||
id=source_file_id,
|
||||
tenant_id=tenant_id,
|
||||
domain=domain,
|
||||
source_filename=source_filename,
|
||||
source_type=source_type,
|
||||
content_sha256=content_sha256,
|
||||
byte_size=byte_size,
|
||||
storage_uri=storage_uri,
|
||||
created_by_api_key_id=created_by_api_key_id,
|
||||
)
|
||||
session.add(source_file)
|
||||
return source_file
|
||||
15
src/infrastructure/postgres/repositories/tenants.py
Normal file
15
src/infrastructure/postgres/repositories/tenants.py
Normal file
@@ -0,0 +1,15 @@
|
||||
"""Tenant lookups (ADR-0009).
|
||||
|
||||
Plain functions over an `AsyncSession` the caller owns. No function here
|
||||
commits, rolls back, or closes the session (ADR-0012).
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.infrastructure.postgres.models.tenant import Tenant
|
||||
|
||||
|
||||
async def get_by_id(session: AsyncSession, tenant_id: uuid.UUID) -> Tenant | None:
|
||||
return await session.get(Tenant, tenant_id)
|
||||
Reference in New Issue
Block a user