refactor(ingestion): give the pipeline package a single async entry point
Why: - The package exposed 8 modules directly, pushing source-type dispatch and the ADR-0017 thread-offload obligation onto every caller. Changes: - Add parse_and_chunk_document as the sole public entry point. - Demote the individual parsers to internal/test-only.
This commit is contained in:
62
tests/unit/application/test_pipeline.py
Normal file
62
tests/unit/application/test_pipeline.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""Source-type dispatch and thread-offload for `parse_and_chunk_document`."""
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from anyio import CapacityLimiter
|
||||
|
||||
from src.application.ingestion import UnsupportedSourceTypeError, parse_and_chunk_document
|
||||
from src.config import ChunkingSettings
|
||||
from tests.support.documents import PROSE_DOCX, load_document
|
||||
|
||||
pytestmark = [pytest.mark.unit, pytest.mark.asyncio]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def settings() -> ChunkingSettings:
|
||||
return ChunkingSettings()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def limiter() -> CapacityLimiter:
|
||||
return CapacityLimiter(2)
|
||||
|
||||
|
||||
async def test_parse_and_chunk_document_docx_returns_chunks(
|
||||
settings: ChunkingSettings, limiter: CapacityLimiter
|
||||
) -> None:
|
||||
chunks = await parse_and_chunk_document(
|
||||
load_document(PROSE_DOCX),
|
||||
source_type="docx",
|
||||
file_id=uuid.uuid4(),
|
||||
settings=settings,
|
||||
limiter=limiter,
|
||||
)
|
||||
|
||||
assert chunks
|
||||
assert all(chunk.chunk_index == index for index, chunk in enumerate(chunks))
|
||||
|
||||
|
||||
async def test_parse_and_chunk_document_csv_returns_chunks(
|
||||
settings: ChunkingSettings, limiter: CapacityLimiter
|
||||
) -> None:
|
||||
data = b"name,value\nfirst,1\nsecond,2\n"
|
||||
|
||||
chunks = await parse_and_chunk_document(
|
||||
data, source_type="csv", file_id=uuid.uuid4(), settings=settings, limiter=limiter
|
||||
)
|
||||
|
||||
assert chunks
|
||||
|
||||
|
||||
async def test_parse_and_chunk_document_unsupported_type_raises(
|
||||
settings: ChunkingSettings, limiter: CapacityLimiter
|
||||
) -> None:
|
||||
with pytest.raises(UnsupportedSourceTypeError):
|
||||
await parse_and_chunk_document(
|
||||
b"whatever",
|
||||
source_type="doc",
|
||||
file_id=uuid.uuid4(),
|
||||
settings=settings,
|
||||
limiter=limiter,
|
||||
)
|
||||
Reference in New Issue
Block a user