Ali Zarinkolah
94684d97ae
Why: - The package exposed 8 modules directly, pushing source-type dispatch and the ADR-0017 thread-offload obligation onto every caller. Changes: - Add parse_and_chunk_document as the sole public entry point. - Demote the individual parsers to internal/test-only.
63 lines
1.7 KiB
Python
63 lines
1.7 KiB
Python
"""Source-type dispatch and thread-offload for `parse_and_chunk_document`."""
|
|
|
|
import uuid
|
|
|
|
import pytest
|
|
from anyio import CapacityLimiter
|
|
|
|
from src.application.ingestion import UnsupportedSourceTypeError, parse_and_chunk_document
|
|
from src.config import ChunkingSettings
|
|
from tests.support.documents import PROSE_DOCX, load_document
|
|
|
|
pytestmark = [pytest.mark.unit, pytest.mark.asyncio]
|
|
|
|
|
|
@pytest.fixture
|
|
def settings() -> ChunkingSettings:
|
|
return ChunkingSettings()
|
|
|
|
|
|
@pytest.fixture
|
|
def limiter() -> CapacityLimiter:
|
|
return CapacityLimiter(2)
|
|
|
|
|
|
async def test_parse_and_chunk_document_docx_returns_chunks(
|
|
settings: ChunkingSettings, limiter: CapacityLimiter
|
|
) -> None:
|
|
chunks = await parse_and_chunk_document(
|
|
load_document(PROSE_DOCX),
|
|
source_type="docx",
|
|
file_id=uuid.uuid4(),
|
|
settings=settings,
|
|
limiter=limiter,
|
|
)
|
|
|
|
assert chunks
|
|
assert all(chunk.chunk_index == index for index, chunk in enumerate(chunks))
|
|
|
|
|
|
async def test_parse_and_chunk_document_csv_returns_chunks(
|
|
settings: ChunkingSettings, limiter: CapacityLimiter
|
|
) -> None:
|
|
data = b"name,value\nfirst,1\nsecond,2\n"
|
|
|
|
chunks = await parse_and_chunk_document(
|
|
data, source_type="csv", file_id=uuid.uuid4(), settings=settings, limiter=limiter
|
|
)
|
|
|
|
assert chunks
|
|
|
|
|
|
async def test_parse_and_chunk_document_unsupported_type_raises(
|
|
settings: ChunkingSettings, limiter: CapacityLimiter
|
|
) -> None:
|
|
with pytest.raises(UnsupportedSourceTypeError):
|
|
await parse_and_chunk_document(
|
|
b"whatever",
|
|
source_type="doc",
|
|
file_id=uuid.uuid4(),
|
|
settings=settings,
|
|
limiter=limiter,
|
|
)
|