"""Token counting for chunk sizing (ADR-0018).

`cl100k_base` is a deliberate proxy for the embedding models' own tokenizers.
`text-embedding-3-large` has an 8191-token window and never binds;
`nomic-embed-text-v2-moe`'s 512-token sequence length is the only real
constraint. cl100k tokenizes Persian inefficiently while nomic's multilingual
tokenizer does not, so a cl100k count reliably over-estimates the nomic count --
safe in the conservative direction, without shipping a second tokenizer and its
model download into the ingestion path.
"""

from functools import lru_cache

import tiktoken


@lru_cache(maxsize=4)
def get_encoder(encoding_name: str) -> tiktoken.Encoding:
    """Return a cached tiktoken encoder.

    Deliberately not a module-level constant: `tiktoken` fetches the BPE
    vocabulary over the network the first time an encoding is used, and
    ADR-0012 forbids external resource setup as an import-time side effect.
    The lifespan warms this at startup so a process fails fast at boot rather
    than inside the first ingestion request. Set `TIKTOKEN_CACHE_DIR` to a
    pre-populated directory for offline deployments.
    """
    return tiktoken.get_encoding(encoding_name)


def count_tokens(text: str, encoding_name: str) -> int:
    """Return the number of tokens `text` encodes to."""
    return len(get_encoder(encoding_name).encode(text))