mistral_common.tokens.tokenizers.base

@abstractmethod
def decode(self, tokens: list[int], special_token_policy: SpecialTokenPolicy) -> str:
    r"""Decode the token IDs to a string.

    Args:
        tokens: The token IDs to decode.
        special_token_policy: The policy to use for special tokens
            (IGNORE, KEEP, or RAISE).

    Returns:
        The decoded string.
    """

@abstractmethod
def encode_fim(self, request: FIMRequestType) -> TokenizedType:
    r"""Encode a FIM request into a Tokenized object.

    Wraps the prompt and suffix with FIM markers ([PREFIX], [SUFFIX],
    [MIDDLE]) so the model completes the middle section.

    Args:
        request: The FIM request to encode.

    Returns:
        The tokenized FIM request.
    """

@abstractmethod
def encode_instruct(self, request: InstructRequestType) -> TokenizedType:
    r"""Encode an instruct request into a Tokenized object.

    Args:
        request: The instruct request to encode.

    Returns:
        The tokenized instruct request.
    """

@abstractmethod
def encode_speech_request(self, request: SpeechRequest) -> TokenizedType:
    r"""Encodes a speech synthesis request into a tokenized format.

    This method processes a speech request containing text input and
    optional reference audio or voice preset, and returns the tokenized output.

    Args:
        request: The speech request object containing the text and voice/audio data.

    Returns:
        Tokenized: The tokenized representation of the speech request.
    """

@abstractmethod
def encode_system_message(self, message: SystemMessage) -> tuple[list[int], list[Audio]]:
    r"""Encode a system message.

    Args:
        message: The system message to encode.

    Returns:
        A tuple of (tokens, audios): the encoded token IDs and the loaded
        audio for this message.
    """
    ...

@abstractmethod
def encode_transcription(self, request: TranscriptionRequest) -> TokenizedType:
    r"""
    Encodes an audio transcription request into a tokenized format.

    This method processes a transcription request containing audio data,
    encodes the user message, and returns the tokenized output.

    Args:
        request: The transcription request object containing
            the audio data to be encoded.

    Returns:
        Tokenized: The tokenized representation of the audio data, including processed audio and tokens
    """

@abstractmethod
def encode_user_content(
    self,
    content: str | list[ContentChunk],
    is_last: bool,
    system_prompt: str | None = None,
    force_img_first: bool = False,
) -> tuple[list[int], list[np.ndarray], list[Audio]]:
    r"""Encode user content.

    Args:
        content: The user content to encode. A plain string or a list of
            content chunks (text, image, audio).
        is_last: `True` if this is the last content of the conversation.
        system_prompt: The system prompt, or `None`. Its placement depends
            on the tokenizer version.
        force_img_first: If `True`, image tokens are placed before text
            tokens.

    Returns:
        A tuple of (tokens, images, audios): the encoded token IDs, the
        loaded images, and the loaded audio for this content.
    """
    ...

@abstractmethod
def encode_user_message(
    self,
    message: UserMessage,
    available_tools: list[Tool] | None,
    is_last: bool,
    is_first: bool,
    system_prompt: str | None = None,
    force_img_first: bool = False,
    settings: ModelSettings = ModelSettings.none(),
) -> tuple[list[int], list[np.ndarray], list[Audio]]:
    r"""Encode a user message.

    Args:
        message: The user message to encode.
        available_tools: The tools available to the assistant, or `None`.
            Depending on the tokenizer version, wrapped in an
            [AVAILABLE_TOOLS] section.
        is_last: `True` if this is the last message of the conversation.
        is_first: `True` if this is the first message of the conversation.
        system_prompt: The system prompt, or `None`. Its placement depends on
            the tokenizer version.
        force_img_first: If `True`, image tokens are placed before text
            tokens within the message.
        settings: The model settings to encode into the token stream.

    Returns:
        A tuple of (tokens, images, audios): the encoded token IDs, the
        loaded images, and the loaded audio for this message.
    """
    ...

@abstractmethod
def decode(self, tokens: list[int], special_token_policy: SpecialTokenPolicy = SpecialTokenPolicy.IGNORE) -> str:
    r"""Decode the token ids to a string.

    Args:
        tokens: The token ids to decode.
        special_token_policy: The policy to use for special tokens.

    Returns:
        The decoded string.
    """

@abstractmethod
def encode(self, s: str, bos: bool, eos: bool) -> list[int]:
    r"""Convert a string to a list of token IDs.

    Args:
        s: The string to encode.
        bos: If `True`, prepend the beginning-of-sentence token ID.
        eos: If `True`, append the end-of-sentence token ID.

    Returns:
        The encoded token IDs.
    """

@abstractmethod
def get_special_token(self, s: str) -> int:
    r"""Get the id of a control token."""

@abstractmethod
def id_to_piece(self, token_id: int) -> str:
    r"""Convert a token id to the token str."""

@abstractmethod
def is_special(self, token: int | np.integer | str) -> bool:
    r"""Check if token id or token str is a special token."""

@abstractmethod
def vocab(self) -> list[str]:
    r"""All tokens in the vocabulary as strings."""