mistral_common.tokens.tokenizers.tekken

def decode(self, tokens: list[int], special_token_policy: SpecialTokenPolicy = SpecialTokenPolicy.IGNORE) -> str:
    r"""Decode a list of token IDs into a string.

    Args:
        tokens: List of token IDs to decode.
        special_token_policy: Policy for handling special tokens:
            - IGNORE: Skip special tokens (default)
            - KEEP: Include special token strings in output
            - RAISE: Raise ValueError if special tokens are present

    Returns:
        The decoded UTF-8 string.

    Raises:
        ValueError: If `special_token_policy` is invalid or RAISE is set and
            special tokens are encountered.
    """
    try:
        special_token_policy = SpecialTokenPolicy(special_token_policy)
    except ValueError as e:
        valid = ", ".join(repr(p.value) for p in SpecialTokenPolicy)
        raise ValueError(
            f"Invalid `special_token_policy` {special_token_policy!r}. Expected one of: {valid}."
        ) from e

    return "".join(self._decode_all(tokens, special_token_policy=special_token_policy))

def encode(self, s: str, bos: bool, eos: bool) -> list[int]:
    r"""Encode a string into a list of token ids.

    Args:
        s: The string to encode.
        bos: If `True`, prepends the beginning-of-sentence token ID to the result.
        eos: If `True`, appends the end-of-sentence token ID to the result.

    Returns:
        List of token IDs. Regular tokens are offset by `num_special_tokens`.
    """
    tokens: list[int] = self._model.encode(s)
    tokens = [t + self.num_special_tokens for t in tokens]
    if bos:
        tokens = [self.bos_id, *tokens]
    if eos:
        tokens = [*tokens, self.eos_id]
    return tokens

@classmethod
def from_file(cls: type["Tekkenizer"], path: str | Path) -> "Tekkenizer":
    r"""Load the tekken tokenizer from a JSON file.

    The file must contain vocab, config, and optionally `special_tokens`, image,
    audio, and `model_settings_builder` sections.

    Args:
        path: Path to the tokenizer JSON file. Must exist and be readable.

    Returns:
        A Tekkenizer instance configured from the file.

    Raises:
        ValueError: If the file has an unknown version, is missing required fields,
            or contains incompatible configuration (e.g., `model_settings_builder`
            with a version that does not support it).
        AssertionError: If the file does not exist.
    """
    if isinstance(path, str):
        path = Path(path)
    assert path.exists(), path
    with open(path, "r", encoding="utf-8") as f:
        untyped = json.load(f)

    _version_str = untyped["config"].get("version")
    if _version_str not in TokenizerVersion.__members__:
        raise ValueError(
            f"Unknown version: {_version_str} in {path}. "
            f"Make sure to use a valid version string: {list(TokenizerVersion.__members__)}"
        )

    assert _version_str is not None
    version = TokenizerVersion(_version_str)

    special_tokens_dicts: list[SpecialTokenInfo] | None = untyped.get("special_tokens", None)
    if special_tokens_dicts is None:
        # Tokenizer > v7 should find special tokens in the tokenizer file
        if version > TokenizerVersion.v7:
            raise ValueError(
                f"Special tokens not found in {path}. "
                "Please update your tokenizer file and include all special tokens you need."
            )
        else:
            special_tokens = list(Tekkenizer.DEPRECATED_SPECIAL_TOKENS)
    else:
        special_tokens = [token for token in special_tokens_dicts]

    untyped["special_tokens"] = special_tokens

    if mm := untyped.get("multimodal"):
        # deprecated - only allowed for tokenizers <= v11
        if version > TokenizerVersion.v11:
            raise ValueError(
                f"The image config has to be called 'image' in {path} for tokenizers of version {version.value}."
            )

        untyped["image"] = ImageConfig(**mm)
    elif image := untyped.get("image"):
        untyped["image"] = ImageConfig(**image)

    if audio := untyped.get("audio"):
        encoding_config = audio.pop("audio_encoding_config")
        encoding_config = AudioSpectrogramConfig(**encoding_config)
        untyped["audio"] = AudioConfig(encoding_config=encoding_config, **audio)

    if (
        model_settings_builder := untyped.get("model_settings_builder")
    ) is not None and not version.supports_model_settings:
        raise ValueError(
            f"model_settings_builder is not supported for {version=} but got {model_settings_builder=}"
        )
    elif model_settings_builder is not None:
        model_settings_builder = ModelSettingsBuilder.model_validate(model_settings_builder)

    model_data: ModelData = untyped

    return cls(
        vocab=model_data["vocab"],
        special_tokens=special_tokens,
        pattern=model_data["config"]["pattern"],
        vocab_size=model_data["config"]["default_vocab_size"],
        num_special_tokens=model_data["config"]["default_num_special_tokens"],
        version=version,
        name=path.name.replace(".json", ""),
        image_config=model_data.get("image"),
        audio_config=model_data.get("audio"),
        model_settings_builder=model_settings_builder,
        _path=path,
    )

def get_control_token(self, s: str) -> int:
    r"""Get the token ID of a control token. Deprecated: use `get_special_token()` instead.

    Deprecated: Use `get_special_token()` instead.

    Args:
        s: The string representation of the control token.

    Returns:
        The integer token ID for the control token.
    """
    warnings.warn("`get_control_token` is deprecated. Use `get_special_token` instead.", FutureWarning)
    return self.get_special_token(s)

def get_special_token(self, s: str) -> int:
    r"""Get the token ID of a special token by its string representation.

    Args:
        s: The string representation of the special token (e.g., "<s>", "<eos>").

    Returns:
        The integer token ID for the special token.

    Raises:
        ValueError: If the special token string is not recognized.
    """
    if s in self._special_tokens_reverse_vocab:
        return self._special_tokens_reverse_vocab[s]
    else:
        raise ValueError(f"Unknown control token {s}")

def id_to_byte_piece(
    self, token_id: int, special_token_policy: SpecialTokenPolicy = SpecialTokenPolicy.IGNORE
) -> bytes:
    r"""Convert a token ID to its byte representation.

    Args:
        token_id: The token ID to convert.
        special_token_policy: Policy for handling special tokens:
            - IGNORE: Return empty bytes for special tokens
            - KEEP: Return UTF-8 encoded special token string
            - RAISE: Raise ValueError if token is special

    Returns:
        The byte representation of the token. For regular tokens, returns the
        original byte sequence from the vocabulary.

    Raises:
        ValueError: If `special_token_policy` is RAISE and `token_id` is a special token,
            or if `special_token_policy` is invalid.
    """
    if token_id < self.num_special_tokens:
        if special_token_policy == SpecialTokenPolicy.KEEP:
            return self._all_special_tokens[token_id]["token_str"].encode("utf-8")
        elif special_token_policy == SpecialTokenPolicy.RAISE:
            raise ValueError(f"{token_id} is a special token")
        elif special_token_policy == SpecialTokenPolicy.IGNORE:
            return b""
        else:
            raise ValueError(f"Unknown special token policy {special_token_policy}")

    return self._model.decode_single_token_bytes(token_id - self.num_special_tokens)

def id_to_piece(self, token_id: int) -> str:
    r"""Convert a token ID to its string representation.

    Args:
        token_id: The token ID to convert.

    Returns:
        The string representation of the token. Special tokens are decoded as their
        token string; regular tokens are decoded using the underlying tiktoken model.
    """
    return self.decode([token_id], special_token_policy=SpecialTokenPolicy.KEEP)

def is_byte(self, token_id: int) -> bool:
    r"""Check if a token ID represents a single byte.

    Args:
        token_id: The token ID to check.

    Returns:
        `True` if the token (after subtracting special token offset) is in the
        range [0, 255], meaning it represents a single byte.
    """
    return 0 <= token_id - self.num_special_tokens < 256

def is_special(self, token: int | np.integer | str) -> bool:
    r"""Check if a token is a special token.

    Args:
        token: Token ID (int or numpy integer) or token string to check.

    Returns:
        `True` if the token is a special token, `False` otherwise.

    Raises:
        TypeError: If token is not an int, numpy integer, or str.
    """
    if isinstance(token, (int, np.integer)):
        return token in self._special_token_ids
    elif isinstance(token, str):
        return token in self._special_tokens_reverse_vocab
    else:
        raise TypeError(f"Expected int or str, got {type(token).__name__}")

def vocab(self) -> list[str]:
    r"""Get all tokens in the vocabulary as strings.

    Note:
        Tokens with decoding errors are collapsed into the "<?>" string. This may
        result in len(set(vocab)) != len(vocab). Use with caution.

    Returns:
        List of all tokens in the vocabulary as strings, in token ID order.
    """
    # when returning self._vocab this will collapse
    # all tokens for which we have a decoding error into
    # the <?> string. This is bad and results in things
    # like len(set(vocab)) != len(vocab))
    # be careful when using self._vocab
    return self._vocab