Skip to content

mistral_common.tokens.tokenizers.sentencepiece

SentencePieceTokenizer(model_path, tokenizer_version=None)

Bases: Tokenizer

SentencePiece tokenizer.

Parameters:

Name Type Description Default
model_path str | Path

The path to the SentencePiece model file.

 required
tokenizer_version TokenizerVersion | None

The version of the tokenizer. If None, inferred from the model path filename.

 None

Raises:

Type Description
AssertionError

If the model file does not exist or its internal vocab size is inconsistent.
TokenizerException

If the version inferred from the filename is unrecognized.
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
def __init__(self, model_path: str | Path, tokenizer_version: TokenizerVersion | None = None) -> None:
    r"""Initialize the SentencePieceTokenizer.

    Args:
        model_path: The path to the SentencePiece model file.
        tokenizer_version: The version of the tokenizer. If `None`, inferred
            from the model path filename.

    Raises:
        AssertionError: If the model file does not exist or its internal
            vocab size is inconsistent.
        TokenizerException: If the version inferred from the filename is
            unrecognized.
    """
    assert_sentencepiece_installed()

    self._logger = logging.getLogger(self.__class__.__name__)
    # reload tokenizer
    assert os.path.isfile(model_path), model_path
    # SentencePiece is optional and conditionally imported, so narrow its dynamic result to this local protocol.
    self._model: _SentencePieceModel = cast(
        _SentencePieceModel,
        SentencePieceProcessor(model_file=model_path if isinstance(model_path, str) else model_path.as_posix()),
    )

    assert self._model.vocab_size() == self._model.get_piece_size()
    self._vocab = [self._model.id_to_piece(i) for i in range(self.n_words)]

    self._version: TokenizerVersion = tokenizer_version or get_spm_version(model_path, raise_deprecated=False)

    self._file_path = Path(model_path)
    super().__init__()

bos_id cached property

The beginning of sentence token id.

eos_id cached property

The end of sentence token id.

file_path property

The path to the tokenizer model.

model_settings_builder property

Always returns None as SentencePiece does not support model_settings_builder.

n_words property

Vocabulary size of the tokenizer.

num_special_tokens property

The number of special tokens of the tokenizer.

pad_id property

The padding token id.

special_ids cached property

Ids of the special tokens.

unk_id property

The unknown token id.

version property

The version of the tokenizer.

decode(tokens, special_token_policy=SpecialTokenPolicy.IGNORE)

Decode the given list of token ids into a string.

Note

Using special_token_policy=SpecialTokenPolicy.KEEP will keep the special tokens and the normal tokens as SentencePiece pieces.

Parameters:

Name Type Description Default
tokens list[int]

The list of token ids.

 required
special_token_policy SpecialTokenPolicy

The policy to use for special tokens.

 IGNORE

Returns:

Type Description
str

The decoded string.
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
def decode(self, tokens: list[int], special_token_policy: SpecialTokenPolicy = SpecialTokenPolicy.IGNORE) -> str:
    r"""Decode the given list of token ids into a string.

    Note:
        Using `special_token_policy=SpecialTokenPolicy.KEEP` will keep the special tokens and the normal tokens as
        SentencePiece pieces.

    Args:
        tokens: The list of token ids.
        special_token_policy: The policy to use for special tokens.

    Returns:
        The decoded string.
    """
    try:
        special_token_policy = SpecialTokenPolicy(special_token_policy)
    except ValueError as e:
        valid = ", ".join(repr(p.value) for p in SpecialTokenPolicy)
        raise ValueError(
            f"Invalid `special_token_policy` {special_token_policy!r}. Expected one of: {valid}."
        ) from e

    if special_token_policy == SpecialTokenPolicy.IGNORE:
        decoded = self._model.decode(tokens)
        assert isinstance(decoded, str), f"Sentencepiece model decoded a {type(decoded)}, not a string."
        return decoded

    return self._decode_with_special_tokens(tokens, special_token_policy)

encode(s, bos, eos)

Encode the given string into a list of token ids.

Parameters:

Name Type Description Default
s str

The string to encode.

 required
bos bool

Whether to add the beginning of sentence token.

 required
eos bool

Whether to add the end of sentence token.

 required

Returns:

Type Description
list[int]

The list of token ids.
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
def encode(self, s: str, bos: bool, eos: bool) -> list[int]:
    r"""Encode the given string into a list of token ids.

    Args:
        s: The string to encode.
        bos: Whether to add the beginning of sentence token.
        eos: Whether to add the end of sentence token.

    Returns:
        The list of token ids.
    """
    assert isinstance(s, str)
    t: list[int] = self._model.encode(s)
    if bos:
        t = [self.bos_id, *t]
    if eos:
        t = [*t, self.eos_id]
    return t

get_control_token(s)

Get the token ID of a control token. Deprecated: use get_special_token() instead.

Parameters:

Name Type Description Default
s str

The special token string.

 required

Returns:

Type Description
int

The token ID for the special token.
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
240
241
242
243
244
245
246
247
248
249
250
def get_control_token(self, s: str) -> int:
    r"""Get the token ID of a control token. Deprecated: use `get_special_token()` instead.

    Args:
        s: The special token string.

    Returns:
        The token ID for the special token.
    """
    warnings.warn("`get_control_token` is deprecated. Use `get_special_token` instead.", FutureWarning)
    return self.get_special_token(s)

get_special_token(s)

Get the token ID for a special token string.

Parameters:

Name Type Description Default
s str

The special token string (e.g., "").

 required

Returns:

Type Description
int

The token ID for the special token. Unknown strings map to the
int

unknown token ID (sentencepiece behavior).
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
228
229
230
231
232
233
234
235
236
237
238
def get_special_token(self, s: str) -> int:
    r"""Get the token ID for a special token string.

    Args:
        s: The special token string (e.g., "<s>").

    Returns:
        The token ID for the special token. Unknown strings map to the
        unknown token ID (sentencepiece behavior).
    """
    return self._model.piece_to_id(s)

id_to_piece(token_id)

Convert the given token id to a token piece.

Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
356
357
358
def id_to_piece(self, token_id: int) -> str:
    r"""Convert the given token id to a token piece."""
    return self._model.id_to_piece(token_id)

is_special(token)

Check if a token is a special (control) token.

Parameters:

Name Type Description Default
token int | integer | str

Token ID (int or numpy integer) or token string to check.

 required

Returns:

Type Description
bool

True if the token is a sentencepiece control token, False otherwise.

Raises:

Type Description
TypeError

If token is not an int, numpy integer, or str.
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
def is_special(self, token: int | np.integer | str) -> bool:
    r"""Check if a token is a special (control) token.

    Args:
        token: Token ID (int or numpy integer) or token string to check.

    Returns:
        `True` if the token is a sentencepiece control token, `False` otherwise.

    Raises:
        TypeError: If token is not an int, numpy integer, or str.
    """
    if isinstance(token, (int, np.integer)):
        return self._model.IsControl(int(token))
    elif isinstance(token, str):
        token_int = self._model.piece_to_id(token)
        return self._model.IsControl(token_int)
    else:
        raise TypeError(f"Expected int or str, got {type(token).__name__}")

vocab()

Get all tokens in the vocabulary as strings.

Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
262
263
264
def vocab(self) -> list[str]:
    r"""Get all tokens in the vocabulary as strings."""
    return self._vocab

get_image_config(tokenizer_filename)

Get the image config from the tokenizer filename.

The multimodal version suffix (e.g., "m1" in "tokenizer.model.v7m1") selects the image configuration.

Parameters:

Name Type Description Default
tokenizer_filename str | Path

The tokenizer model path to parse.

 required

Returns:

Type Description
ImageConfig | None

The ImageConfig for the multimodal version in the filename, or None
ImageConfig | None

if the filename carries no multimodal version.

Raises:

Type Description
TokenizerException

If the multimodal version in the filename is unrecognized.
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
def get_image_config(tokenizer_filename: str | Path) -> ImageConfig | None:
    r"""Get the image config from the tokenizer filename.

    The multimodal version suffix (e.g., "m1" in "tokenizer.model.v7m1")
    selects the image configuration.

    Args:
        tokenizer_filename: The tokenizer model path to parse.

    Returns:
        The ImageConfig for the multimodal version in the filename, or `None`
        if the filename carries no multimodal version.

    Raises:
        TokenizerException: If the multimodal version in the filename is
            unrecognized.
    """
    tokenizer_filename = str(tokenizer_filename)

    _version_str = tokenizer_filename.split(".")[-1]
    if _version_str == "model" or "m" not in _version_str:
        return None

    _mm_version_str = "m" + _version_str.split("m")[-1]

    if _mm_version_str not in MultiModalVersion.__members__:
        raise TokenizerException(f"Unrecognized tokenizer filename: {tokenizer_filename}")

    return MultiModalVersion(_mm_version_str).config

get_spm_version(tokenizer_filename, raise_deprecated=False)

Get the version of the tokenizer from the filename.

Expects filenames like "tokenizer.model.v3" or "tokenizer.model.v7m1". A bare tokenizer.model is treated as v1.

Parameters:

Name Type Description Default
tokenizer_filename str | Path

The tokenizer model path to parse.

 required
raise_deprecated bool

If True, raise for the deprecated unversioned tokenizer.model filename instead of silently returning v1.

 False

Returns:

Type Description
TokenizerVersion

The TokenizerVersion parsed from the filename.

Raises:

Type Description
TokenizerException

If raise_deprecated is True and the filename has no version suffix, or the version is unrecognized.
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def get_spm_version(tokenizer_filename: str | Path, raise_deprecated: bool = False) -> TokenizerVersion:
    r"""Get the version of the tokenizer from the filename.

    Expects filenames like "tokenizer.model.v3" or "tokenizer.model.v7m1".
    A bare `tokenizer.model` is treated as v1.

    Args:
        tokenizer_filename: The tokenizer model path to parse.
        raise_deprecated: If `True`, raise for the deprecated unversioned
            `tokenizer.model` filename instead of silently returning v1.

    Returns:
        The TokenizerVersion parsed from the filename.

    Raises:
        TokenizerException: If `raise_deprecated` is `True` and the filename has
            no version suffix, or the version is unrecognized.
    """
    tokenizer_filename = str(tokenizer_filename)

    _version_str = tokenizer_filename.split(".")[-1]
    if _version_str != "model":  # filter tokenizer_filename == "/path/to/tokenizer.model" case
        _version_str = _version_str.split("m")[0]

    if _version_str == "model":
        if raise_deprecated:
            raise TokenizerException(f"Make sure to rename your tokenizer file to end with {tokenizer_filename}.v1.")

        # tokenizer.model => tokenizer.model.v1
        return TokenizerVersion("v1")

    if _version_str not in TokenizerVersion.__members__:
        raise TokenizerException(f"Unrecognized tokenizer filename: {tokenizer_filename}")

    return TokenizerVersion(_version_str)

is_sentencepiece(path)

Check if the given path is a SentencePiece model.

Recognizes files ending in .model or .model.[], e.g. tokenizer.model.v3 or tokenizer.model.v3m1.

Parameters:

Name Type Description Default
path str | Path

The path to check.

 required

Returns:

Type Description
bool

True if the path is an existing file with a known sentencepiece
bool

suffix, False otherwise.
Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def is_sentencepiece(path: str | Path) -> bool:
    r"""Check if the given path is a SentencePiece model.

    Recognizes files ending in .model or .model.<version>[<mm-version>],
    e.g. tokenizer.model.v3 or tokenizer.model.v3m1.

    Args:
        path: The path to check.

    Returns:
        `True` if the path is an existing file with a known sentencepiece
        suffix, `False` otherwise.
    """
    if isinstance(path, str):
        path = Path(path)

    instruct_versions = list(TokenizerVersion.__members__)
    mm_versions = list(MultiModalVersion.__members__) + [""]  # allow no mm version
    suffixes = [f".model.{v}{m}" for v in instruct_versions for m in mm_versions] + [".model"]

    return path.is_file() and any(path.name.endswith(suffix) for suffix in suffixes)

is_sentencepiece_tokenizer(tokenizer)

Return whether the tokenizer is a SentencePieceTokenizer.

Source code in src/mistral_common/tokens/tokenizers/sentencepiece.py 
398
399
400
def is_sentencepiece_tokenizer(tokenizer: Tokenizer) -> TypeGuard[SentencePieceTokenizer]:
    r"""Return whether the tokenizer is a SentencePieceTokenizer."""
    return isinstance(tokenizer, SentencePieceTokenizer)