Skip to content

mistral_common.guidance.tokenizer

MistralLLGTokenizer(tokenizer)

Wraps a Tekken tokenizer for use with llguidance.

Adapts Mistral special tokens ([...]) to the angle-bracket format (<...>) llguidance expects, and exposes the vocabulary as byte strings.

Special token strings in square brackets ([INST]) are converted to angle brackets () since llguidance only recognizes the latter. All special tokens must be unique and match the <...> format.

Parameters:

Name Type Description Default
tokenizer Tokenizer

The Tekken tokenizer to wrap for llguidance compatibility.

 required

Raises:

Type Description
TypeError

If the tokenizer is not a Tekkenizer.
ValueError

If a special token has an invalid format or is duplicated, or the special token count is inconsistent.
Source code in src/mistral_common/guidance/tokenizer.py 
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
def __init__(self, tokenizer: Tokenizer) -> None:
    r"""Initialize the wrapper.

    Special token strings in square brackets ([INST]) are converted to
    angle brackets (<INST>) since llguidance only recognizes the latter.
    All special tokens must be unique and match the <...> format.

    Args:
        tokenizer: The Tekken tokenizer to wrap for llguidance compatibility.

    Raises:
        TypeError: If the tokenizer is not a Tekkenizer.
        ValueError: If a special token has an invalid format or is
            duplicated, or the special token count is inconsistent.
    """
    assert_llguidance_installed()

    if not is_tekkenizer(tokenizer):
        raise TypeError(f"Guidance only supports Tekken tokenizers, got {type(tokenizer)}")

    self._tokenizer = tokenizer
    self._tokens: list[bytes] = []
    self._special_token_ids: list[int] = []

    seen_special_tokens: set[str] = set()
    for i in range(self._tokenizer.n_words):
        # Convert square brackets to angle brackets for special tokens,
        # since llg only recognizes the latter.
        if i < self._tokenizer.num_special_tokens:
            token_rep = self._tokenizer.id_to_piece(i)
            if match := re.fullmatch(r"\[(.*)\]", token_rep):
                token_rep_llg = f"<{match.group(1)}>"
            else:
                token_rep_llg = token_rep

            if not re.fullmatch(r"<.*>", token_rep_llg):
                raise ValueError(f"Invalid special token: {token_rep_llg} ({token_rep})")
            if token_rep_llg in seen_special_tokens:
                raise ValueError(f"Duplicate special token: {token_rep_llg} (already seen: {seen_special_tokens})")
            seen_special_tokens.add(token_rep_llg)
            self._special_token_ids.append(i)
            self._tokens.append(token_rep_llg.encode("utf-8"))
        else:
            token_bytes = self._tokenizer.id_to_byte_piece(i, SpecialTokenPolicy.RAISE)
            self._tokens.append(token_bytes)

    if len(self._special_token_ids) != self._tokenizer.num_special_tokens:
        raise ValueError(
            f"Expected {self._tokenizer.num_special_tokens} special tokens, but found "
            f"{len(self._special_token_ids)}"
        )

bos_token_id property

The beginning-of-string token ID.

eos_token_id property

The end-of-string token ID.

special_token_ids property

The IDs of all special tokens.

tokens property

The full vocabulary as byte representations.

Returns:

Type Description
list[bytes]

One entry per token ID, in ID order; special tokens use their
list[bytes]

angle-bracket string encoding.

__call__(s)

Tokenizes a string into token ids.

Parameters:

Name Type Description Default
s str

The string to tokenize.

 required

Returns:

Type Description
list[int]

The list of token ids.
Source code in src/mistral_common/guidance/tokenizer.py 
 96
 97
 98
 99
100
101
102
103
104
105
def __call__(self, s: str) -> list[int]:
    r"""Tokenizes a string into token ids.

    Args:
        s: The string to tokenize.

    Returns:
        The list of token ids.
    """
    return self._tokenizer.encode(s, bos=False, eos=False)

from_mistral_tokenizer(tokenizer)

Creates an llguidance tokenizer from a Mistral tokenizer.

Parameters:

Name Type Description Default
tokenizer MistralTokenizer

The Mistral tokenizer to convert. Must wrap a Tekkenizer.

 required

Returns:

Type Description
LLTokenizer

The llguidance tokenizer.

Raises:

Type Description
TypeError

If the underlying tokenizer is not a Tekkenizer.
Source code in src/mistral_common/guidance/tokenizer.py 
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def from_mistral_tokenizer(tokenizer: MistralTokenizer) -> "llg.LLTokenizer":
    r"""Creates an llguidance tokenizer from a Mistral tokenizer.

    Args:
        tokenizer: The Mistral tokenizer to convert. Must wrap a Tekkenizer.

    Returns:
        The llguidance tokenizer.

    Raises:
        TypeError: If the underlying tokenizer is not a Tekkenizer.
    """
    assert_llguidance_installed()
    inner_tokenizer = tokenizer.instruct_tokenizer.tokenizer
    tokenizer_data = MistralLLGTokenizer(inner_tokenizer)
    return llg.LLTokenizer(llg.TokenizerWrapper(tokenizer_data))