Skip to content

mistral_common.experimental.app.main

cli()

Mistral-common CLI.

Source code in src/mistral_common/experimental/app/main.py 
68
69
70
71
@click.group()
def cli() -> None:
    r"""Mistral-common CLI."""
    pass

create_app(tokenizer, validation_mode=ValidationMode.test, engine_url='127.0.0.1', engine_backend=EngineBackend.llama_cpp, timeout=60)

Create a Mistral-common FastAPI app with the given tokenizer and validation mode.

Parameters:

Name Type Description Default
tokenizer Union[str, Path, MistralTokenizer]

The tokenizer path or a MistralTokenizer instance.

 required
validation_mode ValidationMode

The validation mode to use.

 test
engine_url str

The URL of the engine API.

 '127.0.0.1'
timeout int

The timeout of the engine API.

 60

Returns:

Type Description
FastAPI

The Mistral-common FastAPI app.
Source code in src/mistral_common/experimental/app/main.py 
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
def create_app(
    tokenizer: Union[str, Path, MistralTokenizer],
    validation_mode: ValidationMode = ValidationMode.test,
    engine_url: str = "127.0.0.1",
    engine_backend: EngineBackend = EngineBackend.llama_cpp,
    timeout: int = 60,
) -> FastAPI:
    r"""Create a Mistral-common FastAPI app with the given tokenizer and validation mode.

    Args:
        tokenizer: The tokenizer path or a MistralTokenizer instance.
        validation_mode: The validation mode to use.
        engine_url: The URL of the engine API.
        timeout: The timeout of the engine API.

    Returns:
        The Mistral-common FastAPI app.
    """
    if not isinstance(tokenizer, (MistralTokenizer, str, Path)):
        raise ValueError("Tokenizer must be a path or a MistralTokenizer instance.")

    app = FastAPI()
    app.include_router(tokenize_router)
    app.include_router(decode_router)
    app.include_router(main_router)

    @lru_cache
    def get_settings_override() -> Settings:
        settings = Settings(
            engine_url=engine_url,
            engine_backend=engine_backend,
            timeout=timeout,
        )
        if isinstance(tokenizer, MistralTokenizer):
            settings.tokenizer = tokenizer
        else:
            settings._load_tokenizer(tokenizer, validation_mode)
        return settings

    get_settings_override()
    app.dependency_overrides[get_settings] = get_settings_override

    return app

serve(tokenizer_path, validation_mode=ValidationMode.test, host='127.0.0.1', port=0, engine_url='http://127.0.0.1:8080', engine_backend=EngineBackend.llama_cpp.value, timeout=60)

Serve the Mistral-common API with the given tokenizer path and validation mode.

Source code in src/mistral_common/experimental/app/main.py 
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@cli.command(name="serve", context_settings={"auto_envvar_prefix": "UVICORN"})
@click.argument("tokenizer_path", type=str)
@click.argument(
    "validation_mode",
    type=click.Choice([mode.value for mode in ValidationMode], case_sensitive=False),
    default=ValidationMode.test.value,
)
@click.option(
    "--host",
    type=str,
    default="127.0.0.1",
    help="Mistral-common API host",
    show_default=True,
)
@click.option(
    "--port",
    type=int,
    default=0,
    help="Mistral-common API port",
    show_default=True,
)
@click.option(
    "--engine-url",
    type=str,
    default="http://127.0.0.1:8080",
    help="Enginge URL",
    show_default=True,
)
@click.option(
    "--engine-backend",
    type=click.Choice([mode.value for mode in EngineBackend], case_sensitive=False),
    default=EngineBackend.llama_cpp.value,
    help="Engine API backend",
    show_default=True,
)
@click.option(
    "--timeout",
    type=int,
    default=60,
    help="Timeout",
    show_default=True,
)
def serve(
    tokenizer_path: Union[str, Path],
    validation_mode: Union[ValidationMode, str] = ValidationMode.test,
    host: str = "127.0.0.1",
    port: int = 0,
    engine_url: str = "http://127.0.0.1:8080",
    engine_backend: str = EngineBackend.llama_cpp.value,
    timeout: int = 60,
) -> None:
    r"""Serve the Mistral-common API with the given tokenizer path and validation mode."""
    app = create_app(
        tokenizer=tokenizer_path,
        validation_mode=ValidationMode(validation_mode),
        engine_url=engine_url,
        engine_backend=EngineBackend(engine_backend),
        timeout=timeout,
    )
    uvicorn.run(app, host=host, port=port)