diff --git a/components/src/dynamo/common/configuration/utils.py b/components/src/dynamo/common/configuration/utils.py
index f3cf35ca9651..79c07d33a28a 100644
--- a/components/src/dynamo/common/configuration/utils.py
+++ b/components/src/dynamo/common/configuration/utils.py
@@ -11,6 +11,16 @@
T = TypeVar("T")
+def parse_bool(value: str) -> bool:
+ """Parse Dynamo's truthy and falsy configuration values."""
+ normalized = value.strip().lower()
+ if normalized in ("1", "true", "on", "yes"):
+ return True
+ if normalized in ("", "0", "false", "off", "no"):
+ return False
+ raise argparse.ArgumentTypeError("expected one of: true/false, 1/0, on/off, yes/no")
+
+
def split_served_model_names(served_model_name: Any) -> list[str]:
"""Split a ``--served-model-name`` value into individual names.
@@ -98,6 +108,7 @@ def add_argument(
help: str,
obsolete_flag: Optional[str] = None,
arg_type: Optional[Union[type, Callable[..., Any]]] = str,
+ env_value_type: Optional[Union[type, Callable[..., Any]]] = None,
**kwargs: Any,
) -> None:
"""
@@ -114,10 +125,11 @@ def add_argument(
dest: Optional destination name (defaults to flag_name with dashes replaced by underscores)
choices: Optional list of valid values for the argument.
arg_type: Type for the argument (default: str)
+ env_value_type: Optional parser used only for the environment value
"""
arg_dest = _get_dest_name(flag_name, kwargs.get("dest"))
- value_type_for_env: Optional[Union[type, Callable[..., Any]]] = None
- if arg_type is not None and callable(arg_type):
+ value_type_for_env = env_value_type
+ if value_type_for_env is None and arg_type is not None and callable(arg_type):
value_type_for_env = arg_type
if isinstance(default, list) and (arg_type is None or arg_type is str):
value_type_for_env = None
@@ -152,6 +164,7 @@ def add_negatable_bool_argument(
help: str,
dest: Optional[str] = None,
obsolete_flag: Optional[str] = None,
+ env_value_type: Optional[Callable[..., bool]] = None,
) -> None:
"""
Add negatable boolean flag (--foo / --no-foo).
@@ -164,6 +177,7 @@ def add_negatable_bool_argument(
help: Help text
dest: Optional destination name for the parsed value
obsolete_flag: Optional obsolete/legacy flag (for help msg only, must start with '--')
+ env_value_type: Optional strict parser for the environment value
"""
add_argument(
parser,
@@ -174,6 +188,7 @@ def add_negatable_bool_argument(
dest=dest,
obsolete_flag=obsolete_flag,
arg_type=None,
+ env_value_type=env_value_type,
action=argparse.BooleanOptionalAction,
)
diff --git a/components/src/dynamo/common/tests/configuration/test_utils.py b/components/src/dynamo/common/tests/configuration/test_utils.py
index 7952966a4748..fc1c81a47883 100644
--- a/components/src/dynamo/common/tests/configuration/test_utils.py
+++ b/components/src/dynamo/common/tests/configuration/test_utils.py
@@ -11,6 +11,7 @@
add_negatable_bool_argument,
env_or_default,
nullable_float,
+ parse_bool,
)
pytestmark = [
@@ -20,6 +21,28 @@
]
+@pytest.mark.parametrize(
+ ("value", "expected"),
+ [
+ ("true", True),
+ ("1", True),
+ ("on", True),
+ ("yes", True),
+ ("false", False),
+ ("0", False),
+ ("off", False),
+ ("no", False),
+ ],
+)
+def test_parse_bool(value, expected):
+ assert parse_bool(value) is expected
+
+
+def test_parse_bool_rejects_invalid_value():
+ with pytest.raises(argparse.ArgumentTypeError, match="expected one of"):
+ parse_bool("flase")
+
+
class TestEnvOrDefault:
"""Test env_or_default function."""
@@ -251,6 +274,20 @@ def test_uses_env_var_when_set(self, monkeypatch):
args = parser.parse_args([])
assert args.enable_feature is False
+ def test_strict_env_parser_rejects_invalid_value(self, monkeypatch):
+ monkeypatch.setenv("TEST_ENABLE", "flase")
+ parser = argparse.ArgumentParser()
+
+ with pytest.raises(argparse.ArgumentTypeError, match="expected one of"):
+ add_negatable_bool_argument(
+ parser,
+ flag_name="--enable-feature",
+ env_var="TEST_ENABLE",
+ default=True,
+ help="Enable feature",
+ env_value_type=parse_bool,
+ )
+
def test_converts_hyphens_to_underscores(self):
"""Test that flag name with hyphens converts to underscores in dest."""
parser = argparse.ArgumentParser()
diff --git a/components/src/dynamo/frontend/frontend_args.py b/components/src/dynamo/frontend/frontend_args.py
index 722eab142b3c..95b76c5b7ab2 100644
--- a/components/src/dynamo/frontend/frontend_args.py
+++ b/components/src/dynamo/frontend/frontend_args.py
@@ -24,6 +24,7 @@
add_argument,
add_negatable_bool_argument,
env_or_default,
+ parse_bool,
)
from . import __version__
@@ -89,6 +90,7 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AicPerfConfigBase):
exclude_tools_when_tool_choice_none: bool
preprocess_workers: int
tokenizer_backend: str
+ tokenizer_fallback: bool
trust_remote_code: bool
frontend_route_extensions: list[str]
@@ -552,6 +554,23 @@ def add_arguments(self, parser) -> None:
choices=["default", "fastokens", "basetenkenizer"],
)
+ add_negatable_bool_argument(
+ g,
+ flag_name="--tokenizer-fallback",
+ env_var="DYN_TOKENIZER_FALLBACK",
+ default=True,
+ help=(
+ "Automatic fallback to HuggingFace is deprecated and will be "
+ "disabled by default in a future release. The current behavior "
+ "falls back when the selected fastokens or basetenkenizer backend "
+ "cannot load the model tokenizer. Use "
+ "--no-tokenizer-fallback to fail model initialization instead. "
+ "In dynamic mode, discovery retries the load while the frontend "
+ "continues running."
+ ),
+ env_value_type=parse_bool,
+ )
+
add_negatable_bool_argument(
g,
flag_name="--trust-remote-code",
diff --git a/components/src/dynamo/frontend/main.py b/components/src/dynamo/frontend/main.py
index 200ee1ec2ad7..9e0f91ea5bf0 100644
--- a/components/src/dynamo/frontend/main.py
+++ b/components/src/dynamo/frontend/main.py
@@ -412,6 +412,7 @@ def signal_handler():
"enable_streaming_reasoning_dispatch": config.enable_streaming_reasoning_dispatch,
"reasoning_field_name": config.reasoning_field_name,
"tokenizer_backend": config.tokenizer_backend,
+ "tokenizer_fallback": config.tokenizer_fallback,
}
if config.migration_max_seq_len is not None:
kwargs["migration_max_seq_len"] = config.migration_max_seq_len
diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md
index 9ab05b474b43..8db399d0a4ed 100644
--- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md
+++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md
@@ -31,7 +31,7 @@ Use this backend for supported `tokenizer.json` models when you need Baseten Tok
#### Compatibility notes:
- Works with standard BPE `tokenizer.json` files (Qwen, LLaMA, GPT-family, Mistral, DeepSeek, etc.).
-- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace; requests are never dropped.
+- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace by default. Use `--no-tokenizer-fallback` to reject incompatible tokenizers during model initialization.
- Special tokens declared only in a sibling `tokenizer_config.json` are preserved for Baseten encoding and decoding and for Dynamo's L1 prefix-cache boundaries.
- Has no effect on TikToken-format tokenizers (`.model` / `.tiktoken` files), which always use the TikToken backend.
@@ -39,9 +39,14 @@ Use this backend for supported `tokenizer.json` models when you need Baseten Tok
Set the backend with a CLI flag or environment variable. The CLI flag takes precedence.
+> [!WARNING]
+> Automatic tokenizer fallback is deprecated and will be disabled by default in a future release.
+> Set `--no-tokenizer-fallback` or `DYN_TOKENIZER_FALLBACK=false` to adopt the future behavior now.
+
| CLI Argument | Env Var | Valid values | Default |
|---|---|---|---|
| `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens`, `basetenkenizer` | `default` |
+| `--tokenizer-fallback` / `--no-tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | `true`/`false`, `1`/`0`, `on`/`off`, `yes`/`no` | `true` |
**Examples:**
@@ -55,6 +60,9 @@ python -m dynamo.frontend
# Baseten Tokenizer
python -m dynamo.frontend --tokenizer basetenkenizer
+
+# Require Baseten Tokenizer instead of falling back to HuggingFace
+python -m dynamo.frontend --tokenizer basetenkenizer --no-tokenizer-fallback
```
## Dynamo Frontend Behavior
@@ -64,5 +72,5 @@ When a non-default backend is selected:
1. The frontend resolves `--tokenizer` / `DYN_TOKENIZER` and passes the selected backend to the Rust runtime.
2. `ModelDeploymentCard::tokenizer()` loads the HuggingFace tokenizer first for fallback behavior and L1 cache special-token metadata.
3. Dynamo constructs `FastTokenizer` for `fastokens` or `BasetenTokenizer` for `basetenkenizer` from the same `tokenizer.json` file.
-4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace.
+4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--no-tokenizer-fallback`, model initialization fails and reports the backend loading error instead. In dynamic mode, discovery retries the load while the frontend continues running.
5. When the L1 prefix cache is enabled, Dynamo wraps the selected backend with the same special-token boundary metadata and cache metrics used by the default path.
diff --git a/docs/fern/pages/reference/api/python/_core.mdx b/docs/fern/pages/reference/api/python/_core.mdx
index b66351be5757..4c47673bb034 100644
--- a/docs/fern/pages/reference/api/python/_core.mdx
+++ b/docs/fern/pages/reference/api/python/_core.mdx
@@ -316,7 +316,7 @@ The request was cancelled.
from dynamo._core import Cancelled
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3248`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3248)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3250`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3250)
@@ -326,7 +326,7 @@ Failed to establish a connection.
from dynamo._core import CannotConnect
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3233`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3233)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3235`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3235)
@@ -445,7 +445,7 @@ A connection or request timed out.
from dynamo._core import ConnectionTimeout
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3243`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3243)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3245`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3245)
@@ -701,7 +701,7 @@ An established connection was lost.
from dynamo._core import Disconnected
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3238`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3238)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3240`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3240)
@@ -806,7 +806,7 @@ Base exception for all Dynamo error types.
from dynamo._core import DynamoException
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3210`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3210)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3212`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3212)
@@ -949,7 +949,7 @@ The engine process has shut down or crashed.
from dynamo._core import EngineShutdown
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3253`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3253)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3255`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3255)
@@ -970,7 +970,7 @@ from dynamo._core import EntrypointArgs
```
```python
-EntrypointArgs(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None) -> None
+EntrypointArgs(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None, tokenizer_fallback: Optional[bool] = None) -> None
```
[`lib/bindings/python/src/dynamo/_core.pyi#L3076`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3076)
@@ -980,7 +980,7 @@ EntrypointArgs(engine_type: EngineType, model_path: Optional[str] = None, model_
__init__
```python
-__init__(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None) -> None
+__init__(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None, tokenizer_fallback: Optional[bool] = None) -> None
```
Create EntrypointArgs.
@@ -1072,7 +1072,10 @@ Optional streaming tool dispatch override
Optional streaming reasoning dispatch override
-Optional tokenizer backend override ("default" or "fastokens")
+Optional tokenizer backend override ("default", "fastokens", or "basetenkenizer")
+
+
+Whether alternate tokenizer load failures fall back to HuggingFace
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3082)
@@ -1548,7 +1551,7 @@ Invalid input (e.g., prompt exceeds context length).
from dynamo._core import InvalidArgument
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3228`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3228)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3230`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3230)
@@ -3182,7 +3185,7 @@ A request from planner to client to perform a scaling action. Fields: num_prefil
from dynamo._core import PlannerDecision
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3151`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3151)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3153`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3153)
@@ -3474,7 +3477,7 @@ A policy-class queue cap rejected the request.
from dynamo._core import RouterQueueLimitExceeded
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3215`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3215)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3217`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3217)
@@ -3718,7 +3721,7 @@ Raised by `SelectionService` for selector failures that are not malformed input.
from dynamo._core import SelectionServiceError
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3263`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3263)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3265`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3265)
@@ -3806,7 +3809,7 @@ The response stream was terminated before completion.
from dynamo._core import StreamIncomplete
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3258`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3258)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3260`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3260)
@@ -3852,7 +3855,7 @@ Uncategorized or unknown error.
from dynamo._core import Unknown
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3223`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3223)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3225`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3225)
@@ -3866,7 +3869,7 @@ from dynamo._core import VirtualConnectorClient
VirtualConnectorClient(runtime: DistributedRuntime, dynamo_namespace: str) -> None
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3185`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3185)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3187`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3187)
**Public methods**
@@ -3878,7 +3881,7 @@ __init__(runtime: DistributedRuntime, dynamo_namespace: str) -> None
No summary available.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3188)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3190)
get
@@ -3888,7 +3891,7 @@ get() -> PlannerDecision
No summary available.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3191)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3193)
complete
@@ -3898,7 +3901,7 @@ complete(decision: PlannerDecision) -> None
No summary available.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3194)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3196)
wait
@@ -3908,7 +3911,7 @@ wait() -> None
Blocks until there is a new decision to fetch using 'get'
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3197)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3199)
@@ -3922,7 +3925,7 @@ from dynamo._core import VirtualConnectorCoordinator
VirtualConnectorCoordinator(runtime: DistributedRuntime, dynamo_namespace: str, check_interval_secs: int, max_wait_time_secs: int, max_retries: int) -> None
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3161`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3161)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3163`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3163)
**Public methods**
@@ -3934,7 +3937,7 @@ __init__(runtime: DistributedRuntime, dynamo_namespace: str, check_interval_secs
No summary available.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3164)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3166)
async_init
@@ -3944,7 +3947,7 @@ async_init() -> None
Call this before using the object
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3167)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3169)
read_state
@@ -3954,7 +3957,7 @@ read_state() -> PlannerDecision
Get the current values. Most for test / debug.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3171)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3173)
update_scaling_decision
@@ -3964,7 +3967,7 @@ update_scaling_decision(num_prefill: Optional[int] = None, num_decode: Optional[
No summary available.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3175)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3177)
wait_for_scaling_completion
@@ -3974,7 +3977,7 @@ wait_for_scaling_completion() -> None
No summary available.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3178)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3180)
is_scaling_ready
@@ -3984,7 +3987,7 @@ is_scaling_ready() -> bool
Return whether the client acknowledged the current scaling decision.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3181)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3183)
@@ -4077,7 +4080,7 @@ No summary available.
from dynamo._core import backend
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3283`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3283)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3285`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3285)
diff --git a/docs/fern/pages/reference/api/python/frontend.mdx b/docs/fern/pages/reference/api/python/frontend.mdx
index 7df048dd6408..ba31e439525e 100644
--- a/docs/fern/pages/reference/api/python/frontend.mdx
+++ b/docs/fern/pages/reference/api/python/frontend.mdx
@@ -52,7 +52,7 @@ Frontend configuration parameters.
from dynamo.frontend.frontend_args import FrontendArgGroup
```
-[`components/src/dynamo/frontend/frontend_args.py#L175`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L175)
+[`components/src/dynamo/frontend/frontend_args.py#L177`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L177)
**Public methods**
@@ -64,7 +64,7 @@ add_arguments(parser) -> None
No summary available.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L178)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L180)
@@ -74,7 +74,7 @@ Configuration for the Dynamo frontend.
from dynamo.frontend.frontend_args import FrontendConfig
```
-[`components/src/dynamo/frontend/frontend_args.py#L53`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L53)
+[`components/src/dynamo/frontend/frontend_args.py#L54`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L54)
**Public methods**
@@ -86,7 +86,7 @@ validate() -> None
No summary available.
-[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L97)
+[source](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L99)
@@ -565,7 +565,7 @@ graceful_shutdown(runtime: DistributedRuntime) -> None
The DistributedRuntime instance to shut down.
-[`components/src/dynamo/frontend/main.py#L482`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/main.py#L482)
+[`components/src/dynamo/frontend/main.py#L483`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/main.py#L483)
@@ -612,7 +612,7 @@ from dynamo.frontend.main import main
main() -> None
```
-[`components/src/dynamo/frontend/main.py#L491`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/main.py#L491)
+[`components/src/dynamo/frontend/main.py#L492`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/main.py#L492)
@@ -866,7 +866,7 @@ from dynamo.frontend.frontend_args import validate_model_name
validate_model_name(value: str) -> str
```
-[`components/src/dynamo/frontend/frontend_args.py#L35`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L35)
+[`components/src/dynamo/frontend/frontend_args.py#L36`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L36)
@@ -880,7 +880,7 @@ from dynamo.frontend.frontend_args import validate_model_path
validate_model_path(value: str) -> str
```
-[`components/src/dynamo/frontend/frontend_args.py#L44`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L44)
+[`components/src/dynamo/frontend/frontend_args.py#L45`](https://github.com/ai-dynamo/dynamo/blob/main/components/src/dynamo/frontend/frontend_args.py#L45)
diff --git a/docs/fern/pages/reference/api/python/llm.mdx b/docs/fern/pages/reference/api/python/llm.mdx
index 406ea5ab189a..cc97e472ca76 100644
--- a/docs/fern/pages/reference/api/python/llm.mdx
+++ b/docs/fern/pages/reference/api/python/llm.mdx
@@ -53,7 +53,7 @@ from dynamo.llm import EntrypointArgs
```
```python
-EntrypointArgs(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None) -> None
+EntrypointArgs(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None, tokenizer_fallback: Optional[bool] = None) -> None
```
[`lib/bindings/python/src/dynamo/_core.pyi#L3076`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3076)
@@ -63,7 +63,7 @@ EntrypointArgs(engine_type: EngineType, model_path: Optional[str] = None, model_
__init__
```python
-__init__(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None) -> None
+__init__(engine_type: EngineType, model_path: Optional[str] = None, model_name: Optional[str] = None, endpoint_id: Optional[str] = None, template_file: Optional[str] = None, router_config: Optional[RouterConfig] = None, kv_cache_block_size: Optional[int] = None, http_host: Optional[str] = None, http_port: Optional[int] = None, http_metrics_port: Optional[int] = None, tls_cert_path: Optional[str] = None, tls_key_path: Optional[str] = None, extra_engine_args: Optional[str] = None, mocker_engine_args: Optional[MockEngineArgs] = None, runtime_config: Optional[ModelRuntimeConfig] = None, namespace: Optional[str] = None, namespace_prefix: Optional[str] = None, is_prefill: bool = False, is_decode: bool = False, migration_limit: int = 0, migration_max_seq_len: Optional[int] = None, chat_engine_factory: Optional[Callable] = None, aic_perf_config: Optional[AicPerfConfig] = None, *, metrics_prefix: Optional[str] = None, enable_anthropic_api: Optional[bool] = None, strip_anthropic_preamble: Optional[bool] = None, enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None, tokenizer_fallback: Optional[bool] = None) -> None
```
Create EntrypointArgs.
@@ -155,7 +155,10 @@ Optional streaming tool dispatch override
Optional streaming reasoning dispatch override
-Optional tokenizer backend override ("default" or "fastokens")
+Optional tokenizer backend override ("default", "fastokens", or "basetenkenizer")
+
+
+Whether alternate tokenizer load failures fall back to HuggingFace
[source](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3082)
@@ -2220,7 +2223,7 @@ A policy-class queue cap rejected the request.
from dynamo.llm import RouterQueueLimitExceeded
```
-[`lib/bindings/python/src/dynamo/_core.pyi#L3215`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3215)
+[`lib/bindings/python/src/dynamo/_core.pyi#L3217`](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/src/dynamo/_core.pyi#L3217)
diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx
index 8d24ca3399ec..d2d6a00aece8 100644
--- a/docs/fern/pages/reference/components/frontend-configuration.mdx
+++ b/docs/fern/pages/reference/components/frontend-configuration.mdx
@@ -506,6 +506,19 @@ See the [Frontend Guide](../../developer-guide/knowledge-base/modular-components
Environment variable: `DYN_TOKENIZER`
+
+ Fall back to HuggingFace when `fastokens` or `basetenkenizer` cannot load the model tokenizer. Use
+ `--no-tokenizer-fallback` to fail model initialization instead. In dynamic mode, discovery retries
+ the load while the frontend continues running.
+
+
+ Automatic tokenizer fallback is deprecated and will be disabled by default in a future release.
+ Set `--no-tokenizer-fallback` to adopt the future behavior now.
+
+
+ Environment variable: `DYN_TOKENIZER_FALLBACK`
+
+
## Experimental
### Python route extensions
diff --git a/docs/fern/pages/reference/general/releases/deprecations.mdx b/docs/fern/pages/reference/general/releases/deprecations.mdx
index 755a1ebcf748..eff92e4de9e8 100644
--- a/docs/fern/pages/reference/general/releases/deprecations.mdx
+++ b/docs/fern/pages/reference/general/releases/deprecations.mdx
@@ -21,7 +21,7 @@ Upgrading? Pick the version you run today to see the dependency migration and th
## v1.4.0 (upcoming)
-5 entries — 3 behavioral changes, 2 removed.
+6 entries — 3 behavioral changes, 1 deprecated, 2 removed.
Behavioral **Audit subsystem migrated into request trace** (`"log_type":"audit"` records and the `"Audit sinks ready"` readiness marker -> records carrying `"schema":"dynamo.request.trace.v1"` and the `"Request trace sinks ready"` marker); payload logging is unified under the request-trace pipeline and the standalone audit module is gone ([#9390](https://github.com/ai-dynamo/dynamo/pull/9390), [#11180](https://github.com/ai-dynamo/dynamo/pull/11180)).
@@ -35,6 +35,10 @@ Upgrading? Pick the version you run today to see the dependency migration and th
> **Migrate:** Upgrade Codex to a version that sends `thread-id`, or send `X-Dynamo-Session-ID` from a custom client. Dynamo ignores `session-id` when `thread-id` is absent.
+Deprecated **Automatic tokenizer fallback to HuggingFace is deprecated**; fallback remains enabled by default for compatibility when `fastokens` or `basetenkenizer` cannot load a model tokenizer, but it will be disabled by default in a future release ([#12923](https://github.com/ai-dynamo/dynamo/pull/12923)).
+
+ > **Migrate:** Pass `--no-tokenizer-fallback` or set `DYN_TOKENIZER_FALLBACK=false` to fail model initialization when the selected tokenizer backend cannot load the model tokenizer.
+
Removed **Removed the deprecated multimodal worker flags** (`--multimodal-worker` / `--multimodal-encode-worker` -> `--enable-multimodal` with `--disaggregation-mode` / `--dedicated-mm-encoder`) across the vLLM, SGLang, and TensorRT-LLM backends; deprecated in v1.3.0 ([#11887](https://github.com/ai-dynamo/dynamo/pull/11887)).
> **Migrate:** On vLLM and SGLang, replace the removed flags with `--enable-multimodal` plus the appropriate `--disaggregation-mode` / `--dedicated-mm-encoder` combination. On TensorRT-LLM, use `--modality multimodal`, which remains the supported path.
diff --git a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md
index cdcf767d443c..56a1d664e279 100644
--- a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md
+++ b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md
@@ -39,6 +39,10 @@ Stay on the default backend if:
## Enable and Validate FastTokens
+> [!WARNING]
+> Automatic tokenizer fallback is deprecated and will be disabled by default in a future release.
+> Set `--no-tokenizer-fallback` or `DYN_TOKENIZER_FALLBACK=false` to adopt the future behavior now.
+
`fastokens` is a **frontend** setting. Enable it on the Frontend component with the `DYN_TOKENIZER` environment variable:
@@ -66,6 +70,8 @@ Stay on the default backend if:
You can also set the same option with the `--tokenizer fastokens` frontend flag. The flag takes precedence when both settings are present.
+ To require `fastokens`, set `DYN_TOKENIZER_FALLBACK=false` or add `--no-tokenizer-fallback`. The frontend then reports an error during model initialization when `fastokens` cannot load the tokenizer instead of using HuggingFace.
+
To return to the default HuggingFace tokenizer backend, set `DYN_TOKENIZER=default` or omit it. Changing the value requires the Frontend pod to be replaced, which `kubectl apply` does automatically.
No client changes are required. Request payloads, OpenAI-compatible API behavior, and streamed responses remain the same.
@@ -86,7 +92,7 @@ Stay on the default backend if:
Failed to load fastokens, falling back to HuggingFace
```
- The fallback keeps the deployment healthy, but the model does not receive the `fastokens` speedup.
+ The default fallback keeps the deployment healthy, but the model does not receive the `fastokens` speedup. With tokenizer fallback disabled, the same loading failure stops model initialization.
@@ -115,7 +121,7 @@ Stay on the default backend if:
- **Encoding**: `fastokens` converts prompt text to token IDs.
- **Decoding**: HuggingFace `tokenizers` converts generated token IDs back to text.
- Both backends load from the same `tokenizer.json`, so supported tokenizers should produce the same token IDs as the default HuggingFace path. If `fastokens` cannot load the tokenizer file, Dynamo logs a warning and falls back to the default backend instead of dropping requests.
+ Both backends load from the same `tokenizer.json`, so supported tokenizers should produce the same token IDs as the default HuggingFace path. If `fastokens` cannot load the tokenizer file, Dynamo logs a warning and falls back to the default backend. Set `--no-tokenizer-fallback` to make this loading failure stop model initialization.
```mermaid
flowchart TD
@@ -123,7 +129,8 @@ Stay on the default backend if:
Kind -->|BPE tokenizer.json| Load{"fastokens loads?"}
Kind -->|.model / .tiktoken| Other["Use existing TikToken backend"]
Load -->|Yes| Fast["Encode with fastokens
Decode with HuggingFace"]
- Load -->|No| Warn["Log warning"] --> Default["Use HuggingFace backend"]
+ Load -->|No, fallback enabled| Warn["Log warning"] --> Default["Use HuggingFace backend"]
+ Load -->|No, fallback disabled| Fail["Model initialization fails"]
Fast --> Serve["Serve requests"]
Default --> Serve
Other --> Serve
@@ -135,7 +142,7 @@ Stay on the default backend if:
| Tokenizer format | Behavior with `--tokenizer fastokens` |
|---|---|
| BPE `tokenizer.json` | Dynamo tries to encode with `fastokens` and decode with HuggingFace. |
-| BPE `tokenizer.json` with unsupported components | Dynamo logs a warning and falls back to HuggingFace. |
+| BPE `tokenizer.json` with unsupported components | Dynamo logs a warning and falls back to HuggingFace by default. With tokenizer fallback disabled, model initialization fails. |
| TikToken `.model` or `.tiktoken` | Unchanged. Dynamo uses the existing TikToken backend. |
`fastokens` targets BPE tokenizer pipelines. It is focused on inference and does not support every HuggingFace `tokenizers` feature; additional encoding outputs and some normalizers or pre-tokenizers are not available.
@@ -152,13 +159,17 @@ For any new model, validate on representative prompts before rolling out broadly
- The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare it against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` to avoid the warning.
+ The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare it against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` to avoid the warning. For a deployment that requires `fastokens`, use `--no-tokenizer-fallback` so an unsupported tokenizer fails during model initialization.
Use `default`, `fastokens`, or `basetenkenizer` for `DYN_TOKENIZER`. Values such as `fast`, `hf`, or `huggingface` are benchmark-runner aliases, not valid values for the frontend environment variable.
+
+ `DYN_TOKENIZER_FALLBACK` is parsed strictly. Use `true`/`false`, `1`/`0`, `on`/`off`, or `yes`/`no`. An invalid value fails model initialization instead of silently enabling the default fallback behavior. In dynamic mode, discovery retries the load while the frontend continues running.
+
+
The `fastokens` and `basetenkenizer` settings have no effect for TikToken-format tokenizers. Dynamo uses the existing TikToken backend, so you should not expect either alternate-backend activation log or speedup.
diff --git a/lib/bindings/python/rust/llm/entrypoint.rs b/lib/bindings/python/rust/llm/entrypoint.rs
index b7a7227a6c47..3a38ac1733d3 100644
--- a/lib/bindings/python/rust/llm/entrypoint.rs
+++ b/lib/bindings/python/rust/llm/entrypoint.rs
@@ -568,7 +568,7 @@ pub(crate) struct EntrypointArgs {
impl EntrypointArgs {
#[allow(clippy::too_many_arguments)]
#[new]
- #[pyo3(signature = (engine_type, model_path=None, model_name=None, endpoint_id=None, template_file=None, router_config=None, kv_cache_block_size=None, http_host=None, http_port=None, http_metrics_port=None, tls_cert_path=None, tls_key_path=None, extra_engine_args=None, mocker_engine_args=None, runtime_config=None, namespace=None, namespace_prefix=None, is_prefill=false, is_decode=false, migration_limit=0, migration_max_seq_len=None, chat_engine_factory=None, aic_perf_config=None, *, metrics_prefix=None, enable_anthropic_api=None, strip_anthropic_preamble=None, enable_streaming_tool_dispatch=None, enable_streaming_reasoning_dispatch=None, reasoning_field_name=None, tokenizer_backend=None))]
+ #[pyo3(signature = (engine_type, model_path=None, model_name=None, endpoint_id=None, template_file=None, router_config=None, kv_cache_block_size=None, http_host=None, http_port=None, http_metrics_port=None, tls_cert_path=None, tls_key_path=None, extra_engine_args=None, mocker_engine_args=None, runtime_config=None, namespace=None, namespace_prefix=None, is_prefill=false, is_decode=false, migration_limit=0, migration_max_seq_len=None, chat_engine_factory=None, aic_perf_config=None, *, metrics_prefix=None, enable_anthropic_api=None, strip_anthropic_preamble=None, enable_streaming_tool_dispatch=None, enable_streaming_reasoning_dispatch=None, reasoning_field_name=None, tokenizer_backend=None, tokenizer_fallback=None))]
pub fn new(
py: Python<'_>,
engine_type: EngineType,
@@ -601,6 +601,7 @@ impl EntrypointArgs {
enable_streaming_reasoning_dispatch: Option,
reasoning_field_name: Option,
tokenizer_backend: Option,
+ tokenizer_fallback: Option,
) -> PyResult {
let endpoint_id_obj: Option = endpoint_id.as_deref().map(EndpointId::from);
if (tls_cert_path.is_some() && tls_key_path.is_none())
@@ -647,6 +648,11 @@ impl EntrypointArgs {
.inner
.set_tokenizer_backend(Some(tokenizer_backend));
}
+ if let Some(tokenizer_fallback) = tokenizer_fallback {
+ runtime_config
+ .inner
+ .set_tokenizer_fallback_enabled(Some(tokenizer_fallback));
+ }
runtime_config.validate_config()?;
Ok(EntrypointArgs {
diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi
index 8b7aabbea46e..4a7fee8fdb17 100644
--- a/lib/bindings/python/src/dynamo/_core.pyi
+++ b/lib/bindings/python/src/dynamo/_core.pyi
@@ -3111,6 +3111,7 @@ class EntrypointArgs:
enable_streaming_tool_dispatch: Optional[bool] = None,
enable_streaming_reasoning_dispatch: Optional[bool] = None,
tokenizer_backend: Optional[str] = None,
+ tokenizer_fallback: Optional[bool] = None,
) -> None:
"""
Create EntrypointArgs.
@@ -3144,7 +3145,8 @@ class EntrypointArgs:
strip_anthropic_preamble: Optional Anthropic preamble stripping override
enable_streaming_tool_dispatch: Optional streaming tool dispatch override
enable_streaming_reasoning_dispatch: Optional streaming reasoning dispatch override
- tokenizer_backend: Optional tokenizer backend override ("default" or "fastokens")
+ tokenizer_backend: Optional tokenizer backend override ("default", "fastokens", or "basetenkenizer")
+ tokenizer_fallback: Whether alternate tokenizer load failures fall back to HuggingFace
"""
...
diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs
index eae344ea97b9..4e85f5396f07 100644
--- a/lib/llm/src/discovery/watcher.rs
+++ b/lib/llm/src/discovery/watcher.rs
@@ -224,6 +224,8 @@ where
local_model_path: Option,
/// Frontend-level tokenizer backend override for discovered model cards.
tokenizer_backend: Option,
+ /// Frontend-level tokenizer fallback override for discovered model cards.
+ tokenizer_fallback_enabled: Option,
/// Worker capabilities accepted by the frontend's engine-native Generate routes.
/// Keep raw pipelines out of default-off and backend-mismatched paths.
generate_engine_capabilities: Vec<&'static str>,
@@ -353,6 +355,7 @@ where
metrics,
local_model_path: None,
tokenizer_backend: None,
+ tokenizer_fallback_enabled: None,
generate_engine_capabilities: Vec::new(),
worker_selector_factory,
}
@@ -370,6 +373,10 @@ where
self.tokenizer_backend = tokenizer_backend;
}
+ pub fn set_tokenizer_fallback_enabled(&mut self, enabled: Option) {
+ self.tokenizer_fallback_enabled = enabled;
+ }
+
pub(crate) fn set_generate_engine_capabilities(&mut self, capabilities: Vec<&'static str>) {
self.generate_engine_capabilities = capabilities;
}
@@ -381,10 +388,13 @@ where
.collect();
}
- fn apply_tokenizer_backend_override(&self, card: &mut ModelDeploymentCard) {
+ fn apply_tokenizer_overrides(&self, card: &mut ModelDeploymentCard) {
if let Some(tokenizer_backend) = self.tokenizer_backend {
card.runtime_config.tokenizer_backend = Some(tokenizer_backend);
}
+ if let Some(enabled) = self.tokenizer_fallback_enabled {
+ card.runtime_config.tokenizer_fallback_enabled = Some(enabled);
+ }
}
/// Wait until we have at least one chat completions model and return it's name.
@@ -1011,7 +1021,7 @@ where
let mut card = instance.deserialize_model::()?;
normalize_legacy_prefill_topology(&mut card);
- self.apply_tokenizer_backend_override(&mut card);
+ self.apply_tokenizer_overrides(&mut card);
validate_card_shape(&card)?;
anyhow::ensure!(
mcid.model_suffix.is_some() == card.lora.is_some(),
@@ -1327,6 +1337,47 @@ mod tests {
));
}
+ #[tokio::test]
+ async fn tokenizer_fallback_override_applies_to_discovered_card() {
+ use dynamo_runtime::{Runtime, distributed::DistributedConfig};
+
+ let runtime = Runtime::from_current().unwrap();
+ let drt = DistributedRuntime::new(runtime, DistributedConfig::process_local())
+ .await
+ .unwrap();
+ let mut watcher = ModelWatcher::new(
+ drt,
+ Arc::new(ModelManager::new()),
+ RouterConfig::default(),
+ 0,
+ None,
+ None,
+ None,
+ Arc::new(Metrics::new()),
+ );
+ watcher.set_tokenizer_fallback_enabled(Some(false));
+
+ let mut card = ModelDeploymentCard::with_name_only("strict-tokenizer");
+ card.runtime_config.tokenizer_fallback_enabled = Some(true);
+ let instance = DiscoveryInstance::Model {
+ namespace: "ns1".to_string(),
+ component: "workers".to_string(),
+ endpoint: "generate".to_string(),
+ instance_id: 1,
+ card_json: serde_json::to_value(card).unwrap(),
+ model_suffix: None,
+ };
+
+ let desired = watcher
+ .normalize(instance, &NamespaceFilter::Global)
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ desired.card.runtime_config.tokenizer_fallback_enabled,
+ Some(false)
+ );
+ }
+
#[tokio::test]
async fn text_pooling_family_preserves_chat_engine() {
use dynamo_runtime::{Runtime, distributed::DistributedConfig};
diff --git a/lib/llm/src/entrypoint/input/common.rs b/lib/llm/src/entrypoint/input/common.rs
index 6afa35b6e6e5..96cc447771ea 100644
--- a/lib/llm/src/entrypoint/input/common.rs
+++ b/lib/llm/src/entrypoint/input/common.rs
@@ -365,6 +365,9 @@ pub async fn prepare_engine(
watcher.set_local_model_path(Some(local_model.path().to_path_buf()));
}
watcher.set_tokenizer_backend(local_model.runtime_config().tokenizer_backend);
+ watcher.set_tokenizer_fallback_enabled(
+ local_model.runtime_config().tokenizer_fallback_enabled,
+ );
let watch_obj = Arc::new(watcher);
let discovery = distributed_runtime.discovery();
let discovery_stream = discovery
diff --git a/lib/llm/src/entrypoint/input/grpc.rs b/lib/llm/src/entrypoint/input/grpc.rs
index c1ee4b78e717..5da8753bb0b0 100644
--- a/lib/llm/src/entrypoint/input/grpc.rs
+++ b/lib/llm/src/entrypoint/input/grpc.rs
@@ -63,6 +63,7 @@ pub async fn run(
local_model_path,
model.metrics_prefix(),
model.runtime_config().tokenizer_backend,
+ model.runtime_config().tokenizer_fallback_enabled,
)
.await?;
grpc_service
@@ -132,6 +133,7 @@ async fn run_watcher(
local_model_path: Option,
metrics_prefix: Option,
tokenizer_backend: Option,
+ tokenizer_fallback_enabled: Option,
) -> anyhow::Result<()> {
// Start the LoRA allocation controller when LoRA serving is enabled (mirrors http.rs;
// additionally gated on DYN_LORA_ALLOCATION_ENABLED inside start_lora_controller). Without
@@ -154,6 +156,7 @@ async fn run_watcher(
);
watch_obj.set_local_model_path(local_model_path);
watch_obj.set_tokenizer_backend(tokenizer_backend);
+ watch_obj.set_tokenizer_fallback_enabled(tokenizer_fallback_enabled);
tracing::debug!("Waiting for remote model");
let discovery = runtime.discovery();
let discovery_stream = discovery
diff --git a/lib/llm/src/entrypoint/input/http.rs b/lib/llm/src/entrypoint/input/http.rs
index 99be4c14a570..1515881efb37 100644
--- a/lib/llm/src/entrypoint/input/http.rs
+++ b/lib/llm/src/entrypoint/input/http.rs
@@ -221,6 +221,7 @@ where
prefill_load_estimator.clone(),
local_model_path,
model.runtime_config().tokenizer_backend,
+ model.runtime_config().tokenizer_fallback_enabled,
generate_engine_capabilities,
worker_selector_factory.clone(),
)
@@ -304,6 +305,7 @@ async fn run_watcher(
prefill_load_estimator: Option>,
local_model_path: Option,
tokenizer_backend: Option,
+ tokenizer_fallback_enabled: Option,
generate_engine_capabilities: Vec<&'static str>,
worker_selector_factory: WorkerSelectorFactory,
) -> anyhow::Result<()>
@@ -331,6 +333,7 @@ where
);
watch_obj.set_local_model_path(local_model_path);
watch_obj.set_tokenizer_backend(tokenizer_backend);
+ watch_obj.set_tokenizer_fallback_enabled(tokenizer_fallback_enabled);
watch_obj.set_generate_engine_capabilities(generate_engine_capabilities);
tracing::debug!("Waiting for remote model");
let discovery = runtime.discovery();
diff --git a/lib/llm/src/local_model/runtime_config.rs b/lib/llm/src/local_model/runtime_config.rs
index c9d0ddefe7ac..073f57229fa8 100644
--- a/lib/llm/src/local_model/runtime_config.rs
+++ b/lib/llm/src/local_model/runtime_config.rs
@@ -70,6 +70,7 @@ pub enum StructuralTagScope {
}
pub const ENV_TOKENIZER_BACKEND: &str = "DYN_TOKENIZER";
+pub const ENV_TOKENIZER_FALLBACK: &str = "DYN_TOKENIZER_FALLBACK";
/// Worker-advertised support for Dynamo's vLLM-compatible
/// `POST /inference/v1/generate` adapter.
@@ -200,6 +201,11 @@ pub struct ModelRuntimeConfig {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tokenizer_backend: Option,
+ /// Frontend tokenizer fallback override. When unset, direct Rust callers can still use
+ /// `DYN_TOKENIZER_FALLBACK`; when set, this explicit value wins over process environment.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub tokenizer_fallback_enabled: Option,
+
/// Whether structural tag guided decoding is enabled for tool calls.
#[serde(default)]
pub structural_tag_mode: StructuralTagMode,
@@ -333,6 +339,7 @@ impl Default for ModelRuntimeConfig {
reasoning_parser: None,
tool_call_arguments_format: ToolCallArgumentsFormat::JsonString,
tokenizer_backend: None,
+ tokenizer_fallback_enabled: None,
structural_tag_mode: StructuralTagMode::Off,
structural_tag_scope: StructuralTagScope::Auto,
structural_tag_schema: StructuralTagSchemaMode::Auto,
@@ -573,6 +580,32 @@ impl ModelRuntimeConfig {
.unwrap_or_else(TokenizerBackend::from_env_or_default)
}
+ pub fn is_tokenizer_fallback_enabled(&self) -> anyhow::Result {
+ let enabled = if let Some(enabled) = self.tokenizer_fallback_enabled {
+ enabled
+ } else {
+ match std::env::var(ENV_TOKENIZER_FALLBACK) {
+ Ok(value) => dynamo_runtime::config::parse_bool(&value)
+ .map_err(|error| anyhow::anyhow!("{ENV_TOKENIZER_FALLBACK}: {error}"))?,
+ Err(std::env::VarError::NotPresent) => true,
+ Err(std::env::VarError::NotUnicode(_)) => {
+ anyhow::bail!("{ENV_TOKENIZER_FALLBACK} must contain valid UTF-8")
+ }
+ }
+ };
+
+ if enabled && self.effective_tokenizer_backend() != TokenizerBackend::Default {
+ static TOKENIZER_FALLBACK_DEPRECATION_WARNED: std::sync::Once = std::sync::Once::new();
+ TOKENIZER_FALLBACK_DEPRECATION_WARNED.call_once(|| {
+ tracing::warn!(
+ "Automatic tokenizer fallback is deprecated and will be disabled by default in a future release. Set tokenizer fallback to false (`--no-tokenizer-fallback` in the frontend) to adopt the future behavior now."
+ );
+ });
+ }
+
+ Ok(enabled)
+ }
+
/// Resolve the KV-state endpoint, preserving the serving endpoint as the compatibility
/// default for workers that do not advertise an explicit mapping.
pub fn effective_kv_state_endpoint(&self, serving_endpoint: &EndpointId) -> EndpointId {
@@ -589,6 +622,11 @@ impl ModelRuntimeConfig {
self
}
+ pub fn set_tokenizer_fallback_enabled(&mut self, enabled: Option) -> &mut Self {
+ self.tokenizer_fallback_enabled = enabled;
+ self
+ }
+
/// Rebuild canonical topology taints derived from `topology_domains`.
///
/// Existing caller-provided taints outside the reserved topology prefix are preserved; generated
@@ -857,6 +895,59 @@ mod tests {
}
}
+ #[test]
+ #[serial_test::serial]
+ fn tokenizer_fallback_env_default_and_explicit_config_precedence() {
+ temp_env::with_vars([(ENV_TOKENIZER_FALLBACK, Some("false"))], || {
+ let config = ModelRuntimeConfig::default();
+ assert!(!config.is_tokenizer_fallback_enabled().unwrap());
+
+ let config = ModelRuntimeConfig {
+ tokenizer_fallback_enabled: Some(true),
+ ..Default::default()
+ };
+ assert!(config.is_tokenizer_fallback_enabled().unwrap());
+ });
+
+ temp_env::with_vars([(ENV_TOKENIZER_FALLBACK, Some("true"))], || {
+ let config = ModelRuntimeConfig {
+ tokenizer_fallback_enabled: Some(false),
+ ..Default::default()
+ };
+ assert!(!config.is_tokenizer_fallback_enabled().unwrap());
+ });
+
+ temp_env::with_vars_unset([ENV_TOKENIZER_FALLBACK], || {
+ let config = ModelRuntimeConfig::default();
+ assert!(config.is_tokenizer_fallback_enabled().unwrap());
+ });
+
+ temp_env::with_vars([(ENV_TOKENIZER_FALLBACK, Some("flase"))], || {
+ let config = ModelRuntimeConfig::default();
+ let error = config.is_tokenizer_fallback_enabled().unwrap_err();
+ assert!(error.to_string().contains(ENV_TOKENIZER_FALLBACK));
+ });
+ }
+
+ #[test]
+ fn tokenizer_fallback_roundtrips_through_serde_json() {
+ for enabled in [true, false] {
+ let config = ModelRuntimeConfig {
+ tokenizer_fallback_enabled: Some(enabled),
+ ..Default::default()
+ };
+ let json = serde_json::to_string(&config).unwrap();
+ assert!(json.contains(&format!("\"tokenizer_fallback_enabled\":{enabled}")));
+ let parsed: ModelRuntimeConfig = serde_json::from_str(&json).unwrap();
+ assert_eq!(parsed.tokenizer_fallback_enabled, Some(enabled));
+ }
+
+ let json = serde_json::to_string(&ModelRuntimeConfig::default()).unwrap();
+ assert!(!json.contains("tokenizer_fallback_enabled"));
+ let legacy: ModelRuntimeConfig = serde_json::from_str("{}").unwrap();
+ assert_eq!(legacy.tokenizer_fallback_enabled, None);
+ }
+
#[test]
fn tokenizer_backend_string_values_are_strict() {
for backend in [
diff --git a/lib/llm/src/model_card.rs b/lib/llm/src/model_card.rs
index e7a9cbb364e3..24f8b7163a03 100644
--- a/lib/llm/src/model_card.rs
+++ b/lib/llm/src/model_card.rs
@@ -1197,6 +1197,9 @@ impl ModelDeploymentCard {
/// Tokenizer backend controls:
/// - `runtime_config.tokenizer_backend` — select `default`, `fastokens`, or `basetenkenizer`
/// - `DYN_TOKENIZER` — fallback backend for callers without explicit runtime config
+ /// - `runtime_config.tokenizer_fallback_enabled` — control whether an alternate backend load
+ /// failure falls back to HuggingFace
+ /// - `DYN_TOKENIZER_FALLBACK=0` — fallback control for callers without explicit runtime config
/// - `DYN_TOKENIZER_CACHE=0` — disable the L1 prefix cache that records tokenizations
/// at special-token boundaries (enabled by default; any other value keeps it enabled)
/// - `DYN_TOKENIZER_CACHE_BYTES=` — L1 cache byte budget (default 64 MiB)
@@ -1207,6 +1210,7 @@ impl ModelDeploymentCard {
/// fall back to the original hit-without-insert behavior.
pub fn tokenizer(&self) -> anyhow::Result {
let tokenizer_backend = self.runtime_config.effective_tokenizer_backend();
+ let is_fallback_enabled = self.runtime_config.is_tokenizer_fallback_enabled()?;
let cache_enabled =
tokenizer_cache_enabled(std::env::var("DYN_TOKENIZER_CACHE").ok().as_deref());
@@ -1297,6 +1301,11 @@ impl ModelDeploymentCard {
Arc::new(fast)
}
Err(e) => {
+ if !is_fallback_enabled {
+ return Err(e).context(
+ "failed to load fastokens tokenizer backend and fallback is disabled",
+ );
+ }
tracing::warn!(
%e,
"Failed to load fastokens, falling back to HuggingFace"
@@ -1305,6 +1314,12 @@ impl ModelDeploymentCard {
}
}
} else {
+ if !is_fallback_enabled {
+ anyhow::bail!(
+ "failed to load fastokens tokenizer backend because tokenizer path contains non-UTF-8 characters and fallback is disabled: {}",
+ p.display()
+ );
+ }
tracing::warn!(
path = %p.display(),
"Tokenizer path contains non-UTF-8 characters, skipping fastokens; falling back to HuggingFace"
@@ -1320,6 +1335,11 @@ impl ModelDeploymentCard {
Arc::new(baseten)
}
Err(e) => {
+ if !is_fallback_enabled {
+ return Err(e).context(
+ "failed to load basetenkenizer tokenizer backend and fallback is disabled",
+ );
+ }
tracing::warn!(
%e,
"Failed to load basetenkenizer, falling back to HuggingFace"
@@ -1328,6 +1348,12 @@ impl ModelDeploymentCard {
}
}
} else {
+ if !is_fallback_enabled {
+ anyhow::bail!(
+ "failed to load basetenkenizer tokenizer backend because tokenizer path contains non-UTF-8 characters and fallback is disabled: {}",
+ p.display()
+ );
+ }
tracing::warn!(
path = %p.display(),
"Tokenizer path contains non-UTF-8 characters, skipping basetenkenizer; falling back to HuggingFace"
diff --git a/lib/llm/tests/model_card.rs b/lib/llm/tests/model_card.rs
index d2547d53260f..dc4cb7771442 100644
--- a/lib/llm/tests/model_card.rs
+++ b/lib/llm/tests/model_card.rs
@@ -7,12 +7,38 @@ use dynamo_llm::model_card::{ModelDeploymentCard, PromptFormatterArtifact, Token
use dynamo_llm::tokenizers::{
BasetenTokenizer, EncodeSegment, HuggingFaceTokenizer, TikTokenTokenizer, traits::Encoder,
};
-use tempfile::tempdir;
+use std::path::PathBuf;
+use tempfile::{TempDir, tempdir};
use tokenizers::models::bpe::{BPE, Vocab};
const HF_PATH: &str = "tests/data/sample-models/TinyLlama_v1.1";
const TIKTOKEN_PATH: &str = "tests/data/sample-models/mock-tiktoken";
+fn unsupported_alternate_tokenizer() -> (TempDir, PathBuf) {
+ let dir = tempdir().unwrap();
+ let tokenizer_path = dir.path().join("tokenizer.json");
+ std::fs::write(
+ &tokenizer_path,
+ r#"{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [],
+ "normalizer": null,
+ "pre_tokenizer": {"type": "Whitespace"},
+ "post_processor": null,
+ "decoder": null,
+ "model": {
+ "type": "WordLevel",
+ "vocab": {"[UNK]": 0, "hello": 1},
+ "unk_token": "[UNK]"
+ }
+ }"#,
+ )
+ .unwrap();
+ (dir, tokenizer_path)
+}
+
#[tokio::test]
async fn test_model_info_from_hf_like_local_repo() {
let mdc = ModelDeploymentCard::load_from_disk(HF_PATH, None).unwrap();
@@ -156,49 +182,68 @@ fn test_basetenkenizer_model_card_matches_hf_and_uses_prefix_cache() {
#[test]
#[serial_test::serial]
fn test_basetenkenizer_load_failure_falls_back_to_hf() {
- temp_env::with_vars([("DYN_TOKENIZER_CACHE", Some("0"))], || {
- let dir = tempdir().unwrap();
- let tokenizer_path = dir.path().join("tokenizer.json");
- std::fs::write(
- &tokenizer_path,
- r#"{
- "version": "1.0",
- "truncation": null,
- "padding": null,
- "added_tokens": [],
- "normalizer": null,
- "pre_tokenizer": {"type": "Whitespace"},
- "post_processor": null,
- "decoder": null,
- "model": {
- "type": "WordLevel",
- "vocab": {"[UNK]": 0, "hello": 1},
- "unk_token": "[UNK]"
- }
- }"#,
- )
- .unwrap();
+ temp_env::with_vars(
+ [
+ ("DYN_TOKENIZER_CACHE", Some("0")),
+ ("DYN_TOKENIZER_FALLBACK", None),
+ ],
+ || {
+ let (_dir, tokenizer_path) = unsupported_alternate_tokenizer();
- assert!(
- BasetenTokenizer::from_file(tokenizer_path.to_str().unwrap()).is_err(),
- "fixture must remain unsupported by the Baseten BPE backend"
- );
+ assert!(
+ BasetenTokenizer::from_file(tokenizer_path.to_str().unwrap()).is_err(),
+ "fixture must remain unsupported by the Baseten BPE backend"
+ );
- let mut mdc = ModelDeploymentCard::with_name_only("baseten-fallback");
- mdc.tokenizer = Some(TokenizerKind::HfTokenizerJson(
- CheckedFile::from_disk(&tokenizer_path).unwrap(),
- ));
- mdc.runtime_config.tokenizer_backend = Some(TokenizerBackend::Basetenkenizer);
+ let mut mdc = ModelDeploymentCard::with_name_only("baseten-fallback");
+ mdc.tokenizer = Some(TokenizerKind::HfTokenizerJson(
+ CheckedFile::from_disk(&tokenizer_path).unwrap(),
+ ));
+ mdc.runtime_config.tokenizer_backend = Some(TokenizerBackend::Basetenkenizer);
- let tokenizer = mdc
- .tokenizer()
- .expect("unsupported Baseten tokenizer must fall back to HuggingFace");
- assert_eq!(
- tokenizer.encode("hello").unwrap().token_ids(),
- &[1],
- "fallback must remain usable"
- );
- });
+ let tokenizer = mdc
+ .tokenizer()
+ .expect("unsupported Baseten tokenizer must fall back to HuggingFace");
+ assert_eq!(
+ tokenizer.encode("hello").unwrap().token_ids(),
+ &[1],
+ "fallback must remain usable"
+ );
+ },
+ );
+}
+
+#[test]
+#[serial_test::serial]
+fn test_alternate_tokenizer_load_failure_is_rejected_when_fallback_is_disabled() {
+ temp_env::with_vars(
+ [
+ ("DYN_TOKENIZER_CACHE", Some("0")),
+ ("DYN_TOKENIZER_FALLBACK", Some("1")),
+ ],
+ || {
+ for backend in [
+ TokenizerBackend::Fastokens,
+ TokenizerBackend::Basetenkenizer,
+ ] {
+ let (_dir, tokenizer_path) = unsupported_alternate_tokenizer();
+ let mut mdc = ModelDeploymentCard::with_name_only("strict-tokenizer");
+ mdc.tokenizer = Some(TokenizerKind::HfTokenizerJson(
+ CheckedFile::from_disk(&tokenizer_path).unwrap(),
+ ));
+ mdc.runtime_config.tokenizer_backend = Some(backend);
+ mdc.runtime_config.tokenizer_fallback_enabled = Some(false);
+
+ let error = mdc
+ .tokenizer()
+ .err()
+ .expect("unsupported alternate tokenizer must fail without fallback")
+ .to_string();
+ assert!(error.contains(backend.as_str()));
+ assert!(error.contains("fallback is disabled"));
+ }
+ },
+ );
}
#[test]