From 58097b8f7f9b349f904f3ee2450fe277d5009b97 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Mon, 10 Aug 2026 14:17:34 +0800 Subject: [PATCH 1/8] feat(frontend): make tokenizer fallback configurable Signed-off-by: xianlubird --- .../src/dynamo/frontend/frontend_args.py | 13 ++ components/src/dynamo/frontend/main.py | 1 + .../modular-components/frontend/tokenizer.md | 8 +- .../components/frontend-configuration.mdx | 7 + .../use-cases/fastokens-tokenizer/overview.md | 10 +- lib/llm/src/model_card.rs | 33 +++++ lib/llm/tests/model_card.rs | 126 ++++++++++++------ 7 files changed, 151 insertions(+), 47 deletions(-) diff --git a/components/src/dynamo/frontend/frontend_args.py b/components/src/dynamo/frontend/frontend_args.py index 722eab142b3c..2956acd610ee 100644 --- a/components/src/dynamo/frontend/frontend_args.py +++ b/components/src/dynamo/frontend/frontend_args.py @@ -89,6 +89,7 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AicPerfConfigBase): exclude_tools_when_tool_choice_none: bool preprocess_workers: int tokenizer_backend: str + tokenizer_fallback: bool trust_remote_code: bool frontend_route_extensions: list[str] @@ -552,6 +553,18 @@ def add_arguments(self, parser) -> None: choices=["default", "fastokens", "basetenkenizer"], ) + add_negatable_bool_argument( + g, + flag_name="--tokenizer-fallback", + env_var="DYN_TOKENIZER_FALLBACK", + default=True, + help=( + "Fall back to HuggingFace when the selected fastokens or " + "basetenkenizer backend cannot load the model tokenizer. Disable " + "this option to fail during startup instead." + ), + ) + add_negatable_bool_argument( g, flag_name="--trust-remote-code", diff --git a/components/src/dynamo/frontend/main.py b/components/src/dynamo/frontend/main.py index 200ee1ec2ad7..e2e21131225a 100644 --- a/components/src/dynamo/frontend/main.py +++ b/components/src/dynamo/frontend/main.py @@ -390,6 +390,7 @@ def signal_handler(): kv_router_config = None os.environ[MIN_INITIAL_WORKERS_ENV] = str(config.min_initial_workers) + os.environ["DYN_TOKENIZER_FALLBACK"] = str(config.tokenizer_fallback).lower() router_config = RouterConfig( router_mode, kv_router_config, **config.router_kwargs() ) diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md index 9ab05b474b43..ee8a2fdd1d48 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md @@ -31,7 +31,7 @@ Use this backend for supported `tokenizer.json` models when you need Baseten Tok #### Compatibility notes: - Works with standard BPE `tokenizer.json` files (Qwen, LLaMA, GPT-family, Mistral, DeepSeek, etc.). -- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace; requests are never dropped. +- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace by default. Disable tokenizer fallback to reject incompatible configurations during startup. - Special tokens declared only in a sibling `tokenizer_config.json` are preserved for Baseten encoding and decoding and for Dynamo's L1 prefix-cache boundaries. - Has no effect on TikToken-format tokenizers (`.model` / `.tiktoken` files), which always use the TikToken backend. @@ -42,6 +42,7 @@ Set the backend with a CLI flag or environment variable. The CLI flag takes prec | CLI Argument | Env Var | Valid values | Default | |---|---|---|---| | `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens`, `basetenkenizer` | `default` | +| `--tokenizer-fallback` / `--no-tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | Boolean | Enabled | **Examples:** @@ -55,6 +56,9 @@ python -m dynamo.frontend # Baseten Tokenizer python -m dynamo.frontend --tokenizer basetenkenizer + +# Require Baseten Tokenizer instead of falling back to HuggingFace +python -m dynamo.frontend --tokenizer basetenkenizer --no-tokenizer-fallback ``` ## Dynamo Frontend Behavior @@ -64,5 +68,5 @@ When a non-default backend is selected: 1. The frontend resolves `--tokenizer` / `DYN_TOKENIZER` and passes the selected backend to the Rust runtime. 2. `ModelDeploymentCard::tokenizer()` loads the HuggingFace tokenizer first for fallback behavior and L1 cache special-token metadata. 3. Dynamo constructs `FastTokenizer` for `fastokens` or `BasetenTokenizer` for `basetenkenizer` from the same `tokenizer.json` file. -4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. +4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--no-tokenizer-fallback`, startup fails and reports the backend loading error instead. 5. When the L1 prefix cache is enabled, Dynamo wraps the selected backend with the same special-token boundary metadata and cache metrics used by the default path. diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 55bf91b06833..53ea766089ee 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -506,6 +506,13 @@ See the [Frontend Guide](../../developer-guide/knowledge-base/modular-components Environment variable: `DYN_TOKENIZER` + + Fall back to HuggingFace when `fastokens` or `basetenkenizer` cannot load the model tokenizer. Use + `--no-tokenizer-fallback` to fail during startup instead. + + Environment variable: `DYN_TOKENIZER_FALLBACK` + + ## Experimental ### Python route extensions diff --git a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md index cdcf767d443c..52ef1c8b9307 100644 --- a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md +++ b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md @@ -66,6 +66,8 @@ Stay on the default backend if: You can also set the same option with the `--tokenizer fastokens` frontend flag. The flag takes precedence when both settings are present. + To require `fastokens`, set `DYN_TOKENIZER_FALLBACK=false` or add `--no-tokenizer-fallback`. The frontend then reports an error during model initialization when `fastokens` cannot load the tokenizer instead of using HuggingFace. + To return to the default HuggingFace tokenizer backend, set `DYN_TOKENIZER=default` or omit it. Changing the value requires the Frontend pod to be replaced, which `kubectl apply` does automatically. No client changes are required. Request payloads, OpenAI-compatible API behavior, and streamed responses remain the same. @@ -86,7 +88,7 @@ Stay on the default backend if: Failed to load fastokens, falling back to HuggingFace ``` - The fallback keeps the deployment healthy, but the model does not receive the `fastokens` speedup. + The default fallback keeps the deployment healthy, but the model does not receive the `fastokens` speedup. With tokenizer fallback disabled, the same loading failure stops model initialization. @@ -115,7 +117,7 @@ Stay on the default backend if: - **Encoding**: `fastokens` converts prompt text to token IDs. - **Decoding**: HuggingFace `tokenizers` converts generated token IDs back to text. - Both backends load from the same `tokenizer.json`, so supported tokenizers should produce the same token IDs as the default HuggingFace path. If `fastokens` cannot load the tokenizer file, Dynamo logs a warning and falls back to the default backend instead of dropping requests. + Both backends load from the same `tokenizer.json`, so supported tokenizers should produce the same token IDs as the default HuggingFace path. If `fastokens` cannot load the tokenizer file, Dynamo logs a warning and falls back to the default backend. Set `--no-tokenizer-fallback` to make this loading failure stop model initialization. ```mermaid flowchart TD @@ -135,7 +137,7 @@ Stay on the default backend if: | Tokenizer format | Behavior with `--tokenizer fastokens` | |---|---| | BPE `tokenizer.json` | Dynamo tries to encode with `fastokens` and decode with HuggingFace. | -| BPE `tokenizer.json` with unsupported components | Dynamo logs a warning and falls back to HuggingFace. | +| BPE `tokenizer.json` with unsupported components | Dynamo logs a warning and falls back to HuggingFace by default. With tokenizer fallback disabled, model initialization fails. | | TikToken `.model` or `.tiktoken` | Unchanged. Dynamo uses the existing TikToken backend. | `fastokens` targets BPE tokenizer pipelines. It is focused on inference and does not support every HuggingFace `tokenizers` feature; additional encoding outputs and some normalizers or pre-tokenizers are not available. @@ -152,7 +154,7 @@ For any new model, validate on representative prompts before rolling out broadly - The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare it against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` to avoid the warning. + The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare it against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` to avoid the warning. For a deployment that requires `fastokens`, use `--no-tokenizer-fallback` so an unsupported tokenizer fails during model initialization. diff --git a/lib/llm/src/model_card.rs b/lib/llm/src/model_card.rs index 9302272d78c9..5efe8c71537b 100644 --- a/lib/llm/src/model_card.rs +++ b/lib/llm/src/model_card.rs @@ -32,6 +32,7 @@ use crate::preprocessor::media::{MediaDecoder, MediaFetcher}; use crate::protocols::TokenIdType; const DEFAULT_TOKENIZER_CACHE_BYTES: usize = 64 * 1024 * 1024; +const ENV_TOKENIZER_FALLBACK: &str = "DYN_TOKENIZER_FALLBACK"; fn append_indexer_identity_checksum(bytes: &mut Vec, spec: &IndexerIdentitySpec) { bytes.extend_from_slice(b"dynamo/model-card/indexer-identity/v1"); @@ -62,6 +63,12 @@ fn tokenizer_cache_enabled(value: Option<&str>) -> bool { !matches!(value, Some("0")) } +fn is_tokenizer_fallback_enabled(value: Option<&str>) -> bool { + value + .and_then(dynamo_runtime::config::parse_bool_opt) + .unwrap_or(true) +} + fn tokenizer_cache_bytes(value: Option<&str>) -> usize { value .and_then(|value| value.parse::().ok()) @@ -1197,6 +1204,8 @@ impl ModelDeploymentCard { /// Tokenizer backend controls: /// - `runtime_config.tokenizer_backend` — select `default`, `fastokens`, or `basetenkenizer` /// - `DYN_TOKENIZER` — fallback backend for callers without explicit runtime config + /// - `DYN_TOKENIZER_FALLBACK=0` — fail if the selected alternate backend cannot load + /// instead of falling back to HuggingFace (fallback is enabled by default) /// - `DYN_TOKENIZER_CACHE=0` — disable the L1 prefix cache that records tokenizations /// at special-token boundaries (enabled by default; any other value keeps it enabled) /// - `DYN_TOKENIZER_CACHE_BYTES=` — L1 cache byte budget (default 64 MiB) @@ -1207,6 +1216,8 @@ impl ModelDeploymentCard { /// fall back to the original hit-without-insert behavior. pub fn tokenizer(&self) -> anyhow::Result { let tokenizer_backend = self.runtime_config.effective_tokenizer_backend(); + let is_fallback_enabled = + is_tokenizer_fallback_enabled(std::env::var(ENV_TOKENIZER_FALLBACK).ok().as_deref()); let cache_enabled = tokenizer_cache_enabled(std::env::var("DYN_TOKENIZER_CACHE").ok().as_deref()); @@ -1297,6 +1308,11 @@ impl ModelDeploymentCard { Arc::new(fast) } Err(e) => { + if !is_fallback_enabled { + return Err(e).context( + "failed to load fastokens tokenizer backend and fallback is disabled", + ); + } tracing::warn!( %e, "Failed to load fastokens, falling back to HuggingFace" @@ -1305,6 +1321,12 @@ impl ModelDeploymentCard { } } } else { + if !is_fallback_enabled { + anyhow::bail!( + "failed to load fastokens tokenizer backend because tokenizer path contains non-UTF-8 characters and fallback is disabled: {}", + p.display() + ); + } tracing::warn!( path = %p.display(), "Tokenizer path contains non-UTF-8 characters, skipping fastokens; falling back to HuggingFace" @@ -1320,6 +1342,11 @@ impl ModelDeploymentCard { Arc::new(baseten) } Err(e) => { + if !is_fallback_enabled { + return Err(e).context( + "failed to load basetenkenizer tokenizer backend and fallback is disabled", + ); + } tracing::warn!( %e, "Failed to load basetenkenizer, falling back to HuggingFace" @@ -1328,6 +1355,12 @@ impl ModelDeploymentCard { } } } else { + if !is_fallback_enabled { + anyhow::bail!( + "failed to load basetenkenizer tokenizer backend because tokenizer path contains non-UTF-8 characters and fallback is disabled: {}", + p.display() + ); + } tracing::warn!( path = %p.display(), "Tokenizer path contains non-UTF-8 characters, skipping basetenkenizer; falling back to HuggingFace" diff --git a/lib/llm/tests/model_card.rs b/lib/llm/tests/model_card.rs index d2547d53260f..a21389ed3830 100644 --- a/lib/llm/tests/model_card.rs +++ b/lib/llm/tests/model_card.rs @@ -7,12 +7,38 @@ use dynamo_llm::model_card::{ModelDeploymentCard, PromptFormatterArtifact, Token use dynamo_llm::tokenizers::{ BasetenTokenizer, EncodeSegment, HuggingFaceTokenizer, TikTokenTokenizer, traits::Encoder, }; -use tempfile::tempdir; +use std::path::PathBuf; +use tempfile::{TempDir, tempdir}; use tokenizers::models::bpe::{BPE, Vocab}; const HF_PATH: &str = "tests/data/sample-models/TinyLlama_v1.1"; const TIKTOKEN_PATH: &str = "tests/data/sample-models/mock-tiktoken"; +fn unsupported_alternate_tokenizer() -> (TempDir, PathBuf) { + let dir = tempdir().unwrap(); + let tokenizer_path = dir.path().join("tokenizer.json"); + std::fs::write( + &tokenizer_path, + r#"{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [], + "normalizer": null, + "pre_tokenizer": {"type": "Whitespace"}, + "post_processor": null, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": {"[UNK]": 0, "hello": 1}, + "unk_token": "[UNK]" + } + }"#, + ) + .unwrap(); + (dir, tokenizer_path) +} + #[tokio::test] async fn test_model_info_from_hf_like_local_repo() { let mdc = ModelDeploymentCard::load_from_disk(HF_PATH, None).unwrap(); @@ -156,49 +182,67 @@ fn test_basetenkenizer_model_card_matches_hf_and_uses_prefix_cache() { #[test] #[serial_test::serial] fn test_basetenkenizer_load_failure_falls_back_to_hf() { - temp_env::with_vars([("DYN_TOKENIZER_CACHE", Some("0"))], || { - let dir = tempdir().unwrap(); - let tokenizer_path = dir.path().join("tokenizer.json"); - std::fs::write( - &tokenizer_path, - r#"{ - "version": "1.0", - "truncation": null, - "padding": null, - "added_tokens": [], - "normalizer": null, - "pre_tokenizer": {"type": "Whitespace"}, - "post_processor": null, - "decoder": null, - "model": { - "type": "WordLevel", - "vocab": {"[UNK]": 0, "hello": 1}, - "unk_token": "[UNK]" - } - }"#, - ) - .unwrap(); + temp_env::with_vars( + [ + ("DYN_TOKENIZER_CACHE", Some("0")), + ("DYN_TOKENIZER_FALLBACK", None), + ], + || { + let (_dir, tokenizer_path) = unsupported_alternate_tokenizer(); - assert!( - BasetenTokenizer::from_file(tokenizer_path.to_str().unwrap()).is_err(), - "fixture must remain unsupported by the Baseten BPE backend" - ); + assert!( + BasetenTokenizer::from_file(tokenizer_path.to_str().unwrap()).is_err(), + "fixture must remain unsupported by the Baseten BPE backend" + ); - let mut mdc = ModelDeploymentCard::with_name_only("baseten-fallback"); - mdc.tokenizer = Some(TokenizerKind::HfTokenizerJson( - CheckedFile::from_disk(&tokenizer_path).unwrap(), - )); - mdc.runtime_config.tokenizer_backend = Some(TokenizerBackend::Basetenkenizer); + let mut mdc = ModelDeploymentCard::with_name_only("baseten-fallback"); + mdc.tokenizer = Some(TokenizerKind::HfTokenizerJson( + CheckedFile::from_disk(&tokenizer_path).unwrap(), + )); + mdc.runtime_config.tokenizer_backend = Some(TokenizerBackend::Basetenkenizer); - let tokenizer = mdc - .tokenizer() - .expect("unsupported Baseten tokenizer must fall back to HuggingFace"); - assert_eq!( - tokenizer.encode("hello").unwrap().token_ids(), - &[1], - "fallback must remain usable" - ); - }); + let tokenizer = mdc + .tokenizer() + .expect("unsupported Baseten tokenizer must fall back to HuggingFace"); + assert_eq!( + tokenizer.encode("hello").unwrap().token_ids(), + &[1], + "fallback must remain usable" + ); + }, + ); +} + +#[test] +#[serial_test::serial] +fn test_alternate_tokenizer_load_failure_is_rejected_when_fallback_is_disabled() { + temp_env::with_vars( + [ + ("DYN_TOKENIZER_CACHE", Some("0")), + ("DYN_TOKENIZER_FALLBACK", Some("0")), + ], + || { + for backend in [ + TokenizerBackend::Fastokens, + TokenizerBackend::Basetenkenizer, + ] { + let (_dir, tokenizer_path) = unsupported_alternate_tokenizer(); + let mut mdc = ModelDeploymentCard::with_name_only("strict-tokenizer"); + mdc.tokenizer = Some(TokenizerKind::HfTokenizerJson( + CheckedFile::from_disk(&tokenizer_path).unwrap(), + )); + mdc.runtime_config.tokenizer_backend = Some(backend); + + let error = mdc + .tokenizer() + .err() + .expect("unsupported alternate tokenizer must fail without fallback") + .to_string(); + assert!(error.contains(backend.as_str())); + assert!(error.contains("fallback is disabled")); + } + }, + ); } #[test] From 82d9f0b7eb98f4aa0e513e8510e57c2d73282ccb Mon Sep 17 00:00:00 2001 From: xianlubird Date: Mon, 10 Aug 2026 14:33:06 +0800 Subject: [PATCH 2/8] fix(frontend): accept explicit tokenizer fallback value Signed-off-by: xianlubird --- .../src/dynamo/common/configuration/utils.py | 10 ++++++++++ .../common/tests/configuration/test_utils.py | 14 ++++++++++++++ components/src/dynamo/frontend/frontend_args.py | 9 ++++++--- .../modular-components/frontend/tokenizer.md | 6 +++--- .../components/frontend-configuration.mdx | 6 +++--- .../use-cases/fastokens-tokenizer/overview.md | 6 +++--- 6 files changed, 39 insertions(+), 12 deletions(-) diff --git a/components/src/dynamo/common/configuration/utils.py b/components/src/dynamo/common/configuration/utils.py index f3cf35ca9651..fb97eaca0584 100644 --- a/components/src/dynamo/common/configuration/utils.py +++ b/components/src/dynamo/common/configuration/utils.py @@ -11,6 +11,16 @@ T = TypeVar("T") +def parse_bool(value: str) -> bool: + """Parse a case-insensitive true or false CLI value.""" + normalized = value.strip().lower() + if normalized == "true": + return True + if normalized == "false": + return False + raise argparse.ArgumentTypeError("expected 'true' or 'false'") + + def split_served_model_names(served_model_name: Any) -> list[str]: """Split a ``--served-model-name`` value into individual names. diff --git a/components/src/dynamo/common/tests/configuration/test_utils.py b/components/src/dynamo/common/tests/configuration/test_utils.py index 7952966a4748..4c505ad4de19 100644 --- a/components/src/dynamo/common/tests/configuration/test_utils.py +++ b/components/src/dynamo/common/tests/configuration/test_utils.py @@ -11,6 +11,7 @@ add_negatable_bool_argument, env_or_default, nullable_float, + parse_bool, ) pytestmark = [ @@ -20,6 +21,19 @@ ] +@pytest.mark.parametrize( + ("value", "expected"), + [("true", True), ("TRUE", True), ("false", False), ("FALSE", False)], +) +def test_parse_bool(value, expected): + assert parse_bool(value) is expected + + +def test_parse_bool_rejects_invalid_value(): + with pytest.raises(argparse.ArgumentTypeError, match="expected 'true' or 'false'"): + parse_bool("yes") + + class TestEnvOrDefault: """Test env_or_default function.""" diff --git a/components/src/dynamo/frontend/frontend_args.py b/components/src/dynamo/frontend/frontend_args.py index 2956acd610ee..65a31f2c89e1 100644 --- a/components/src/dynamo/frontend/frontend_args.py +++ b/components/src/dynamo/frontend/frontend_args.py @@ -24,6 +24,7 @@ add_argument, add_negatable_bool_argument, env_or_default, + parse_bool, ) from . import __version__ @@ -553,16 +554,18 @@ def add_arguments(self, parser) -> None: choices=["default", "fastokens", "basetenkenizer"], ) - add_negatable_bool_argument( + add_argument( g, flag_name="--tokenizer-fallback", env_var="DYN_TOKENIZER_FALLBACK", default=True, help=( "Fall back to HuggingFace when the selected fastokens or " - "basetenkenizer backend cannot load the model tokenizer. Disable " - "this option to fail during startup instead." + "basetenkenizer backend cannot load the model tokenizer. Set this " + "option to false to fail during startup instead." ), + arg_type=parse_bool, + metavar="{true,false}", ) add_negatable_bool_argument( diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md index ee8a2fdd1d48..0f1c4084da30 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md @@ -42,7 +42,7 @@ Set the backend with a CLI flag or environment variable. The CLI flag takes prec | CLI Argument | Env Var | Valid values | Default | |---|---|---|---| | `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens`, `basetenkenizer` | `default` | -| `--tokenizer-fallback` / `--no-tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | Boolean | Enabled | +| `--tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | `true`, `false` | `true` | **Examples:** @@ -58,7 +58,7 @@ python -m dynamo.frontend python -m dynamo.frontend --tokenizer basetenkenizer # Require Baseten Tokenizer instead of falling back to HuggingFace -python -m dynamo.frontend --tokenizer basetenkenizer --no-tokenizer-fallback +python -m dynamo.frontend --tokenizer basetenkenizer --tokenizer-fallback false ``` ## Dynamo Frontend Behavior @@ -68,5 +68,5 @@ When a non-default backend is selected: 1. The frontend resolves `--tokenizer` / `DYN_TOKENIZER` and passes the selected backend to the Rust runtime. 2. `ModelDeploymentCard::tokenizer()` loads the HuggingFace tokenizer first for fallback behavior and L1 cache special-token metadata. 3. Dynamo constructs `FastTokenizer` for `fastokens` or `BasetenTokenizer` for `basetenkenizer` from the same `tokenizer.json` file. -4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--no-tokenizer-fallback`, startup fails and reports the backend loading error instead. +4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--tokenizer-fallback false`, startup fails and reports the backend loading error instead. 5. When the L1 prefix cache is enabled, Dynamo wraps the selected backend with the same special-token boundary metadata and cache metrics used by the default path. diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 53ea766089ee..54a48d8a366f 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -506,9 +506,9 @@ See the [Frontend Guide](../../developer-guide/knowledge-base/modular-components Environment variable: `DYN_TOKENIZER` - - Fall back to HuggingFace when `fastokens` or `basetenkenizer` cannot load the model tokenizer. Use - `--no-tokenizer-fallback` to fail during startup instead. + + Fall back to HuggingFace when `fastokens` or `basetenkenizer` cannot load the model tokenizer. Set + this option to `false` to fail during startup instead. Environment variable: `DYN_TOKENIZER_FALLBACK` diff --git a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md index 52ef1c8b9307..7bcd23547b0e 100644 --- a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md +++ b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md @@ -66,7 +66,7 @@ Stay on the default backend if: You can also set the same option with the `--tokenizer fastokens` frontend flag. The flag takes precedence when both settings are present. - To require `fastokens`, set `DYN_TOKENIZER_FALLBACK=false` or add `--no-tokenizer-fallback`. The frontend then reports an error during model initialization when `fastokens` cannot load the tokenizer instead of using HuggingFace. + To require `fastokens`, set `DYN_TOKENIZER_FALLBACK=false` or add `--tokenizer-fallback false`. The frontend then reports an error during model initialization when `fastokens` cannot load the tokenizer instead of using HuggingFace. To return to the default HuggingFace tokenizer backend, set `DYN_TOKENIZER=default` or omit it. Changing the value requires the Frontend pod to be replaced, which `kubectl apply` does automatically. @@ -117,7 +117,7 @@ Stay on the default backend if: - **Encoding**: `fastokens` converts prompt text to token IDs. - **Decoding**: HuggingFace `tokenizers` converts generated token IDs back to text. - Both backends load from the same `tokenizer.json`, so supported tokenizers should produce the same token IDs as the default HuggingFace path. If `fastokens` cannot load the tokenizer file, Dynamo logs a warning and falls back to the default backend. Set `--no-tokenizer-fallback` to make this loading failure stop model initialization. + Both backends load from the same `tokenizer.json`, so supported tokenizers should produce the same token IDs as the default HuggingFace path. If `fastokens` cannot load the tokenizer file, Dynamo logs a warning and falls back to the default backend. Set `--tokenizer-fallback false` to make this loading failure stop model initialization. ```mermaid flowchart TD @@ -154,7 +154,7 @@ For any new model, validate on representative prompts before rolling out broadly - The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare it against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` to avoid the warning. For a deployment that requires `fastokens`, use `--no-tokenizer-fallback` so an unsupported tokenizer fails during model initialization. + The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare it against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` to avoid the warning. For a deployment that requires `fastokens`, use `--tokenizer-fallback false` so an unsupported tokenizer fails during model initialization. From 01159fcc8a491d274fbd7db20f068478fa4a4809 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Mon, 10 Aug 2026 14:46:04 +0800 Subject: [PATCH 3/8] fix(frontend): pass tokenizer fallback explicitly Signed-off-by: xianlubird --- components/src/dynamo/frontend/main.py | 2 +- .../use-cases/fastokens-tokenizer/overview.md | 3 +- lib/bindings/python/rust/llm/entrypoint.rs | 8 ++- lib/bindings/python/src/dynamo/_core.pyi | 4 +- lib/llm/src/local_model/runtime_config.rs | 69 +++++++++++++++++++ lib/llm/src/model_card.rs | 15 ++-- lib/llm/tests/model_card.rs | 3 +- 7 files changed, 88 insertions(+), 16 deletions(-) diff --git a/components/src/dynamo/frontend/main.py b/components/src/dynamo/frontend/main.py index e2e21131225a..9e0f91ea5bf0 100644 --- a/components/src/dynamo/frontend/main.py +++ b/components/src/dynamo/frontend/main.py @@ -390,7 +390,6 @@ def signal_handler(): kv_router_config = None os.environ[MIN_INITIAL_WORKERS_ENV] = str(config.min_initial_workers) - os.environ["DYN_TOKENIZER_FALLBACK"] = str(config.tokenizer_fallback).lower() router_config = RouterConfig( router_mode, kv_router_config, **config.router_kwargs() ) @@ -413,6 +412,7 @@ def signal_handler(): "enable_streaming_reasoning_dispatch": config.enable_streaming_reasoning_dispatch, "reasoning_field_name": config.reasoning_field_name, "tokenizer_backend": config.tokenizer_backend, + "tokenizer_fallback": config.tokenizer_fallback, } if config.migration_max_seq_len is not None: kwargs["migration_max_seq_len"] = config.migration_max_seq_len diff --git a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md index 7bcd23547b0e..449c2aa540dc 100644 --- a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md +++ b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md @@ -125,7 +125,8 @@ Stay on the default backend if: Kind -->|BPE tokenizer.json| Load{"fastokens loads?"} Kind -->|.model / .tiktoken| Other["Use existing TikToken backend"] Load -->|Yes| Fast["Encode with fastokens
Decode with HuggingFace"] - Load -->|No| Warn["Log warning"] --> Default["Use HuggingFace backend"] + Load -->|No, fallback true| Warn["Log warning"] --> Default["Use HuggingFace backend"] + Load -->|No, fallback false| Fail["Model initialization fails"] Fast --> Serve["Serve requests"] Default --> Serve Other --> Serve diff --git a/lib/bindings/python/rust/llm/entrypoint.rs b/lib/bindings/python/rust/llm/entrypoint.rs index d3dc3babb59e..22ebfdbb93ef 100644 --- a/lib/bindings/python/rust/llm/entrypoint.rs +++ b/lib/bindings/python/rust/llm/entrypoint.rs @@ -568,7 +568,7 @@ pub(crate) struct EntrypointArgs { impl EntrypointArgs { #[allow(clippy::too_many_arguments)] #[new] - #[pyo3(signature = (engine_type, model_path=None, model_name=None, endpoint_id=None, template_file=None, router_config=None, kv_cache_block_size=None, http_host=None, http_port=None, http_metrics_port=None, tls_cert_path=None, tls_key_path=None, extra_engine_args=None, mocker_engine_args=None, runtime_config=None, namespace=None, namespace_prefix=None, is_prefill=false, is_decode=false, migration_limit=0, migration_max_seq_len=None, chat_engine_factory=None, aic_perf_config=None, *, metrics_prefix=None, enable_anthropic_api=None, strip_anthropic_preamble=None, enable_streaming_tool_dispatch=None, enable_streaming_reasoning_dispatch=None, reasoning_field_name=None, tokenizer_backend=None))] + #[pyo3(signature = (engine_type, model_path=None, model_name=None, endpoint_id=None, template_file=None, router_config=None, kv_cache_block_size=None, http_host=None, http_port=None, http_metrics_port=None, tls_cert_path=None, tls_key_path=None, extra_engine_args=None, mocker_engine_args=None, runtime_config=None, namespace=None, namespace_prefix=None, is_prefill=false, is_decode=false, migration_limit=0, migration_max_seq_len=None, chat_engine_factory=None, aic_perf_config=None, *, metrics_prefix=None, enable_anthropic_api=None, strip_anthropic_preamble=None, enable_streaming_tool_dispatch=None, enable_streaming_reasoning_dispatch=None, reasoning_field_name=None, tokenizer_backend=None, tokenizer_fallback=None))] pub fn new( py: Python<'_>, engine_type: EngineType, @@ -601,6 +601,7 @@ impl EntrypointArgs { enable_streaming_reasoning_dispatch: Option, reasoning_field_name: Option, tokenizer_backend: Option, + tokenizer_fallback: Option, ) -> PyResult { let endpoint_id_obj: Option = endpoint_id.as_deref().map(EndpointId::from); if (tls_cert_path.is_some() && tls_key_path.is_none()) @@ -647,6 +648,11 @@ impl EntrypointArgs { .inner .set_tokenizer_backend(Some(tokenizer_backend)); } + if let Some(tokenizer_fallback) = tokenizer_fallback { + runtime_config + .inner + .set_tokenizer_fallback_enabled(Some(tokenizer_fallback)); + } runtime_config.validate_config()?; Ok(EntrypointArgs { diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index d7d034d68f5f..63cb6d02aaa5 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -3100,6 +3100,7 @@ class EntrypointArgs: enable_streaming_tool_dispatch: Optional[bool] = None, enable_streaming_reasoning_dispatch: Optional[bool] = None, tokenizer_backend: Optional[str] = None, + tokenizer_fallback: Optional[bool] = None, ) -> None: """ Create EntrypointArgs. @@ -3133,7 +3134,8 @@ class EntrypointArgs: strip_anthropic_preamble: Optional Anthropic preamble stripping override enable_streaming_tool_dispatch: Optional streaming tool dispatch override enable_streaming_reasoning_dispatch: Optional streaming reasoning dispatch override - tokenizer_backend: Optional tokenizer backend override ("default" or "fastokens") + tokenizer_backend: Optional tokenizer backend override ("default", "fastokens", or "basetenkenizer") + tokenizer_fallback: Whether alternate tokenizer load failures fall back to HuggingFace """ ... diff --git a/lib/llm/src/local_model/runtime_config.rs b/lib/llm/src/local_model/runtime_config.rs index b2c459ef85a1..49cdeb99117c 100644 --- a/lib/llm/src/local_model/runtime_config.rs +++ b/lib/llm/src/local_model/runtime_config.rs @@ -70,6 +70,7 @@ pub enum StructuralTagScope { } pub const ENV_TOKENIZER_BACKEND: &str = "DYN_TOKENIZER"; +pub const ENV_TOKENIZER_FALLBACK: &str = "DYN_TOKENIZER_FALLBACK"; /// Worker-advertised support for Dynamo's vLLM-compatible /// `POST /inference/v1/generate` adapter. @@ -170,6 +171,11 @@ pub struct ModelRuntimeConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub tokenizer_backend: Option, + /// Frontend tokenizer fallback override. When unset, direct Rust callers can still use + /// `DYN_TOKENIZER_FALLBACK`; when set, this explicit value wins over process environment. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tokenizer_fallback_enabled: Option, + /// Whether structural tag guided decoding is enabled for tool calls. #[serde(default)] pub structural_tag_mode: StructuralTagMode, @@ -302,6 +308,7 @@ impl Default for ModelRuntimeConfig { tool_call_parser: None, reasoning_parser: None, tokenizer_backend: None, + tokenizer_fallback_enabled: None, structural_tag_mode: StructuralTagMode::Off, structural_tag_scope: StructuralTagScope::Auto, structural_tag_schema: StructuralTagSchemaMode::Auto, @@ -542,6 +549,16 @@ impl ModelRuntimeConfig { .unwrap_or_else(TokenizerBackend::from_env_or_default) } + pub fn is_tokenizer_fallback_enabled(&self) -> bool { + self.tokenizer_fallback_enabled.unwrap_or_else(|| { + std::env::var(ENV_TOKENIZER_FALLBACK) + .ok() + .as_deref() + .and_then(dynamo_runtime::config::parse_bool_opt) + .unwrap_or(true) + }) + } + /// Resolve the KV-state endpoint, preserving the serving endpoint as the compatibility /// default for workers that do not advertise an explicit mapping. pub fn effective_kv_state_endpoint(&self, serving_endpoint: &EndpointId) -> EndpointId { @@ -558,6 +575,11 @@ impl ModelRuntimeConfig { self } + pub fn set_tokenizer_fallback_enabled(&mut self, enabled: Option) -> &mut Self { + self.tokenizer_fallback_enabled = enabled; + self + } + /// Rebuild canonical topology taints derived from `topology_domains`. /// /// Existing caller-provided taints outside the reserved topology prefix are preserved; generated @@ -826,6 +848,53 @@ mod tests { } } + #[test] + #[serial_test::serial] + fn tokenizer_fallback_env_default_and_explicit_config_precedence() { + temp_env::with_vars([(ENV_TOKENIZER_FALLBACK, Some("false"))], || { + let config = ModelRuntimeConfig::default(); + assert!(!config.is_tokenizer_fallback_enabled()); + + let config = ModelRuntimeConfig { + tokenizer_fallback_enabled: Some(true), + ..Default::default() + }; + assert!(config.is_tokenizer_fallback_enabled()); + }); + + temp_env::with_vars([(ENV_TOKENIZER_FALLBACK, Some("true"))], || { + let config = ModelRuntimeConfig { + tokenizer_fallback_enabled: Some(false), + ..Default::default() + }; + assert!(!config.is_tokenizer_fallback_enabled()); + }); + + temp_env::with_vars_unset([ENV_TOKENIZER_FALLBACK], || { + let config = ModelRuntimeConfig::default(); + assert!(config.is_tokenizer_fallback_enabled()); + }); + } + + #[test] + fn tokenizer_fallback_roundtrips_through_serde_json() { + for enabled in [true, false] { + let config = ModelRuntimeConfig { + tokenizer_fallback_enabled: Some(enabled), + ..Default::default() + }; + let json = serde_json::to_string(&config).unwrap(); + assert!(json.contains(&format!("\"tokenizer_fallback_enabled\":{enabled}"))); + let parsed: ModelRuntimeConfig = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.tokenizer_fallback_enabled, Some(enabled)); + } + + let json = serde_json::to_string(&ModelRuntimeConfig::default()).unwrap(); + assert!(!json.contains("tokenizer_fallback_enabled")); + let legacy: ModelRuntimeConfig = serde_json::from_str("{}").unwrap(); + assert_eq!(legacy.tokenizer_fallback_enabled, None); + } + #[test] fn tokenizer_backend_string_values_are_strict() { for backend in [ diff --git a/lib/llm/src/model_card.rs b/lib/llm/src/model_card.rs index 5efe8c71537b..79be3fa83959 100644 --- a/lib/llm/src/model_card.rs +++ b/lib/llm/src/model_card.rs @@ -32,7 +32,6 @@ use crate::preprocessor::media::{MediaDecoder, MediaFetcher}; use crate::protocols::TokenIdType; const DEFAULT_TOKENIZER_CACHE_BYTES: usize = 64 * 1024 * 1024; -const ENV_TOKENIZER_FALLBACK: &str = "DYN_TOKENIZER_FALLBACK"; fn append_indexer_identity_checksum(bytes: &mut Vec, spec: &IndexerIdentitySpec) { bytes.extend_from_slice(b"dynamo/model-card/indexer-identity/v1"); @@ -63,12 +62,6 @@ fn tokenizer_cache_enabled(value: Option<&str>) -> bool { !matches!(value, Some("0")) } -fn is_tokenizer_fallback_enabled(value: Option<&str>) -> bool { - value - .and_then(dynamo_runtime::config::parse_bool_opt) - .unwrap_or(true) -} - fn tokenizer_cache_bytes(value: Option<&str>) -> usize { value .and_then(|value| value.parse::().ok()) @@ -1204,8 +1197,9 @@ impl ModelDeploymentCard { /// Tokenizer backend controls: /// - `runtime_config.tokenizer_backend` — select `default`, `fastokens`, or `basetenkenizer` /// - `DYN_TOKENIZER` — fallback backend for callers without explicit runtime config - /// - `DYN_TOKENIZER_FALLBACK=0` — fail if the selected alternate backend cannot load - /// instead of falling back to HuggingFace (fallback is enabled by default) + /// - `runtime_config.tokenizer_fallback_enabled` — control whether an alternate backend load + /// failure falls back to HuggingFace + /// - `DYN_TOKENIZER_FALLBACK=0` — fallback control for callers without explicit runtime config /// - `DYN_TOKENIZER_CACHE=0` — disable the L1 prefix cache that records tokenizations /// at special-token boundaries (enabled by default; any other value keeps it enabled) /// - `DYN_TOKENIZER_CACHE_BYTES=` — L1 cache byte budget (default 64 MiB) @@ -1216,8 +1210,7 @@ impl ModelDeploymentCard { /// fall back to the original hit-without-insert behavior. pub fn tokenizer(&self) -> anyhow::Result { let tokenizer_backend = self.runtime_config.effective_tokenizer_backend(); - let is_fallback_enabled = - is_tokenizer_fallback_enabled(std::env::var(ENV_TOKENIZER_FALLBACK).ok().as_deref()); + let is_fallback_enabled = self.runtime_config.is_tokenizer_fallback_enabled(); let cache_enabled = tokenizer_cache_enabled(std::env::var("DYN_TOKENIZER_CACHE").ok().as_deref()); diff --git a/lib/llm/tests/model_card.rs b/lib/llm/tests/model_card.rs index a21389ed3830..dc4cb7771442 100644 --- a/lib/llm/tests/model_card.rs +++ b/lib/llm/tests/model_card.rs @@ -219,7 +219,7 @@ fn test_alternate_tokenizer_load_failure_is_rejected_when_fallback_is_disabled() temp_env::with_vars( [ ("DYN_TOKENIZER_CACHE", Some("0")), - ("DYN_TOKENIZER_FALLBACK", Some("0")), + ("DYN_TOKENIZER_FALLBACK", Some("1")), ], || { for backend in [ @@ -232,6 +232,7 @@ fn test_alternate_tokenizer_load_failure_is_rejected_when_fallback_is_disabled() CheckedFile::from_disk(&tokenizer_path).unwrap(), )); mdc.runtime_config.tokenizer_backend = Some(backend); + mdc.runtime_config.tokenizer_fallback_enabled = Some(false); let error = mdc .tokenizer() From b7b4a6e40289424670f012074e3b6c21560aa310 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Tue, 11 Aug 2026 09:22:39 +0800 Subject: [PATCH 4/8] fix(frontend): propagate tokenizer fallback override Signed-off-by: xianlubird --- .../src/dynamo/common/configuration/utils.py | 12 ++-- .../common/tests/configuration/test_utils.py | 29 +++++++++- .../src/dynamo/frontend/frontend_args.py | 9 ++- .../modular-components/frontend/tokenizer.md | 6 +- .../components/frontend-configuration.mdx | 6 +- .../use-cases/fastokens-tokenizer/overview.md | 10 ++-- lib/llm/src/discovery/watcher.rs | 55 ++++++++++++++++++- lib/llm/src/entrypoint/input/common.rs | 3 + lib/llm/src/entrypoint/input/grpc.rs | 3 + lib/llm/src/entrypoint/input/http.rs | 3 + lib/llm/src/local_model/runtime_config.rs | 35 ++++++++---- lib/llm/src/model_card.rs | 2 +- 12 files changed, 134 insertions(+), 39 deletions(-) diff --git a/components/src/dynamo/common/configuration/utils.py b/components/src/dynamo/common/configuration/utils.py index fb97eaca0584..1893b5c34f5b 100644 --- a/components/src/dynamo/common/configuration/utils.py +++ b/components/src/dynamo/common/configuration/utils.py @@ -12,13 +12,13 @@ def parse_bool(value: str) -> bool: - """Parse a case-insensitive true or false CLI value.""" + """Parse Dynamo's truthy and falsy configuration values.""" normalized = value.strip().lower() - if normalized == "true": + if normalized in ("1", "true", "on", "yes"): return True - if normalized == "false": + if normalized in ("", "0", "false", "off", "no"): return False - raise argparse.ArgumentTypeError("expected 'true' or 'false'") + raise argparse.ArgumentTypeError("expected one of: true/false, 1/0, on/off, yes/no") def split_served_model_names(served_model_name: Any) -> list[str]: @@ -162,6 +162,7 @@ def add_negatable_bool_argument( help: str, dest: Optional[str] = None, obsolete_flag: Optional[str] = None, + env_value_type: Optional[Callable[..., bool]] = None, ) -> None: """ Add negatable boolean flag (--foo / --no-foo). @@ -174,6 +175,7 @@ def add_negatable_bool_argument( help: Help text dest: Optional destination name for the parsed value obsolete_flag: Optional obsolete/legacy flag (for help msg only, must start with '--') + env_value_type: Optional strict parser for the environment value """ add_argument( parser, @@ -183,7 +185,7 @@ def add_negatable_bool_argument( help=help, dest=dest, obsolete_flag=obsolete_flag, - arg_type=None, + arg_type=env_value_type, action=argparse.BooleanOptionalAction, ) diff --git a/components/src/dynamo/common/tests/configuration/test_utils.py b/components/src/dynamo/common/tests/configuration/test_utils.py index 4c505ad4de19..fc1c81a47883 100644 --- a/components/src/dynamo/common/tests/configuration/test_utils.py +++ b/components/src/dynamo/common/tests/configuration/test_utils.py @@ -23,15 +23,24 @@ @pytest.mark.parametrize( ("value", "expected"), - [("true", True), ("TRUE", True), ("false", False), ("FALSE", False)], + [ + ("true", True), + ("1", True), + ("on", True), + ("yes", True), + ("false", False), + ("0", False), + ("off", False), + ("no", False), + ], ) def test_parse_bool(value, expected): assert parse_bool(value) is expected def test_parse_bool_rejects_invalid_value(): - with pytest.raises(argparse.ArgumentTypeError, match="expected 'true' or 'false'"): - parse_bool("yes") + with pytest.raises(argparse.ArgumentTypeError, match="expected one of"): + parse_bool("flase") class TestEnvOrDefault: @@ -265,6 +274,20 @@ def test_uses_env_var_when_set(self, monkeypatch): args = parser.parse_args([]) assert args.enable_feature is False + def test_strict_env_parser_rejects_invalid_value(self, monkeypatch): + monkeypatch.setenv("TEST_ENABLE", "flase") + parser = argparse.ArgumentParser() + + with pytest.raises(argparse.ArgumentTypeError, match="expected one of"): + add_negatable_bool_argument( + parser, + flag_name="--enable-feature", + env_var="TEST_ENABLE", + default=True, + help="Enable feature", + env_value_type=parse_bool, + ) + def test_converts_hyphens_to_underscores(self): """Test that flag name with hyphens converts to underscores in dest.""" parser = argparse.ArgumentParser() diff --git a/components/src/dynamo/frontend/frontend_args.py b/components/src/dynamo/frontend/frontend_args.py index 65a31f2c89e1..de8cb2467caf 100644 --- a/components/src/dynamo/frontend/frontend_args.py +++ b/components/src/dynamo/frontend/frontend_args.py @@ -554,18 +554,17 @@ def add_arguments(self, parser) -> None: choices=["default", "fastokens", "basetenkenizer"], ) - add_argument( + add_negatable_bool_argument( g, flag_name="--tokenizer-fallback", env_var="DYN_TOKENIZER_FALLBACK", default=True, help=( "Fall back to HuggingFace when the selected fastokens or " - "basetenkenizer backend cannot load the model tokenizer. Set this " - "option to false to fail during startup instead." + "basetenkenizer backend cannot load the model tokenizer. Use " + "--no-tokenizer-fallback to fail during startup instead." ), - arg_type=parse_bool, - metavar="{true,false}", + env_value_type=parse_bool, ) add_negatable_bool_argument( diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md index 0f1c4084da30..0e22fa29ad16 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md @@ -42,7 +42,7 @@ Set the backend with a CLI flag or environment variable. The CLI flag takes prec | CLI Argument | Env Var | Valid values | Default | |---|---|---|---| | `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens`, `basetenkenizer` | `default` | -| `--tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | `true`, `false` | `true` | +| `--tokenizer-fallback` / `--no-tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | enabled, disabled | enabled | **Examples:** @@ -58,7 +58,7 @@ python -m dynamo.frontend python -m dynamo.frontend --tokenizer basetenkenizer # Require Baseten Tokenizer instead of falling back to HuggingFace -python -m dynamo.frontend --tokenizer basetenkenizer --tokenizer-fallback false +python -m dynamo.frontend --tokenizer basetenkenizer --no-tokenizer-fallback ``` ## Dynamo Frontend Behavior @@ -68,5 +68,5 @@ When a non-default backend is selected: 1. The frontend resolves `--tokenizer` / `DYN_TOKENIZER` and passes the selected backend to the Rust runtime. 2. `ModelDeploymentCard::tokenizer()` loads the HuggingFace tokenizer first for fallback behavior and L1 cache special-token metadata. 3. Dynamo constructs `FastTokenizer` for `fastokens` or `BasetenTokenizer` for `basetenkenizer` from the same `tokenizer.json` file. -4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--tokenizer-fallback false`, startup fails and reports the backend loading error instead. +4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--no-tokenizer-fallback`, startup fails and reports the backend loading error instead. 5. When the L1 prefix cache is enabled, Dynamo wraps the selected backend with the same special-token boundary metadata and cache metrics used by the default path. diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 54a48d8a366f..53ea766089ee 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -506,9 +506,9 @@ See the [Frontend Guide](../../developer-guide/knowledge-base/modular-components Environment variable: `DYN_TOKENIZER` - - Fall back to HuggingFace when `fastokens` or `basetenkenizer` cannot load the model tokenizer. Set - this option to `false` to fail during startup instead. + + Fall back to HuggingFace when `fastokens` or `basetenkenizer` cannot load the model tokenizer. Use + `--no-tokenizer-fallback` to fail during startup instead. Environment variable: `DYN_TOKENIZER_FALLBACK` diff --git a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md index 449c2aa540dc..6099467949c9 100644 --- a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md +++ b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md @@ -66,7 +66,7 @@ Stay on the default backend if: You can also set the same option with the `--tokenizer fastokens` frontend flag. The flag takes precedence when both settings are present. - To require `fastokens`, set `DYN_TOKENIZER_FALLBACK=false` or add `--tokenizer-fallback false`. The frontend then reports an error during model initialization when `fastokens` cannot load the tokenizer instead of using HuggingFace. + To require `fastokens`, set `DYN_TOKENIZER_FALLBACK=false` or add `--no-tokenizer-fallback`. The frontend then reports an error during model initialization when `fastokens` cannot load the tokenizer instead of using HuggingFace. To return to the default HuggingFace tokenizer backend, set `DYN_TOKENIZER=default` or omit it. Changing the value requires the Frontend pod to be replaced, which `kubectl apply` does automatically. @@ -117,7 +117,7 @@ Stay on the default backend if: - **Encoding**: `fastokens` converts prompt text to token IDs. - **Decoding**: HuggingFace `tokenizers` converts generated token IDs back to text. - Both backends load from the same `tokenizer.json`, so supported tokenizers should produce the same token IDs as the default HuggingFace path. If `fastokens` cannot load the tokenizer file, Dynamo logs a warning and falls back to the default backend. Set `--tokenizer-fallback false` to make this loading failure stop model initialization. + Both backends load from the same `tokenizer.json`, so supported tokenizers should produce the same token IDs as the default HuggingFace path. If `fastokens` cannot load the tokenizer file, Dynamo logs a warning and falls back to the default backend. Set `--no-tokenizer-fallback` to make this loading failure stop model initialization. ```mermaid flowchart TD @@ -125,8 +125,8 @@ Stay on the default backend if: Kind -->|BPE tokenizer.json| Load{"fastokens loads?"} Kind -->|.model / .tiktoken| Other["Use existing TikToken backend"] Load -->|Yes| Fast["Encode with fastokens
Decode with HuggingFace"] - Load -->|No, fallback true| Warn["Log warning"] --> Default["Use HuggingFace backend"] - Load -->|No, fallback false| Fail["Model initialization fails"] + Load -->|No, fallback enabled| Warn["Log warning"] --> Default["Use HuggingFace backend"] + Load -->|No, fallback disabled| Fail["Model initialization fails"] Fast --> Serve["Serve requests"] Default --> Serve Other --> Serve @@ -155,7 +155,7 @@ For any new model, validate on representative prompts before rolling out broadly
- The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare it against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` to avoid the warning. For a deployment that requires `fastokens`, use `--tokenizer-fallback false` so an unsupported tokenizer fails during model initialization. + The model's tokenizer file uses a feature that `fastokens` does not support, or it is not a BPE `tokenizer.json` path. Dynamo has already fallen back to HuggingFace and should keep serving traffic. Check the tokenizer format, compare it against the [tested models list](https://github.com/crusoecloud/fastokens#tested-models), and use `--tokenizer default` to avoid the warning. For a deployment that requires `fastokens`, use `--no-tokenizer-fallback` so an unsupported tokenizer fails during model initialization. diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index eae344ea97b9..4e85f5396f07 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -224,6 +224,8 @@ where local_model_path: Option, /// Frontend-level tokenizer backend override for discovered model cards. tokenizer_backend: Option, + /// Frontend-level tokenizer fallback override for discovered model cards. + tokenizer_fallback_enabled: Option, /// Worker capabilities accepted by the frontend's engine-native Generate routes. /// Keep raw pipelines out of default-off and backend-mismatched paths. generate_engine_capabilities: Vec<&'static str>, @@ -353,6 +355,7 @@ where metrics, local_model_path: None, tokenizer_backend: None, + tokenizer_fallback_enabled: None, generate_engine_capabilities: Vec::new(), worker_selector_factory, } @@ -370,6 +373,10 @@ where self.tokenizer_backend = tokenizer_backend; } + pub fn set_tokenizer_fallback_enabled(&mut self, enabled: Option) { + self.tokenizer_fallback_enabled = enabled; + } + pub(crate) fn set_generate_engine_capabilities(&mut self, capabilities: Vec<&'static str>) { self.generate_engine_capabilities = capabilities; } @@ -381,10 +388,13 @@ where .collect(); } - fn apply_tokenizer_backend_override(&self, card: &mut ModelDeploymentCard) { + fn apply_tokenizer_overrides(&self, card: &mut ModelDeploymentCard) { if let Some(tokenizer_backend) = self.tokenizer_backend { card.runtime_config.tokenizer_backend = Some(tokenizer_backend); } + if let Some(enabled) = self.tokenizer_fallback_enabled { + card.runtime_config.tokenizer_fallback_enabled = Some(enabled); + } } /// Wait until we have at least one chat completions model and return it's name. @@ -1011,7 +1021,7 @@ where let mut card = instance.deserialize_model::()?; normalize_legacy_prefill_topology(&mut card); - self.apply_tokenizer_backend_override(&mut card); + self.apply_tokenizer_overrides(&mut card); validate_card_shape(&card)?; anyhow::ensure!( mcid.model_suffix.is_some() == card.lora.is_some(), @@ -1327,6 +1337,47 @@ mod tests { )); } + #[tokio::test] + async fn tokenizer_fallback_override_applies_to_discovered_card() { + use dynamo_runtime::{Runtime, distributed::DistributedConfig}; + + let runtime = Runtime::from_current().unwrap(); + let drt = DistributedRuntime::new(runtime, DistributedConfig::process_local()) + .await + .unwrap(); + let mut watcher = ModelWatcher::new( + drt, + Arc::new(ModelManager::new()), + RouterConfig::default(), + 0, + None, + None, + None, + Arc::new(Metrics::new()), + ); + watcher.set_tokenizer_fallback_enabled(Some(false)); + + let mut card = ModelDeploymentCard::with_name_only("strict-tokenizer"); + card.runtime_config.tokenizer_fallback_enabled = Some(true); + let instance = DiscoveryInstance::Model { + namespace: "ns1".to_string(), + component: "workers".to_string(), + endpoint: "generate".to_string(), + instance_id: 1, + card_json: serde_json::to_value(card).unwrap(), + model_suffix: None, + }; + + let desired = watcher + .normalize(instance, &NamespaceFilter::Global) + .unwrap() + .unwrap(); + assert_eq!( + desired.card.runtime_config.tokenizer_fallback_enabled, + Some(false) + ); + } + #[tokio::test] async fn text_pooling_family_preserves_chat_engine() { use dynamo_runtime::{Runtime, distributed::DistributedConfig}; diff --git a/lib/llm/src/entrypoint/input/common.rs b/lib/llm/src/entrypoint/input/common.rs index 1413b7ccf632..0c0bfd4c2123 100644 --- a/lib/llm/src/entrypoint/input/common.rs +++ b/lib/llm/src/entrypoint/input/common.rs @@ -402,6 +402,9 @@ pub async fn prepare_engine( watcher.set_local_model_path(Some(local_model.path().to_path_buf())); } watcher.set_tokenizer_backend(local_model.runtime_config().tokenizer_backend); + watcher.set_tokenizer_fallback_enabled( + local_model.runtime_config().tokenizer_fallback_enabled, + ); let watch_obj = Arc::new(watcher); let discovery = distributed_runtime.discovery(); let discovery_stream = discovery diff --git a/lib/llm/src/entrypoint/input/grpc.rs b/lib/llm/src/entrypoint/input/grpc.rs index c1ee4b78e717..5da8753bb0b0 100644 --- a/lib/llm/src/entrypoint/input/grpc.rs +++ b/lib/llm/src/entrypoint/input/grpc.rs @@ -63,6 +63,7 @@ pub async fn run( local_model_path, model.metrics_prefix(), model.runtime_config().tokenizer_backend, + model.runtime_config().tokenizer_fallback_enabled, ) .await?; grpc_service @@ -132,6 +133,7 @@ async fn run_watcher( local_model_path: Option, metrics_prefix: Option, tokenizer_backend: Option, + tokenizer_fallback_enabled: Option, ) -> anyhow::Result<()> { // Start the LoRA allocation controller when LoRA serving is enabled (mirrors http.rs; // additionally gated on DYN_LORA_ALLOCATION_ENABLED inside start_lora_controller). Without @@ -154,6 +156,7 @@ async fn run_watcher( ); watch_obj.set_local_model_path(local_model_path); watch_obj.set_tokenizer_backend(tokenizer_backend); + watch_obj.set_tokenizer_fallback_enabled(tokenizer_fallback_enabled); tracing::debug!("Waiting for remote model"); let discovery = runtime.discovery(); let discovery_stream = discovery diff --git a/lib/llm/src/entrypoint/input/http.rs b/lib/llm/src/entrypoint/input/http.rs index aba76975ed87..e9d466fd0f32 100644 --- a/lib/llm/src/entrypoint/input/http.rs +++ b/lib/llm/src/entrypoint/input/http.rs @@ -215,6 +215,7 @@ where prefill_load_estimator.clone(), local_model_path, model.runtime_config().tokenizer_backend, + model.runtime_config().tokenizer_fallback_enabled, generate_engine_capabilities, worker_selector_factory.clone(), ) @@ -298,6 +299,7 @@ async fn run_watcher( prefill_load_estimator: Option>, local_model_path: Option, tokenizer_backend: Option, + tokenizer_fallback_enabled: Option, generate_engine_capabilities: Vec<&'static str>, worker_selector_factory: WorkerSelectorFactory, ) -> anyhow::Result<()> @@ -325,6 +327,7 @@ where ); watch_obj.set_local_model_path(local_model_path); watch_obj.set_tokenizer_backend(tokenizer_backend); + watch_obj.set_tokenizer_fallback_enabled(tokenizer_fallback_enabled); watch_obj.set_generate_engine_capabilities(generate_engine_capabilities); tracing::debug!("Waiting for remote model"); let discovery = runtime.discovery(); diff --git a/lib/llm/src/local_model/runtime_config.rs b/lib/llm/src/local_model/runtime_config.rs index 49cdeb99117c..d4e47e60c7e5 100644 --- a/lib/llm/src/local_model/runtime_config.rs +++ b/lib/llm/src/local_model/runtime_config.rs @@ -549,14 +549,19 @@ impl ModelRuntimeConfig { .unwrap_or_else(TokenizerBackend::from_env_or_default) } - pub fn is_tokenizer_fallback_enabled(&self) -> bool { - self.tokenizer_fallback_enabled.unwrap_or_else(|| { - std::env::var(ENV_TOKENIZER_FALLBACK) - .ok() - .as_deref() - .and_then(dynamo_runtime::config::parse_bool_opt) - .unwrap_or(true) - }) + pub fn is_tokenizer_fallback_enabled(&self) -> anyhow::Result { + if let Some(enabled) = self.tokenizer_fallback_enabled { + return Ok(enabled); + } + + match std::env::var(ENV_TOKENIZER_FALLBACK) { + Ok(value) => dynamo_runtime::config::parse_bool(&value) + .map_err(|error| anyhow::anyhow!("{ENV_TOKENIZER_FALLBACK}: {error}")), + Err(std::env::VarError::NotPresent) => Ok(true), + Err(std::env::VarError::NotUnicode(_)) => { + anyhow::bail!("{ENV_TOKENIZER_FALLBACK} must contain valid UTF-8") + } + } } /// Resolve the KV-state endpoint, preserving the serving endpoint as the compatibility @@ -853,13 +858,13 @@ mod tests { fn tokenizer_fallback_env_default_and_explicit_config_precedence() { temp_env::with_vars([(ENV_TOKENIZER_FALLBACK, Some("false"))], || { let config = ModelRuntimeConfig::default(); - assert!(!config.is_tokenizer_fallback_enabled()); + assert!(!config.is_tokenizer_fallback_enabled().unwrap()); let config = ModelRuntimeConfig { tokenizer_fallback_enabled: Some(true), ..Default::default() }; - assert!(config.is_tokenizer_fallback_enabled()); + assert!(config.is_tokenizer_fallback_enabled().unwrap()); }); temp_env::with_vars([(ENV_TOKENIZER_FALLBACK, Some("true"))], || { @@ -867,12 +872,18 @@ mod tests { tokenizer_fallback_enabled: Some(false), ..Default::default() }; - assert!(!config.is_tokenizer_fallback_enabled()); + assert!(!config.is_tokenizer_fallback_enabled().unwrap()); }); temp_env::with_vars_unset([ENV_TOKENIZER_FALLBACK], || { let config = ModelRuntimeConfig::default(); - assert!(config.is_tokenizer_fallback_enabled()); + assert!(config.is_tokenizer_fallback_enabled().unwrap()); + }); + + temp_env::with_vars([(ENV_TOKENIZER_FALLBACK, Some("flase"))], || { + let config = ModelRuntimeConfig::default(); + let error = config.is_tokenizer_fallback_enabled().unwrap_err(); + assert!(error.to_string().contains(ENV_TOKENIZER_FALLBACK)); }); } diff --git a/lib/llm/src/model_card.rs b/lib/llm/src/model_card.rs index 79be3fa83959..ac0cfcf2429a 100644 --- a/lib/llm/src/model_card.rs +++ b/lib/llm/src/model_card.rs @@ -1210,7 +1210,7 @@ impl ModelDeploymentCard { /// fall back to the original hit-without-insert behavior. pub fn tokenizer(&self) -> anyhow::Result { let tokenizer_backend = self.runtime_config.effective_tokenizer_backend(); - let is_fallback_enabled = self.runtime_config.is_tokenizer_fallback_enabled(); + let is_fallback_enabled = self.runtime_config.is_tokenizer_fallback_enabled()?; let cache_enabled = tokenizer_cache_enabled(std::env::var("DYN_TOKENIZER_CACHE").ok().as_deref()); From ad47f562f2d12a860968dde66105d8b6354cf8c8 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Tue, 11 Aug 2026 10:46:43 +0800 Subject: [PATCH 5/8] fix(frontend): separate env and CLI bool parsing Signed-off-by: xianlubird --- components/src/dynamo/common/configuration/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/components/src/dynamo/common/configuration/utils.py b/components/src/dynamo/common/configuration/utils.py index 1893b5c34f5b..79c07d33a28a 100644 --- a/components/src/dynamo/common/configuration/utils.py +++ b/components/src/dynamo/common/configuration/utils.py @@ -108,6 +108,7 @@ def add_argument( help: str, obsolete_flag: Optional[str] = None, arg_type: Optional[Union[type, Callable[..., Any]]] = str, + env_value_type: Optional[Union[type, Callable[..., Any]]] = None, **kwargs: Any, ) -> None: """ @@ -124,10 +125,11 @@ def add_argument( dest: Optional destination name (defaults to flag_name with dashes replaced by underscores) choices: Optional list of valid values for the argument. arg_type: Type for the argument (default: str) + env_value_type: Optional parser used only for the environment value """ arg_dest = _get_dest_name(flag_name, kwargs.get("dest")) - value_type_for_env: Optional[Union[type, Callable[..., Any]]] = None - if arg_type is not None and callable(arg_type): + value_type_for_env = env_value_type + if value_type_for_env is None and arg_type is not None and callable(arg_type): value_type_for_env = arg_type if isinstance(default, list) and (arg_type is None or arg_type is str): value_type_for_env = None @@ -185,7 +187,8 @@ def add_negatable_bool_argument( help=help, dest=dest, obsolete_flag=obsolete_flag, - arg_type=env_value_type, + arg_type=None, + env_value_type=env_value_type, action=argparse.BooleanOptionalAction, ) From fe28206aab01c9d5d22a4bcc60107c9154d32870 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Tue, 11 Aug 2026 14:15:50 +0800 Subject: [PATCH 6/8] docs(frontend): clarify tokenizer fallback failure Signed-off-by: xianlubird --- components/src/dynamo/frontend/frontend_args.py | 4 +++- .../knowledge-base/modular-components/frontend/tokenizer.md | 4 ++-- .../pages/reference/components/frontend-configuration.mdx | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/components/src/dynamo/frontend/frontend_args.py b/components/src/dynamo/frontend/frontend_args.py index de8cb2467caf..b6fe35e79948 100644 --- a/components/src/dynamo/frontend/frontend_args.py +++ b/components/src/dynamo/frontend/frontend_args.py @@ -562,7 +562,9 @@ def add_arguments(self, parser) -> None: help=( "Fall back to HuggingFace when the selected fastokens or " "basetenkenizer backend cannot load the model tokenizer. Use " - "--no-tokenizer-fallback to fail during startup instead." + "--no-tokenizer-fallback to fail model initialization instead. " + "In dynamic mode, discovery retries the load while the frontend " + "continues running." ), env_value_type=parse_bool, ) diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md index 0e22fa29ad16..760731cb6e06 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md @@ -31,7 +31,7 @@ Use this backend for supported `tokenizer.json` models when you need Baseten Tok #### Compatibility notes: - Works with standard BPE `tokenizer.json` files (Qwen, LLaMA, GPT-family, Mistral, DeepSeek, etc.). -- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace by default. Disable tokenizer fallback to reject incompatible configurations during startup. +- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace by default. Disable tokenizer fallback to reject incompatible tokenizers during model initialization. In dynamic mode, discovery retries the load while the frontend continues running. - Special tokens declared only in a sibling `tokenizer_config.json` are preserved for Baseten encoding and decoding and for Dynamo's L1 prefix-cache boundaries. - Has no effect on TikToken-format tokenizers (`.model` / `.tiktoken` files), which always use the TikToken backend. @@ -68,5 +68,5 @@ When a non-default backend is selected: 1. The frontend resolves `--tokenizer` / `DYN_TOKENIZER` and passes the selected backend to the Rust runtime. 2. `ModelDeploymentCard::tokenizer()` loads the HuggingFace tokenizer first for fallback behavior and L1 cache special-token metadata. 3. Dynamo constructs `FastTokenizer` for `fastokens` or `BasetenTokenizer` for `basetenkenizer` from the same `tokenizer.json` file. -4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--no-tokenizer-fallback`, startup fails and reports the backend loading error instead. +4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--no-tokenizer-fallback`, model initialization fails and reports the backend loading error instead. In dynamic mode, discovery retries the load while the frontend continues running. 5. When the L1 prefix cache is enabled, Dynamo wraps the selected backend with the same special-token boundary metadata and cache metrics used by the default path. diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index af4b6adc2191..0c0956cab109 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -508,7 +508,8 @@ See the [Frontend Guide](../../developer-guide/knowledge-base/modular-components Fall back to HuggingFace when `fastokens` or `basetenkenizer` cannot load the model tokenizer. Use - `--no-tokenizer-fallback` to fail during startup instead. + `--no-tokenizer-fallback` to fail model initialization instead. In dynamic mode, discovery retries + the load while the frontend continues running. Environment variable: `DYN_TOKENIZER_FALLBACK` From 9b1a49d08b2eaeb54cf469577b8125c81bd12ce0 Mon Sep 17 00:00:00 2001 From: xianlubird Date: Wed, 12 Aug 2026 11:24:01 +0800 Subject: [PATCH 7/8] docs(frontend): fix tokenizer fallback guidance Signed-off-by: xianlubird --- .../knowledge-base/modular-components/frontend/tokenizer.md | 4 ++-- docs/fern/pages/use-cases/fastokens-tokenizer/overview.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md index 760731cb6e06..d33aedd14586 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md @@ -31,7 +31,7 @@ Use this backend for supported `tokenizer.json` models when you need Baseten Tok #### Compatibility notes: - Works with standard BPE `tokenizer.json` files (Qwen, LLaMA, GPT-family, Mistral, DeepSeek, etc.). -- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace by default. Disable tokenizer fallback to reject incompatible tokenizers during model initialization. In dynamic mode, discovery retries the load while the frontend continues running. +- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace by default. Use `--no-tokenizer-fallback` to reject incompatible tokenizers during model initialization. - Special tokens declared only in a sibling `tokenizer_config.json` are preserved for Baseten encoding and decoding and for Dynamo's L1 prefix-cache boundaries. - Has no effect on TikToken-format tokenizers (`.model` / `.tiktoken` files), which always use the TikToken backend. @@ -42,7 +42,7 @@ Set the backend with a CLI flag or environment variable. The CLI flag takes prec | CLI Argument | Env Var | Valid values | Default | |---|---|---|---| | `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens`, `basetenkenizer` | `default` | -| `--tokenizer-fallback` / `--no-tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | enabled, disabled | enabled | +| `--tokenizer-fallback` / `--no-tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | `true`/`false`, `1`/`0`, `on`/`off`, `yes`/`no` | `true` | **Examples:** diff --git a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md index 6099467949c9..c3269ad34a9b 100644 --- a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md +++ b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md @@ -162,6 +162,10 @@ For any new model, validate on representative prompts before rolling out broadly Use `default`, `fastokens`, or `basetenkenizer` for `DYN_TOKENIZER`. Values such as `fast`, `hf`, or `huggingface` are benchmark-runner aliases, not valid values for the frontend environment variable. + + `DYN_TOKENIZER_FALLBACK` is parsed strictly. Use `true`/`false`, `1`/`0`, `on`/`off`, or `yes`/`no`. An invalid value fails model initialization instead of silently enabling the default fallback behavior. In dynamic mode, discovery retries the load while the frontend continues running. + + The `fastokens` and `basetenkenizer` settings have no effect for TikToken-format tokenizers. Dynamo uses the existing TikToken backend, so you should not expect either alternate-backend activation log or speedup. From 87d75bf345187da4b7b5d5f499e7da444442400f Mon Sep 17 00:00:00 2001 From: xianlubird Date: Thu, 13 Aug 2026 09:21:45 +0800 Subject: [PATCH 8/8] docs(frontend): deprecate automatic tokenizer fallback Signed-off-by: xianlubird --- .../src/dynamo/frontend/frontend_args.py | 6 ++-- .../modular-components/frontend/tokenizer.md | 4 +++ .../components/frontend-configuration.mdx | 5 +++ .../general/releases/deprecations.mdx | 6 +++- .../use-cases/fastokens-tokenizer/overview.md | 4 +++ lib/llm/src/local_model/runtime_config.rs | 31 +++++++++++++------ 6 files changed, 43 insertions(+), 13 deletions(-) diff --git a/components/src/dynamo/frontend/frontend_args.py b/components/src/dynamo/frontend/frontend_args.py index b6fe35e79948..95b76c5b7ab2 100644 --- a/components/src/dynamo/frontend/frontend_args.py +++ b/components/src/dynamo/frontend/frontend_args.py @@ -560,8 +560,10 @@ def add_arguments(self, parser) -> None: env_var="DYN_TOKENIZER_FALLBACK", default=True, help=( - "Fall back to HuggingFace when the selected fastokens or " - "basetenkenizer backend cannot load the model tokenizer. Use " + "Automatic fallback to HuggingFace is deprecated and will be " + "disabled by default in a future release. The current behavior " + "falls back when the selected fastokens or basetenkenizer backend " + "cannot load the model tokenizer. Use " "--no-tokenizer-fallback to fail model initialization instead. " "In dynamic mode, discovery retries the load while the frontend " "continues running." diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md index d33aedd14586..8db399d0a4ed 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md @@ -39,6 +39,10 @@ Use this backend for supported `tokenizer.json` models when you need Baseten Tok Set the backend with a CLI flag or environment variable. The CLI flag takes precedence. +> [!WARNING] +> Automatic tokenizer fallback is deprecated and will be disabled by default in a future release. +> Set `--no-tokenizer-fallback` or `DYN_TOKENIZER_FALLBACK=false` to adopt the future behavior now. + | CLI Argument | Env Var | Valid values | Default | |---|---|---|---| | `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens`, `basetenkenizer` | `default` | diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index e7f71d79e3e8..d2d6a00aece8 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -511,6 +511,11 @@ See the [Frontend Guide](../../developer-guide/knowledge-base/modular-components `--no-tokenizer-fallback` to fail model initialization instead. In dynamic mode, discovery retries the load while the frontend continues running. + + Automatic tokenizer fallback is deprecated and will be disabled by default in a future release. + Set `--no-tokenizer-fallback` to adopt the future behavior now. + + Environment variable: `DYN_TOKENIZER_FALLBACK` diff --git a/docs/fern/pages/reference/general/releases/deprecations.mdx b/docs/fern/pages/reference/general/releases/deprecations.mdx index 755a1ebcf748..eff92e4de9e8 100644 --- a/docs/fern/pages/reference/general/releases/deprecations.mdx +++ b/docs/fern/pages/reference/general/releases/deprecations.mdx @@ -21,7 +21,7 @@ Upgrading? Pick the version you run today to see the dependency migration and th ## v1.4.0 (upcoming) -5 entries — 3 behavioral changes, 2 removed. +6 entries — 3 behavioral changes, 1 deprecated, 2 removed. Behavioral **Audit subsystem migrated into request trace** (`"log_type":"audit"` records and the `"Audit sinks ready"` readiness marker -> records carrying `"schema":"dynamo.request.trace.v1"` and the `"Request trace sinks ready"` marker); payload logging is unified under the request-trace pipeline and the standalone audit module is gone ([#9390](https://github.com/ai-dynamo/dynamo/pull/9390), [#11180](https://github.com/ai-dynamo/dynamo/pull/11180)). @@ -35,6 +35,10 @@ Upgrading? Pick the version you run today to see the dependency migration and th > **Migrate:** Upgrade Codex to a version that sends `thread-id`, or send `X-Dynamo-Session-ID` from a custom client. Dynamo ignores `session-id` when `thread-id` is absent. +Deprecated **Automatic tokenizer fallback to HuggingFace is deprecated**; fallback remains enabled by default for compatibility when `fastokens` or `basetenkenizer` cannot load a model tokenizer, but it will be disabled by default in a future release ([#12923](https://github.com/ai-dynamo/dynamo/pull/12923)). + + > **Migrate:** Pass `--no-tokenizer-fallback` or set `DYN_TOKENIZER_FALLBACK=false` to fail model initialization when the selected tokenizer backend cannot load the model tokenizer. + Removed **Removed the deprecated multimodal worker flags** (`--multimodal-worker` / `--multimodal-encode-worker` -> `--enable-multimodal` with `--disaggregation-mode` / `--dedicated-mm-encoder`) across the vLLM, SGLang, and TensorRT-LLM backends; deprecated in v1.3.0 ([#11887](https://github.com/ai-dynamo/dynamo/pull/11887)). > **Migrate:** On vLLM and SGLang, replace the removed flags with `--enable-multimodal` plus the appropriate `--disaggregation-mode` / `--dedicated-mm-encoder` combination. On TensorRT-LLM, use `--modality multimodal`, which remains the supported path. diff --git a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md index c3269ad34a9b..56a1d664e279 100644 --- a/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md +++ b/docs/fern/pages/use-cases/fastokens-tokenizer/overview.md @@ -39,6 +39,10 @@ Stay on the default backend if: ## Enable and Validate FastTokens +> [!WARNING] +> Automatic tokenizer fallback is deprecated and will be disabled by default in a future release. +> Set `--no-tokenizer-fallback` or `DYN_TOKENIZER_FALLBACK=false` to adopt the future behavior now. + `fastokens` is a **frontend** setting. Enable it on the Frontend component with the `DYN_TOKENIZER` environment variable: diff --git a/lib/llm/src/local_model/runtime_config.rs b/lib/llm/src/local_model/runtime_config.rs index e1757212686d..073f57229fa8 100644 --- a/lib/llm/src/local_model/runtime_config.rs +++ b/lib/llm/src/local_model/runtime_config.rs @@ -581,18 +581,29 @@ impl ModelRuntimeConfig { } pub fn is_tokenizer_fallback_enabled(&self) -> anyhow::Result { - if let Some(enabled) = self.tokenizer_fallback_enabled { - return Ok(enabled); - } - - match std::env::var(ENV_TOKENIZER_FALLBACK) { - Ok(value) => dynamo_runtime::config::parse_bool(&value) - .map_err(|error| anyhow::anyhow!("{ENV_TOKENIZER_FALLBACK}: {error}")), - Err(std::env::VarError::NotPresent) => Ok(true), - Err(std::env::VarError::NotUnicode(_)) => { - anyhow::bail!("{ENV_TOKENIZER_FALLBACK} must contain valid UTF-8") + let enabled = if let Some(enabled) = self.tokenizer_fallback_enabled { + enabled + } else { + match std::env::var(ENV_TOKENIZER_FALLBACK) { + Ok(value) => dynamo_runtime::config::parse_bool(&value) + .map_err(|error| anyhow::anyhow!("{ENV_TOKENIZER_FALLBACK}: {error}"))?, + Err(std::env::VarError::NotPresent) => true, + Err(std::env::VarError::NotUnicode(_)) => { + anyhow::bail!("{ENV_TOKENIZER_FALLBACK} must contain valid UTF-8") + } } + }; + + if enabled && self.effective_tokenizer_backend() != TokenizerBackend::Default { + static TOKENIZER_FALLBACK_DEPRECATION_WARNED: std::sync::Once = std::sync::Once::new(); + TOKENIZER_FALLBACK_DEPRECATION_WARNED.call_once(|| { + tracing::warn!( + "Automatic tokenizer fallback is deprecated and will be disabled by default in a future release. Set tokenizer fallback to false (`--no-tokenizer-fallback` in the frontend) to adopt the future behavior now." + ); + }); } + + Ok(enabled) } /// Resolve the KV-state endpoint, preserving the serving endpoint as the compatibility