From 057a38d30864760bb53bea2b02b6a8a3e839b7d0 Mon Sep 17 00:00:00 2001 From: Joel Gustafson Date: Wed, 16 Sep 2026 15:37:09 -0400 Subject: [PATCH] fix(sglang): publish SGLang's full Mooncake key prefix for the HiCache shared cache `--shared-cache-type hicache` makes the frontend re-derive Mooncake object keys from the worker's `sglang_hicache_mooncake` runtime config and look them up in the master's KV event index. SGLang's Mooncake backend tags every object key and group id with `config_prefix` -- the configured `extra_backend_tag`, then the served model name with `/` replaced by `-`, joined by `_` -- but register.py published only the bare tag. `shared_cache.rs` (`maybe_prefix_key`, `sglang_group_id`) therefore derived `__k` while the master announced `___k`. The lookups never matched, so `router_shared_cache_hit_rate` sat at zero and worker selection quietly fell back to device-overlap-only scoring. No `extra_backend_tag` value fixes this from the outside, because SGLang appends the model name after the tag. SGLang added the model-name component in sgl-project/sglang#31920, first released in v0.5.17. #13343 moved Dynamo from 0.5.16 to 0.5.17 on 2026-08-21 without adapting the derivation, and main has skewed since. Publish the prefix SGLang actually uses, replicating its construction exactly -- including a present-but-empty tag contributing a leading separator and a non-string tag being stringified. `has_same_layout` already compares this field, so a fleet with mismatched prefixes is still detected. Signed-off-by: Joel Gustafson Co-Authored-By: Claude Opus 5 --- components/src/dynamo/sglang/register.py | 39 ++++++++-- .../sglang/tests/test_runtime_metadata.py | 75 +++++++++++++++++++ 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index 6fb2f1afb4ee..a51fa2005b8d 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -280,6 +280,38 @@ def _parse_hicache_storage_extra_config( return {} +def _mooncake_config_prefix( + server_args: ServerArgs, extra_config: dict[str, Any] +) -> Optional[str]: + """Derive the ``config_prefix`` SGLang tags Mooncake object keys and group ids with. + + Mirrors ``MooncakeStore.__init__`` (``mooncake_store.py``): the + ``extra_backend_tag`` from the HiCache extra config, then the served model + name with ``/`` replaced by ``-``, joined by ``_``. The router prepends the + result verbatim when it re-derives keys (``shared_cache.rs`` + ``maybe_prefix_key`` / ``sglang_group_id``), so it must match SGLang's + exactly. + """ + parts: list[str] = [] + + # SGLang tests ``is not None`` and applies ``str()``, so empty and + # non-string tags contribute a part too. + tag = extra_config.get("extra_backend_tag") + if tag is not None: + parts.append(str(tag)) + + # SGLang passes the resolved served model name as + # ``HiCacheStorageConfig.model_name`` (``hiradix_cache.py``); ``args.py`` + # resolves it to a single name, falling back to ``model_path``. + model_name = ( + getattr(server_args, "served_model_name", None) or server_args.model_path + ) + if model_name: + parts.append("-".join(model_name.split("/"))) + + return "_".join(parts) if parts else None + + def _get_mooncake_runtime_data(server_args: ServerArgs) -> Optional[dict[str, Any]]: if getattr(server_args, "hicache_storage_backend", None) != "mooncake": return None @@ -321,10 +353,6 @@ def _get_mooncake_runtime_data(server_args: ServerArgs) -> Optional[dict[str, An and tp_lcm_size % tp_size == 0 ) - extra_backend_tag = extra_config.get("extra_backend_tag") - if not isinstance(extra_backend_tag, str) or not extra_backend_tag: - extra_backend_tag = None - return { "backend": "mooncake", "page_size": int(getattr(server_args, "page_size", 1) or 1), @@ -334,7 +362,8 @@ def _get_mooncake_runtime_data(server_args: ServerArgs) -> Optional[dict[str, An "is_eagle": is_eagle, "tp_lcm_size": tp_lcm_size, "should_split_heads": should_split_heads, - "extra_backend_tag": extra_backend_tag, + # Holds the full ``config_prefix``, which the router prepends verbatim. + "extra_backend_tag": _mooncake_config_prefix(server_args, extra_config), "kv_events_endpoint": os.getenv("DYN_MOONCAKE_KV_EVENTS_ENDPOINT") or None, } diff --git a/components/src/dynamo/sglang/tests/test_runtime_metadata.py b/components/src/dynamo/sglang/tests/test_runtime_metadata.py index 76ff8c13ee6f..a384ae3f5377 100644 --- a/components/src/dynamo/sglang/tests/test_runtime_metadata.py +++ b/components/src/dynamo/sglang/tests/test_runtime_metadata.py @@ -300,3 +300,78 @@ def fail_hicache_publish(self, key, value): assert ( "Failed to attach native offloading capacity from SGLang HiCache" in caplog.text ) + + +@pytest.mark.parametrize( + "extra_config, served_model_name, model_path, expected", + [ + ( + {}, + "meta-llama/Llama-3.1-8B", + "meta-llama/Llama-3.1-8B", + "meta-llama-Llama-3.1-8B", + ), + ( + {"extra_backend_tag": "t6-rev1p4"}, + "meta-llama/Llama-3.1-8B", + "meta-llama/Llama-3.1-8B", + "t6-rev1p4_meta-llama-Llama-3.1-8B", + ), + # Unset served name falls back to model_path, as args.py does. + ( + {"extra_backend_tag": "t6-rev1p4"}, + None, + "/models/local/llama", + "t6-rev1p4_-models-local-llama", + ), + # An empty tag is present, so it contributes a leading separator. + ( + {"extra_backend_tag": ""}, + "Qwen/Qwen3-32B", + "Qwen/Qwen3-32B", + "_Qwen-Qwen3-32B", + ), + # A non-string tag is stringified. + ( + {"extra_backend_tag": 123}, + "Qwen/Qwen3-32B", + "Qwen/Qwen3-32B", + "123_Qwen-Qwen3-32B", + ), + # An explicit null tag contributes nothing. + ({"extra_backend_tag": None}, "a/b/c", "a/b/c", "a-b-c"), + ], +) +def test_mooncake_config_prefix_matches_sglang( + extra_config, served_model_name, model_path, expected +): + from dynamo.sglang.register import _mooncake_config_prefix + + server_args = SimpleNamespace( + served_model_name=served_model_name, model_path=model_path + ) + + assert _mooncake_config_prefix(server_args, extra_config) == expected + + +def test_mooncake_runtime_data_publishes_full_key_prefix(): + from dynamo.sglang.register import _get_mooncake_runtime_data + + server_args = SimpleNamespace( + hicache_storage_backend="mooncake", + hicache_storage_backend_extra_config=json.dumps( + {"extra_backend_tag": "t6-rev1p4"} + ), + hicache_mem_layout="layer_first", + served_model_name="meta-llama/Llama-3.1-8B", + model_path="meta-llama/Llama-3.1-8B", + page_size=64, + tp_size=1, + pp_size=1, + speculative_algorithm="NONE", + ) + + runtime_data = _get_mooncake_runtime_data(server_args) + + assert runtime_data is not None + assert runtime_data["extra_backend_tag"] == "t6-rev1p4_meta-llama-Llama-3.1-8B"