Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 34 additions & 5 deletions components/src/dynamo/sglang/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,38 @@ def _parse_hicache_storage_extra_config(
return {}


def _mooncake_config_prefix(
server_args: ServerArgs, extra_config: dict[str, Any]
) -> Optional[str]:
"""Derive the ``config_prefix`` SGLang tags Mooncake object keys and group ids with.

Mirrors ``MooncakeStore.__init__`` (``mooncake_store.py``): the
``extra_backend_tag`` from the HiCache extra config, then the served model
name with ``/`` replaced by ``-``, joined by ``_``. The router prepends the
result verbatim when it re-derives keys (``shared_cache.rs``
``maybe_prefix_key`` / ``sglang_group_id``), so it must match SGLang's
exactly.
"""
parts: list[str] = []

# SGLang tests ``is not None`` and applies ``str()``, so empty and
# non-string tags contribute a part too.
tag = extra_config.get("extra_backend_tag")
if tag is not None:
parts.append(str(tag))

# SGLang passes the resolved served model name as
# ``HiCacheStorageConfig.model_name`` (``hiradix_cache.py``); ``args.py``
# resolves it to a single name, falling back to ``model_path``.
model_name = (
getattr(server_args, "served_model_name", None) or server_args.model_path
)
if model_name:
parts.append("-".join(model_name.split("/")))

return "_".join(parts) if parts else None


def _get_mooncake_runtime_data(server_args: ServerArgs) -> Optional[dict[str, Any]]:
if getattr(server_args, "hicache_storage_backend", None) != "mooncake":
return None
Expand Down Expand Up @@ -321,10 +353,6 @@ def _get_mooncake_runtime_data(server_args: ServerArgs) -> Optional[dict[str, An
and tp_lcm_size % tp_size == 0
)

extra_backend_tag = extra_config.get("extra_backend_tag")
if not isinstance(extra_backend_tag, str) or not extra_backend_tag:
extra_backend_tag = None

return {
"backend": "mooncake",
"page_size": int(getattr(server_args, "page_size", 1) or 1),
Expand All @@ -334,7 +362,8 @@ def _get_mooncake_runtime_data(server_args: ServerArgs) -> Optional[dict[str, An
"is_eagle": is_eagle,
"tp_lcm_size": tp_lcm_size,
"should_split_heads": should_split_heads,
"extra_backend_tag": extra_backend_tag,
# Holds the full ``config_prefix``, which the router prepends verbatim.
"extra_backend_tag": _mooncake_config_prefix(server_args, extra_config),
"kv_events_endpoint": os.getenv("DYN_MOONCAKE_KV_EVENTS_ENDPOINT") or None,
}

Expand Down
75 changes: 75 additions & 0 deletions components/src/dynamo/sglang/tests/test_runtime_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,3 +300,78 @@ def fail_hicache_publish(self, key, value):
assert (
"Failed to attach native offloading capacity from SGLang HiCache" in caplog.text
)


@pytest.mark.parametrize(
"extra_config, served_model_name, model_path, expected",
[
(
{},
"meta-llama/Llama-3.1-8B",
"meta-llama/Llama-3.1-8B",
"meta-llama-Llama-3.1-8B",
),
(
{"extra_backend_tag": "t6-rev1p4"},
"meta-llama/Llama-3.1-8B",
"meta-llama/Llama-3.1-8B",
"t6-rev1p4_meta-llama-Llama-3.1-8B",
),
# Unset served name falls back to model_path, as args.py does.
(
{"extra_backend_tag": "t6-rev1p4"},
None,
"/models/local/llama",
"t6-rev1p4_-models-local-llama",
),
# An empty tag is present, so it contributes a leading separator.
(
{"extra_backend_tag": ""},
"Qwen/Qwen3-32B",
"Qwen/Qwen3-32B",
"_Qwen-Qwen3-32B",
),
# A non-string tag is stringified.
(
{"extra_backend_tag": 123},
"Qwen/Qwen3-32B",
"Qwen/Qwen3-32B",
"123_Qwen-Qwen3-32B",
),
# An explicit null tag contributes nothing.
({"extra_backend_tag": None}, "a/b/c", "a/b/c", "a-b-c"),
],
)
def test_mooncake_config_prefix_matches_sglang(
extra_config, served_model_name, model_path, expected
):
from dynamo.sglang.register import _mooncake_config_prefix

server_args = SimpleNamespace(
served_model_name=served_model_name, model_path=model_path
)

assert _mooncake_config_prefix(server_args, extra_config) == expected


def test_mooncake_runtime_data_publishes_full_key_prefix():
from dynamo.sglang.register import _get_mooncake_runtime_data

server_args = SimpleNamespace(
hicache_storage_backend="mooncake",
hicache_storage_backend_extra_config=json.dumps(
{"extra_backend_tag": "t6-rev1p4"}
),
hicache_mem_layout="layer_first",
served_model_name="meta-llama/Llama-3.1-8B",
model_path="meta-llama/Llama-3.1-8B",
page_size=64,
tp_size=1,
pp_size=1,
speculative_algorithm="NONE",
)

runtime_data = _get_mooncake_runtime_data(server_args)

assert runtime_data is not None
assert runtime_data["extra_backend_tag"] == "t6-rev1p4_meta-llama-Llama-3.1-8B"
Loading