Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions components/src/dynamo/vllm/tests/test_vllm_worker_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -1857,3 +1857,51 @@ async def test_passes_the_disabling_default_through_unchanged(self) -> None:
handler_cls = await self._create_encode_worker(0.0)

assert handler_cls.call_args.kwargs["embedding_cache_capacity_gb"] == 0.0


class TestShadowStandbyEntry:
"""Cover the real ``_maybe_wait_for_failover_lock``.

Every other test in this file replaces it with an ``AsyncMock``, so the
bound on the pause would otherwise never be executed.
"""

@staticmethod
def _handler_that_never_pauses():
class _NeverPauses:
async def pause(self, *_args: object) -> bool:
await asyncio.sleep(3600)
return True

return SimpleNamespace(_pause_controller=_NeverPauses())

@pytest.mark.timeout(30)
async def test_pause_that_never_returns_raises_naming_engine_and_stage(
self, monkeypatch
):
monkeypatch.setenv("ENGINE_ID", "2")
monkeypatch.setenv("DYN_GMS_SHADOW_PAUSE_TIMEOUT_SECONDS", "0.2")
elect_and_wake = AsyncMock()
monkeypatch.setattr("dynamo.vllm.worker_factory.elect_and_wake", elect_and_wake)
runtime = Mock()

with pytest.raises(RuntimeError) as excinfo:
await _make_factory()._maybe_wait_for_failover_lock(
self._handler_that_never_pauses(),
runtime,
_make_config(gms_shadow_mode=True),
)

message = str(excinfo.value)
assert "engine-2" in message
assert "did not finish pausing for standby" in message
# The stalled stage has to be named, or a cluster log cannot tell a
# weights-admission stall from any other unfinished startup step.
assert "sleep" in message
# The bound has to print as configured: rounding a sub-second bound to
# "0s" would print the value that disables the bound.
assert "0.2s" in message
# A shadow that never paused holds no lock and cannot take a
# promotion, so it must not report itself Ready.
runtime.set_health_status.assert_not_called()
elect_and_wake.assert_not_awaited()
33 changes: 32 additions & 1 deletion components/src/dynamo/vllm/worker_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@
# serving nor error propagation may hang on it.
WORKER_GC_STOP_TIMEOUT_SECONDS = 30.0

# Bound for a shadow engine's pause before standby; generous because it spans a
# real engine sleep. Override with the env var below; "0" waits unbounded.
SHADOW_PAUSE_TIMEOUT_SECONDS = 900.0
ENV_SHADOW_PAUSE_TIMEOUT_SECONDS = "DYN_GMS_SHADOW_PAUSE_TIMEOUT_SECONDS"

# (engine_client, vllm_config, default_sampling_params, cleanup_resource, component_gauges)
# component_gauges is None on the embedding-worker path: pooling engines
# have no KV cache / scheduler gauges, so setup_vllm_engine() skips the
Expand Down Expand Up @@ -1155,7 +1160,33 @@ async def _maybe_wait_for_failover_lock(
if config.gms_shadow_mode is not True:
return False

await handler._pause_controller.pause(1)
engine_id = os.environ.get("ENGINE_ID", "0")
raw_timeout = os.environ.get(ENV_SHADOW_PAUSE_TIMEOUT_SECONDS)
timeout_s = float(raw_timeout) if raw_timeout else SHADOW_PAUSE_TIMEOUT_SECONDS
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
# %g, not a fixed number of decimals: rounding a sub-second bound down
# to "0" would print the one value that means "wait forever".
logger.info(
"[Shadow] engine-%s pausing before standby (bound %gs)",
engine_id,
timeout_s,
)
try:
if timeout_s > 0:
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
await asyncio.wait_for(
handler._pause_controller.pause(1), timeout=timeout_s
)
else:
await handler._pause_controller.pause(1)
except asyncio.TimeoutError as exc:
# Health is deliberately not published here: a shadow that never
# paused holds no lock and cannot take a promotion.
raise RuntimeError(
f"[Shadow] engine-{engine_id} did not finish pausing for standby "
f"within {timeout_s:g}s; the engine pause (pause_generation, then "
"the engine sleep, which asks GMS to unmap and abort its weights "
"and kv_cache mappings) never completed, so this engine cannot "
"enter standby"
) from exc
lock = await elect_and_wake(
handler._pause_controller,
runtime,
Expand Down
7 changes: 6 additions & 1 deletion lib/gpu_memory_service/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -610,4 +610,9 @@ To bound how long a restored engine waits for the published weights layout befor
--model-loader-extra-config '{"gms_ro_connect_timeout_ms": 300000}'
```

The default is `null`, which waits indefinitely. Set an integer value to fail after that many milliseconds.
The value bounds both weight admissions an engine performs: the first one, during device init, and the remap after a shadow is promoted. An expiry during the remap is fatal and exits the worker.

An explicit value always wins. When the key is absent, the default depends on the engine:

- a primary engine (`ENGINE_ID=0`) and any engine outside shadow mode default to `null`, which waits indefinitely;
- a failover shadow (`ENGINE_ID` other than `0`, under `--gms-shadow-mode`) defaults to `900000` ms. GMS prefers writers, so with two or more shadows a reader can be refused admission behind a queued writer, and an untimed refusal parks the worker with no log and no traceback.
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
20 changes: 20 additions & 0 deletions lib/gpu_memory_service/integrations/vllm/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ def configure_gms_worker_logging() -> None:
gms_root.propagate = False


# Default weight-admission deadline for a shadow engine, in ms. Generous: it
# sits behind a real writer's load, and must stay under the 2h startup probe.
SHADOW_RO_CONNECT_TIMEOUT_MS = 900_000


def configure_gms_lock_mode(engine_args) -> None:
"""Set gms_read_only in model_loader_extra_config based on ENGINE_ID.

Expand All @@ -52,6 +57,13 @@ def configure_gms_lock_mode(engine_args) -> None:
deadlock: if multiple engines tried to acquire RW locks across TP ranks
simultaneously, they could block each other indefinitely.

A non-primary engine also gets a default gms_ro_connect_timeout_ms when the
operator has not chosen one. GMS prefers writers, so with two or more
shadows a reader can be refused admission behind a queued writer; an
untimed refusal parks the worker silently. A single-engine deployment is
left alone and keeps waiting indefinitely, and an explicit value always
wins.

Raises if user-specified gms_read_only conflicts with ENGINE_ID.
"""
engine_id = os.environ.get("ENGINE_ID", "0")
Expand All @@ -72,6 +84,14 @@ def configure_gms_lock_mode(engine_args) -> None:
f"but gms_read_only=False was explicitly set."
)
extra["gms_read_only"] = True
if extra.get("gms_ro_connect_timeout_ms") is None:
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
extra["gms_ro_connect_timeout_ms"] = SHADOW_RO_CONNECT_TIMEOUT_MS
logger.info(
"[GMS] ENGINE_ID=%s is a shadow; defaulting "
"gms_ro_connect_timeout_ms to %d ms",
engine_id,
SHADOW_RO_CONNECT_TIMEOUT_MS,
)

engine_args.model_loader_extra_config = extra

Expand Down
3 changes: 3 additions & 0 deletions lib/gpu_memory_service/integrations/vllm/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,11 +175,14 @@ def init_device(self) -> None:
)
mode = get_gms_lock_mode(extra)
self.gms_ro_connect_timeout_ms = get_gms_ro_connect_timeout_ms(extra)
# Must honour the same deadline as the wake_up remap below: without it,
# a shadow refused admission blocks with no log and no traceback.
get_or_create_gms_client_memory_manager(
get_socket_path(device, "weights"),
device,
mode=mode,
tag="weights",
timeout_ms=self.gms_ro_connect_timeout_ms,
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
)
# Parent will set device again (harmless) and do memory checks
super().init_device()
Expand Down
71 changes: 71 additions & 0 deletions lib/gpu_memory_service/tests/test_runtime_flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@
_GPU_MEMORY_RECOVERY_TIMEOUT_SECONDS = 30.0
_FAST_POLL_INTERVAL_SECONDS = 0.01
_SLOW_POLL_INTERVAL_SECONDS = 0.1
# Stand in for the production shadow default, which is minutes long.
_SHADOW_CONNECT_TIMEOUT_MS = 200
_QUEUED_WRITER_TIMEOUT_MS = 2000


def _gpu_memory_free_bytes(device: int = 0) -> int:
Expand Down Expand Up @@ -494,6 +497,74 @@ def open_writer() -> None:
waiting_writer.close()


@pytest.mark.timeout(_SOCKET_TEST_TIMEOUT_SECONDS)
def test_third_shadow_client_times_out_behind_queued_writer(running_gms, monkeypatch):
"""Three clients on one weights socket: the shape a numShadows=2 pool makes.

A shadow engine resolves its admission deadline through
``configure_gms_lock_mode``. Without the shadow default that deadline is
``None``, and the third client's handshake never returns.
"""
from types import SimpleNamespace

from gpu_memory_service.integrations.common.utils import (
get_gms_ro_connect_timeout_ms,
)
from gpu_memory_service.integrations.vllm import utils as vllm_gms_utils

server, socket_path = running_gms

# engine-0, the primary, publishes the weight layout.
primary = _GMSClientSession(socket_path, RequestedLockType.RW, None)
primary.commit()

# engine-1, the first shadow, attaches read-only and stays attached.
first_shadow = _GMSClientSession(socket_path, RequestedLockType.RO, None)

queued_writer: dict[str, object] = {}

def open_writer() -> None:
try:
queued_writer["session"] = _GMSClientSession(
socket_path,
RequestedLockType.RW,
_QUEUED_WRITER_TIMEOUT_MS,
)
except Exception as exc:
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
queued_writer["error"] = exc

thread = threading.Thread(target=open_writer)
thread.start()
try:
# A writer is now queued behind the attached reader, so no further
# reader can be admitted.
_wait_for_waiting_writers(server, 1)

monkeypatch.setattr(
vllm_gms_utils,
"SHADOW_RO_CONNECT_TIMEOUT_MS",
_SHADOW_CONNECT_TIMEOUT_MS,
)
monkeypatch.setenv("ENGINE_ID", "2")
engine_args = SimpleNamespace(model_loader_extra_config=None)
vllm_gms_utils.configure_gms_lock_mode(engine_args)
timeout_ms = get_gms_ro_connect_timeout_ms(
engine_args.model_loader_extra_config
)
assert timeout_ms == _SHADOW_CONNECT_TIMEOUT_MS

# engine-2, the second shadow, is refused and must say so.
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
with pytest.raises(TimeoutError, match="Timeout waiting for lock"):
_GMSClientSession(socket_path, RequestedLockType.RO, timeout_ms)
finally:
first_shadow.close()
thread.join(timeout=_BLOCKED_WRITER_JOIN_TIMEOUT_SECONDS)
granted = queued_writer.get("session")
if isinstance(granted, _GMSClientSession):
granted.close()
primary.close()


@pytest.mark.timeout(_SOCKET_TEST_TIMEOUT_SECONDS)
def test_rw_or_ro_times_out_while_writer_waits_behind_reader(running_gms):
server, socket_path = running_gms
Expand Down
112 changes: 112 additions & 0 deletions lib/gpu_memory_service/tests/test_vllm_worker.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Unit tests for GMSWorker's early weights connection."""

from __future__ import annotations

from types import SimpleNamespace

import pytest
from _deps import HAS_GMS, HAS_TORCH

if not HAS_GMS:
pytest.skip(
"gpu_memory_service package is not available in this test image",
allow_module_level=True,
)

if not HAS_TORCH:
pytest.skip("torch is required", allow_module_level=True)

pytest.importorskip("vllm", reason="vLLM is required")

from gpu_memory_service.common.locks import RequestedLockType # noqa: E402
from gpu_memory_service.common.vmm import VMMDeviceType # noqa: E402
from gpu_memory_service.integrations.vllm import worker as gms_worker # noqa: E402

pytestmark = [
pytest.mark.pre_merge,
pytest.mark.unit,
pytest.mark.vllm,
pytest.mark.core,
pytest.mark.gpu_0,
]

_RO_CONNECT_TIMEOUT_MS = 4200


@pytest.fixture
def init_device_calls(monkeypatch):
"""Run ``GMSWorker.init_device`` with every device collaborator replaced.

Returns the keyword arguments the worker handed to the client-manager
factory. Patching that factory in the module under test mirrors how the
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
SGLang memory-saver tests cover the same plumbing.
"""
calls: list[dict[str, object]] = []

def fake_factory(socket_path, device, mode=None, *, tag=None, timeout_ms=None):
calls.append(
{
"socket_path": socket_path,
"device": device,
"mode": mode,
"tag": tag,
"timeout_ms": timeout_ms,
}
)
return SimpleNamespace()

monkeypatch.setattr(
gms_worker, "get_or_create_gms_client_memory_manager", fake_factory
)
monkeypatch.setattr(gms_worker, "get_vmm_device_type", lambda: VMMDeviceType.CUDA)
monkeypatch.setattr(
"vllm.platforms.current_platform",
SimpleNamespace(set_device=lambda _device: None),
)
# The parent's device init allocates on a real GPU; this test is only about
# the arguments GMS resolves before it runs.
monkeypatch.setattr(gms_worker._BaseWorker, "init_device", lambda self: None)

def run(extra_config: dict) -> list[dict[str, object]]:
worker = gms_worker.GMSWorker.__new__(gms_worker.GMSWorker)
worker.local_rank = 0
worker.parallel_config = SimpleNamespace(
distributed_executor_backend="mp",
data_parallel_backend="mp",
nnodes_within_dp=1,
data_parallel_rank_local=0,
data_parallel_index=0,
pipeline_parallel_size=1,
tensor_parallel_size=1,
)
worker.vllm_config = SimpleNamespace(
load_config=SimpleNamespace(model_loader_extra_config=extra_config)
)
worker.init_device()
return calls

return run


def test_init_device_forwards_the_resolved_ro_connect_timeout(init_device_calls):
"""A shadow's weights admission must carry the deadline, not drop it.

``init_device`` resolves ``gms_ro_connect_timeout_ms`` for its own use on
the wake path. If it does not also hand the value to the factory, the very
first weights handshake is unbounded, and a shadow refused admission parks
there with no log and no traceback.
"""
calls = init_device_calls(
{
"gms_read_only": True,
"gms_ro_connect_timeout_ms": _RO_CONNECT_TIMEOUT_MS,
}
)

assert len(calls) == 1
assert calls[0]["tag"] == "weights"
assert calls[0]["mode"] == RequestedLockType.RO
assert calls[0]["timeout_ms"] == _RO_CONNECT_TIMEOUT_MS
Loading