From 27791a4f435efb7c1beac71fe2f6c557fca03a47 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 24 Oct 2026 13:04:57 -0700 Subject: [PATCH 01/82] feat(dynamo)!: serve any vLLM generation config through Dynamo's backend SDK `policy.generation.dynamo: true` on a vLLM generation config is now the whole Dynamo switch. NeMo RL's own vLLM workers register their AsyncLLM with a managed dynamo.frontend through Dynamo's engine-agnostic backend SDK (dynamo.common.backend), from inside each worker's actor loop. No Dynamo-owned vLLM code runs, so refit, pause, quantization, speculative decoding and parallelism are the native backend's own, and a vLLM bump never waits on Dynamo. - One request pipeline (vllm/request_pipeline.py) serves native generate_async, the native OpenAI server and the Dynamo adapter: the sampling-profile check, clamps, R3, token fields, capture, prefix splice and media. - Rollouts go through the frontend's KV-aware router. The engine publishes KV events, and functional overlays fail if the router applies none. - NeMo-Gym calls go through the frontend in Gym's Dynamo dialect (NVIDIA-NeMo/Gym#4817), with engine-side token and media capture. - Startup rejects what Dynamo cannot serve: colocated generation, fleet health, the generation router, a bearer token, R3 with Gym but no capture, parser plugins and chat settings with no frontend counterpart. - Removed: backend: dynamo, DynamoGeneration, the dynamo.vllm launcher, its refit routes, token wrapper, metrics scraper and isolated venv. Old keys are rejected with the migration. - ai-dynamo and ai-dynamo-runtime 1.5.1 join the vllm extra, without the planner deps. BUILD_DYNAMO installs only etcd and nats-server. - CI: the Dynamo lane runs native SingleController tests as overlays: math, Gym token capture, reload refit, and R3 with deferred-route capture. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .github/workflows/cicd-main.yml | 4 +- .github/workflows/lockfile-check.yml | 5 - 3rdparty/Gym-workspace/Gym | 2 +- docker/Dockerfile | 11 +- docker/dynamo/install.sh | 51 +- docker/dynamo/pyproject.toml | 9 - docker/dynamo/uv.lock | 4572 ----------------- docs/design-docs/dynamo-integration.md | 169 +- docs/guides/async-grpo.md | 2 +- docs/guides/dynamo-generation.md | 389 +- docs/guides/single-controller.md | 4 +- docs/index.md | 4 +- docs/observability/configuration.md | 2 +- docs/observability/metrics.md | 2 +- examples/configs/grpo_math_1B.yaml | 2 +- examples/configs/grpo_math_1B_dynamo.yaml | 97 - ...grpo_math_1B_dynamo_single_controller.yaml | 68 +- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 21 +- .../async_utils/trajectory_collector.py | 23 +- nemo_rl/algorithms/distillation.py | 2 +- nemo_rl/algorithms/grpo.py | 80 +- .../single_controller_utils/config.py | 4 +- .../single_controller_utils/setup.py | 58 +- nemo_rl/distributed/actor_environments.py | 1 - nemo_rl/distributed/virtual_cluster.py | 4 +- nemo_rl/environments/nemo_gym.py | 40 +- nemo_rl/experience/rollouts.py | 3 - nemo_rl/models/generation/__init__.py | 66 +- nemo_rl/models/generation/constants.py | 1 - nemo_rl/models/generation/dynamo/__init__.py | 8 +- nemo_rl/models/generation/dynamo/arguments.py | 167 +- nemo_rl/models/generation/dynamo/config.py | 507 +- .../generation/dynamo/dynamo_generation.py | 767 --- .../models/generation/dynamo/dynamo_worker.py | 304 -- .../models/generation/dynamo/http_client.py | 41 +- .../generation/dynamo/managed_runtime.py | 259 +- nemo_rl/models/generation/dynamo/metrics.py | 238 - .../models/generation/dynamo/native_engine.py | 930 ++++ nemo_rl/models/generation/dynamo/refit.py | 320 -- .../generation/dynamo/rollout_client.py | 114 + .../models/generation/dynamo/token_wrapper.py | 556 -- .../dynamo/validate_dynamo_vllm_args.py | 86 - nemo_rl/models/generation/dynamo/venv.py | 62 +- .../models/generation/dynamo/worker_pool.py | 246 - nemo_rl/models/generation/interfaces.py | 3 - .../models/generation/openai_server_utils.py | 72 +- nemo_rl/models/generation/vllm/config.py | 8 + .../models/generation/vllm/metric_names.py | 2 +- .../generation/vllm/request_pipeline.py | 217 + nemo_rl/models/generation/vllm/utils.py | 243 +- .../models/generation/vllm/vllm_generation.py | 200 + .../generation/vllm/vllm_worker_async.py | 425 +- nemo_rl/utils/outdated_config_checks.py | 52 + nemo_rl/utils/routed_experts_codec.py | 8 + .../collective_weight_synchronizer.py | 10 +- nemo_rl/weight_sync/factory.py | 4 +- pyproject.toml | 11 +- pyrefly.toml | 10 +- .../functional/L1_Functional_Tests_Dynamo.sh | 9 +- tests/functional/check_dynamo_overlay.sh | 53 + .../grpo_async_gym_single_controller.sh | 8 + tests/functional/grpo_dp_single_controller.sh | 4 + tests/functional/grpo_dynamo.sh | 79 - .../grpo_dynamo_single_controller.sh | 86 - ...lm_token_capture_single_controller_1n2g.sh | 4 + ...nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh | 4 +- tests/unit/algorithms/test_async_utils.py | 28 - tests/unit/algorithms/test_distillation.py | 2 +- tests/unit/algorithms/test_grpo.py | 280 +- .../test_nemo_gym_token_capture.py | 79 +- .../unit/environments/test_nemo_gym_utils.py | 7 +- tests/unit/experience/test_rollouts.py | 29 - .../generation/test_dynamo_arguments.py | 424 -- .../models/generation/test_dynamo_config.py | 202 + .../generation/test_dynamo_generation.py | 667 --- .../generation/test_dynamo_http_client.py | 123 +- .../generation/test_dynamo_managed_runtime.py | 810 +-- .../generation/test_dynamo_native_engine.py | 746 +++ .../test_dynamo_native_engine_capture.py | 232 + .../generation/test_dynamo_rollout_client.py | 67 + .../generation/test_dynamo_token_wrapper.py | 646 --- .../generation/test_dynamo_vllm_decoupling.py | 91 + .../generation/test_openai_server_utils.py | 92 + .../generation/test_swe1_dynamo_config.py | 21 +- .../models/generation/test_vllm_generation.py | 61 +- .../generation/test_vllm_request_pipeline.py | 146 + .../test_vllm_token_capture_hosting.py | 1 + .../unit/models/generation/test_vllm_utils.py | 27 + .../policy/test_worker_refit_signatures.py | 1 - tests/unit/single_controller/test_setup.py | 194 +- tests/unit/test_dependency_pins.py | 18 + .../unit/utils/test_outdated_config_checks.py | 43 + .../test_reconcile_communicator.py | 7 +- .../weight_sync/test_weight_synchronizer.py | 12 - uv.lock | 185 +- 95 files changed, 5006 insertions(+), 12048 deletions(-) delete mode 100644 docker/dynamo/pyproject.toml delete mode 100644 docker/dynamo/uv.lock delete mode 100644 examples/configs/grpo_math_1B_dynamo.yaml delete mode 100644 nemo_rl/models/generation/dynamo/dynamo_generation.py delete mode 100644 nemo_rl/models/generation/dynamo/dynamo_worker.py delete mode 100644 nemo_rl/models/generation/dynamo/metrics.py create mode 100644 nemo_rl/models/generation/dynamo/native_engine.py delete mode 100644 nemo_rl/models/generation/dynamo/refit.py create mode 100644 nemo_rl/models/generation/dynamo/rollout_client.py delete mode 100644 nemo_rl/models/generation/dynamo/token_wrapper.py delete mode 100644 nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py delete mode 100644 nemo_rl/models/generation/dynamo/worker_pool.py create mode 100644 nemo_rl/models/generation/vllm/request_pipeline.py create mode 100755 tests/functional/check_dynamo_overlay.sh delete mode 100644 tests/functional/grpo_dynamo.sh delete mode 100644 tests/functional/grpo_dynamo_single_controller.sh delete mode 100644 tests/unit/models/generation/test_dynamo_arguments.py create mode 100644 tests/unit/models/generation/test_dynamo_config.py delete mode 100644 tests/unit/models/generation/test_dynamo_generation.py create mode 100644 tests/unit/models/generation/test_dynamo_native_engine.py create mode 100644 tests/unit/models/generation/test_dynamo_native_engine_capture.py create mode 100644 tests/unit/models/generation/test_dynamo_rollout_client.py delete mode 100644 tests/unit/models/generation/test_dynamo_token_wrapper.py create mode 100644 tests/unit/models/generation/test_dynamo_vllm_decoupling.py create mode 100644 tests/unit/models/generation/test_vllm_request_pipeline.py diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index c7bad64967a..ca7de9393e7 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -238,8 +238,8 @@ jobs: h100_functional_test_scripts=$(jq -c \ '[.functional[] | select((ascii_downcase | contains("gb200")) | not)]' \ <<< "$test_plan") - # The managed Dynamo runtime is currently amd64-only and is not - # installed in the arm64 GB200 image. + # The GB200 image is built without BUILD_DYNAMO, so it has no etcd or + # nats-server for the managed Dynamo runtime. gb200_functional_test_scripts=$(jq -c \ '[.functional[] | select(. != "L1_Functional_Tests_Dynamo")]' \ <<< "$test_plan") diff --git a/.github/workflows/lockfile-check.yml b/.github/workflows/lockfile-check.yml index 7ecaf0d2684..7d54bf94049 100644 --- a/.github/workflows/lockfile-check.yml +++ b/.github/workflows/lockfile-check.yml @@ -26,8 +26,6 @@ on: - "pyproject.toml" - "uv.lock" - "3rdparty/**" - - "docker/dynamo/pyproject.toml" - - "docker/dynamo/uv.lock" - ".github/workflows/lockfile-check.yml" jobs: @@ -50,6 +48,3 @@ jobs: # uv.lock is slow (~5 min); cached runs are fast. - name: Check lockfile is up to date run: uv lock --check - - - name: Check Dynamo lockfile is up to date - run: uv lock --check --directory docker/dynamo diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 3f6f20593a1..5e2fb03bbab 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 3f6f20593a126ca9484793769a51d186aa1486c3 +Subproject commit 5e2fb03bbabe89543291cf99c471e3d32effdd83 diff --git a/docker/Dockerfile b/docker/Dockerfile index 4e8393a3eb5..cfa1afa949b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,8 @@ # --build-arg SKIP_VLLM_BUILD=1 # Skip vLLM dependencies # --build-arg SKIP_SGLANG_BUILD=1 # Skip SGLang dependencies # --build-arg SKIP_TRTLLM_BUILD=1 # Skip TRT-LLM dependencies -# Optional isolated managed Dynamo runtime (Dynamo + vLLM only): +# Optional etcd and nats-server binaries for Dynamo serving +# (policy.generation.dynamo=true; ai-dynamo itself ships in the vLLM venvs): # --build-arg BUILD_DYNAMO=1 ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.05-cuda13.2-devel-ubuntu24.04 @@ -587,7 +588,6 @@ FROM hermetic AS release # Re-declare build args for this stage ARG BUILD_DYNAMO -ARG DYNAMO_PYTHON_VERSION=3.12.11 ARG ETCD_VERSION=v3.5.21 ARG NATS_VERSION=v2.11.6 # Space-separated config paths whose NeMo Gym venvs are prefetched into the image @@ -606,7 +606,7 @@ ENV NVIDIA_BUILD_REF=${NVIDIA_BUILD_REF:-} LABEL com.nvidia.build.id="${NVIDIA_BUILD_ID}" LABEL com.nvidia.build.ref="${NVIDIA_BUILD_REF}" -ENV NEMO_RL_DYNAMO_VENV_DIR=/opt/dynamo_venv +ENV NEMO_RL_DYNAMO_BIN_DIR=/opt/dynamo/bin # AWS EFA OFI plugin discovery (p4d / p5 / p5en multi-node NCCL via SRD). # The host's aws-efa-installer places libnccl-net-ofi.so under @@ -623,11 +623,10 @@ ENV LD_LIBRARY_PATH="/opt/amazon/ofi-nccl/lib:/opt/amazon/efa/lib:${LD_LIBRARY_P # Exclude pyproject.toml and uv.lock since those may be altered by build-custom-vllm.sh COPY --from=nemo-rl --exclude=pyproject.toml --exclude=uv.lock . /opt/nemo-rl -# Keep Dynamo's Python 3.12 dependency graph isolated from NeMo-RL's normal -# Ray/vLLM environments. Default image builds do not create this environment. +# The control-plane binaries Dynamo serving starts. Default image builds skip them. RUN <<"EOF" bash -exu -o pipefail if [[ "${BUILD_DYNAMO:-0}" != "1" ]]; then - echo "BUILD_DYNAMO is not 1; skipping the managed Dynamo runtime" + echo "BUILD_DYNAMO is not 1; skipping the Dynamo service binaries" exit 0 fi bash docker/dynamo/install.sh diff --git a/docker/dynamo/install.sh b/docker/dynamo/install.sh index 12c4ecf99e1..7797a308988 100644 --- a/docker/dynamo/install.sh +++ b/docker/dynamo/install.sh @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Install the etcd and nats-server binaries the managed Dynamo services run. +# ai-dynamo itself ships in NeMo RL's vLLM worker venvs (the vllm extra), so +# Dynamo serves the same vLLM release NeMo RL pins; nothing here touches Python. + set -euo pipefail -script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -project_dir=${NEMO_RL_DYNAMO_PROJECT_DIR:-${script_dir}} -repo_root=$(realpath "${script_dir}/../..") -dynamo_venv_dir=${NEMO_RL_DYNAMO_VENV_DIR:-${repo_root}/venvs/dynamo} -dynamo_python_version=${DYNAMO_PYTHON_VERSION:-3.12.11} +bin_dir=${NEMO_RL_DYNAMO_BIN_DIR:-/opt/dynamo/bin} etcd_version=${ETCD_VERSION:-v3.5.21} nats_version=${NATS_VERSION:-v2.11.6} @@ -42,36 +42,7 @@ case "${target_arch}" in ;; esac -uv python install "${dynamo_python_version}" -uv venv --python "${dynamo_python_version}" "${dynamo_venv_dir}" -UV_PROJECT_ENVIRONMENT="${dynamo_venv_dir}" uv sync \ - --directory "${project_dir}" \ - --locked \ - --no-dev \ - --no-install-project \ - --link-mode copy - -dynamo_python=${dynamo_venv_dir}/bin/python -vllm_version=$("${dynamo_python}" -c \ - 'from importlib.metadata import version; print(version("vllm"))') -if [[ "${vllm_version}" != "0.28.0" ]]; then - echo "Expected vllm==0.28.0 from ai-dynamo[vllm]==1.5.1; got ${vllm_version}" >&2 - exit 1 -fi - -# Layerwise weight reload must count only the elements each composed loader -# actually writes (vLLM PR #44814). Without it, trailing NemotronH/Mamba2 -# parameters such as mixer.D stay unloaded after a refit and corrupt logits. -# vLLM 0.28.0 ships the fix; fail the build if a future pin drops it. -"${dynamo_python}" - <<'PY' -import inspect - -import vllm.model_executor.model_loader.reload.meta as meta - -if "min(numel," not in inspect.getsource(meta): - raise SystemExit("vLLM is missing the PR #44814 layerwise-reload fix") -PY - +mkdir -p "${bin_dir}" download_dir=$(mktemp -d "${TMPDIR:-/tmp}/nemorl-dynamo-install.XXXXXX") trap 'rm -rf "${download_dir}"' EXIT @@ -81,7 +52,7 @@ curl --fail --location --retry 3 \ tar -xzf "${download_dir}/etcd.tgz" -C "${download_dir}" install -m 0755 \ "${download_dir}/etcd-${etcd_version}-linux-${target_arch}/etcd" \ - "${dynamo_venv_dir}/bin/etcd" + "${bin_dir}/etcd" curl --fail --location --retry 3 \ "https://github.com/nats-io/nats-server/releases/download/${nats_version}/nats-server-${nats_version}-linux-${target_arch}.tar.gz" \ @@ -89,9 +60,7 @@ curl --fail --location --retry 3 \ tar -xzf "${download_dir}/nats.tgz" -C "${download_dir}" install -m 0755 \ "${download_dir}/nats-server-${nats_version}-linux-${target_arch}/nats-server" \ - "${dynamo_venv_dir}/bin/nats-server" + "${bin_dir}/nats-server" -"${dynamo_python}" -c \ - 'import importlib.metadata as m; assert m.version("ai-dynamo") == "1.5.1"; assert m.version("vllm") == "0.28.0"; assert m.version("nvidia-nccl-cu13") == "2.30.7"' -"${dynamo_venv_dir}/bin/etcd" --version -"${dynamo_venv_dir}/bin/nats-server" --version +"${bin_dir}/etcd" --version +"${bin_dir}/nats-server" --version diff --git a/docker/dynamo/pyproject.toml b/docker/dynamo/pyproject.toml deleted file mode 100644 index f1eff7a2aaf..00000000000 --- a/docker/dynamo/pyproject.toml +++ /dev/null @@ -1,9 +0,0 @@ -[project] -name = "nemo-rl-dynamo-runtime" -version = "0.0.0" -requires-python = "==3.12.*" -dependencies = ["ai-dynamo[vllm]==1.5.1"] - -[tool.uv] -package = false -override-dependencies = ["nvidia-nccl-cu13==2.30.7; sys_platform == 'linux'"] diff --git a/docker/dynamo/uv.lock b/docker/dynamo/uv.lock deleted file mode 100644 index b00d829e528..00000000000 --- a/docker/dynamo/uv.lock +++ /dev/null @@ -1,4572 +0,0 @@ -version = 1 -revision = 3 -requires-python = "==3.12.*" -resolution-markers = [ - "sys_platform == 'darwin'", - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] - -[manifest] -overrides = [{ name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'", specifier = "==2.30.7" }] - -[[package]] -name = "absl-py" -version = "2.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/1d/58e2b5a6e4d703ccb2a029943d665974cb3d5a4fb2b3e3675dd03a9df10e/absl_py-2.5.1.tar.gz", hash = "sha256:286e71c82c1a38e75bbcf185f9b37d0305ad7786535107cb49bf4df9ff2e1f95", size = 118387, upload-time = "2026-10-09T08:38:58.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/7d/01e62f59e4166af1be6238d9f2b7f3453630c51b8044b412d05b5606532a/absl_py-2.5.1-py3-none-any.whl", hash = "sha256:721200f2f0e9960f2ca9dc3a2a706b201f5f75d812c158f059cbbe29eeafbdb8", size = 137673, upload-time = "2026-10-09T08:38:56.629Z" }, -] - -[[package]] -name = "ai-dynamo" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ai-dynamo-runtime" }, - { name = "aiohttp" }, - { name = "aisimulate" }, - { name = "kubernetes" }, - { name = "msgspec" }, - { name = "prometheus-client" }, - { name = "pyzmq" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "typing-extensions" }, - { name = "zstandard" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/19/bc00c468b1d8093abafa7f3ab88362d12cfe06f9be2e65baf0ef03d370d1/ai_dynamo-1.5.1-py3-none-any.whl", hash = "sha256:86d21cb7fbebbf1c3d0933303729b1be222ad0fb4a50b17a03316165d33377b1", size = 3512837, upload-time = "2026-10-07T05:08:09.942Z" }, -] - -[package.optional-dependencies] -vllm = [ - { name = "blake3" }, - { name = "librosa" }, - { name = "nixl", extra = ["cu13"] }, - { name = "ray" }, - { name = "soundfile" }, - { name = "uvloop" }, - { name = "vllm", extra = ["otel", "runai"] }, -] - -[[package]] -name = "ai-dynamo-runtime" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "uvloop" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/63/f1cde7bb2d74aebc954251fe40ee093298f0975d6f22c47f1f3c66b11949/ai_dynamo_runtime-1.5.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:88e5580daf32056dbe2da9d4189ddd2f4e4cc2472ddfebdd19412d223ac4a596", size = 65897955, upload-time = "2026-10-07T05:08:55.817Z" }, - { url = "https://files.pythonhosted.org/packages/df/05/777f6f79e8c9359d259cf500ec62fcaf6bd9ce148d3bf47910ec02d996e3/ai_dynamo_runtime-1.5.1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b5297625a4f407b3871e3039a53f49e09102d67039fc1339349802038b84ec3e", size = 67290732, upload-time = "2026-10-07T05:09:45.438Z" }, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, -] - -[[package]] -name = "aiohttp" -version = "3.14.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohappyeyeballs" }, - { name = "aiosignal" }, - { name = "attrs" }, - { name = "frozenlist" }, - { name = "multidict" }, - { name = "propcache" }, - { name = "typing-extensions" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, - { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, - { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, - { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, - { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, - { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, - { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, - { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, - { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, - { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, - { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, - { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, - { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, - { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, - { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, -] - -[[package]] -name = "aiosignal" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "frozenlist" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, -] - -[[package]] -name = "aisimulate" -version = "0.12.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "bokeh" }, - { name = "chex" }, - { name = "flax" }, - { name = "google-vizier", extra = ["jax"] }, - { name = "jax" }, - { name = "jaxlib" }, - { name = "jinja2" }, - { name = "matplotlib" }, - { name = "munch" }, - { name = "numpy" }, - { name = "nvidia-ml-py" }, - { name = "optax" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "plotext" }, - { name = "plotly" }, - { name = "prettytable" }, - { name = "pyarrow" }, - { name = "pydantic" }, - { name = "pyyaml" }, - { name = "scipy" }, - { name = "tfp-nightly", extra = ["jax"] }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/75/8031a245717bcd8eb742fb59405576c296a3c8f554bc5cea0b4a9e25992a/aisimulate-0.12.0.tar.gz", hash = "sha256:131665696adf122f94a7e8ba50dd853718c0009070f3f5057b7680d3d75ebf7b", size = 20458, upload-time = "2026-09-17T18:00:29.629Z" } - -[[package]] -name = "annotated-doc" -version = "0.0.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, -] - -[[package]] -name = "annotated-types" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, -] - -[[package]] -name = "anthropic" -version = "0.120.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "docstring-parser" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d7/10/4ca013cb166f226bd89e0aeb0fcaff94f45ddf716d4925ce89475d3c587b/anthropic-0.120.2.tar.gz", hash = "sha256:9722efc10c27a30a69f5338ddacdb35bc6a64297a4e4ba729bf83af873d5fb3a", size = 1008421, upload-time = "2026-07-28T17:38:26.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/af/0f5db57b9397a0f3b7fc204cbef143401a7cadaf982330f97f1ce3d39f34/anthropic-0.120.2-py3-none-any.whl", hash = "sha256:0f0bc2b381dc0eb41c8d886b815d79c2041cd2374f83aed36f574b6dc9c579c1", size = 1022851, upload-time = "2026-07-28T17:38:25.466Z" }, -] - -[[package]] -name = "anyio" -version = "4.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, -] - -[[package]] -name = "apache-tvm-ffi" -version = "0.1.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/3d/4b9226cd45aa800a6904603dda9b323d728f3c3869952a673f3483b78b19/apache_tvm_ffi-0.1.11.tar.gz", hash = "sha256:153cd2c5a9717804cb0bcd9b2709f22a1e5f80ed05b5a490faf5949b136eedba", size = 2798354, upload-time = "2026-05-04T17:48:43.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/9d/0f81ca556e5836b3ca64818cdae3f47dc7822bd35d22ddef7a54106d801d/apache_tvm_ffi-0.1.11-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:6ae51cc7df415b5f373a9df4baa1165a65608e519bea81e7dd23428f00eeb689", size = 2418793, upload-time = "2026-05-04T17:47:57.879Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a9/f48e5dd4ae1f6f0c5ffac259c0a9531b7d6a7c0a4c45bc2229d55de6adf8/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da2c8d07fdc737d1ba75f4de25c29f156905b9dc980f1da90c395b4db525f522", size = 2605176, upload-time = "2026-05-04T17:47:59.676Z" }, - { url = "https://files.pythonhosted.org/packages/36/99/2848df4e8ed5bf51df1d286d1718510584fa61e88adbc9c5b23d71b38f7c/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78aa1857b04a2ea718317041ab3f01288b3d496e6036eb1b99ebdc9da0fdaef5", size = 2725887, upload-time = "2026-05-04T17:48:01.381Z" }, - { url = "https://files.pythonhosted.org/packages/7d/80/963c991934a4eb0fa0c0178f51963333fe14a96b732009da642b6bf6b42e/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a8b845c8dff498fb981c1dda36c954549204191b485a385845e604966594d0b2", size = 2513121, upload-time = "2026-05-04T17:48:03.43Z" }, - { url = "https://files.pythonhosted.org/packages/4d/18/95569107ee83619d61a3bb0d28743a0599f85c5161981e3e098c82c2b185/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2843f084cdc94dedacd8b257a395a2b71b8a3dc7fc99711b148bf1d161983128", size = 2697683, upload-time = "2026-05-04T17:48:05.222Z" }, - { url = "https://files.pythonhosted.org/packages/dc/99/f352cf1cce8f6f05584c4adf11de9eca07e6d217229bad6af35fb372926c/apache_tvm_ffi-0.1.11-cp312-abi3-win_amd64.whl", hash = "sha256:bd67e03759d25ff59f4e0ed9c8630a16872afc9dd8792f46ac3c927554015e60", size = 2365545, upload-time = "2026-05-04T17:48:07.295Z" }, -] - -[[package]] -name = "astor" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, -] - -[[package]] -name = "attrs" -version = "26.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, -] - -[[package]] -name = "audioread" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/4a/874ecf9b472f998130c2b5e145dcdb9f6131e84786111489103b66772143/audioread-3.1.0.tar.gz", hash = "sha256:1c4ab2f2972764c896a8ac61ac53e261c8d29f0c6ccd652f84e18f08a4cab190", size = 20082, upload-time = "2025-10-26T19:44:13.484Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/16/fbe8e1e185a45042f7cd3a282def5bb8d95bb69ab9e9ef6a5368aa17e426/audioread-3.1.0-py3-none-any.whl", hash = "sha256:b30d1df6c5d3de5dcef0fb0e256f6ea17bdcf5f979408df0297d8a408e2971b4", size = 23143, upload-time = "2025-10-26T19:44:12.016Z" }, -] - -[[package]] -name = "azure-core" -version = "1.41.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, -] - -[[package]] -name = "azure-identity" -version = "1.25.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "msal" }, - { name = "msal-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/0e/3a63efb48aa4a5ae2cfca61ee152fbcb668092134d3eb8bfda472dd5c617/azure_identity-1.25.3.tar.gz", hash = "sha256:ab23c0d63015f50b630ef6c6cf395e7262f439ce06e5d07a64e874c724f8d9e6", size = 286304, upload-time = "2026-03-13T01:12:20.892Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/9a/417b3a533e01953a7c618884df2cb05a71e7b68bdbce4fbdb62349d2a2e8/azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c", size = 192138, upload-time = "2026-03-13T01:12:22.951Z" }, -] - -[[package]] -name = "azure-storage-blob" -version = "12.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "isodate" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/48/84a820d898267f662b5c06f7cd76fdb8a9e272b44aa9376cef3ec0f6a294/azure_storage_blob-12.30.0.tar.gz", hash = "sha256:2cd74d4d5731e5eb6b8d5c5056ee115a5e88f8fdf22517b739836fda685018be", size = 618229, upload-time = "2026-06-08T11:45:35.575Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/0b/e106f0fd7fa785867d9ffcc47dc9e6237c0e58f51058473b777487a98edc/azure_storage_blob-12.30.0-py3-none-any.whl", hash = "sha256:d415ac50b67a8da6b3ae7e9f1014b1b55cd7aafa0b8d4ca9b380568dc7360423", size = 435610, upload-time = "2026-06-08T11:45:37.213Z" }, -] - -[[package]] -name = "blake3" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/6a/4cc5a9dd40fd8a6d283fd3761e5f59c490109571ef8e3c73245417e5a305/blake3-1.0.9.tar.gz", hash = "sha256:5fa374fa5070ca084368776c19b420157eb0f2d3f091343d6bc59189929d62e2", size = 116872, upload-time = "2026-06-22T18:02:25.366Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/d2/9bdf8345c70993aaef635398f52edfb915d6e8ad2c000c801204e387c456/blake3-1.0.9-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a70c20542d5e7960983a0ff32999049a2b0e5ef1f22dbbbdfb51cf04828a4156", size = 344587, upload-time = "2026-06-22T18:00:34.244Z" }, - { url = "https://files.pythonhosted.org/packages/36/9d/be8b1f7f85b12bb45a0fade6ca7bdbf83a507d23d0b6141ba29fe69c8cea/blake3-1.0.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:72cdecf088a9d25e6ec79948a578995649b0dbee407e7a46c543a9ecc0f6f281", size = 328864, upload-time = "2026-06-22T18:00:35.59Z" }, - { url = "https://files.pythonhosted.org/packages/f2/78/66580635d744c826671fd219938caffb16281a26f62c4f856695d4233677/blake3-1.0.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42fa57bf462285ef16400601b0fd32214c248ba92505bbb94b1221ab9af5a092", size = 373795, upload-time = "2026-06-22T18:00:36.887Z" }, - { url = "https://files.pythonhosted.org/packages/b1/79/b5b17d3004bb81a5732c0b176c812703d200ed8c652b3b7713b9633bbe10/blake3-1.0.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b25ccde5a64be070f20e5c7a81da70292db40b164b6c77588cbd6230856badbb", size = 374183, upload-time = "2026-06-22T18:00:38.205Z" }, - { url = "https://files.pythonhosted.org/packages/3c/63/0d209c44b2041bbe130ced12a23c92dd995fbfe5bce7ee77fffea16f5cb0/blake3-1.0.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a800b87433955f37691b5f361ad29c7dd3ee089c9cd109adc5aea8e24bc4c1f", size = 446783, upload-time = "2026-06-22T18:00:39.493Z" }, - { url = "https://files.pythonhosted.org/packages/c5/51/efd1f9b8a9d3e9a0e235f3ced99a738529a1019fe78b3988e29d9c2fbba6/blake3-1.0.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6879739e7904b9c42afbedbcc2e8c36cebe140fb3fc3f5c492993579cf5cd516", size = 487369, upload-time = "2026-06-22T18:00:40.875Z" }, - { url = "https://files.pythonhosted.org/packages/8d/3f/a8dcaea9e0b26e419a540ca0cd6203c9fbb505e85b02b03c5a59bf9e6a45/blake3-1.0.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6edeb3d49a24c307995899b70dd47aa901d0e9ad51d2f8a79aba4f074f32d8c5", size = 383845, upload-time = "2026-06-22T18:00:42.251Z" }, - { url = "https://files.pythonhosted.org/packages/f6/10/e9907f5b86410d5071982aaf05d149ca4d4fd8acab7e77eebbc9a333c7b4/blake3-1.0.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd56a7a972c4185070f7042ccc20166927eec3c0f98b8405f375d007b604a0b", size = 383851, upload-time = "2026-06-22T18:00:43.715Z" }, - { url = "https://files.pythonhosted.org/packages/34/cf/c7863a185550706a9624f6aa7b6d46470aaed0bb46a827c5cda2a7d03151/blake3-1.0.9-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:a288664d08dee154cc496e06e62517fc9e655ecec12b0d7db538d244ac79edf1", size = 380067, upload-time = "2026-06-22T18:00:45.249Z" }, - { url = "https://files.pythonhosted.org/packages/54/0a/e7af679c719368b400c9ba9c3460072aac2ba077ddbd4bc806fef28cda03/blake3-1.0.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:91db52a809b68b5bebe7c413ddcd230e1f759398e7fa7a873104595a4fa648b6", size = 549471, upload-time = "2026-06-22T18:00:46.793Z" }, - { url = "https://files.pythonhosted.org/packages/2c/3c/37c1dd3539b7bd9b6d2eef019802aacdb4a3d48ab484b140603bbf9c5b5a/blake3-1.0.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfaa671b07eb73883162ca940442193868358b0b904cfa266e4b74131ce966da", size = 591396, upload-time = "2026-06-22T18:00:48.122Z" }, - { url = "https://files.pythonhosted.org/packages/ae/55/4f0a23b72795292e74084834130900ea778c0583004519c86698dfffe1a5/blake3-1.0.9-cp312-cp312-win32.whl", hash = "sha256:ae47c3d5729ff89baa6ddf6de47fcfcc915985d39eb1bfcd6db653331f3c6fcc", size = 229271, upload-time = "2026-06-22T18:00:49.377Z" }, - { url = "https://files.pythonhosted.org/packages/12/91/7db93e4689f0f145bcb954dc62936e5f5090548a9fa20c6bbebfaeaa648a/blake3-1.0.9-cp312-cp312-win_amd64.whl", hash = "sha256:15566065ff90ab3da46ec0be1417406f00507af902b6fb0fbc6563e77f02fc42", size = 218220, upload-time = "2026-06-22T18:00:50.659Z" }, -] - -[[package]] -name = "bokeh" -version = "3.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "narwhals" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyyaml" }, - { name = "tornado", marker = "sys_platform != 'emscripten'" }, - { name = "xyzservices" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/ab/19f33e7442fe2f09378bdafa95d088a1ed0dd078ea1f3d676c28d3cf2ed7/bokeh-3.10.1.tar.gz", hash = "sha256:e1dc21c4fa02f793b7c6acb2b439311924f46912b1d1ae1c7b058f8245b54fe8", size = 6053211, upload-time = "2026-10-13T16:53:00.445Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/9c/aaf2f11e31a75f0fffc79659ad7f0be01f39bb018bbc83c2a8a02657589e/bokeh-3.10.1-py3-none-any.whl", hash = "sha256:d73412b6317f70df0ffe6796956af609617523dfdfbc624d4f7f496cd09e8101", size = 6721259, upload-time = "2026-10-13T16:52:58.309Z" }, -] - -[[package]] -name = "boto3" -version = "1.43.58" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/95/bd6276870084c9c1a94e17d1dc73b2de275e59bd7a0ad8bfff0a1598cec4/boto3-1.43.58.tar.gz", hash = "sha256:12871fb50c383f1b9aa4ed6dd386ba689062baef730552e79d5a9cd782b53058", size = 112685, upload-time = "2026-07-28T19:35:09.336Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/85/b0709066efb4ce7b86aba4092c739ad0e458be9d62cf32550fc4c130c93f/boto3-1.43.58-py3-none-any.whl", hash = "sha256:ce1a20cbcfaa1d0b3c8f568e2b6c7fbd34b842ea00e729d49a4b4de522828db9", size = 140026, upload-time = "2026-07-28T19:35:06.993Z" }, -] - -[[package]] -name = "botocore" -version = "1.43.58" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jmespath" }, - { name = "python-dateutil" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/64/5cd46a7e72b0647e6d78fc8da016259ad66b9ae0818f4c5d629c75e7ca49/botocore-1.43.58.tar.gz", hash = "sha256:e110ca53f65c128fe98df4d6d36a459b1db17c6114671c8a18becf151bf20909", size = 15742412, upload-time = "2026-07-28T19:34:57.468Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/82/6f8fbbea47b773734ba0199643d2d851e5c2f75bc3699fe99db8af344d96/botocore-1.43.58-py3-none-any.whl", hash = "sha256:f516159f0732da8249206163ccea3bd1f82ad2a9d184fe6ed447e1abdba4330e", size = 15426503, upload-time = "2026-07-28T19:34:53.508Z" }, -] - -[[package]] -name = "cachetools" -version = "7.1.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/af/861ebc2e318a5c3300e3eb63bc4d30f3d70a46d13b360093728ac0705eed/cachetools-7.1.6.tar.gz", hash = "sha256:c7a79e7f30ba9943c1cefd08cc36f006aaae086e017af9166f1d59d6170c47e1", size = 40572, upload-time = "2026-07-23T22:47:53.737Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/f2/2086ba18a925a73586c4d4e61d25f4a6058e56fd00d77ce8f1d361ab4c9b/cachetools-7.1.6-py3-none-any.whl", hash = "sha256:2c12e255780330af28b91bb7fb96cce4c766f04e38396b9a24510190a5827096", size = 16954, upload-time = "2026-07-23T22:47:52.397Z" }, -] - -[[package]] -name = "cbor2" -version = "6.1.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3a/6f/07b4af8da8bd27f640362b1ac8271d80895407f2ede0c2bcc9433c06e1ca/cbor2-6.1.3.tar.gz", hash = "sha256:8d70680acb55c04ea5b5ad86da094f9612b53d5a8a65d0f5b3aafc3ce917ecbb", size = 89503, upload-time = "2026-07-04T10:36:48.793Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/16/cff14259c3d19a7f0ae88b6996fe4c85f6ff1764dad889ac8a39e843e39c/cbor2-6.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3d939f55097c21e032f5a2d67592fcc57298986281f219356e2f519e4466f4ea", size = 412779, upload-time = "2026-07-04T10:36:04.975Z" }, - { url = "https://files.pythonhosted.org/packages/50/6c/f3641d19b7b85a63cb2756c10164131489c2cb46b379ec51ae22283fefb9/cbor2-6.1.3-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b025009478d644dab407164fd60e3ef4381af284f5af6966df94c663756d949e", size = 457781, upload-time = "2026-07-04T10:36:06.349Z" }, - { url = "https://files.pythonhosted.org/packages/55/85/0c55a66f3037056bfb8e1c7184168085fdea67ae5830404498bcf466233b/cbor2-6.1.3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:2226d32e102e375737656ad5d141ad8c6ae3e705e04e263f24756f0eb379c6c1", size = 468373, upload-time = "2026-07-04T10:36:07.769Z" }, - { url = "https://files.pythonhosted.org/packages/46/74/40f7db3e0d880560193916a5c9b744fcf299558bed7113f77c28237c7c29/cbor2-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e61d465244d66ffed36492eef3b44d43795d76a2bba0663a2f15c186af7f7513", size = 523844, upload-time = "2026-07-04T10:36:09.404Z" }, - { url = "https://files.pythonhosted.org/packages/b5/a1/b5e07d6a08441c3a552fe2ae48ccb7e9dfc5065b9f6a3bae9879b4f0fbc0/cbor2-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87fe7be8fab6ec4796aa127c1a52e09e79dbafd2aa31caf809cf04b8080a5975", size = 536238, upload-time = "2026-07-04T10:36:10.914Z" }, - { url = "https://files.pythonhosted.org/packages/c9/99/e166be0fd74bf3a91f5a0d103e34883efbc438d970f72cc8200e274787e5/cbor2-6.1.3-cp312-cp312-win32.whl", hash = "sha256:da25d345f01e6a40b2e5c57ef96b4dcff7be69394fb62f0f70e07f437f2376a9", size = 279858, upload-time = "2026-07-04T10:36:12.247Z" }, - { url = "https://files.pythonhosted.org/packages/d2/1b/90b4a121e40aba189c55a5822dd3c698eaf487e1d4a780ab18c804a5ef1c/cbor2-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:d5514f693db6fa6f433b4096e9b604e6a7bf151c9ef1d2db86d0858e4c5e768f", size = 300929, upload-time = "2026-07-04T10:36:13.564Z" }, - { url = "https://files.pythonhosted.org/packages/19/db/52c58a8d33464927389dde8103997b3fa51b081ce29b347ac2cc4fd0dfbf/cbor2-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:3d43183d7beb3d3cd198d69b31bd2ee487ed704a1150c75cb0a66d6ad63d8c1a", size = 290908, upload-time = "2026-07-04T10:36:14.857Z" }, -] - -[[package]] -name = "certifi" -version = "2026.7.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, -] - -[[package]] -name = "cffi" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, - { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, - { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, - { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, - { url = "https://files.pythonhosted.org/packages/58/0c/f528df19cc94b675087324d4760d9e6d5bfae97d6217aa4fac43de4f5fcc/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7", size = 208859, upload-time = "2026-07-06T21:32:52.512Z" }, - { url = "https://files.pythonhosted.org/packages/62/f2/c9522a81c32132799a1972c39f5c5f8b4c8b9f00488a23feaa6c06f07741/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66", size = 221844, upload-time = "2026-07-06T21:32:53.704Z" }, - { url = "https://files.pythonhosted.org/packages/6e/28/bd53988b9833e8f8ad539d26f4c07a6b3f6bcb1e9e02e7ca038250b3428d/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe", size = 225287, upload-time = "2026-07-06T21:32:54.907Z" }, - { url = "https://files.pythonhosted.org/packages/79/99/0d0fd37f055224085f42bbb2c022d002e17dde4a97972822327b07d84101/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b", size = 223681, upload-time = "2026-07-06T21:32:56.329Z" }, - { url = "https://files.pythonhosted.org/packages/b0/80/c138990aa2a70b1a269f6e06348729836d733d6f970867943f61d367f8cc/cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a", size = 175269, upload-time = "2026-07-06T21:32:57.777Z" }, - { url = "https://files.pythonhosted.org/packages/a8/eb/f636456ff21a83fc13c032b58cc5dde061691546ac79efa284b2989b7982/cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384", size = 185881, upload-time = "2026-07-06T21:32:59.253Z" }, - { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, - { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, - { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, - { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, - { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, - { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, - { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, - { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, - { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, - { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, - { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, - { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, - { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, - { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, -] - -[[package]] -name = "chex" -version = "0.1.87" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "jax" }, - { name = "jaxlib" }, - { name = "numpy" }, - { name = "setuptools" }, - { name = "toolz" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ba/8a/857474810b64ab135a0c3e594b0453c7f39f140757c4cd26a32bccadcbc4/chex-0.1.87.tar.gz", hash = "sha256:0096d89cc8d898bb521ef4bfbf5c24549022b0e5b301f529ab57238896fe6c5d", size = 90063, upload-time = "2024-09-30T14:40:56.93Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/dd/c1ff2eb8fbf95a8ca804abb1cc3ce70b283ee7b4bc653c3abac245670400/chex-0.1.87-py3-none-any.whl", hash = "sha256:ce536475661fd96d21be0c1728ecdbedd03f8ff950c662dfc338c92ea782cb16", size = 99369, upload-time = "2024-09-30T14:40:54.862Z" }, -] - -[[package]] -name = "click" -version = "8.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, -] - -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "compressed-tensors" -version = "0.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "loguru" }, - { name = "pydantic" }, - { name = "torch" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2c/9e/d7f18bd9a0354088abc11a0c1f2c7698f7c49e5a709faedf6a46e388f693/compressed_tensors-0.17.0.tar.gz", hash = "sha256:15c20d06bdbcf35b51fc99fd125e7b9be1e1855567c33b7a46dfac26ad6fb126", size = 257091, upload-time = "2026-06-03T16:49:17.208Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/63/6edf0415b072fff0bf8b546074dea3f0f9b148e49b601ac98bdc60a76c68/compressed_tensors-0.17.0-py3-none-any.whl", hash = "sha256:4a1b89b508f7efb8ffb4eee8a6e69e0452d9b080cae130146025c64fbe9fa9aa", size = 211714, upload-time = "2026-06-03T16:49:15.672Z" }, -] - -[[package]] -name = "contourpy" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/5a/a55177dd22553a277388e8a1b3220e92de91bacb28356cdc73caa240121d/contourpy-1.4.0.tar.gz", hash = "sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1", size = 13323726, upload-time = "2026-09-11T19:05:05.808Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/ba/01bde9753bdbba04a6da9d2bff3881f021bc708f654655e95fae26d3b3a3/contourpy-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:186ba929df36d61b6127da2e89cd1357e4cb79aca647381ab1a6feb0b152877b", size = 297302, upload-time = "2026-09-11T19:02:45.103Z" }, - { url = "https://files.pythonhosted.org/packages/dd/f5/c5bf67522d49a2222f3154fe46879451d26f0e35ece41770758a64ea78cb/contourpy-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c76f3a5318164db7d9401132fc1b5364b784c613c93fa506e3b5e6d1bf353ec8", size = 284921, upload-time = "2026-09-11T19:02:48.049Z" }, - { url = "https://files.pythonhosted.org/packages/0f/cc/cb989599eec12fda312e127eb8e04a8b21a7a6c94bf7ff1bc68273334a42/contourpy-1.4.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:43c3ccbb32c6294b183dcc8e8c46dacf5ecef809497e3d48a5be298eef185ad0", size = 356332, upload-time = "2026-09-11T19:02:51.408Z" }, - { url = "https://files.pythonhosted.org/packages/47/fb/6f620d7602507817b0b4c21dd790ed68688f1f4ae9c29aacf21f08db5cf1/contourpy-1.4.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86d05cec773c9507a3950122e0e40ce77c23c75ceeb2fc189514e71893cbb34b", size = 405878, upload-time = "2026-09-11T19:02:54.093Z" }, - { url = "https://files.pythonhosted.org/packages/32/4e/693c6d6bdece679f0953775eef3a1b66d56be55478d936e7deb1f3004422/contourpy-1.4.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2deb580178ca19437a84bd77e4bc2cd91a8ccad212413a83c273900868b5978c", size = 408296, upload-time = "2026-09-11T19:02:56.528Z" }, - { url = "https://files.pythonhosted.org/packages/78/f9/b6831508960d559581c448532ba4df312217072975486dbf2b24fcc5b76f/contourpy-1.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:875f42444c9cf48d56f724f2637e60d0f73b3b12c9041e1484580a233edf9591", size = 383243, upload-time = "2026-09-11T19:02:58.638Z" }, - { url = "https://files.pythonhosted.org/packages/f2/21/52903825a0ae7bb625e8bd30a09816d4c3c5d6174a469c10a616166cf780/contourpy-1.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f863c6100bf926cf47d13f3cd75f9bb8ebd98aaab230eeb4468b91f98f39a6b3", size = 1357306, upload-time = "2026-09-11T19:03:01.036Z" }, - { url = "https://files.pythonhosted.org/packages/d8/79/d68b1ce8539e4071518fed9523f558398f34dcd078b8927b109c72dad2ef/contourpy-1.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3863ef2e2b13fe93f8c0ebb08ee400cb07153b8b0e91c5acb26e4537f283634f", size = 1427243, upload-time = "2026-09-11T19:03:03.866Z" }, - { url = "https://files.pythonhosted.org/packages/ac/f0/b75a10e9d0616b97a30de277b6dfe83f1879880854c7330337309530eb48/contourpy-1.4.0-cp312-cp312-win32.whl", hash = "sha256:5450f091ac1be0be3ad3a2a3b3f23b5e443e78c670ced4fd347d626f92a28fd2", size = 347345, upload-time = "2026-09-11T19:03:06.192Z" }, - { url = "https://files.pythonhosted.org/packages/50/a9/dab08786bb4d77ef9a046b3c1be923be4e73c5d07d91f5a54e8ea9b09e41/contourpy-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e697d94e69f499ff6bebb899cae97a58d5d14f0e1fe9568b43a0248d2f9af8c", size = 234083, upload-time = "2026-09-11T19:03:08.122Z" }, - { url = "https://files.pythonhosted.org/packages/47/b9/3ba509755a970dbde5948e7142ac21f266be72f38017cc4087a05fdceee1/contourpy-1.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:0c7a4c2716a4e98342221954416a836cca77c996c14ddbf22b5f01d5d93ca09c", size = 559521, upload-time = "2026-09-11T19:03:10.047Z" }, -] - -[[package]] -name = "cryptography" -version = "49.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" }, - { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, - { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, - { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, - { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, - { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, - { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, - { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, - { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, - { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, - { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, - { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" }, - { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, - { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, - { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, - { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, - { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, - { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, - { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, - { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, - { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, - { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, - { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, -] - -[[package]] -name = "cuda-bindings" -version = "13.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-pathfinder" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, - { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" }, - { url = "https://files.pythonhosted.org/packages/7c/95/872a0392122f1fb43fcb06869790ef3171f37beee9f7db8f441739113570/cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff", size = 5875099, upload-time = "2026-05-29T23:11:54.635Z" }, -] - -[[package]] -name = "cuda-core" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-pathfinder" }, - { name = "numpy" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/a0/1daeae599cadd612689dbbf70d7da1c01883964fc2fbc7386f3c630a68cf/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6816dc020aee6103d8071bc02d8e4e1d91f2b49596f666896d608d92224d79d1", size = 4789856, upload-time = "2026-05-12T20:11:30.862Z" }, - { url = "https://files.pythonhosted.org/packages/a1/4d/603557ab3cb171cc2a61d3678a39cb4dae3fd21275078bfbd1c0b0b5230b/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be7b65311bf78964b7905adbf3c0f8f717d432f2854dc45169277729bf60f1e2", size = 5106023, upload-time = "2026-05-12T20:11:33.509Z" }, - { url = "https://files.pythonhosted.org/packages/c2/1a/ae079963c9df7f4274227eb63cf8f6083a532a6443adb340d951fd21c626/cuda_core-1.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:1a5c1aa3b738a7599ea289498d038fe625d259fd7ab795394541eee58a8e29bc", size = 4663076, upload-time = "2026-05-12T20:11:35.784Z" }, -] - -[[package]] -name = "cuda-pathfinder" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/b4/d088047afe39827556df21118cac9ffd20cc3f968c99a7681494d1eb333c/cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51", size = 54591, upload-time = "2026-07-21T15:03:56.224Z" }, -] - -[[package]] -name = "cuda-python" -version = "13.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings" }, - { name = "cuda-core" }, - { name = "cuda-pathfinder" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/31/7ff3f7768eded7535c621abc2fecb9d181a34ea4cae3afe682feb796f242/cuda_python-13.3.1-py3-none-any.whl", hash = "sha256:280b014139ab447b6dd70a377db1596f310d6e887d9d342e6651b919ec145fb3", size = 8295, upload-time = "2026-05-29T23:28:47.012Z" }, -] - -[[package]] -name = "cuda-tile" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/6d/cc2fb5a25689a501564a2eced4acf654f307e801a2c1506be97c0d100491/cuda_tile-1.5.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:87652483baa9c81a9a24e4450f016e4ee78fd205d8422dad8996571bd1f2622e", size = 322641, upload-time = "2026-07-08T01:49:23.388Z" }, - { url = "https://files.pythonhosted.org/packages/1c/f5/b4ba9d0fc71198d939ebf9a090228179995d8411ee9def8f638a0e3ccdc5/cuda_tile-1.5.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:cef6d30acc37557643ece0de3770fc4c33497c4af40209e424f72fbfcbe6ea5a", size = 324990, upload-time = "2026-07-08T01:49:17.739Z" }, - { url = "https://files.pythonhosted.org/packages/a5/6e/7a60f317c503580ab7946dbb7fd080438fe953d0ddfdc81904beb9a1fab7/cuda_tile-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:16d97a60ed1d33388abbca85ea08cdae6325cc700476b3135f190d0fb50329f4", size = 304817, upload-time = "2026-07-08T01:49:38.853Z" }, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.3.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, -] - -[package.optional-dependencies] -cublas = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cudart = [ - { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -curand = [ - { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cusolver = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -nvrtc = [ - { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] - -[[package]] -name = "cycler" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, -] - -[[package]] -name = "decorator" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, -] - -[[package]] -name = "depyf" -version = "0.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astor" }, - { name = "dill" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/88/35/83fb0178212279aa0af031031905804c6de5618435d229f41ed21bb9ad2c/depyf-0.20.0.tar.gz", hash = "sha256:fb7683bd72c44f67b56029df2c47721e9a02ffa4d7b19095f1c54c4ebf797a98", size = 6168761, upload-time = "2025-10-13T12:33:38.589Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/65/4df6936130b56e1429114e663e7c1576cf845f3aef1b2dd200c0a5d19dba/depyf-0.20.0-py3-none-any.whl", hash = "sha256:d31effad4261cebecb58955d832e448ace88f432328f95f82fd99c30fd9308d4", size = 39381, upload-time = "2025-10-13T12:33:33.647Z" }, -] - -[[package]] -name = "detect-installer" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/ce/6897d812825e9d4c53e3c7112726e800cc5231b013b2223bf64f653ff362/detect_installer-0.1.0.tar.gz", hash = "sha256:00ad7ba0a36e3cf7d08a40d3643011746dbc112597c7d475cc91c416710ca4e7", size = 3049, upload-time = "2026-02-23T10:40:22.567Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/34/8cc73273414405086c58852916e4031812a6a30fe04c057e37ad99397b7f/detect_installer-0.1.0-py3-none-any.whl", hash = "sha256:034fb20fd665c36e6ba52b8821525ea07fb4f7f938cac459df889fb33801528a", size = 4539, upload-time = "2026-02-23T10:40:23.807Z" }, -] - -[[package]] -name = "dill" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, -] - -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - -[[package]] -name = "dm-tree" -version = "0.1.10" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "attrs" }, - { name = "numpy" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/66/a3ec619d22b6baffa5ab853e8dc6ec9d0c837127948af59bb15b988d7312/dm_tree-0.1.10.tar.gz", hash = "sha256:22f37b599e01cc3402a17f79c257a802aebd8d326de05b54657650845956208a", size = 35748, upload-time = "2026-03-31T17:35:39.03Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/a1/17e0d68eec978c483db4712b14d083ee01484381b29ea85edb2b20210bd0/dm_tree-0.1.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94af18e4fd22ce69eccae89eeed8ed498b6b4cc4957f4ed10b4160e59f620e1d", size = 315976, upload-time = "2026-03-31T17:35:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/cc/6f/ed603715fbc29c887a8985252e2cfe0d449497aea96bac51010159771617/dm_tree-0.1.10-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b442a0c1e9d0960e0314a2e4af81fd328a87921b6d6db6dc41bfa420536884d6", size = 184053, upload-time = "2026-03-31T17:35:16.512Z" }, - { url = "https://files.pythonhosted.org/packages/83/eb/1d55c679cee9a54e552480d308535753c72e2250cf720d7aa777bff2a4fe/dm_tree-0.1.10-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:012c2b376e88d3685c73a4b5c23be41fe933e14e380dcd90172971690b0e02d2", size = 186506, upload-time = "2026-03-31T17:35:17.593Z" }, - { url = "https://files.pythonhosted.org/packages/89/2d/adef6924f8dc7f1665eea4ce066387820c14a629d0e1005568892d56ea6a/dm_tree-0.1.10-cp312-cp312-win_amd64.whl", hash = "sha256:da8d5b8995bea1b6bb93f457e0dad5d16e6e2344a6488ced55320e7f3fd50f56", size = 112708, upload-time = "2026-03-31T17:35:18.699Z" }, -] - -[[package]] -name = "dnspython" -version = "2.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, -] - -[[package]] -name = "docstring-parser" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, -] - -[[package]] -name = "durationpy" -version = "0.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/a4/e44218c2b394e31a6dd0d6b095c4e1f32d0be54c2a4b250032d717647bab/durationpy-0.10.tar.gz", hash = "sha256:1fa6893409a6e739c9c72334fc65cca1f355dbdd93405d30f726deb5bde42fba", size = 3335, upload-time = "2025-05-17T13:52:37.26Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/0d/9feae160378a3553fa9a339b0e9c1a048e147a4127210e286ef18b730f03/durationpy-0.10-py3-none-any.whl", hash = "sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286", size = 3922, upload-time = "2025-05-17T13:52:36.463Z" }, -] - -[[package]] -name = "einops" -version = "0.8.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/77/850bef8d72ffb9219f0b1aac23fbc1bf7d038ee6ea666f331fa273031aa2/einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827", size = 56261, upload-time = "2026-01-26T04:13:17.638Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193", size = 65638, upload-time = "2026-01-26T04:13:18.546Z" }, -] - -[[package]] -name = "email-validator" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dnspython" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, -] - -[[package]] -name = "equinox" -version = "0.11.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jax" }, - { name = "jaxtyping" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/48/ed7ba0849bea18b8be49f471b8e597ca6e5975f75ed05c3be84c1e0ef7c4/equinox-0.11.7.tar.gz", hash = "sha256:96e0216a9d822ec4b1465b0cbfbab14a36fb7e7d62c55f521287db3aaaa251be", size = 139909, upload-time = "2024-09-18T17:10:15.415Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/3f/2fa4627a902a38cd836aa8f7e5a0753114bb23d6b4d2a73784dd9f5ff2e6/equinox-0.11.7-py3-none-any.whl", hash = "sha256:1177354e1795061fbad71535fe54096d8887dc059b4ab7d400fea2d47dfaa283", size = 178416, upload-time = "2024-09-18T17:10:13.699Z" }, -] - -[[package]] -name = "etils" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/ce/6e067242fde898841922ac6fc82b0bb2fe35c38e995880bdffdfbe30182a/etils-1.14.0.tar.gz", hash = "sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4", size = 108127, upload-time = "2026-03-04T17:41:36.291Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/3d/589663aeeacd59bb2f3e8596bfd3e81cf0fb18d70bb433199041f469771b/etils-1.14.0-py3-none-any.whl", hash = "sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4", size = 172934, upload-time = "2026-03-04T17:41:35.01Z" }, -] - -[package.optional-dependencies] -epath = [ - { name = "fsspec" }, - { name = "typing-extensions" }, - { name = "zipp" }, -] -epy = [ - { name = "typing-extensions" }, -] - -[[package]] -name = "fastapi" -version = "0.136.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "pydantic" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/2d/ff8d91d7b564d464629a0fd50a4489c97fcb836ac230bf3a7269232a9b1f/fastapi-0.136.3.tar.gz", hash = "sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab", size = 396410, upload-time = "2026-05-23T18:53:15.192Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl", hash = "sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620", size = 117481, upload-time = "2026-05-23T18:53:16.924Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "email-validator" }, - { name = "fastapi-cli", extra = ["standard"] }, - { name = "fastar" }, - { name = "httpx" }, - { name = "jinja2" }, - { name = "pydantic-extra-types" }, - { name = "pydantic-settings" }, - { name = "python-multipart" }, - { name = "uvicorn", extra = ["standard"] }, -] - -[[package]] -name = "fastapi-cli" -version = "0.0.32" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "rich-toolkit" }, - { name = "typer" }, - { name = "uvicorn", extra = ["standard"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/eb/3b534c6f8e157f9ddbf2a153512307c886cad0b258739c200dd8ff8c4452/fastapi_cli-0.0.32.tar.gz", hash = "sha256:38024d2345275e1b37ce8848727a580d84901b570e96b3256d9d36a9a5039424", size = 26636, upload-time = "2026-07-16T12:16:58.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/53/56ae5ae17bb0a5d89d1d31e5320eb1865553ebbfbde91cdc4c221245f2a8/fastapi_cli-0.0.32-py3-none-any.whl", hash = "sha256:8dcc286fa32f01bbd3f65dd09cfd5a2540ed5f2230b77db7fd30978d6165f3c4", size = 14670, upload-time = "2026-07-16T12:16:57.297Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "fastapi-cloud-cli" }, - { name = "uvicorn", extra = ["standard"] }, -] - -[[package]] -name = "fastapi-cloud-cli" -version = "0.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "detect-installer" }, - { name = "fastar" }, - { name = "httpx" }, - { name = "pydantic", extra = ["email"] }, - { name = "rich-toolkit" }, - { name = "rignore" }, - { name = "sentry-sdk" }, - { name = "typer" }, - { name = "uvicorn", extra = ["standard"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/00/dc/63aaf9913f455e39a7027c27140edd887a87d47d65ac43532d77a51718e5/fastapi_cloud_cli-0.23.0.tar.gz", hash = "sha256:840895bb8d14309aeffc905e0dcd1334d18c6f5da54b735413a8f1cb385e581e", size = 95295, upload-time = "2026-07-28T14:03:33.463Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/96/7e9aba6fabce3cb05f320abeef5b81efd5134823ae85d1a517872cb83cbc/fastapi_cloud_cli-0.23.0-py3-none-any.whl", hash = "sha256:1cd2ffa56e92e92c1fc63acc426c214dd928cbeed2a4c7c6a9a5fc85ea73de16", size = 78058, upload-time = "2026-07-28T14:03:34.386Z" }, -] - -[[package]] -name = "fastar" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/0f/0aeb3fc50046617702acc0078b277b58367fd62eb727b9ec733ae0e8bbcc/fastar-0.11.0.tar.gz", hash = "sha256:aa7f100f7313c03fdb20f1385927ba95671071ba308ad0c1763fef295e1895ce", size = 70238, upload-time = "2026-04-13T17:11:17.143Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/06/a5773706afc8bd496769786590bbc56d2d0ee419a299cc12ea3f5717fcf3/fastar-0.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3c51f1c2cdddbd1420d2897ace7738e36c65e17f6ae84e0bfe763f8d1068bb97", size = 708394, upload-time = "2026-04-13T17:09:57.269Z" }, - { url = "https://files.pythonhosted.org/packages/cc/a6/d5e2a4e48495616440a21eed07558219ca90243ad00b0502586f95bd4833/fastar-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0d9d6b052baf5380baea866675dab6ccd04ec2460d12b1c46f10ce3f4ee6a820", size = 628417, upload-time = "2026-04-13T17:09:42.145Z" }, - { url = "https://files.pythonhosted.org/packages/ab/69/9816d69ac8265c9e50456637a487ccfb7a9c566efd9dbcd673df9c2558c2/fastar-0.11.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bd2f05666d4df7e14885b5c38fefd92a785917387513d33d837ff42ec143a22f", size = 863950, upload-time = "2026-04-13T17:09:11.506Z" }, - { url = "https://files.pythonhosted.org/packages/5b/0d/f88daad53aff2e754b6b5ff2a7113f72447a34f6ef17cc23ca99988117b7/fastar-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e6e74aba1ae77ca4aedcaf1697cd413319f4c88a5ccbe5b42c709517c5097e", size = 760737, upload-time = "2026-04-13T17:07:55.958Z" }, - { url = "https://files.pythonhosted.org/packages/2f/a6/82ef4ecd969d50d92ed3ed9dbd8fe77faa24be5e5736f716edc9f4ce8d62/fastar-0.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38ef77fe940bbc9b37a98bd838727f844b11731cd39358a2640ff864fb385086", size = 757603, upload-time = "2026-04-13T17:08:10.623Z" }, - { url = "https://files.pythonhosted.org/packages/03/35/50249f0d827251f8ac511495e2eacccebda80a00a0ad73e9615b8113b84f/fastar-0.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8955e61b32d6aff82c983217abf80933fd823b0e727586fc72f08043d996fd59", size = 923952, upload-time = "2026-04-13T17:08:25.526Z" }, - { url = "https://files.pythonhosted.org/packages/7b/d8/faee41659e9c379d906d24eaee6d6833ac8cfef0a5df480e5c2a8d3efb33/fastar-0.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:483532442cdb08fbff0169510224eae0836f2f672cea6aacb52847d90fefdc46", size = 816574, upload-time = "2026-04-13T17:08:56.076Z" }, - { url = "https://files.pythonhosted.org/packages/22/47/0448ea7992b997dad2bf004bfd98eca74b5858630eae080b50c7b17d9ddc/fastar-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef5a6071121e05d8287fc75bccb054bcbac8bb0501200a0c0a8feeace5303ea4", size = 819382, upload-time = "2026-04-13T17:09:26.66Z" }, - { url = "https://files.pythonhosted.org/packages/33/ef/0d63eb43586831b7a6f8b22c4d77125a7c594423af1f4f090fa9541b9b40/fastar-0.11.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e45e598af5afe8412197d4786efd6cf29be02e7d3d4f6a3461149eae5d7e94f1", size = 885254, upload-time = "2026-04-13T17:08:40.9Z" }, - { url = "https://files.pythonhosted.org/packages/01/25/edd584675d69e49a165052c3ee886df1c5d574f3e7d813c990306387c623/fastar-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2e160919b1c47ddb8538e7e8eb4cd527281b40f0bf75110a75993838ef61f286", size = 971239, upload-time = "2026-04-13T17:10:12.997Z" }, - { url = "https://files.pythonhosted.org/packages/a5/37/e8bb24f506ba2b08fbaf36c5800e843bd4d542954e9331f00418e2d23349/fastar-0.11.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4bb4dc0fc8f7a6807febcebce8a2f3626ba4955a9263d81ecc630aad83be84c0", size = 1035185, upload-time = "2026-04-13T17:10:30.207Z" }, - { url = "https://files.pythonhosted.org/packages/9a/bf/be753736296338149ee4cb3e92e2b5423d6ba17c7b951d15218fd7e99bbf/fastar-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4ec95af56aa173f6e320e1183001bf108ba59beaf13edd1fc8200648db203588", size = 1072191, upload-time = "2026-04-13T17:10:47.072Z" }, - { url = "https://files.pythonhosted.org/packages/d2/cd/a81c1aaafb5a22ce57c98ae22f39c89413ed53e4ee6e1b1444b0bd666a6c/fastar-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:136cf342735464091c39dc3708168f9fdeb9ebea40b1ead937c61afaf46143d9", size = 1028054, upload-time = "2026-04-13T17:11:04.293Z" }, - { url = "https://files.pythonhosted.org/packages/ec/88/1ce4eed3d70627c95f49ca017f6bbbf2ddcc4b0c601d293259de7689bc20/fastar-0.11.0-cp312-cp312-win32.whl", hash = "sha256:35f23c11b556cc4d3704587faacbc0037f7bdf6c4525cd1d09c70bda4b1c6809", size = 454198, upload-time = "2026-04-13T17:11:45.168Z" }, - { url = "https://files.pythonhosted.org/packages/8f/1d/26ce92f4331cd61a69840db9ca6115829805eec24f285481a854f578e917/fastar-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:920bc56c3c0b8a8ca492904941d1883c1c947c858cd93343356c29122a38f44c", size = 486697, upload-time = "2026-04-13T17:11:31.084Z" }, - { url = "https://files.pythonhosted.org/packages/ed/96/e6eda4480559c69b05d466e7b5ea9170e81fef3795a73e059959a3258319/fastar-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:395248faf89e8a6bd5dc1fd544c8465113b627cb6d7c8b296796b60ebea33593", size = 462591, upload-time = "2026-04-13T17:11:20.577Z" }, -] - -[[package]] -name = "fastsafetensors" -version = "0.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/07/4c/f17bd54c933fd23648ce1e4272adf27d8d7e99b788c8859544bbd39f02e7/fastsafetensors-0.3.3.tar.gz", hash = "sha256:ba4fb59be8a6adbc91723848c3c6f57a9a9a5d2247d9768f84511380d01e554c", size = 77817, upload-time = "2026-07-07T07:21:47.121Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/3a/f95f7fc099ac1fc4c22aa46257d159eac88e29dd0765d21c4fc91caedb01/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c2f788a936ffd17938484360339645812e14cf1b2bdf4c18c035c713218e5a9", size = 1887326, upload-time = "2026-07-07T07:21:34.624Z" }, - { url = "https://files.pythonhosted.org/packages/92/8c/e3347b2a44a8ab9aced94fa450df4f309baa21f7f2981a8a7bd6a977f4d3/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3587bc66b8dec560ad903becf9540889013d4d47f0e10cf45f19bedc7b7bffa7", size = 1915478, upload-time = "2026-07-07T07:21:35.901Z" }, - { url = "https://files.pythonhosted.org/packages/80/65/388a55e6b2b3023fb732843335de14803f16a6d92f0cd47f1125d28b77ac/fastsafetensors-0.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:39c252a5528fa8653366f979d2ab8fa81159db4456b7c72cb5c288adb7699079", size = 424934, upload-time = "2026-07-07T07:21:37.216Z" }, -] - -[[package]] -name = "filelock" -version = "3.32.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, -] - -[[package]] -name = "flashinfer-python" -version = "0.6.16.post3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "click" }, - { name = "cuda-python" }, - { name = "cuda-tile" }, - { name = "einops" }, - { name = "nccl4py" }, - { name = "ninja" }, - { name = "numpy" }, - { name = "nvidia-cudnn-frontend" }, - { name = "nvidia-cutlass-dsl" }, - { name = "nvidia-ml-py" }, - { name = "packaging" }, - { name = "requests" }, - { name = "tabulate" }, - { name = "torch" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c2/46/17045cb47b7b93fcb4e5303e398e6b129b86239930ac0875942a83d8b962/flashinfer_python-0.6.16.post3.tar.gz", hash = "sha256:9b146cfcc1454c80f3f99444db48013b3eadaeb32f2a399bd76c9471ecb72f04", size = 11076656, upload-time = "2026-08-08T01:14:16.408Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/dc/5367cb601fc9190cc75b4c141f5f7e9a224d1c26a1e983151823e02a25b3/flashinfer_python-0.6.16.post3-py3-none-any.whl", hash = "sha256:caf686b9b079abe1c9d65ab505698bd325e8072de40afd822f2c74f2ac3bc601", size = 15836034, upload-time = "2026-08-08T01:14:13.709Z" }, -] - -[[package]] -name = "flax" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jax" }, - { name = "msgpack" }, - { name = "numpy" }, - { name = "optax" }, - { name = "orbax-checkpoint" }, - { name = "pyyaml" }, - { name = "rich" }, - { name = "tensorstore" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e6/d8/767804f923b473790e283b6e5d0812d792195e140cacfad08a2760e0afe8/flax-0.10.0.tar.gz", hash = "sha256:8025607abc6077e29554af00996041454e89d10d6d2939d7cb4b86ecbc3fe360", size = 3623416, upload-time = "2024-10-16T23:26:25.812Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/00/c19d0108f3ce4da076f41e59019149d53f3319ee96a3db880c5f01ba40ac/flax-0.10.0-py3-none-any.whl", hash = "sha256:b3025c5c7d33ba400e123656ad66b0a1900ec67685365bf322ffa47c40a1d16c", size = 420218, upload-time = "2024-10-16T23:26:23.225Z" }, -] - -[[package]] -name = "fonttools" -version = "4.66.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/87/b6/126c659ab7e0e03e01a5f5d223abf7b2c0691ae92718085a212a3924a2a3/fonttools-4.66.1.tar.gz", hash = "sha256:64967c6ddb0d4c610dfd8cb1485981b2d27972ddfb7d4bbbd9e199d2a089c450", size = 3694174, upload-time = "2026-09-29T16:11:53.706Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/1b/fcb22638f2f5918c855abfbab203701e4b03739953d26a59f3e6e59b8f30/fonttools-4.66.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8526b2b7ec4db6b81efb83438be52b1264eda9a4994d867163cfe8c65581ce8d", size = 3097433, upload-time = "2026-09-29T16:10:07.869Z" }, - { url = "https://files.pythonhosted.org/packages/7f/0d/f51141407f9a64efc9fb39b94b0194c4a99c1ffff50834e7ca7a3f1cdd53/fonttools-4.66.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6946fe7bfb28590a1fd4061a17609c9a843952deb65dcf30d1fe725070c3e7a4", size = 2587566, upload-time = "2026-09-29T16:10:10.099Z" }, - { url = "https://files.pythonhosted.org/packages/75/c0/5810d73f9102eb1a08f26b8f7a6c22498622b43e05e684cd5ec602be8ffd/fonttools-4.66.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09ae73bd219e1245debd8376077a0fa6e03175e255c4f51bae5f6a271bfe384a", size = 5416460, upload-time = "2026-09-29T16:10:12.182Z" }, - { url = "https://files.pythonhosted.org/packages/a6/6e/babde908b879a3b51ffc230919d06a804912559dd4cf8c94f3fa2af69fec/fonttools-4.66.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b8ff9e0edbcee2fbf7dff0c41b9041c1901c26acf64e23adb67495012df11de", size = 5397326, upload-time = "2026-09-29T16:10:14.133Z" }, - { url = "https://files.pythonhosted.org/packages/e6/98/8522cc7a5e5ad64a2b2b6e9598489809ed4956c532b887c65b131b8500b7/fonttools-4.66.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38ce8f5fbd5c17dd2153d47d7c8d4108f3deda3f2b4a79b60ddc470a58faded3", size = 5353857, upload-time = "2026-09-29T16:10:16.375Z" }, - { url = "https://files.pythonhosted.org/packages/15/f9/ab87d67c23178886e57f24397b3113b4ac35297f086467c2c4d4231671a8/fonttools-4.66.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8aed2bbcd6216253ef1b015763593365ee8084f621dfa53bb957c19d5e05f7cd", size = 5517604, upload-time = "2026-09-29T16:10:18.791Z" }, - { url = "https://files.pythonhosted.org/packages/d4/20/e126062610aea31919310b0b3de4d0bbe73ccbdcb3d0409cc52027db93ee/fonttools-4.66.1-cp312-cp312-win32.whl", hash = "sha256:9ea6c93091cbf83161a544388746a0911550bd98cb911faca3591cf5ead166ac", size = 2435084, upload-time = "2026-09-29T16:10:21.145Z" }, - { url = "https://files.pythonhosted.org/packages/f0/af/5c245a0587e5b7b3dd209f640b9f70d8e63404ad8dc93819ba578d685985/fonttools-4.66.1-cp312-cp312-win_amd64.whl", hash = "sha256:261d8dc95845e751f975fe8d6075600593ee253470d46d1b84801688051b09f6", size = 2486987, upload-time = "2026-09-29T16:10:22.924Z" }, - { url = "https://files.pythonhosted.org/packages/f6/10/d45b74135d5d642cb3a4fb0a957c1613ef93de4c8548671dfc3a5bf38299/fonttools-4.66.1-py3-none-any.whl", hash = "sha256:7234ae9e28db64273fbbfa72caebd0a97e3bdba6b05064114741b9539ef339d0", size = 1202222, upload-time = "2026-09-29T16:11:51.678Z" }, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, - { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, - { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, - { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, - { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, - { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, - { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, - { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, - { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, - { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, - { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, - { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, - { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, - { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, - { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, -] - -[[package]] -name = "fsspec" -version = "2026.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, -] - -[[package]] -name = "gast" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/91/f6/e73969782a2ecec280f8a176f2476149dd9dba69d5f8779ec6108a7721e6/gast-0.7.0.tar.gz", hash = "sha256:0bb14cd1b806722e91ddbab6fb86bba148c22b40e7ff11e248974e04c8adfdae", size = 33630, upload-time = "2025-11-29T15:30:05.266Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/33/f1c6a276de27b7d7339a34749cc33fa87f077f921969c47185d34a887ae2/gast-0.7.0-py3-none-any.whl", hash = "sha256:99cbf1365633a74099f69c59bd650476b96baa5ef196fec88032b00b31ba36f7", size = 22966, upload-time = "2025-11-29T15:30:03.983Z" }, -] - -[[package]] -name = "google-api-core" -version = "2.33.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "googleapis-common-protos" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/87/62/8fb1fb647d2788c950d69d6a769cd9d55c918ac1fc57be2f90b7e4029787/google_api_core-2.33.0.tar.gz", hash = "sha256:3a36bcc3e319783f4c97da41f6f45ea6ffcaa55848e341de16e09cb70243c2bb", size = 181607, upload-time = "2026-07-22T16:28:28.027Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/31/5056a347bb934ea04583c8b27916ef1501729c72638629545bce26ff4223/google_api_core-2.33.0-py3-none-any.whl", hash = "sha256:a2e22a0c1d0f03eafff1858b38cf46f832d5902b0c052235bf0ab8402929fbdc", size = 176462, upload-time = "2026-07-22T16:28:22.447Z" }, -] - -[[package]] -name = "google-auth" -version = "2.56.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyasn1-modules" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c7/33/dbc946a407401b975f0719658f18e664ece2109f79ffd1ff3bf226c205f4/google_auth-2.56.2.tar.gz", hash = "sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051", size = 365820, upload-time = "2026-07-21T21:53:28.188Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/63/50636aae68c9bf17c891c7eb18b49baa9bd6b31d2a97b8de4813a9fc8d1c/google_auth-2.56.2-py3-none-any.whl", hash = "sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6", size = 258588, upload-time = "2026-07-21T21:53:26.399Z" }, -] - -[[package]] -name = "google-cloud-core" -version = "2.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a8/dd/1eef226e470369b26824a505c34482c0b493bc35fe8e0c6b003b5feca21a/google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83", size = 36001, upload-time = "2026-05-07T08:04:04.124Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390, upload-time = "2026-05-07T08:02:34.672Z" }, -] - -[[package]] -name = "google-cloud-storage" -version = "3.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e3/25/355ed97c1723c787dfaa888808d55db18371f82c38ff862357b1e902cd19/google_cloud_storage-3.13.0.tar.gz", hash = "sha256:d11d8706ea1520fba0f21043bcb7897caf7015d76ce1ad9a4f60237e4d7a9f6c", size = 17340960, upload-time = "2026-07-13T19:10:07.524Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/e8/b3678a0931ee7d4b3fdaf0813e6206d66e0922b2c26d912f308728b5b95a/google_cloud_storage-3.13.0-py3-none-any.whl", hash = "sha256:648af3ef8a6acc674e1359d3c920c67eb89a7a5ab66b336bd3ac43fed6b5ab84", size = 341428, upload-time = "2026-07-13T19:09:52.39Z" }, -] - -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, - { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, -] - -[[package]] -name = "google-resumable-media" -version = "2.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-crc32c" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/f8/1ca5781d6be9cb9f73f7d40f4958c4bd1226a60598e3e39e1d6aaf838c4b/google_resumable_media-2.10.0.tar.gz", hash = "sha256:e324bc9d0fdae4c52a08ae90456edc4e71ece858399e1217ac0eb3a51d6bc6ee", size = 2164570, upload-time = "2026-06-03T16:14:26.103Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/d8/00c6854ac1512bb9eaf13bd3f8f28222f7674947fc510a4ff7616f2efc80/google_resumable_media-2.10.0-py3-none-any.whl", hash = "sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c", size = 81533, upload-time = "2026-06-03T16:13:12.51Z" }, -] - -[[package]] -name = "google-vizier" -version = "0.1.21" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "attrs" }, - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "grpcio-tools" }, - { name = "numpy" }, - { name = "portpicker" }, - { name = "protobuf" }, - { name = "sqlalchemy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/25/38c4cd3a3cc9786fd87998016a709cae88ebacc11eb3b5395737b5eccebb/google-vizier-0.1.21.tar.gz", hash = "sha256:b0ef8514675ee618ae126c4c418c7c38c2934f4f0ddfb1a3670c01d6499e17f3", size = 515482, upload-time = "2025-01-07T21:02:04.894Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/b6/00c7d87a565d98727dcd857e65b3bf6e2c09d060e680fbea6e44abc90fc5/google_vizier-0.1.21-py3-none-any.whl", hash = "sha256:ae457cb42ab55f531223dcbc2b973f2050850eba40808d4927afb3ab0d7e4d2c", size = 799598, upload-time = "2025-01-07T21:02:03.047Z" }, -] - -[package.optional-dependencies] -jax = [ - { name = "chex" }, - { name = "equinox" }, - { name = "flax" }, - { name = "jax" }, - { name = "jaxlib" }, - { name = "jaxopt" }, - { name = "jaxtyping" }, - { name = "optax" }, - { name = "tfp-nightly", extra = ["jax"] }, - { name = "typeguard" }, -] - -[[package]] -name = "googleapis-common-protos" -version = "1.75.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, -] - -[[package]] -name = "grpcio" -version = "1.83.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, - { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, - { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, - { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, - { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, - { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, - { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, - { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" }, - { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" }, -] - -[[package]] -name = "grpcio-tools" -version = "1.81.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/b3/1c5951352d6777fd7f99a0ccee04617fdfd8a5dbf2918a1f58c8b2b280b8/grpcio_tools-1.81.1.tar.gz", hash = "sha256:a22a3870180927fdd84e2b27d079ef5b7f5f8c6110181b6736afc17a463481f1", size = 6236155, upload-time = "2026-06-11T12:51:21.235Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/8a/824a9ca20bcdce8a568bb8c9f98bfeb7fad62129235e6d2ae7576fd1250a/grpcio_tools-1.81.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:353b1fafcc739c31ed42271052709595b340d34f27c459beeb78a32938305bb5", size = 2585927, upload-time = "2026-06-11T12:50:05.671Z" }, - { url = "https://files.pythonhosted.org/packages/2f/35/e5f9f671378b1b89a896150d3e4fa2c6ec61a5e1e9e5107ce4c140ccc931/grpcio_tools-1.81.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:768f584c2423cbeb6cb6867817a39365b987ff16b8259a3adbc6546b9e303a4e", size = 5815665, upload-time = "2026-06-11T12:50:08.178Z" }, - { url = "https://files.pythonhosted.org/packages/c6/02/631b628e4072e988c669bd8f1b2406ef3c9a4cfcb2625bbf2a308a07b71d/grpcio_tools-1.81.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1680b35a84f4694401819ac4acac42dda6dbc7bb8fc74112fd1a60425a07adf4", size = 2635518, upload-time = "2026-06-11T12:50:10.391Z" }, - { url = "https://files.pythonhosted.org/packages/de/7c/2e3537e3ea3d1c0ddd6766cf6a7c62b487d89fb005713df2781d5f21483a/grpcio_tools-1.81.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f64e665c8ec639278ecf009beb92cbdcc5994f617c1af3d58036e1f70b1423ec", size = 2958252, upload-time = "2026-06-11T12:50:12.677Z" }, - { url = "https://files.pythonhosted.org/packages/35/68/14013cb2942bdac354746b643b4c37dd91906da8dce00f41c616e88bf33d/grpcio_tools-1.81.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba8f1ae82ad199f43448995715445cc623fb20d3882382e4be61f0da8ccb3f0e", size = 2698439, upload-time = "2026-06-11T12:50:15.017Z" }, - { url = "https://files.pythonhosted.org/packages/bd/45/000c14c0338a7ad36054b9f17ea41842deb7841c05c067dd36cc831bc0f4/grpcio_tools-1.81.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7b6d1e986d5923751bfe2b5cca9c4cb3d5653446e4fa4aacd438033e2dc360a", size = 3152160, upload-time = "2026-06-11T12:50:17.3Z" }, - { url = "https://files.pythonhosted.org/packages/41/97/881930ca3967d2c8a95649bea8ebc991a7cf2331bc96679fd3600450dccc/grpcio_tools-1.81.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7f208c207aca639dcb34648d3826c38d7cf3485118fb2065117e9fc4827406b3", size = 3710468, upload-time = "2026-06-11T12:50:19.479Z" }, - { url = "https://files.pythonhosted.org/packages/c7/b5/67baeba7366162652cdc1dbd962289accde07241bc8f42f6f02b305efcc6/grpcio_tools-1.81.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:724ecb69af63d2f6d4ccea3e6fa0ca110ed9c5824d48c2f887c631bbb03c1c3c", size = 3370797, upload-time = "2026-06-11T12:50:21.501Z" }, - { url = "https://files.pythonhosted.org/packages/f2/5d/34f2dce2125ccb107e32b57f5a9c1257edcc0793b0d2fef1e8b13a6bac3c/grpcio_tools-1.81.1-cp312-cp312-win32.whl", hash = "sha256:895a6782cec86beac71ccebb4b9848259c6f04a3028b8e42fa8d40cfe5146593", size = 1008453, upload-time = "2026-06-11T12:50:23.358Z" }, - { url = "https://files.pythonhosted.org/packages/8a/be/09da8256ec8d2a5ce8a1acc51cbbc4ca52a462d78ed3412778440a56502e/grpcio_tools-1.81.1-cp312-cp312-win_amd64.whl", hash = "sha256:0265fd1386b7458302f79542558345880d484f8fa92ae196c0c0268242c5f23a", size = 1174857, upload-time = "2026-06-11T12:50:25.685Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "hf-xet" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9e/27/06d899ea7bd721d272f84aac98bdb238de98af4cc767a69056d967d68c71/hf_xet-1.7.0.tar.gz", hash = "sha256:d406ec79053c0871817f700c2ac8c36ba0d87f9c34b7458b0f0063bb218b0466", size = 985689, upload-time = "2026-10-06T20:18:43.89Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/0b/b03be21ffaada749ba0d3197d8aefbf1aa698bac149580421c15239b299e/hf_xet-1.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e88a7a75d7d95cbee1f37dc31341d6201124cf21c6c4b1dfab8ccba9b09e0f", size = 3796096, upload-time = "2026-10-06T20:18:28.43Z" }, - { url = "https://files.pythonhosted.org/packages/c3/47/a26ebdce7056a61e931f228439bc0ab08cbec239d1690f965e5e637cba79/hf_xet-1.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:59fba37039233c7fcbe196817d6cdcf1b40dfb17b410f229d85b0cf0a1848da4", size = 3560352, upload-time = "2026-10-06T20:18:30.365Z" }, - { url = "https://files.pythonhosted.org/packages/a3/4c/2bf3b66c215d409655f28de1622393dde04c9461280d48c7924bb3b2decd/hf_xet-1.7.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2814a6e999d13464c4d679b788cc5d784eb5a4edfc638a31f10e9a11ab531ef8", size = 4212180, upload-time = "2026-10-06T20:18:32.292Z" }, - { url = "https://files.pythonhosted.org/packages/49/0c/a2f703a5a78267556e89e03316fa0805c86b72b50829bc67665746e8ebf0/hf_xet-1.7.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fcfd6c22418e57dd5b3aea649e813b2e2cfb2aebf317b210d90f1fe4b3018b52", size = 3990011, upload-time = "2026-10-06T20:18:34.21Z" }, - { url = "https://files.pythonhosted.org/packages/a4/77/e52e4201b1cbf571530a61cc57f70182045a39a230089ee5f1df182a4de2/hf_xet-1.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80f79dae613ce9e0ea1fd1ae15616ca9ac74aed4c770aabc199c4f03ebecc863", size = 4190628, upload-time = "2026-10-06T20:18:36.062Z" }, - { url = "https://files.pythonhosted.org/packages/6c/dc/03a21b89f118664a0926ff25b0f8e44a519bf22724a6a8fc7a9abbc188b6/hf_xet-1.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0a9e802f33bf50c851abe45fc5380e61f959e2d369647d6742b79ad9d6c27cab", size = 4418814, upload-time = "2026-10-06T20:18:37.888Z" }, - { url = "https://files.pythonhosted.org/packages/4d/59/b35106dfa71b6eef605dc88bd038fe99c7f86fb132a15b60d0bf2f235b2c/hf_xet-1.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:2b7bb5727889b0f2436dbaaad8fc4c3e66b8240d992716989e0c086b4278b1bc", size = 3822644, upload-time = "2026-10-06T20:18:40.052Z" }, - { url = "https://files.pythonhosted.org/packages/48/cd/072313585f74fe9d441e2eb5e0a4703c30586cd709810ea369675f61b74e/hf_xet-1.7.0-cp38-abi3-win_arm64.whl", hash = "sha256:acc3851cf2576a8fb2ae926da863f4efabe21303cf292e9a44332802ab0dcc6a", size = 3662436, upload-time = "2026-10-06T20:18:42.205Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpcore2" -version = "2.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "h11" }, - { name = "truststore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/a8/20ed1ed79cbc2ecdf5301c0968ab7c85547212e2a7bd126ddd2d986e206e/httpcore2-2.9.1.tar.gz", hash = "sha256:4d8acbf8b306f48c9d6046591fd5ba4037d1b1b1000d140fc2c3eab1e9a0c0e2", size = 67089, upload-time = "2026-07-24T09:21:03.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/fb/46c52b781975c335a2bcf1072c7bbc007cbdc8d674217f5ee1daba2c848b/httpcore2-2.9.1-py3-none-any.whl", hash = "sha256:6182472379e855fe4221246a2bb7ecede403bc61c6798062ae1787d051ccde26", size = 82809, upload-time = "2026-07-24T09:21:01.178Z" }, -] - -[[package]] -name = "httptools" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/88/1d21a36da8f5cb0fa49eafd4b169eba5608d57e75bbcf61845cbc6243216/httptools-0.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d", size = 208247, upload-time = "2026-05-25T22:17:07.843Z" }, - { url = "https://files.pythonhosted.org/packages/a5/42/cc4feea2945cb3051038f090c9b36bd5b8a9d7f5a894a506a8983e33fd1c/httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5", size = 113064, upload-time = "2026-05-25T22:17:09.136Z" }, - { url = "https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2", size = 523851, upload-time = "2026-05-25T22:17:10.106Z" }, - { url = "https://files.pythonhosted.org/packages/b7/e4/f90a0df0b83beff265b7e3b65f2a4cefd95792d4be0ac3e16049f2acd3c2/httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09", size = 518842, upload-time = "2026-05-25T22:17:11.218Z" }, - { url = "https://files.pythonhosted.org/packages/9e/2d/0c9ac76dd2c893841fbf6498d6acec4f2442e1b7067f6e3e316a80e494e8/httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a", size = 501238, upload-time = "2026-05-25T22:17:12.728Z" }, - { url = "https://files.pythonhosted.org/packages/ca/42/906adc91ae3a5fa9c59c0a2f21c139725bd7e5b41ae6acd485cd14123ebf/httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745", size = 509567, upload-time = "2026-05-25T22:17:13.842Z" }, - { url = "https://files.pythonhosted.org/packages/05/0b/4240efeb672751ee5b9b380cb0e3fdc050bc05f68adc7a8aefc4fcd9a69a/httptools-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150", size = 90918, upload-time = "2026-05-25T22:17:15.155Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[[package]] -name = "httpx2" -version = "2.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "httpcore2" }, - { name = "idna" }, - { name = "truststore" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/14/38128fbafd7e0ed41d874df6c9a653d47c2d111cfe59e2b4ac95161b4abd/httpx2-2.9.1.tar.gz", hash = "sha256:1932a768737e3666291582833da748cc4e563c337cf96706fccc04fa6e58764a", size = 95458, upload-time = "2026-07-24T09:21:04.972Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/b8/cfd91c4ab9134d386d48f0b6ac662ff3d4be6efdee59ee1c67ebc3c0487c/httpx2-2.9.1-py3-none-any.whl", hash = "sha256:1820fe14a9ab1107bfeff39259987429450b070ec0ff38cc87eb0d8c97fdc71a", size = 91191, upload-time = "2026-07-24T09:21:02.6Z" }, -] - -[[package]] -name = "huggingface-hub" -version = "1.33.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "httpx" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/2a/484d112c0d8fc5f665d7b65137ac9cdb2953c982391598c3597968a12ee7/huggingface_hub-1.33.0.tar.gz", hash = "sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e", size = 1043548, upload-time = "2026-09-24T09:49:29.971Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/16/963096d224b80909432dc16561a615fd33d2d13beef3ce4c63fa25e40867/huggingface_hub-1.33.0-py3-none-any.whl", hash = "sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899", size = 846435, upload-time = "2026-09-24T09:49:28.059Z" }, -] - -[[package]] -name = "humanize" -version = "4.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/ea/13a1ef3c12d12662905801495283530251918b70d62d368f1d2e0272c70d/humanize-4.16.0.tar.gz", hash = "sha256:7dc2244a2f84a4bfb1d36c37bac80cd78e35cdc5c119206d87b018e1445f3a3f", size = 89515, upload-time = "2026-06-30T16:17:29.859Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/aa/0b7365d30fed43e7a3449aba1fe20a0a7174d9cf13e282af4e69ac825441/humanize-4.16.0-py3-none-any.whl", hash = "sha256:353eb2f34c09d098b2880eee8bef21832eae6d174f48c5762fff7e5fcb74d01d", size = 137209, upload-time = "2026-06-30T16:17:28.36Z" }, -] - -[[package]] -name = "humming-kernels" -version = "0.1.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings" }, - { name = "jinja2" }, - { name = "numpy" }, - { name = "nvidia-ml-py" }, - { name = "safetensors" }, - { name = "tabulate" }, - { name = "torch" }, - { name = "tqdm" }, - { name = "triton" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/b9/53797fff4b1b31b4567453a365646d724469b1bb0d76c07f8746617d0cb6/humming_kernels-0.1.12.tar.gz", hash = "sha256:c96c37c95f74379067d2d2d0c5154d12ee9a8cfa29a7c3370a7f3e34f207ca72", size = 259323, upload-time = "2026-07-31T14:28:23.124Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/0f/01f871dc26f8d7e1683df9464890d48cfd5c4e46c4a264d7ee0868749197/humming_kernels-0.1.12-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:988e8b9da41e3679ae2816bdfb51acab53c05cc56254b5ca98e847f4efcf24fd", size = 318019, upload-time = "2026-07-31T14:28:19.925Z" }, - { url = "https://files.pythonhosted.org/packages/ec/a2/f96912ee7d68f56e61c15555657ffa057b8afbaa21241c63342b5969bb74/humming_kernels-0.1.12-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:cd3ef712a93f3a9075ea99de2c72bcd3ec89dab3759b3a248d869f5507b60331", size = 322656, upload-time = "2026-07-31T14:28:21.511Z" }, -] - -[package.optional-dependencies] -cu13 = [ - { name = "nvidia-cuda-cccl" }, - { name = "nvidia-cuda-nvcc", version = "13.2.86", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, - { name = "nvidia-cuda-nvcc", version = "13.3.73", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, - { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, - { name = "nvidia-cuda-nvrtc", version = "13.3.33", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, - { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, - { name = "nvidia-cuda-runtime", version = "13.3.29", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, -] - -[[package]] -name = "idna" -version = "3.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, -] - -[[package]] -name = "ijson" -version = "3.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3a/06/b31f040a8764336a11152e474a7abcb3782fedb0d1cdf78f442b82878c56/ijson-3.5.1.tar.gz", hash = "sha256:af40bd1a85f55db0b8b30715c858761306bd92d5590148636f75c3309e6e76bd", size = 69913, upload-time = "2026-07-06T17:37:42.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/6e/f3ded1ebb85ccc89a30f7b10a0076f30db70ae1d1e0b6423ff93c57b7539/ijson-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ee60c7741012671867678eae71c51872cac938b76f3d4ca40a778e6c361774d2", size = 88643, upload-time = "2026-07-06T17:36:28.529Z" }, - { url = "https://files.pythonhosted.org/packages/ee/f2/18f14a1d79ef4898e746b4f50dcdbe60abab317cc2bd8390f043b9553c4e/ijson-3.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:11c1d7d36a13054b5872ecd5d745dc4009d9abdbcba2312de69e66c2f92a46d2", size = 60611, upload-time = "2026-07-06T17:36:29.597Z" }, - { url = "https://files.pythonhosted.org/packages/30/c7/6e3e591324fd4c7a7a9e1bc23548bacbd84c0d91766b71f09f13e945e7e9/ijson-3.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b9517efbe6604bce16f3e50d49b0cd1bdc58917f98cf2eab026599c5c0422991", size = 60447, upload-time = "2026-07-06T17:36:30.747Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a5/9af7be670381ddac26dd55107ed0110b50f5161673b053311db67f510dcc/ijson-3.5.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ea4fd7bec203a600b1cc88a492dfe6b75ce4b1b87488a66adcd5406022213f64", size = 139092, upload-time = "2026-07-06T17:36:31.749Z" }, - { url = "https://files.pythonhosted.org/packages/41/fb/f9c1664d75467453e6bd4e5f9cd2211b730b09e049445ab64cbac68cc6a3/ijson-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350caea815e53151994b597abc80cf669454276b5ac6aadcec69ef6d48f7e90b", size = 149921, upload-time = "2026-07-06T17:36:32.912Z" }, - { url = "https://files.pythonhosted.org/packages/43/80/d20b1c49c4aa7cc6644131e2e57192b45346ef4816566ed1cd9fd05bae38/ijson-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4fcebfe1685bb7ba06a8255a5d428ea6b4b895d7acf979cb637d8bbc9db2f47", size = 149848, upload-time = "2026-07-06T17:36:34.032Z" }, - { url = "https://files.pythonhosted.org/packages/fd/fc/5baa710869f5ab939e6233583ced1546889b55c35f35b844c518ac10abc3/ijson-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d78f362f51c8691798758a9e6ac3c9d385ee1228cb82987c91562a2fae235cd3", size = 150810, upload-time = "2026-07-06T17:36:35.19Z" }, - { url = "https://files.pythonhosted.org/packages/54/16/a12b3d987a5c1677b04557c6f9b9feb7e04b7d4171e9a344856cb9136e9b/ijson-3.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0b184180d45f85fd4479659582749b109e49f4a29c21ac700ccc9c2280fe015e", size = 142989, upload-time = "2026-07-06T17:36:36.23Z" }, - { url = "https://files.pythonhosted.org/packages/ed/63/1026c535671fc334fc85aeb78f0945c825e7a338575edc753c0f455459ae/ijson-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e353891d33a2e6aa5caf72c2a5fbadd7a46f5f9b32dcfd0c84113b2444c255b8", size = 151702, upload-time = "2026-07-06T17:36:37.296Z" }, - { url = "https://files.pythonhosted.org/packages/cb/af/b58aa3a2bf4d31c388ea78b49826605f60932891ce97e404d196766b4ea3/ijson-3.5.1-cp312-cp312-win32.whl", hash = "sha256:936f28671f018f8ac4d3f003ae9fa01d0467ab4ef4cfd0c97f23beda485b61c6", size = 52613, upload-time = "2026-07-06T17:36:38.345Z" }, - { url = "https://files.pythonhosted.org/packages/04/66/ce70a92949c2a753dad91fdd5761dc14f3a44517e80cfc3c26612982ed61/ijson-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:322c783f3ee0c6b383bbd4db88370b10172168808cc2a0bf811f1253f7435602", size = 54729, upload-time = "2026-07-06T17:36:39.337Z" }, - { url = "https://files.pythonhosted.org/packages/a5/ff/e17784240c9cf1d58de2f2853ebaf9cc54f6bce117a1f12a6150bbb4a5aa/ijson-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:e2ac204b59f09e38e16d277f906240e9fd38780e42076599419265af183dc4b4", size = 53714, upload-time = "2026-07-06T17:36:40.308Z" }, -] - -[[package]] -name = "interegular" -version = "0.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/9d/8b6dde58a028a3962ce17e84d5fe73758df61378e00ef8ac3d85da34b0ff/interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600", size = 24705, upload-time = "2024-01-06T23:01:22.372Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/01/72d6472f80651673716d1deda2a5bbb633e563ecf94f4479da5519d69d25/interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c", size = 23635, upload-time = "2024-01-06T23:01:20.829Z" }, -] - -[[package]] -name = "isodate" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, -] - -[[package]] -name = "jax" -version = "0.4.38" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jaxlib" }, - { name = "ml-dtypes" }, - { name = "numpy" }, - { name = "opt-einsum" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/e5/c4aa9644bb96b7f6747bd7c9f8cda7665ca5e194fa2542b2dea3ff730701/jax-0.4.38.tar.gz", hash = "sha256:43bae65881628319e0a2148e8f81a202fbc2b8d048e35c7cb1df2416672fa4a8", size = 1930034, upload-time = "2024-12-17T23:03:47.623Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/49/b4418a7a892c0dd64442bbbeef54e1cdfe722dfc5a7bf0d611d3f5f90e99/jax-0.4.38-py3-none-any.whl", hash = "sha256:78987306f7041ea8500d99df1a17c33ed92620c2268c4c3677fb24e06712be64", size = 2236864, upload-time = "2024-12-17T23:03:44.433Z" }, -] - -[[package]] -name = "jaxlib" -version = "0.4.38" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ml-dtypes" }, - { name = "numpy" }, - { name = "scipy" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/df/08b94c593c0867c7eaa334592807ba74495de4be90580f360db8b96221dc/jaxlib-0.4.38-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:3fefea985f0415816f3bbafd3f03a437050275ef9bac9a72c1314e1644ac57c1", size = 99737849, upload-time = "2024-12-17T23:06:20.406Z" }, - { url = "https://files.pythonhosted.org/packages/ab/b1/c9d2a7ba9ebeabb7ac37082f4c466364f475dc7550a79358c0f0aa89fdf2/jaxlib-0.4.38-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f33bcafe32c97a562ecf6894d7c41674c80c0acdedfa5423d49af51147149874", size = 79509242, upload-time = "2024-12-17T23:06:33.73Z" }, - { url = "https://files.pythonhosted.org/packages/53/25/dd670d8bdf3799ece76d12cfe6a6a250ea256057aa4b0fcace4753a99d2d/jaxlib-0.4.38-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:496f45b0e001a2341309cd0c74af0b670537dced79c168cb230cfcc773f0aa86", size = 93251503, upload-time = "2024-12-17T23:06:48.243Z" }, - { url = "https://files.pythonhosted.org/packages/f9/cc/37fce5162f6b9070203fd76cc0f298d9b3bfdf01939a78935a6078d63621/jaxlib-0.4.38-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:dad6c0a96567c06d083c0469fec40f201210b099365bd698be31a6d2ec88fd59", size = 101792792, upload-time = "2024-12-17T23:07:01.615Z" }, - { url = "https://files.pythonhosted.org/packages/6f/7a/8515950a60a4ea5b13cc98fc0a42e36553b2db5a6eedc00d3bd7836f77b5/jaxlib-0.4.38-cp312-cp312-win_amd64.whl", hash = "sha256:966cdec36cfa978f5b4582bcb4147fe511725b94c1a752dac3a5f52ce46b6fa3", size = 64288223, upload-time = "2024-12-17T23:07:15.024Z" }, -] - -[[package]] -name = "jaxopt" -version = "0.8.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jax" }, - { name = "jaxlib" }, - { name = "numpy" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/da/ff7d7fbd13b8ed5e8458e80308d075fc649062b9f8676d3fc56f2dc99a82/jaxopt-0.8.5.tar.gz", hash = "sha256:2790bd68ef132b216c083a8bc7a2704eceb35a92c0fc0a1e652e79dfb1e9e9ab", size = 121709, upload-time = "2025-04-14T17:59:01.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/d8/55e0901103c93d57bab3b932294c216f0cbd49054187ce29f8f13808d530/jaxopt-0.8.5-py3-none-any.whl", hash = "sha256:ff221d1a86908ec759eb1e219ee1d12bf208a70707e961bf7401076fe7cf4d5e", size = 172434, upload-time = "2025-04-14T17:59:00.342Z" }, -] - -[[package]] -name = "jaxtyping" -version = "0.3.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wadler-lindig" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/c1/091b8852bd7cbf50bd655543c8506033cf4029300c67f8c176c1286879a9/jaxtyping-0.3.11.tar.gz", hash = "sha256:b09c14acf6686feb9e0df5b0d8c6e7c5b6f8d36bf059ee54cd522a186c2ef050", size = 46489, upload-time = "2026-06-13T18:35:23.167Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/38/c66bbdc5047f4776c2bd3e47e5295a350e3fa44d5b8942105e71c2a876a0/jaxtyping-0.3.11-py3-none-any.whl", hash = "sha256:8a4bedc4e3f963fa82df41bd13c7ebc2bad925601eb48614c65798f21329d4e3", size = 56593, upload-time = "2026-06-13T18:35:22.01Z" }, -] - -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - -[[package]] -name = "jiter" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, - { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, - { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, - { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, - { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, - { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, - { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, - { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, - { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, - { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, - { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, - { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, - { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, - { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, - { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, - { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, -] - -[[package]] -name = "jmespath" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, -] - -[[package]] -name = "joblib" -version = "1.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, -] - -[[package]] -name = "jsonschema" -version = "4.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rpds-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, -] - -[[package]] -name = "jsonschema-specifications" -version = "2025.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "referencing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, -] - -[[package]] -name = "kiwisolver" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/07/bd78e6a8fae171ea041ef5bba3ed21a003522fa088834b069b1909981f30/kiwisolver-1.5.1.tar.gz", hash = "sha256:f1303ef2eec81262a4b708c3e858afe58d7c75ad91c1c05266eda7673369859a", size = 104395, upload-time = "2026-08-28T10:28:27.153Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/9b/65b302742389c6f96f2956bef5decf26011309feb2fc5d79613af18adea4/kiwisolver-1.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:63fb7294b768f444eb4b068965f2662f28c2fd4161e23bd60fcf3ff27b74c046", size = 123876, upload-time = "2026-08-28T10:25:27.44Z" }, - { url = "https://files.pythonhosted.org/packages/71/74/c21f339956f6f691b2ed7e31d5f3ae767304df6c460192739fc830853051/kiwisolver-1.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0ebdef3eae5336568147c39a55be6a2036ffde53faa9ca2d978989ae7c2da12c", size = 66487, upload-time = "2026-08-28T10:25:28.728Z" }, - { url = "https://files.pythonhosted.org/packages/84/e5/bdb34e21523e01dceda064d63713f3bdec91388af24fba1eca7ea5e85864/kiwisolver-1.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1798e83840c3f627246104c4d8a9639c60fa068adf9ce92b61791781fa8a68c1", size = 64660, upload-time = "2026-08-28T10:25:30.071Z" }, - { url = "https://files.pythonhosted.org/packages/fc/f4/dadfec469313c7f428efa7e84b4aba9732f813c13ea7131a24b7b008ef57/kiwisolver-1.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34633ecf50d16187ab8e5528b7a2530f2feb4e23f300db4672538b51cfc5cd38", size = 1477929, upload-time = "2026-08-28T10:25:31.495Z" }, - { url = "https://files.pythonhosted.org/packages/6f/35/09c58daac34e6f6ea5c6dee0094b422118e5a7c265586008a95fd135ac5f/kiwisolver-1.5.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d27c2123977cb9269c30a49ba45f03a4323017ef693e19db4ec9dbe1299a3002", size = 1278499, upload-time = "2026-08-28T10:25:33.375Z" }, - { url = "https://files.pythonhosted.org/packages/19/32/739765e24fbad29d13f83e546ea4abc215a78cea9d677ca09025b027724d/kiwisolver-1.5.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6a797a1cefc8b9c93170db580337e1fe3d011ad18b1299943231279406342048", size = 1296677, upload-time = "2026-08-28T10:25:35.059Z" }, - { url = "https://files.pythonhosted.org/packages/df/32/03304d1010e2cc45e5b3b52cef7e43fed3a2a5cd6c87a89b4a88e1d85b5d/kiwisolver-1.5.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2551cf9917af48ee7c4b29cc82320489508cf96fd26a51f6fc124de661cd44c7", size = 1346037, upload-time = "2026-08-28T10:25:36.705Z" }, - { url = "https://files.pythonhosted.org/packages/3e/57/4c49377bfd274450dd72ecaa13eaac32ea804a03363e4d1db0c5aa999ceb/kiwisolver-1.5.1-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:38f6e0deb4d0a4615efe0c4efc5990b06ae450ab50a0b321c0b078b6d238c083", size = 988248, upload-time = "2026-08-28T10:25:38.299Z" }, - { url = "https://files.pythonhosted.org/packages/cb/c3/38df144a08b6c5d75ca4504e5cc3141bb3bfef64c04f4ef48204f42711b6/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bfd1de989b3330420e29de39352f5c049905c9e3ee67233a50d550e3d652c148", size = 2228722, upload-time = "2026-08-28T10:25:40.038Z" }, - { url = "https://files.pythonhosted.org/packages/e7/11/3221838a89cd64d9b386353e000cd8a296069a20fbe3584507fdfd5bebae/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1209042a623ddfda5497e4066c7b77651dde8e1d3a9dd97599dc7e97f3b9b78c", size = 2325216, upload-time = "2026-08-28T10:25:41.699Z" }, - { url = "https://files.pythonhosted.org/packages/83/d4/075c219230697bb5db910d37262b9bacf880f92b4811a02ab81ed073a253/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:26e8268480be5061d509e29669d59103c067a26377a56491630ece11762e3858", size = 1977689, upload-time = "2026-08-28T10:25:43.559Z" }, - { url = "https://files.pythonhosted.org/packages/bb/08/1d219c3c2dd960983d0d4da623d916e9de6385df2b0bab3d1af0e9b8fccc/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d79308fa689fac89cbcfbd4dbfc80b5f95c54c5a7fd4d194be221f9d33d026e6", size = 2491443, upload-time = "2026-08-28T10:25:45.242Z" }, - { url = "https://files.pythonhosted.org/packages/ba/d3/024208ec1079d273f1047468d1bdffbf38bb75b7b268090fd3a0301b9d9a/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b03af77d77e50edba2030fd5f7c352ff209314b09030a3cba7c14edf9a09a444", size = 2295200, upload-time = "2026-08-28T10:25:46.984Z" }, - { url = "https://files.pythonhosted.org/packages/6e/7c/7b210498f9f92e1cd7855f260fa69ef056881087b199ee20c208f0e4189a/kiwisolver-1.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:06a6917674de9e0fe3f66f5430787f59a9f2ddb64af9b714eaec547e29ef5c19", size = 70748, upload-time = "2026-08-28T10:25:48.444Z" }, - { url = "https://files.pythonhosted.org/packages/94/61/ef0daa157c8bb23672f7423e0d14c39db1dc6ef8ed47e6bc54c9c1bef3bf/kiwisolver-1.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:ad8b9671348d7c8716715652ae11f85ed0eb99e265a2df2ca490577d69860b2c", size = 68324, upload-time = "2026-08-28T10:25:49.81Z" }, - { url = "https://files.pythonhosted.org/packages/a9/c4/1407df7512a5b36cc79840e01710dc575733c461b13ab866cae77eaf87f3/kiwisolver-1.5.1-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:482676e5bd48d70ac99d9fc78863469845421e01184fa83f1f9366dc49f7e974", size = 134002, upload-time = "2026-08-28T10:28:08.881Z" }, - { url = "https://files.pythonhosted.org/packages/16/45/c37a21ad5c0ab581a93c55ad544721aaa1f0ae94edb29c6a678a23d013e6/kiwisolver-1.5.1-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:072bdb15a3c19a5b5dbc8f8fb1f4e1884bf4f3507eeb4cc6334401274d37a5c0", size = 194292, upload-time = "2026-08-28T10:28:11.06Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c1/69f00d627949580e43d57af0aa465df46868d7c29801c137a55374101294/kiwisolver-1.5.1-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:a5a00665d1a0e26763a7338d7e911d4598fbc1d50dd0d6b7919b7dc6c5d6569f", size = 73362, upload-time = "2026-08-28T10:28:12.449Z" }, -] - -[[package]] -name = "kubernetes" -version = "32.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "durationpy" }, - { name = "google-auth" }, - { name = "oauthlib" }, - { name = "python-dateutil" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "requests-oauthlib" }, - { name = "six" }, - { name = "urllib3" }, - { name = "websocket-client" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/e8/0598f0e8b4af37cd9b10d8b87386cf3173cb8045d834ab5f6ec347a758b3/kubernetes-32.0.1.tar.gz", hash = "sha256:42f43d49abd437ada79a79a16bd48a604d3471a117a8347e87db693f2ba0ba28", size = 946691, upload-time = "2025-02-18T21:06:34.148Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/10/9f8af3e6f569685ce3af7faab51c8dd9d93b9c38eba339ca31c746119447/kubernetes-32.0.1-py2.py3-none-any.whl", hash = "sha256:35282ab8493b938b08ab5526c7ce66588232df00ef5e1dbe88a419107dc10998", size = 1988070, upload-time = "2025-02-18T21:06:31.391Z" }, -] - -[[package]] -name = "lark" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, -] - -[[package]] -name = "lazy-loader" -version = "0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/ac/21a1f8aa3777f5658576777ea76bfb124b702c520bbe90edf4ae9915eafa/lazy_loader-0.5.tar.gz", hash = "sha256:717f9179a0dbed357012ddad50a5ad3d5e4d9a0b8712680d4e687f5e6e6ed9b3", size = 15294, upload-time = "2026-03-06T15:45:09.054Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/a1/8d812e53a5da1687abb10445275d41a8b13adb781bbf7196ddbcf8d88505/lazy_loader-0.5-py3-none-any.whl", hash = "sha256:ab0ea149e9c554d4ffeeb21105ac60bed7f3b4fd69b1d2360a4add51b170b005", size = 8044, upload-time = "2026-03-06T15:45:07.668Z" }, -] - -[[package]] -name = "librosa" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "audioread" }, - { name = "decorator" }, - { name = "joblib" }, - { name = "lazy-loader" }, - { name = "msgpack" }, - { name = "numba" }, - { name = "numpy" }, - { name = "pooch" }, - { name = "scikit-learn" }, - { name = "scipy" }, - { name = "soundfile" }, - { name = "soxr" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/36/360b5aafa0238e29758729e9486c6ed92a6f37fa403b7875e06c115cdf4a/librosa-0.11.0.tar.gz", hash = "sha256:f5ed951ca189b375bbe2e33b2abd7e040ceeee302b9bbaeeffdfddb8d0ace908", size = 327001, upload-time = "2025-03-11T15:09:54.884Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/ba/c63c5786dfee4c3417094c4b00966e61e4a63efecee22cb7b4c0387dda83/librosa-0.11.0-py3-none-any.whl", hash = "sha256:0b6415c4fd68bff4c29288abe67c6d80b587e0e1e2cfb0aad23e4559504a7fa1", size = 260749, upload-time = "2025-03-11T15:09:52.982Z" }, -] - -[[package]] -name = "llguidance" -version = "1.7.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/91/6bc8bb503dc259e46d253b5424385a54fe06c38a4c7a12befe69a3c2455a/llguidance-1.7.6.tar.gz", hash = "sha256:db7febbe412ed2015501904646750071d7e00e6df7f85c4b956ad4f206fd2df7", size = 1156574, upload-time = "2026-06-03T20:13:25.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/1d/5a9a13421b1f3f1c1acf82beb63ed72fa4d302e65099b72f4a4fe5a098ab/llguidance-1.7.6-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:eabf4572c8731734c0444c353b9ea06bc5c156986d2ff0a4ec0499159271381f", size = 3227892, upload-time = "2026-06-03T20:13:09.533Z" }, - { url = "https://files.pythonhosted.org/packages/46/fe/bb185f11bad82f2637e3cd8cbf6b200cbb6ed56ac395de47ea05a60d4649/llguidance-1.7.6-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:9c54c899db8cb4b4fba128a7d844730066576c70d806c95ada92b2bd2d6ab498", size = 3138127, upload-time = "2026-06-03T20:13:11.649Z" }, - { url = "https://files.pythonhosted.org/packages/51/b9/dc76d7716e04dc7b3427cae52eaa32bd20771382d4d1dd9f4538a9dd2086/llguidance-1.7.6-cp39-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:e70fa25ed550c2b50c2fd70baa9e2808b4ecb859d01e453bd5459aff62ba38c3", size = 2899993, upload-time = "2026-06-03T20:13:13.563Z" }, - { url = "https://files.pythonhosted.org/packages/1a/64/d74336f22242ef94356a456057d4ff1be7c1bc9c7dbc867171c6982a5512/llguidance-1.7.6-cp39-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:ceec951d29a74309984e3be0fe7f5f56c1362434cd937abd517b259a60908b1e", size = 3074809, upload-time = "2026-06-03T20:13:15.498Z" }, - { url = "https://files.pythonhosted.org/packages/49/37/99d700f0e2c83acf25a8d8946b2bee9f5eac47bc530bfbd53ba3126c667f/llguidance-1.7.6-cp39-abi3-win_amd64.whl", hash = "sha256:ace7e81cd31950a87186356ab24bd7f75fbc10a05ca9d9f7f8748f931963f763", size = 2879207, upload-time = "2026-06-03T20:13:23.341Z" }, -] - -[[package]] -name = "llvmlite" -version = "0.47.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/88/a8952b6d5c21e74cbf158515b779666f692846502623e9e3c39d8e8ba25f/llvmlite-0.47.0.tar.gz", hash = "sha256:62031ce968ec74e95092184d4b0e857e444f8fdff0b8f9213707699570c33ccc", size = 193614, upload-time = "2026-03-31T18:29:53.497Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/48/4b7fe0e34c169fa2f12532916133e0b219d2823b540733651b34fdac509a/llvmlite-0.47.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:306a265f408c259067257a732c8e159284334018b4083a9e35f67d19792b164f", size = 37232769, upload-time = "2026-03-31T18:28:43.735Z" }, - { url = "https://files.pythonhosted.org/packages/e6/4b/e3f2cd17822cf772a4a51a0a8080b0032e6d37b2dbe8cfb724eac4e31c52/llvmlite-0.47.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5853bf26160857c0c2573415ff4efe01c4c651e59e2c55c2a088740acfee51cd", size = 56275178, upload-time = "2026-03-31T18:28:48.342Z" }, - { url = "https://files.pythonhosted.org/packages/b6/55/a3b4a543185305a9bdf3d9759d53646ed96e55e7dfd43f53e7a421b8fbae/llvmlite-0.47.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:003bcf7fa579e14db59c1a1e113f93ab8a06b56a4be31c7f08264d1d4072d077", size = 55128632, upload-time = "2026-03-31T18:28:52.901Z" }, - { url = "https://files.pythonhosted.org/packages/2f/f5/d281ae0f79378a5a91f308ea9fdb9f9cc068fddd09629edc0725a5a8fde1/llvmlite-0.47.0-cp312-cp312-win_amd64.whl", hash = "sha256:f3079f25bdc24cd9d27c4b2b5e68f5f60c4fdb7e8ad5ee2b9b006007558f9df7", size = 38138692, upload-time = "2026-03-31T18:28:57.147Z" }, -] - -[[package]] -name = "lm-format-enforcer" -version = "0.11.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "interegular" }, - { name = "packaging" }, - { name = "pydantic" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/84/d5/41cd417ba7dfdbbcfe46cebf81fb3dfd7c591b89897560ad05bb410a465d/lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da", size = 40258, upload-time = "2025-08-24T19:37:47.527Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/ef/11292bb0b85cf4c93447cab5a29f64576ed14d3ab4280e35ddd23486594a/lm_format_enforcer-0.11.3-py3-none-any.whl", hash = "sha256:cf586350875def1ae7a8fba84fcbbfc8371424b6c9d05c1fcba70aa233fbf06f", size = 45418, upload-time = "2025-08-24T19:37:46.325Z" }, -] - -[[package]] -name = "loguru" -version = "0.7.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "win32-setctime", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, -] - -[[package]] -name = "markupsafe" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, - { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, - { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, - { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, - { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, - { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, - { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, - { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, -] - -[[package]] -name = "matplotlib" -version = "3.11.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "contourpy" }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e7/c8/9aa712a0afb882649424dd8de8ad9aa6235e796e84c6052e8f6dc1598d0d/matplotlib-3.11.2.tar.gz", hash = "sha256:cec596316640f2b394b8f0daa0ea61a8eae82d017b620b9f202befb972a59ea4", size = 32660610, upload-time = "2026-09-11T19:05:31.214Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/ce/1bfcc4873b121597791ad74032943b123218c0613af0b97e8dd05e916fb1/matplotlib-3.11.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ef752769cd962f39ea0b6ffc82d1ea43a0012c5a6157c7a075212fa509cfcff2", size = 9476466, upload-time = "2026-09-11T19:03:19.45Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c4/7f5f3601ee69baf072c0c7d3ce60c03e0618621c5a56c34a62a460e29d11/matplotlib-3.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ef31985c4dedb5f1424e1aec6849a47dd37689cb7fa3c20b1b82187f26806261", size = 9305583, upload-time = "2026-09-11T19:03:22.39Z" }, - { url = "https://files.pythonhosted.org/packages/b8/90/2b3fd67ee273163faeda6d514be70b5596eaa0fd77b60ffc294ad0b34f5f/matplotlib-3.11.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df4f7784aca81a94f254c0a2767d592ee25f407e488f5fa7203e51093fb6ca27", size = 9860353, upload-time = "2026-09-11T19:03:25.049Z" }, - { url = "https://files.pythonhosted.org/packages/f4/84/32549e7a462dc311aed2ab62e5d2538028840b5c53a8be0195c789937c3d/matplotlib-3.11.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b9a7ad579856284135e401ecc918c5f8a017ee30539298862a109f51b971710", size = 10670348, upload-time = "2026-09-11T19:03:28.108Z" }, - { url = "https://files.pythonhosted.org/packages/84/39/02e21b74f7439bd643d717ea846006d46e309e6d29b632b57751265311d6/matplotlib-3.11.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3aa4b8516fd26659e4363abbf317c703d9116496c5db2e9d0609a2866dd39dd2", size = 10810580, upload-time = "2026-09-11T19:03:31.402Z" }, - { url = "https://files.pythonhosted.org/packages/04/93/0d239bde12b308262265c2d98909b2f0ecad2b5deeae96241642e822e9c7/matplotlib-3.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:c5c1c68ee401fc98271263410f0e5ce88285abacf7627132914e8adf3d70ff43", size = 9349409, upload-time = "2026-09-11T19:03:34.195Z" }, - { url = "https://files.pythonhosted.org/packages/49/a8/06baf901c02246c8a222b655cc4540ef9c15e1549a04e07927f9cde716c3/matplotlib-3.11.2-cp312-cp312-win_arm64.whl", hash = "sha256:643ff850d8e0f5b8319337f87ed3cb59506afb3df3cc48de777d85871233be7b", size = 9028084, upload-time = "2026-09-11T19:03:37.032Z" }, -] - -[[package]] -name = "mcp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "httpx2" }, - { name = "jsonschema" }, - { name = "mcp-types" }, - { name = "opentelemetry-api" }, - { name = "pydantic" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "python-multipart" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, - { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/33/32d4dff2c95bb5d897c3ef4c83649a08996b17b58f0a326d2495d4c81179/mcp-2.0.0.tar.gz", hash = "sha256:0f440e735c13ece8bb19bc62cf0b86f4313448432fbb77d35e14034f4e050728", size = 1662284, upload-time = "2026-07-28T13:45:32.346Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/72/7d7897418912c1d12e87556630dfb7bf0eac71160e9bef8b447960804ee3/mcp-2.0.0-py3-none-any.whl", hash = "sha256:1cb4c75d2d2c7b8c1d756355e5d82a39f2822cc7f13e22a2051d7ca3592349d6", size = 349980, upload-time = "2026-07-28T13:45:28.853Z" }, -] - -[[package]] -name = "mcp-types" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bb/56/9b8e1c152f61f6c6b07c4b5896c88c7d0ae90bac6ee6306f852fcc5c1eb0/mcp_types-2.0.0.tar.gz", hash = "sha256:d7d939b9285c9961ae8866ba75ef85da34d12bafe276efbf4eb6a131786d8379", size = 66632, upload-time = "2026-07-28T13:45:33.804Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/4c/c78d78c3d52b0ac594ad7cc8ef5972adfe070e3597a8a4c6ce0cd39196ea/mcp_types-2.0.0-py3-none-any.whl", hash = "sha256:6b2de797ca2797f568b79529e1b25948e34de511bcc0bd82fef1039a6d1b8eb0", size = 69649, upload-time = "2026-07-28T13:45:30.713Z" }, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - -[[package]] -name = "mistral-common" -version = "1.11.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jsonschema" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "pydantic" }, - { name = "pydantic-extra-types", extra = ["pycountry"] }, - { name = "requests" }, - { name = "tiktoken" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/d0/61b2c24be62a8e2f0e46a1c16de23de386c8644408da249bc66768a6681b/mistral_common-1.11.7.tar.gz", hash = "sha256:d3b79583595cf6d96a2ab33e42cb8449768383147b8c56cac5a4f193be19d20d", size = 6387178, upload-time = "2026-07-23T09:21:17.206Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/a4/bc2850eb33cc2d633a21f51530756350dca325ee69b07c9202552f2bbadb/mistral_common-1.11.7-py3-none-any.whl", hash = "sha256:a9511b88eacacbe7dacddd9d3498c1739f56847b7fdddbd5a22e7844fd9def95", size = 6553583, upload-time = "2026-07-23T09:21:19.818Z" }, -] - -[package.optional-dependencies] -image = [ - { name = "opencv-python-headless" }, -] - -[[package]] -name = "ml-dtypes" -version = "0.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, - { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, - { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, - { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, - { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, -] - -[[package]] -name = "model-hosting-container-standards" -version = "0.1.16" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fastapi" }, - { name = "httpx" }, - { name = "jmespath" }, - { name = "pydantic" }, - { name = "setuptools" }, - { name = "starlette" }, - { name = "supervisor" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2d/5f/bc0d0fce1bd0a35378696aa13b21feffa18d9cda837f4e1be124e45ee090/model_hosting_container_standards-0.1.16.tar.gz", hash = "sha256:d34589633900e53c3ee5f7c78280a7cf7e4f6532c35e763341a262fc85cbe84a", size = 94130, upload-time = "2026-06-15T21:29:34.771Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/ef/6eabeb251d2a0598cb5f9a274159e05ae07a1e3fe6a1473bf6035793252a/model_hosting_container_standards-0.1.16-py3-none-any.whl", hash = "sha256:47f4f65713120bc3a69feb022981a38db9e557aedf88dbd72077f20588caa12b", size = 125666, upload-time = "2026-06-15T21:29:33.415Z" }, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, -] - -[[package]] -name = "msal" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9a/99/d840198ecf6e8057bbc937f129ae940404485d736cda73253bbff9537f01/msal-1.37.0.tar.gz", hash = "sha256:1b1672a33ee467c1d70b341bb16cafd51bb3c817147a95b93263794b03971bec", size = 182444, upload-time = "2026-05-29T19:49:05.561Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/b0/d807279f4b55d16d1f120d5ac4344c6e39b56732e2a224d40bded7fd67ad/msal-1.37.0-py3-none-any.whl", hash = "sha256:dd17e95a7c71bce75e8108113438ba7c4a086b3bcad4f57a8c09b7af3d753c2d", size = 123725, upload-time = "2026-05-29T19:49:04.335Z" }, -] - -[[package]] -name = "msal-extensions" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, -] - -[[package]] -name = "msgpack" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" }, - { url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" }, - { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, - { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, - { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, - { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, - { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" }, - { url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" }, - { url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" }, -] - -[[package]] -name = "msgspec" -version = "0.21.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/60/f79b9b013a16fa3a58350c9295ddc6789f2e335f36ea61ed10a21b215364/msgspec-0.21.1.tar.gz", hash = "sha256:2313508e394b0d208f8f56892ca9b2799e2561329de9763b19619595a6c0f72c", size = 319193, upload-time = "2026-04-12T21:44:50.394Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/cf/317224852c00248c620a9bcf4b26e2e4ab8afd752f18d2a6ef73ebd423b6/msgspec-0.21.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4248cf0b6129b7d230eacd493c17cc2d4f3989f3bb7f633a928a85b7dcfa251", size = 196188, upload-time = "2026-04-12T21:44:07.181Z" }, - { url = "https://files.pythonhosted.org/packages/6d/81/074612945c0666078f7366f40000013de9f6ba687491d450df699bceebc9/msgspec-0.21.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5102c7e9b3acff82178449b85006d96310e690291bb1ea0142f1b24bcb8aabcb", size = 188473, upload-time = "2026-04-12T21:44:08.736Z" }, - { url = "https://files.pythonhosted.org/packages/8a/37/655101799590bcc5fddb2bd3fe0e6194e816c2d1da7c361725f5eb89a910/msgspec-0.21.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:846758412e9518252b2ac9bffd6f0e54d9ff614f5f9488df7749f81ff5c80920", size = 218871, upload-time = "2026-04-12T21:44:09.917Z" }, - { url = "https://files.pythonhosted.org/packages/b5/d1/d4cd9fe89c7d400d7a18f86ccc94daa3f0927f53558846fcb60791dce5d6/msgspec-0.21.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21995e74b5c598c2e004110ad66ec7f1b8c20bf2bcf3b2de8fd9a3094422d3ff", size = 225025, upload-time = "2026-04-12T21:44:11.191Z" }, - { url = "https://files.pythonhosted.org/packages/24/bf/e20549e602b9edccadeeff98760345a416f9cce846a657e8b18e3396b212/msgspec-0.21.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6129f0cca52992e898fd5344187f7c8127b63d810b2fd73e36fca73b4c6475ee", size = 222672, upload-time = "2026-04-12T21:44:12.481Z" }, - { url = "https://files.pythonhosted.org/packages/b4/68/04d7a8f0f786545cf9b8c280c57aa6befb5977af6e884b8b54191cbe44b3/msgspec-0.21.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ef3ec2296248d1f8b9231acb051b6d471dfde8f21819e86c9adaaa9f42918521", size = 227303, upload-time = "2026-04-12T21:44:13.709Z" }, - { url = "https://files.pythonhosted.org/packages/cc/4d/619866af2840875be408047bf9e70ceafbae6ab50660de7134ed1b25eb86/msgspec-0.21.1-cp312-cp312-win_amd64.whl", hash = "sha256:d4ab834a054c6f0cbeef6df9e7e1b33d5f1bc7b86dea1d2fd7cad003873e783d", size = 190017, upload-time = "2026-04-12T21:44:14.977Z" }, - { url = "https://files.pythonhosted.org/packages/5e/2e/a8f9eca8fd00e097d7a9e99ba8a4685db994494448e3d4f0b7f6e9a3c0f7/msgspec-0.21.1-cp312-cp312-win_arm64.whl", hash = "sha256:628aaa35c74950a8c59da330d7e98917e1c7188f983745782027748ee4ca573e", size = 175345, upload-time = "2026-04-12T21:44:16.431Z" }, -] - -[[package]] -name = "multidict" -version = "6.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, - { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, - { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, - { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, - { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, - { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, - { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, - { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, - { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, - { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, - { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, - { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, - { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, - { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, - { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, - { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, - { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, - { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, -] - -[[package]] -name = "munch" -version = "4.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/2b/45098135b5f9f13221820d90f9e0516e11a2a0f55012c13b081d202b782a/munch-4.0.0.tar.gz", hash = "sha256:542cb151461263216a4e37c3fd9afc425feeaf38aaa3025cd2a981fadb422235", size = 19089, upload-time = "2023-07-01T09:49:35.98Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/b3/7c69b37f03260a061883bec0e7b05be7117c1b1c85f5212c72c8c2bc3c8c/munch-4.0.0-py2.py3-none-any.whl", hash = "sha256:71033c45db9fb677a0b7eb517a4ce70ae09258490e419b0e7f00d1e386ecb1b4", size = 9950, upload-time = "2023-07-01T09:49:34.472Z" }, -] - -[[package]] -name = "narwhals" -version = "2.24.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/58946e5aab18393e793bd4add6985b95d0e01c3a2d832f38f54468b10dcd/narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d", size = 661143, upload-time = "2026-07-13T10:49:19.086Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, -] - -[[package]] -name = "nccl4py" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-core" }, - { name = "cuda-pathfinder" }, - { name = "numpy" }, - { name = "packaging" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/2d/86a96d31ad8092f86f15f45d051d599a3fa7a7044b0176f27a50ff16260c/nccl4py-0.6.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f5b6cac4a7edaeb1b626afabbf29b0d316b847738dace109fb9dd71f2c6b56", size = 3626706, upload-time = "2026-09-23T17:14:14.74Z" }, - { url = "https://files.pythonhosted.org/packages/bf/c5/bd7d2f1251215881233580f402606e5a480b60f469f7dbea01d30fb99066/nccl4py-0.6.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4006cf8c12c26ed29811b80858a58f8f31e4b42ea85b6c1d23401981449ab9d", size = 3695883, upload-time = "2026-09-23T17:14:37.411Z" }, -] - -[[package]] -name = "nemo-rl-dynamo-runtime" -version = "0.0.0" -source = { virtual = "." } -dependencies = [ - { name = "ai-dynamo", extra = ["vllm"] }, -] - -[package.metadata] -requires-dist = [{ name = "ai-dynamo", extras = ["vllm"], specifier = "==1.5.1" }] - -[[package]] -name = "nest-asyncio" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, -] - -[[package]] -name = "networkx" -version = "3.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, -] - -[[package]] -name = "ninja" -version = "1.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/73/79a0b22fc731989c708068427579e840a6cf4e937fe7ae5c5d0b7356ac22/ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978", size = 242558, upload-time = "2025-08-11T15:10:19.421Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/74/d02409ed2aa865e051b7edda22ad416a39d81a84980f544f8de717cab133/ninja-1.13.0-py3-none-macosx_10_9_universal2.whl", hash = "sha256:fa2a8bfc62e31b08f83127d1613d10821775a0eb334197154c4d6067b7068ff1", size = 310125, upload-time = "2025-08-11T15:09:50.971Z" }, - { url = "https://files.pythonhosted.org/packages/8e/de/6e1cd6b84b412ac1ef327b76f0641aeb5dcc01e9d3f9eee0286d0c34fd93/ninja-1.13.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630", size = 177467, upload-time = "2025-08-11T15:09:52.767Z" }, - { url = "https://files.pythonhosted.org/packages/c8/83/49320fb6e58ae3c079381e333575fdbcf1cca3506ee160a2dcce775046fa/ninja-1.13.0-py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:be7f478ff9f96a128b599a964fc60a6a87b9fa332ee1bd44fa243ac88d50291c", size = 187834, upload-time = "2025-08-11T15:09:54.115Z" }, - { url = "https://files.pythonhosted.org/packages/56/c7/ba22748fb59f7f896b609cd3e568d28a0a367a6d953c24c461fe04fc4433/ninja-1.13.0-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:60056592cf495e9a6a4bea3cd178903056ecb0943e4de45a2ea825edb6dc8d3e", size = 202736, upload-time = "2025-08-11T15:09:55.745Z" }, - { url = "https://files.pythonhosted.org/packages/79/22/d1de07632b78ac8e6b785f41fa9aad7a978ec8c0a1bf15772def36d77aac/ninja-1.13.0-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1c97223cdda0417f414bf864cfb73b72d8777e57ebb279c5f6de368de0062988", size = 179034, upload-time = "2025-08-11T15:09:57.394Z" }, - { url = "https://files.pythonhosted.org/packages/ed/de/0e6edf44d6a04dabd0318a519125ed0415ce437ad5a1ec9b9be03d9048cf/ninja-1.13.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa", size = 180716, upload-time = "2025-08-11T15:09:58.696Z" }, - { url = "https://files.pythonhosted.org/packages/54/28/938b562f9057aaa4d6bfbeaa05e81899a47aebb3ba6751e36c027a7f5ff7/ninja-1.13.0-py3-none-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4be9c1b082d244b1ad7ef41eb8ab088aae8c109a9f3f0b3e56a252d3e00f42c1", size = 146843, upload-time = "2025-08-11T15:10:00.046Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fb/d06a3838de4f8ab866e44ee52a797b5491df823901c54943b2adb0389fbb/ninja-1.13.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6739d3352073341ad284246f81339a384eec091d9851a886dfa5b00a6d48b3e2", size = 154402, upload-time = "2025-08-11T15:10:01.657Z" }, - { url = "https://files.pythonhosted.org/packages/31/bf/0d7808af695ceddc763cf251b84a9892cd7f51622dc8b4c89d5012779f06/ninja-1.13.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:11be2d22027bde06f14c343f01d31446747dbb51e72d00decca2eb99be911e2f", size = 552388, upload-time = "2025-08-11T15:10:03.349Z" }, - { url = "https://files.pythonhosted.org/packages/9d/70/c99d0c2c809f992752453cce312848abb3b1607e56d4cd1b6cded317351a/ninja-1.13.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:aa45b4037b313c2f698bc13306239b8b93b4680eb47e287773156ac9e9304714", size = 472501, upload-time = "2025-08-11T15:10:04.735Z" }, - { url = "https://files.pythonhosted.org/packages/9f/43/c217b1153f0e499652f5e0766da8523ce3480f0a951039c7af115e224d55/ninja-1.13.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f8e1e8a1a30835eeb51db05cf5a67151ad37542f5a4af2a438e9490915e5b72", size = 638280, upload-time = "2025-08-11T15:10:06.512Z" }, - { url = "https://files.pythonhosted.org/packages/8c/45/9151bba2c8d0ae2b6260f71696330590de5850e5574b7b5694dce6023e20/ninja-1.13.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:3d7d7779d12cb20c6d054c61b702139fd23a7a964ec8f2c823f1ab1b084150db", size = 642420, upload-time = "2025-08-11T15:10:08.35Z" }, - { url = "https://files.pythonhosted.org/packages/3c/fb/95752eb635bb8ad27d101d71bef15bc63049de23f299e312878fc21cb2da/ninja-1.13.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:d741a5e6754e0bda767e3274a0f0deeef4807f1fec6c0d7921a0244018926ae5", size = 585106, upload-time = "2025-08-11T15:10:09.818Z" }, - { url = "https://files.pythonhosted.org/packages/c1/31/aa56a1a286703800c0cbe39fb4e82811c277772dc8cd084f442dd8e2938a/ninja-1.13.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:e8bad11f8a00b64137e9b315b137d8bb6cbf3086fbdc43bf1f90fd33324d2e96", size = 707138, upload-time = "2025-08-11T15:10:11.366Z" }, - { url = "https://files.pythonhosted.org/packages/34/6f/5f5a54a1041af945130abdb2b8529cbef0cdcbbf9bcf3f4195378319d29a/ninja-1.13.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b4f2a072db3c0f944c32793e91532d8948d20d9ab83da9c0c7c15b5768072200", size = 581758, upload-time = "2025-08-11T15:10:13.295Z" }, - { url = "https://files.pythonhosted.org/packages/95/97/51359c77527d45943fe7a94d00a3843b81162e6c4244b3579fe8fc54cb9c/ninja-1.13.0-py3-none-win32.whl", hash = "sha256:8cfbb80b4a53456ae8a39f90ae3d7a2129f45ea164f43fadfa15dc38c4aef1c9", size = 267201, upload-time = "2025-08-11T15:10:15.158Z" }, - { url = "https://files.pythonhosted.org/packages/29/45/c0adfbfb0b5895aa18cec400c535b4f7ff3e52536e0403602fc1a23f7de9/ninja-1.13.0-py3-none-win_amd64.whl", hash = "sha256:fb8ee8719f8af47fed145cced4a85f0755dd55d45b2bddaf7431fa89803c5f3e", size = 309975, upload-time = "2025-08-11T15:10:16.697Z" }, - { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, -] - -[[package]] -name = "nixl" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nixl-cu12" }, - { name = "nixl-cu13" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/b3/c7f635c70c8ae7b0b920b1c82ecabf67617614bcb2abd8df0cd727ac8520/nixl-1.3.2-py3-none-any.whl", hash = "sha256:4324569b19aaf6e16d9d7b27ef1965a2e36a170fb66c6d1b6597a76bf4761cbe", size = 10303, upload-time = "2026-07-24T20:19:58.019Z" }, -] - -[package.optional-dependencies] -cu13 = [ - { name = "nixl-cu13" }, -] - -[[package]] -name = "nixl-cu12" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/28/8ba9eab9faa5f9455d8ebd322630398573f4e109b84ef8615d85d4bca3b4/nixl_cu12-1.3.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ea79ec205168cf819614127265ce1fff1e61d9d126d6d56cbd1ecaa29c980723", size = 80286069, upload-time = "2026-07-24T20:14:48.56Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d3/2964339654b3fe85e7aa62fdce4da3b97ee40337f3b72466aa79251f1196/nixl_cu12-1.3.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ef8ccdffcd54978e8a799de59287efcb3af0b7ba3bf02e04bc4df4c842f1f569", size = 82188539, upload-time = "2026-07-24T20:12:52.953Z" }, -] - -[[package]] -name = "nixl-cu13" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/2f/154a40352b7b6cb5afc41e1b794821c9631974712a6536ade78ef681a0b6/nixl_cu13-1.3.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5c3f5c1265fcec50f608a56fe828fad892322a1170755cdade9cc37c6545bb73", size = 64425110, upload-time = "2026-07-24T20:18:51.96Z" }, - { url = "https://files.pythonhosted.org/packages/99/d8/5768b907b85d8856c07674ddd0ffeb736ed987ff530a12e8f17a273cab3b/nixl_cu13-1.3.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:22fcd7183b2cd831b3da781c9e9991c5f6ef77a238ee6b7ac05d42558ea469a9", size = 66330361, upload-time = "2026-07-24T20:16:38.296Z" }, -] - -[[package]] -name = "numba" -version = "0.65.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llvmlite" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/61/7299643b9c18d669e04be7c5bcb64d985070d07553274817b45b049e7bfe/numba-0.65.0.tar.gz", hash = "sha256:edad0d9f6682e93624c00125a471ae4df186175d71fd604c983c377cdc03e68b", size = 2764131, upload-time = "2026-04-01T03:52:01.946Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/2f/8bd31a1ea43c01ac215283d83aa5f8d5acbe7a36c85b82f1757bfe9ccb31/numba-0.65.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:b27ee4847e1bfb17e9604d100417ee7c1d10f15a6711c6213404b3da13a0b2aa", size = 2680705, upload-time = "2026-04-01T03:51:32.597Z" }, - { url = "https://files.pythonhosted.org/packages/73/36/88406bd58600cc696417b8e5dd6a056478da808f3eaf48d18e2421e0c2d9/numba-0.65.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a52d92ffd297c10364bce60cd1fcb88f99284ab5df085f2c6bcd1cb33b529a6f", size = 3801411, upload-time = "2026-04-01T03:51:34.321Z" }, - { url = "https://files.pythonhosted.org/packages/0c/61/ce753a1d7646dd477e16d15e89473703faebb8995d2f71d7ad69a540b565/numba-0.65.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da8e371e328c06d0010c3d8b44b21858652831b85bcfba78cb22c042e22dbd8e", size = 3501622, upload-time = "2026-04-01T03:51:36.348Z" }, - { url = "https://files.pythonhosted.org/packages/7d/86/db87a5393f1b1fabef53ac3ba4e6b938bb27e40a04ad7cc512098fcae032/numba-0.65.0-cp312-cp312-win_amd64.whl", hash = "sha256:59bb9f2bb9f1238dfd8e927ba50645c18ae769fef4f3d58ea0ea22a2683b91f5", size = 2749979, upload-time = "2026-04-01T03:51:37.88Z" }, -] - -[[package]] -name = "numpy" -version = "2.3.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/65/21b3bc86aac7b8f2862db1e808f1ea22b028e30a225a34a5ede9bf8678f2/numpy-2.3.5.tar.gz", hash = "sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0", size = 20584950, upload-time = "2025-11-16T22:52:42.067Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/37/e669fe6cbb2b96c62f6bbedc6a81c0f3b7362f6a59230b23caa673a85721/numpy-2.3.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:74ae7b798248fe62021dbf3c914245ad45d1a6b0cb4a29ecb4b31d0bfbc4cc3e", size = 16733873, upload-time = "2025-11-16T22:49:49.84Z" }, - { url = "https://files.pythonhosted.org/packages/c5/65/df0db6c097892c9380851ab9e44b52d4f7ba576b833996e0080181c0c439/numpy-2.3.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee3888d9ff7c14604052b2ca5535a30216aa0a58e948cdd3eeb8d3415f638769", size = 12259838, upload-time = "2025-11-16T22:49:52.863Z" }, - { url = "https://files.pythonhosted.org/packages/5b/e1/1ee06e70eb2136797abe847d386e7c0e830b67ad1d43f364dd04fa50d338/numpy-2.3.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:612a95a17655e213502f60cfb9bf9408efdc9eb1d5f50535cc6eb365d11b42b5", size = 5088378, upload-time = "2025-11-16T22:49:55.055Z" }, - { url = "https://files.pythonhosted.org/packages/6d/9c/1ca85fb86708724275103b81ec4cf1ac1d08f465368acfc8da7ab545bdae/numpy-2.3.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3101e5177d114a593d79dd79658650fe28b5a0d8abeb8ce6f437c0e6df5be1a4", size = 6628559, upload-time = "2025-11-16T22:49:57.371Z" }, - { url = "https://files.pythonhosted.org/packages/74/78/fcd41e5a0ce4f3f7b003da85825acddae6d7ecb60cf25194741b036ca7d6/numpy-2.3.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b973c57ff8e184109db042c842423ff4f60446239bd585a5131cc47f06f789d", size = 14250702, upload-time = "2025-11-16T22:49:59.632Z" }, - { url = "https://files.pythonhosted.org/packages/b6/23/2a1b231b8ff672b4c450dac27164a8b2ca7d9b7144f9c02d2396518352eb/numpy-2.3.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28", size = 16606086, upload-time = "2025-11-16T22:50:02.127Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c5/5ad26fbfbe2012e190cc7d5003e4d874b88bb18861d0829edc140a713021/numpy-2.3.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:51c1e14eb1e154ebd80e860722f9e6ed6ec89714ad2db2d3aa33c31d7c12179b", size = 16025985, upload-time = "2025-11-16T22:50:04.536Z" }, - { url = "https://files.pythonhosted.org/packages/d2/fa/dd48e225c46c819288148d9d060b047fd2a6fb1eb37eae25112ee4cb4453/numpy-2.3.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b46b4ec24f7293f23adcd2d146960559aaf8020213de8ad1909dba6c013bf89c", size = 18542976, upload-time = "2025-11-16T22:50:07.557Z" }, - { url = "https://files.pythonhosted.org/packages/05/79/ccbd23a75862d95af03d28b5c6901a1b7da4803181513d52f3b86ed9446e/numpy-2.3.5-cp312-cp312-win32.whl", hash = "sha256:3997b5b3c9a771e157f9aae01dd579ee35ad7109be18db0e85dbdbe1de06e952", size = 6285274, upload-time = "2025-11-16T22:50:10.746Z" }, - { url = "https://files.pythonhosted.org/packages/2d/57/8aeaf160312f7f489dea47ab61e430b5cb051f59a98ae68b7133ce8fa06a/numpy-2.3.5-cp312-cp312-win_amd64.whl", hash = "sha256:86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa", size = 12782922, upload-time = "2025-11-16T22:50:12.811Z" }, - { url = "https://files.pythonhosted.org/packages/78/a6/aae5cc2ca78c45e64b9ef22f089141d661516856cf7c8a54ba434576900d/numpy-2.3.5-cp312-cp312-win_arm64.whl", hash = "sha256:f28620fe26bee16243be2b7b874da327312240a7cdc38b769a697578d2100013", size = 10194667, upload-time = "2025-11-16T22:50:16.16Z" }, -] - -[[package]] -name = "nvidia-cublas" -version = "13.1.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" } }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, -] - -[[package]] -name = "nvidia-cuda-cccl" -version = "13.3.3.4.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/bd/572971ffc14bd36676c821fc15d991b08fe6179cb09368250147475f954d/nvidia_cuda_cccl-13.3.3.4.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:067d19b4b3c9d0f2ebec9f29a311b2863db96bf98e058bbc331597d51ce818cf", size = 3454030, upload-time = "2026-06-29T16:41:49.092Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ab/049726d90147865a3ea53bae6cb7c35b98bf1fdf96cdb967101329625f83/nvidia_cuda_cccl-13.3.3.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc0adc188d570b09f4d606c7dc05a42aa3d8aa082e0d60f7bbfc5b6435f627c6", size = 3454034, upload-time = "2026-06-29T16:42:07.435Z" }, - { url = "https://files.pythonhosted.org/packages/24/d3/b1afcd9c40ceca72022579215fcaf5318cd747fd896cb928d4a1de924ff8/nvidia_cuda_cccl-13.3.3.4.1-py3-none-win_amd64.whl", hash = "sha256:d7c92cc03047031fa7af30866636d35ce4af409c28fc7dd8f69cb17053741399", size = 3454014, upload-time = "2026-06-29T17:09:09.012Z" }, -] - -[[package]] -name = "nvidia-cuda-crt" -version = "13.3.73" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/41/2089e411507d66458d67208bdd1bc562d492bb6458c3d2aea4603072a219/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:60aacc0b5e1e8b40c62abe4d1ab16440add91b99bd2f17f62dd091586b73d166", size = 157353, upload-time = "2026-06-29T16:42:38.163Z" }, - { url = "https://files.pythonhosted.org/packages/7e/ce/16d76f4b5b3f7460f5ebd17516685495c149c66651ffdd381f90e4d4e65c/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df14a17ae1c5c3171265411212246654d780f89344ea85344466c6b955247543", size = 157352, upload-time = "2026-06-29T16:43:09.209Z" }, - { url = "https://files.pythonhosted.org/packages/49/b3/6791ffba6f4b8e0d3ed875285aad8078ee407afa464ecd934ae298c205b1/nvidia_cuda_crt-13.3.73-py3-none-win_amd64.whl", hash = "sha256:af04e75148db1f0eea30958f33a9ec5a5a2dc2afa99ca4323f9a93b840602ca5", size = 158286, upload-time = "2026-06-29T17:09:28.621Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, -] - -[[package]] -name = "nvidia-cuda-nvcc" -version = "13.2.86" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "nvidia-cuda-crt" }, - { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-nvvm", version = "13.2.86", source = { registry = "https://pypi.org/simple" } }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/a3/403638f80960e677ae8ecc78059d8c85dc2519b85ed8c0229840dc6f3b54/nvidia_cuda_nvcc-13.2.86-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:909140a1f942b943982b2eff120e618c94e29d75d9e33f5cd074f0e64eb411e8", size = 38716270, upload-time = "2026-07-16T09:37:12.754Z" }, - { url = "https://files.pythonhosted.org/packages/38/7b/ad9b5f84e8820af24afa8db66b5b2785b9bdc3b8724dffcd5e6ef2d40e53/nvidia_cuda_nvcc-13.2.86-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d3f56c8d705bad35bbba69eb0470d90856fd5d4dc48c6a6173aaf1e9f887cf5", size = 44042846, upload-time = "2026-07-16T09:37:47.98Z" }, - { url = "https://files.pythonhosted.org/packages/03/c1/01dad23300993e52933773751d0dd8e795175daa830b798ce324db377e3a/nvidia_cuda_nvcc-13.2.86-py3-none-win_amd64.whl", hash = "sha256:4171face482ef8ca35b5b2d59cfd25d2d0e2a2e2534fe34d2ce20695437868d9", size = 31816369, upload-time = "2026-07-16T10:01:34.327Z" }, -] - -[[package]] -name = "nvidia-cuda-nvcc" -version = "13.3.73" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'darwin'", -] -dependencies = [ - { name = "nvidia-cuda-crt" }, - { name = "nvidia-cuda-runtime", version = "13.3.29", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-nvvm", version = "13.3.73", source = { registry = "https://pypi.org/simple" } }, -] - -[[package]] -name = "nvidia-cuda-nvdisasm" -version = "13.4.92" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/88/2382bc8b0200cd5a39e4ccc4d3930310c4699ac24bf91761ba780c00bb85/nvidia_cuda_nvdisasm-13.4.92-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b007c08c8248b29281ed9a174b478d557b58ea8a921ab78369a2843f0718f77d", size = 5163390, upload-time = "2026-09-16T20:38:33.02Z" }, - { url = "https://files.pythonhosted.org/packages/59/ea/b83b98288fd3d37b16c0da8846987d9c8719e9ff49ab974c1988d9e63f5a/nvidia_cuda_nvdisasm-13.4.92-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f35a5b6ddb64b6758c22c0b078c5b6ff6d54f7c240cd74bf131ab5bf13b6c3fe", size = 5203296, upload-time = "2026-09-16T20:38:39.558Z" }, - { url = "https://files.pythonhosted.org/packages/bf/73/acabcf7b9ac28e9bcd8326ae72f958f5ec05e32fb8683263868e1990bedf/nvidia_cuda_nvdisasm-13.4.92-py3-none-win_amd64.whl", hash = "sha256:600c43bde6b323be4d950e33a248aa97ff9cdea28a4b7dae083680f092bdd230", size = 5470680, upload-time = "2026-09-16T20:53:29.634Z" }, - { url = "https://files.pythonhosted.org/packages/25/26/2edceaba240f549c7a4d5acc3ccde21085fd76913abc854aa68825c98e52/nvidia_cuda_nvdisasm-13.4.92-py3-none-win_arm64.whl", hash = "sha256:624a3406572a983079edce35284cc5b6b1a3f23b02a62057b49a79be33d76252", size = 5422156, upload-time = "2026-09-16T20:57:06.408Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, - { url = "https://files.pythonhosted.org/packages/4a/af/345fedb9f4c76c84ab4fa445b36bd4048a4d9db60e6bc76b4f913ff4b852/nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872", size = 76807835, upload-time = "2025-09-04T08:39:15.274Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.3.33" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'darwin'", -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, - { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, - { url = "https://files.pythonhosted.org/packages/b7/94/6b867483bec07da24ffa32736c79fabb94ef3a7af4d787a9d4a974868576/nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492", size = 2927037, upload-time = "2025-10-09T09:04:23.782Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.3.29" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'darwin'", -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.20.0.48" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, - { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, -] - -[[package]] -name = "nvidia-cudnn-frontend" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/c4/3f587b73ac2eb6e391aebffb7a7a9ac9ed70e1e0e6a1d90ec0fdcb1a516f/nvidia_cudnn_frontend-1.26.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee50df3468f672aa31402fde4c911ad545d08e1d5e133e15bc55c3679d9fd9ca", size = 3471242, upload-time = "2026-07-07T20:55:30.929Z" }, - { url = "https://files.pythonhosted.org/packages/9b/31/64818fbaa117456349241b42ddfaef3ae6b050f5e99bb0e9d78eb831279b/nvidia_cudnn_frontend-1.26.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a1223c4e2e8bbe6f620148d6848f4eb773dd94bef534d5b748e91232b5be618", size = 3627033, upload-time = "2026-07-07T20:55:47.089Z" }, - { url = "https://files.pythonhosted.org/packages/f9/99/04a37f34b271ed157c6108c3191bd1993ab3fad8d5d66516970bd1e7c12d/nvidia_cudnn_frontend-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f104a40cb6f25cf01b7d7e84cb99af7dedd32b1cb8264384c2f363b7a3f7fb6", size = 2998730, upload-time = "2026-07-07T20:56:09.848Z" }, -] - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, - { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, -] - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, - { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, -] - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, - { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, - { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, -] - -[[package]] -name = "nvidia-cutlass-dsl" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cutlass-dsl-libs-base" }, - { name = "nvidia-cutlass-dsl-libs-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/ec/14e6ecbfed31ec35bbd1bb6965ae61f879370906a8f9c2e851e292704ba4/nvidia_cutlass_dsl-4.6.2-py3-none-any.whl", hash = "sha256:06ac62deb182a852dfb053032cf945bf02b9aa1bf502a18b129d280d7babb26c", size = 10460, upload-time = "2026-08-05T14:39:41.657Z" }, -] - -[package.optional-dependencies] -cu13 = [ - { name = "nvidia-cutlass-dsl-libs-cu13" }, -] - -[[package]] -name = "nvidia-cutlass-dsl-libs-base" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-python" }, - { name = "numpy" }, - { name = "nvidia-cuda-nvdisasm" }, - { name = "nvidia-cutlass-dsl-libs-core" }, - { name = "protobuf" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/f3/4be98f2faa283471e355a42ecbc20956fb2c3b6dbc71861f483be277e99a/nvidia_cutlass_dsl_libs_base-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e129db7155d56ba4478be098c0f819e37690a6ce5d43a9a9a83bf1300d32c57f", size = 3321894, upload-time = "2026-08-05T14:12:30.139Z" }, - { url = "https://files.pythonhosted.org/packages/54/1b/cf4a0837054bb4b9dc36cdd86f9d2e7fead3629e3373aaa7f7ec1e5e1542/nvidia_cutlass_dsl_libs_base-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:24dfaddad6077fd0de14eecaf66a72762f2f5f30ae1872231f5bf8b243ac2eac", size = 2824987, upload-time = "2026-08-05T14:12:49.618Z" }, -] - -[[package]] -name = "nvidia-cutlass-dsl-libs-core" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-python" }, - { name = "numpy" }, - { name = "nvidia-cuda-nvdisasm" }, - { name = "protobuf" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/ef/10c0089234a32f1379e1f213f28f8c7521a9ff0154e3acc58f613d5ef3be/nvidia_cutlass_dsl_libs_core-4.6.2-py3-none-any.whl", hash = "sha256:571d4b46fca1bfbb123d0dc348eaa7fd80af0014ddffc07b849e8195b2364333", size = 772393, upload-time = "2026-08-05T14:09:50.76Z" }, -] - -[[package]] -name = "nvidia-cutlass-dsl-libs-cu12" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-python" }, - { name = "numpy" }, - { name = "nvidia-cuda-nvdisasm" }, - { name = "nvidia-cutlass-dsl-libs-base" }, - { name = "protobuf" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/b7/019a8f74ad30ad9b6190dd5963d8921fc2b03777316e47260eb822624a0a/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ea3b96dd6b627fe20637acfa7876ea1adfa8d81a2b106ffff0bb2d042c0c2731", size = 87013242, upload-time = "2026-08-05T14:18:57.703Z" }, - { url = "https://files.pythonhosted.org/packages/80/6e/70c545a600b52b3f0fbbe01608aefc4e675c924ba886485ad72541f5c88e/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7f510307369d522da8e7d666557b9b9e7df06b94748bd5dc0198d59dfd0d918a", size = 88454261, upload-time = "2026-08-05T14:21:14.733Z" }, -] - -[[package]] -name = "nvidia-cutlass-dsl-libs-cu13" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-python" }, - { name = "numpy" }, - { name = "nvidia-cuda-nvdisasm" }, - { name = "nvidia-cutlass-dsl-libs-base" }, - { name = "protobuf" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/38/bbe5b8f1ef1a15e7bd1e2ea86686c38f15eeee0a4ba440a5d00d8222cb8e/nvidia_cutlass_dsl_libs_cu13-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b9543257622ce70d696c6a2bda7e9617ef5b1cc9f29bfe66f4b38d1d1e29a8cb", size = 86712108, upload-time = "2026-08-05T14:27:35.488Z" }, - { url = "https://files.pythonhosted.org/packages/ca/05/7874c37d50e112177a64a96dd6c7eaeacb1661ebb864e0fe5cfd9f706767/nvidia_cutlass_dsl_libs_cu13-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a299a623fad75eb752c1a4e447b924e9049ed396b037509c41a48da82d1f340a", size = 88036030, upload-time = "2026-08-05T14:27:58.991Z" }, -] - -[[package]] -name = "nvidia-ml-py" -version = "13.610.43" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f0/b5/a8fbc356f768fa5c9cfd646668fd7d34bf55bdd1c6e20754642a64d930d4/nvidia_ml_py-13.610.43.tar.gz", hash = "sha256:65437eb73d68d0c62c931ca4d45038472faff03bd0b8729abba4b899f70d60f2", size = 52109, upload-time = "2026-06-01T18:54:08.829Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/45/caa600acfab94560807a20a64b5830d2cd3c3202b7f1328644d70b7d6bd8/nvidia_ml_py-13.610.43-py3-none-any.whl", hash = "sha256:f13c72698edef492f985cc225f14faafe68ae065a2e407f45bdf6f4b9b43fde8", size = 53163, upload-time = "2026-06-01T18:54:07.704Z" }, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.30.7" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/21/a73174c6157101bdf1ffc22b517f76ff0082613989dd9bc8f43e8034caac/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:ca786ffa5a647c75d4d1f5cc72a6c4f537947e2ba8823d7c8aaf768e7a7b9f77", size = 215983881, upload-time = "2026-06-09T03:23:15.633Z" }, - { url = "https://files.pythonhosted.org/packages/3f/34/c500f90c7ae641b8e0f98965b36b8a7ac79cc8b296e8d251fe3eb592ee54/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:cefa7fdb9710efd0f39c5f1be1d61ff6fc9a996c451265bd7fbdcf9455ed4b50", size = 215965170, upload-time = "2026-06-09T03:23:39.73Z" }, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, - { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, -] - -[[package]] -name = "nvidia-nvvm" -version = "13.2.86" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/1d/b50325516e259d4de9fca78b069840d39b6a172c5ba012b88d9985e01fa3/nvidia_nvvm-13.2.86-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a9244f3209922d655612c11ebb4117ea1f80983cb2b215c9cccded127282cebc", size = 64280112, upload-time = "2026-07-16T09:58:27.625Z" }, - { url = "https://files.pythonhosted.org/packages/e6/64/039ad70d68355634581d3b66f99b7aa8f75ca6078e1a6a2c9223677bbee3/nvidia_nvvm-13.2.86-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a502dcc2859f17a925adba495c222d44a221b9eb10e7d111a7046dc2cc883b69", size = 61886756, upload-time = "2026-07-16T09:57:51.818Z" }, - { url = "https://files.pythonhosted.org/packages/c8/1c/a1e27ca53d767f5043eab9332564581e976953a437e1bc8c3642679a622f/nvidia_nvvm-13.2.86-py3-none-win_amd64.whl", hash = "sha256:7d140c2dd2d177af71240a31bbca20f40b4aee9a4e331156083e387855222b49", size = 56750837, upload-time = "2026-07-16T10:11:39.216Z" }, -] - -[[package]] -name = "nvidia-nvvm" -version = "13.3.73" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'darwin'", -] - -[[package]] -name = "nvtx" -version = "0.2.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/dd/692765e87de30bae1522cdffaa0f2b52949658a92a0fa6d96b1a01eae9d2/nvtx-0.2.15.tar.gz", hash = "sha256:2287d3be05b85661deb386f878d1f536c2e532774aa9ec7a50c434942ed81ae5", size = 121230, upload-time = "2026-03-18T10:01:25.547Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/07/698355285a03a366ef63ea9762fc1feef3f9f25483e1655408f72d827090/nvtx-0.2.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2cc530cd0f1a2c14a3a7e683833db509888ac5ed4ead94e5c9e2c7317c6937a7", size = 807159, upload-time = "2026-03-18T10:09:49.232Z" }, - { url = "https://files.pythonhosted.org/packages/c0/d1/08f22448d83481408d663065764ba583df091a7de629ed38fc97e522f1af/nvtx-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ca8030a6d197952318013dd1c12c22da1d4b9feb76ba72e0fcd449961183c2c", size = 806187, upload-time = "2026-03-18T10:13:32.972Z" }, - { url = "https://files.pythonhosted.org/packages/54/23/c97c39e3b7ba256aa343cb828ca0d1c8421f705ca84795658ecd14ca95ed/nvtx-0.2.15-cp312-cp312-win_amd64.whl", hash = "sha256:70a1e768964e0520b68ccabc4df391cc227537c45936a7eba6507bc65e617e00", size = 129178, upload-time = "2026-03-18T10:02:55.299Z" }, -] - -[[package]] -name = "oauthlib" -version = "3.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, -] - -[[package]] -name = "openai" -version = "2.49.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d3/9a/275bee8349b766906741223b98c668fcd4e17e1982a87597778743b4156f/openai-2.49.0.tar.gz", hash = "sha256:80f934333b5b83cef2fde9af7151dacaa72e150f43f92b7675f7647ca6157f48", size = 1080973, upload-time = "2026-07-27T22:51:40.014Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/ca/53357e460a1172e831ecbe43dd0c37342b7211a1eb09f4cf21a412adbbdf/openai-2.49.0-py3-none-any.whl", hash = "sha256:b694201eaa42a1ccf2aa125fe29458150108fb22df1abfb55d7188599da81d8c", size = 1648589, upload-time = "2026-07-27T22:51:38Z" }, -] - -[[package]] -name = "openai-harmony" -version = "0.0.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3e/92/2d038d096f29179c7c9571b431f9e739f87a487121901725e23fe338dd9d/openai_harmony-0.0.8.tar.gz", hash = "sha256:6e43f98e6c242fa2de6f8ea12eab24af63fa2ed3e89c06341fb9d92632c5cbdf", size = 284777, upload-time = "2025-11-05T19:07:06.727Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/c6/2502f416d46be3ec08bb66d696cccffb57781a499e3ff2e4d7c174af4e8f/openai_harmony-0.0.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:029ec25ca74abe48fdb58eb9fdd2a8c1618581fc33ce8e5653f8a1ffbfbd9326", size = 2627806, upload-time = "2025-11-05T19:06:57.063Z" }, - { url = "https://files.pythonhosted.org/packages/d3/d2/ce6953ca87db9cae3e775024184da7d1c5cb88cead19a2d75b42f00a959c/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4f709815924ec325b9a890e6ab2bbb0ceec8e319a4e257328eb752cf36b2efc", size = 2948463, upload-time = "2025-11-05T19:06:48.17Z" }, - { url = "https://files.pythonhosted.org/packages/fa/4c/b553c9651662d6ce102ca7f3629d268b23df1abe5841e24bed81e8a8e949/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5cfcfd963b50a41fc656c84d3440ca6eecdccd6c552158ce790b8f2e33dfb5a9", size = 2704083, upload-time = "2025-11-05T19:06:50.205Z" }, - { url = "https://files.pythonhosted.org/packages/9b/af/4eec8f9ab9c27bcdb444460c72cf43011d176fc44c79d6e113094ca1e152/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a3a16972aa1cee38ea958470cd04ac9a2d5ac38fdcf77ab686611246220c158", size = 2959765, upload-time = "2025-11-05T19:06:53.62Z" }, - { url = "https://files.pythonhosted.org/packages/11/3c/33f3374e4624e0e776f6b13b73c45a7ead7f9c4529f8369ed5bfcaa30cac/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4d5cfa168e74d08f8ba6d58a7e49bc7daef4d58951ec69b66b0d56f4927a68d", size = 3427031, upload-time = "2025-11-05T19:06:51.829Z" }, - { url = "https://files.pythonhosted.org/packages/25/3f/1a192b93bb47c6b44cd98ba8cc1d3d2a9308f1bb700c3017e6352da11bda/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c007d277218a50db8839e599ed78e0fffe5130f614c3f6d93ae257f282071a29", size = 2953260, upload-time = "2025-11-05T19:06:55.406Z" }, - { url = "https://files.pythonhosted.org/packages/5b/f8/93b582cad3531797c3db7c2db5400fd841538ccddfd9f5e3df61be99a630/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8565d4f5a0638da1bffde29832ed63c9e695c558611053add3b2dc0b56c92dbc", size = 3127044, upload-time = "2025-11-05T19:06:59.553Z" }, - { url = "https://files.pythonhosted.org/packages/1d/10/4327dbf87f75ae813405fd9a9b4a5cde63d506ffed0a096a440a4cabd89c/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:cbaa3bda75ef0d8836e1f8cc84af62f971b1d756d740efc95c38c3e04c0bfde2", size = 2932931, upload-time = "2025-11-05T19:07:01.437Z" }, - { url = "https://files.pythonhosted.org/packages/8a/c8/1774eec4f6f360ef57618fb8f52e3d3af245b2491bd0297513aa09eec04b/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:772922a9bd24e133950fad71eb1550836f415a88e8c77870e12d0c3bd688ddc2", size = 2996140, upload-time = "2025-11-05T19:07:03.438Z" }, - { url = "https://files.pythonhosted.org/packages/60/c3/3d1e01e2dba517a91760e4a03e4f20ffc75039a6fe584d0e6f9b5c78fd15/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:007b0476a1f331f8130783f901f1da6f5a7057af1a4891f1b6a31dec364189b5", size = 3205080, upload-time = "2025-11-05T19:07:05.078Z" }, - { url = "https://files.pythonhosted.org/packages/14/63/119de431572d7c70a7bf1037034a9be6ed0a7502a7498ba7302bca5b3242/openai_harmony-0.0.8-cp38-abi3-win32.whl", hash = "sha256:a9b5f893326b28d9e935ade14b4f655f5a840942473bc89b201c25f7a15af9cf", size = 2082457, upload-time = "2025-11-05T19:07:09.631Z" }, - { url = "https://files.pythonhosted.org/packages/40/1f/c83cf5a206c263ee70448a5ae4264682555f4d0b5bed0d2cc6ca1108103d/openai_harmony-0.0.8-cp38-abi3-win_amd64.whl", hash = "sha256:39d44f0d8f466bd56698e7ead708bead3141e27b9b87e3ab7d5a6d0e4a869ee5", size = 2438369, upload-time = "2025-11-05T19:07:08.1Z" }, -] - -[[package]] -name = "opencv-python-headless" -version = "5.0.0.93" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/99/76b7c80252aa83c1af16393454aafd125a0287101afe8deb0a6821af0e30/opencv_python_headless-5.0.0.93.tar.gz", hash = "sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c", size = 81817738, upload-time = "2026-07-02T07:01:06.039Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/7c/8c8097891c509d98cd128493835c95631c80be6a8f37ed9d25716c2e16f1/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f", size = 48322581, upload-time = "2026-07-02T05:50:34.207Z" }, - { url = "https://files.pythonhosted.org/packages/90/8c/eab2ad388c3cbab2a350c10c2ef19ce6bd099240afc31789032c996bab52/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:1e55af3abfb462eeeabe5c775f12bdb36216d8a93a3583d69e6bd6e1d6ba7d00", size = 34782894, upload-time = "2026-07-02T05:51:39.856Z" }, - { url = "https://files.pythonhosted.org/packages/ec/78/afca939f40ffe2b2380bfa86f812b2f7d4acc5a27b27dc41b49cad7ce7b4/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10818d91510e05c04568ae12b5cd120779c70c01bf897b001a6221fe430df80f", size = 36521085, upload-time = "2026-07-02T06:55:24.429Z" }, - { url = "https://files.pythonhosted.org/packages/2b/97/8170e9819764c47e436c130d3ff6cfb73b58f923eae9d3a03d8982b04aec/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09a872a157c1376ab922a69bbf22f9a95bcc7b658a9d8b436a60212b02b2eeb4", size = 56563598, upload-time = "2026-07-02T06:55:47.355Z" }, - { url = "https://files.pythonhosted.org/packages/3a/98/1a28a7101e31801042b3098871a74b76c61581d328ef40774ff4edb53a56/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:840bd717c21e5c11cadadc022a823315ea417f961213d06b4df010e019eb16f4", size = 39648433, upload-time = "2026-07-02T06:56:04.255Z" }, - { url = "https://files.pythonhosted.org/packages/9b/21/f6ef335f6e65724aa78b8d792b48d40a48c381715f1e62f5a5049e09d07e/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed709fdf9aa0bd1f2ed8549e71d19449b03a675bb581eb292285f6861953be37", size = 61204038, upload-time = "2026-07-02T06:56:41.823Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8f/b8756467ea991449a293797f6b3fa80fcfdd29598a0a60d1cd5715b96e61/opencv_python_headless-5.0.0.93-cp37-abi3-win32.whl", hash = "sha256:c6bcd96b185975ea240d22cfdb15a1f6d080cc95264cfbe2621f21bb144d89b9", size = 35411237, upload-time = "2026-07-02T05:50:12.901Z" }, - { url = "https://files.pythonhosted.org/packages/b8/88/763b967f7efd7226b82c9fae16d560cba049b1f0c036647e65c610fd636e/opencv_python_headless-5.0.0.93-cp37-abi3-win_amd64.whl", hash = "sha256:829717b6a95554f273e49e357cee3b3a2a26b6f4842fbc1bed2b45bdd8f87e0e", size = 43825962, upload-time = "2026-07-02T05:50:09.627Z" }, -] - -[[package]] -name = "opentelemetry-api" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f2/45/7af37fe54e5d3e66e7dcd7ba8b8aeee73f202bfac909cc94b8c4e428f9ac/opentelemetry_exporter_otlp-1.44.0.tar.gz", hash = "sha256:af1cde7c33ea8ed624bf04ac49a885730fe44c1f1ad698656e592c38f70ce106", size = 6090, upload-time = "2026-07-16T15:25:34.585Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/c3/7b466a9463944e70b37b744072a0c1b88a425dade3fff0631adec66c9bcc/opentelemetry_exporter_otlp-1.44.0-py3-none-any.whl", hash = "sha256:4a498fa8d8fd8be9e8e2d175fe5524a3fe581ccffadd8509db86526a5fb97051", size = 6727, upload-time = "2026-07-16T15:25:14.445Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-proto" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/47/80d9e9d468dc5de3af5096f5ccdb065fa4dd1470f74495cc53e59e397f47/opentelemetry_exporter_otlp_proto_grpc-1.44.0.tar.gz", hash = "sha256:40d1ae9e03fcc36de3cbac610cc99f35894938bff9cfd90fc4ec68bd85448463", size = 27225, upload-time = "2026-07-16T15:25:38.308Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/29/6ae42ba32b153ae0a44ae125f0caff2188bbe62d99c82d1768da30864e72/opentelemetry_exporter_otlp_proto_grpc-1.44.0-py3-none-any.whl", hash = "sha256:6a1a645ea182a2f59440c51fa8301d309f3324a8f9d65f8395584b064b67ee4e", size = 19624, upload-time = "2026-07-16T15:25:19.096Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-http" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, -] - -[[package]] -name = "opentelemetry-proto" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, -] - -[[package]] -name = "opentelemetry-sdk" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.65b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions-ai" -version = "0.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/24/02/10aeacc37a38a3a8fa16ff67bec1ae3bf882539f6f9efb0f70acf802ca2d/opentelemetry_semantic_conventions_ai-0.5.1.tar.gz", hash = "sha256:153906200d8c1d2f8e09bd78dbef526916023de85ac3dab35912bfafb69ff04c", size = 26533, upload-time = "2026-03-26T14:20:38.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/55/22/41fb05f1dc5fda2c468e05a41814c20859016c85117b66c8a257cae814f6/opentelemetry_semantic_conventions_ai-0.5.1-py3-none-any.whl", hash = "sha256:25aeb22bd261543b4898a73824026d96770e5351209c7d07a0b1314762b1f6e4", size = 11250, upload-time = "2026-03-26T14:20:37.108Z" }, -] - -[[package]] -name = "opt-einsum" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" }, -] - -[[package]] -name = "optax" -version = "0.2.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "chex" }, - { name = "etils", extra = ["epy"] }, - { name = "jax" }, - { name = "jaxlib" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/af/b5/f88a0d851547b2e6b2c7e7e6509ad66236b3e7019f1f095bb03dbaa61fa1/optax-0.2.4.tar.gz", hash = "sha256:4e05d3d5307e6dde4c319187ae36e6cd3a0c035d4ed25e9e992449a304f47336", size = 229717, upload-time = "2024-11-12T21:52:17.439Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/24/28d0bb21600a78e46754947333ec9a297044af884d360092eb8561575fe9/optax-0.2.4-py3-none-any.whl", hash = "sha256:db35c04e50b52596662efb002334de08c2a0a74971e4da33f467e84fac08886a", size = 319212, upload-time = "2024-11-12T21:52:15.446Z" }, -] - -[[package]] -name = "orbax-checkpoint" -version = "0.11.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "etils", extra = ["epath", "epy"] }, - { name = "humanize" }, - { name = "jax" }, - { name = "msgpack" }, - { name = "nest-asyncio" }, - { name = "numpy" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "simplejson" }, - { name = "tensorstore" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7e/b4/262439a3fe00064b53a5182c0149447304f5a2d5a328a92d64390c18d189/orbax_checkpoint-0.11.5.tar.gz", hash = "sha256:8331ff594980a241ba43eb59dd683e5b590b339cff32a7b72d78cb5a350030b4", size = 249258, upload-time = "2025-02-11T19:33:45.364Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/80/e659696b5b1c2ced427efedd2d9d29c1bc31d841ac8a031215aa38f6b2ae/orbax_checkpoint-0.11.5-py3-none-any.whl", hash = "sha256:b55a7a254ea0ab18237e8234a6ca8bf5522f589fcc2ac698cf6893d5e7ae3500", size = 342800, upload-time = "2025-02-11T19:33:43.507Z" }, -] - -[[package]] -name = "outlines-core" -version = "0.2.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/04/4a0812eb27c086cfd2e66e7ec9150f33e105912a9b7f8b335e3479f03a06/outlines_core-0.2.14.tar.gz", hash = "sha256:64808deed1591ca3029ff64346ceb974cd5d780c916ea82504951fe83523039e", size = 191539, upload-time = "2026-01-09T15:59:10.016Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/93/30b9188648a479b32be429a24166db47a7bfdb0f9a8aac4c6dcf569e0a52/outlines_core-0.2.14-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:95e6476d9702d2fcc4e85370dbbfb6933a46c816e9c90107f6ce36eb68b5d64a", size = 2049651, upload-time = "2026-01-09T15:58:28.549Z" }, - { url = "https://files.pythonhosted.org/packages/0d/06/f3557daa8e87d5b95f64de269a301d73ec3c2202ab897c3e1f1cb93eb1db/outlines_core-0.2.14-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:f04731a5e29a190e2cc9f692a1f3fb2414a645355ca7d01b83df43439c38bea8", size = 2201046, upload-time = "2026-01-09T15:58:29.958Z" }, - { url = "https://files.pythonhosted.org/packages/0c/67/d8acf778990964c951080d568284e858d466f27dfd6f2674781927faba1c/outlines_core-0.2.14-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:0e4c69f0a8565edb56464c4c9b6c291a10805f3a96dff84182980e90ae1a5e2f", size = 2049558, upload-time = "2026-01-09T15:58:31.003Z" }, - { url = "https://files.pythonhosted.org/packages/17/e1/0320b14b49b8379ced1ab195ecf5875dbd2267b90148847541f43bfde6c1/outlines_core-0.2.14-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:63f53cfd9614e754499ae86dd699f3abcecf42d6a4e58d80fd80347881d85960", size = 2197854, upload-time = "2026-01-09T15:58:32.39Z" }, - { url = "https://files.pythonhosted.org/packages/29/29/3a04944407207a5d214879ca5ca33c2bd3e65199a4e927051c1bdaaa4d50/outlines_core-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bb2060c240c4507f334965a8948dbeeb22007560d797f6debd92346c0b620cb", size = 2341426, upload-time = "2026-01-09T15:58:33.553Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a7/a77f746272504bac3f628047d56ea1731b61549a3e1d9bbfd226f2968246/outlines_core-0.2.14-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1de34681c7e0e7e1551fc9036e4fa3c57986336c905a10536591ceb6d869c258", size = 2236941, upload-time = "2026-01-09T15:58:35.118Z" }, - { url = "https://files.pythonhosted.org/packages/99/0d/9f599d938923ab8ceeff26fdf2f9ea53bea3c962085c4927a08338a32349/outlines_core-0.2.14-cp312-cp312-win32.whl", hash = "sha256:870e8e038853818cb202ccc8cde92251f300f96805bfcc3be1c883adda7b5297", size = 1842940, upload-time = "2026-01-09T15:58:36.544Z" }, - { url = "https://files.pythonhosted.org/packages/f8/df/0f145c52ebd156d80273e2f5278227ea57e0275b2aa863bed33f44f77923/outlines_core-0.2.14-cp312-cp312-win_amd64.whl", hash = "sha256:87b42440478764cce1353a87d8560ef82f3b39b9d753bfe93195ea3584f369e3", size = 2137266, upload-time = "2026-01-09T15:58:37.831Z" }, -] - -[[package]] -name = "packaging" -version = "26.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, -] - -[[package]] -name = "pandas" -version = "3.0.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e2/17/d7b106e05bfa642e8694451e7d3d759c6a241c5386a5d962e4f66c047e06/pandas-3.0.6.tar.gz", hash = "sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10", size = 4667686, upload-time = "2026-09-17T23:23:18.345Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/4c/597d588c055d4373cff19cbbc32d4dd046c7be8fadee957585b5ba9e5b24/pandas-3.0.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7dac2d65e9087e8e7b5a45fe15c4920911a221df061ab629943ce016489145c7", size = 10397411, upload-time = "2026-09-17T23:20:49.465Z" }, - { url = "https://files.pythonhosted.org/packages/18/8f/48907c7c707b61a8e5018c32e1a3f70623209bfb59020a2f6196159d3aa7/pandas-3.0.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9dab635a549e58a053c7b0fa054dc0bd7be22f0ed9a720f4a85d5fb993276172", size = 10051279, upload-time = "2026-09-17T23:20:52.409Z" }, - { url = "https://files.pythonhosted.org/packages/67/fa/613d867c3d9554a61bafdec6f79565c8a3e73235feb52cc4a72ad2e0fa6a/pandas-3.0.6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3dccb584123b399c07562ac4d62543e90ede49ddf8ce3c13ffc64cbe828c281", size = 10303429, upload-time = "2026-09-17T23:20:55.597Z" }, - { url = "https://files.pythonhosted.org/packages/cb/67/0c0f18e38d7f2d2af8c24b3315bc4046e73bbdd4a5540405506671ad0c0d/pandas-3.0.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0704044b676496b8350e023b09f174a26772456c974a2b11c36bebb558c9490d", size = 10788193, upload-time = "2026-09-17T23:20:58.617Z" }, - { url = "https://files.pythonhosted.org/packages/39/53/1b57f3162501fe36687e4870e1b918a6458ca7386b173af75663ace95857/pandas-3.0.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e7c1905ef02c3d6d43d9dbd5b6ccb4da4870a0b0c821bbc103fbdb6f3ad2707b", size = 11323501, upload-time = "2026-09-17T23:21:01.911Z" }, - { url = "https://files.pythonhosted.org/packages/f2/d2/b1182e8d39100369d7f13f4a125a3fb6b096fef112c46d0566c25781ff68/pandas-3.0.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:569e114072b24fc4970c12e2b4bab252671668a40b324318903380cab0254c0c", size = 11838173, upload-time = "2026-09-17T23:21:04.85Z" }, - { url = "https://files.pythonhosted.org/packages/c7/33/5b717af24d2f27995e51e0875a269dddd373045216e34cf62e3aa764eaa1/pandas-3.0.6-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:2a8fc94be2ee5f1d86f97aacd8cc566f81680b6498e76f3007421bb5d98151bf", size = 7118499, upload-time = "2026-09-17T23:21:07.661Z" }, - { url = "https://files.pythonhosted.org/packages/bd/2a/14b3b17cd75cef4a1ee1af4234eb41cc1afe4103b98c2d80e8916abfd42b/pandas-3.0.6-cp312-cp312-win_amd64.whl", hash = "sha256:3ef908d28590b3f42d7070e7ad8f9b34b442b260b7f3c1afb57e0040c58cdb1b", size = 9661439, upload-time = "2026-09-17T23:21:10.959Z" }, - { url = "https://files.pythonhosted.org/packages/3b/11/3d580a604a1e35d69f6676847bd12db7d14344bc677b717f4413e79c5d0d/pandas-3.0.6-cp312-cp312-win_arm64.whl", hash = "sha256:f4e7c52eb108d752e7592268108fd3e98efd76d83a3125cdd06c621c2e44359b", size = 8959806, upload-time = "2026-09-17T23:21:13.851Z" }, -] - -[[package]] -name = "partial-json-parser" -version = "0.2.1.1.post7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/6d/eed37d7ebc1e0bcd27b831c0cf1fe94881934316187c4b30d23f29ea0bd4/partial_json_parser-0.2.1.1.post7.tar.gz", hash = "sha256:86590e1ba6bcb6739a2dfc17d2323f028cb5884f4c6ce23db376999132c9a922", size = 10296, upload-time = "2025-11-17T07:27:41.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/32/658973117bf0fd82a24abbfb94fe73a5e86216e49342985e10acce54775a/partial_json_parser-0.2.1.1.post7-py3-none-any.whl", hash = "sha256:145119e5eabcf80cbb13844a6b50a85c68bf99d376f8ed771e2a3c3b03e653ae", size = 10877, upload-time = "2025-11-17T07:27:40.457Z" }, -] - -[[package]] -name = "pillow" -version = "12.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, - { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, - { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, - { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, - { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, - { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, - { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, - { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, -] - -[[package]] -name = "platformdirs" -version = "4.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/78/9b/560e4be8e26f6fd133a03630a8df0c663b9e8d61b4ade152b72005aec83b/platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0", size = 31953, upload-time = "2026-07-21T13:09:36.565Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/68/d8d58938dfb1370b266a1a729e6d77a985be23689a0496498ee17b2cbf90/platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74", size = 23247, upload-time = "2026-07-21T13:09:35.422Z" }, -] - -[[package]] -name = "plotext" -version = "5.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/d7/f75f397af966fe252d0d34ffd3cae765317fce2134f925f95e7d6725d1ce/plotext-5.3.2.tar.gz", hash = "sha256:52d1e932e67c177bf357a3f0fe6ce14d1a96f7f7d5679d7b455b929df517068e", size = 61967, upload-time = "2024-09-24T15:13:37.728Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/1e/12fe7c40cd2099a1f454518754ed229b01beaf3bbb343127f0cc13ce6c22/plotext-5.3.2-py3-none-any.whl", hash = "sha256:394362349c1ddbf319548cfac17ca65e6d5dfc03200c40dfdc0503b3e95a2283", size = 64047, upload-time = "2024-09-24T15:13:36.296Z" }, -] - -[[package]] -name = "plotly" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "narwhals" }, - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/c3/72b369f5ed7701b04ab0ea3dcf83e9bbce71c0b3bc6f07f87568550d09ea/plotly-7.1.0.tar.gz", hash = "sha256:f860166a4a3d78c69cb1f4a15f28a5c8283eade98a282a698f3bb853a449ace5", size = 6689315, upload-time = "2026-09-15T19:21:21.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/7d/905a3a3d51087515719058c94cfbda2ff0fc14417c20d557ae3e82d8b250/plotly-7.1.0-py3-none-any.whl", hash = "sha256:dbb7fa18afce40d0a8e80d1bf162eceb3faa0ce5a77fe741ad09a74cf78f53f3", size = 9692368, upload-time = "2026-09-15T19:21:18.331Z" }, -] - -[[package]] -name = "pooch" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "platformdirs" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/43/85ef45e8b36c6a48546af7b266592dc32d7f67837a6514d111bced6d7d75/pooch-1.9.0.tar.gz", hash = "sha256:de46729579b9857ffd3e741987a2f6d5e0e03219892c167c6578c0091fb511ed", size = 61788, upload-time = "2026-01-30T19:15:09.649Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl", hash = "sha256:f265597baa9f760d25ceb29d0beb8186c243d6607b0f60b83ecf14078dbc703b", size = 67175, upload-time = "2026-01-30T19:15:08.36Z" }, -] - -[[package]] -name = "portpicker" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "psutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4d/d0/cda2fc582f09510c84cd6b7d7b9e22a02d4e45dbad2b2ef1c6edd7847e00/portpicker-1.6.0.tar.gz", hash = "sha256:bd507fd6f96f65ee02781f2e674e9dc6c99bbfa6e3c39992e3916204c9d431fa", size = 25676, upload-time = "2023-08-15T04:37:08.865Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/2d/440e4d7041fff89f28f483733eb617127aa866135c2dc719e05893f089e1/portpicker-1.6.0-py3-none-any.whl", hash = "sha256:b2787a41404cf7edbe29b07b9e0ed863b09f2665dcc01c1eb0c2261c1e7d0755", size = 16613, upload-time = "2023-08-15T04:37:07.327Z" }, -] - -[[package]] -name = "prettytable" -version = "3.18.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/74/ba08d81e668ccfe8658d7520a307e63c19862c08eb4ccb26f356c5239a7a/prettytable-3.18.0.tar.gz", hash = "sha256:439217116152244369caf3d9f1caf2f9fe29b03bd79e88d2928c8e718c95d680", size = 76373, upload-time = "2026-06-22T16:07:50.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/be/2e6798ace5cc036f5d05d36b7b2fd85346f1a708c87060890b070d0ec607/prettytable-3.18.0-py3-none-any.whl", hash = "sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a", size = 37357, upload-time = "2026-06-22T16:07:48.595Z" }, -] - -[[package]] -name = "prometheus-client" -version = "0.26.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/73/f1334c29c2af4cd9dba6c7817e61b611bd0215e2eb5565c6064a4de18802/prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b", size = 92910, upload-time = "2026-07-24T19:36:41.893Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" }, -] - -[[package]] -name = "prometheus-fastapi-instrumentator" -version = "8.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "prometheus-client" }, - { name = "starlette" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/f4/cdcebf7094b03b99fba71ac8f56bd6f227973642662f49d272332d8419b3/prometheus_fastapi_instrumentator-8.1.0.tar.gz", hash = "sha256:b77f3043665e8d28e2bbd21017506195a43d9adf1d402d01bf95b494b7e560e1", size = 20492, upload-time = "2026-07-26T11:12:44.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/b9/91a2246e6cf01b7ccb14479803c8a50f9c258ae5c6a0f16ba3294820632b/prometheus_fastapi_instrumentator-8.1.0-py3-none-any.whl", hash = "sha256:b9f40b2cff3f7891ca0610b3ae4fc6ec723fd326b04bb659819aaeb821a0fc7d", size = 19649, upload-time = "2026-07-26T11:12:45.168Z" }, -] - -[[package]] -name = "propcache" -version = "0.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, - { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, - { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, - { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, - { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, - { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, - { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, - { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, - { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, - { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, - { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, - { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, - { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, - { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, - { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, - { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, - { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, -] - -[[package]] -name = "proto-plus" -version = "1.28.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/73/3e/29e0d6a2c5adde6ab5772253fd16ab346324026b89a66e354689c86d0584/proto_plus-1.28.2.tar.gz", hash = "sha256:26d843eb99c1e32fdf1d20ff0faae56607f7748fe774acf9ecd5cfe6c6472501", size = 58063, upload-time = "2026-07-22T16:28:29.119Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/84/4e9a53a062d4073c74897a6bd20fff74d55307341b3e85c081002462b3ef/proto_plus-1.28.2-py3-none-any.whl", hash = "sha256:b874236fcac2358f601e4330bcb76cb8b89c851303ccf4078408b3d4774d1c52", size = 50693, upload-time = "2026-07-22T16:28:24.059Z" }, -] - -[[package]] -name = "protobuf" -version = "6.33.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, -] - -[[package]] -name = "psutil" -version = "7.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, - { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, - { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, - { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, - { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, - { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, - { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, - { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, -] - -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - -[[package]] -name = "pyarrow" -version = "26.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/34/17c34cb38e5d940e38f0f0d9fdfa0e8a506676409ea9b85aff7e3079f831/pyarrow-26.0.0.tar.gz", hash = "sha256:0cccd36e00ea3afeb52ded61f2721ce71f604853d70c45365c58324eb773d6ae", size = 1239433, upload-time = "2026-10-09T08:26:25.315Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/60/6793778f2617cce469383dac0ba08c4f2401cf342df0c7b9ca53939d9b46/pyarrow-26.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:90ddaf7c625307ad52f31a9b25c34fe5e4897c7529ee3481135822b2b6842ff1", size = 36333953, upload-time = "2026-10-09T08:14:00.387Z" }, - { url = "https://files.pythonhosted.org/packages/db/81/f944cc63ce8a753e5fbff25de6d1d475ebd7fffdf9cf98c65130294fc896/pyarrow-26.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee341973f78a0b46e073d065e88e75026a9c584051e97f98a0d05d96c6bac7dd", size = 38688456, upload-time = "2026-10-09T08:14:04.344Z" }, - { url = "https://files.pythonhosted.org/packages/f5/2d/7e5c722fa5d5d9f3b75e62fe11694b34217664d4f05ac88031197166b277/pyarrow-26.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:01c863a18bd9c8412453dd0d92de6d0ee7b2b3d6fb079d9734a4b2a3c8bd4453", size = 50867603, upload-time = "2026-10-09T08:14:09.115Z" }, - { url = "https://files.pythonhosted.org/packages/88/e4/9cd356d906e71bd79b0c3fc5c9a54e01a0020dcf14c152ccfbcb503c7298/pyarrow-26.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6a628922ba20705fa964ca73e4ef959c2fb2f14b9bbec5589a6a1e68e6257c85", size = 53931932, upload-time = "2026-10-09T08:14:24.051Z" }, - { url = "https://files.pythonhosted.org/packages/bb/e4/5bae3133b7fe04c24907a20f3bc1fba388cbbde659199e7b76445982047a/pyarrow-26.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:954d971b363b16ee41f89389a4053315dc71265f2ce5c2468eb0a910b1166268", size = 54444720, upload-time = "2026-10-09T08:14:31.214Z" }, - { url = "https://files.pythonhosted.org/packages/ba/b4/ee422493bb6dafdbef776cfe2c2a73106a1063a79bf4e78d1e5f51176885/pyarrow-26.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5d5768d03426abe6526d5274adefa00abf00a7f81118c46e98b5a46390f5549e", size = 57388949, upload-time = "2026-10-09T08:14:38.964Z" }, - { url = "https://files.pythonhosted.org/packages/54/3c/1783aab1dac28e175dcf26dfc7123725efc474caecaed91e8a34cb89cad0/pyarrow-26.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cc903e1069e9dd5e9dcf780324c0112e27e051e422ecfaff574fb33ed65d9160", size = 28567581, upload-time = "2026-10-09T08:14:44.279Z" }, -] - -[[package]] -name = "pyasn1" -version = "0.6.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - -[[package]] -name = "pybase64" -version = "1.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/b8/4ed5c7ad5ec15b08d35cc79ace6145d5c1ae426e46435f4987379439dfea/pybase64-1.4.3.tar.gz", hash = "sha256:c2ed274c9e0ba9c8f9c4083cfe265e66dd679126cd9c2027965d807352f3f053", size = 137272, upload-time = "2025-12-06T13:27:04.013Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/a7/efcaa564f091a2af7f18a83c1c4875b1437db56ba39540451dc85d56f653/pybase64-1.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:18d85e5ab8b986bb32d8446aca6258ed80d1bafe3603c437690b352c648f5967", size = 38167, upload-time = "2025-12-06T13:23:16.821Z" }, - { url = "https://files.pythonhosted.org/packages/db/c7/c7ad35adff2d272bf2930132db2b3eea8c44bb1b1f64eb9b2b8e57cde7b4/pybase64-1.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3f5791a3491d116d0deaf4d83268f48792998519698f8751efb191eac84320e9", size = 31673, upload-time = "2025-12-06T13:23:17.835Z" }, - { url = "https://files.pythonhosted.org/packages/43/1b/9a8cab0042b464e9a876d5c65fe5127445a2436da36fda64899b119b1a1b/pybase64-1.4.3-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f0b3f200c3e06316f6bebabd458b4e4bcd4c2ca26af7c0c766614d91968dee27", size = 68210, upload-time = "2025-12-06T13:23:18.813Z" }, - { url = "https://files.pythonhosted.org/packages/62/f7/965b79ff391ad208b50e412b5d3205ccce372a2d27b7218ae86d5295b105/pybase64-1.4.3-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb632edfd132b3eaf90c39c89aa314beec4e946e210099b57d40311f704e11d4", size = 71599, upload-time = "2025-12-06T13:23:20.195Z" }, - { url = "https://files.pythonhosted.org/packages/03/4b/a3b5175130b3810bbb8ccfa1edaadbd3afddb9992d877c8a1e2f274b476e/pybase64-1.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:356ef1d74648ce997f5a777cf8f1aefecc1c0b4fe6201e0ef3ec8a08170e1b54", size = 59922, upload-time = "2025-12-06T13:23:21.487Z" }, - { url = "https://files.pythonhosted.org/packages/da/5d/c38d1572027fc601b62d7a407721688b04b4d065d60ca489912d6893e6cf/pybase64-1.4.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:c48361f90db32bacaa5518419d4eb9066ba558013aaf0c7781620279ecddaeb9", size = 56712, upload-time = "2025-12-06T13:23:22.77Z" }, - { url = "https://files.pythonhosted.org/packages/e7/d4/4e04472fef485caa8f561d904d4d69210a8f8fc1608ea15ebd9012b92655/pybase64-1.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:702bcaa16ae02139d881aeaef5b1c8ffb4a3fae062fe601d1e3835e10310a517", size = 59300, upload-time = "2025-12-06T13:23:24.543Z" }, - { url = "https://files.pythonhosted.org/packages/86/e7/16e29721b86734b881d09b7e23dfd7c8408ad01a4f4c7525f3b1088e25ec/pybase64-1.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:53d0ffe1847b16b647c6413d34d1de08942b7724273dd57e67dcbdb10c574045", size = 60278, upload-time = "2025-12-06T13:23:25.608Z" }, - { url = "https://files.pythonhosted.org/packages/b1/02/18515f211d7c046be32070709a8efeeef8a0203de4fd7521e6b56404731b/pybase64-1.4.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9a1792e8b830a92736dae58f0c386062eb038dfe8004fb03ba33b6083d89cd43", size = 54817, upload-time = "2025-12-06T13:23:26.633Z" }, - { url = "https://files.pythonhosted.org/packages/e7/be/14e29d8e1a481dbff151324c96dd7b5d2688194bb65dc8a00ca0e1ad1e86/pybase64-1.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d468b1b1ac5ad84875a46eaa458663c3721e8be5f155ade356406848d3701f6", size = 58611, upload-time = "2025-12-06T13:23:27.684Z" }, - { url = "https://files.pythonhosted.org/packages/b4/8a/a2588dfe24e1bbd742a554553778ab0d65fdf3d1c9a06d10b77047d142aa/pybase64-1.4.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e97b7bdbd62e71898cd542a6a9e320d9da754ff3ebd02cb802d69087ee94d468", size = 52404, upload-time = "2025-12-06T13:23:28.714Z" }, - { url = "https://files.pythonhosted.org/packages/27/fc/afcda7445bebe0cbc38cafdd7813234cdd4fc5573ff067f1abf317bb0cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b33aeaa780caaa08ffda87fc584d5eab61e3d3bbb5d86ead02161dc0c20d04bc", size = 68817, upload-time = "2025-12-06T13:23:30.079Z" }, - { url = "https://files.pythonhosted.org/packages/d3/3a/87c3201e555ed71f73e961a787241a2438c2bbb2ca8809c29ddf938a3157/pybase64-1.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c0efcf78f11cf866bed49caa7b97552bc4855a892f9cc2372abcd3ed0056f0d", size = 57854, upload-time = "2025-12-06T13:23:31.17Z" }, - { url = "https://files.pythonhosted.org/packages/fd/7d/931c2539b31a7b375e7d595b88401eeb5bd6c5ce1059c9123f9b608aaa14/pybase64-1.4.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66e3791f2ed725a46593f8bd2761ff37d01e2cdad065b1dceb89066f476e50c6", size = 54333, upload-time = "2025-12-06T13:23:32.422Z" }, - { url = "https://files.pythonhosted.org/packages/de/5e/537601e02cc01f27e9d75f440f1a6095b8df44fc28b1eef2cd739aea8cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:72bb0b6bddadab26e1b069bb78e83092711a111a80a0d6b9edcb08199ad7299b", size = 56492, upload-time = "2025-12-06T13:23:33.515Z" }, - { url = "https://files.pythonhosted.org/packages/96/97/2a2e57acf8f5c9258d22aba52e71f8050e167b29ed2ee1113677c1b600c1/pybase64-1.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5b3365dbcbcdb0a294f0f50af0c0a16b27a232eddeeb0bceeefd844ef30d2a23", size = 70974, upload-time = "2025-12-06T13:23:36.27Z" }, - { url = "https://files.pythonhosted.org/packages/75/2e/a9e28941c6dab6f06e6d3f6783d3373044be9b0f9a9d3492c3d8d2260ac0/pybase64-1.4.3-cp312-cp312-win32.whl", hash = "sha256:7bca1ed3a5df53305c629ca94276966272eda33c0d71f862d2d3d043f1e1b91a", size = 33686, upload-time = "2025-12-06T13:23:37.848Z" }, - { url = "https://files.pythonhosted.org/packages/83/e3/507ab649d8c3512c258819c51d25c45d6e29d9ca33992593059e7b646a33/pybase64-1.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:9f2da8f56d9b891b18b4daf463a0640eae45a80af548ce435be86aa6eff3603b", size = 35833, upload-time = "2025-12-06T13:23:38.877Z" }, - { url = "https://files.pythonhosted.org/packages/bc/8a/6eba66cd549a2fc74bb4425fd61b839ba0ab3022d3c401b8a8dc2cc00c7a/pybase64-1.4.3-cp312-cp312-win_arm64.whl", hash = "sha256:0631d8a2d035de03aa9bded029b9513e1fee8ed80b7ddef6b8e9389ffc445da0", size = 31185, upload-time = "2025-12-06T13:23:39.908Z" }, - { url = "https://files.pythonhosted.org/packages/17/45/92322aec1b6979e789b5710f73c59f2172bc37c8ce835305434796824b7b/pybase64-1.4.3-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:2baaa092f3475f3a9c87ac5198023918ea8b6c125f4c930752ab2cbe3cd1d520", size = 38746, upload-time = "2025-12-06T13:26:25.869Z" }, - { url = "https://files.pythonhosted.org/packages/11/94/f1a07402870388fdfc2ecec0c718111189732f7d0f2d7fe1386e19e8fad0/pybase64-1.4.3-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:cde13c0764b1af07a631729f26df019070dad759981d6975527b7e8ecb465b6c", size = 32573, upload-time = "2025-12-06T13:26:27.792Z" }, - { url = "https://files.pythonhosted.org/packages/fa/8f/43c3bb11ca9bacf81cb0b7a71500bb65b2eda6d5fe07433c09b543de97f3/pybase64-1.4.3-graalpy312-graalpy250_312_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5c29a582b0ea3936d02bd6fe9bf674ab6059e6e45ab71c78404ab2c913224414", size = 43461, upload-time = "2025-12-06T13:26:28.906Z" }, - { url = "https://files.pythonhosted.org/packages/2d/4c/2a5258329200be57497d3972b5308558c6de42e3749c6cc2aa1cbe34b25a/pybase64-1.4.3-graalpy312-graalpy250_312_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6b664758c804fa919b4f1257aa8cf68e95db76fc331de5f70bfc3a34655afe1", size = 36058, upload-time = "2025-12-06T13:26:30.092Z" }, - { url = "https://files.pythonhosted.org/packages/ea/6d/41faa414cde66ec023b0ca8402a8f11cb61731c3dc27c082909cbbd1f929/pybase64-1.4.3-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:f7537fa22ae56a0bf51e4b0ffc075926ad91c618e1416330939f7ef366b58e3b", size = 36231, upload-time = "2025-12-06T13:26:31.656Z" }, -] - -[[package]] -name = "pycountry" -version = "26.2.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/1d/061b9e7a48b85cfd69f33c33d2ef784a531c359399ad764243399673c8f5/pycountry-26.2.16.tar.gz", hash = "sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801", size = 7711342, upload-time = "2026-02-17T03:42:52.367Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/42/7703bd45b62fecd44cd7d3495423097e2f7d28bc2e99e7c1af68892ab157/pycountry-26.2.16-py3-none-any.whl", hash = "sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a", size = 8044600, upload-time = "2026-02-17T03:42:49.777Z" }, -] - -[[package]] -name = "pycparser" -version = "3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, -] - -[[package]] -name = "pydantic" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/84/6b/69fd5c7194b21ebde0f8637e2a4ddc766ada29d472bfa6a5ca533d79549a/pydantic-2.13.0.tar.gz", hash = "sha256:b89b575b6e670ebf6e7448c01b41b244f471edd276cd0b6fe02e7e7aca320070", size = 843468, upload-time = "2026-04-13T10:51:35.571Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/01/d7/c3a52c61f5b7be648e919005820fbac33028c6149994cd64453f49951c17/pydantic-2.13.0-py3-none-any.whl", hash = "sha256:ab0078b90da5f3e2fd2e71e3d9b457ddcb35d0350854fbda93b451e28d56baaf", size = 471872, upload-time = "2026-04-13T10:51:33.343Z" }, -] - -[package.optional-dependencies] -email = [ - { name = "email-validator" }, -] - -[[package]] -name = "pydantic-core" -version = "2.46.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/0a/9414cddf82eda3976b14048cc0fa8f5b5d1aecb0b22e1dcd2dbfe0e139b1/pydantic_core-2.46.0.tar.gz", hash = "sha256:82d2498c96be47b47e903e1378d1d0f770097ec56ea953322f39936a7cf34977", size = 471441, upload-time = "2026-04-13T09:06:33.813Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/d2/206c72ad47071559142a35f71efc29eb16448a4a5ae9487230ab8e4e292b/pydantic_core-2.46.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:66ccedb02c934622612448489824955838a221b3a35875458970521ef17b2f9c", size = 2117060, upload-time = "2026-04-13T09:04:47.443Z" }, - { url = "https://files.pythonhosted.org/packages/17/2c/7a53b33f91c8b77e696b1a6aa3bed609bf9374bdc0f8dcda681bc7d922b8/pydantic_core-2.46.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a44f27f4d2788ef9876ec47a43739b118c5904d74f418f53398f6ced3bbcacf2", size = 1951802, upload-time = "2026-04-13T09:05:34.591Z" }, - { url = "https://files.pythonhosted.org/packages/fc/20/90e548c1f6d38800ef11c915881525770ce270d8e5e887563ff046a08674/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f26a1032bcce6ca4b4670eb3f7d8195bd0a8b8f255f1307823e217ca3cfa7c27", size = 1976621, upload-time = "2026-04-13T09:04:03.909Z" }, - { url = "https://files.pythonhosted.org/packages/20/3c/9c5810ca70b60c623488cdd80f7e9ee1a0812df81e97098b64788719860f/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b8d1412f725060527e56675904b17a2d421dddcf861eecf7c75b9dda47921a4", size = 2056721, upload-time = "2026-04-13T09:04:40.992Z" }, - { url = "https://files.pythonhosted.org/packages/1a/a3/d6e5f4cdec84278431c75540f90838c9d0a4dfe9402a8f3902073660ff28/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc3d1569edd859cabaa476cabce9eecd05049a7966af7b4a33b541bfd4ca1104", size = 2239634, upload-time = "2026-04-13T09:03:52.478Z" }, - { url = "https://files.pythonhosted.org/packages/46/42/ef58aacf330d8de6e309d62469aa1f80e945eaf665929b4037ac1bfcebc1/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:38108976f2d8afaa8f5067fd1390a8c9f5cc580175407cda636e76bc76e88054", size = 2315739, upload-time = "2026-04-13T09:05:04.971Z" }, - { url = "https://files.pythonhosted.org/packages/8b/86/c63b12fafa2d86a515bfd1840b39c23a49302f02b653161bf9c3a0566c50/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5a06d8ed01dad5575056b5187e5959b336793c6047920a3441ee5b03533836", size = 2098169, upload-time = "2026-04-13T09:07:27.151Z" }, - { url = "https://files.pythonhosted.org/packages/76/19/b5b33a2f6be4755b21a20434293c4364be255f4c1a108f125d101d4cc4ee/pydantic_core-2.46.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:04017ace142da9ce27cafd423a480872571b5c7e80382aec22f7d715ca8eb870", size = 2170830, upload-time = "2026-04-13T09:04:39.448Z" }, - { url = "https://files.pythonhosted.org/packages/99/ae/7559f99a29b7d440012ddb4da897359304988a881efaca912fd2f655652e/pydantic_core-2.46.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2629ad992ed1b1c012e6067f5ffafd3336fcb9b54569449fabb85621f1444ed3", size = 2203901, upload-time = "2026-04-13T09:04:01.048Z" }, - { url = "https://files.pythonhosted.org/packages/dd/0e/b0ef945a39aeb4ac58da316813e1106b7fbdfbf20ac141c1c27904355ac5/pydantic_core-2.46.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3068b1e7bd986aebc88f6859f8353e72072538dcf92a7fb9cf511a0f61c5e729", size = 2191789, upload-time = "2026-04-13T09:06:39.915Z" }, - { url = "https://files.pythonhosted.org/packages/90/f4/830484e07188c1236b013995818888ab93bab8fd88aa9689b1d8fd22220d/pydantic_core-2.46.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:1e366916ff69ff700aa9326601634e688581bc24c5b6b4f8738d809ec7d72611", size = 2344423, upload-time = "2026-04-13T09:05:12.252Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ba/e455c18cbdc333177af754e740be4fe9d1de173d65bbe534daf88da02ac0/pydantic_core-2.46.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:485a23e8f4618a1b8e23ac744180acde283fffe617f96923d25507d5cade62ec", size = 2384037, upload-time = "2026-04-13T09:06:24.503Z" }, - { url = "https://files.pythonhosted.org/packages/78/1f/b35d20d73144a41e78de0ae398e60fdd8bed91667daa1a5a92ab958551ba/pydantic_core-2.46.0-cp312-cp312-win32.whl", hash = "sha256:520940e1b702fe3b33525d0351777f25e9924f1818ca7956447dabacf2d339fd", size = 1967068, upload-time = "2026-04-13T09:05:23.374Z" }, - { url = "https://files.pythonhosted.org/packages/d1/84/4b6252e9606e8295647b848233cc4137ee0a04ebba8f0f9fb2977655b38c/pydantic_core-2.46.0-cp312-cp312-win_amd64.whl", hash = "sha256:90d2048e0339fa365e5a66aefe760ddd3b3d0a45501e088bc5bc7f4ed9ff9571", size = 2071008, upload-time = "2026-04-13T09:05:21.392Z" }, - { url = "https://files.pythonhosted.org/packages/39/95/d08eb508d4d5560ccbd226ee5971e5ef9b749aba9b413c0c4ed6e406d4f6/pydantic_core-2.46.0-cp312-cp312-win_arm64.whl", hash = "sha256:a70247649b7dffe36648e8f34be5ce8c5fa0a27ff07b071ea780c20a738c05ce", size = 2036634, upload-time = "2026-04-13T09:05:48.299Z" }, - { url = "https://files.pythonhosted.org/packages/74/0c/106ed5cc50393d90523f09adcc50d05e42e748eb107dc06aea971137f02d/pydantic_core-2.46.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:bc0e2fefe384152d7da85b5c2fe8ce2bf24752f68a58e3f3ea42e28a29dfdeb2", size = 2104968, upload-time = "2026-04-13T09:06:26.967Z" }, - { url = "https://files.pythonhosted.org/packages/f5/71/b494cef3165e3413ee9bbbb5a9eedc9af0ea7b88d8638beef6c2061b110e/pydantic_core-2.46.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:a2ab0e785548be1b4362a62c4004f9217598b7ee465f1f420fc2123e2a5b5b02", size = 1940442, upload-time = "2026-04-13T09:06:29.332Z" }, - { url = "https://files.pythonhosted.org/packages/7e/3e/a4d578c8216c443e26a1124f8c1e07c0654264ce5651143d3883d85ff140/pydantic_core-2.46.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16d45aecb18b8cba1c68eeb17c2bb2d38627ceed04c5b30b882fc9134e01f187", size = 1999672, upload-time = "2026-04-13T09:04:42.798Z" }, - { url = "https://files.pythonhosted.org/packages/cd/c1/9114560468685525a21770138382fd0cb849aaf351ff2c7b97f760d121e0/pydantic_core-2.46.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5078f6c377b002428e984259ac327ef8902aacae6c14b7de740dd4869a491501", size = 2154533, upload-time = "2026-04-13T09:04:50.868Z" }, -] - -[[package]] -name = "pydantic-extra-types" -version = "2.11.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/71/dba38ee2651f84f7842206adbd2233d8bbdb59fb85e9fa14232486a8c471/pydantic_extra_types-2.11.1.tar.gz", hash = "sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049", size = 172002, upload-time = "2026-03-16T08:08:03.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl", hash = "sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1", size = 79526, upload-time = "2026-03-16T08:08:02.533Z" }, -] - -[package.optional-dependencies] -pycountry = [ - { name = "pycountry" }, -] - -[[package]] -name = "pydantic-settings" -version = "2.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, -] - -[[package]] -name = "pygments" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, -] - -[[package]] -name = "pyjwt" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, -] - -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - -[[package]] -name = "pynvvideocodec" -version = "2.0.4" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/1c/78f6fdf85133157a6a3405eab5ef4c2bc8048194dbda1c91bb9b8645bb36/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bad9e25f494abdcfa8f9dffa33a840509eda3ffcdf6e7cf6465d73be307c0c82", size = 28630316, upload-time = "2026-07-08T04:25:54.596Z" }, - { url = "https://files.pythonhosted.org/packages/ca/49/98da271686e00676f41b1197ba5431ddc341b96d8efb68ea9d68e2b0d870/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b59cec7a1a3f78fad13fead78cad8b6d9686827f9ff4477080245457675a01d0", size = 43176147, upload-time = "2026-05-27T04:04:08.297Z" }, - { url = "https://files.pythonhosted.org/packages/42/80/7b13c12fd5f3243b01190130ce098a44ddf62e030e6ed712911cbfe40311/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:a0daa28b09705806c8c6b26326df217c45e60c0a12a673ea3ea6ee5e2e7193b0", size = 35754893, upload-time = "2026-05-27T04:04:43.866Z" }, - { url = "https://files.pythonhosted.org/packages/76/50/eb1571ab1cee8ebb8a7bdfc355078beebe4b2bb2e5c6ad5d0e18ab8585db/pynvvideocodec-2.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:46e2adb82dc6ac333d3535cc76e4e25c7e8d80dd272b1aba0c28702b861d5261", size = 25692590, upload-time = "2026-05-27T04:05:17.164Z" }, -] - -[[package]] -name = "pyparsing" -version = "3.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/11/b213bebff182584360cb8d17c72c1677fec5c5c228de439e63bcf8ab1c8f/pyparsing-3.3.3.tar.gz", hash = "sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36", size = 1050487, upload-time = "2026-09-20T20:59:05.609Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4", size = 126420, upload-time = "2026-09-20T20:59:04.025Z" }, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - -[[package]] -name = "python-dotenv" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, -] - -[[package]] -name = "python-json-logger" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/ff/3cc9165fd44106973cd7ac9facb674a65ed853494592541d339bdc9a30eb/python_json_logger-4.1.0.tar.gz", hash = "sha256:b396b9e3ed782b09ff9d6e4f1683d46c83ad0d35d2e407c09a9ebbf038f88195", size = 17573, upload-time = "2026-03-29T04:39:56.805Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/be/0631a861af4d1c875f096c07d34e9a63639560a717130e7a87cbc82b7e3f/python_json_logger-4.1.0-py3-none-any.whl", hash = "sha256:132994765cf75bf44554be9aa49b06ef2345d23661a96720262716438141b6b2", size = 15021, upload-time = "2026-03-29T04:39:55.266Z" }, -] - -[[package]] -name = "python-multipart" -version = "0.0.32" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, -] - -[[package]] -name = "pywin32" -version = "312" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" }, - { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" }, - { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" }, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, -] - -[[package]] -name = "pyzmq" -version = "27.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:452631b640340c928fa343801b0d07eb0c3789a5ffa843f6e1a9cee0ba4eb4fc", size = 1306279, upload-time = "2025-09-08T23:08:03.807Z" }, - { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, - { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, - { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, - { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, - { url = "https://files.pythonhosted.org/packages/e6/2f/104c0a3c778d7c2ab8190e9db4f62f0b6957b53c9d87db77c284b69f33ea/pyzmq-27.1.0-cp312-abi3-win32.whl", hash = "sha256:250e5436a4ba13885494412b3da5d518cd0d3a278a1ae640e113c073a5f88edd", size = 559184, upload-time = "2025-09-08T23:08:15.163Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7f/a21b20d577e4100c6a41795842028235998a643b1ad406a6d4163ea8f53e/pyzmq-27.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:9ce490cf1d2ca2ad84733aa1d69ce6855372cb5ce9223802450c9b2a7cba0ccf", size = 619480, upload-time = "2025-09-08T23:08:17.192Z" }, - { url = "https://files.pythonhosted.org/packages/78/c2/c012beae5f76b72f007a9e91ee9401cb88c51d0f83c6257a03e785c81cc2/pyzmq-27.1.0-cp312-abi3-win_arm64.whl", hash = "sha256:75a2f36223f0d535a0c919e23615fc85a1e23b71f40c7eb43d7b1dedb4d8f15f", size = 552993, upload-time = "2025-09-08T23:08:18.926Z" }, -] - -[[package]] -name = "quack-kernels" -version = "0.6.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "einops" }, - { name = "nvidia-cutlass-dsl" }, - { name = "torch" }, - { name = "torch-c-dlpack-ext" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/75/5c/67c4a0d54cbb8d4d05af73132d620d1900a193b33e0d5ea6a25829b941d2/quack_kernels-0.6.4.tar.gz", hash = "sha256:8bf08a0e1a85aecc892ce84f4b6ed7abb6a44ec7c68b83040abe174bc8c2b936", size = 845485, upload-time = "2026-08-07T23:25:31.94Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/f5/36ec7e075e90bd92bdf30fee0606f189348c2bbb0af6aebaf7382927c622/quack_kernels-0.6.4-py3-none-any.whl", hash = "sha256:e77c5d1f1299b0b38487fe8737df6c6975daa16bca7f7eb883bd1a74d09e7e78", size = 728458, upload-time = "2026-08-07T23:25:30.549Z" }, -] - -[[package]] -name = "ray" -version = "2.56.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "jsonschema" }, - { name = "msgpack" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "requests" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/50/d9/a17feef16a123f5d32d4c5fa7de853c59ba702f8404cf452a7ce20faca13/ray-2.56.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:44bc0000c5bfad85b2ff6e0ef91e95f901d1a2d2fdd72f94f08a046eb494cd61", size = 66346989, upload-time = "2026-07-17T21:28:39.4Z" }, - { url = "https://files.pythonhosted.org/packages/05/19/c3b1bcccd09decaf2a2e3370041ae67070bb1a8638f2665d36edfbb0261d/ray-2.56.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:8fdd6b096215906cf1f9acdc7898c9d6140606f2d27245778b8385a9f19e6cb0", size = 73319289, upload-time = "2026-07-17T21:28:44.502Z" }, - { url = "https://files.pythonhosted.org/packages/d2/7f/577a61bf2c8eff26e942afe53a6b03f4dbf5b4f233b832c228417d0c954e/ray-2.56.1-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:e5d3173696831134c76bd09451dfe95c32d72c271253b0d6b09d2df9994aa660", size = 74194147, upload-time = "2026-07-17T21:28:50.567Z" }, - { url = "https://files.pythonhosted.org/packages/00/e9/0fd1223597f9ca98ec496ec726043918a5301547789b1be95a635ec82649/ray-2.56.1-cp312-cp312-win_amd64.whl", hash = "sha256:8052573ee5ef8c4fdd7aeb6a257c80542e69c48f3f6d117101f95c970ffdc7e2", size = 28373294, upload-time = "2026-07-17T21:28:55.122Z" }, -] - -[[package]] -name = "referencing" -version = "0.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "rpds-py" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, -] - -[[package]] -name = "regex" -version = "2026.7.19" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, - { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, - { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, - { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, - { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, - { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, - { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, - { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, - { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, - { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, - { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, - { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, - { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, - { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, - { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, -] - -[[package]] -name = "requests" -version = "2.34.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, -] - -[[package]] -name = "requests-oauthlib" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "oauthlib" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, -] - -[[package]] -name = "rich" -version = "15.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, -] - -[[package]] -name = "rich-toolkit" -version = "0.20.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "rich" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/3a/a258c2fbc6c6bdf428611388f5698ba5d57ffdf0755e1cab474d9cc47813/rich_toolkit-0.20.3.tar.gz", hash = "sha256:223dd2cfba325ed55e94933b9e53f3aca13e9fdf76622bd564c18109a2273c1b", size = 205355, upload-time = "2026-07-13T14:38:06.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/ce/639d0d0ce3d25c5edbd1afecd308bb35dc04883a45ac9f0855c8aee4e919/rich_toolkit-0.20.3-py3-none-any.whl", hash = "sha256:419aa87516d5f3849cca553c6dcf707c02a36d508fcf996946606725d34a3002", size = 36195, upload-time = "2026-07-13T14:38:05.687Z" }, -] - -[[package]] -name = "rignore" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/77/6ba90ab4a538d3ec244329c57c4d26a78c8313ea6fa72c8768d46f11c1c9/rignore-0.8.0.tar.gz", hash = "sha256:2e5ad6b19834f04a877d26fe863fd77ed851ed4019fdca097fb1b744311e3562", size = 55358, upload-time = "2026-07-17T19:01:21.257Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/7e/0d270c1ed723b82bea8ccd504185acb5d71830975260e8de02af7acff728/rignore-0.8.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a5e285ec58b3b66284a7f48805e4db0ea948370deb484a4935b147187ecf1e25", size = 847184, upload-time = "2026-07-17T18:58:31.348Z" }, - { url = "https://files.pythonhosted.org/packages/ff/dc/841941f8b0883a8038f9d540607238c456df20e98243a61142fd15699806/rignore-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:367d3cf401b477a5ba7eb05b5b94c491b0d704507d9eaf80378a8d843fe00674", size = 816590, upload-time = "2026-07-17T18:58:32.571Z" }, - { url = "https://files.pythonhosted.org/packages/57/53/9e047a6cd95b553519703350f7a3530ddd31d309c9ade1ec913db5882f74/rignore-0.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d49870c032abcc28db2210ad92b3bd5706ec792ab25c9295a6c536a4b92226ce", size = 884130, upload-time = "2026-07-17T18:58:33.921Z" }, - { url = "https://files.pythonhosted.org/packages/07/5a/ae444f30dfa47716ccadf9ed9512736494c0bf5222f330845a637f1c569c/rignore-0.8.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d50632823c273ee19212fd939b04e9b55d5a6e0c9b998fadef6dc9a0c2f6aecc", size = 857230, upload-time = "2026-07-17T18:58:35.32Z" }, - { url = "https://files.pythonhosted.org/packages/ba/67/b2ddfbf5a42a8ea89e6a0330189f7f76f9113e0d8c6a4c68b68ff68f139c/rignore-0.8.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b3337e69856ba18c9d079e0ce8b342dc256f48585ae3e1c7aac67682b672e83a", size = 1133331, upload-time = "2026-07-17T18:58:36.895Z" }, - { url = "https://files.pythonhosted.org/packages/8e/76/291be16f7260ac87dedab41394f7738834bb1cd05d6e1b878eea8d598520/rignore-0.8.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1264c48aaff09c6431b428144e9e9fbd0d7864673f7bd752b51d2409971275e", size = 912520, upload-time = "2026-07-17T18:58:38.178Z" }, - { url = "https://files.pythonhosted.org/packages/0c/99/cd0df2def95ecfa93c781431ce7c58ea52f90742550ff327ed0f1f715f14/rignore-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8912e3cb4034b972ba75c9ecabb79bc41a1076cdf726e53019e45b71b82da56", size = 927109, upload-time = "2026-07-17T18:58:40Z" }, - { url = "https://files.pythonhosted.org/packages/e4/58/ce8af7214b903a2f86c3223dabd23e03a0c499df6559be5c9e2b15b4344c/rignore-0.8.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:43b20be0f1c8ae3f3a2575dbee4a38bd406608855f6e79cde8fda1c6bff9e203", size = 892314, upload-time = "2026-07-17T18:58:41.391Z" }, - { url = "https://files.pythonhosted.org/packages/47/3f/e8f4ec6cdf04d0cb9915590679771b55a6f579a7478704d1e6be45a2f792/rignore-0.8.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02cf38d96b6f54b69719b726877e308b34ac2c9c9f636dc8872ede2ff6825579", size = 962528, upload-time = "2026-07-17T18:58:42.736Z" }, - { url = "https://files.pythonhosted.org/packages/58/57/d10a43221644177f776d658ed9a03204e49d9bfd6805fbece841ea6720ac/rignore-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5c820d3fa597ee419f3643f8c098a51249a868ca8515296a56a1505c01a8083d", size = 1060799, upload-time = "2026-07-17T18:58:44.279Z" }, - { url = "https://files.pythonhosted.org/packages/a9/0e/d769dfa933861dd469c9158ffc9bc2cb3bc6d46f88000fb6f41cf2905a3a/rignore-0.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e3e863b2cb1db481384bd43a25730ef00f7eccdd49c82a0e1d481a6412dc2653", size = 1132462, upload-time = "2026-07-17T18:58:45.583Z" }, - { url = "https://files.pythonhosted.org/packages/c1/3d/57fc6264ebf8d9b95850589899e6192d9f92b67ea48e56fc32969b75474a/rignore-0.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:90cc363aedc7a93b4b15933f2104bfcae1c6c1635a9bc479665cae83044577d9", size = 1139679, upload-time = "2026-07-17T18:58:46.928Z" }, - { url = "https://files.pythonhosted.org/packages/fa/be/24b12a8464e19d348aeb388267f897ede0fae6618052c3148b747acae214/rignore-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:303a9fd02d3612d15e6dc3474ae7d3eae1d07b668b7b4943fd4df305be4d2f76", size = 1138125, upload-time = "2026-07-17T18:58:48.42Z" }, - { url = "https://files.pythonhosted.org/packages/d6/8d/89f7cba3491164c04f8a64056ca973494536ffdf48bfba41565f54dfa122/rignore-0.8.0-cp312-cp312-win32.whl", hash = "sha256:e17a0914378fa15d1e29effce4b39fd2031f253e52d307ff4b71c443d1d5d30e", size = 637663, upload-time = "2026-07-17T18:58:49.76Z" }, - { url = "https://files.pythonhosted.org/packages/3f/ff/823a5ead8bba0a2054cbf2dbef99989204557904ad8578f514bec1b9b4e7/rignore-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:d5c8f1dff84c4d4114f1553564b5d909a0c4c09dbe1b5916a0506d719f1fc3b5", size = 728265, upload-time = "2026-07-17T18:58:51.028Z" }, - { url = "https://files.pythonhosted.org/packages/8f/ce/73c919505f9f270ee3e34205ff2dbe0b0d73e945f8c569922acff148bcf4/rignore-0.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:6cdea3f85de8286a38ae75a0f9092cd3afc4d33ce6ee2e3f6005f97f7da9d249", size = 665216, upload-time = "2026-07-17T18:58:52.308Z" }, -] - -[[package]] -name = "rpds-py" -version = "2026.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, - { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, - { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, - { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, - { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, - { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, - { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, - { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, - { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, - { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, - { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, - { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, - { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, -] - -[[package]] -name = "runai-model-streamer" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "humanize" }, - { name = "numpy" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/55/ebf29e73c7c4a66d093e0018b212ef8ccc828231f37c3362edda2535ecb3/runai_model_streamer-0.16.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:035cc282935d7d55032ce2da97507b3d94201e647d622bd44476a6ef9f04f824", size = 649145, upload-time = "2026-07-13T07:51:59.982Z" }, - { url = "https://files.pythonhosted.org/packages/ee/3d/32656788027d72d6bb54804a1479429ad1ccb36d212ff408c4f513bdd4a5/runai_model_streamer-0.16.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:0f9958ecdef6786347b855f345d3076a039201f6f2c0e22dd52c74eef8941430", size = 650434, upload-time = "2026-07-13T07:52:01.341Z" }, -] - -[package.optional-dependencies] -azure = [ - { name = "runai-model-streamer-azure" }, -] -gcs = [ - { name = "runai-model-streamer-gcs" }, -] -s3 = [ - { name = "runai-model-streamer-s3" }, -] - -[[package]] -name = "runai-model-streamer-azure" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-identity" }, - { name = "azure-storage-blob" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/13/16cf17e039ce7f7bc93a32c87876917a6cd946d668a3195990ce9f1494d3/runai_model_streamer_azure-0.16.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:77e00e797852167d9036c36a952a6cb46cabbb7d78ddf30a159ef424803f1d20", size = 5905985, upload-time = "2026-07-13T07:52:17.805Z" }, - { url = "https://files.pythonhosted.org/packages/cc/d4/39c97a540199a0f48a522bf44e5cb8bcc4e9f5a6bd1f051b14677c8c8fec/runai_model_streamer_azure-0.16.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:3c66172e3c22405d68cb47bccf022d7cab299f1571400ade684771b860420483", size = 5627580, upload-time = "2026-07-13T07:52:19.411Z" }, -] - -[[package]] -name = "runai-model-streamer-gcs" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "google-cloud-storage" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/f5/347ae314c8a76c76976c7edfc82764e1fefc9c75b393cc16c492e96dcfb0/runai_model_streamer_gcs-0.16.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4a536d2058c025a4452afa2cf869514ccd77d8b1437edd7b5edb9f881fc8ca25", size = 23302179, upload-time = "2026-07-13T07:52:11.08Z" }, - { url = "https://files.pythonhosted.org/packages/7a/ca/b34853e30bab59285d7b742af3d5e840213ebea816187b14d0b142c23b4f/runai_model_streamer_gcs-0.16.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:295f1af31452f47c32ae02d07787ab0922393971988070c0b9c618528d8ac8ec", size = 23249177, upload-time = "2026-07-13T07:52:13.385Z" }, -] - -[[package]] -name = "runai-model-streamer-s3" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "boto3" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/c9/36d39711463f48ce753e8fb7e21facff97a9789773a8f7d67f25897bed7d/runai_model_streamer_s3-0.16.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:b7ac88beda87b8f2864e8666c4b8974a0c1e886f9d307ae9de6fbfe18e3b9253", size = 6189831, upload-time = "2026-07-13T07:52:04.955Z" }, - { url = "https://files.pythonhosted.org/packages/58/40/275cf91d178abfc6e1feda301b7bc4ac85d8dbafeebbd77ffdb370938cf6/runai_model_streamer_s3-0.16.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:fb225b5dfbecf8087fe9073f5eb6916b56d19b7f7b49625269053f7205d7c0fd", size = 5931724, upload-time = "2026-07-13T07:52:06.589Z" }, -] - -[[package]] -name = "s3transfer" -version = "0.19.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, -] - -[[package]] -name = "safetensors" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, - { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, - { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, - { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, - { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, - { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, - { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, - { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, - { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, - { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, - { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, - { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, - { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, - { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, - { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, -] - -[[package]] -name = "scikit-learn" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy" }, - { name = "scipy" }, - { name = "threadpoolctl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/20/75f915ff375d6249e6550ac740fdbbd66159a068fd3af1400ff62036b07a/scikit_learn-1.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2bd41b0d201bc81575531b96b713d3eb5e5f50fb0b82101ff0f92294fdc236ac", size = 8741122, upload-time = "2026-06-02T11:53:24.08Z" }, - { url = "https://files.pythonhosted.org/packages/cc/d5/2b5148f2279196775e1db2aeb85d14b70ac80e7e32b3b28e7ebeafb0901d/scikit_learn-1.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5be45aa4a42a68a533913a6ed736cf309de2226411c79ef8d609a5456f1939b1", size = 8261512, upload-time = "2026-06-02T11:53:27.183Z" }, - { url = "https://files.pythonhosted.org/packages/a0/ee/5adbc77656b71f9456a2f5a7a9fdb4bcf9207a6b962889f1c2f9323afa4e/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e50ed4da51974e86e940690e9a3d82e729b62b5a49f7c9bac534d515d39d86f", size = 8837603, upload-time = "2026-06-02T11:53:30.328Z" }, - { url = "https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8", size = 9132097, upload-time = "2026-06-02T11:53:33.456Z" }, - { url = "https://files.pythonhosted.org/packages/83/a4/c8e67227c680e2259c8864ae72ff48b06e16a6f51253a22167aa02a8aa4e/scikit_learn-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:4306775fad04cc4b472a1b15af1ae9cede1540fbfcc17fbce3767cd8dc7ae283", size = 8211173, upload-time = "2026-06-02T11:53:36.602Z" }, - { url = "https://files.pythonhosted.org/packages/cf/fd/3c0863792e98e67e9184aa4029288a175935eb65443afcd30d4f143450cf/scikit_learn-1.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:26e22435f63bcdcf396b574273f29f13dd531f5ea035801f5be10ba1540a4e60", size = 7867451, upload-time = "2026-06-02T11:53:39.075Z" }, -] - -[[package]] -name = "scipy" -version = "1.18.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a", size = 31056519, upload-time = "2026-06-19T14:59:45.203Z" }, - { url = "https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b", size = 28681889, upload-time = "2026-06-19T14:59:48.103Z" }, - { url = "https://files.pythonhosted.org/packages/9b/2d/11dd93d21e147a73ba22bd75c0b9208d3a2e0ec76d53170ce7d9029b1015/scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9", size = 20423580, upload-time = "2026-06-19T14:59:50.665Z" }, - { url = "https://files.pythonhosted.org/packages/9c/01/93552f75e0d2a7dd115a45e59209c51e8d514daff02fc887d2623be06fe1/scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8", size = 23054441, upload-time = "2026-06-19T14:59:53.564Z" }, - { url = "https://files.pythonhosted.org/packages/3c/23/21f5e703643d66f21faa6b4c73195bfcad70c55efcb4f1ab327cd7c4101a/scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab", size = 33968720, upload-time = "2026-06-19T14:59:56.415Z" }, - { url = "https://files.pythonhosted.org/packages/dd/aa/1b939f6c67ed68635bb538e6752d3dacc02f66535182e939a89581a44e9c/scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2", size = 35287115, upload-time = "2026-06-19T14:59:59.411Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ff/eec46be7e9234208f801062b53e1983085eddebd693f6c9bfb03b459830d/scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11", size = 35577989, upload-time = "2026-06-19T15:00:02.235Z" }, - { url = "https://files.pythonhosted.org/packages/84/ca/210d4759c7210bb7d269437421959b39a33434e2776b60c5cb8a763bb30a/scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de", size = 37421717, upload-time = "2026-06-19T15:00:05.102Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/9a9edb45345bd6744da5ddfb6628e5d5185920494c6a67ec45b6381004cb/scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132", size = 36597428, upload-time = "2026-06-19T15:00:08.112Z" }, - { url = "https://files.pythonhosted.org/packages/99/0e/33f32a2a58987e26aec0f7df252cbbad1e90ae77bdbc76f40dd4ed0cf0ea/scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76", size = 24351481, upload-time = "2026-06-19T15:00:11.103Z" }, -] - -[[package]] -name = "sentencepiece" -version = "0.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/33/ea3cb3839607eb175da835244a798f797f478c5ddf0e8ecdf57ea85a4c70/sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6", size = 8218435, upload-time = "2026-07-12T08:39:34.701Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/13/7a562289c8d5b49ebdf3f9c1e8ab67cf14a8743b1d90c8f406bfdec36b72/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b", size = 2188384, upload-time = "2026-07-12T08:38:28.486Z" }, - { url = "https://files.pythonhosted.org/packages/85/d1/912f14fd5eae168aba726ffb6a9a2dc1c71fe7676c53da6f5c442b886d4a/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820", size = 1441553, upload-time = "2026-07-12T08:38:30.552Z" }, - { url = "https://files.pythonhosted.org/packages/bd/44/caa9cab5f261a019e2808bc5046152775dc57352ba9cbae7525e9e7a1ed4/sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838", size = 1347176, upload-time = "2026-07-12T08:38:32.211Z" }, - { url = "https://files.pythonhosted.org/packages/19/90/cd798935668cff71d309d8ff10385844ecf216b1fe454f1993ed8bf2cb91/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a", size = 1325200, upload-time = "2026-07-12T08:38:33.689Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2d/37e3da037318a70066ded0d51bc2a7f35491ae6338dd993d5eb1503fc3b5/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da", size = 1397736, upload-time = "2026-07-12T08:38:35.771Z" }, - { url = "https://files.pythonhosted.org/packages/8d/11/753fca2e6b109be3ab7867abf357dfe48677fe726ae5a5363d0b54ca9450/sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383", size = 1248030, upload-time = "2026-07-12T08:38:37.604Z" }, - { url = "https://files.pythonhosted.org/packages/e2/0a/70efbe861ca182d7d4b6e1a20f58e043400848fa9f2915229f082e221648/sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d", size = 1187325, upload-time = "2026-07-12T08:38:39.348Z" }, -] - -[[package]] -name = "sentry-sdk" -version = "2.66.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7f/6f/d59cad0889d15fde85254cf58e701484de3f3f0406003b3197746910b19b/sentry_sdk-2.66.1.tar.gz", hash = "sha256:f882fb08710c5f8bfc603aafa3e901b384009a19cc3f76a572b863392ee81cdc", size = 940543, upload-time = "2026-07-22T12:26:54.553Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/d3/726bd88f0eece09ddf431bea4c9191c18e7a8d070b854eb0014d447712ee/sentry_sdk-2.66.1-py3-none-any.whl", hash = "sha256:86002793161d9a95ef04bdd8d442e9bfece5d989b755f05d6360215094a7aff6", size = 505555, upload-time = "2026-07-22T12:26:52.71Z" }, -] - -[[package]] -name = "setproctitle" -version = "1.3.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/48/49393a96a2eef1ab418b17475fb92b8fcfad83d099e678751b05472e69de/setproctitle-1.3.7.tar.gz", hash = "sha256:bc2bc917691c1537d5b9bca1468437176809c7e11e5694ca79a9ca12345dcb9e", size = 27002, upload-time = "2025-09-05T12:51:25.278Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/f0/2dc88e842077719d7384d86cc47403e5102810492b33680e7dadcee64cd8/setproctitle-1.3.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2dc99aec591ab6126e636b11035a70991bc1ab7a261da428491a40b84376654e", size = 18049, upload-time = "2025-09-05T12:49:36.241Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b4/50940504466689cda65680c9e9a1e518e5750c10490639fa687489ac7013/setproctitle-1.3.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdd8aa571b7aa39840fdbea620e308a19691ff595c3a10231e9ee830339dd798", size = 13079, upload-time = "2025-09-05T12:49:38.088Z" }, - { url = "https://files.pythonhosted.org/packages/d0/99/71630546b9395b095f4082be41165d1078204d1696c2d9baade3de3202d0/setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2906b6c7959cdb75f46159bf0acd8cc9906cf1361c9e1ded0d065fe8f9039629", size = 32932, upload-time = "2025-09-05T12:49:39.271Z" }, - { url = "https://files.pythonhosted.org/packages/50/22/cee06af4ffcfb0e8aba047bd44f5262e644199ae7527ae2c1f672b86495c/setproctitle-1.3.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6915964a6dda07920a1159321dcd6d94fc7fc526f815ca08a8063aeca3c204f1", size = 33736, upload-time = "2025-09-05T12:49:40.565Z" }, - { url = "https://files.pythonhosted.org/packages/5c/00/a5949a8bb06ef5e7df214fc393bb2fb6aedf0479b17214e57750dfdd0f24/setproctitle-1.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff72899861c765bd4021d1ff1c68d60edc129711a2fdba77f9cb69ef726a8b6", size = 35605, upload-time = "2025-09-05T12:49:42.362Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3a/50caca532a9343828e3bf5778c7a84d6c737a249b1796d50dd680290594d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7cb05bd446687ff816a3aaaf831047fc4c364feff7ada94a66024f1367b448c", size = 33143, upload-time = "2025-09-05T12:49:43.515Z" }, - { url = "https://files.pythonhosted.org/packages/ca/14/b843a251296ce55e2e17c017d6b9f11ce0d3d070e9265de4ecad948b913d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3a57b9a00de8cae7e2a1f7b9f0c2ac7b69372159e16a7708aa2f38f9e5cc987a", size = 34434, upload-time = "2025-09-05T12:49:45.31Z" }, - { url = "https://files.pythonhosted.org/packages/c8/b7/06145c238c0a6d2c4bc881f8be230bb9f36d2bf51aff7bddcb796d5eed67/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8828b356114f6b308b04afe398ed93803d7fca4a955dd3abe84430e28d33739", size = 32795, upload-time = "2025-09-05T12:49:46.419Z" }, - { url = "https://files.pythonhosted.org/packages/ef/dc/ef76a81fac9bf27b84ed23df19c1f67391a753eed6e3c2254ebcb5133f56/setproctitle-1.3.7-cp312-cp312-win32.whl", hash = "sha256:b0304f905efc845829ac2bc791ddebb976db2885f6171f4a3de678d7ee3f7c9f", size = 12552, upload-time = "2025-09-05T12:49:47.635Z" }, - { url = "https://files.pythonhosted.org/packages/e2/5b/a9fe517912cd6e28cf43a212b80cb679ff179a91b623138a99796d7d18a0/setproctitle-1.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:9888ceb4faea3116cf02a920ff00bfbc8cc899743e4b4ac914b03625bdc3c300", size = 13247, upload-time = "2025-09-05T12:49:49.16Z" }, -] - -[[package]] -name = "setuptools" -version = "80.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/95/faf61eb8363f26aa7e1d762267a8d602a1b26d4f3a1e758e92cb3cb8b054/setuptools-80.10.2.tar.gz", hash = "sha256:8b0e9d10c784bf7d262c4e5ec5d4ec94127ce206e8738f29a437945fbc219b70", size = 1200343, upload-time = "2026-01-25T22:38:17.252Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" }, -] - -[[package]] -name = "shellingham" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, -] - -[[package]] -name = "simplejson" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/f0/ea064bba6c9afda0168ddb834f1c75a93351031e25aee35c046108e7f292/simplejson-4.2.0.tar.gz", hash = "sha256:55b121b70a560f4610bd3a355ab2015aca4f39978f6a82353f24d2013fe85861", size = 123986, upload-time = "2026-10-03T03:34:23.27Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/24/87a310dcd8bd02876bbc33164d41f1d7b78a14ae98210b187cae22f560fc/simplejson-4.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:aa067739b28c661deb4421ee9ec1d7bad5ee06b7c50f8cf0d009e7945abe7d52", size = 117330, upload-time = "2026-10-03T03:32:06.623Z" }, - { url = "https://files.pythonhosted.org/packages/4d/cf/b1fc78e122ab98a2bae6fd6d66dec18d1ed436cafb875278f4d9e7677cbd/simplejson-4.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f458e7a2dd3d1b8b90dc12900c9e5a0f8b863fa7b02286fee13086962244f70a", size = 95482, upload-time = "2026-10-03T03:32:07.969Z" }, - { url = "https://files.pythonhosted.org/packages/27/ec/bad733020b3eef7e8414385fdfd4ba6afd5b8a56a0c4eb4eaa21be380f2a/simplejson-4.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c490ec62ed1b66a27afd5085e743e7f93b745c515257373de8433f4d51e5c3bb", size = 95297, upload-time = "2026-10-03T03:32:09.864Z" }, - { url = "https://files.pythonhosted.org/packages/2d/29/fb579920d8ec86ebb1f9a8cc8c9d17f3dcdb32a885dbd8332d21aa855575/simplejson-4.2.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8c1e156ad810704994439719b9c03694e267052d4938ca188d91a1769d6f742b", size = 198576, upload-time = "2026-10-03T03:32:11.003Z" }, - { url = "https://files.pythonhosted.org/packages/8e/7d/11fee9bebb22944c9e294139d5035cfa584f692192dabf3659e9c9bc0148/simplejson-4.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8910997afb7bae918b1ccf766e106e37707c8f8b4c61ac6ce433c4c86c5848f", size = 196424, upload-time = "2026-10-03T03:32:12.299Z" }, - { url = "https://files.pythonhosted.org/packages/a4/9e/cfb4d64d68e589f93aa83ecdf0ab9e62c787d7a5ef41c8a31f74eb62857c/simplejson-4.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f0767e82c062486211af7ee88cbe4732ca24250ce8127ffebd47732455439b69", size = 203804, upload-time = "2026-10-03T03:32:13.758Z" }, - { url = "https://files.pythonhosted.org/packages/7f/12/0e752142cdfbbd442f9ac614eb92b61c3def9a6fdd7aad4771ec9acc4abe/simplejson-4.2.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:797f086f589395e701ab077e9996dc0522a0b60158993e703e42749c4a17127c", size = 188048, upload-time = "2026-10-03T03:32:15.5Z" }, - { url = "https://files.pythonhosted.org/packages/4f/ff/4da29e068b788803681f17693153ed7f472c4190400eb724698b8e3dcd4b/simplejson-4.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2f53916dc840f4424dbafca0da7e8a3bafa7372ce7e1866c764966e36271f7bb", size = 193244, upload-time = "2026-10-03T03:32:16.694Z" }, - { url = "https://files.pythonhosted.org/packages/3b/4a/8e4167770b595ad5f7d2e20f0605e83f3467fa0ca6b85d50ee09019c4607/simplejson-4.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:703f532ec018562bb0c8eaf4b4851f5736c0f60d02e23ba8736ce885fa361eda", size = 201201, upload-time = "2026-10-03T03:32:18.001Z" }, - { url = "https://files.pythonhosted.org/packages/c6/5a/b8095e99e96a0f33f49d4da723d3a8f3b4295a13362c6fc26810f5dc51c4/simplejson-4.2.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2c1772c43537c7cc616fc217344acb00dec8312cfa76e725b6c5a6a4d5f80fb5", size = 186277, upload-time = "2026-10-03T03:32:19.326Z" }, - { url = "https://files.pythonhosted.org/packages/3d/87/fd79ac0e3841cc173f4b44d9158d595eaac072603691b505dafab6bde0aa/simplejson-4.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:471f30cd51ffdda1a0c421dc9963ada31e9d29bd688a3198041d2c69d18d65c4", size = 195937, upload-time = "2026-10-03T03:32:20.615Z" }, - { url = "https://files.pythonhosted.org/packages/c5/3f/45f19753465fa2ec337259b608236620e44a6aa410c1167edda010f30ce1/simplejson-4.2.0-cp312-cp312-win32.whl", hash = "sha256:85bde07e265b39be9593c0dd5e144c2308aa51d2dd1c18f495b46fa942f336d7", size = 92129, upload-time = "2026-10-03T03:32:21.839Z" }, - { url = "https://files.pythonhosted.org/packages/78/f0/08a6cffc4545112ca4c9918110e8c5227146a69fd270b0c64ff22a445ac6/simplejson-4.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:733acb0a25795becbbb6c5564f5c1c2e839889a931a72249fb0cc1c176659d83", size = 93961, upload-time = "2026-10-03T03:32:22.996Z" }, - { url = "https://files.pythonhosted.org/packages/e9/4c/9acdf4ae4f41c09a09ad17427e5ee912f35aa56ea1d1723a9d927d659d4e/simplejson-4.2.0-py3-none-any.whl", hash = "sha256:c2a2e5f43287cbe3413f7b73b04d5a6f75c7bd93d783e628f5978853a2ef738d", size = 72826, upload-time = "2026-10-03T03:34:21.667Z" }, -] - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "sniffio" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, -] - -[[package]] -name = "soundfile" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d2/db/949331952a6fb1c5b12e9de80fd08747966c2039d1a61db4764fbd3981c2/soundfile-0.14.0.tar.gz", hash = "sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11", size = 47842, upload-time = "2026-06-06T08:58:47.869Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/d1/5e338af9ca6ed0786cd5bb03f6d60de1c325728c1189014f3b59aae7403c/soundfile-0.14.0-py2.py3-none-any.whl", hash = "sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8", size = 26799, upload-time = "2026-06-06T08:58:33.269Z" }, - { url = "https://files.pythonhosted.org/packages/7e/72/c6b21e58d3113596e7e8de0a08d6f1d95173492cfbca0a4db14148cbba2a/soundfile-0.14.0-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4", size = 1144568, upload-time = "2026-06-06T08:58:35.231Z" }, - { url = "https://files.pythonhosted.org/packages/63/7a/dfdd6f8c748988427119f75eb860a3cedd858d1aea1fe28f39ad8559ef22/soundfile-0.14.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c", size = 1103726, upload-time = "2026-06-06T08:58:37.948Z" }, - { url = "https://files.pythonhosted.org/packages/4a/f8/fc39fad6f879633461d27394cd1ddaf1f769ffa0597dca35872f51b16461/soundfile-0.14.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377", size = 1238050, upload-time = "2026-06-06T08:58:39.932Z" }, - { url = "https://files.pythonhosted.org/packages/7b/a2/70fd4432b924684c372df8b0a45708c36c057ef3596c9eb53e0a806b980b/soundfile-0.14.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d", size = 1315963, upload-time = "2026-06-06T08:58:41.716Z" }, - { url = "https://files.pythonhosted.org/packages/d9/34/c9e80783d83eab739a9531fdee03675d53e0bf1b2ccb4bb3af5844675046/soundfile-0.14.0-py2.py3-none-win32.whl", hash = "sha256:0a6ae43c50c71b4e020cc55382925cb89451c1ed1a0c3d0f5d802da269226849", size = 902199, upload-time = "2026-06-06T08:58:43.289Z" }, - { url = "https://files.pythonhosted.org/packages/ed/97/b39c18ac1df45e755ca22b8b00e872929da5d107998a207a5e4ac831bfda/soundfile-0.14.0-py2.py3-none-win_amd64.whl", hash = "sha256:299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e", size = 1021480, upload-time = "2026-06-06T08:58:45.016Z" }, - { url = "https://files.pythonhosted.org/packages/f4/83/55c65e61cf457805ce2ec157c1c6ae17715d0851aa2374422de0538838ca/soundfile-0.14.0-py2.py3-none-win_arm64.whl", hash = "sha256:e090704718e124e7c844695236f1fce8d18a5e761eaf7c82dfcd124620805f98", size = 888858, upload-time = "2026-06-06T08:58:46.593Z" }, -] - -[[package]] -name = "soxr" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ed/11/27cebce4a108f77afea7c80545115536b45e3f11ebfb914f638fdd9ba847/soxr-1.1.0.tar.gz", hash = "sha256:9f228ae21c78fa9359ca98d8a5e8e91f30639e438e574133dace62c5b5309e44", size = 173067, upload-time = "2026-05-03T00:15:18.214Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/8a/f3da7973b5f1b05d2d7e94d5376b881dcbc05297900cae6c3d33d95b209b/soxr-1.1.0-cp312-abi3-macosx_10_14_x86_64.whl", hash = "sha256:e0e09fa633ce2e67df08b298afced4d184f6e753fc330f241022250f1d0d61da", size = 204124, upload-time = "2026-05-03T00:14:54.505Z" }, - { url = "https://files.pythonhosted.org/packages/03/dc/200013a74641f8774664bbcd2346c695c05c2e300ea792adcb40a293eed0/soxr-1.1.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d6a7ad82b8d5f3fcc04b1d2ca055562b96af571e1d4fa7c6c61d0fb509ac43b4", size = 165457, upload-time = "2026-05-03T00:14:56.007Z" }, - { url = "https://files.pythonhosted.org/packages/88/2b/2e5eba817a762a2ec589ff165b8bc5955b25a0ad140045f7cd8e45410543/soxr-1.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf98c0d7b7d5ef5bf072fee8d3020e8b664f2d195933ea7bc5089267c2e22a06", size = 206529, upload-time = "2026-05-03T00:14:57.646Z" }, - { url = "https://files.pythonhosted.org/packages/5c/f1/0e55195893228609c9a08c3b13b7a83a46c3a992cd00d3304f0f320cfb07/soxr-1.1.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b033078e86f3c4a658e5697fac8995764fad9e799563616b630136b613167f1", size = 240413, upload-time = "2026-05-03T00:14:59.363Z" }, - { url = "https://files.pythonhosted.org/packages/b0/4d/621e4150e4815246ad552d215a8a294a90143fedd19ee442cf82d3b3abc8/soxr-1.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:6ae2a174bffea94e8ead857dad85999d3f49f091774dbad5b046c0417d7092f4", size = 174357, upload-time = "2026-05-03T00:15:00.724Z" }, -] - -[[package]] -name = "sqlalchemy" -version = "2.1.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/44/311bac6b6ef81e4dfd0287d04900108b1f5c00c9761dd3c0a2b7b9d0f86b/sqlalchemy-2.1.4.tar.gz", hash = "sha256:7bd7ad604487daa7eab8716471c29a7185f17b5287ce73bb7bc79fea050d8cfd", size = 10544216, upload-time = "2026-10-07T17:33:59.116Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/5e/cb5b078e007340661b010fa8bd31ce27468f88e09b35266544df4e0c52ca/sqlalchemy-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f953be9ba26039a24a5205c65d33518b608ce6f4f0f4e9b9c14eaf42a10dfc52", size = 2466469, upload-time = "2026-10-07T18:17:24.049Z" }, - { url = "https://files.pythonhosted.org/packages/b1/98/44e2fdc5bc053dae559bf4f4eb7967ceecbad162299ecfc8de2edc3fcbe7/sqlalchemy-2.1.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ac64fce94c5b389062d2e3806db5dc780447591e0dfd5ead218c884f0703f2e", size = 4668213, upload-time = "2026-10-07T18:37:42.294Z" }, - { url = "https://files.pythonhosted.org/packages/08/25/ed2262f964687b06f10c2c98b2dc9c9ed211f7cc11702879969a9ac217e4/sqlalchemy-2.1.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e5045fb6aadbb0f978ab9b9d8822f7b7a97d2281814e7d13d791155664eace3", size = 4720857, upload-time = "2026-10-07T18:24:46.842Z" }, - { url = "https://files.pythonhosted.org/packages/4d/d4/fab64c61d5d22ddbb077afd1e6b29b498bdacdf6406a03f53566e7e01686/sqlalchemy-2.1.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e3a026436c51f296aa1d01243909a3b76490950e927824b10899a083cc26e7c3", size = 4369351, upload-time = "2026-10-07T18:59:45.483Z" }, - { url = "https://files.pythonhosted.org/packages/d9/e4/33413f0fafbcf3b332320aac2c1e40f3b4f17e56359a9474cb10de4bee8b/sqlalchemy-2.1.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:71040390ef01c85e9d26e5c83cb0c5942dcc8725c49186430af160ce2f54234d", size = 4591496, upload-time = "2026-10-07T18:37:44.433Z" }, - { url = "https://files.pythonhosted.org/packages/bb/65/19821440cbd5c93da053d627b3e402eff11ff252bfae37700645b3c155a4/sqlalchemy-2.1.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:07c60abaffb980b7382f2c75be8a5279c2b5df2626a0f5d751dd942799bf3b5c", size = 4362379, upload-time = "2026-10-07T18:59:48.278Z" }, - { url = "https://files.pythonhosted.org/packages/01/e3/168a0f93efd6ec40f59645a7e45ab08918e0bc8ecf07656e4ca09acdcc30/sqlalchemy-2.1.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a577e2127e52b0fe2bc54c73abb375a20ffe6f59fbc5568ccafc233f5bfcf8ef", size = 4667997, upload-time = "2026-10-07T18:24:48.72Z" }, - { url = "https://files.pythonhosted.org/packages/54/79/0a852ef65864acd8d577d7aa6f67146167382bd6faee7a7586b9e6e28275/sqlalchemy-2.1.4-cp312-cp312-win32.whl", hash = "sha256:6c79e0c824d51c586757ecd342160bbdede9010df04bb71b9bbfffd5c7b6ee29", size = 2376664, upload-time = "2026-10-07T18:25:00.637Z" }, - { url = "https://files.pythonhosted.org/packages/27/b9/a5934263bb1d712f743289ca224ab3b87e3570ac157802291e37ab85d365/sqlalchemy-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:dffa69d2f3ba1933c1c1882dbef8fb3231b33eb19263e8b8c5cea24995071f06", size = 2429357, upload-time = "2026-10-07T18:25:02.565Z" }, - { url = "https://files.pythonhosted.org/packages/a5/fa/a2323d81384ff214aa189057b7455b63623e66f28208b982e86c3cb042f5/sqlalchemy-2.1.4-cp312-cp312-win_arm64.whl", hash = "sha256:e30524ae24e31d83e1b5f734862882c442f4158e3566f2c5f5e9bd3c659bb517", size = 2388756, upload-time = "2026-10-07T18:22:36.025Z" }, - { url = "https://files.pythonhosted.org/packages/f7/62/dbf11a262f6fbb41390cab2d8e47a30ec0961018b68201607b599dd489f5/sqlalchemy-2.1.4-py3-none-any.whl", hash = "sha256:0b96edcc2cd60fe1e35f67a46f4eb076e57297841b9eae949ac5f196593f00a7", size = 2054935, upload-time = "2026-10-07T18:01:16.403Z" }, -] - -[[package]] -name = "sse-starlette" -version = "3.4.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "starlette" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6c/10/a34c656829ffc1c4b22ef36d70d9ebb6b99c020e2aeb17cee5485099f028/sse_starlette-3.4.6.tar.gz", hash = "sha256:725f8a1bd6d26ae1b2c9610c0ef5065dfdd496f3988d28adcf8c4b49dc25c627", size = 32542, upload-time = "2026-07-20T14:16:32.201Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/36/e10c1d1b7ca881d2625db2ec28508578499187bb1c389952c398474e1834/sse_starlette-3.4.6-py3-none-any.whl", hash = "sha256:56217ab4c9a9f9c5db7b21e08732d3e7c2b807f45231ad23de0551a24c4a41f6", size = 16516, upload-time = "2026-07-20T14:16:30.978Z" }, -] - -[[package]] -name = "starlette" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, -] - -[[package]] -name = "supervisor" -version = "4.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/b5/37e7a3706de436a8a2d75334711dad1afb4ddffab09f25e31d89e467542f/supervisor-4.3.0.tar.gz", hash = "sha256:4a2bf149adf42997e1bb44b70c43b613275ec9852c3edacca86a9166b27e945e", size = 468912, upload-time = "2025-08-23T18:25:02.418Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/65/5e726c372da8a5e35022a94388b12252710aad0c2351699c3d76ae8dba78/supervisor-4.3.0-py2.py3-none-any.whl", hash = "sha256:0bcb763fddafba410f35cbde226aa7f8514b9fb82eb05a0c85f6588d1c13f8db", size = 320736, upload-time = "2025-08-23T18:25:00.767Z" }, -] - -[[package]] -name = "sympy" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mpmath" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, -] - -[[package]] -name = "tabulate" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, -] - -[[package]] -name = "tensorstore" -version = "0.1.85" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ml-dtypes" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/16/a6530387ed22d07e974af2ef55e34fb19a7251a6911c85a0be029a4dcf9c/tensorstore-0.1.85.tar.gz", hash = "sha256:26698bded0278e98e0988eb8f7d76ad248762afc5f55a38b9122ffd1474e505f", size = 7317948, upload-time = "2026-08-05T15:35:56.098Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/81/612d3f58009ea1380f9befe2dced9d98c0a0fd608cd3e9d89389f9a04757/tensorstore-0.1.85-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:6b9453f28de853c804339bf882d0bc3eb523c4c55600b764239240bdec4460a8", size = 17015828, upload-time = "2026-08-05T15:35:03.426Z" }, - { url = "https://files.pythonhosted.org/packages/fa/3c/74050e09a490c0cfed6016430226710d29fe4e165e12145368c7de4c6426/tensorstore-0.1.85-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af9189ca9f692bf83ce53e760a8b53fcf46ad0bf365930726faa0e301d0ab147", size = 15411026, upload-time = "2026-08-05T15:35:05.89Z" }, - { url = "https://files.pythonhosted.org/packages/88/7e/a1100ea66dbb028e0ea02f6d2e413e05285534d82d560f1f7d6145603640/tensorstore-0.1.85-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2f42d1e57df7e07655b819dd07f3de258fde97e39b896a53f089d7d7067e21", size = 19973651, upload-time = "2026-08-05T15:35:09.016Z" }, - { url = "https://files.pythonhosted.org/packages/78/49/c311f02b82299c6d79e18c7012c8b816141c54721253cbcba1c7c59550d4/tensorstore-0.1.85-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de27f2d6e063050f4283d1fdfe41e6c0ebc6ceacaab10ca2f7e9ecbf35221962", size = 21580735, upload-time = "2026-08-05T15:35:11.727Z" }, - { url = "https://files.pythonhosted.org/packages/88/da/3d66562291808e1edd433704bfdd4f212c1a6c53920293fb5f7296d644a5/tensorstore-0.1.85-cp312-cp312-win_amd64.whl", hash = "sha256:a5421480cd6ab144039c88fc7018ea5cefb8d00038e6dd2470cf776ef1bf45a5", size = 14129526, upload-time = "2026-08-05T15:35:14.98Z" }, -] - -[[package]] -name = "tfp-nightly" -version = "0.26.0.dev20260626" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "cloudpickle" }, - { name = "decorator" }, - { name = "dm-tree" }, - { name = "gast" }, - { name = "numpy" }, - { name = "six" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/5c/e24a6f85b2038cbf209844944774102bb37e3a1e67ae524c71a7aad0e0a6/tfp_nightly-0.26.0.dev20260626-py2.py3-none-any.whl", hash = "sha256:57d024ecbc1da39e33af522f98bdfc95cb118af04e27a0b3391618f29be546ee", size = 6975580, upload-time = "2026-06-26T08:45:22.208Z" }, -] - -[package.optional-dependencies] -jax = [ - { name = "jax" }, - { name = "jaxlib" }, -] - -[[package]] -name = "threadpoolctl" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, -] - -[[package]] -name = "tiktoken" -version = "0.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "regex" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/8e/144bde4e01df66b34bb865557c7cd754ed08b036217ebd79c9db5e9048a9/tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791", size = 1034888, upload-time = "2026-05-15T04:50:31.579Z" }, - { url = "https://files.pythonhosted.org/packages/36/18/d4ac9d20956cdebca04841316660ed584c2fecdc2b81722a28bc7ad3b1e4/tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b", size = 982970, upload-time = "2026-05-15T04:50:32.961Z" }, - { url = "https://files.pythonhosted.org/packages/74/ed/6bb8d05b9f731f749fee5c6f5ca63e981143c826a5985877330507bd13b7/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7", size = 1115741, upload-time = "2026-05-15T04:50:34.475Z" }, - { url = "https://files.pythonhosted.org/packages/34/de/2ca96b07a82d972b74fe4b46de055b79c904e45c7eab699354a0bfa697dc/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649", size = 1136523, upload-time = "2026-05-15T04:50:35.782Z" }, - { url = "https://files.pythonhosted.org/packages/ee/dc/9dafec002c2d4424378563cf4cf5c7fb93631d2a55013c8b87554ee4012c/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b", size = 1181954, upload-time = "2026-05-15T04:50:36.99Z" }, - { url = "https://files.pythonhosted.org/packages/a1/d0/1f8578c45b2f24759b46f0b50d31878c63c73e6bf0f2227e10ec5c5408dc/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91", size = 1240069, upload-time = "2026-05-15T04:50:38.221Z" }, - { url = "https://files.pythonhosted.org/packages/aa/90/28d7f154888610aa9237e541986beb62b479df29d193a5a0617dbb1514d0/tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41", size = 874748, upload-time = "2026-05-15T04:50:39.587Z" }, -] - -[[package]] -name = "tilelang" -version = "0.1.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "cloudpickle" }, - { name = "ml-dtypes" }, - { name = "numpy" }, - { name = "psutil" }, - { name = "setuptools", marker = "sys_platform == 'darwin'" }, - { name = "torch" }, - { name = "torch-c-dlpack-ext" }, - { name = "tqdm" }, - { name = "typing-extensions" }, - { name = "z3-solver" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/9a/5776adb5b6e03e141c8cff020e16c45b87c9b7a7a5bdfe83416d6cc23933/tilelang-0.1.12.tar.gz", hash = "sha256:595b39581d099a53f875bd8d553863e7e4f58998052b58764f5472cbfbb87043", size = 93565090, upload-time = "2026-07-08T04:22:52.901Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/51/4ae1fb532406082a4ebf16c890f5f98db14b8588f2c486334ba23578b4bc/tilelang-0.1.12-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:8741d6cd519782645d55c17abd8986402c62948bc2122d3417c445d11de5dd16", size = 38347295, upload-time = "2026-07-08T04:22:35.691Z" }, - { url = "https://files.pythonhosted.org/packages/d1/53/f281a0bd9ee7e03d6a97828fc0e443321ed26ea2b0bd74bf9f1d9451d30f/tilelang-0.1.12-cp38-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbeb5573cbe2544a51a5c9a6cc737c5e3b5c2d95411caa3687fd9b08eb9d5f97", size = 50501074, upload-time = "2026-07-08T04:22:39.973Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a4/ff0b14d00d29fe418063cd97490f55aa9cad6f981a6fbbf29124b024637d/tilelang-0.1.12-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:a1701eba4ad603d73f1b128b88f6a2350f1b1161729e5b2d1e540a1ec7a82d52", size = 46281933, upload-time = "2026-07-08T04:22:44.244Z" }, - { url = "https://files.pythonhosted.org/packages/cb/15/1a8a5b5df0a23ece80b4da8e49c43418244e5ab165ed1938a1cc2656cda2/tilelang-0.1.12-cp38-abi3-win_amd64.whl", hash = "sha256:daeb3d529ab9e7665adb94c4bf0d47732294709aa9b8b9bd673fa48001385422", size = 34020025, upload-time = "2026-07-08T04:22:48.066Z" }, -] - -[[package]] -name = "tokenizers" -version = "0.22.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, - { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, - { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, - { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, - { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, - { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, - { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, - { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, - { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, - { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, - { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, - { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, - { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, - { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, - { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, -] - -[[package]] -name = "tokenspeed-mla" -version = "0.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "nvidia-cutlass-dsl" }, - { name = "tokenspeed-triton" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/65/81d7e9f14472bc4c6abb576c9b1edd8e40ab01832027c4e647bfe2890749/tokenspeed_mla-0.1.8-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:952209cf4b29a54e6b6e7e088be9d4a40f24792b06fdfa330de8077b9926a7d9", size = 752341, upload-time = "2026-06-24T03:36:28.619Z" }, - { url = "https://files.pythonhosted.org/packages/27/df/0037ade72b165ac97859040919e006aa3d80cb8cc3a79420fb6c03eb16a0/tokenspeed_mla-0.1.8-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6a7526d7327746893f8c20d24aa63ba5b8a123d0dfd6e66388e13b768b6452c6", size = 755827, upload-time = "2026-06-24T03:36:29.96Z" }, -] - -[[package]] -name = "tokenspeed-triton" -version = "3.8.10.post20260721" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/44/89740db8951918c9acd8731243eef8b44d0eb92ea423552639265c46018e/tokenspeed_triton-3.8.10.post20260721-cp312-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d793ad0eaebb1d08272c97a2b8f2c31304231748b03de9a08e70a362de92a6e0", size = 82966664, upload-time = "2026-07-21T17:14:38.568Z" }, - { url = "https://files.pythonhosted.org/packages/91/53/f46b401e8ec8998f5b9c39cff0614b796bf49113a09f588cfdfa342789a3/tokenspeed_triton-3.8.10.post20260721-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66cba8d32a1539afd0ff3eec1782b082d01b4db6824d68017d1d789a03d0be37", size = 87210295, upload-time = "2026-07-21T17:14:42.173Z" }, -] - -[[package]] -name = "toolz" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/6f/ae20c212a07aa2d156c787383d8088a5e045ee39628661edb190c97e1659/toolz-1.2.0.tar.gz", hash = "sha256:9667a038e9d6ecba37995e26cb2f59ec6420b6ad8dd9677de59db9b956b08490", size = 55442, upload-time = "2026-10-07T04:16:25.639Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/17/4c8beb6c8c4176c6bf143bfd7e1e4dd6719b00ced90738c7ac471b71c1df/toolz-1.2.0-py3-none-any.whl", hash = "sha256:890f820b1cb8152785aaf9386d8707770110809035800985ca65cb24ce1120ef", size = 60789, upload-time = "2026-10-07T04:16:24.173Z" }, -] - -[[package]] -name = "torch" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/3a/ed0f4d4d1dcde03bced7aac9a28e800abcdc0cbd06b6775044c9fbd877b7/torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027", size = 111213045, upload-time = "2026-07-08T16:05:22.997Z" }, - { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" }, - { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" }, - { url = "https://files.pythonhosted.org/packages/21/d6/e8f3c6f7e01f626f77259de9860d2a78bc84c40539e28e79b7e98b0bb659/torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d", size = 122057313, upload-time = "2026-07-08T16:03:53.43Z" }, -] - -[[package]] -name = "torch-c-dlpack-ext" -version = "0.1.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "torch" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/37/de/921b6491efce5c389a5ef9bbed3d2d6660005840dae488124173180859ab/torch_c_dlpack_ext-0.1.5.tar.gz", hash = "sha256:d06f0357d575d22a168cc77acb9020fc4bae30968ceb6718a055dcbe92bacabe", size = 12913, upload-time = "2026-01-12T11:25:08.484Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/67/10d236698525d7b7db4d74ec0a4b01f5b2db33968995fdd9ac6b4635e327/torch_c_dlpack_ext-0.1.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c0f2bd51fcd99c0e5b50314e1985f2728c4941bfa821f065e6c30951d1f995ca", size = 5291237, upload-time = "2026-01-12T11:24:44.011Z" }, - { url = "https://files.pythonhosted.org/packages/87/06/8d760997307a5c3be4384424667bf31aae0a42060838c532c7d846516175/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3562ee411258676f9c38b8ad39306d1c8d027b6a86f6a87c920d2d009a9d1510", size = 443069, upload-time = "2026-01-12T11:24:45.451Z" }, - { url = "https://files.pythonhosted.org/packages/e2/79/a914539b4785f3e44f891aa012a886edb8bc10fe081c440981c57543ce21/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6f9da4bb9af70e27facc777458be62e10dbbbddda7672d16138db0553c5a524", size = 897846, upload-time = "2026-01-12T11:24:48.168Z" }, - { url = "https://files.pythonhosted.org/packages/3a/e6/7d7a97a3953208d6d6ce749180c34d1dab48464ded9a76cecabe9d021ce6/torch_c_dlpack_ext-0.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:670fbbab70123cc228bed41693a3720757af57a0ad22669063c9db25321e8f55", size = 1482855, upload-time = "2026-01-12T11:24:49.581Z" }, -] - -[[package]] -name = "torchaudio" -version = "2.11.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/b1/77658817acacd01a72b714440c62f419efc4d90170e704e8e7a2c0918988/torchaudio-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a1cf1acc883bee9cb906a933572fed6a8a933f86ef34e9ea7d803f72317e8c1b", size = 684226, upload-time = "2026-03-23T18:13:40.023Z" }, - { url = "https://files.pythonhosted.org/packages/78/28/c7adc053039f286c2aca0038b766cbe3294e66fec6b29a820e95128f9ede/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bc653defca1c16154398517a1adc98d0fb7f1dd08e58ced217558d213c2c6e29", size = 1626670, upload-time = "2026-03-23T18:13:42.162Z" }, - { url = "https://files.pythonhosted.org/packages/88/d8/d6d0f896e064aa67377484efef4911cdcc07bce2929474e1417cc0af18c2/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6503c0bdb29daf2e6281bb70ea2dfe2c3553b782b619eb5d73bdadd8a3f7cecf", size = 1771992, upload-time = "2026-03-23T18:13:33.188Z" }, - { url = "https://files.pythonhosted.org/packages/23/a8/941277ecc39f7a0a169d554302a1f1afd87c1d94a8aec828891916cea59a/torchaudio-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:478110f981e5d40a8d82221732c57a56c85a1d5895fb8fe646e86ee15eded3bd", size = 328663, upload-time = "2026-03-23T18:13:19.218Z" }, -] - -[[package]] -name = "torchcodec" -version = "0.17.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/53/33acd60b2880ed4b0d6526436f691a0f447de7b53aceb42eb58a9719e474/torchcodec-0.17.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3d69d30fbd2d5f9cc6d63c6d970acb5fa3aeddb5524a059f0ca538a83972f65c", size = 4219893, upload-time = "2026-09-30T12:52:54.86Z" }, - { url = "https://files.pythonhosted.org/packages/75/20/5692a1177b4be33086770970dcf83effa821376f447899cea3ba850ea2fa/torchcodec-0.17.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:81655cf25e168e74f27aee29db9fc16e37a9f5b2a188a0f7fe25cdfaab2ddd2e", size = 8854331, upload-time = "2026-09-30T12:52:56.445Z" }, - { url = "https://files.pythonhosted.org/packages/05/44/d00a4df9c8985c6ce9e6d1e3f861496cdcde8b5b4b7543358d49e03f7337/torchcodec-0.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:defd5f0901999c9e981f6465a0f4aac8b5a5147e38e539b77c0711c015cf5349", size = 9972897, upload-time = "2026-09-30T12:52:59.059Z" }, - { url = "https://files.pythonhosted.org/packages/f4/e4/0f2a356d6d08b1e2ece096288a02f2978806d077cac7a8340adf312b21ed/torchcodec-0.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:0771bc114ff75a30f0ea8aa82cc7b259c518214eafe73fee2e606d745d77942b", size = 6810114, upload-time = "2026-09-30T12:53:00.966Z" }, -] - -[[package]] -name = "torchvision" -version = "0.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pillow" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/49/c1cab1ecbb3ff1a380a3f99283db1dee61b8afe354f6352c643b65937130/torchvision-0.28.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:e9f54c30cd52e3ef7fd034cc69b7bb7e0964e1c8f8743e018ab92e95b40f9eee", size = 1856020, upload-time = "2026-07-08T16:07:52.182Z" }, - { url = "https://files.pythonhosted.org/packages/f0/4c/95233776e2def960e5abb7a07931230a545f43717a56a1e1140162033598/torchvision-0.28.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5cf78ebc401ce64ae19b8c55de866bb836797d559a4de9c25ccbe74cfa642d3a", size = 7842127, upload-time = "2026-07-08T16:07:53.446Z" }, - { url = "https://files.pythonhosted.org/packages/93/e4/e9b2495d0d57b9f60d63c57d0a910410a81b4b073bf70917bef815291119/torchvision-0.28.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:028a3d481b37d785605620d7cdad897064c5a55bae2aa1f2658766333e291940", size = 7675040, upload-time = "2026-07-08T16:07:58.017Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9c/55ed9cb6dfe3ee9c837df5cd0e758372e5829aa38b8dd71343aa632cc4e2/torchvision-0.28.0-cp312-cp312-win_amd64.whl", hash = "sha256:87dc16b2df427c1318ad335f1e2be2b3b15b2cf20f7934c83b0505a48425ee5d", size = 4085785, upload-time = "2026-07-08T16:07:50.928Z" }, -] - -[[package]] -name = "tornado" -version = "6.5.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/61/53d562a57b28c08eda40b258c0f975e360541943ad7c7bef897a40caafda/tornado-6.5.10.tar.gz", hash = "sha256:a6b1ccd08c04b4a06fb5aeb381be99de5ad1e5375c1785e31d78c880feb57687", size = 537910, upload-time = "2026-09-15T13:47:48.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/5b/ff5fc58fa2427c30dea74c90053f4fc5eda1e7f3833ed3ecc7147fe2b311/tornado-6.5.10-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9261783640e23258694a9ff0795df430a5a7b0a651d3dd53dd0969ad6be16da7", size = 465883, upload-time = "2026-09-15T13:47:35.463Z" }, - { url = "https://files.pythonhosted.org/packages/ad/f5/cd7be26c34a3315532f3aef5f092465da8f59c334dd439d3c14aaef16461/tornado-6.5.10-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:83e6cf438b106c6b3852d70960967bb1b70c87438050dca0981e4b9aa751a4c1", size = 464046, upload-time = "2026-09-15T13:47:37.178Z" }, - { url = "https://files.pythonhosted.org/packages/60/33/df6d7d04854a58619f8349a51e3edb138324130a7562b0bb21f115bb940f/tornado-6.5.10-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bdf942448169e5336451d0494d7e3d81cfa726d5aa312affdc4682dd62a62f6d", size = 467096, upload-time = "2026-09-15T13:47:38.559Z" }, - { url = "https://files.pythonhosted.org/packages/29/17/cc35dff68272d685cffd8600ffafbd8067e7d05e7348d9f80caddffbbd5f/tornado-6.5.10-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69acca6501eed74582b76dbbceee2a91613f54728e3e418346000d7103101676", size = 468067, upload-time = "2026-09-15T13:47:40.085Z" }, - { url = "https://files.pythonhosted.org/packages/c3/01/6e5349b4e1a53a4b4972a6716785e1fe7407f312063c3972690af8ff301b/tornado-6.5.10-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:66aaa3f57d30c6e6becee83ff28055d5930ac724214bde99393eefda83d5e015", size = 467901, upload-time = "2026-09-15T13:47:41.576Z" }, - { url = "https://files.pythonhosted.org/packages/28/5e/b4facf94370dba006819c8d304376f8b9fbec6b935b5e51bf45823a9790b/tornado-6.5.10-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4bd192b959f9128fb99b8898148070ba4574c9589b78bce42d1851131fe85828", size = 467308, upload-time = "2026-09-15T13:47:43.145Z" }, - { url = "https://files.pythonhosted.org/packages/56/ae/047938e828cafc8eca4c908fafb6588fee944e3af39a0af9d7b602499ae5/tornado-6.5.10-cp39-abi3-win32.whl", hash = "sha256:302eb1e0e3e159314eb591920529fdea80acca92df5510a2cec5bbd4f099ec72", size = 468387, upload-time = "2026-09-15T13:47:44.556Z" }, - { url = "https://files.pythonhosted.org/packages/d8/d4/5901517f05affd752490f6a654ba31b7474664e8dd80bd045a00c220bd88/tornado-6.5.10-cp39-abi3-win_amd64.whl", hash = "sha256:37ae8f150cecfdbf747fc4e12f5e9a97ecd8cf1d4cdb3f119e2de84b11196918", size = 468828, upload-time = "2026-09-15T13:47:45.961Z" }, - { url = "https://files.pythonhosted.org/packages/f3/1a/fd497f3a7f7b74bb04f4b94536b5c9f80742b5d50501fd27977652ddec16/tornado-6.5.10-cp39-abi3-win_arm64.whl", hash = "sha256:ce045d3c298fddd30e89a2777f97039d1b641eb9518ac7b26a4721903539c694", size = 467847, upload-time = "2026-09-15T13:47:47.283Z" }, -] - -[[package]] -name = "tqdm" -version = "4.70.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, -] - -[[package]] -name = "transformers" -version = "5.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "regex" }, - { name = "safetensors" }, - { name = "tokenizers" }, - { name = "tqdm" }, - { name = "typer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, -] - -[[package]] -name = "triton" -version = "3.7.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/fa/f856e24deb462d5f18bd4b5a746957862ab9b6ee5834bda60605ec348366/triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1", size = 184692359, upload-time = "2026-06-17T20:03:48.288Z" }, - { url = "https://files.pythonhosted.org/packages/c4/6f/fb96d15db6f36d6eae4cafb998c2e0353bf59d7c4ea1662d7497f269134a/triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728", size = 197719725, upload-time = "2026-06-17T19:53:20.419Z" }, -] - -[[package]] -name = "truststore" -version = "0.10.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, -] - -[[package]] -name = "typeguard" -version = "2.13.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3a/38/c61bfcf62a7b572b5e9363a802ff92559cb427ee963048e1442e3aef7490/typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4", size = 40604, upload-time = "2021-12-10T21:09:39.158Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/bb/d43e5c75054e53efce310e79d63df0ac3f25e34c926be5dffb7d283fb2a8/typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1", size = 17605, upload-time = "2021-12-10T21:09:37.844Z" }, -] - -[[package]] -name = "typer" -version = "0.27.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "rich" }, - { name = "shellingham" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, -] - -[[package]] -name = "typing-inspection" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, -] - -[[package]] -name = "tzdata" -version = "2026.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, -] - -[[package]] -name = "uvicorn" -version = "0.51.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a2/65/b7c6c443ccc58678c91e1e973bbe2a878591538655d6e1d47f24ba1c51f3/uvicorn-0.51.0.tar.gz", hash = "sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0", size = 94412, upload-time = "2026-07-08T10:59:05.962Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/ec/dbb7e5a6b91f86bfb9eb7d2988a2730907b6a729875b949c7f022e8b88fa/uvicorn-0.51.0-py3-none-any.whl", hash = "sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b", size = 73219, upload-time = "2026-07-08T10:59:04.44Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "httptools" }, - { name = "python-dotenv" }, - { name = "pyyaml" }, - { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, - { name = "watchfiles" }, - { name = "websockets" }, -] - -[[package]] -name = "uvloop" -version = "0.22.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, - { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, - { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, - { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, - { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, -] - -[[package]] -name = "vllm" -version = "0.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "anthropic" }, - { name = "apache-tvm-ffi" }, - { name = "blake3" }, - { name = "cachetools" }, - { name = "cbor2" }, - { name = "cloudpickle" }, - { name = "compressed-tensors" }, - { name = "depyf" }, - { name = "einops" }, - { name = "fastapi", extra = ["standard"] }, - { name = "fastsafetensors" }, - { name = "filelock" }, - { name = "flashinfer-python" }, - { name = "huggingface-hub" }, - { name = "humming-kernels", extra = ["cu13"] }, - { name = "ijson" }, - { name = "jsonschema" }, - { name = "lark" }, - { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, - { name = "lm-format-enforcer" }, - { name = "mcp" }, - { name = "mistral-common", extra = ["image"] }, - { name = "model-hosting-container-standards" }, - { name = "msgspec" }, - { name = "ninja" }, - { name = "numba" }, - { name = "numpy" }, - { name = "nvidia-cudnn-frontend" }, - { name = "nvidia-cutlass-dsl", extra = ["cu13"] }, - { name = "nvtx" }, - { name = "openai" }, - { name = "openai-harmony" }, - { name = "opencv-python-headless" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp" }, - { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions-ai" }, - { name = "outlines-core" }, - { name = "partial-json-parser" }, - { name = "pillow" }, - { name = "prometheus-client" }, - { name = "prometheus-fastapi-instrumentator" }, - { name = "protobuf" }, - { name = "psutil" }, - { name = "py-cpuinfo" }, - { name = "pybase64" }, - { name = "pydantic" }, - { name = "pynvvideocodec" }, - { name = "python-json-logger" }, - { name = "pyyaml" }, - { name = "pyzmq" }, - { name = "quack-kernels" }, - { name = "regex" }, - { name = "requests" }, - { name = "safetensors" }, - { name = "sentencepiece" }, - { name = "setproctitle" }, - { name = "setuptools" }, - { name = "six" }, - { name = "starlette" }, - { name = "tiktoken" }, - { name = "tilelang" }, - { name = "tokenizers" }, - { name = "tokenspeed-mla", marker = "sys_platform == 'linux'" }, - { name = "torch" }, - { name = "torchaudio" }, - { name = "torchcodec" }, - { name = "torchvision" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "typing-extensions" }, - { name = "watchfiles" }, - { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/39/096d88cfe34ee43a8b6e7c957d1ad6338fa5256474d6ff9a32008c6f8cd6/vllm-0.28.0.tar.gz", hash = "sha256:ac96dd0ec5be9c13f2aa4bfb50498c4727110406f6e4980b58a4097e4d18634a", size = 39991441, upload-time = "2026-08-26T10:08:30.56Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/31/007ac11dc756f7e69600b90b08e39f573eed9aaee6258198f9d9a2b06355/vllm-0.28.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:817b8181f7f61b4a62dc1d5d9ab39f2bfb60a6cb86c29879a78a147b85756787", size = 308873410, upload-time = "2026-08-26T10:08:13.738Z" }, - { url = "https://files.pythonhosted.org/packages/87/d7/97f6ecc2ae883e601e08d7cef87cb54ececeefcfe6b5e12d5d92f8d06d6b/vllm-0.28.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:addb0ffdaafd8155d75e9b3f5ddb3da28fdee9e8a7097ede91f7db2e9e1a3889", size = 314500103, upload-time = "2026-08-26T10:07:25.091Z" }, -] - -[package.optional-dependencies] -otel = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp" }, - { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions-ai" }, -] -runai = [ - { name = "runai-model-streamer", extra = ["azure", "gcs", "s3"] }, -] - -[[package]] -name = "wadler-lindig" -version = "0.1.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1e/67/cbae4bf7683a64755c2c1778c418fea96d00e34395bb91743f08bd951571/wadler_lindig-0.1.7.tar.gz", hash = "sha256:81d14d3fe77d441acf3ebd7f4aefac20c74128bf460e84b512806dccf7b2cd55", size = 15842, upload-time = "2025-06-18T07:00:42.843Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/96/04e7b441807b26b794da5b11e59ed7f83b2cf8af202bd7eba8ad2fa6046e/wadler_lindig-0.1.7-py3-none-any.whl", hash = "sha256:e3ec83835570fd0a9509f969162aeb9c65618f998b1f42918cfc8d45122fe953", size = 20516, upload-time = "2025-06-18T07:00:41.684Z" }, -] - -[[package]] -name = "watchfiles" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/2f/e42c992d2afda3108ea1c02acecc991b9f31d05c14adc2a7cee9ee211fc4/watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26", size = 400115, upload-time = "2026-05-18T04:32:02.06Z" }, - { url = "https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c", size = 393659, upload-time = "2026-05-18T04:30:50.951Z" }, - { url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" }, - { url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" }, - { url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" }, - { url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" }, - { url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" }, - { url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" }, - { url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" }, - { url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" }, - { url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" }, - { url = "https://files.pythonhosted.org/packages/fa/08/d9e2e0f9e8e6791d33aefc694ad7eefa7f901f63caff84a81ded38692f9c/watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898", size = 275480, upload-time = "2026-05-18T04:30:31.307Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e6/9d42569c0102645cc8cea5d8c7d8a1e9d4ada2cb7f05f75e554b8aa2202a/watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379", size = 288718, upload-time = "2026-05-18T04:32:10.745Z" }, - { url = "https://files.pythonhosted.org/packages/0a/26/88e0dc6ee3898169d7fa22bb6a69cabf2502d2ee25cb8c876d1262d204f8/watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5", size = 281026, upload-time = "2026-05-18T04:30:22.23Z" }, -] - -[[package]] -name = "wcwidth" -version = "0.9.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f0/b4/7830542634bb2d3e62aa3b586a72d5b3b6c91c3168929e7000ef3fed041d/wcwidth-0.9.2.tar.gz", hash = "sha256:ae0ef90b90f6af38b54f1fe6d58662ec33b3cb4b8391958a62416d654231727b", size = 955039, upload-time = "2026-10-05T00:24:05.521Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/1e/4532a81fb9dfbf4114a816775e0a36c3a64ee1d1f4bba2094e2da50be5dc/wcwidth-0.9.2-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:7ef5a940bd5e30bac6e721f1a48fce0cd7bb3ece19e9c5d139e72c76c35cfd07", size = 605343, upload-time = "2026-10-05T00:23:22.649Z" }, - { url = "https://files.pythonhosted.org/packages/a0/07/cb6940e81134b7ed25fa312ee9ab536a63db0793b149f88a90e603ceace9/wcwidth-0.9.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ae0800c5339423cc53d33a266ad264b42ba8aaa16d4464f6e6b1bee607f50b17", size = 609960, upload-time = "2026-10-05T00:23:27.049Z" }, - { url = "https://files.pythonhosted.org/packages/a4/80/15ad05d40bfa99155639fb9e13b3d77083aa0fab893c816db2543d29005c/wcwidth-0.9.2-cp310-abi3-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:9e542f1f8475b78452a295495d7a5bc3ead565112e9446a64dc93462a41c2a79", size = 762566, upload-time = "2026-10-05T00:23:38.322Z" }, - { url = "https://files.pythonhosted.org/packages/bc/f0/b8ef7758003d66b60f093695831a86dcc726aac01ee6446ffcbda27b61e3/wcwidth-0.9.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:674b518af28d38ee645ff97b74f5760abee5fad4bac74413bfc4b881ef2ce724", size = 771632, upload-time = "2026-10-05T00:23:32.448Z" }, - { url = "https://files.pythonhosted.org/packages/db/6c/f940133c71427c208575910e981942bd78c98b1f7cd0d1425ca4b7457c04/wcwidth-0.9.2-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:751bef0ab404b6a1dc028b56b4b85d46486be1c55833f80da533e42dc691f389", size = 769763, upload-time = "2026-10-05T00:23:40.175Z" }, - { url = "https://files.pythonhosted.org/packages/92/8f/285f862826f721964ec7c42f81dc53d23afbd723a0f4cd989651f8218e25/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c3d80f39ba4653a595edae9aa46a509d14883790a8fc23c5db221ceb207f64b7", size = 782134, upload-time = "2026-10-05T00:23:33.926Z" }, - { url = "https://files.pythonhosted.org/packages/c2/2d/64aa54882a5d556d3654c1f926d9118b797461033e23a158409941a37c8f/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:0a47e03d8293590ecce66c45dc20ff7b4b885e3c78093722239585eca0d77ab2", size = 782851, upload-time = "2026-10-05T00:23:41.974Z" }, - { url = "https://files.pythonhosted.org/packages/59/39/52389f6de7fe2e9c14ceb8253dd99034bd86e1c87847ea3c100a97dded9a/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67d901a4ad99249eb775b4ee4769ca97fa405d35a75f46e83166910a47003f04", size = 784621, upload-time = "2026-10-05T00:23:43.449Z" }, - { url = "https://files.pythonhosted.org/packages/b3/8b/20225500a076ace27bbcc8a6fd7c55125133c57a618816c7b7b8b73070b1/wcwidth-0.9.2-cp310-abi3-win32.whl", hash = "sha256:ee1fd0db9d9fd711a70f3e7765e0e04c05d26982fa05361456163062549d7da4", size = 593123, upload-time = "2026-10-05T00:23:55.953Z" }, - { url = "https://files.pythonhosted.org/packages/5a/d6/b0690f55ea0483530a18bac917fbadbf54f35122510446fc370f5f1c2453/wcwidth-0.9.2-cp310-abi3-win_amd64.whl", hash = "sha256:2a9746de704242bd4fdaabb31dd46b82f694a56a8d21081ad89b679a89da9fec", size = 597827, upload-time = "2026-10-05T00:23:57.489Z" }, - { url = "https://files.pythonhosted.org/packages/e5/11/6ecf4e9e268ab1a4ec617ffcccc2ee4a71301625f5490912dbaba462fa9c/wcwidth-0.9.2-cp310-abi3-win_arm64.whl", hash = "sha256:b9c6ab615e03723b7f8760ea2f27758d656e7e13b51515c9dca5c3e8b04612fa", size = 598415, upload-time = "2026-10-05T00:23:51.517Z" }, - { url = "https://files.pythonhosted.org/packages/9c/b4/0bfa065af506540d9d558e3e5548cff00bc1f9b24e6e2a8512498e8628de/wcwidth-0.9.2-py3-none-any.whl", hash = "sha256:89ca642c5bf0101157a09366be69fad0379db1f700ae39a920e103234573670e", size = 301667, upload-time = "2026-10-05T00:23:21.097Z" }, -] - -[[package]] -name = "websocket-client" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, -] - -[[package]] -name = "websockets" -version = "16.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/f7/bc3a25c5ec26ce62ce487690becc2f3710bbc7b33338f005ad390db0b986/websockets-16.1.1.tar.gz", hash = "sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57", size = 182204, upload-time = "2026-07-17T22:51:05.858Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/9d/681cda21c9eee743203a6cb79b9d3d05adad9aa60ec660c6c9bf4dd619ca/websockets-16.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00", size = 179600, upload-time = "2026-07-17T22:49:13.92Z" }, - { url = "https://files.pythonhosted.org/packages/fb/8d/6195a88b45e8d2a8f745fc2046e36f885a3c9763e6767d2c46229bf9510c/websockets-16.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b", size = 177272, upload-time = "2026-07-17T22:49:15.453Z" }, - { url = "https://files.pythonhosted.org/packages/73/e3/fe2d498c64dea0095c9a9f9a351af4cd6eef31b618395582bc1f38ba45ff/websockets-16.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175", size = 177542, upload-time = "2026-07-17T22:49:16.875Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ed/f1831681fce0e3242346e5458486003c5f124ed69e5e0b847fd029db4973/websockets-16.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1", size = 187137, upload-time = "2026-07-17T22:49:18.323Z" }, - { url = "https://files.pythonhosted.org/packages/6f/79/4ff9dcc1bb46f6b4c536936dde1fd60f9b564f3304307274db97f4c9496d/websockets-16.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15", size = 188374, upload-time = "2026-07-17T22:49:19.65Z" }, - { url = "https://files.pythonhosted.org/packages/62/c3/5c49b6efb36cab733d23773f6de575e1dba65736ead17d5d2b2a1daef779/websockets-16.1.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa", size = 191155, upload-time = "2026-07-17T22:49:21.331Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f6/56ccceda3a4838d18f1d40821480da4775397e8b1eecf4031e20c50e2e90/websockets-16.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab", size = 189011, upload-time = "2026-07-17T22:49:22.889Z" }, - { url = "https://files.pythonhosted.org/packages/86/d6/ad5286241a2bce1107e2798d3bfbd62cf79aee167bdb654f8cb1e9dbf949/websockets-16.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847", size = 187766, upload-time = "2026-07-17T22:49:24.339Z" }, - { url = "https://files.pythonhosted.org/packages/bc/67/d65c970b7e347fdca69479beb7811c2060529956730a7a4e3ae7c66b0e31/websockets-16.1.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428", size = 185173, upload-time = "2026-07-17T22:49:25.743Z" }, - { url = "https://files.pythonhosted.org/packages/1d/5b/14af3cd4ee69d8ea9baca58f3dc3cfb1ba78332a347fd478cb096549d60e/websockets-16.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf", size = 187809, upload-time = "2026-07-17T22:49:27.147Z" }, - { url = "https://files.pythonhosted.org/packages/7b/11/be301710d70de97e3e7b3586e6d492c9c06d6a61bf1c2202c36cf0c75607/websockets-16.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751", size = 186412, upload-time = "2026-07-17T22:49:28.611Z" }, - { url = "https://files.pythonhosted.org/packages/db/07/fe1435bf6fe738a3d3b54dbe0c18dabf12cba4d909ac8b58b539ce27c1f4/websockets-16.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f", size = 188290, upload-time = "2026-07-17T22:49:29.965Z" }, - { url = "https://files.pythonhosted.org/packages/8a/0a/81f394aff8efcbb01208c1ced77df0a3c7fcce584a88c7273663697946c2/websockets-16.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2", size = 185844, upload-time = "2026-07-17T22:49:31.447Z" }, - { url = "https://files.pythonhosted.org/packages/39/5c/dd485b995473f415510251fe9bd708f2d24458f439fce958daf8d66dc7c6/websockets-16.1.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383", size = 186823, upload-time = "2026-07-17T22:49:33.104Z" }, - { url = "https://files.pythonhosted.org/packages/9d/0b/f78de76ff446f1e66af12b43c48a35f31744de93cfdec2f4ea67d5d7bbf1/websockets-16.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3", size = 187102, upload-time = "2026-07-17T22:49:34.616Z" }, - { url = "https://files.pythonhosted.org/packages/37/a1/4cf892007778eaf84ad162bfc98046e0ed89b63ac55949e3236626b2a23f/websockets-16.1.1-cp312-cp312-win32.whl", hash = "sha256:1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747", size = 179943, upload-time = "2026-07-17T22:49:36.213Z" }, - { url = "https://files.pythonhosted.org/packages/d9/de/6abe251d28c3a3f217096575400b27750b18e0b1d2fff3a2a239960fea07/websockets-16.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7", size = 180243, upload-time = "2026-07-17T22:49:37.626Z" }, - { url = "https://files.pythonhosted.org/packages/be/4d/2d0d67834092e354d2b0498f014a41249a89556bc406cf86f3e1557bb463/websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3", size = 173814, upload-time = "2026-07-17T22:51:04.184Z" }, -] - -[[package]] -name = "win32-setctime" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, -] - -[[package]] -name = "wrapt" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3e/d2/a254a26d8ceaea87e0eee2e89fcfe53ddc1858418647493bb2937549ab6f/wrapt-2.5.0.tar.gz", hash = "sha256:c48cdb6c904dca76d9915a579e4a5fab6b0c25f650c1019ce78a78effaf7a345", size = 183288, upload-time = "2026-09-27T01:41:56.874Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/a6/44589f9b34160280a1fbccbcf206b18df034a1b3e9584336d3a7f039a33c/wrapt-2.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b898caea081303006decc562c7fca5126f7c96507e78dd8f1ae3285dfa50ddc7", size = 106192, upload-time = "2026-09-27T01:40:00.941Z" }, - { url = "https://files.pythonhosted.org/packages/ce/2b/94db3ba2e9528400e4173cbb67a56822a4644d4c2b319feb0cc638094b36/wrapt-2.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8837fbe708cb9d8a2d32a37dee836d24a531f02560db26418e2b181986fa21cb", size = 105828, upload-time = "2026-09-27T01:40:02.494Z" }, - { url = "https://files.pythonhosted.org/packages/11/0e/3ce67af67525c0068680637c0eba823ce2f0fc6eff2d1779cab656fb8c3d/wrapt-2.5.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cabb9c17ab79b2549d1f23b36f436473ad9253ef53995a817feba26fae69d5b", size = 250177, upload-time = "2026-09-27T01:40:03.819Z" }, - { url = "https://files.pythonhosted.org/packages/d3/3b/262b2c3c38aca6fa32cfea6407dbb346ce26ff788179b98137b3b3f3d8ca/wrapt-2.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6761765cc520ff9616fb035c02a85d1d744f7f70edd4649718fd0d09c589eacf", size = 250332, upload-time = "2026-09-27T01:40:05.15Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ec/d54d273a2223d1964ecd5b5954aca18e8fbf2c5d8471e60ef3cbaaed9325/wrapt-2.5.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a145a7826eddea3eb5814903f98f93756042b919bb5305544cb1331daa2705b1", size = 229600, upload-time = "2026-09-27T01:40:06.47Z" }, - { url = "https://files.pythonhosted.org/packages/8e/88/33c75ac47b13b0bf77b44d96d02b1d0a2e6db66180909bacce86ef8fcd8e/wrapt-2.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6a9ee62a970075738909909bdbef3a7da9f7ae03dfca584db283547a29503b56", size = 246890, upload-time = "2026-09-27T01:40:07.838Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0e/c3a3a158801e5bee7a0d9aa3561a127bd80f166b02b7b520a39e45d8f9b9/wrapt-2.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:691671ea05684f921ffc2e935fd3f9311c1795a10fbbfa006b46269733668f66", size = 226463, upload-time = "2026-09-27T01:40:09.206Z" }, - { url = "https://files.pythonhosted.org/packages/7c/2c/4c48ba51698a87e2e8299fc04856bbef391cb725f195bc93f0bc6196e1f2/wrapt-2.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e716f47c7f61e11709d3c0904213c94fc22999abf0c41461276cef886c1e8b4d", size = 238583, upload-time = "2026-09-27T01:40:10.631Z" }, - { url = "https://files.pythonhosted.org/packages/33/d2/077835618ed96b131730f74301a7236efd841ec2f341afe36303389b56ef/wrapt-2.5.0-cp312-cp312-win32.whl", hash = "sha256:5421acb5c363a9bc959122a8645e3f1f42010c932dc53885b11a5ff5b5a6d730", size = 100221, upload-time = "2026-09-27T01:40:11.91Z" }, - { url = "https://files.pythonhosted.org/packages/96/9f/9e56db2a3492275be809082c6f63a7a063f52b3e1812aae1eb03ec33f64c/wrapt-2.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:ab45839c912777e2738fed369636589c8b2a6d9c44ca56de0fd0814581d467c2", size = 105786, upload-time = "2026-09-27T01:40:13.208Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e4/e37ff75e5254564aa50e13bc4313383d02d5ec78f5bd3b9b1994dd7052d5/wrapt-2.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:ce4cab32c37ef71e69cf88f909b7febd0dd79543e5ae3650e2b874e0f3d3b975", size = 102937, upload-time = "2026-09-27T01:40:14.534Z" }, - { url = "https://files.pythonhosted.org/packages/87/7d/5ed859fad4b5eddd598a846150aaab2703730ed4886c5c5e03b0df0cfdd5/wrapt-2.5.0-py3-none-any.whl", hash = "sha256:107eea1a511e98a3a5033b0c2cb403fbb37f05dee6ac1fb85c0460d311ec278c", size = 81586, upload-time = "2026-09-27T01:41:55.479Z" }, -] - -[[package]] -name = "xgrammar" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "numpy" }, - { name = "pydantic" }, - { name = "torch" }, - { name = "transformers" }, - { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/f4/e71693d8cec60b7e36dab660784ecc5a6aa51e478a83b556011645c58c87/xgrammar-0.2.3.tar.gz", hash = "sha256:f76423630ae3ac4e090cb38ce1e30e7bcc69b3dee4d22d94353944386a4c6f18", size = 2447704, upload-time = "2026-06-27T04:45:24.759Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/1c/0cdb22fc799e6d158b3243eeb895ae2e086825487b57767838c98d4864ee/xgrammar-0.2.3-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:173e167d43a5cf4171eee2be86097decff8803b0a0853d7baaf446c732a7d3a9", size = 23284489, upload-time = "2026-06-27T04:44:23.927Z" }, - { url = "https://files.pythonhosted.org/packages/cf/bc/994dc6f222189174840c29a1f5b4c175e69dfe13ed2e25b6dbbe9f200a29/xgrammar-0.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aa35f24835a59c822e249ecc80912eea4de03fc8b04afb2f82c8b950a56be6ef", size = 23240027, upload-time = "2026-06-27T04:44:26.255Z" }, - { url = "https://files.pythonhosted.org/packages/e4/79/0bb37937bf847c738c64b64dc50ddc12e7c526b34c5ab82cebe58da5ec8f/xgrammar-0.2.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11255f184971489fc72b948b096e2917f482ba2dca975177f5411562cedb9c6d", size = 44314481, upload-time = "2026-06-27T04:44:28.875Z" }, - { url = "https://files.pythonhosted.org/packages/d4/fd/5ebd5d14b8993cb225151bbb8f2011742fc7a7d94a3bdbc3ec3954b9b62d/xgrammar-0.2.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdf081fab29694302d41d61dcf52fad7d253879a718bc6afc68db0a0dabd7f19", size = 44855110, upload-time = "2026-06-27T04:44:31.586Z" }, - { url = "https://files.pythonhosted.org/packages/17/66/67239f43b0244f65aec4639f51ab95905db42eb66e532ee2a4e5cdce32de/xgrammar-0.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:e7787dd8321a04f86116b756aa3dadd622e3607a3559b1e986cc5f77da00d68e", size = 15780277, upload-time = "2026-06-27T04:44:34.081Z" }, -] - -[[package]] -name = "xyzservices" -version = "2026.9.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c4/bd/b75d5c2312caec607ec995200fa1eb8453225fd869947ec743919c279544/xyzservices-2026.9.1.tar.gz", hash = "sha256:8d1a39bf6b192940cc5db5264eeefc1b20dd184f8b83b1303b078743744f5943", size = 1134047, upload-time = "2026-09-04T10:35:41.399Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/71/9219fe65ef32bddc0b5d14e1d98dfefe9d2c9c1b1d1e93efca1f29fcbd4d/xyzservices-2026.9.1-py3-none-any.whl", hash = "sha256:af4fddac0f1fa5f7951834e2c58e10109e722be548cb1fe4f1a8e8b18cef4c09", size = 98829, upload-time = "2026-09-04T10:35:40.08Z" }, -] - -[[package]] -name = "yarl" -version = "1.24.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "multidict" }, - { name = "propcache" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/31/33/ebe9e3d1f86c7a0b51094c0a146392045ca1631d2664889539dec8088a33/yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f", size = 228679, upload-time = "2026-07-20T02:07:45.435Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/84/71d051c850b5af41d168c679d9eb67eb7c55283ac4ee131673edf134bc4e/yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d", size = 136035, upload-time = "2026-07-20T02:05:25.489Z" }, - { url = "https://files.pythonhosted.org/packages/03/4d/8ad27f9a1b7e69313cca5d695b925b48efe51208d3490e0844bae97cabc0/yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec", size = 97642, upload-time = "2026-07-20T02:05:27.429Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b4/05b4131c407006cd1e410e9c6539f16a0945724677e5364447313c15ea3e/yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c", size = 97323, upload-time = "2026-07-20T02:05:29.441Z" }, - { url = "https://files.pythonhosted.org/packages/20/16/e618c875c73e0e39611f20a581b3d5e8d59b8857bf001bee3263044c6deb/yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54", size = 107741, upload-time = "2026-07-20T02:05:31.367Z" }, - { url = "https://files.pythonhosted.org/packages/d9/9a/c4defeaf3ed33fcb346aacf9c6e971a8d4e2bde04a0310e79abb208e7965/yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12", size = 103570, upload-time = "2026-07-20T02:05:33.303Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e7/0e0e0de5865ebd5914537ef486f36c727a59865c3ac0cf5ff1b32aececbf/yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d", size = 115815, upload-time = "2026-07-20T02:05:35.292Z" }, - { url = "https://files.pythonhosted.org/packages/2b/27/ca56b700cb170aba25a3893b75355b213935657dc5714d2383354a270e62/yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1", size = 116025, upload-time = "2026-07-20T02:05:37.503Z" }, - { url = "https://files.pythonhosted.org/packages/d6/d0/d56c859b8222116f5d68459199f48359e0bf121b6f65a69bf329b3602ba0/yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9", size = 109835, upload-time = "2026-07-20T02:05:39.506Z" }, - { url = "https://files.pythonhosted.org/packages/70/a2/3a35557e4d1a79425040eba202ccaf08bdc8717680fc77e2498a1ad2e0a5/yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027", size = 108884, upload-time = "2026-07-20T02:05:41.584Z" }, - { url = "https://files.pythonhosted.org/packages/e4/35/ef4c26356b7913c68983bac2d72a4212b3347af551cb8d250b99b5ed7b7f/yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b", size = 107308, upload-time = "2026-07-20T02:05:43.697Z" }, - { url = "https://files.pythonhosted.org/packages/d5/91/ff0dc66c2ccf3e0153ab97ff61eabab4400e6a5264af427ab30cd69f1857/yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293", size = 103646, upload-time = "2026-07-20T02:05:45.895Z" }, - { url = "https://files.pythonhosted.org/packages/74/f0/33b9271c7f881766359d58266fa0811d2e5210ed860e28da7dc6d7786344/yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e", size = 115305, upload-time = "2026-07-20T02:05:47.832Z" }, - { url = "https://files.pythonhosted.org/packages/ef/65/fd79fb1868c4a80db8661091de525bf430f63c3bea1b20e8b6a84fc7d359/yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b", size = 108404, upload-time = "2026-07-20T02:05:49.604Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ba/dbabe6b262f17a816c70cfc09558dbf03ece3ec76684d02f911a3d3a189c/yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce", size = 115940, upload-time = "2026-07-20T02:05:51.741Z" }, - { url = "https://files.pythonhosted.org/packages/a5/43/fab2d1dad9d340a268cdde63756a123d069723efff6a372d123fa74a9517/yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba", size = 110006, upload-time = "2026-07-20T02:05:53.554Z" }, - { url = "https://files.pythonhosted.org/packages/c4/27/41eb51bbd1b8d89546b83897cfb0164f1e109304fd408dbb151b639eec0f/yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b", size = 97618, upload-time = "2026-07-20T02:05:55.57Z" }, - { url = "https://files.pythonhosted.org/packages/3c/25/b2553764b3d65db711d8f45416351ec4f420847558eb669edcbcaadf5780/yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c", size = 93018, upload-time = "2026-07-20T02:05:57.554Z" }, - { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" }, -] - -[[package]] -name = "z3-solver" -version = "4.15.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/8e/0c8f17309549d2e5cde9a3ccefa6365437f1e7bafe71878eaf9478e47b18/z3_solver-4.15.4.0.tar.gz", hash = "sha256:928c29b58c4eb62106da51c1914f6a4a55d0441f8f48a81b9da07950434a8946", size = 5018600, upload-time = "2025-10-29T18:12:03.062Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/33/a3d5d2eaeb0f7b3174d57d405437eabb2075d4d50bd9ea0957696c435c7b/z3_solver-4.15.4.0-py3-none-macosx_13_0_arm64.whl", hash = "sha256:407e825cc9211f95ef46bdc8d151bf630e7ab2d62a21d24cd74c09cc5b73f3aa", size = 37052538, upload-time = "2025-10-29T18:11:46.233Z" }, - { url = "https://files.pythonhosted.org/packages/47/84/fd7ffac1551cd9f8d44fe41358f738be670fc4c24dfd514fab503f2cf3e7/z3_solver-4.15.4.0-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:00bd10c5a6a5f6112d3a9a810d0799227e52f76caa860dafa5e00966bb47eb13", size = 39807925, upload-time = "2025-10-29T18:11:49.81Z" }, - { url = "https://files.pythonhosted.org/packages/21/c9/bb51a96af0091324c81b803f16c49f719f9f6ea0b0bb52200f5c97ec4892/z3_solver-4.15.4.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e103a6f203f505b8b8b8e5c931cc407c95b61556512d4921c1ddc0b3f41b08e", size = 29268352, upload-time = "2025-10-29T18:11:53.032Z" }, - { url = "https://files.pythonhosted.org/packages/bf/2e/0b49f7e4e53817cfb09a0f6585012b782dfe0b666e8abefcb4fac0570606/z3_solver-4.15.4.0-py3-none-manylinux_2_34_aarch64.whl", hash = "sha256:62c7e9cbdd711932301f29919ad9158de9b2f58b4d281dd259bbcd0a2f408ba1", size = 27226534, upload-time = "2025-10-29T18:11:55.59Z" }, - { url = "https://files.pythonhosted.org/packages/26/91/33de49538444d4aafbe47415c450c2f9abab1733e1226f276b496672f46c/z3_solver-4.15.4.0-py3-none-win32.whl", hash = "sha256:be3bc916545c96ffbf89e00d07104ff14f78336e55db069177a1bfbcc01b269d", size = 13191672, upload-time = "2025-10-29T18:11:58.424Z" }, - { url = "https://files.pythonhosted.org/packages/03/d6/a0b135e4419df475177ae78fc93c422430b0fd8875649486f9a5989772e6/z3_solver-4.15.4.0-py3-none-win_amd64.whl", hash = "sha256:00e35b02632ed085ea8199fb230f6015e6fc40554a6680c097bd5f060e827431", size = 16259597, upload-time = "2025-10-29T18:12:01.14Z" }, -] - -[[package]] -name = "zipp" -version = "4.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/23/655a1802fe8041302c959774ca7c80b53bc24737ff3ef45cb50ef11bd96c/zipp-4.1.1.tar.gz", hash = "sha256:7ebb7a44c021b29fd8dbd7cce6812d0d7b5b454521f93cc71af6ccd155aaa70b", size = 27649, upload-time = "2026-10-03T17:03:03.452Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/98/df615823cd9419131ce19fba00de53a663794369e198aade064a244b385d/zipp-4.1.1-py3-none-any.whl", hash = "sha256:8979f52d874162f485ff2981e3891f3a3317b7a3dd43ff1e1775b9304f307a9c", size = 10582, upload-time = "2026-10-03T17:03:02.506Z" }, -] - -[[package]] -name = "zstandard" -version = "0.25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, - { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, - { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, - { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, - { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, - { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, - { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, - { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, - { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, - { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, - { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, - { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, - { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, - { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, - { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, - { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, - { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, -] diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 955d0d07b49..8aa388e138d 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -1,63 +1,106 @@ -# Managed Dynamo generation design - -The managed Dynamo backend owns a fixed vLLM fleet inside the Ray allocation. -It is deliberately narrower than Dynamo itself: there is no external-runtime, -Kubernetes, DGD, multi-node engine-group, or non-vLLM mode. - -## Ownership and placement - -Constructing `ManagedDynamoRuntime` is inert. Its explicit `start()` method -allocates ports, launches etcd and NATS JetStream, creates one Ray-managed -`dynamo.vllm` process per model-parallel group, and starts the frontend. A -worker group must fit on one node. Its world size is derived from vLLM tensor -parallelism times pipeline parallelism; expert parallelism must be either one -or equal to tensor parallelism. - -Startup completes only after the frontend sees the same fixed membership at -the generation and RL endpoints and advertises the configured model. Worker -handles are recorded before readiness checks so partial startup failures can -be torn down. Shutdown is idempotent and guards the frontend, worker pool, -NATS, etcd, and temporary state independently. - -## Generation state - -Both GRPO trainers use `DynamoGeneration.generate_async()` against the managed -frontend. NeMo-Gym traffic passes through a process-local token wrapper. It -uses the policy tokenizer, preserves caller `nvext.extra_fields`, adds Dynamo -engine metadata, and translates rendered multi-turn prefixes back to the exact -caller token IDs. - -Serialized rollout copies contain only frontend URLs, immutable worker admin -endpoints, and the prepared refit metadata. They cannot own or stop services. -That is enough for the SingleController actor to issue weight updates and for -AREAL-style post-refit cache invalidation; Magistral keeps its existing -driver-side invalidation lifecycle. - -## Weight refit - -Dynamo uses `CollectiveWeightSynchronizer`. If each engine has world size `E`, -worker `i` starts at rank `training_world_size + i * E`. The policy sender uses -vLLM's peer initialization and a packed transfer with two 1-GiB buffers. vLLM -reads the packing mode and geometry once, from the init info that NeMo-RL sends -to `init_weight_transfer_engine`; each update round carries only tensor names, -dtypes, and shapes. Before a worker starts, the isolated vLLM environment checks -that its weight-transfer API accepts that init and update shape. - -Generation is drained before refit. The worker then runs vLLM's native -`start_weight_update`, `update_weights`, and `finish_weight_update` transaction. -KV-cache invalidation stays outside the generic synchronizer because GRPO's -cache mode determines where it runs. - -## Dependency isolation - -`BUILD_DYNAMO=1` adds a Python 3.12 `/opt/dynamo_venv` to the standard image. -It contains only `ai-dynamo[vllm]==1.5.1`, its pinned vLLM 0.28.0, etcd, -and NATS. NeMo-RL's normal Ray and engine environments are unchanged; the -standard NeMo-RL vLLM environment currently uses vLLM 0.29.0. - -Layerwise reload depends on vLLM PR #44814, which fixes reload accounting for -composed loaders. vLLM 0.28.0 includes it, and the installer fails if the -installed vLLM does not. - -See [Managed Dynamo generation on Slurm](../guides/dynamo-generation.md) for -build, configuration, and launch instructions. +# Dynamo integration design + +`policy.generation.dynamo: true` serves NeMo RL's own vLLM engines through +Dynamo's frontend and KV-aware router. Two requirements shape the design: + +- A vLLM bump must never be blocked by Dynamo. +- Dynamo CI coverage must stay consistent with native vLLM's. + +Both hold only if the engine process contains no Dynamo-owned vLLM code. Then a +vLLM bump can break Dynamo serving only where it also breaks native vLLM. + +## Engine hosting + +`VllmGeneration` builds its workers exactly as without Dynamo. Each data-parallel +leader (`VllmAsyncGenerationWorkerImpl`) owns an `AsyncLLM`. With Dynamo on, it +also runs `dynamo.common.backend.Worker` as a task on its own actor event loop, +which is the loop that owns the `AsyncLLM`. + +`dynamo.common.backend` is Dynamo's engine-agnostic backend SDK and imports no +vLLM. The adapter, `nemo_rl/models/generation/dynamo/native_engine.py`, +implements its `LLMEngine` over the worker's `AsyncLLM`: + +- `generate` maps Dynamo's token request onto vLLM `SamplingParams` the way + vLLM's OpenAI server does, and streams token deltas back. Rollouts that NeMo RL + itself sends carry an annotation. For those, the engine uses the same code as + native `generate_async`: + - the worker's `_rollout_max_new_tokens` and `_rollout_sampling_params` build + the request; + - `request_pipeline.rollout_result` builds the result, routed experts + included. + + The result travels in `engine_data`, and the driver assembles the same + output with `request_pipeline.rollout_output`. + + NeMo-Gym's calls also carry an annotation (Gym's Dynamo dialect). The engine + applies the native chat endpoint's rules to them: the on-policy sampling check + (`vllm/utils.off_policy_sampling_error`), the prefix splice + (`openai_server_utils.splice_prefix_tokens`) and the context clamp. A + token-capture call stages through the worker's own capture lifecycle + (`_begin_request_capture` / `_finish_request_capture`), and the token fields + and routed experts come from the native endpoint's helpers + (`chat_generation_tokens`, `chat_routed_experts`). + + A request with media carries the frontend's tokens, with one unexpanded + placeholder per item, and the media URLs. The adapter loads the media with + vLLM's media connector. It then builds the processed prompt with the engine's + renderer (`process_for_engine_async`), the same step that follows rendering + on the native endpoint. The splice, the placeholder remap + (`remap_multimodal_placeholders`) and media capture (`_capture_request_media`) + all run on that processed prompt. +- `abort` cancels the vLLM request on client disconnect. +- Registration advertises the context length, the KV block size (from vLLM's + KV-cache group metadata), the block count and the scheduler limits. It + publishes no token budget. The engine owns the context check and the output + cap, because a prefix splice changes the prompt after the frontend has seen it. +- `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ + publisher. It is set only when the engine caches prefixes, and it binds a + wildcard endpoint, because vLLM binds only wildcard endpoints. A vLLM stat + logger publishes the router's per-engine load signal. + +Because the Worker runs on the actor's own loop, the worker's other methods +interleave with Dynamo requests exactly as with native requests: +- refit (every transport); +- `pause_generation_async` / `resume_generation_async`. + +Colocated generation is rejected at startup: a sleeping engine stays +registered, so the frontend would keep routing to it. + +No HTTP control routes are involved. The Rust Worker installs a SIGTERM handler +that chains to Ray's, so actor shutdown is unchanged, and discovery drops an +engine when its process exits. + +## Driver + +`VllmGeneration` starts `ManagedDynamoRuntime`, which runs etcd, NATS and +`dynamo.frontend`. After the engines load, it asks every DP leader to register +and waits until the frontend routes to all of them. Each poll also asks the +leaders whether their endpoint task stopped, so a failed registration fails +with its cause instead of a timeout. + +`generate_async` on text prompts posts a token-prompt completion to the frontend +(`dynamo/rollout_client.py`). It sets top-level `ignore_eos` and a context-sized +`max_tokens`, so the frontend's own stop rules never end a stream before the +engine reports its result. Everything else keeps native dispatch; the guide +lists what goes where. + +## Dependency coupling + +The vLLM workers' venv (the `vllm` extra) carries the released `ai-dynamo` and +`ai-dynamo-runtime` wheels, never `ai-dynamo[vllm]`. The engine process only +imports `dynamo.common.backend`, `dynamo.llm.exceptions` and `dynamo._core`. + +What a Dynamo upgrade has to re-check: +- the `dynamo.common.backend` contract (`LLMEngine`, `Worker`, `WorkerConfig`, + publisher and logprob helpers); +- the request fields of Dynamo's preprocessed request; +- the frontend's `nvext` extensions and annotation forwarding; +- the parser names the native settings map to (`dynamo/config.py`). + +The one vLLM↔Dynamo wire coupling is vLLM's KV-event msgpack, which Dynamo's +router decodes. A change there degrades routing, not serving. The managed +runtime logs the router's applied-event counts at shutdown, and the Dynamo +functional overlays fail if the engines published events but none were +applied (`tests/functional/check_dynamo_overlay.sh`). + +`ai-dynamo-runtime` caps pydantic, which pins the vLLM venvs' pydantic version. diff --git a/docs/guides/async-grpo.md b/docs/guides/async-grpo.md index 9a76ceae772..f4d3562e023 100644 --- a/docs/guides/async-grpo.md +++ b/docs/guides/async-grpo.md @@ -221,7 +221,7 @@ If no `replay_buffer.pt` file is found in the latest checkpoint directory, train 3. **Resource Allocation**: Ensure sufficient GPU memory for both the training and generation clusters -4. **In-Flight Weight Updates**: Enable `in_flight_weight_updates: true` to refit without waiting for the longest in-flight generation to finish. Except for managed Dynamo, the collector requests a generation pause and resume from every async backend around the weight transfer. Async vLLM implements this contract while preserving request state. SGLang acknowledges these hooks through its weight synchronizer, which pauses, flushes KV, transfers weights, and resumes generation; its configured pause mode determines whether requests are requeued or aborted. A backend that does not implement the hook emits a warning once per backend type per process and refits without a collector-side pause or drain. Managed Dynamo always drains active trajectories before refit. vLLM requires `async_engine: true`; the Megatron backend is always async-engine. +4. **In-Flight Weight Updates**: Enable `in_flight_weight_updates: true` to refit without waiting for the longest in-flight generation to finish. The collector requests a generation pause and resume from every async backend around the weight transfer. Async vLLM implements this contract while preserving request state. SGLang acknowledges these hooks through its weight synchronizer, which pauses, flushes KV, transfers weights, and resumes generation; its configured pause mode determines whether requests are requeued or aborted. A backend that does not implement the hook emits a warning once per backend type per process and refits without a collector-side pause or drain. vLLM requires `async_engine: true`; the Megatron backend is always async-engine. 5. **Recompute KV Cache After Weight Updates**: `recompute_kv_cache_after_weight_updates` defaults to `true` to invalidate reusable KV/prefix caches when weights change. On the native async vLLM in-flight path, caches are cleared while generation is paused, so preserved requests recompute their KV after resuming. This repeats prefill for those requests and can reduce throughput. SGLang always flushes KV inside its weight synchronizer before the transfer, regardless of this flag; on its in-flight path, the collector skips a second flush after generation resumes. Other refit paths keep their existing post-update invalidation behavior. Set the flag to `false` to retain pre-update KV caches and avoid this recomputation cost on backends that support doing so. On the Megatron generation backend, the flag must agree with `policy.generation.mcore_generation_config.kv_cache_management_mode`: use `true` with `recompute`, or explicitly set `false` with `persist`/`offload`; setup errors on a mismatch. diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index da804901718..09915d3f276 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -1,20 +1,165 @@ -# Managed Dynamo generation on Slurm +# Dynamo serving for vLLM generation -NeMo RL can launch and own Dynamo's control plane, frontend, and a fixed vLLM -worker fleet inside a Slurm-backed Ray allocation. This mode supports direct -GRPO and NeMo-Gym rollouts, NCCL weight refits, cache invalidation, and -`generation_metrics/*` telemetry sent to enabled loggers such as W&B. -See the [Dynamo integration design](../design-docs/dynamo-integration.md) for -the service ownership, startup, and weight-refit architecture. +Any vLLM generation config can serve its rollouts through +[Dynamo](https://github.com/ai-dynamo/dynamo)'s OpenAI frontend and KV-aware +router. Add one line: + +```yaml +policy: + generation: + backend: vllm + dynamo: true +``` -This integration is managed and vLLM-only. It does not connect to an external -Dynamo deployment and does not support Kubernetes, DGD, SGLang, TensorRT-LLM, -speculative decoding, quantized generation, or model-parallel engine groups -that span nodes. +The same NeMo RL vLLM workers run the same engines, with the same refit +transports, pause and resume, quantization, speculative decoding and metrics. +Each engine also registers with a managed `dynamo.frontend`, and NeMo RL's own +rollouts (`generate_async`) go through that frontend, so each request lands on +the engine that already holds most of its prompt's KV cache. The few settings +Dynamo cannot serve fail at startup; see [Not supported](#not-supported). + +See the [Dynamo integration design](../design-docs/dynamo-integration.md) for +how the engines are hosted and why Dynamo upgrades and vLLM bumps stay +independent. + +## What happens at startup + +With `dynamo: true`: + +- `vllm_cfg.async_engine` is turned on, because Dynamo drives the engine's + `AsyncLLM`. Every rollout path the sync engine supports also runs on the async + one. +- The driver starts etcd, NATS and `dynamo.frontend`. The frontend runs from the + vLLM workers' own venv, so it is the same ai-dynamo release. +- Each engine turns on vLLM's KV-cache events for `router_mode: kv` and + registers with the frontend from inside its own worker process. Setup waits + until the frontend routes to every engine. +- vLLM publishes KV-cache events only for blocks its prefix cache stores. With + `vllm_cfg.enable_prefix_caching: false`, the engines publish none, and the + KV router routes on load alone. +- Shutdown stops the services after the engines. + +## Configure + +`dynamo: true` is a complete configuration. `policy.generation.dynamo_cfg` only +tunes the frontend and registration. Its fields and defaults live in +`nemo_rl/models/generation/dynamo/config.py`, and +[`examples/configs/grpo_math_1B_dynamo_single_controller.yaml`](../../examples/configs/grpo_math_1B_dynamo_single_controller.yaml) +shows them all: + +| Key | Default | Meaning | +|---|---|---| +| `startup_timeout_s` | 600 | Bounds service start-up and engine registration | +| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timed-out rollout is not retried; NeMo-Gym's calls keep Gym's timeouts | +| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | +| `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | +| `frontend_args.router_mode` | `kv` | Frontend routing policy | +| `frontend_args.session_affinity_ttl_secs` | 1800 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it | +| `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | +| `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | + +The etcd and nats-server binaries are found in `NEMO_RL_DYNAMO_BIN_DIR` when it +is set (the image sets it to `/opt/dynamo/bin`), and on `PATH` otherwise. + +## Which requests go through the frontend + +| Request | Path | +|---|---| +| `generate_async` on text token prompts, router replay (R3) included | Frontend (KV-aware); the engine samples and assembles it exactly as native `generate_async` does | +| NeMo RL's own multimodal rollouts | Native shard dispatch. Those rollouts send media to the engine directly rather than as OpenAI content | +| `generate_text_async`, which evaluation uses | Native shard dispatch | +| NeMo-Gym's OpenAI requests | Frontend, in Gym's Dynamo dialect; the engine serves them as the native chat endpoint does. See [NeMo-Gym](#nemo-gym) | + +A rollout through the frontend has the same `GenerationOutputSpec` as the +native path: token IDs, log probabilities, truncation, routed experts with +their `r3_routed_experts_*` counts, and `gen_leader_worker_idx`. Both paths +build it with `nemo_rl/models/generation/vllm/request_pipeline.py`. + +OpenAI clients of the frontend get what vLLM's own OpenAI server applies: +- the model's generation-config sampling defaults; +- `n > 1`; +- guided decoding (`guided_*`, `response_format`); +- `bad_words_token_ids` and `allowed_token_ids`; +- prompt log probabilities (`nvext.extra_fields: ["prompt_logprobs"]`); +- vLLM's queue limits, rejected with HTTP 503; +- a 400 naming the "maximum context length" for a prompt that leaves no room + for output. The engine, not the frontend, checks the context, so it checks + the prompt it actually runs. + +## NeMo-Gym + +NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect +(`vllm_model.dynamo_frontend`, from +[NVIDIA-NeMo/Gym#4817](https://github.com/NVIDIA-NeMo/Gym/pull/4817)). Gym marks +every call with an annotation, and the engine serves a marked call as the native +chat endpoint does: + +- Sampling must match `temperature`/`top_p` or `val_temperature`/`val_top_p`, + with `top_k` unset. vLLM samples with the configured values. +- The token prefix Gym echoes replaces the history the frontend rendered. +- Output is clamped to the context, and a prompt that leaves no room is refused + with the native endpoint's "maximum context length" error. +- Stop strings stop vLLM itself, so the token IDs match the native endpoint's. +- The response carries `prompt_token_ids`, `completion_token_ids` and + `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. +- Images, video and audio in the messages are processed as on the native + endpoint. The frontend passes each media URL through with one unexpanded + placeholder per item. The engine loads the media with vLLM's media connector + and runs vLLM's multimodal processor on the frontend's tokens. The splice, + placeholder remap and media capture then work on that processed prompt. + +With `token_capture.enabled`, Gym uses its `dynamo_worker` capture backend. The +same vLLM workers stage each call's tokens, log probabilities, routed experts and +processed media from their own process, exactly as the native endpoint does, and +only the commit coordinates travel back through the frontend. The frontend keeps a rollout on +one engine (`session_affinity_ttl_secs`), as the native backend pins each Gym +session to one server. + +The native server's chat settings carry over: + +| Native setting (`vllm_cfg`) | Under Dynamo | +|---|---| +| `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | +| `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | +| `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | +| `http_server_serving_chat_kwargs.default_chat_template_kwargs` | Sent by Gym under its own `chat_template_kwargs` | +| `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | + +Where the behavior differs from the native endpoint: + +- The engine sees the rendered prompt, not the messages. It finds where the + echoed prefix ends by counting the prefix's EOS tokens, not by rendering the + history. +- When a request sets `chat_template_kwargs.force_nonempty_content`, Dynamo + reports a reasoning-only turn as content. + +## Not supported + +Startup rejects these with `dynamo: true`: + +- `colocated.enabled: true`: a colocated engine sleeps while the policy trains, + and the frontend would keep routing to it. The SingleController never runs + colocated vLLM; the other entrypoints need non-colocated generation. +- `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, + and a restarted shard does not register with it yet. +- `async_rl.generation_router`: the frontend already routes the fleet behind one + URL. +- `vllm_cfg.http_server_api_key_required`: the frontend checks no bearer token. +- On the SingleController, router replay with Gym but without token capture: + Gym's dialect does not return routed experts in responses. Other entrypoints + do not check this, and such rollouts replay no routes. +- Parser plugins with no Dynamo parser, and the chat settings above that the + frontend has no counterpart for. +- `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without + `dynamo: true`. + +A Gym call with `n > 1` is refused when it arrives. ## Build the image -The normal image is unchanged unless `BUILD_DYNAMO` is set: +ai-dynamo is part of the `vllm` extra, so every vLLM worker venv carries it. +`BUILD_DYNAMO=1` adds only the etcd and nats-server binaries, under +`/opt/dynamo/bin`: ```bash docker buildx build \ @@ -26,210 +171,36 @@ docker buildx build \ . ``` -The opt-in layer installs `ai-dynamo[vllm]==1.5.1` in isolated Python -3.12 under `/opt/dynamo_venv`, along with etcd v3.5.21 and NATS Server v2.11.6. -It does not replace NeMo RL's normal Ray or vLLM dependencies: the standard -NeMo RL vLLM environment currently uses vLLM 0.29.0, while this isolated -Dynamo environment uses Dynamo's vLLM 0.28.0 pin. Both environments pin -`nvidia-nccl-cu13==2.30.7` so their NCCL communicators use the same release. -For a local source checkout, the same environment can be installed under -`venvs/dynamo`: +For a local checkout, install the binaries with: ```bash -bash docker/dynamo/install.sh +NEMO_RL_DYNAMO_BIN_DIR=$PWD/venvs/dynamo-bin bash docker/dynamo/install.sh ``` -Set `NEMO_RL_DYNAMO_VENV_DIR` to choose another location. The installer checks -that Dynamo resolved vLLM 0.28.0 and NCCL 2.30.7, and that the installed vLLM -contains the PR #44814 layerwise-reload fix. - -A Dynamo upgrade must update and reverify these coupled locations: - -- `docker/dynamo/pyproject.toml` and `docker/dynamo/uv.lock`: the Dynamo pin - and resolved dependency set -- root `pyproject.toml` and `uv.lock`, plus the isolated Dynamo project and - lockfile: the `nvidia-nccl-cu13` pins must remain identical -- `docker/dynamo/install.sh`, `tests/functional/grpo_dynamo.sh`, and - `tests/functional/grpo_dynamo_single_controller.sh`: the vLLM version, the - #44814 fix check, and runtime assertions -- `nemo_rl/models/generation/dynamo/refit.py` and - `validate_dynamo_vllm_args.py`: the vLLM weight-transfer payloads (packing is - declared in the init info; update rounds carry names, dtypes, and shapes; - `start_weight_update` takes no arguments) -- `tests/unit/distributed/test_stateless_process_group.py`: vLLM's - `broadcast_from/0/0` weight-transfer wire key -- `nemo_rl/models/generation/dynamo/token_wrapper.py`: the real Dynamo response - keys `nvext.engine_data.{prompt_token_ids,completion_token_ids,completion_logprobs}`; - reverify them against real Dynamo output because unit tests validate only the - expected local response shape -- `nemo_rl/models/generation/dynamo/managed_runtime.py`: the managed - `DYN_ENABLE_EXPERIMENTAL_PARSERS_V2=1` setting. Dynamo 1.3.0's legacy tool - jail removes `nvext.engine_data`; remove this setting only after an upgraded - Dynamo preserves the token metadata for `tool_choice=auto` -- this guide and the Dynamo design document: the stated versions - -## Configure Dynamo - -Start with [`examples/configs/grpo_math_1B_dynamo.yaml`](../../examples/configs/grpo_math_1B_dynamo.yaml). -The important boundary is: +## Run -```yaml -policy: - generation: - backend: dynamo - dynamo_cfg: - engine: vllm - frontend_args: - router_mode: kv - vllm_cfg: - tensor_parallel_size: 1 - pipeline_parallel_size: 1 - expert_parallel_size: 1 - colocated: - enabled: false - resources: - gpus_per_node: 1 - num_nodes: 1 -``` - -NeMo RL derives each engine's world size from TP times PP. EP must be one or -equal to TP. Parser settings belong under `dynamo_cfg.worker_args`; inherited -vLLM HTTP-parser settings are rejected with the corresponding Dynamo field. -Service ports and the namespace are runtime-owned rather than public config. - -`vllm_cfg` settings are handled in five explicit classes: - -| Class | Behavior | Examples | -| --- | --- | --- | -| Translated | Forwarded to `dynamo.vllm` | TP, PP, EP, dtype, model length | -| Moved | Startup error naming the Dynamo replacement | tool and reasoning parsers, HTTP serving chat kwargs | -| Unsupported | Warning when active, or an error when it requests unsupported low precision | tokenizer skipping, MX and mixed BF16/FP8 helpers | -| Managed runtime | Consumed or enforced by NeMo RL rather than forwarded | HTTP-wrapper enablement, metrics sampling, processed rollout logprobs | -| Inapplicable | Ignored because the managed path owns that behavior | async mode, progress bars, NeMo RL HTTP/ZMQ refit ports | - -The shared GRPO base config also supplies `mcore_generation_config` and -`refit_cfg`. Dynamo accepts these inherited sections but does not use them; -worker arguments come from `vllm_cfg`, and refit uses the collective weight -synchronizer. Any other active top-level `policy.generation` setting that -Dynamo does not honor, such as `bad_words` or `ignore_eos`, draws a startup -warning. - -Enable and filter worker telemetry with both managed configuration sections: - -```yaml -policy: - generation: - vllm_cfg: - enable_vllm_metrics_logger: true - vllm_metrics_logger_interval: 1.0 - dynamo_cfg: - metrics_include_prefixes: null # null selects the curated defaults - metrics_exclude_prefixes: null # null excludes python_ and process_ -``` - -The NCCL sender also selects vLLM's peer protocol: the policy publishes both -the raw NeMo RL unique ID and vLLM's pickled `ncclUniqueId`, then uses the -all-reduce warmup expected by `PyNcclCommunicator`. This protocol choice and -the packed 1-GiB/two-buffer geometry come from the generation backend rather -than GRPO-specific branches. - -The fixed port layout is: - -- `1313-1399`: driver-local etcd and NATS control plane -- `3000-3999`: frontend and token-wrapper HTTP endpoints -- `4000-4099`: node-local `DYN_SYSTEM_PORT` -- `7000 + slot * 100`: node-local vLLM rendezvous ports - -## Run the two-GPU smoke - -Convert the image to the format required by the Slurm site, then submit from -the repository root: +Use any non-colocated vLLM recipe and add the switch: ```bash -export CONTAINER=/shared/images/nemo-rl-dynamo.sqsh -export MOUNTS="$PWD:$PWD" -export GPUS_PER_NODE=2 -export BASE_LOG_DIR="$PWD/results/dynamo-smoke/logs" -printf -v COMMAND '%q ' \ - /opt/nemo_rl_venv/bin/python -u "$PWD/examples/run_grpo.py" \ - --config "$PWD/examples/configs/grpo_math_1B_dynamo.yaml" -export COMMAND - -sbatch \ - --nodes=1 \ - --gres=gpu:2 \ - --exclusive \ - --account= \ - --partition= \ - ray.sub +uv run examples/run_grpo_single_controller.py \ + --config examples/configs/grpo_math_1B_megatron_single_controller.yaml \ + ++policy.generation.dynamo=true ``` -The recipe assigns one GPU to training and one to a TP1 Dynamo worker. Its two -steps exercise generation, refit, post-refit cache invalidation, telemetry, -and cleanup. For a matched control, run the same seed/model/batch settings with -the standard non-colocated vLLM backend and compare post-refit output validity. - -## Run under the SingleController - -The async SingleController trainer can also use managed Dynamo. It follows the -same data path as the native vLLM backend: the SingleController sends each turn -to the Dynamo frontend, builds the rollout record from the response, and commits -it to the TransferQueue data plane, from which the policy workers read their -shards. No Dynamo-side component writes to TransferQueue. - -Start from [`examples/configs/grpo_math_1B_dynamo_single_controller.yaml`](../../examples/configs/grpo_math_1B_dynamo_single_controller.yaml) -and launch it with `examples/run_grpo_single_controller.py`. Before every weight -update the SingleController drains and pauses each Dynamo worker (requests the -engine is running finish on the old weights; queued and new requests wait and -run on the new weights), runs the NCCL refit, and then resumes the workers. With -`async_rl.recompute_kv_cache_after_weight_updates` (the default), the drain also -clears each worker's KV and prefix cache. The drain is bounded by -`dynamo_cfg.control_timeout_s`. - -Current limits on this path: - -- Native rollouts only. NeMo-Gym under the SingleController supports only - `backend: vllm` and `backend: megatron`. -- Text policies only, and no MoE router replay: `policy.is_vlm` and - `policy.router_replay.enabled` must be false. -- Fleet health is not supported: keep `async_rl.generation_fleet_health.enabled` - false (the Dynamo frontend owns request routing) and set - `async_rl.generation_fleet_health.refit_timeout_s: null` (default 300), since - Dynamo workers cannot abort their refit collective. -- `dynamo_cfg.control_timeout_s` must be at least `dynamo_cfg.request_timeout_s`, - because it bounds the drain. -- `vllm_cfg.enable_vllm_metrics_logger` requires - `rollout_checkpointing.telemetry_interval_s`: the rollout telemetry pump drains - the Dynamo metrics sampler and reports each engine's generated tokens, running - and waiting requests, and KV-cache usage. - -Setup rejects any of these before allocating clusters. - -## Run SWE1 with W&B - -The three-node nightly recipe targets -`nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16`. It uses two 8-GPU training -nodes and one 8-GPU inference node. The inference node runs two TP4/EP4 Dynamo -engines. Download the standard SWE1 split under -`${HF_HOME}/superv3_data/swe1`, then run the registered test-suite driver: - -```bash -HF_HOME=/shared/huggingface \ -WANDB_API_KEY= \ -bash tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh -``` - -A successful acceptance run completes four training steps, produces valid -generations after refit, and records worker timelines under -`generation_metrics/*` in TensorBoard and W&B. - -## Operational notes - -- The driver owns all services. Do not start a separate etcd, NATS, frontend, - or worker fleet for this mode. -- Startup validates fixed worker membership; a dead or replaced worker fails - refit instead of serving mixed model versions. -- Shutdown is idempotent and terminates whole subprocess groups, including - partial-startup failures. -- Fault tolerance and a multi-controller architecture remain follow-up work. +is the same run with every `dynamo_cfg` default spelled out. + +## Migrating from `backend: dynamo` + +`backend: dynamo` launched Dynamo's own vLLM engine and was removed. Startup +rejects it with the migration: + +- Set `backend: vllm` and `dynamo: true`. `vllm_cfg` and `vllm_kwargs` keep their + meaning, and every vLLM feature is now available. +- Remove these from `dynamo_cfg`: + - `engine`; + - `control_timeout_s`: refit and pause use the vLLM backend's controls; + - `metrics_include_prefixes` / `metrics_exclude_prefixes`: use + `vllm_cfg.enable_vllm_metrics_logger`; + - `worker_args.extra_cli_args`: engines are configured through `vllm_cfg` and + `vllm_kwargs`. diff --git a/docs/guides/single-controller.md b/docs/guides/single-controller.md index db23125c7f4..e9f1cd4f72b 100644 --- a/docs/guides/single-controller.md +++ b/docs/guides/single-controller.md @@ -542,12 +542,12 @@ The SC path is still under active development. Feature gaps are tracked in [issu Gym rollouts; multimodal/VLM MOPD is not yet supported. See [Multi-Teacher On-Policy Distillation](../about/algorithms/mopd.md#running-mopd). - Train backend: only Megatron is supported and validated; the AutoModel training path has not been tested on SC. -- Generation backend: vLLM and Megatron generation are supported (Megatron in both non-colocated and colocated modes). Managed Dynamo (`backend: dynamo`) is supported on the native rollout path; see [Managed Dynamo generation](dynamo-generation.md#run-under-the-singlecontroller). SGLang and TRT-LLM have not been tested on SC. +- Generation backend: vLLM and Megatron generation are supported (Megatron in both non-colocated and colocated modes). vLLM generation can also serve through Dynamo (`policy.generation.dynamo: true`); see [Dynamo serving](dynamo-generation.md). SGLang and TRT-LLM have not been tested on SC. - Validation is not yet supported (setup raises on `val_period > 0`, `val_at_start`, or `val_at_end`); checkpointing is. - (PPO) Rollout drop budgets — `async_rl.rollout_failure.max_skipped_prompts` and `max_consecutive_dropped_prompts` must both be `0`. A drop shortens the step, and the critic shards it against the configured `value.train_global_batch_size` rather than its actual size, so setup rejects a non-zero budget. The resiliency layer stays available on GRPO. - Reward shaping and sample filtering — `reward_shaping`, `reward_scaling`, and `use_dynamic_sampling` are implemented on neither algorithm block, so setup rejects them rather than silently skipping the shaping. Environment-flagged sample masking and `overlong_filtering` are supported; truncated completions are excluded from the loss through `sample_mask`, and a step in which every completion is filtered is rejected rather than skipped. - The `windowed` sampler has no `over_sampling_ratio` cap — over-produced groups aged past the window are evicted, wasting rollout compute. -- The drain gate in refit is not yet supported. Managed Dynamo pauses its engines before every refit because its weight update is unsafe while requests run, but that is not the drain gate: running requests finish on the old weights, while queued requests and later turns of a multi-turn rollout run on the new ones. +- The drain gate in refit is not yet supported. ### Full training-data dumps diff --git a/docs/index.md b/docs/index.md index e88dfe3c699..9cfda03a275 100644 --- a/docs/index.md +++ b/docs/index.md @@ -79,11 +79,11 @@ Measure model and agent capabilities with repeatable benchmarks and custom evalu Configure and launch NeMo RL on multi-node Slurm or Kubernetes clusters for distributed computing. ::: -:::{grid-item-card} {octicon}`workflow` Managed Dynamo Generation +:::{grid-item-card} {octicon}`workflow` Dynamo Serving :link: guides/dynamo-generation :link-type: doc -Run a fixed Dynamo vLLM fleet with NCCL refit and W&B telemetry inside a Slurm Ray allocation. +Serve any vLLM generation config through Dynamo's frontend and KV-aware router with `dynamo: true`. ::: :::: diff --git a/docs/observability/configuration.md b/docs/observability/configuration.md index 9fab877b536..36e42c3b9e1 100644 --- a/docs/observability/configuration.md +++ b/docs/observability/configuration.md @@ -100,7 +100,7 @@ Each process sets stable-for-the-run values on its OTel `Resource` — `init_tel | `nv.dl.training.config.optimizer` | on Megatron `policy.megatron_cfg.optimizer.optimizer` (`adam`), otherwise `policy.optimizer.name` (`torch.optim.AdamW`) — the backends both spell it and store it differently, and only the enabled one is read | | `nv.dl.training.config.recompute_granularity` | Megatron runs only: `policy.megatron_cfg.recompute_granularity`, and only when `activation_checkpointing` is on. Megatron ignores the granularity otherwise, so reporting it would describe recompute that is not happening | | `nv.dl.training.target.train_iters` | `.max_num_steps`, when positive. This is the configured cap, not a prediction: many configs set a placeholder (`grpo_math_1B` uses `1000000` with `max_num_epochs: 1`) and stop earlier on epochs. Looked up by algorithm name, so an entry point that keeps its settings under a differently named section reports no target at all — `sft_v2` reads `master_config.sft`, and omits this. `rm` spells "train for one epoch" as `-1`, which is omitted rather than reported as a target | -| `nv.dl.software.torch`, `.cuda`, `.nccl` | versions as loaded in the process that opened the span, not as pinned in the lockfile — they are per process, so filter on `rl.worker_group` to read one backend's versions. CUDA and NCCL are omitted on a CPU-only build. SGLang, TRT-LLM and Dynamo workers do not report versions yet | +| `nv.dl.software.torch`, `.cuda`, `.nccl` | versions as loaded in the process that opened the span, not as pinned in the lockfile — they are per process, so filter on `rl.worker_group` to read one backend's versions. CUDA and NCCL are omitted on a CPU-only build. SGLang and TRT-LLM workers do not report versions yet | | `nv.dl.software.transformer_engine` | the installed distribution's version, read from package metadata rather than by importing it — importing transformer-engine loads its PyTorch extension in every process that sets up telemetry. Omitted when it is not installed, which is every process except Megatron and Automodel workers: its absence on a driver, controller or vLLM span does not mean the run lacked TE | | `nv.dl.rank`, `nv.dl.world_size` | this process's rank and group size (`RANK` / `WORLD_SIZE`, or `0` / `1` for the driver and singleton actors) | | `nv.dl.local_rank` | worker processes only: `LOCAL_RANK`, which `RayWorkerGroup` exports beside `RANK` as the worker's bundle index in its placement group. Node-local only when each node has its own placement group — a generation backend spanning nodes requests a single unified one (SGLang always; vLLM, TRT-LLM and Megatron when their parallelism crosses nodes) and the index then runs across nodes. Not a physical device index either, since Ray gives each worker its own `CUDA_VISIBLE_DEVICES` | diff --git a/docs/observability/metrics.md b/docs/observability/metrics.md index ce58a5f7b31..3824e9bf1d3 100644 --- a/docs/observability/metrics.md +++ b/docs/observability/metrics.md @@ -187,7 +187,7 @@ The counters carry the step's delta, so summing them across steps reconstructs t **What counts as failed.** vLLM's finish reasons are `stop`, `length`, `abort`, `error` and `repetition`. Only `stop` and `length` leave a usable sample behind, and `length` is a normal RL outcome — a rollout routinely runs to `max_tokens` — so those two are `ok` and *everything else* is `failed`. Counting the remainder rather than an explicit deny-list keeps `ok + failed` equal to the engine's own total even if a future vLLM adds a reason we have not heard of. -These are the family names vLLM registers. The reader returns family names, which `prometheus_client` strips of `_total`, so no suffixed alias is needed on this path — unlike the HTTP-scrape path in `nemo_rl/models/generation/dynamo/metrics.py`, where sample names do carry it. `test_engine_metric_names_match_vllm` checks the copies against vLLM itself. A series that matches nothing is **omitted**, leaving a visible gap in the dashboard rather than a plausible-looking zero, and so is one whose total went backwards, which means an engine restarted mid-step. +These are the family names vLLM registers. The reader returns family names, which `prometheus_client` strips of `_total`, so no suffixed alias is needed on this path. `test_engine_metric_names_match_vllm` checks the copies against vLLM itself. A series that matches nothing is **omitted**, leaving a visible gap in the dashboard rather than a plausible-looking zero, and so is one whose total went backwards, which means an engine restarted mid-step. ## Driver-side generation metrics (`gen_ai.*`) diff --git a/examples/configs/grpo_math_1B.yaml b/examples/configs/grpo_math_1B.yaml index c55645114c8..d354b952783 100644 --- a/examples/configs/grpo_math_1B.yaml +++ b/examples/configs/grpo_math_1B.yaml @@ -504,7 +504,7 @@ policy: # Non-colocated vLLM also supports vllm_s3_sparse, vllm_zmq_sparse, and nixl. # Non-colocated vLLM or Megatron generation supports nccl_reshard (M-to-N). refit_transport: null - # vLLM backend only - sglang/megatron/trtllm/dynamo require this to remain null. + # vLLM backend only - sglang/megatron/trtllm require this to remain null. # Selects the NeMo-RL Ray actor class, NOT vLLM's own `worker_extension_cls`. worker_extension_cls_fqn: null # optional registered vLLM generation-worker subclass; mutually exclusive with quant_cfg # Keep real-quant export tensors on CPU by default. Set false only for diff --git a/examples/configs/grpo_math_1B_dynamo.yaml b/examples/configs/grpo_math_1B_dynamo.yaml deleted file mode 100644 index fcf30538306..00000000000 --- a/examples/configs/grpo_math_1B_dynamo.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Two-step, two-GPU smoke for the Ray-managed Dynamo vLLM backend. -defaults: grpo_math_1B.yaml - -grpo: - num_prompts_per_step: 2 - num_generations_per_prompt: 2 - max_num_steps: 2 - val_period: 0 - val_at_start: false - val_at_end: false - async_grpo: - enabled: true - in_flight_weight_updates: false - recompute_kv_cache_after_weight_updates: true - -loss_fn: - use_importance_sampling_correction: true - truncated_importance_sampling_type: tis - truncated_importance_sampling_ratio: 2.0 - -checkpointing: - enabled: false - -policy: - train_global_batch_size: 4 - train_micro_batch_size: 1 - logprob_batch_size: 1 - max_total_sequence_length: 512 - sequence_packing: - enabled: false - automodel_cfg: - enabled: true - cpu_offload: false - activation_checkpointing: false - megatron_cfg: - enabled: false - generation: - backend: dynamo - max_new_tokens: 128 - temperature: 0.7 - top_p: 0.95 - top_k: 50 - dynamo_cfg: - engine: vllm - startup_timeout_s: 600 - request_timeout_s: 900 - control_timeout_s: 600 - metrics_include_prefixes: null - metrics_exclude_prefixes: null - worker_args: - tool_call_parser: null - reasoning_parser: null - exclude_tools_when_tool_choice_none: true - enable_structural_tag: false - structural_tag_scope: auto - structural_tag_schema: auto - custom_jinja_template: null - endpoint_types: [chat, completions] - extra_cli_args: [] - frontend_args: - tokenizer: default - tokenizer_cache: false - tokenizer_cache_bytes: 52428800 - router_mode: kv - extra_cli_args: [] - vllm_cfg: - async_engine: true - tensor_parallel_size: 1 - pipeline_parallel_size: 1 - expert_parallel_size: 1 - precision: bfloat16 - gpu_memory_utilization: 0.6 - max_model_len: ${policy.max_total_sequence_length} - kv_cache_dtype: auto - enforce_eager: true - expose_http_server: false - enable_vllm_metrics_logger: true - vllm_metrics_logger_interval: 1.0 - env_vars: - NCCL_MNNVL_ENABLE: "0" - vllm_kwargs: - max_num_seqs: 16 - colocated: - enabled: false - resources: - gpus_per_node: 1 - num_nodes: 1 - -logger: - log_dir: results/grpo-math-1b-dynamo - wandb_enabled: false - tensorboard_enabled: true - monitor_gpus: false - -cluster: - gpus_per_node: 2 - num_nodes: 1 diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index da974c294c4..291aac186b6 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -1,73 +1,45 @@ -# SingleController variant with the Ray-managed Dynamo vLLM backend. +# grpo_math_1B_megatron_single_controller.yaml served through Dynamo. +# `policy.generation.dynamo: true` is the whole switch: the same vLLM workers +# register with a managed dynamo.frontend, and rollouts go through its KV-aware +# router. Every other vLLM setting keeps its meaning; Dynamo implies +# vllm_cfg.async_engine and needs non-colocated generation. # -# Rollouts follow the native-backend path: the SingleController sends each turn -# to the Dynamo frontend over HTTP, builds the rollout record from the response, -# and commits it to the TransferQueue data plane. Before every weight update the -# SingleController drains and pauses the Dynamo workers (running requests -# finish on the old weights; queued and new requests wait and run on the new -# weights), then resumes them afterwards. +# dynamo_cfg is optional. The values below are its defaults +# (nemo_rl/models/generation/dynamo/config.py). defaults: grpo_math_1B_megatron_single_controller.yaml -async_rl: - generation_fleet_health: - # Dynamo workers cannot abort their refit collective, so no refit deadline. - refit_timeout_s: null - policy: - # Training uses the Megatron backend inherited from - # grpo_math_1B_megatron_single_controller.yaml; the SingleController's split - # training API is implemented by the Megatron policy worker. generation: - backend: dynamo + dynamo: true dynamo_cfg: - engine: vllm + # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: 600 + # Timeout for one rollout routed through the frontend (in place of + # NRL_VLLM_ASYNC_TIMEOUT_SECONDS); a timed-out rollout is not retried. request_timeout_s: 900 - # Bounds the pre-refit drain, which waits for running requests to finish, - # so it must be at least request_timeout_s. - control_timeout_s: 900 - metrics_include_prefixes: null - metrics_exclude_prefixes: null worker_args: + # Dynamo's built-in parser names. null follows the native server's + # vllm_cfg.http_server_serving_chat_kwargs (tool_parser, reasoning_parser). tool_call_parser: null reasoning_parser: null - exclude_tools_when_tool_choice_none: true + exclude_tools_when_tool_choice_none: false enable_structural_tag: false structural_tag_scope: auto structural_tag_schema: auto + # A chat template that replaces the model's own. null follows + # vllm_cfg.http_server_serving_chat_kwargs.chat_template. custom_jinja_template: null endpoint_types: [chat, completions] - extra_cli_args: [] frontend_args: tokenizer: default tokenizer_cache: false tokenizer_cache_bytes: 52428800 + # kv: route each request to the engine holding most of its prefix. router_mode: kv + # Keep a NeMo-Gym token-capture rollout on one engine while it idles up + # to this many seconds (null disables it). + session_affinity_ttl_secs: 1800 extra_cli_args: [] - vllm_cfg: - async_engine: true - tensor_parallel_size: 1 - pipeline_parallel_size: 1 - expert_parallel_size: 1 - precision: bfloat16 - gpu_memory_utilization: 0.6 - max_model_len: ${policy.max_total_sequence_length} - kv_cache_dtype: auto - enforce_eager: true - expose_http_server: false - # Under the SingleController the rollout telemetry pump reports these - # metrics, so enabling this requires rollout_checkpointing.telemetry_interval_s. - enable_vllm_metrics_logger: false - vllm_metrics_logger_interval: 1.0 - env_vars: - NCCL_MNNVL_ENABLE: "0" - vllm_kwargs: - max_num_seqs: 16 - colocated: - enabled: false - resources: - gpus_per_node: 1 - num_nodes: 1 logger: log_dir: results/grpo-math-1b-dynamo-single-controller diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml index 75869d90917..010fee9cb49 100644 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml @@ -82,26 +82,11 @@ policy: sequence_packing: fuse_loss: true generation: - backend: dynamo + backend: vllm + dynamo: true dynamo_cfg: - engine: vllm startup_timeout_s: 1800 request_timeout_s: 7200 - control_timeout_s: 600 - metrics_include_prefixes: null - metrics_exclude_prefixes: null - worker_args: - tool_call_parser: qwen3_coder - reasoning_parser: nemotron_nano - exclude_tools_when_tool_choice_none: true - enable_structural_tag: false - structural_tag_scope: auto - structural_tag_schema: auto - custom_jinja_template: null - endpoint_types: - - chat - - completions - extra_cli_args: [] frontend_args: tokenizer: fastokens tokenizer_cache: true @@ -115,8 +100,6 @@ policy: enable_vllm_metrics_logger: true vllm_metrics_logger_interval: 0.5 env_vars: null - reasoning_parser_plugin: null - http_server_serving_chat_kwargs: null vllm_kwargs: attention_backend: FLASH_ATTN moe_backend: triton diff --git a/nemo_rl/algorithms/async_utils/trajectory_collector.py b/nemo_rl/algorithms/async_utils/trajectory_collector.py index fc8fd1c8eee..8986e8dda90 100644 --- a/nemo_rl/algorithms/async_utils/trajectory_collector.py +++ b/nemo_rl/algorithms/async_utils/trajectory_collector.py @@ -1048,10 +1048,10 @@ def resume(self) -> None: def prepare_for_refit(self) -> None: """Pause new generation starts and optionally wait for pending generations. - Every async backend configured for in-flight weight updates, except managed - Dynamo, is asked to pause generation. vLLM preserves in-flight request state - with its native keep-mode pause. Backends without pause support warn and - retain their existing behavior. Managed Dynamo drains active trajectories. + Every async backend configured for in-flight weight updates is asked to + pause generation. vLLM preserves in-flight request state with its native + keep-mode pause. Backends without pause support warn and retain their + existing behavior. For non-async engines, waits for all pending generations to complete before refit. """ @@ -1067,16 +1067,7 @@ def prepare_for_refit(self) -> None: # Check if we're using async engine generation_cfg = self.master_config.policy["generation"] backend = generation_cfg["backend"] - if backend == "dynamo": - # Dynamo's native layerwise reload temporarily materializes model - # parameters while the NCCL update is in progress. It is not safe - # to execute an already-issued vLLM request concurrently with that - # reload (in particular for NemotronH/Mamba parameters), even when - # the update route accepts allow_unpaused=True. Stop new trajectory - # starts above and drain every active trajectory before refitting. - is_async_engine = False - else: - is_async_engine = should_use_async_rollouts(generation_cfg) + is_async_engine = should_use_async_rollouts(generation_cfg) in_flight_weight_updates = self.async_config.in_flight_weight_updates if is_async_engine and in_flight_weight_updates: @@ -1144,10 +1135,6 @@ def resume_after_refit(self) -> None: ) except Exception as e: print(f"⚠️ Failed to invalidate generation backend KV caches: {e}") - if self.master_config.policy["generation"]["backend"] == "dynamo": - raise RuntimeError( - "Managed Dynamo KV cache invalidation failed after refit" - ) from e finally: self._generation_pause_requested_for_refit = False self._generation_paused_for_refit = False diff --git a/nemo_rl/algorithms/distillation.py b/nemo_rl/algorithms/distillation.py index b861d73522c..74a2691af88 100644 --- a/nemo_rl/algorithms/distillation.py +++ b/nemo_rl/algorithms/distillation.py @@ -549,7 +549,7 @@ def init_vllm_deferred(): def init_nemo_gym(): shard_set = build_nemo_gym_actors( env_configs, - base_urls=cast(list[str], deferred_vllm.dp_openai_server_base_urls), + base_urls=cast(list[str], deferred_vllm.client_base_urls), model_name=generation_config["model_name"], tokenizer=tokenizer, # Distillation does not configure vLLM for router replay. diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 5a96be314a3..9d808421d49 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -126,7 +126,6 @@ run_nemo_gym_rollout_sync, should_mask_flagged_samples, ) -from nemo_rl.models.generation.dynamo import DynamoConfig, DynamoGeneration from nemo_rl.models.generation.interfaces import ( GenerationConfig, GenerationInterface, @@ -628,16 +627,11 @@ def _needs_hf_refit_handshake( def _uses_managed_noncolocated_refit( - generation_backend: str, nccl_reshard_refit_enabled: bool, offload_policy_before_refit: bool, ) -> bool: """Whether non-colocated setup must attach a weight synchronizer.""" - return ( - nccl_reshard_refit_enabled - or generation_backend == "dynamo" - or offload_policy_before_refit - ) + return nccl_reshard_refit_enabled or offload_policy_before_refit def _shutdown_completed_nemo_gym_startup( @@ -707,20 +701,6 @@ def setup( if generation_config["backend"] == "vllm": normalize_nvfp4_pertoken_policy_config(policy_config, entry_point="grpo") normalize_vllm_refit_config(cast(VllmConfig, generation_config)) - elif generation_config["backend"] == "dynamo": - # Validate the complete managed-Dynamo boundary before allocating Ray - # placement groups or starting any external services. - if grpo_config.async_grpo.in_flight_weight_updates: - raise ValueError( - "grpo.async_grpo.in_flight_weight_updates must be false when " - "policy.generation.backend='dynamo'; managed Dynamo drains " - "rollouts before layerwise weight refit" - ) - generation_config.setdefault("vllm_kwargs", {})["hf_overrides"] = ( - policy_config.get("hf_config_overrides") or {} - ) - generation_config = DynamoConfig.model_validate(generation_config).model_dump() - policy_config["generation"] = generation_config # generation.disaggregation is written in backend-agnostic terms, but only # the TRT-LLM backend implements it today: every other backend would @@ -1267,10 +1247,6 @@ def _spinup_nemo_gym(base_urls, model_name, policy_api_key=None): gpus_per_instance = replica_width( cast(TrtllmConfig, generation_config) ) - elif generation_config["backend"] == "dynamo": - gpus_per_instance = DynamoConfig.model_validate( - generation_config - ).engine_world_size else: sglang_cfg = generation_config.get("sglang_cfg", {}) gpus_per_instance = sglang_cfg.get("gpus_per_server", 1) @@ -1344,10 +1320,6 @@ def _spinup_nemo_gym(base_urls, model_name, policy_api_key=None): MegatronGeneration.init_cluster_placement_groups( inference_cluster, policy_config ) - elif generation_config["backend"] == "dynamo": - # Managed Dynamo creates one single-node engine per placement - # group and does not need a backend-specific PG strategy. - inference_cluster.get_placement_groups() else: { "vllm": VllmGeneration, @@ -1776,8 +1748,8 @@ def init_megatron_generation_task(policy_future): ) vllm_reserve_time = time.perf_counter() - vllm_reserve_t0 print( - f" ✓ Reserved {len(deferred_vllm.dp_openai_server_base_urls)} vLLM server URLs: " - f"{deferred_vllm.dp_openai_server_base_urls}", + f" ✓ Reserved {len(deferred_vllm.client_base_urls)} vLLM server URLs: " + f"{deferred_vllm.client_base_urls}", flush=True, ) @@ -1791,7 +1763,7 @@ def init_vllm_deferred(): def init_nemo_gym(): """Spin up NeMo Gym servers with the pre-assigned vLLM URLs.""" return _spinup_nemo_gym( - deferred_vllm.dp_openai_server_base_urls, + deferred_vllm.client_base_urls, generation_config["model_name"], policy_api_key=deferred_vllm.http_server_api_key, ) @@ -1901,37 +1873,6 @@ def init_trtllm(): ) setup_timing_metrics.nemo_gym_init_time_s = nemo_gym_time - elif backend == "dynamo": - # Managed Dynamo owns a fixed worker fleet on the inference virtual cluster. - - def init_dynamo(): - t0 = time.perf_counter() - generation = DynamoGeneration( - cluster=inference_cluster, - config=generation_config, - tokenizer=tokenizer, - tokenizer_config=policy_config["tokenizer"], - ) - return generation, time.perf_counter() - t0 - - policy_generation, policy = initialize_generation_with_policy( - init_generation_fn=init_dynamo, - colocated_inference=False, - setup_timing_metrics=setup_timing_metrics, - ) - - if enable_nemo_gym: - nemo_gym_actor, nemo_gym_time = _spinup_nemo_gym( - policy_generation.dp_openai_server_base_urls, - generation_config["model_name"], - ) - setup_timing_metrics.nemo_gym_init_time_s = nemo_gym_time - - print( - f" ✓ Using Dynamo backend (frontend: {policy_generation.frontend_url})", - flush=True, - ) - # Record when worker initialization completes (for calculating other setup time) worker_init_complete_time = time.perf_counter() - setup_start_time @@ -1973,7 +1914,6 @@ def init_dynamo(): t0 = time.perf_counter() # init collective if _uses_managed_noncolocated_refit( - generation_backend=backend, nccl_reshard_refit_enabled=nccl_reshard_refit_enabled, offload_policy_before_refit=offload_policy_before_refit, ): @@ -4910,17 +4850,17 @@ def async_grpo_train( media to NeMo Gym prompt rows. """ # Ensure we are running with a compatible async generation backend. - # Async GRPO supports vLLM, Megatron, TRT-LLM, and Dynamo; + # Async GRPO supports vLLM, Megatron, and TRT-LLM; # SGLang async rollouts do not support the async GRPO replay path. generation_config = master_config.policy["generation"] backend = generation_config.get("backend", "") if generation_config else "" - assert backend in ("vllm", "megatron", "trtllm", "dynamo"), ( - "Async GRPO supports the vLLM, Megatron, TRT-LLM, and Dynamo generation backends; " + assert backend in ("vllm", "megatron", "trtllm"), ( + "Async GRPO supports the vLLM, Megatron, and TRT-LLM generation backends; " f"got policy.generation.backend={backend!r}." ) assert should_use_async_rollouts(generation_config), ( - "Async GRPO requires Dynamo, Megatron, or an async vLLM or TRT-LLM " - "generation engine. Set policy.generation.backend=dynamo, " + "Async GRPO requires Megatron, or an async vLLM or TRT-LLM " + "generation engine. Set " "policy.generation.vllm_cfg.async_engine=true (vLLM), or " "policy.generation.trtllm_cfg.async_engine=true (TRT-LLM). " "Megatron Inference always uses its async engine." @@ -5217,7 +5157,7 @@ def _flush_collector_telemetry() -> None: raise # Generation must hold the policy's real weights before any backend starts - # collecting. In particular, vLLM and Dynamo start with dummy weights when + # collecting. In particular, vLLM starts with dummy weights when # the first refit supplies model parameters. ray.get(trajectory_collector.set_weight_version.remote(weight_version)) ray.get(trajectory_collector.start_collection.remote(CyclingDataLoader(dataloader))) diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py index 6a61eb838e2..bd5f9fa7f3a 100644 --- a/nemo_rl/algorithms/single_controller_utils/config.py +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -664,7 +664,9 @@ class TokenCaptureConfig(BaseModel, extra="allow"): capture_dir: Optional[str] = None # Generation backend hosting token capture. This is derived from # policy.generation.backend during setup; users should not set it separately. - generation_backend: Optional[Literal["vllm", "megatron"]] = None + # "dynamo" is the vLLM backend served through Dynamo (policy.generation.dynamo): + # the same workers host capture, and NeMo-Gym reaches them through the frontend. + generation_backend: Optional[Literal["vllm", "megatron", "dynamo"]] = None # Keep routed_experts out of canonical rows and assemble them on policy # workers from strict staged-fragment plans. defer_routed_experts_to_policy: bool = False diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index e3ffbb1ba11..302783c2b18 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -120,7 +120,7 @@ should_mask_flagged_samples, ) from nemo_rl.models.generation import resolve_generation_class -from nemo_rl.models.generation.dynamo import DynamoConfig, DynamoGeneration +from nemo_rl.models.generation.dynamo.config import dynamo_enabled from nemo_rl.models.generation.fleet_health import ( FleetHealthPolicy, GenerationFleetHealth, @@ -130,7 +130,6 @@ GenerationRouterActor, GenerationRouterImpl, ) -from nemo_rl.models.generation.interfaces import GenerationConfig from nemo_rl.models.generation.megatron.megatron_generation import MegatronGeneration from nemo_rl.models.generation.sglang.config import SGLangConfig from nemo_rl.models.generation.sglang.sglang_generation import SGLangGeneration @@ -517,10 +516,6 @@ def _build_clusters( gpus_per_instance = vllm_cfg["tensor_parallel_size"] * vllm_cfg.get( "pipeline_parallel_size", 1 ) - elif backend == "dynamo": - gpus_per_instance = DynamoConfig.model_validate( - generation_config_dict - ).engine_world_size elif backend == "sglang": gpus_per_instance = generation_config_dict["sglang_cfg"].get( "gpus_per_server", 1 @@ -532,7 +527,7 @@ def _build_clusters( else: raise ValueError( "single_controller_utils.setup only supports vllm, sglang, " - f"megatron, or dynamo generation; got {backend!r}" + f"or megatron generation; got {backend!r}" ) nodes_per_instance = ( gpus_per_instance + inference_gpus_per_node - 1 @@ -595,7 +590,7 @@ def _build_generation( tokenizer: Optional[PreTrainedTokenizerBase] = None, processor: Optional[AutoProcessor] = None, ) -> tuple[Any, float]: - """Spin up the generation backend (vLLM, SGLang, Megatron, or managed Dynamo). + """Spin up the generation backend (vLLM, SGLang, or Megatron). Args: inference_cluster: Ray virtual cluster the generation workers run on. @@ -608,8 +603,8 @@ def _build_generation( Returns: A tuple of (generation object, wall time spent in this call). The - generation object is a VllmGeneration, SGLangGeneration, MegatronGeneration, - or DynamoGeneration. + generation object is a VllmGeneration, SGLangGeneration, or + MegatronGeneration. """ t0 = time.perf_counter() generation_config = master_config.policy["generation"] @@ -659,21 +654,10 @@ def _build_generation( skip_weight_load=True, ) - elif backend == "dynamo": - assert not defer_model_load, ( - "defer_model_load is only supported for the vllm backend" - ) - # setup_single_controller validated and normalized this config through - # DynamoConfig before allocating clusters. - gen = DynamoGeneration( - cluster=inference_cluster, - config=cast(dict[str, Any], generation_config), - ) - else: raise ValueError( - "single_controller_utils.setup only supports vllm, sglang, megatron, or " - f"dynamo generation; got {backend!r}" + "single_controller_utils.setup only supports vllm, sglang, or megatron " + f"generation; got {backend!r}" ) if not defer_model_load: @@ -825,12 +809,12 @@ def _spinup_gym( def _generation_max_seq_len(generation_config) -> int: """Return the per-backend max sequence length. - vllm and dynamo (which manages vLLM workers) use vllm_cfg.max_model_len; + vllm uses vllm_cfg.max_model_len; sglang uses sglang_cfg.context_length; megatron uses mcore_generation_config.max_model_len. """ backend = generation_config["backend"] - if backend in ("vllm", "dynamo"): + if backend == "vllm": return generation_config["vllm_cfg"]["max_model_len"] if backend == "sglang": return generation_config["sglang_cfg"]["context_length"] @@ -1213,7 +1197,7 @@ def setup_single_controller( telemetry_interval_s = master_config.rollout_checkpointing.telemetry_interval_s if telemetry_interval_s is not None: generation_backend = generation_config["backend"] - if generation_backend not in ("vllm", "dynamo"): + if generation_backend != "vllm": warnings.warn( "rollout_checkpointing.telemetry_interval_s is enabled with " f"policy.generation.backend={generation_backend!r}. Canonical " @@ -1231,7 +1215,7 @@ def setup_single_controller( "vLLM token, request, and KV-cache signals will be absent.", stacklevel=2, ) - elif generation_backend == "vllm" and not vllm_cfg["async_engine"]: + elif not vllm_cfg["async_engine"]: warnings.warn( "rollout_checkpointing.telemetry_interval_s and " "policy.generation.vllm_cfg.enable_vllm_metrics_logger are " @@ -1257,18 +1241,6 @@ def setup_single_controller( if checkpointing_pretrained is not None: policy_config["pretrained_checkpoint"] = checkpointing_pretrained - if generation_config["backend"] == "dynamo": - # Validate the complete managed-Dynamo boundary before allocating Ray - # placement groups or starting any external services, as grpo.setup does. - dynamo_config = cast(dict[str, Any], generation_config) - dynamo_config.setdefault("vllm_kwargs", {})["hf_overrides"] = ( - policy_config.get("hf_config_overrides") or {} - ) - generation_config = cast( - GenerationConfig, DynamoConfig.model_validate(dynamo_config).model_dump() - ) - policy_config["generation"] = generation_config - # Token capture: validate the supported combination loudly at setup # (NeMo-Gym rollout path; vLLM with async_engine=true, or Megatron with # expose_http_server=true). The serving worker's venv already carries @@ -1368,7 +1340,11 @@ def setup_single_controller( # Fill the derived ledger-hosting fields (see TokenCaptureConfig): a # per-run control-plane bearer token, the process-shared capture # directory used by every Gym worker, and the capture-host backend. - token_capture_cfg.generation_backend = generation_config["backend"] + token_capture_cfg.generation_backend = ( + "dynamo" + if dynamo_enabled(generation_config) + else generation_config["backend"] + ) if token_capture_cfg.control_auth_token is None: # Deferred import: only needed on the capture path. import secrets @@ -1852,7 +1828,7 @@ def _build_trainer_then_megatron_generation() -> tuple[ defer_model_load=True, ) defer_generation_model_load = True - gym_base_urls = generation.dp_openai_server_base_urls + gym_base_urls = generation.client_base_urls # Before the Gym task is built, so Gym can be handed the router's single URL. # These two statements are the only failable ones related to the port holder's creation. try: diff --git a/nemo_rl/distributed/actor_environments.py b/nemo_rl/distributed/actor_environments.py index 8bd9ccaf943..25e2cc4791a 100644 --- a/nemo_rl/distributed/actor_environments.py +++ b/nemo_rl/distributed/actor_environments.py @@ -54,7 +54,6 @@ "nemo_gym", ], "nemo_rl.models.generation.sglang.sglang_worker.SGLangGenerationWorker": ["sglang"], - "nemo_rl.models.generation.dynamo.dynamo_worker.DynamoVllmWorker": None, "nemo_rl.models.policy.workers.automodel_policy_worker.AutomodelPolicyWorker": [ "automodel" ], diff --git a/nemo_rl/distributed/virtual_cluster.py b/nemo_rl/distributed/virtual_cluster.py index f87fe68f224..049ddde289f 100644 --- a/nemo_rl/distributed/virtual_cluster.py +++ b/nemo_rl/distributed/virtual_cluster.py @@ -157,8 +157,8 @@ def uv_py_executable(extras: Sequence[str]) -> str: # 1313-1399 Dynamo etcd/NATS control plane (driver-local allocation) # 1400-1999 Master address / TCPStore (cluster.master_port_range_low/high) # [3000, 4999) Shared NeMo RL generation range (policy.generation.port_range_low/high) -# [3000, 4000) Dynamo frontend/token-wrapper HTTP endpoints -# [4000, 4100) Dynamo worker system endpoints (node-local free-port selection) +# [3000, 4000) Dynamo frontend HTTP endpoint +# [4000, 4100) Dynamo engine KV-event endpoints (node-local free-port selection) # 5000-5999 NeMo Gym HTTP servers (env.nemo_gym.port_range_low/high) # 6000 NeMo-Skills sandbox Nginx (NEMO_SKILLS_SANDBOX_PORT; ray.sub starts one # sidecar per allocated node, driver included) diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 0900200b4b1..017f5d152dd 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -92,6 +92,10 @@ RolloutDataFailure, http_status_is_infra, ) +from nemo_rl.models.generation.dynamo.config import ( + dynamo_enabled, + native_default_chat_template_kwargs, +) from nemo_rl.models.generation.interfaces import ( resolve_routed_experts_dtype_name_for_model, should_use_async_rollouts, @@ -300,10 +304,6 @@ def should_use_nemo_gym(master_config: NemoGymCompatibleConfig) -> bool: should_expose_http_server = generation_config.get("trtllm_cfg", {}).get( "expose_http_server" ) - elif generation_config["backend"] == "dynamo": - should_expose_http_server = generation_config.get("vllm_cfg", {}).get( - "expose_http_server" - ) else: should_expose_http_server = False assert should_expose_http_server, ( @@ -446,9 +446,11 @@ def _external_staging_backend(token_capture: Dict[str, Any]) -> str: return "vllm_worker" if generation_backend == "megatron": return "megatron_worker" + if generation_backend == "dynamo": + return "dynamo_worker" raise ValueError( "token_capture.enabled requires setup-derived generation_backend to be " - f"'vllm' or 'megatron'; got {generation_backend!r}" + f"'vllm', 'megatron' or 'dynamo'; got {generation_backend!r}" ) @@ -1870,8 +1872,7 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config = config.policy["generation"] # Enable the backend's OpenAI-compatible server. - if generation_config["backend"] in ("vllm", "dynamo"): - # Dynamo uses these flags to expose its token wrapper to Gym. + if generation_config["backend"] == "vllm": generation_config["vllm_cfg"]["async_engine"] = True generation_config["vllm_cfg"]["expose_http_server"] = True elif generation_config["backend"] == "megatron": @@ -1887,7 +1888,7 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["trtllm_cfg"]["expose_http_server"] = True else: raise ValueError( - "NeMo-Gym setup supports vllm, dynamo, megatron, or trtllm " + "NeMo-Gym setup supports vllm, megatron, or trtllm " f"generation; got {generation_config['backend']!r}" ) @@ -1895,6 +1896,29 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_strings"] = None generation_config["stop_token_ids"] = None + if dynamo_enabled(generation_config): + # Gym's policy model calls the Dynamo frontend, which takes token data in + # nvext (Gym's Dynamo dialect) instead of vLLM's request extensions. + vllm_model = ( + config.env.setdefault("nemo_gym", {}) + .setdefault("policy_model", {}) + .setdefault("responses_api_models", {}) + .setdefault("vllm_model", {}) + ) + vllm_model["dynamo_frontend"] = True + # The frontend reads `reasoning` as another name for `reasoning_content` and + # refuses an echoed assistant turn that carries both, Gym's default. + vllm_model.setdefault("reasoning_field", "reasoning_content") + # The native server merges these under every request's own template + # kwargs; the frontend has no such default, so Gym sends them. They sit + # under the kwargs Gym sends: vllm_model's own, else extra_body's. + default_kwargs = native_default_chat_template_kwargs(generation_config) + if default_kwargs: + gym_kwargs = vllm_model.get("chat_template_kwargs") or ( + (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} + ) + vllm_model["chat_template_kwargs"] = {**default_kwargs, **gym_kwargs} + # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for # multi-turn multimodal postprocessing). diff --git a/nemo_rl/experience/rollouts.py b/nemo_rl/experience/rollouts.py index 0384ef6f111..486fccfffaa 100644 --- a/nemo_rl/experience/rollouts.py +++ b/nemo_rl/experience/rollouts.py @@ -753,7 +753,6 @@ async def generate_responses_async( # SGLang exposes ``sglang_cfg`` and gates on ``use_async_rollouts``; # vLLM exposes ``cfg`` and gates on ``vllm_cfg.async_engine``; # TRT-LLM requires its flag; the Megatron backend is always async. - # Managed Dynamo always exposes its rollout frontend asynchronously. vllm_cfg = getattr(policy_generation, "cfg", None) sglang_cfg = getattr(policy_generation, "sglang_cfg", None) generation_config = vllm_cfg or sglang_cfg or {} @@ -765,8 +764,6 @@ async def generate_responses_async( use_async_generation = bool( generation_config.get("vllm_cfg", {}).get("async_engine", False) ) - elif backend == "dynamo": - use_async_generation = True elif backend == "trtllm": assert generation_config.get("trtllm_cfg", {}).get("async_engine", False), ( "TRT-LLM backend requires trtllm_cfg.async_engine=true; the " diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index e043d018150..888015266e3 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -17,6 +17,12 @@ from transformers import PreTrainedTokenizerBase +from nemo_rl.models.generation.dynamo.config import ( + check_dynamo_serving_chat_kwargs, + dynamo_cfg, + dynamo_enabled, + dynamo_parsers, +) from nemo_rl.models.generation.interfaces import GenerationConfig, GenerationInterface from nemo_rl.models.generation.trtllm.config import ( TrtllmConfig, @@ -106,10 +112,6 @@ def resolve_generation_class( from nemo_rl.models.generation.trtllm import TrtllmGeneration return TrtllmGeneration - if backend == "dynamo": - from nemo_rl.models.generation.dynamo import DynamoGeneration - - return DynamoGeneration raise ValueError(f"Unknown generation backend: {backend!r}") @@ -153,6 +155,19 @@ def configure_generation_config( "extend the training worker instead." ) + if dynamo_enabled(config) and config["backend"] != "vllm": + raise ValueError( + "policy.generation.dynamo=true serves NeMo RL's vLLM workers through " + f"Dynamo and does not apply to backend={config['backend']!r}. Set " + "policy.generation.backend=vllm, or remove policy.generation.dynamo." + ) + if config.get("dynamo_cfg") is not None and not dynamo_enabled(config): + raise ValueError( + "policy.generation.dynamo_cfg is set but policy.generation.dynamo is not " + "true, so it would be ignored. Set policy.generation.dynamo=true to " + "serve generation through Dynamo, or remove dynamo_cfg." + ) + # tokenizer setting if "_pad_token_id" in config: warnings.warn( @@ -164,19 +179,42 @@ def configure_generation_config( if config["stop_token_ids"] is None: config["stop_token_ids"] = [tokenizer.eos_token_id] - # vLLM setting shared by the standard and managed Dynamo backends. - if config["backend"] in ("vllm", "dynamo"): - vllm_backed_config = cast(VllmConfig, config) + if config["backend"] == "vllm": + config = cast(VllmConfig, config) # VLM training can require real startup weights for modules that are # frozen and therefore never refitted (for example, the vision tower). # Preserve an explicit recipe override while retaining the lightweight # dummy default for ordinary training. - vllm_backed_config["vllm_cfg"].setdefault( - "load_format", "auto" if is_eval else "dummy" - ) - - if config["backend"] == "vllm": - config = cast(VllmConfig, config) + config["vllm_cfg"].setdefault("load_format", "auto" if is_eval else "dummy") + if dynamo_enabled(config): + # Validate before any Ray allocation; dynamo_parsers also rejects + # parser plugins that Dynamo's Rust frontend cannot run. + dynamo_cfg(config) + dynamo_parsers(config) + check_dynamo_serving_chat_kwargs(config) + if config["vllm_cfg"].get("http_server_api_key_required"): + raise ValueError( + "policy.generation.vllm_cfg.http_server_api_key_required is not " + "supported with policy.generation.dynamo=true: NeMo-Gym calls the " + "Dynamo frontend, which checks no bearer token." + ) + colocated = config.get("colocated") + if colocated is not None and colocated["enabled"]: + raise ValueError( + "policy.generation.dynamo=true requires non-colocated generation " + "(policy.generation.colocated.enabled=false): a colocated engine " + "sleeps while the policy trains, and the Dynamo frontend would " + "keep routing requests to it." + ) + if not config["vllm_cfg"]["async_engine"]: + # Dynamo serves the engine's AsyncLLM. The async engine runs every + # rollout path the sync one does, so turning it on is the switch. + print( + " [Dynamo] policy.generation.dynamo=true: using the async " + "vLLM engine (vllm_cfg.async_engine=true)", + flush=True, + ) + config["vllm_cfg"]["async_engine"] = True validate_vllm_quantization_config(config) validate_nvfp4_pertoken_generation(config, is_eval=is_eval) if config.get("real_quant"): @@ -232,6 +270,8 @@ def configure_generation_config( is_eval or config["stop_strings"] is not None or config["vllm_cfg"].get("expose_http_server", None) + # Dynamo serves OpenAI clients as the exposed server does. + or dynamo_enabled(config) ): config["vllm_cfg"]["skip_tokenizer_init"] = False else: diff --git a/nemo_rl/models/generation/constants.py b/nemo_rl/models/generation/constants.py index 06564f055ef..c00f2297b56 100644 --- a/nemo_rl/models/generation/constants.py +++ b/nemo_rl/models/generation/constants.py @@ -21,5 +21,4 @@ VLLM_BACKEND = "vllm" SGLANG_BACKEND = "sglang" MEGATRON_BACKEND = "megatron" -DYNAMO_BACKEND = "dynamo" TRTLLM_BACKEND = "trtllm" diff --git a/nemo_rl/models/generation/dynamo/__init__.py b/nemo_rl/models/generation/dynamo/__init__.py index 082e6260c19..40376929afa 100644 --- a/nemo_rl/models/generation/dynamo/__init__.py +++ b/nemo_rl/models/generation/dynamo/__init__.py @@ -13,16 +13,16 @@ # limitations under the License. from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, - DynamoConfig, DynamoFrontendArgs, DynamoWorkerArgs, + dynamo_cfg, + dynamo_enabled, ) -from nemo_rl.models.generation.dynamo.dynamo_generation import DynamoGeneration __all__ = [ "DynamoCfg", - "DynamoConfig", "DynamoFrontendArgs", - "DynamoGeneration", "DynamoWorkerArgs", + "dynamo_cfg", + "dynamo_enabled", ] diff --git a/nemo_rl/models/generation/dynamo/arguments.py b/nemo_rl/models/generation/dynamo/arguments.py index 07041493f71..c08d9873200 100644 --- a/nemo_rl/models/generation/dynamo/arguments.py +++ b/nemo_rl/models/generation/dynamo/arguments.py @@ -12,23 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Deterministic argument and environment construction for managed Dynamo.""" +"""Argument construction and log redaction for the managed Dynamo frontend.""" import json from collections.abc import Mapping, Sequence from typing import Any -from nemo_rl.models.generation.dynamo.config import ( - DYNAMO_VLLM_FLAGS, - DynamoCfg, -) - -_MANAGED_FLAGS = { - "--component", - "--model-name", - "--model-path", - "--endpoint", -} +from nemo_rl.models.generation.dynamo.config import DynamoCfg _CREDENTIAL_FLAGS = { "--access-token", @@ -49,10 +39,6 @@ def _normalise_flag(flag: str) -> str: return flag -def _flag_for_key(key: str) -> str: - return "--" + key.replace("_", "-") - - def _serialise_value(value: Any) -> str: if isinstance(value, (dict, list, tuple)): return json.dumps(value, separators=(",", ":"), sort_keys=True) @@ -69,7 +55,7 @@ def add(self, flag: str, value: Any = None, *, source: str) -> None: prior = self.sources.get(normalised) if prior is not None: raise ValueError( - f"Dynamo worker option {normalised} is set by both {prior} and {source}." + f"Dynamo frontend option {normalised} is set by both {prior} and {source}." ) self.sources[normalised] = source if isinstance(value, bool): @@ -89,13 +75,9 @@ def add_raw(self, args: Sequence[str], *, source: str) -> None: current_flag = token current_has_value = "=" in token normalised = _normalise_flag(token) - if normalised in _MANAGED_FLAGS: - raise ValueError( - f"{source} may not override managed option {normalised}." - ) if normalised in self.sources: raise ValueError( - f"Dynamo worker option {normalised} is set by both " + f"Dynamo frontend option {normalised} is set by both " f"{self.sources[normalised]} and {source}." ) self.sources[normalised] = source @@ -109,111 +91,6 @@ def add_raw(self, args: Sequence[str], *, source: str) -> None: self.argv.extend(args) -def build_dynamo_vllm_argv( - *, - model_name: str, - namespace: str, - seed: int, - vllm_cfg: Mapping[str, Any], - vllm_kwargs: Mapping[str, Any], - dynamo_cfg: DynamoCfg, -) -> list[str]: - """Build a conflict-free ``python -m dynamo.vllm`` argument list.""" - builder = _ArgvBuilder() - builder.add("--model", model_name, source="managed runtime") - builder.add("--served-model-name", model_name, source="managed runtime") - builder.add("--namespace", namespace, source="managed runtime") - builder.add("--discovery-backend", "etcd", source="managed runtime") - builder.add("--request-plane", "tcp", source="managed runtime") - builder.add("--event-plane", "nats", source="managed runtime") - builder.add("--enable-rl", source="managed runtime") - builder.add( - "--weight-transfer-config", - {"backend": "nccl"}, - source="managed runtime", - ) - builder.add("--trust-remote-code", source="managed runtime") - builder.add("--seed", seed, source="managed runtime") - - for key, flag in DYNAMO_VLLM_FLAGS.items(): - value = vllm_cfg.get(key) - if value is not None: - builder.add(flag, value, source=f"vllm_cfg.{key}") - - if int(vllm_cfg["expert_parallel_size"]) > 1: - builder.add( - "--enable-expert-parallel", - source="vllm_cfg.expert_parallel_size", - ) - - worker_args = dynamo_cfg.worker_args - if worker_args.tool_call_parser is not None: - builder.add( - "--dyn-tool-call-parser", - worker_args.tool_call_parser, - source="dynamo_cfg.worker_args.tool_call_parser", - ) - if worker_args.reasoning_parser is not None: - builder.add( - "--dyn-reasoning-parser", - worker_args.reasoning_parser, - source="dynamo_cfg.worker_args.reasoning_parser", - ) - builder.add( - "--exclude-tools-when-tool-choice-none", - worker_args.exclude_tools_when_tool_choice_none, - source="dynamo_cfg.worker_args.exclude_tools_when_tool_choice_none", - ) - builder.add( - "--dyn-enable-structural-tag", - worker_args.enable_structural_tag, - source="dynamo_cfg.worker_args.enable_structural_tag", - ) - builder.add( - "--dyn-structural-tag-scope", - worker_args.structural_tag_scope, - source="dynamo_cfg.worker_args.structural_tag_scope", - ) - builder.add( - "--dyn-structural-tag-schema", - worker_args.structural_tag_schema, - source="dynamo_cfg.worker_args.structural_tag_schema", - ) - if worker_args.custom_jinja_template is not None: - builder.add( - "--custom-jinja-template", - worker_args.custom_jinja_template, - source="dynamo_cfg.worker_args.custom_jinja_template", - ) - builder.add( - "--endpoint-types", - ",".join(worker_args.endpoint_types), - source="dynamo_cfg.worker_args.endpoint_types", - ) - - for key, value in vllm_kwargs.items(): - if value is None: - continue - if key == "speculative_config": - raise ValueError( - "policy.generation.vllm_kwargs.speculative_config is not " - "supported by backend='dynamo' because draft weights are not " - "refit after step 0" - ) - flag = _flag_for_key(key) - normalised = _normalise_flag(flag) - source = f"vllm_kwargs.{key}" - if normalised in _MANAGED_FLAGS: - raise ValueError(f"{source} may not override managed option {normalised}.") - builder.add(flag, value, source=source) - - builder.add_raw( - worker_args.extra_cli_args, - source="dynamo_cfg.worker_args.extra_cli_args", - ) - return builder.argv - - def build_dynamo_frontend_argv( *, host: str, @@ -234,6 +111,12 @@ def build_dynamo_frontend_argv( dynamo_cfg.frontend_args.router_mode, source="dynamo_cfg.frontend_args.router_mode", ) + if dynamo_cfg.frontend_args.session_affinity_ttl_secs is not None: + builder.add( + "--router-session-affinity-ttl-secs", + dynamo_cfg.frontend_args.session_affinity_ttl_secs, + source="dynamo_cfg.frontend_args.session_affinity_ttl_secs", + ) builder.add_raw( dynamo_cfg.frontend_args.extra_cli_args, source="dynamo_cfg.frontend_args.extra_cli_args", @@ -241,36 +124,6 @@ def build_dynamo_frontend_argv( return builder.argv -def build_managed_worker_env( - *, - base_env: Mapping[str, str], - configured_env: Mapping[str, str], - manager_env: Mapping[str, str], -) -> dict[str, str]: - """Return a reproducible worker environment with semantic overrides blocked.""" - reserved_env_keys = set(manager_env) - forbidden = sorted( - key - for key in configured_env - if key.startswith("DYN_") or key in reserved_env_keys - ) - if forbidden: - raise ValueError( - "vllm_cfg.env_vars may not override managed Dynamo settings: " - + ", ".join(forbidden) - ) - - env = { - key: value - for key, value in base_env.items() - if not key.startswith(("DYN_", "ETCD_", "NATS_")) - and key not in reserved_env_keys - } - env.update(configured_env) - env.update(manager_env) - return env - - def redact_argv(argv: Sequence[str]) -> list[str]: """Redact values following explicit credential options for safe logging.""" redacted = list(argv) diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 2bf99ba4e12..d08758ce3ed 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -12,143 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Validated configuration for the managed Dynamo generation backend.""" - -import warnings -from typing import Annotated, Any, Literal - -from pydantic import ( - BaseModel, - BeforeValidator, - PositiveFloat, - PositiveInt, - model_validator, -) - -# Packed weight-transfer geometry. NeMo RL declares it to each Dynamo worker in -# the NCCL init info, and the trainer packs with the same values, so both sides -# recompute identical chunk boundaries. -VLLM_PACKED_BUFFER_SIZE_BYTES = 1024**3 -VLLM_PACKED_NUM_BUFFERS = 2 - -DYNAMO_VLLM_FLAGS: dict[str, str] = { - "tensor_parallel_size": "--tensor-parallel-size", - "pipeline_parallel_size": "--pipeline-parallel-size", - "gpu_memory_utilization": "--gpu-memory-utilization", - "max_model_len": "--max-model-len", - "kv_cache_dtype": "--kv-cache-dtype", - "load_format": "--load-format", - "precision": "--dtype", - "enforce_eager": "--enforce-eager", -} - -_VLLM_CFG_STRUCTURAL = { - "env_vars", - "expert_parallel_size", -} - -_VLLM_CFG_MOVED = { - "http_server_serving_chat_kwargs": ( - "dynamo_cfg.worker_args.custom_jinja_template and tool_call_parser" - ), - "reasoning_parser_plugin": "dynamo_cfg.worker_args.reasoning_parser", - "tool_parser_plugin": "dynamo_cfg.worker_args.tool_call_parser", -} - -_VLLM_CFG_UNSUPPORTED = { - # Applied inside NeMo RL's in-process vLLM worker; it cannot configure the - # managed ``dynamo.vllm`` subprocess. - "cap_max_tokens_to_context", - # Enforced by NeMo RL's own HTTP server; the managed Dynamo frontend checks - # no bearer token, so a truthy value is rejected below rather than warned. - "http_server_api_key_required", - "is_mx", - "num_first_layers_in_bf16", - "num_last_layers_in_bf16", - "skip_tokenizer_init", - "use_deep_gemm", -} - -_VLLM_CFG_MANAGED_RUNTIME = { - "enable_vllm_metrics_logger", - "expose_http_server", - "logprobs_mode", - "vllm_metrics_logger_interval", -} - -_VLLM_CFG_INAPPLICABLE = { - "async_engine", - "enable_return_routed_experts", - "http_refit_api_key_env_var", - "http_refit_server_port", - "use_tqdm", - "zmq_refit_server_port", -} - -_VLLM_SINGLE_RANK_ONLY_FIELDS = { - "data_parallel_size", - "decode_context_parallel_size", - "prefill_context_parallel_size", -} - -# Top-level ``policy.generation`` keys outside the declared fields that are accounted -# for; any other active key would be silently dropped, so it is warned about. -_GENERATION_CLASSIFIED = { - # Read by Dynamo generation, or validated in DynamoConfig. - "model_name", - "max_new_tokens", - "temperature", - "top_p", - "top_k", - "stop_token_ids", - "stop_strings", - "_pad_token_id", - "colocated", - "refit_transport", - "quant_cfg", - "real_quant", - "sglang_cfg", - "trtllm_cfg", - # Read by the algorithm, rollouts, or NeMo-Gym rather than the engine. - "val_temperature", - "val_top_p", - "val_top_k", - "port_range_low", - "port_range_high", - "use_async_rollouts", - "_debug_payload_metrics", - "_draft_weights_from_refit", - "_mtp_weights_from_refit", - "_draft_full_refit", - # Shared-YAML sections for other backends and refit paths. - "mcore_generation_config", - "refit_cfg", - "real_quant_export_cpu_offload", - "real_quant_ignore", -} - - -def _is_active(value: Any) -> bool: - """Whether a setting is on; a block with an ``enabled`` switch counts only then.""" - if isinstance(value, BaseModel): - value = value.model_dump() - if isinstance(value, dict) and "enabled" in value: - return bool(value["enabled"]) - return bool(value) +"""Configuration for serving NeMo RL's vLLM workers behind Dynamo. + +``policy.generation.dynamo: true`` on any vLLM config is the whole switch; +``policy.generation.dynamo_cfg`` only tunes the frontend and registration, and +every field below carries its default. +""" + +from typing import Any, Literal, Mapping + +from pydantic import BaseModel, Field, PositiveFloat, PositiveInt, model_validator class DynamoWorkerArgs(BaseModel, extra="forbid"): - """Structured arguments passed to every managed ``dynamo.vllm`` worker.""" - - tool_call_parser: str | None - reasoning_parser: str | None - exclude_tools_when_tool_choice_none: bool - enable_structural_tag: bool - structural_tag_scope: Literal["auto", "always"] - structural_tag_schema: Literal["auto", "strict"] - custom_jinja_template: str | None - endpoint_types: list[Literal["chat", "completions"]] - extra_cli_args: list[str] + """How each vLLM engine registers with Dynamo (``WorkerConfig`` fields).""" + + # Dynamo's built-in parser names (dynamo-parsers). None follows the native + # server's vllm_cfg.http_server_serving_chat_kwargs (see dynamo_parsers). + tool_call_parser: str | None = None + reasoning_parser: str | None = None + # False, as vLLM's OpenAIServingChat (the native server) defaults it. + exclude_tools_when_tool_choice_none: bool = False + enable_structural_tag: bool = False + structural_tag_scope: Literal["auto", "always"] = "auto" + structural_tag_schema: Literal["auto", "strict"] = "auto" + # A Jinja chat template file that replaces the model's own. None follows + # vllm_cfg.http_server_serving_chat_kwargs.chat_template. + custom_jinja_template: str | None = None + endpoint_types: list[Literal["chat", "completions"]] = ["chat", "completions"] @model_validator(mode="after") def _validate_endpoint_types(self) -> "DynamoWorkerArgs": @@ -160,11 +51,13 @@ def _validate_endpoint_types(self) -> "DynamoWorkerArgs": class DynamoFrontendArgs(BaseModel, extra="forbid"): - """Structured arguments passed to the managed Dynamo frontend.""" + """Arguments for the managed ``dynamo.frontend`` process.""" - tokenizer: Literal["default", "fastokens"] - tokenizer_cache: bool - tokenizer_cache_bytes: PositiveInt + tokenizer: Literal["default", "fastokens"] = "default" + tokenizer_cache: bool = False + tokenizer_cache_bytes: PositiveInt = 50 * 1024**2 + # "kv" routes each request to the engine holding most of its prefix, fed by + # the engines' vLLM KV-cache events. router_mode: Literal[ "round-robin", "random", @@ -173,222 +66,154 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "direct", "least-loaded", "device-aware-weighted", - ] - extra_cli_args: list[str] + ] = "kv" + # Idle TTL in seconds for keeping one session on one engine, or None to + # disable it. NeMo-Gym names the session on token-capture calls only (its + # rollout id), as the native backend pins each Gym session to one server; the + # engine splices a call's staged prefix after routing, so the KV router alone + # cannot see where that prefix is cached. + session_affinity_ttl_secs: PositiveInt | None = 1800 + extra_cli_args: list[str] = Field(default_factory=list) class DynamoCfg(BaseModel, extra="forbid"): - """Driver-owned Dynamo service and worker-fleet configuration.""" - - engine: Literal["vllm"] - startup_timeout_s: PositiveFloat - request_timeout_s: PositiveFloat - control_timeout_s: PositiveFloat - worker_args: DynamoWorkerArgs - frontend_args: DynamoFrontendArgs - metrics_include_prefixes: list[str] | None - metrics_exclude_prefixes: list[str] | None + """``policy.generation.dynamo_cfg``: the frontend and engine registration.""" + + # Bounds etcd/NATS/frontend start-up and every engine's registration. + startup_timeout_s: PositiveFloat = 600.0 + # Timeout for one NeMo RL rollout routed through the frontend, in place of + # NRL_VLLM_ASYNC_TIMEOUT_SECONDS; a timed-out rollout is not retried. + # NeMo-Gym's own calls keep Gym's timeouts. + request_timeout_s: PositiveFloat = 900.0 + worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) + frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) + + +# vLLM parser names whose Dynamo built-in parser goes by another name. nano_v3 is +# NeMo RL's vLLM plugin: DeepSeek-R1 reasoning parsing, with the whole output as +# content when chat_template_kwargs.enable_thinking is false, as Dynamo's +# nemotron_v3 parser does. +_DYNAMO_REASONING_PARSER_NAMES = {"nano_v3": "nemotron_v3"} +_DYNAMO_TOOL_PARSER_NAMES = {"openai": "harmony"} + + +def _dynamo_parser( + configured: str | None, + native: str | None, + plugin: str | None, + renames: Mapping[str, str], + *, + kind: str, + worker_arg: str, +) -> str | None: + if configured is not None or native is None: + return configured + if native in renames: + return renames[native] + if plugin: + raise ValueError( + f"policy.generation.vllm_cfg.{kind}_parser_plugin loads a Python parser " + "into vLLM's OpenAI server, which Dynamo's Rust frontend cannot run. Set " + f"policy.generation.dynamo_cfg.worker_args.{worker_arg} to the Dynamo " + f"built-in parser for {native!r}." + ) + return native -class DynamoVllmConfig(BaseModel, extra="allow"): - """Known vLLM settings consumed by ``dynamo.vllm``. +def dynamo_parsers( + generation_config: Mapping[str, Any], +) -> tuple[str | None, str | None]: + """The ``(tool_call_parser, reasoning_parser)`` Dynamo's frontend applies. - Additional fields remain visible in ``model_extra`` so argument construction - can warn rather than silently dropping an inherited vLLM setting. + ``dynamo_cfg.worker_args`` names them explicitly. Otherwise they follow the + native server's ``vllm_cfg.http_server_serving_chat_kwargs``, renamed where + Dynamo calls the same parser differently. A parser plugin without such a + rename has no Dynamo equivalent and is rejected. """ + worker_args = dynamo_cfg(generation_config).worker_args + vllm_cfg = generation_config["vllm_cfg"] + serving = vllm_cfg.get("http_server_serving_chat_kwargs") or {} + tool_call_parser = _dynamo_parser( + worker_args.tool_call_parser, + serving.get("tool_parser"), + vllm_cfg.get("tool_parser_plugin"), + _DYNAMO_TOOL_PARSER_NAMES, + kind="tool", + worker_arg="tool_call_parser", + ) + reasoning_parser = _dynamo_parser( + worker_args.reasoning_parser, + serving.get("reasoning_parser"), + vllm_cfg.get("reasoning_parser_plugin"), + _DYNAMO_REASONING_PARSER_NAMES, + kind="reasoning", + worker_arg="reasoning_parser", + ) + return tool_call_parser, reasoning_parser + + +def check_dynamo_serving_chat_kwargs(generation_config: Mapping[str, Any]) -> None: + """Reject native chat-server settings Dynamo's frontend has no counterpart for. + + The frontend renders templates and parses tool calls itself: it always + parses ``tool_choice="auto"`` calls when a tool parser is set, and has no + content-format setting. The native defaults (``enable_auto_tools: true``, + ``chat_template_content_format: auto``) match it; other values would + silently change what the policy sees. + """ + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + content_format = serving.get("chat_template_content_format") + if content_format not in (None, "auto"): + raise ValueError( + "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." + f"chat_template_content_format={content_format!r} is not supported " + "with policy.generation.dynamo=true: Dynamo's frontend renders chat " + "templates itself and has no content-format setting. Remove it or " + "set it to 'auto'." + ) + if serving.get("enable_auto_tools") is False: + raise ValueError( + "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." + "enable_auto_tools=false is not supported with " + "policy.generation.dynamo=true: Dynamo's frontend always parses " + "tool_choice='auto' calls when a tool parser is set. Remove it, or " + "remove the tool parser." + ) - async_engine: bool - tensor_parallel_size: PositiveInt - pipeline_parallel_size: PositiveInt - expert_parallel_size: PositiveInt - gpu_memory_utilization: float - max_model_len: PositiveInt - kv_cache_dtype: str - load_format: str - precision: str - enforce_eager: bool - expose_http_server: bool - enable_vllm_metrics_logger: bool - vllm_metrics_logger_interval: PositiveFloat - env_vars: dict[str, str] | None - @model_validator(mode="after") - def _validate_parallelism_and_precision(self) -> "DynamoVllmConfig": - if self.expert_parallel_size not in (1, self.tensor_parallel_size): - raise ValueError( - "backend='dynamo' requires expert_parallel_size to be 1 or " - "equal tensor_parallel_size" - ) - if self.precision.lower() not in { - "bf16", - "bfloat16", - }: - raise ValueError( - f"policy.generation.vllm_cfg.precision={self.precision!r} is not " - "supported by backend='dynamo'; managed weight refit currently " - "supports BF16 generation only" - ) - if self.kv_cache_dtype != "auto": - raise ValueError( - f"policy.generation.vllm_cfg.kv_cache_dtype={self.kv_cache_dtype!r} " - "is not supported by backend='dynamo'; use 'auto'" - ) - extra = self.model_extra or {} - if extra.get("is_mx"): - raise ValueError( - "policy.generation.vllm_cfg.is_mx is not supported by " - "backend='dynamo'; use backend='vllm' for MXFP8 generation" - ) - if extra.get("http_server_api_key_required"): - raise ValueError( - "policy.generation.vllm_cfg.http_server_api_key_required is not " - "supported by backend='dynamo'; the managed Dynamo frontend checks " - "no bearer token, so the exposed server would stay open" - ) - if ( - int(extra.get("num_first_layers_in_bf16") or 0) != 0 - or int(extra.get("num_last_layers_in_bf16") or 0) != 0 - ): - raise ValueError( - "mixed BF16/FP8 generation is not supported by backend='dynamo'; " - "use backend='vllm'" - ) - logprobs_mode = extra.get("logprobs_mode") - if logprobs_mode not in (None, "processed_logprobs"): - raise ValueError( - "policy.generation.vllm_cfg.logprobs_mode must be " - "'processed_logprobs' when backend='dynamo'; the managed " - "--enable-rl option selects processed rollout log probabilities" - ) - - configured_fields = self.model_fields_set | set(extra) - for key, replacement in _VLLM_CFG_MOVED.items(): - if extra.get(key) is not None: - raise ValueError( - f"policy.generation.vllm_cfg.{key} is not read by the " - f"Dynamo backend; set {replacement} instead" - ) - for key in sorted(_VLLM_CFG_UNSUPPORTED & configured_fields): - if not extra.get(key): - continue - warnings.warn( - f"policy.generation.vllm_cfg.{key} is ignored by backend='dynamo'", - stacklevel=2, - ) - classified = ( - set(DYNAMO_VLLM_FLAGS) - | _VLLM_CFG_STRUCTURAL - | set(_VLLM_CFG_MOVED) - | _VLLM_CFG_UNSUPPORTED - | _VLLM_CFG_MANAGED_RUNTIME - | _VLLM_CFG_INAPPLICABLE - | _VLLM_SINGLE_RANK_ONLY_FIELDS - ) - unclassified = { - key for key in configured_fields if getattr(self, key, None) is not None - } - classified - if unclassified: - warnings.warn( - "vllm_cfg keys ignored by backend='dynamo': " - f"{sorted(unclassified)}. Add them to DYNAMO_VLLM_FLAGS or an " - "explicit not-forwarded classification.", - stacklevel=2, - ) - return self +def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: + """The chat template the frontend renders with: a file path or literal Jinja. + ``dynamo_cfg.worker_args.custom_jinja_template``, else the native server's + ``http_server_serving_chat_kwargs.chat_template``; None uses the model's own. + """ + configured = dynamo_cfg(generation_config).worker_args.custom_jinja_template + if configured is not None: + return configured + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + return serving.get("chat_template") -def _require_nonempty_vllm_config(value: Any) -> Any: - if not isinstance(value, dict) or not value: - raise ValueError( - "policy.generation.vllm_cfg must be a nonempty mapping when " - "backend='dynamo'" - ) - return value +def native_default_chat_template_kwargs( + generation_config: Mapping[str, Any], +) -> dict[str, Any]: + """The template kwargs the native server merges under every request's own. -class DynamoConfig(BaseModel, extra="allow"): - """Validated boundary for ``policy.generation.backend=dynamo``.""" + ``default_chat_template_kwargs``, or the older ``chat_template_kwargs`` + spelling, of ``vllm_cfg.http_server_serving_chat_kwargs``. + """ + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + kwargs = serving.get("default_chat_template_kwargs") + if kwargs is None: + kwargs = serving.get("chat_template_kwargs") + return dict(kwargs or {}) - backend: Literal["dynamo"] - dynamo_cfg: DynamoCfg - vllm_cfg: Annotated[ - DynamoVllmConfig, BeforeValidator(_require_nonempty_vllm_config) - ] - vllm_kwargs: dict[str, Any] - @property - def engine_world_size(self) -> int: - """Return the derived ranks in each single-node vLLM engine.""" - return self.vllm_cfg.tensor_parallel_size * self.vllm_cfg.pipeline_parallel_size +def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: + """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" + return bool(generation_config.get("dynamo")) - @model_validator(mode="after") - def _validate_backend_boundary(self) -> "DynamoConfig": - extra = self.model_extra or {} - # Shared GRPO YAML inheritance supplies mcore_generation_config and - # refit_cfg. Managed Dynamo uses vLLM arguments and the collective - # synchronizer instead, so these two sections are intentionally ignored. - for backend_cfg in ("sglang_cfg", "trtllm_cfg"): - if extra.get(backend_cfg): - raise ValueError( - f"policy.generation.{backend_cfg} is not valid when " - "backend='dynamo'; Dynamo manages vLLM only" - ) - colocated = extra.get("colocated") - if isinstance(colocated, dict) and colocated.get("enabled"): - raise ValueError( - "policy.generation.colocated.enabled must be false when " - "backend='dynamo'" - ) - if extra.get("refit_transport") is not None: - raise ValueError( - "policy.generation.refit_transport must be null when " - "backend='dynamo'; managed Dynamo supports NCCL collective refit only" - ) - for quantization_field in ("quant_cfg", "real_quant"): - if extra.get(quantization_field): - raise ValueError( - f"policy.generation.{quantization_field} is not supported " - "when backend='dynamo'" - ) - speculative_config = self.vllm_kwargs.get("speculative_config") or ( - self.vllm_cfg.model_extra or {} - ).get("speculative_config") - if speculative_config: - raise ValueError( - "policy.generation.vllm_kwargs.speculative_config is not " - "supported by backend='dynamo' because draft weights are not " - "refit after step 0" - ) - if self.vllm_kwargs.get("quantization") is not None: - raise ValueError( - "policy.generation.vllm_kwargs.quantization is not supported " - "when backend='dynamo'" - ) - vllm_extra = self.vllm_cfg.model_extra or {} - for field in sorted(_VLLM_SINGLE_RANK_ONLY_FIELDS): - for source, value in ( - ("vllm_cfg", vllm_extra.get(field)), - ("vllm_kwargs", self.vllm_kwargs.get(field)), - ): - if value is not None and int(value) != 1: - raise ValueError( - f"policy.generation.{source}.{field} must be 1 when " - "backend='dynamo'; managed refit rank geometry is TP × PP" - ) - stop_strings = extra.get("stop_strings") - if stop_strings is not None and len(stop_strings) > 32: - raise ValueError( - "policy.generation.stop_strings supports at most 32 values when " - "backend='dynamo'" - ) - ignored = sorted( - key - for key, value in extra.items() - if key not in _GENERATION_CLASSIFIED and _is_active(value) - ) - if ignored: - warnings.warn( - f"policy.generation keys ignored by backend='dynamo': {ignored}", - stacklevel=2, - ) - return self + +def dynamo_cfg(generation_config: Mapping[str, Any]) -> DynamoCfg: + """The validated ``dynamo_cfg`` block; an absent block means all defaults.""" + return DynamoCfg.model_validate(generation_config.get("dynamo_cfg") or {}) diff --git a/nemo_rl/models/generation/dynamo/dynamo_generation.py b/nemo_rl/models/generation/dynamo/dynamo_generation.py deleted file mode 100644 index f94439feeb1..00000000000 --- a/nemo_rl/models/generation/dynamo/dynamo_generation.py +++ /dev/null @@ -1,767 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Generation and NCCL refit through a driver-owned Dynamo vLLM fleet.""" - -import asyncio -import logging -from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, cast - -import ray -import torch - -from nemo_rl.data.multimodal_utils import VLLM_PROMPT_KEYS -from nemo_rl.distributed.batched_data_dict import BatchedDataDict -from nemo_rl.distributed.virtual_cluster import RayVirtualCluster -from nemo_rl.models.generation.dynamo.config import DynamoConfig -from nemo_rl.models.generation.dynamo.http_client import ( - async_http_post_json, - format_dynamo_error, -) -from nemo_rl.models.generation.dynamo.managed_runtime import ManagedDynamoRuntime -from nemo_rl.models.generation.dynamo.metrics import DynamoMetricsSampler -from nemo_rl.models.generation.dynamo.refit import DynamoRefitChannel -from nemo_rl.models.generation.dynamo.token_wrapper import DynamoTokenWrapperServer -from nemo_rl.models.generation.interfaces import ( - CollectiveSenderSpec, - GenerationDatumSpec, - GenerationInterface, - GenerationOutputSpec, - reject_unenforceable_refit_deadline, - verify_right_padding, -) -from nemo_rl.models.megatron.router_replay import router_replay_enabled - -if TYPE_CHECKING: - from nemo_rl.algorithms.single_controller_utils.config import MasterConfig - -LOGGER = logging.getLogger(__name__) - -_HTTP_MAX_ATTEMPTS = 3 -_HTTP_RETRY_DELAY_S = 1.0 -_RETRYABLE_HTTP_STATUS_CODES = {408, 429} - - -def _is_retryable_http_response(response: Any) -> bool: - """Return whether an internal HTTP error shape represents a transient error.""" - if not isinstance(response, dict): - return False - if "transport_error" in response or response.get("json_decode_error") is True: - return True - status = response.get("http_status") - return isinstance(status, int) and ( - status in _RETRYABLE_HTTP_STATUS_CODES or 500 <= status < 600 - ) - - -def _parse_dynamo_completion_response( - response: dict[str, Any], *, request_url: str -) -> tuple[list[int], list[float], bool]: - """Parse the Dynamo OpenAI completion response for direct generation.""" - if not isinstance(response, dict): - raise RuntimeError( - f"Dynamo completion response from {request_url} was not a JSON object." - ) - if response.get("status") == "error": - raise RuntimeError( - f"Dynamo completion request to {request_url} failed: " - f"{format_dynamo_error(response)}" - ) - - choices = response.get("choices") - if not isinstance(choices, list) or not choices: - raise RuntimeError( - f"Dynamo completion response from {request_url} did not include choices." - ) - choice = choices[0] - if not isinstance(choice, dict): - raise RuntimeError( - f"Dynamo completion response from {request_url} has invalid choice shape." - ) - - nvext = response.get("nvext") - if not isinstance(nvext, dict): - raise RuntimeError( - f"Dynamo completion response from {request_url} did not include nvext." - ) - completion_token_ids = nvext.get("completion_token_ids") - if not isinstance(completion_token_ids, list): - raise RuntimeError( - "Dynamo completion response did not include " - "nvext.completion_token_ids. Ensure the Dynamo frontend is " - "configured to return completion token IDs." - ) - generated_token_ids = [int(token_id) for token_id in completion_token_ids] - - if not generated_token_ids: - return ( - generated_token_ids, - [], - choice.get("finish_reason") == "length", - ) - - logprobs = choice.get("logprobs") - if not isinstance(logprobs, dict): - raise RuntimeError( - f"Dynamo completion response from {request_url} did not include " - "choice.logprobs." - ) - token_logprobs = logprobs.get("token_logprobs") - if not isinstance(token_logprobs, list): - raise RuntimeError( - f"Dynamo completion response from {request_url} did not include " - "choice.logprobs.token_logprobs." - ) - if len(token_logprobs) != len(generated_token_ids): - raise RuntimeError( - f"Dynamo completion response from {request_url} returned " - f"{len(token_logprobs)} token logprobs for " - f"{len(generated_token_ids)} generated tokens." - ) - - generated_logprobs = [] - for idx, logprob in enumerate(token_logprobs): - if not isinstance(logprob, (int, float)) or isinstance(logprob, bool): - raise RuntimeError( - f"Dynamo completion response from {request_url} returned invalid " - f"logprob {logprob!r} for generated token {idx}." - ) - generated_logprobs.append(float(logprob)) - - return ( - generated_token_ids, - generated_logprobs, - choice.get("finish_reason") == "length", - ) - - -class DynamoGeneration(GenerationInterface): - """Own a fixed Dynamo service fleet and expose it for NeMo-RL rollouts.""" - - @classmethod - def validate_settings(cls, master_config: "MasterConfig") -> None: - """Reject SingleController settings the managed Dynamo backend cannot honor.""" - # Both would otherwise fail only after the engines are up: R3 in the Megatron - # worker's setup, a VLM policy at its first multimodal prompt. - if router_replay_enabled(master_config.policy): - raise ValueError( - "policy.router_replay.enabled=true requires " - "policy.generation.backend='vllm'; managed Dynamo does not return " - "routed experts." - ) - if master_config.policy.get("is_vlm"): - raise ValueError( - "policy.is_vlm=true is not supported with " - "policy.generation.backend='dynamo'; managed Dynamo serves text " - "prompts only." - ) - fleet_health = master_config.async_rl.generation_fleet_health - if fleet_health.enabled: - # The Dynamo frontend routes every request across its worker fleet, so - # there are no NeMo RL-selected shards for fleet health to track. - raise ValueError( - "async_rl.generation_fleet_health.enabled=true is not supported with " - "policy.generation.backend='dynamo'; the Dynamo frontend owns request " - "routing. Set async_rl.generation_fleet_health.enabled=false." - ) - if fleet_health.refit_timeout_s is not None: - # Dynamo workers cannot abort their refit collective, so a deadline would - # be refused at the first weight update; fail before anything builds. - raise ValueError( - "async_rl.generation_fleet_health.refit_timeout_s must be null with " - "policy.generation.backend='dynamo'; Dynamo workers cannot enforce a " - "refit deadline." - ) - # Setup has already normalized this through DynamoConfig. - generation = cast(dict[str, Any], master_config.policy["generation"]) - dynamo_cfg = generation["dynamo_cfg"] - if dynamo_cfg["control_timeout_s"] < dynamo_cfg["request_timeout_s"]: - raise ValueError( - "policy.generation.dynamo_cfg.control_timeout_s " - f"({dynamo_cfg['control_timeout_s']}) must be >= request_timeout_s " - f"({dynamo_cfg['request_timeout_s']}): it bounds the pre-refit drain, " - "which waits for running requests to finish." - ) - if ( - generation["vllm_cfg"]["enable_vllm_metrics_logger"] - and master_config.rollout_checkpointing.telemetry_interval_s is None - ): - # The rollout telemetry pump is the only reader of Dynamo's metrics on - # this path; without it the flag would silently do nothing. - raise ValueError( - "policy.generation.vllm_cfg.enable_vllm_metrics_logger requires " - "rollout_checkpointing.telemetry_interval_s with " - "policy.generation.backend='dynamo' under the SingleController; " - "the rollout telemetry pump is what reports Dynamo's metrics." - ) - - def __init__( - self, - cluster: Optional[RayVirtualCluster], - config: dict[str, Any], - tokenizer: Any | None = None, - tokenizer_config: Optional[dict[str, Any]] = None, - ): - validated_config = DynamoConfig.model_validate(config) - self.cfg = validated_config.model_dump() - self._dynamo_cfg = validated_config.dynamo_cfg - dynamo_cfg = self._dynamo_cfg - vllm_cfg = validated_config.vllm_cfg - expose_http_server = vllm_cfg.expose_http_server - tokenizer_chat_template_kwargs: Optional[dict[str, Any]] = None - if expose_http_server: - if tokenizer is None: - raise RuntimeError( - "DynamoGeneration requires a tokenizer when exposing an " - "OpenAI-compatible rollout server." - ) - if ( - tokenizer_config is not None - and "chat_template_kwargs" in tokenizer_config - and tokenizer_config["chat_template_kwargs"] is not None - ): - chat_template_kwargs = tokenizer_config["chat_template_kwargs"] - if not isinstance(chat_template_kwargs, dict): - raise RuntimeError( - "policy.tokenizer.chat_template_kwargs must be a dictionary." - ) - tokenizer_chat_template_kwargs = dict(chat_template_kwargs) - if cluster is None: - raise RuntimeError( - "Managed Dynamo requires a non-colocated inference RayVirtualCluster." - ) - self._managed_runtime: Optional[ManagedDynamoRuntime] = ManagedDynamoRuntime( - cluster=cluster, - config=self.cfg, - ) - self._token_wrapper_server: Optional[DynamoTokenWrapperServer] = None - self._dynamo_frontend_base_url = "" - self.dp_openai_server_base_urls: list[Optional[str]] = [] - self._refit_channel: DynamoRefitChannel | None = None - self._metrics_sampler: DynamoMetricsSampler | None = None - try: - self._managed_runtime.start() - url = self._managed_runtime.frontend_url - self._dynamo_frontend_base_url = url - workers = self._managed_runtime.refit_workers() - self._refit_channel = DynamoRefitChannel( - workers, - engine_world_size=validated_config.engine_world_size, - control_timeout_s=dynamo_cfg.control_timeout_s, - validate_workers=self._managed_runtime.validate_workers, - ) - - if expose_http_server: - self._token_wrapper_server = DynamoTokenWrapperServer( - dynamo_frontend_base_url=url, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=( - dynamo_cfg.worker_args.exclude_tools_when_tool_choice_none - ), - request_timeout_s=dynamo_cfg.request_timeout_s, - ) - wrapper_url = self._token_wrapper_server.start() - self.dp_openai_server_base_urls = [wrapper_url] - print( - " [Dynamo] Forwarding rollout chat requests through token " - f"wrapper {wrapper_url} -> {url}", - flush=True, - ) - else: - self.dp_openai_server_base_urls = [None] - print(f" [Dynamo] Forwarding rollouts to {url}", flush=True) - except Exception: - self.shutdown() - raise - - # ------------------------------------------------------------------ - # GenerationInterface — lifecycle - # ------------------------------------------------------------------ - - def prepare_for_generation(self, *args: Any, **kwargs: Any) -> bool: - return True - - @property - def frontend_url(self) -> str: - """Return the internal managed Dynamo OpenAI frontend URL.""" - if not self._dynamo_frontend_base_url: - raise RuntimeError("DynamoGeneration does not have a frontend URL.") - return self._dynamo_frontend_base_url - - def finish_generation(self, *args: Any, **kwargs: Any) -> bool: - """Invalidate cached rollout state after synchronous generation.""" - return self.invalidate_kv_cache() - - def _logger_sampler(self) -> DynamoMetricsSampler | None: - """Return this process's metrics sampler, starting it on first use. - - The sampler owns a thread and locks, so it cannot travel with a pickled - copy: it starts in whichever process reads the metrics (the driver for - the legacy loops, the SingleController actor for its telemetry pump). - """ - vllm_cfg = self.cfg["vllm_cfg"] - channel = self._refit_channel - if ( - self._metrics_sampler is None - and vllm_cfg["enable_vllm_metrics_logger"] - and channel is not None - ): - self._metrics_sampler = DynamoMetricsSampler( - channel.workers, - interval_s=vllm_cfg["vllm_metrics_logger_interval"], - include_prefixes=self._dynamo_cfg.metrics_include_prefixes, - exclude_prefixes=self._dynamo_cfg.metrics_exclude_prefixes, - ) - self._metrics_sampler.start() - return self._metrics_sampler - - def get_logger_metrics(self) -> dict[str, Any]: - """Return per-worker Dynamo metric timelines for generation logging.""" - sampler = self._logger_sampler() - return {} if sampler is None else sampler.snapshot() - - def clear_logger_metrics(self) -> None: - """Clear the Dynamo metric timelines for the next logging window.""" - sampler = self._logger_sampler() - if sampler is not None: - sampler.clear() - - def drain_latest_logger_metrics(self) -> dict[str, Any]: - """Return each worker's latest Dynamo metrics for rollout telemetry.""" - sampler = self._logger_sampler() - return {} if sampler is None else sampler.drain_latest() - - def get_inference_world_size(self) -> int: - """Return the number of vLLM ranks across all discovered workers.""" - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.inference_world_size - - def get_collective_sender_spec(self) -> CollectiveSenderSpec: - """Return vLLM's NCCL protocol and packed-transfer geometry.""" - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.sender_spec - - def shutdown(self) -> bool: - """Stop process-local helpers and any driver-owned managed runtime.""" - sampler = self._metrics_sampler - self._metrics_sampler = None - if sampler is not None: - try: - sampler.shutdown() - except Exception: - LOGGER.exception("Failed to stop the Dynamo metrics sampler") - token_wrapper_server = self._token_wrapper_server - self._token_wrapper_server = None - if token_wrapper_server is not None: - try: - token_wrapper_server.shutdown() - except Exception: - LOGGER.exception("Failed to stop the Dynamo token wrapper") - managed_runtime = self._managed_runtime - self._managed_runtime = None - if managed_runtime is not None: - try: - managed_runtime.shutdown() - except Exception: - LOGGER.exception("Failed to stop the managed Dynamo runtime") - self._refit_channel = None - return True - - # ------------------------------------------------------------------ - # Pickling — async rollouts ship the GenerationInterface across Ray actors - # ------------------------------------------------------------------ - - def __getstate__(self) -> dict[str, Any]: - """Serialize only HTTP clients needed by Ray rollout actors. - - Driver-owned subprocesses, threads, and Ray worker handles are excluded. - The refit channel's client copy is retained so weight updates and - AREAL-style cache invalidation still reach every managed worker after - deserialization. - """ - refit_channel = self._refit_channel - return { - "cfg": self.cfg, - "dp_openai_server_base_urls": self.dp_openai_server_base_urls, - "_dynamo_frontend_base_url": self._dynamo_frontend_base_url, - "_refit_channel": ( - None if refit_channel is None else refit_channel.client_copy() - ), - } - - def __setstate__(self, state: dict[str, Any]) -> None: - """Restore a client-only rollout copy with no service ownership.""" - self.cfg = state["cfg"] - validated_config = DynamoConfig.model_validate(self.cfg) - self._dynamo_cfg = validated_config.dynamo_cfg - self.dp_openai_server_base_urls = state["dp_openai_server_base_urls"] - frontend_url = state["_dynamo_frontend_base_url"] - if not isinstance(frontend_url, str) or not frontend_url: - raise RuntimeError("Pickled DynamoGeneration has no frontend URL.") - self._dynamo_frontend_base_url = frontend_url - self._token_wrapper_server = None - self._managed_runtime = None - self._metrics_sampler = None - self._refit_channel = state["_refit_channel"] - - def _completion_url(self) -> str: - base_url = self._dynamo_frontend_base_url - if not base_url: - raise RuntimeError("DynamoGeneration does not have a frontend URL.") - return f"{base_url.rstrip('/')}/completions" - - def _request_timeout_s(self) -> float: - return self._dynamo_cfg.request_timeout_s - - def _merge_stop_strings(self, batch_stop_strings: Any) -> Optional[list[str]]: - stop_set: set[str] = set() - - configured_stop_strings = self.cfg.get("stop_strings") - if configured_stop_strings is not None: - stop_set.update(configured_stop_strings) - - if batch_stop_strings is not None: - for sample_stop_strings in batch_stop_strings: - if not sample_stop_strings: - continue - if isinstance(sample_stop_strings, str): - stop_set.add(sample_stop_strings) - else: - stop_set.update(sample_stop_strings) - - if len(stop_set) > 32: - raise ValueError( - "Dynamo supports at most 32 stop strings after merging configured " - "and per-sample values" - ) - return list(stop_set) if stop_set else None - - def _prompt_token_ids( - self, - data: BatchedDataDict["GenerationDatumSpec"], - sample_idx: int, - ) -> list[int]: - if VLLM_PROMPT_KEYS & data.keys(): - raise NotImplementedError( - "DynamoGeneration direct generate() supports token-ID LLM " - "prompts only; multimodal vLLM prompt data is not supported." - ) - - input_length = int(data["input_lengths"][sample_idx].item()) - return data["input_ids"][sample_idx, :input_length].tolist() - - def _build_completion_request( - self, - *, - prompt_token_ids: list[int], - greedy: bool, - stop_strings: Optional[list[str]], - max_new_tokens: int, - ) -> dict[str, Any]: - top_k_cfg = self.cfg["top_k"] - top_k_val = 1 if greedy else (top_k_cfg if top_k_cfg is not None else -1) - - payload: dict[str, Any] = { - "model": self.cfg["model_name"], - "prompt": prompt_token_ids, - "max_tokens": int(max_new_tokens), - "temperature": 0.0 if greedy else self.cfg["temperature"], - "top_p": self.cfg["top_p"], - "top_k": top_k_val, - "n": 1, - "logprobs": 0, - "include_stop_str_in_output": True, - "nvext": {"extra_fields": ["completion_token_ids"]}, - } - - if self.cfg["stop_token_ids"] is not None: - payload["stop_token_ids"] = self.cfg["stop_token_ids"] - if stop_strings is not None: - payload["stop"] = stop_strings - - return payload - - def _allowed_new_tokens(self, input_length: int) -> int: - """Return the generation budget for a prompt.""" - remaining_ctx = int(self.cfg["vllm_cfg"]["max_model_len"]) - input_length - if remaining_ctx <= 0: - raise ValueError( - f"Dynamo prompt length {input_length} must be less than " - f"vllm_cfg.max_model_len={self.cfg['vllm_cfg']['max_model_len']}" - ) - return min(self.cfg["max_new_tokens"], remaining_ctx) - - def _assert_response_within_context( - self, *, input_length: int, generated_length: int - ) -> None: - response_length = input_length + generated_length - max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) - if response_length > max_model_len: - raise AssertionError( - "Dynamo response length exceeded " - f"vllm_cfg.max_model_len: {response_length} > {max_model_len}" - ) - - async def _post_completion_request( - self, - *, - prompt_token_ids: list[int], - greedy: bool, - stop_strings: Optional[list[str]], - max_new_tokens: int, - ) -> tuple[list[int], list[float], bool]: - request_url = self._completion_url() - payload = self._build_completion_request( - prompt_token_ids=prompt_token_ids, - greedy=greedy, - stop_strings=stop_strings, - max_new_tokens=max_new_tokens, - ) - response: dict[str, Any] = {} - for attempt in range(1, _HTTP_MAX_ATTEMPTS + 1): - response = await async_http_post_json( - request_url, - payload, - self._request_timeout_s(), - ) - if not _is_retryable_http_response(response): - break - if attempt == _HTTP_MAX_ATTEMPTS: - break - LOGGER.warning( - "Dynamo completion attempt %d/%d failed; retrying in %.1fs: %s", - attempt, - _HTTP_MAX_ATTEMPTS, - _HTTP_RETRY_DELAY_S, - format_dynamo_error(response), - ) - await asyncio.sleep(_HTTP_RETRY_DELAY_S) - return _parse_dynamo_completion_response(response, request_url=request_url) - - def _single_sample_output( - self, - *, - input_ids: torch.Tensor, - input_length: int, - generated_token_ids: list[int], - generated_logprobs: list[float], - truncated: bool, - ) -> BatchedDataDict["GenerationOutputSpec"]: - output_length = input_length + len(generated_token_ids) - self._assert_response_within_context( - input_length=input_length, - generated_length=len(generated_token_ids), - ) - output_ids = torch.full( - (output_length,), - self.cfg["_pad_token_id"], - dtype=input_ids.dtype, - device=input_ids.device, - ) - output_ids[:input_length] = input_ids[:input_length] - if generated_token_ids: - output_ids[input_length:output_length] = torch.tensor( - generated_token_ids, - dtype=input_ids.dtype, - device=input_ids.device, - ) - - logprobs = torch.zeros( - (1, output_length), - dtype=torch.float32, - device=input_ids.device, - ) - for idx, logprob in enumerate(generated_logprobs[: len(generated_token_ids)]): - logprobs[0, input_length + idx] = logprob - - return BatchedDataDict[GenerationOutputSpec]( - { - "output_ids": output_ids.unsqueeze(0), - "logprobs": logprobs, - "generation_lengths": torch.tensor( - [len(generated_token_ids)], - dtype=torch.long, - device=input_ids.device, - ), - "unpadded_sequence_lengths": torch.tensor( - [output_length], - dtype=torch.long, - device=input_ids.device, - ), - "truncated": torch.tensor( - [truncated], - dtype=torch.bool, - device=input_ids.device, - ), - } - ) - - def generate( - self, - data: BatchedDataDict["GenerationDatumSpec"], - greedy: bool = False, - ) -> BatchedDataDict["GenerationOutputSpec"]: - """Reject the unused blocking interface. - - Both synchronous and asynchronous GRPO trainers use ``generate_async`` - for the managed HTTP frontend. - """ - raise NotImplementedError( - "Dynamo generation uses generate_async() for both synchronous and " - "asynchronous GRPO trainers" - ) - - async def generate_async( - self, - data: BatchedDataDict["GenerationDatumSpec"], - greedy: bool = False, - ) -> AsyncGenerator[tuple[int, BatchedDataDict["GenerationOutputSpec"]], None]: - """Generate one token-ID prompt asynchronously through the managed frontend.""" - assert isinstance(data, BatchedDataDict), ( - f"data must be a BatchedDataDict, got type: {type(data)}" - ) - assert "input_ids" in data and "input_lengths" in data, ( - "input_ids and input_lengths are required in data for Dynamo generation" - ) - if len(data["input_ids"]) == 0: - return - - verify_right_padding(data, pad_value=self.cfg["_pad_token_id"]) - - input_ids_batch = data["input_ids"] - input_lengths_batch = data["input_lengths"] - batch_size = input_ids_batch.shape[0] - assert batch_size == 1, ( - "generate_async is restricted to handle only single samples, " - f"but received batch_size={batch_size}. Please handle batching " - "outside this method." - ) - sample_idx = 0 - input_length = int(input_lengths_batch[sample_idx].item()) - batch_stop_strings = data.get("stop_strings", [[] for _ in range(batch_size)]) - per_sample_stop_strings = None - if batch_stop_strings and sample_idx < len(batch_stop_strings): - per_sample_stop_strings = batch_stop_strings[sample_idx] - final_stop_strings = self._merge_stop_strings( - [per_sample_stop_strings] if per_sample_stop_strings else None - ) - - allowed_new_tokens = self._allowed_new_tokens(input_length) - input_ids = input_ids_batch[sample_idx] - ( - generated_token_ids, - generated_logprobs, - truncated, - ) = await self._post_completion_request( - prompt_token_ids=self._prompt_token_ids(data, sample_idx), - greedy=greedy, - stop_strings=final_stop_strings, - max_new_tokens=allowed_new_tokens, - ) - - yield ( - sample_idx, - self._single_sample_output( - input_ids=input_ids, - input_length=input_length, - generated_token_ids=generated_token_ids, - generated_logprobs=generated_logprobs, - truncated=truncated, - ), - ) - - def init_collective( - self, - ip: str, - port: int, - world_size: int, - *, - train_world_size: int, - ) -> list[ray.ObjectRef]: - """Initialize native vLLM NCCL transfer on every managed worker.""" - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.init_collective( - ip, - port, - world_size, - train_world_size=train_world_size, - ) - - def prepare_refit_info( - self, state_dict_info: Optional[dict[str, Any]] - ) -> Optional[list[str]]: - """Serialize checkpoint-format tensor metadata for native vLLM refit.""" - if state_dict_info is None: - return None - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - channel.prepare(state_dict_info) - return None - - def update_weights_via_ipc_zmq(self) -> list[ray.ObjectRef]: - raise NotImplementedError( - "DynamoGeneration only supports NCCL weight transfer." - ) - - def update_weights_from_collective( - self, refit_timeout_s: Optional[float] = None - ) -> list[ray.ObjectRef]: - """Receive packed checkpoint-format weights on every Dynamo worker.""" - reject_unenforceable_refit_deadline("Dynamo", refit_timeout_s) - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.update_weights() - - def invalidate_kv_cache(self) -> bool: - """Flush every fixed Dynamo worker's prefix/KV cache.""" - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.flush_cache() - - def pause_generation_for_refit( - self, *, clear_cache: bool, timeout_s: Optional[float] = None - ) -> bool: - """Drain and pause every worker; layerwise reload is unsafe mid-request. - - Unlike vLLM's frozen pause, running requests finish on the current - weights; queued and new requests wait and run on the new ones. Because - the drain waits for running requests, ``dynamo_cfg.control_timeout_s`` - bounds it rather than ``timeout_s``. - """ - del timeout_s - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - channel.pause(clear_cache=clear_cache) - return True - - def resume_generation_after_refit( - self, *, timeout_s: Optional[float] = None - ) -> bool: - """Resume every worker after a weight update.""" - del timeout_s - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - channel.resume() - return True diff --git a/nemo_rl/models/generation/dynamo/dynamo_worker.py b/nemo_rl/models/generation/dynamo/dynamo_worker.py deleted file mode 100644 index 624d58ca09a..00000000000 --- a/nemo_rl/models/generation/dynamo/dynamo_worker.py +++ /dev/null @@ -1,304 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Ray actors used by the fixed, managed Dynamo vLLM fleet.""" - -import json -import os -import signal -import socket -import subprocess -import time -from pathlib import Path -from typing import Any - -import ray - -from nemo_rl.distributed.virtual_cluster import _get_free_port_local, _get_node_ip_local -from nemo_rl.models.generation.dynamo.arguments import ( - build_dynamo_vllm_argv, - build_managed_worker_env, - redact_argv, - redact_environment, -) -from nemo_rl.models.generation.dynamo.config import ( - VLLM_PACKED_BUFFER_SIZE_BYTES, - VLLM_PACKED_NUM_BUFFERS, - DynamoConfig, -) -from nemo_rl.models.generation.dynamo.venv import ( - get_dynamo_python, - get_dynamo_venv_dir, -) - - -@ray.remote(num_cpus=0) -class DynamoGpuReservation: # pragma: no cover - """Hold one placement-group GPU while a sibling actor owns the engine.""" - - def __init__(self) -> None: - self._process_pid: int | None = None - - def metadata(self) -> dict[str, Any]: - gpu_ids = [int(float(gpu_id)) for gpu_id in ray.get_gpu_ids()] - if len(gpu_ids) != 1: - raise RuntimeError( - f"Expected one GPU for Dynamo reservation, got {gpu_ids}." - ) - return {"node_ip": _get_node_ip_local(), "gpu_id": gpu_ids[0]} - - def select_free_port( - self, - *, - port_range_low: int, - port_range_high: int, - excluded_ports: list[int], - ) -> int: - """Select an unused node-local port from a half-open range.""" - return _get_free_port_local( - port_range_low, - port_range_high, - max_retries=None, - excluded_ports=set(excluded_ports), - ) - - def register_process_group(self, pid: int) -> bool: - """Record the colocated worker process group for failure cleanup.""" - if self._process_pid not in (None, pid): - raise RuntimeError( - "Dynamo GPU reservation already owns process group " - f"{self._process_pid}; cannot register {pid}." - ) - self._process_pid = pid - return True - - def cleanup_process_group(self) -> bool: - """Best-effort cleanup if the subprocess-owning actor died first.""" - pid = self._process_pid - if pid is None: - return True - try: - os.killpg(pid, signal.SIGTERM) - except ProcessLookupError: - self._process_pid = None - return True - time.sleep(2) - try: - os.killpg(pid, signal.SIGKILL) - except ProcessLookupError: - pass - self._process_pid = None - return True - - -@ray.remote(num_cpus=0) -class DynamoVllmWorker: # pragma: no cover - """Own one ``dynamo.vllm`` subprocess for a model-parallel GPU group.""" - - def __init__( - self, - config: dict[str, Any], - *, - namespace: str, - group_name: str, - cuda_devices: list[int], - system_port: int, - vllm_port: int, - manager_env: dict[str, str], - startup_timeout_s: float, - seed: int, - cleanup_reservation: ray.actor.ActorHandle, - ) -> None: - self._group_name = group_name - self._node_ip = _get_node_ip_local() - self._system_port = system_port - self._vllm_port = vllm_port - self._process: subprocess.Popen | None = None - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: - probe.bind(("0.0.0.0", system_port)) - except OSError as exc: - raise RuntimeError( - f"DYN_SYSTEM_PORT {system_port} is unavailable for {group_name}." - ) from exc - - validated_config = DynamoConfig.model_validate(config) - dynamo_cfg = validated_config.dynamo_cfg - dynamo_python = get_dynamo_python() - dynamo_venv = str(get_dynamo_venv_dir()) - vllm_cfg = validated_config.vllm_cfg.model_dump() - vllm_kwargs = dict(validated_config.vllm_kwargs) - configured_env = dict(vllm_cfg.get("env_vars") or {}) - worker_env = build_managed_worker_env( - base_env=os.environ, - configured_env=configured_env, - manager_env={ - **manager_env, - "CUDA_VISIBLE_DEVICES": ",".join(str(gpu) for gpu in cuda_devices), - "DYN_SYSTEM_PORT": str(system_port), - "PYTHONHASHSEED": "0", - "VLLM_PORT": str(vllm_port), - "VLLM_SKIP_P2P_CHECK": "1", - "VIRTUAL_ENV": dynamo_venv, - "UV_PROJECT_ENVIRONMENT": dynamo_venv, - }, - ) - argv = build_dynamo_vllm_argv( - model_name=config["model_name"], - namespace=namespace, - seed=seed, - vllm_cfg=vllm_cfg, - vllm_kwargs=vllm_kwargs, - dynamo_cfg=dynamo_cfg, - ) - self._validate_argv( - dynamo_python, - argv, - worker_env, - timeout_s=startup_timeout_s, - ) - - command = [dynamo_python, "-m", "dynamo.vllm", *argv] - relevant_env = { - key: value - for key, value in worker_env.items() - if key.startswith(("CUDA_", "DYN_", "ETCD_", "NATS_", "NCCL_", "VLLM_")) - } - print( - f" [Dynamo:{group_name}] launching argv={redact_argv(command)!r} " - f"env={redact_environment(relevant_env)!r} " - f"system_url={self.system_url}", - flush=True, - ) - try: - self._process = subprocess.Popen( - command, env=worker_env, start_new_session=True - ) - ray.get( - cleanup_reservation.register_process_group.remote(self._process.pid) - ) - self._wait_for_system_port(startup_timeout_s) - except Exception: - self._stop_process() - raise - - @property - def system_url(self) -> str: - host = f"[{self._node_ip}]" if ":" in self._node_ip else self._node_ip - return f"http://{host}:{self._system_port}" - - @staticmethod - def _validate_argv( - dynamo_python: str, - argv: list[str], - env: dict[str, str], - *, - timeout_s: float, - ) -> None: - validator = Path(__file__).with_name("validate_dynamo_vllm_args.py") - command = [ - dynamo_python, - str(validator), - json.dumps(argv), - json.dumps( - { - "buffer_size_bytes": VLLM_PACKED_BUFFER_SIZE_BYTES, - "num_buffers": VLLM_PACKED_NUM_BUFFERS, - } - ), - ] - try: - result = subprocess.run( - command, - env=env, - capture_output=True, - text=True, - check=False, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as error: - raise RuntimeError( - "Resolved dynamo.vllm argument validation exceeded " - f"startup_timeout_s={timeout_s}." - ) from error - if result.returncode != 0: - raise RuntimeError( - "Resolved dynamo.vllm arguments failed validation: " - f"stdout={result.stdout!r}, stderr={result.stderr!r}" - ) - - def _wait_for_system_port(self, timeout_s: float) -> None: - deadline = time.monotonic() + timeout_s - while time.monotonic() < deadline: - process = self._process - if process is not None and process.poll() is not None: - raise RuntimeError( - f"dynamo.vllm for {self._group_name} exited with " - f"code {process.returncode} before its system endpoint was ready." - ) - try: - with socket.create_connection( - ("127.0.0.1", self._system_port), timeout=1 - ): - return - except OSError: - time.sleep(0.5) - raise RuntimeError( - f"dynamo.vllm for {self._group_name} did not open DYN_SYSTEM_PORT " - f"{self._system_port} within {timeout_s}s." - ) - - def metadata(self) -> dict[str, Any]: - process = self._process - if process is None: - raise RuntimeError(f"dynamo.vllm for {self._group_name} is not running.") - return { - "instance_id": self._group_name, - "system_url": self.system_url, - "process_pid": process.pid, - "vllm_port": self._vllm_port, - } - - def is_alive(self) -> bool: - return self._process is not None and self._process.poll() is None - - def _stop_process(self) -> None: - process = self._process - if process is None: - return - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass - if process.poll() is None: - try: - process.wait(timeout=15) - except subprocess.TimeoutExpired: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.wait(timeout=5) - self._process = None - - def shutdown(self) -> bool: - process = self._process - if process is None: - return True - self._stop_process() - print( - f" [Dynamo:{self._group_name}] stopped pid={process.pid}", - flush=True, - ) - return True diff --git a/nemo_rl/models/generation/dynamo/http_client.py b/nemo_rl/models/generation/dynamo/http_client.py index 8f69bee6b4d..b7d81f3de15 100644 --- a/nemo_rl/models/generation/dynamo/http_client.py +++ b/nemo_rl/models/generation/dynamo/http_client.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Small HTTP client helpers shared by managed Dynamo components.""" +"""HTTP helpers for requests to the managed Dynamo frontend.""" import json -import urllib.error -import urllib.request from typing import Any import aiohttp @@ -37,31 +35,6 @@ def _decode_json_object(body: bytes) -> dict[str, Any]: return decoded -def http_post_json( - url: str, payload: dict[str, Any], timeout_s: float -) -> dict[str, Any]: - """POST JSON and return either the decoded object or an error object.""" - data = json.dumps(payload).encode("utf-8") - request = urllib.request.Request( - url, - data=data, - headers={"Content-Type": "application/json"}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=timeout_s) as response: - body = response.read() - except urllib.error.HTTPError as error: - raw = error.read().decode("utf-8", "replace") if error.fp else "" - return {"status": "error", "http_status": error.code, "raw": raw} - except (urllib.error.URLError, TimeoutError) as error: - return { - "status": "error", - "transport_error": f"{type(error).__name__}: {error}", - } - return _decode_json_object(body) - - async def async_http_post_json( url: str, payload: dict[str, Any], timeout_s: float ) -> dict[str, Any]: @@ -77,7 +50,13 @@ async def async_http_post_json( "http_status": response.status, "raw": body.decode("utf-8", "replace"), } - except (aiohttp.ClientError, TimeoutError) as error: + except TimeoutError: + # Before ClientError: aiohttp's ServerTimeoutError is both. + return { + "status": "error", + "timeout_error": f"no response within {timeout_s}s", + } + except aiohttp.ClientError as error: return { "status": "error", "transport_error": f"{type(error).__name__}: {error}", @@ -86,11 +65,13 @@ async def async_http_post_json( def format_dynamo_error(response: dict[str, Any]) -> str: - """Format an error object returned by :func:`http_post_json`.""" + """Format an error object returned by :func:`async_http_post_json`.""" if "http_status" in response: return f"HTTP {response['http_status']}: {response.get('raw', '')}" if "transport_error" in response: return str(response["transport_error"]) + if "timeout_error" in response: + return str(response["timeout_error"]) if "raw" in response: return str(response["raw"]) return str(response) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index 34e0eb77696..df75a7d9f12 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Driver-owned lifecycle for a fixed Ray-managed Dynamo deployment.""" +"""Driver-owned Dynamo services: etcd, NATS and the OpenAI frontend. + +The vLLM engines are NeMo RL's own workers; each registers itself with these +services (see ``native_engine``), so the runtime only waits to see them. +""" import atexit import json @@ -27,14 +31,13 @@ import time import urllib.error import urllib.request -from typing import Any +from typing import Callable from nemo_rl.distributed.virtual_cluster import ( DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH, DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, - RayVirtualCluster, _get_free_port_local, _get_node_ip_local, ) @@ -43,14 +46,21 @@ redact_argv, redact_environment, ) -from nemo_rl.models.generation.dynamo.config import DynamoConfig -from nemo_rl.models.generation.dynamo.venv import ( - get_dynamo_executable, - get_dynamo_python, -) -from nemo_rl.models.generation.dynamo.worker_pool import FixedDynamoWorkerPool +from nemo_rl.models.generation.dynamo.config import DynamoCfg +from nemo_rl.models.generation.dynamo.venv import get_dynamo_service_executable + +# The engines register on this component and endpoint (see native_engine). +DYNAMO_COMPONENT = "backend" +DYNAMO_ENDPOINT = "generate" LOGGER = logging.getLogger(__name__) +# The KV router's applied-event counter (Dynamo's kvrouter.KV_CACHE_EVENTS_APPLIED), +# one sample per label set. +_KV_EVENTS_APPLIED = re.compile( + r'^\w*kv_cache_events_applied(?:_total)?\{(?=[^}]*status="(?P[^"]*)")' + r"[^}]*\}\s+(?P\S+)$", + re.MULTILINE, +) def _managed_namespace() -> str: @@ -62,25 +72,22 @@ def _managed_namespace() -> str: class ManagedDynamoRuntime: - """Own etcd, NATS, frontend, and a fixed Ray actor worker fleet.""" + """Own etcd, NATS and the frontend that NeMo RL's vLLM engines register with.""" def __init__( - self, - *, - cluster: RayVirtualCluster, - config: dict[str, Any], + self, *, dynamo_cfg: DynamoCfg, model_name: str, frontend_python: str ) -> None: - validated_config = DynamoConfig.model_validate(config) - self._cluster = cluster - self._config = validated_config.model_dump() - self._dynamo_cfg = validated_config.dynamo_cfg - self._engine_world_size = validated_config.engine_world_size - if self._engine_world_size > cluster.num_gpus_per_node: - raise ValueError( - "Managed Dynamo requires each TP/PP engine group to fit on one " - f"node: tp*pp={self._engine_world_size} exceeds " - f"cluster.num_gpus_per_node={cluster.num_gpus_per_node}" - ) + """Describe the services; nothing starts until :meth:`start`. + + Args: + dynamo_cfg: The validated ``policy.generation.dynamo_cfg``. + model_name: The served model; the frontend is ready once it lists it. + frontend_python: An interpreter whose environment has ``ai-dynamo``: + the vLLM workers' own. + """ + self._dynamo_cfg = dynamo_cfg + self._model_name = model_name + self._frontend_python = frontend_python self._namespace = _managed_namespace() self._host = "" self._etcd_port = 0 @@ -95,7 +102,10 @@ def __init__( self._frontend_process: subprocess.Popen | None = None self._etcd_data_dir: str | None = None self._nats_data_dir: str | None = None - self._pool: FixedDynamoWorkerPool | None = None + + @property + def namespace(self) -> str: + return self._namespace @property def frontend_url(self) -> str: @@ -104,13 +114,20 @@ def frontend_url(self) -> str: host = f"[{self._host}]" if ":" in self._host else self._host return f"http://{host}:{self._frontend_port}/v1" + @property + def engine_env(self) -> dict[str, str]: + """The environment each engine's Dynamo worker needs to register.""" + if not self._started: + raise RuntimeError("Managed Dynamo runtime has not been started") + return dict(self._manager_env) + def start(self) -> None: - """Start the complete managed service fleet.""" + """Start etcd, NATS and the frontend.""" if self._started: raise RuntimeError("Managed Dynamo runtime is already started") # Managed services use new process groups and would survive an - # exception that escapes GRPO setup. Keep an interpreter-exit fallback - # in addition to the normal explicit shutdown path. + # exception that escapes setup. Keep an interpreter-exit fallback in + # addition to the normal explicit shutdown path. atexit.register(self.shutdown) self._atexit_registered = True self._host = _get_node_ip_local() @@ -147,15 +164,16 @@ def allocate_port(*, low: int, high: int) -> int: "NATS_SERVER": f"nats://{self._host}:{self._nats_port}", "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", - # Dynamo 1.3.0's legacy tool jail rebuilds response chunks with - # nvext=None. Its qwen3/deepseek v2 path preserves engine_data, - # which NeMo-Gym needs for exact token IDs and log probabilities. + # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. + # Its v2 parser path preserves engine_data, which NeMo-Gym needs for + # exact token IDs and log probabilities. "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", + # A canary request would stall behind a refit pause; engine liveness + # is the Ray actor's, as for every other vLLM worker. "DYN_HEALTH_CHECK_ENABLED": "false", "DYN_SDK_DISABLE_ANSI_LOGGING": "1", - "DYN_RL_INIT_WEIGHTS_TIMEOUT_S": str(self._dynamo_cfg.control_timeout_s), } print( f" [Dynamo] managed environment={redact_environment(self._manager_env)!r}", @@ -164,17 +182,7 @@ def allocate_port(*, low: int, high: int) -> int: try: self._start_etcd() self._start_nats() - self._pool = FixedDynamoWorkerPool( - cluster=self._cluster, - config=self._config, - namespace=self._namespace, - engine_world_size=self._engine_world_size, - manager_env=self._manager_env, - startup_timeout_s=self._dynamo_cfg.startup_timeout_s, - ) - self._pool.start() self._start_frontend() - self._wait_for_frontend(self._pool.size) self._started = True except Exception: self.shutdown() @@ -223,7 +231,7 @@ def _start_etcd(self) -> None: self._etcd_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_etcd_") peer_url = f"http://{self._host}:{self._etcd_peer_port}" command = [ - get_dynamo_executable("etcd"), + get_dynamo_service_executable("etcd"), "--listen-client-urls", f"http://0.0.0.0:{self._etcd_port}", "--advertise-client-urls", @@ -263,7 +271,7 @@ def _start_nats(self) -> None: self._nats_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_nats_") self._nats_process = subprocess.Popen( [ - get_dynamo_executable("nats-server"), + get_dynamo_service_executable("nats-server"), "-js", "-sd", self._nats_data_dir, @@ -298,7 +306,7 @@ def _start_frontend(self) -> None: dynamo_cfg=self._dynamo_cfg, ) command = [ - get_dynamo_python(), + self._frontend_python, "-m", "dynamo.frontend", *argv, @@ -307,6 +315,9 @@ def _start_frontend(self) -> None: f" [Dynamo] launching frontend argv={redact_argv(command)!r}", flush=True ) frontend_env = self._frontend_env() + dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) + frontend_env["VIRTUAL_ENV"] = dynamo_venv + frontend_env["UV_PROJECT_ENVIRONMENT"] = dynamo_venv tokenizer_env = { key: value for key, value in frontend_env.items() @@ -320,102 +331,66 @@ def _start_frontend(self) -> None: command, env=frontend_env, start_new_session=True ) - def _wait_for_frontend(self, expected_workers: int) -> None: + def wait_for_engines( + self, expected_engines: int, check_engines: Callable[[], None] + ) -> None: + """Block until the frontend routes to ``expected_engines`` registered engines. + + ``/health`` lists discovery registrations before the frontend's model + watcher has installed its OpenAI routes, so readiness also needs the + served model on ``/v1/models``; otherwise an immediate request can race + the watcher and get a transient 404. + + ``check_engines`` runs on every poll and raises when an engine exited or + its endpoint stopped, so a failed registration fails at once. + """ health_url = f"http://127.0.0.1:{self._frontend_port}/health" models_url = f"http://127.0.0.1:{self._frontend_port}/v1/models" - expected_model = str(self._config["model_name"]) deadline = time.monotonic() + self._dynamo_cfg.startup_timeout_s - last_counts = (0, 0) - last_models: set[str] = set() + registered = 0 + served: set[str] = set() while time.monotonic() < deadline: - if self._pool is None or not self._pool.is_alive(): - raise RuntimeError( - "A Ray-managed Dynamo vLLM worker exited while the frontend " - "was waiting for model registration." - ) - if ( - self._frontend_process is not None - and self._frontend_process.poll() is not None - ): - raise RuntimeError( - f"Dynamo frontend exited with code {self._frontend_process.returncode}." - ) + self._assert_services_alive() + check_engines() try: with urllib.request.urlopen(health_url, timeout=5) as response: - payload = json.loads(response.read()) + health = json.loads(response.read()) + with urllib.request.urlopen(models_url, timeout=5) as response: + models = json.loads(response.read()) except (urllib.error.URLError, TimeoutError, json.JSONDecodeError): time.sleep(1) continue - generate_ids: set[str] = set() - rl_ids: set[str] = set() - for instance in payload.get("instances", []): - if not isinstance(instance, dict): - continue - if instance.get("namespace") != self._namespace: - continue - if instance.get("component") != "backend": - continue - instance_id = instance.get("instance_id") - if instance_id is None: - continue - if instance.get("endpoint") == "generate": - generate_ids.add(str(instance_id)) - elif instance.get("endpoint") == "rl": - rl_ids.add(str(instance_id)) - last_counts = (len(generate_ids), len(rl_ids)) - if ( - last_counts == (expected_workers, expected_workers) - and generate_ids == rl_ids - ): - # /health reflects discovery registrations before the frontend's - # model watcher has necessarily installed its OpenAI routes. Do - # not expose frontend_url until the served model is visible; - # otherwise an immediate /v1/completions request can race with - # watcher setup and receive a transient 404. - try: - with urllib.request.urlopen(models_url, timeout=5) as response: - models_payload = json.loads(response.read()) - except ( - urllib.error.URLError, - TimeoutError, - json.JSONDecodeError, - ): - time.sleep(1) - continue - last_models = { - str(model["id"]) - for model in models_payload.get("data", []) - if isinstance(model, dict) and model.get("id") is not None + registered = len( + { + str(instance["instance_id"]) + for instance in health.get("instances", []) + if isinstance(instance, dict) + and instance.get("namespace") == self._namespace + and instance.get("component") == DYNAMO_COMPONENT + and instance.get("endpoint") == DYNAMO_ENDPOINT + and instance.get("instance_id") is not None } - if expected_model not in last_models: - time.sleep(1) - continue + ) + served = { + str(model["id"]) + for model in models.get("data", []) + if isinstance(model, dict) and model.get("id") is not None + } + if registered >= expected_engines and self._model_name in served: print( - f" [Dynamo] frontend ready with {expected_workers} generation " - "and RL workers", + f" [Dynamo] frontend {self.frontend_url} routes to " + f"{registered} engine(s)", flush=True, ) return time.sleep(1) raise RuntimeError( - "Dynamo frontend did not observe the fixed worker fleet within " - f"{self._dynamo_cfg.startup_timeout_s}s: expected={expected_workers}, " - f"last_generate={last_counts[0]}, last_rl={last_counts[1]}, " - f"expected_model={expected_model!r}, last_models={sorted(last_models)!r}." + "Dynamo frontend did not see every vLLM engine within " + f"{self._dynamo_cfg.startup_timeout_s}s: expected={expected_engines}, " + f"registered={registered}, model={self._model_name!r}, " + f"served={sorted(served)!r}." ) - def refit_workers(self) -> list[dict[str, Any]]: - self._assert_services_alive() - if self._pool is None: - raise RuntimeError("Managed Dynamo worker pool is not running.") - return self._pool.refit_workers() - - def validate_workers(self, expected: list[dict[str, Any]]) -> list[dict[str, Any]]: - self._assert_services_alive() - if self._pool is None: - raise RuntimeError("Managed Dynamo worker pool is not running.") - return self._pool.validate(expected) - def _assert_services_alive(self) -> None: for label, process in ( ("etcd", self._etcd_process), @@ -428,20 +403,42 @@ def _assert_services_alive(self) -> None: f"Managed Dynamo {label} is not alive (code={code})." ) + def _log_router_kv_events(self) -> None: + """Log how many engine KV-cache events the KV router applied, by status. + + Read from the frontend's metrics before it stops. A KV router that never + applied one routed on load alone. + """ + if self._frontend_process is None or self._frontend_process.poll() is not None: + return + url = f"http://127.0.0.1:{self._frontend_port}/metrics" + try: + with urllib.request.urlopen(url, timeout=5) as response: + metrics = response.read().decode() + except (urllib.error.URLError, TimeoutError) as error: + print( + f" [Dynamo] router KV events: metrics unavailable ({error})", + flush=True, + ) + return + applied: dict[str, float] = {} + for match in _KV_EVENTS_APPLIED.finditer(metrics): + applied[match["status"]] = applied.get(match["status"], 0.0) + float( + match["value"] + ) + print(f" [Dynamo] router applied KV events: {json.dumps(applied)}", flush=True) + def shutdown(self) -> None: """Best-effort, idempotent teardown of every owned resource.""" + try: + self._log_router_kv_events() + except Exception: + LOGGER.exception("Failed to read the Dynamo router's KV-event metrics") try: self._stop_process(self._frontend_process, "frontend") except Exception: LOGGER.exception("Failed to stop the managed Dynamo frontend") self._frontend_process = None - pool = self._pool - self._pool = None - if pool is not None: - try: - pool.shutdown() - except Exception: - LOGGER.exception("Failed to stop the managed Dynamo worker pool") try: self._stop_process(self._nats_process, "NATS") except Exception: diff --git a/nemo_rl/models/generation/dynamo/metrics.py b/nemo_rl/models/generation/dynamo/metrics.py deleted file mode 100644 index daf7a611c32..00000000000 --- a/nemo_rl/models/generation/dynamo/metrics.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Prometheus polling lifecycle for managed Dynamo workers.""" - -import logging -import threading -import urllib.error -import urllib.request -from collections.abc import Sequence -from typing import Any - -from nemo_rl.models.generation.dynamo.refit import DynamoWorkerEndpoint - -LOGGER = logging.getLogger(__name__) - -DEFAULT_METRICS_EXCLUDE_PREFIXES = ("python_", "process_") -CURATED_METRICS_INCLUDE_PREFIXES = ( - "dynamo_component_gpu_cache_usage", - "dynamo_component_inflight_requests", - "dynamo_work_handler_queue_depth", - "dynamo_component_requests_total", - "dynamo_work_handler_time_to_first_response", - "vllm:num_requests_running", - "vllm:num_requests_waiting", - "vllm:kv_cache_usage_perc", - "vllm:generation_tokens", - "vllm:prompt_tokens_total", - "vllm:inter_token_latency", -) -CANONICAL_LOGGER_ALIASES = { - "inflight_batch_sizes": [ - "dynamo_component_inflight_requests", - "vllm_num_requests_running", - ], - "num_pending_samples": [ - "dynamo_work_handler_queue_depth", - "vllm_num_requests_waiting", - ], - "kv_cache_usage_perc": [ - "dynamo_component_gpu_cache_usage_percent", - "vllm_kv_cache_usage_perc", - "vllm_gpu_cache_usage_perc", - ], - "generation_tokens": [ - "vllm_generation_tokens_total", - "vllm_generation_tokens", - ], -} - - -def _http_get_text(url: str, timeout_s: float) -> str | None: - try: - with urllib.request.urlopen(url, timeout=timeout_s) as response: - return response.read().decode("utf-8", "replace") - except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError): - return None - - -def parse_prometheus_metrics( - text: str, - include_prefixes: tuple[str, ...] | None = None, - exclude_prefixes: tuple[str, ...] = DEFAULT_METRICS_EXCLUDE_PREFIXES, -) -> dict[str, float]: - """Parse Prometheus text exposition into summed scalar values.""" - metrics: dict[str, float] = {} - for raw_line in text.splitlines(): - line = raw_line.strip() - if not line or line.startswith("#"): - continue - if "{" in line: - name = line[: line.index("{")] - try: - value_text = line[line.rindex("}") + 1 :] - except ValueError: - continue - else: - parts = line.split(None, 1) - if len(parts) != 2: - continue - name, value_text = parts - if name.endswith(("_bucket", "_created")): - continue - if include_prefixes and not name.startswith(include_prefixes): - continue - if exclude_prefixes and name.startswith(exclude_prefixes): - continue - value_parts = value_text.split() - if not value_parts: - continue - try: - value = float(value_parts[0]) - except ValueError: - continue - key = name.replace(":", "_") - metrics[key] = metrics.get(key, 0.0) + value - return metrics - - -def _with_canonical_aliases(metrics: dict[str, Any]) -> dict[str, Any]: - """Rename the first available source of each canonical logger metric.""" - for alias, sources in CANONICAL_LOGGER_ALIASES.items(): - if alias in metrics: - continue - source = next((name for name in sources if name in metrics), None) - metrics[alias] = dict(metrics[source]) if source is not None else {} - if source is not None: - del metrics[source] - return metrics - - -class DynamoMetricsSampler: - """Own polling thread, samples, and deterministic sampler shutdown.""" - - def __init__( - self, - workers: Sequence[dict[str, Any] | DynamoWorkerEndpoint], - *, - interval_s: float, - include_prefixes: list[str] | None, - exclude_prefixes: list[str] | None, - ) -> None: - self._workers = tuple( - DynamoWorkerEndpoint.from_metadata(worker) - if isinstance(worker, dict) - else worker - for worker in workers - ) - if not self._workers: - raise ValueError("Managed Dynamo metrics require at least one worker") - self._interval_s = interval_s - self._include_prefixes = ( - tuple(include_prefixes) - if include_prefixes is not None - else CURATED_METRICS_INCLUDE_PREFIXES - ) - self._exclude_prefixes = ( - tuple(exclude_prefixes) - if exclude_prefixes is not None - else DEFAULT_METRICS_EXCLUDE_PREFIXES - ) - self._samples: dict[str, dict[int, list[float]]] = {} - self._lock = threading.Lock() - self._stop = threading.Event() - self._thread: threading.Thread | None = None - - def start(self) -> None: - if self._thread is not None: - raise RuntimeError("Dynamo metrics sampler is already started") - self._thread = threading.Thread( - target=self._run, - name="dynamo-metrics-sampler", - daemon=True, - ) - self._thread.start() - - def _run(self) -> None: - self._stop.wait(min(2.0, self._interval_s)) - while not self._stop.is_set(): - for ordinal, worker in enumerate(self._workers): - text = _http_get_text( - f"{worker.system_url}/metrics", - timeout_s=self._interval_s + 2.0, - ) - if self._stop.is_set(): - break - if not text: - continue - metrics = parse_prometheus_metrics( - text, - self._include_prefixes, - self._exclude_prefixes, - ) - with self._lock: - for name, value in metrics.items(): - self._samples.setdefault(name, {}).setdefault( - ordinal, [] - ).append(value) - self._stop.wait(self._interval_s) - - def snapshot(self) -> dict[str, Any]: - with self._lock: - metrics = self._copy_samples() - return _with_canonical_aliases(metrics) - - def drain_latest(self) -> dict[str, Any]: - """Return each worker's latest sample of every metric and drop older ones. - - The SingleController's rollout telemetry polls this, as it does the vLLM - backend's ``drain_latest_logger_metrics``, which keeps histories bounded. - """ - with self._lock: - self._samples = { - name: { - worker_id: samples[-1:] - for worker_id, samples in worker_metrics.items() - } - for name, worker_metrics in self._samples.items() - } - metrics = self._copy_samples() - return _with_canonical_aliases(metrics) - - def _copy_samples(self) -> dict[str, dict[int, list[float]]]: - return { - name: { - worker_id: list(samples) - for worker_id, samples in worker_metrics.items() - } - for name, worker_metrics in self._samples.items() - } - - def clear(self) -> None: - with self._lock: - self._samples = {} - - def shutdown(self) -> None: - self._stop.set() - thread = self._thread - if ( - thread is not None - and thread is not threading.current_thread() - and thread.is_alive() - ): - thread.join(timeout=max(5.0, self._interval_s + 3.0)) - if thread.is_alive(): - LOGGER.warning("Dynamo metrics sampler did not stop before shutdown") - self._thread = None diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py new file mode 100644 index 00000000000..2eb085d3e5d --- /dev/null +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -0,0 +1,930 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serve a NeMo RL vLLM worker's own AsyncLLM through Dynamo's backend SDK. + +The engine process holds no Dynamo-owned vLLM code. This adapter implements +``dynamo.common.backend.LLMEngine``, which imports no vLLM, over the AsyncLLM +that ``VllmAsyncGenerationWorkerImpl`` owns, so refit, pause, sleep, +quantization and speculative decoding are that worker's own. +""" + +from __future__ import annotations + +import asyncio +import functools +import hashlib +import json +import logging +import os +import tempfile +import time +from collections.abc import AsyncGenerator +from typing import TYPE_CHECKING, Any, Optional + +import numpy as np +import torch +from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names +from dynamo.common.backend import ( + EngineConfig, + GenerateChunk, + GenerateRequest, + LLMEngine, + LlmRegistration, + Worker, + WorkerConfig, +) +from dynamo.common.backend.logprobs import ( + extract_from_completion_output, + extract_prompt_logprobs_from_completion_output, + parse_logprob_options, +) +from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource +from dynamo.llm.exceptions import HttpError, InvalidArgument +from vllm import envs as vllm_envs +from vllm.entrypoints.chat_utils import load_chat_template +from vllm.exceptions import GracefulHTTPError +from vllm.inputs import TokensPrompt +from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY +from vllm.sampling_params import ( + RequestOutputKind, + SamplingParams, + StructuredOutputsParams, +) +from vllm.transformers_utils.processor import get_video_processor_cls_name +from vllm.v1.metrics.loggers import StatLoggerBase + +from nemo_rl.data.captured_media import MediaCaptureRejected +from nemo_rl.distributed.virtual_cluster import ( + DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH, + DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, + _get_free_port_local, +) +from nemo_rl.models.generation.dynamo.config import ( + DynamoCfg, + dynamo_cfg, + dynamo_chat_template, + dynamo_parsers, +) +from nemo_rl.models.generation.dynamo.managed_runtime import ( + DYNAMO_COMPONENT, + DYNAMO_ENDPOINT, +) +from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION +from nemo_rl.models.generation.openai_server_utils import ( + derive_prefix_eos_count, + splice_prefix_tokens, +) +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_result, + rollout_result_to_wire, +) +from nemo_rl.models.generation.vllm.utils import ( + chat_generation_tokens, + chat_routed_experts, + off_policy_sampling_error, + remap_multimodal_placeholders, + validate_rollout_prompt, +) + +if TYPE_CHECKING: + from nemo_rl.models.generation.vllm.vllm_worker_async import ( + VllmAsyncGenerationWorkerImpl, + ) + +LOGGER = logging.getLogger(__name__) + +# Annotations NeMo-Gym's Dynamo dialect (vllm_model with dynamo_frontend, Gym#4817) +# sets on its requests; Dynamo forwards nvext.annotations to the engine verbatim. +GYM_ROLLOUT_ANNOTATION = "nemo_gym:rollout" +# A token-capture call's admission JSON. +GYM_CAPTURE_ANNOTATION_PREFIX = "ng_capture:" +# Without capture: the last echoed assistant turn's prompt + generation token +# IDs, which the native server reads from the messages instead. +GYM_PREFIX_ANNOTATION_PREFIX = "ng_prefix:" +_GYM_SAMPLING_KEYS = ("temperature", "top_p", "val_temperature", "val_top_p") + +# The vLLM modality each key of Dynamo's multi_modal_data holds. +_DYNAMO_MEDIA_MODALITIES = { + "image_url": "image", + "video_url": "video", + "audio_url": "audio", +} + +# Filled from the model's generation config when a request leaves them unset, +# as vLLM's OpenAI server fills them. +_GENERATION_CONFIG_KEYS = ( + "repetition_penalty", + "temperature", + "top_k", + "top_p", + "min_p", +) +_SAMPLING_KEYS = ( + *_GENERATION_CONFIG_KEYS, + "seed", + "presence_penalty", + "frequency_penalty", +) + + +def dynamo_kv_events_config( + cfg: DynamoCfg, *, prefix_caching: bool +) -> Optional[dict[str, Any]]: + """The ``kv_events_config`` that feeds Dynamo's KV router. + + None unless the frontend routes by KV and the engine caches prefixes: vLLM + publishes block events only for the blocks its prefix cache stores. + """ + if cfg.frontend_args.router_mode != "kv" or not prefix_caching: + return None + port = _get_free_port_local( + DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH + ) + return { + "enable_kv_cache_events": True, + "publisher": "zmq", + # A wildcard host makes vLLM's publisher bind; any other TCP endpoint + # makes it connect, and Dynamo's ZMQ source connects too, so no event + # would flow. kv_event_sources subscribes on 127.0.0.1. + "endpoint": f"tcp://*:{port}", + } + + +def _annotation_value(request: GenerateRequest, prefix: str) -> Optional[str]: + """The value of the first ```` annotation, or None.""" + for annotation in request.get("annotations") or (): + if annotation.startswith(prefix): + return annotation[len(prefix) :] + return None + + +def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: + """The rollout spec if ``request`` is one of NeMo RL's own rollouts. + + Those come from ``VllmGeneration.generate_async`` and are built here + exactly as the native path builds them. + """ + rollout = _annotation_value(request, ROLLOUT_ANNOTATION) + return None if rollout is None else json.loads(rollout) + + +def _as_f32(value: Optional[float]) -> Optional[float]: + """Dynamo's frontend carries sampling floats as f32 (0.7 arrives as 0.69999998).""" + return None if value is None else float(np.float32(value)) + + +def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: + """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. + + Dynamo sets at most one constraint. The decoding backend is the engine's + own (``structured_outputs_config``), as for the native server. + """ + structural_tag = guided_decoding.get("structural_tag") + if isinstance(structural_tag, dict): + structural_tag = json.dumps(structural_tag) + return StructuredOutputsParams( + json=guided_decoding.get("json"), + regex=guided_decoding.get("regex"), + choice=guided_decoding.get("choice"), + grammar=guided_decoding.get("grammar"), + whitespace_pattern=guided_decoding.get("whitespace_pattern"), + structural_tag=structural_tag, + ) + + +def build_sampling_params( + request: GenerateRequest, + defaults: dict[str, Any], + *, + output_room: int, + overrides: Optional[dict[str, Any]] = None, + output_kind: RequestOutputKind = RequestOutputKind.DELTA, +) -> SamplingParams: + """Map an OpenAI-shaped Dynamo request onto vLLM SamplingParams. + + As vLLM's OpenAI server builds them: the model's generation-config + ``defaults`` for what the request leaves unset, and ``max_tokens`` capped + by the context. Stop strings are applied by Dynamo's Rust detokenizer, so + only token-level stops reach vLLM unless ``overrides`` sets them. + + Args: + request: Dynamo's preprocessed request. + defaults: ``ModelConfig.get_diff_sampling_param()``. + output_room: The context left after the prompt the engine runs, which + media expansion or a prefix splice can make longer than the + request's ``token_ids``. + overrides: SamplingParams fields that replace the request's. + output_kind: How vLLM reports outputs. + """ + sampling = request.get("sampling_options") or {} + passthrough = (request.get("extra_args") or {}).get("sampling_options") or {} + stop = request.get("stop_conditions") or {} + n = sampling.get("n") or 1 + if sampling.get("best_of") not in (None, n): + raise InvalidArgument("best_of is not supported by vLLM's V1 engine") + + kwargs: dict[str, Any] = { + key: defaults[key] for key in _GENERATION_CONFIG_KEYS if key in defaults + } + kwargs.update( + {key: sampling[key] for key in _SAMPLING_KEYS if sampling.get(key) is not None} + ) + kwargs["n"] = n + kwargs["max_tokens"] = min( + limit + for limit in ( + max(1, output_room), + stop.get("max_tokens"), + defaults.get("max_tokens"), + ) + if limit is not None + ) + if stop.get("min_tokens") is not None: + kwargs["min_tokens"] = stop["min_tokens"] + if stop.get("ignore_eos") is not None: + kwargs["ignore_eos"] = stop["ignore_eos"] + stop_ids = set(stop.get("stop_token_ids") or ()) | set( + stop.get("stop_token_ids_hidden") or () + ) + if stop_ids: + kwargs["stop_token_ids"] = sorted(stop_ids) + if sampling.get("guided_decoding"): + kwargs["structured_outputs"] = structured_outputs(sampling["guided_decoding"]) + for key in ("allowed_token_ids", "logprob_token_ids"): + if passthrough.get(key) is not None: + kwargs[key] = passthrough[key] + + logprobs, prompt_logprobs = parse_logprob_options( + request.get("output_options") or {} + ) + # Explicit logprob_token_ids replace top-k selection; vLLM rejects both. + if logprobs is not None and "logprob_token_ids" not in kwargs: + kwargs["logprobs"] = logprobs + if prompt_logprobs is not None: + kwargs["prompt_logprobs"] = prompt_logprobs + kwargs["detokenize"] = False + kwargs.update(overrides or {}) + params = SamplingParams(**kwargs, output_kind=output_kind) + if passthrough.get("bad_words_token_ids") is not None: + # vLLM only tokenizes bad_words strings into this field; a token-in + # client sends the token IDs themselves. + if not hasattr(params, "_bad_words_token_ids"): + raise RuntimeError( + "vLLM SamplingParams has no _bad_words_token_ids; update the " + "bad_words_token_ids passthrough for this vLLM" + ) + params._bad_words_token_ids = passthrough["bad_words_token_ids"] + return params + + +class SnapshotStatLogger(StatLoggerBase): + """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" + + publisher: Optional[Any] = None + num_gpu_blocks: int = 0 + + def __init__(self, vllm_config: Any, engine_index: int = 0) -> None: + self.dp_rank = engine_index + + def record(self, scheduler_stats, iteration_stats, *args, **kwargs) -> None: + cls = type(self) + if scheduler_stats is None or cls.publisher is None: + return + total = max(1, cls.num_gpu_blocks) + usage = scheduler_stats.kv_cache_usage + hit_rate = None + prefix = getattr(scheduler_stats, "prefix_cache_stats", None) + if prefix is not None and prefix.queries > 0: + hit_rate = prefix.hits / prefix.queries + cls.publisher.publish( + self.dp_rank, + ComponentSnapshot( + kv_used_blocks=int(total * usage), + kv_total_blocks=total, + gpu_cache_usage=usage, + kv_cache_hit_rate=hit_rate, + dp_rank=self.dp_rank, + ), + ) + + def log_engine_initialized(self) -> None: + pass + + +class NativeVllmEngine(LLMEngine): + """LLMEngine over a running NeMo RL vLLM worker. Does not own its lifecycle.""" + + def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int): + self._host = host + self._leader_worker_idx = leader_worker_idx + self._llm = host.llm + self._max_model_len = self._llm.vllm_config.model_config.max_model_len + self._defaults = self._llm.model_config.get_diff_sampling_param() + # The train and validation sampling NeMo-Gym rollouts must match, at the + # f32 precision the frontend delivers. + self._gym_sampling_profile = { + key: _as_f32(host.cfg[key]) for key in _GYM_SAMPLING_KEYS + } + + @classmethod + async def from_args(cls, argv=None): + raise NotImplementedError("constructed in-process by the vLLM worker") + + async def start(self, worker_id: int) -> EngineConfig: + del worker_id + vllm_config = self._llm.vllm_config + cache = vllm_config.cache_config + scheduler = vllm_config.scheduler_config + # The KV-event block size is the main-attention group's, which can + # differ from cache_config.block_size on hybrid models. + block_size = cache.block_size + for group in await self._llm.engine_core.call_utility_async( + "get_kv_cache_group_metadata" + ): + if group.get("kind") in ("full_attention", "mla_attention"): + block_size = group["block_size"] + break + SnapshotStatLogger.num_gpu_blocks = cache.num_gpu_blocks or 0 + model = self._host.model_name + return EngineConfig( + model=model, + served_model_name=model, + # No token budget: the engine owns prompt overflow and the output + # cap, as the native server does. A budget would make the frontend + # fill an omitted max_tokens from the prompt it rendered, which a + # NeMo-Gym prefix splice then changes, and refuse NeMo RL's own + # rollouts that fill the context. + llm=LlmRegistration( + context_length=self._max_model_len, + kv_cache_block_size=block_size, + total_kv_blocks=cache.num_gpu_blocks, + max_num_seqs=scheduler.max_num_seqs, + max_num_batched_tokens=scheduler.max_num_batched_tokens, + ), + ) + + async def generate( + self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + if request.get("prompt_embeds"): + raise InvalidArgument( + "Embedding prompts are not served through Dynamo by NeMo RL's " + "vLLM engines." + ) + rollout = _rollout_request(request) + if rollout is not None: + async for chunk in self._generate_rollout(request, rollout, context): + yield chunk + return + annotations = request.get("annotations") or () + if GYM_ROLLOUT_ANNOTATION in annotations: + async for chunk in self._generate_gym(request, context): + yield chunk + return + engine_input = await self._engine_input(request) + prompt_len = len(engine_input["prompt_token_ids"]) + self._check_prompt_fits(prompt_len) + sampling_params = build_sampling_params( + request, self._defaults, output_room=self._max_model_len - prompt_len + ) + try: + # vLLM's queue limits (max_num_queued_reqs / max_num_queued_tokens), + # checked before every request as the native server checks them. + self._llm.check_admission(sampling_params.n, context.id()) + except GracefulHTTPError as error: + raise HttpError(int(error.http_status), error.message) from error + logprobs_requested, prompt_logprobs_requested = parse_logprob_options( + request.get("output_options") or {} + ) + tokenizer = self._llm.tokenizer + completion_tokens: dict[int, int] = {} + prompt_output = None + async for output in self._llm.generate( + engine_input, sampling_params, context.id() + ): + # Under DELTA, vLLM sends every prompt logprob once, at the end of + # prefill; later outputs carry an empty list. + if getattr(output, "prompt_logprobs", None): + prompt_output = output + for completion in output.outputs: + token_ids = list(completion.token_ids) + completion_tokens[completion.index] = completion_tokens.get( + completion.index, 0 + ) + len(token_ids) + if not token_ids and completion.finish_reason is None: + continue + chunk: GenerateChunk = { + "token_ids": token_ids, + "index": completion.index, + } + log_probs, top_logprobs = extract_from_completion_output( + completion, 0, tokenizer=tokenizer, include_bytes=True + ) + if log_probs is not None: + chunk["log_probs"] = log_probs + if logprobs_requested: + chunk["top_logprobs"] = top_logprobs + if completion.finish_reason is not None: + chunk["finish_reason"] = completion.finish_reason + chunk["completion_usage"] = _usage( + output, sum(completion_tokens.values()) + ) + if prompt_logprobs_requested is not None and prompt_output: + prompt_logprobs = ( + extract_prompt_logprobs_from_completion_output( + prompt_output, tokenizer=tokenizer + ) + ) + if prompt_logprobs is not None: + chunk["engine_data"] = {"prompt_logprobs": prompt_logprobs} + yield chunk + + async def _generate_rollout( + self, request: GenerateRequest, rollout: dict[str, Any], context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + """One NeMo RL rollout, served exactly as ``generate_async`` serves it. + + The worker builds the SamplingParams and the shared request pipeline + builds the result, routed experts included, from vLLM's final output. + One chunk carries it all in ``engine_data``; the driver assembles the + same output ``generate_async`` returns. + """ + host = self._host + prompt_token_ids = list(request["token_ids"]) + max_new_tokens = host._rollout_max_new_tokens(len(prompt_token_ids)) + final_output = None + if max_new_tokens == 0: + result = RolloutResult.nothing_generated() + else: + sampling_params = host._rollout_sampling_params( + max_new_tokens=max_new_tokens, + greedy=rollout["greedy"], + stop_strings=rollout["stop_strings"], + ) + async for output in self._llm.generate( + TokensPrompt(prompt_token_ids=prompt_token_ids), + sampling_params, + context.id(), + ): + final_output = output + if final_output is None: + raise RuntimeError(f"No output received for request {context.id()}") + validate_rollout_prompt(prompt_token_ids, final_output.prompt_token_ids) + result = rollout_result( + final_output, + input_length=len(prompt_token_ids), + return_routed_experts=host._return_routed_experts_enabled(), + routed_experts_dtype=host.routed_experts_dtype, + device=torch.device("cpu"), + ) + chunk: GenerateChunk = { + "token_ids": result.token_ids, + "index": 0, + "finish_reason": ( + "length" + if final_output is None + else final_output.outputs[0].finish_reason + ), + "engine_data": { + "rollout": rollout_result_to_wire(result), + "gen_leader_worker_idx": self._leader_worker_idx, + }, + } + if final_output is not None: + chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) + yield chunk + + async def _generate_gym( + self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + """One NeMo-Gym model call, served as the native chat endpoint serves it. + + The request was rendered by the frontend. As on the native server, it must + sample on-policy, its echoed or staged prefix replaces the rendered + history, output is clamped to the context, and a token-capture call stages + its delta from this process. Tokens, log probabilities and routed experts + come from vLLM's final output through the native endpoint's helpers. One + chunk carries the whole completion and, in ``engine_data``, the token + arrays Gym's dialect reads, or a captured call's commit coordinates. + """ + host = self._host + sampling: dict[str, Any] = request.get("sampling_options") or {} + sampling_error = off_policy_sampling_error( + temperature=_as_f32(sampling.get("temperature")), + top_p=_as_f32(sampling.get("top_p")), + top_k=sampling.get("top_k"), + generation_config=self._gym_sampling_profile, + ) + if sampling_error is not None: + raise InvalidArgument(sampling_error) + if (sampling.get("n") or 1) != 1: + raise InvalidArgument( + "NeMo-Gym calls through Dynamo take n=1: engine_data carries one " + "choice's tokens." + ) + # The profile's own values, so vLLM samples exactly as natively. + validation = ( + _as_f32(sampling["temperature"]) + != self._gym_sampling_profile["temperature"] + or _as_f32(sampling["top_p"]) != self._gym_sampling_profile["top_p"] + ) + temperature, top_p = ( + (host.cfg["val_temperature"], host.cfg["val_top_p"]) + if validation + else (host.cfg["temperature"], host.cfg["top_p"]) + ) + + capture_context = _annotation_value(request, GYM_CAPTURE_ANNOTATION_PREFIX) + admission = ( + None + if capture_context is None + else host._parse_capture_admission(json.loads(capture_context)) + ) + prefix_token_ids = None + if admission is not None: + if admission.mode == "token_in": + # A blocking data-plane read; keep it off the engine loop. + prefix_token_ids = await asyncio.to_thread( + host._resolve_admission_prefix, admission + ) + elif ( + echoed := _annotation_value(request, GYM_PREFIX_ANNOTATION_PREFIX) + ) is not None: + prefix_token_ids = json.loads(echoed) + # As the native endpoint does: a captured call reads the processor's own + # pixels, not cache references (#4124). + engine_input = await self._engine_input( + request, skip_mm_cache=admission is not None and host._capture_media + ) + template_token_ids = engine_input["prompt_token_ids"] + self._check_prompt_fits(len(template_token_ids)) + token_ids, splice = template_token_ids, None + if prefix_token_ids is not None: + # The frontend rendered the whole conversation; replace its history + # with the exact tokens the model generated for it. The engine sees no + # messages, so the prefix's EOS count stands in for the native + # endpoint's render of the history. + tokenizer = self._llm.tokenizer + splice = splice_prefix_tokens( + tokenizer=tokenizer, + model_prefix_token_ids=prefix_token_ids, + template_token_ids=template_token_ids, + template_prefix_eos_count=derive_prefix_eos_count( + prefix_token_ids, eos_token_id=tokenizer.eos_token_id + ), + ) + token_ids = splice.token_ids + self._check_prompt_fits(len(token_ids)) + try: + # Off the loop: a retained-media check reads staged calls. + media = await asyncio.to_thread( + host._capture_request_media, + engine_input, + admission=admission, + splice=splice, + ) + except MediaCaptureRejected as error: + raise InvalidArgument(f"{error} ({error.code})") from error + if ( + media is None + and splice is not None + and (placeholders := engine_input.get("mm_placeholders")) + ): + # Media capture already moved them to spliced coordinates. + engine_input["mm_placeholders"] = remap_multimodal_placeholders( + template_token_ids=template_token_ids, + final_token_ids=token_ids, + mm_placeholders=placeholders, + splice=splice, + ) + engine_input["prompt_token_ids"] = token_ids + + stop = (request.get("stop_conditions") or {}).get("stop") + overrides: dict[str, Any] = { + "temperature": temperature, + "top_p": top_p, + # As the native endpoint does: Gym requests leave top_k to NeMo RL. + "top_k": -1, + } + if stop: + # vLLM stops on them as natively, so the token arrays match; the + # frontend then cuts the text at the same place. + overrides.update(stop=stop, detokenize=True) + if admission is not None: + # A captured call stages the sampled tokens' log probabilities. + overrides["logprobs"] = 0 + sampling_params = build_sampling_params( + request, + self._defaults, + output_room=self._max_model_len - len(token_ids), + overrides=overrides, + output_kind=RequestOutputKind.FINAL_ONLY, + ) + try: + self._llm.check_admission(1, context.id()) + except GracefulHTTPError as error: + raise HttpError(int(error.http_status), error.message) from error + + if admission is not None: + host._begin_request_capture( + request, + token_ids, + admission=admission, + prefix_token_ids=prefix_token_ids, + media=media, + ) + staged = False + try: + final_output = None + async for output in self._llm.generate( + engine_input, sampling_params, context.id() + ): + final_output = output + if final_output is None: + raise RuntimeError(f"No output received for request {context.id()}") + completion = final_output.outputs[0] + chunk: GenerateChunk = { + "token_ids": list(completion.token_ids), + "index": 0, + "finish_reason": completion.finish_reason, + "completion_usage": _usage(final_output, len(completion.token_ids)), + } + if admission is not None: + chunk["engine_data"] = { + "ng_commit_coords": await self._stage_gym_call( + request, final_output + ) + } + staged = True + else: + chunk["engine_data"] = self._gym_token_data(final_output) + log_probs, top_logprobs = extract_from_completion_output( + completion, 0, tokenizer=self._llm.tokenizer, include_bytes=True + ) + if log_probs is not None: + chunk["log_probs"] = log_probs + chunk["top_logprobs"] = top_logprobs + yield chunk + finally: + if admission is not None and not staged: + host._abort_request_capture( + request, + reason=( + "client_disconnect" if context.is_stopped() else "engine_error" + ), + ) + + async def _load_media( + self, request: GenerateRequest + ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: + """The request's media and their UUIDs by modality, as vLLM's chat parser loads them. + + Dynamo passes each media URL through (data: URIs included); the engine + fetches and decodes it with vLLM's own media connector and settings. + """ + media = request.get("multi_modal_data") + if not media: + return None + model_config = self._llm.model_config + mm_config = model_config.multimodal_config + connector = MEDIA_CONNECTOR_REGISTRY.load( + vllm_envs.VLLM_MEDIA_CONNECTOR, + media_io_kwargs=request.get("media_io_kwargs") + or (mm_config.media_io_kwargs if mm_config else None), + allowed_local_media_path=model_config.allowed_local_media_path, + allowed_media_domains=model_config.allowed_media_domains, + ) + fetch = { + "image": connector.fetch_image_async, + "audio": connector.fetch_audio_async, + } + if "video_url" in media: + fetch["video"] = functools.partial( + connector.fetch_video_async, + video_processor=get_video_processor_cls_name(model_config), + ) + uuids = request.get("multi_modal_uuids") or {} + mm_data: dict[str, list[Any]] = {} + mm_uuids: dict[str, list[Any]] = {} + for key, items in media.items(): + modality = _DYNAMO_MEDIA_MODALITIES.get(key) + if modality is None: + raise InvalidArgument(f"Unsupported media type {key!r}") + if any("Url" not in item for item in items): + raise InvalidArgument( + f"{key} items must carry their URL: NeMo RL's vLLM engines load " + "media themselves" + ) + try: + mm_data[modality] = list( + await asyncio.gather( + *(fetch[modality](item["Url"]) for item in items) + ) + ) + except ValueError as error: + raise InvalidArgument(f"Cannot load {key}: {error}") from error + mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) + return mm_data, mm_uuids + + async def _engine_input( + self, request: GenerateRequest, *, skip_mm_cache: bool = False + ) -> dict[str, Any]: + """The processed vLLM prompt for the request, as the native chat endpoint builds it. + + The frontend's tokens carry one unexpanded placeholder per media item; + vLLM's processor expands them and processes the media, exactly as after + the native endpoint renders and tokenizes the messages. + """ + prompt = TokensPrompt(prompt_token_ids=list(request["token_ids"])) + media = await self._load_media(request) + if media is not None: + prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media + if request.get("mm_processor_kwargs"): + prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] + return await self._llm.renderer.process_for_engine_async( + prompt, time.time(), skip_mm_cache=skip_mm_cache + ) + + def _check_prompt_fits(self, prompt_len: int) -> None: + """Refuse a prompt that leaves no room for output, as the native endpoint does. + + NeMo-Gym reads "context length" in a 400 as a length refusal rather than an + engine failure. + """ + if prompt_len >= self._max_model_len: + raise InvalidArgument( + f"Prompt length ({prompt_len}) fills or exceeds this model's " + f"maximum context length ({self._max_model_len}). No room for " + "output tokens." + ) + + @staticmethod + def _gym_token_data(final_output: Any) -> dict[str, Any]: + """The token arrays Gym's dialect lifts onto the message, as the native endpoint reports them.""" + token_data: dict[str, Any] = { + "prompt_token_ids": list(final_output.prompt_token_ids), + "completion_token_ids": list(final_output.outputs[0].token_ids), + } + if final_output.outputs[0].logprobs is not None: + _, token_data["completion_logprobs"] = chat_generation_tokens( + final_output.outputs[0] + ) + return token_data + + async def _stage_gym_call(self, request: GenerateRequest, final_output: Any) -> Any: + """Stage a captured call from this process; return its commit coordinates.""" + host = self._host + completion = final_output.outputs[0] + generation_token_ids, generation_log_probs = chat_generation_tokens(completion) + message: dict[str, Any] = { + "generation_token_ids": generation_token_ids, + "generation_log_probs": generation_log_probs, + } + if host._return_routed_experts_enabled(): + message["routed_experts"] = chat_routed_experts( + final_output, + completion, + prompt_token_count=len(final_output.prompt_token_ids), + device=torch.device("cpu"), + logger=LOGGER, + routed_experts_dtype=host.routed_experts_dtype, + ) + # The native endpoint's response shape; the prompt is the one recorded at + # begin_call. + content = { + "choices": [ + { + "index": 0, + "finish_reason": completion.finish_reason, + "message": message, + } + ] + } + # The sink write is a blocking data-plane round trip. + content = await asyncio.to_thread( + host._finish_request_capture, request, content + ) + return content["ng_commit_coords"] + + async def abort(self, context: Context) -> None: + await self._llm.abort(context.id()) + + async def kv_event_sources(self) -> list[KvEventSource]: + kv_events = self._llm.vllm_config.kv_events_config + if kv_events is None or not kv_events.enable_kv_cache_events: + return [] + return [ZmqSource(endpoint=kv_events.endpoint.replace("*", "127.0.0.1"))] + + def component_metrics_dp_ranks(self) -> list[int]: + return [0] + + def attach_snapshot_publisher(self, publisher: Any) -> None: + SnapshotStatLogger.publisher = publisher + + async def cleanup(self) -> None: + # The worker owns the AsyncLLM; its shutdown() tears the engine down. + SnapshotStatLogger.publisher = None + + +def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: + prompt_tokens = len(output.prompt_token_ids or ()) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": num_output_tokens, + "total_tokens": prompt_tokens + num_output_tokens, + "prompt_tokens_details": {"cached_tokens": output.num_cached_tokens or 0}, + } + + +def _check_parser_name(name: Optional[str], known: list[str], field: str) -> None: + if name is not None and name not in known: + raise ValueError( + f"Dynamo has no {field.replace('_', ' ')} named {name!r}; set " + f"policy.generation.dynamo_cfg.worker_args.{field} to one of " + f"{sorted(known)}." + ) + + +def _chat_template_file(chat_template: Optional[str]) -> Optional[str]: + """A file holding ``chat_template``, which Dynamo reads by path. + + Resolved as the native server resolves it (vLLM's ``load_chat_template``): + a file, a vLLM built-in template name, or literal Jinja. Named by content, + so every engine on a node, and every restart, shares one file. + """ + if chat_template is None: + return None + content = load_chat_template(chat_template) + digest = hashlib.sha256(content.encode()).hexdigest()[:16] + path = os.path.join(tempfile.gettempdir(), f"nemo_rl_chat_template_{digest}.jinja") + if not os.path.exists(path): + # Written aside and renamed, so a concurrent reader never sees a part. + with tempfile.NamedTemporaryFile( + "w", dir=os.path.dirname(path), suffix=".jinja", delete=False + ) as template_file: + template_file.write(content) + os.replace(template_file.name, path) + return path + + +def start_dynamo_endpoint( + host: "VllmAsyncGenerationWorkerImpl", + *, + namespace: str, + engine_env: dict[str, str], + leader_worker_idx: int, +) -> asyncio.Task: + """Register ``host``'s engine with Dynamo and serve it on the running loop. + + The Rust Worker serves until the process gets SIGTERM. It installs a chained + handler, so Ray's actor handler still runs and the actor exits as usual; + discovery drops the engine with its process. + """ + # The Rust runtime reads these when the first Worker is constructed. + os.environ.update(engine_env) + kv_events = host.llm.vllm_config.kv_events_config + publishes = kv_events is not None and kv_events.enable_kv_cache_events + # The Dynamo functional overlays check the router applied these. + print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) + worker_args = dynamo_cfg(host.cfg).worker_args + tool_call_parser, reasoning_parser = dynamo_parsers(host.cfg) + _check_parser_name(tool_call_parser, get_tool_parser_names(), "tool_call_parser") + _check_parser_name( + reasoning_parser, get_reasoning_parser_names(), "reasoning_parser" + ) + config = WorkerConfig( + namespace=namespace, + component=DYNAMO_COMPONENT, + endpoint=DYNAMO_ENDPOINT, + model_name=host.model_name, + served_model_name=host.model_name, + endpoint_types=",".join(worker_args.endpoint_types), + discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], + request_plane=engine_env["DYN_REQUEST_PLANE"], + event_plane=engine_env["DYN_EVENT_PLANE"], + custom_jinja_template=_chat_template_file(dynamo_chat_template(host.cfg)), + tool_call_parser=tool_call_parser, + reasoning_parser=reasoning_parser, + exclude_tools_when_tool_choice_none=( + worker_args.exclude_tools_when_tool_choice_none + ), + structural_tag_mode="on" if worker_args.enable_structural_tag else "off", + structural_tag_scope=worker_args.structural_tag_scope, + structural_tag_schema=worker_args.structural_tag_schema, + ) + return asyncio.get_running_loop().create_task( + Worker(NativeVllmEngine(host, leader_worker_idx), config).run() + ) diff --git a/nemo_rl/models/generation/dynamo/refit.py b/nemo_rl/models/generation/dynamo/refit.py deleted file mode 100644 index 4df856747b6..00000000000 --- a/nemo_rl/models/generation/dynamo/refit.py +++ /dev/null @@ -1,320 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""NCCL refit protocol for a fixed managed Dynamo worker fleet.""" - -from collections.abc import Callable, Sequence -from dataclasses import dataclass -from typing import Any - -import ray - -from nemo_rl.models.generation.dynamo.config import ( - VLLM_PACKED_BUFFER_SIZE_BYTES, - VLLM_PACKED_NUM_BUFFERS, -) -from nemo_rl.models.generation.dynamo.http_client import ( - format_dynamo_error, - http_post_json, -) -from nemo_rl.models.generation.interfaces import CollectiveSenderSpec - - -@dataclass(frozen=True) -class DynamoWorkerEndpoint: - """Serializable identity and admin endpoint for one Dynamo vLLM engine.""" - - instance_id: str - system_url: str - - @classmethod - def from_metadata(cls, metadata: dict[str, Any]) -> "DynamoWorkerEndpoint": - return cls( - instance_id=str(metadata["instance_id"]), - system_url=str(metadata["system_url"]), - ) - - -@ray.remote(num_cpus=0) -def _post_worker_route( # pragma: no cover - *, - system_url: str, - route: str, - payload: dict[str, Any], - timeout_s: float, -) -> bool: - response = http_post_json( - f"{system_url}/engine/{route}", - payload, - timeout_s, - ) - if response.get("status") != "ok": - raise RuntimeError( - f"Dynamo worker {system_url} route {route} failed: " - f"{format_dynamo_error(response)}" - ) - return True - - -@ray.remote(num_cpus=0) -def _update_worker_weights( # pragma: no cover - *, - system_url: str, - update_info: dict[str, Any], - timeout_s: float, -) -> bool: - common = {"allow_unpaused": True, "reset_prefix_cache": False} - steps: tuple[tuple[str, dict[str, Any]], ...] = ( - ("start_weight_update", {}), - ("update_weights", {"update_info": update_info}), - ("finish_weight_update", {}), - ) - for engine_rpc, kwargs in steps: - response = http_post_json( - f"{system_url}/engine/update_weights_from_distributed", - {"engine_rpc": engine_rpc, **common, **kwargs}, - timeout_s, - ) - if response.get("status") != "ok": - raise RuntimeError( - f"Dynamo worker {system_url} RPC {engine_rpc} failed: " - f"{format_dynamo_error(response)}" - ) - return True - - -class DynamoRefitChannel: - """Closed refit protocol shared by driver and serialized rollout copies.""" - - def __init__( - self, - workers: Sequence[dict[str, Any] | DynamoWorkerEndpoint], - *, - engine_world_size: int, - control_timeout_s: float, - validate_workers: Callable[[list[dict[str, Any]]], list[dict[str, Any]]] - | None = None, - ) -> None: - self._worker_metadata = [ - dict(worker) for worker in workers if isinstance(worker, dict) - ] - self._workers = tuple( - DynamoWorkerEndpoint.from_metadata(worker) - if isinstance(worker, dict) - else worker - for worker in workers - ) - if not self._workers: - raise ValueError("Dynamo refit requires at least one worker endpoint") - self._engine_world_size = engine_world_size - self._control_timeout_s = control_timeout_s - self._validate_workers = validate_workers - self._update_info: dict[str, Any] | None = None - - def client_copy(self) -> "DynamoRefitChannel": - """Return a serializable copy without the driver-owned worker validator. - - The prepared update metadata travels with the copy: SingleController - prepares refit on the driver and then pickles the synchronizer (and - this generation client) into its actor, which issues every update. - """ - copy = DynamoRefitChannel( - self._workers, - engine_world_size=self._engine_world_size, - control_timeout_s=self._control_timeout_s, - ) - copy._update_info = self._update_info - return copy - - def _validated_workers(self) -> tuple[DynamoWorkerEndpoint, ...]: - if self._validate_workers is None: - return self._workers - current = self._validate_workers(self._worker_metadata) - endpoints = tuple(DynamoWorkerEndpoint.from_metadata(item) for item in current) - if endpoints != self._workers: - raise RuntimeError( - "Managed Dynamo worker membership changed after collective setup" - ) - return endpoints - - @property - def workers(self) -> tuple[DynamoWorkerEndpoint, ...]: - """Return the managed workers' endpoints, as fixed at collective setup.""" - return self._workers - - @property - def inference_world_size(self) -> int: - return len(self._workers) * self._engine_world_size - - @property - def sender_spec(self) -> CollectiveSenderSpec: - """Return the peer and packing geometry declared to workers at init.""" - return CollectiveSenderSpec( - nccl_peer="vllm", - buffer_size_bytes=VLLM_PACKED_BUFFER_SIZE_BYTES, - num_buffers=VLLM_PACKED_NUM_BUFFERS, - use_legacy_packing=True, - ) - - def prepare(self, state_dict_info: dict[str, Any] | None) -> None: - if state_dict_info is None: - raise ValueError("state_dict_info must not be None for Dynamo refit") - names: list[str] = [] - dtype_names: list[str] = [] - shapes: list[list[int]] = [] - for name, (shape, dtype) in state_dict_info.items(): - names.append(name) - dtype_names.append(str(dtype).removeprefix("torch.")) - shapes.append(list(shape)) - self._update_info = { - "names": names, - "dtype_names": dtype_names, - "shapes": shapes, - } - - def init_collective( - self, - ip: str, - port: int, - world_size: int, - *, - train_world_size: int, - ) -> list[ray.ObjectRef]: - expected_world_size = train_world_size + self.inference_world_size - if world_size != expected_world_size: - raise ValueError( - f"NCCL world_size={world_size} does not match expected " - f"{expected_world_size}" - ) - workers = self._validated_workers() - return [ - _post_worker_route.remote( - system_url=worker.system_url, - route="init_weights_update_group", - payload={ - "engine_rpc": "init_weight_transfer_engine", - "init_info": { - "master_address": ip, - "master_port": port, - "rank_offset": train_world_size - + worker_index * self._engine_world_size, - "world_size": world_size, - # vLLM reads packing once at init and defaults it off; - # omitting it desynchronizes the packed broadcast. - "packed": True, - "packed_buffer_size_bytes": VLLM_PACKED_BUFFER_SIZE_BYTES, - "packed_num_buffers": VLLM_PACKED_NUM_BUFFERS, - }, - }, - timeout_s=self._control_timeout_s, - ) - for worker_index, worker in enumerate(workers) - ] - - def update_weights(self) -> list[ray.ObjectRef]: - if self._update_info is None: - raise RuntimeError( - "prepare_refit_info() must be called before Dynamo weight updates" - ) - return [ - _update_worker_weights.remote( - system_url=worker.system_url, - update_info=self._update_info, - timeout_s=self._control_timeout_s, - ) - for worker in self._validated_workers() - ] - - def _pause_workers( - self, *, clear_cache: bool - ) -> tuple[list[DynamoWorkerEndpoint], list[str]]: - """Drain in-flight requests, optionally clear caches; new requests wait.""" - pause_futures = [ - _post_worker_route.remote( - system_url=worker.system_url, - route="pause_generation", - payload={"mode": "wait", "clear_cache": clear_cache}, - timeout_s=self._control_timeout_s, - ) - for worker in self._workers - ] - paused_workers: list[DynamoWorkerEndpoint] = [] - pause_errors: list[str] = [] - for worker, future in zip(self._workers, pause_futures, strict=True): - try: - ray.get(future) - except Exception as error: - pause_errors.append(f"{worker.system_url}: {error}") - else: - paused_workers.append(worker) - return paused_workers, pause_errors - - def _resume_workers(self, workers: Sequence[DynamoWorkerEndpoint]) -> list[str]: - resume_futures = [ - ( - worker, - _post_worker_route.remote( - system_url=worker.system_url, - route="resume_generation", - payload={}, - timeout_s=self._control_timeout_s, - ), - ) - for worker in workers - ] - resume_errors: list[str] = [] - for worker, future in resume_futures: - try: - ray.get(future) - except Exception as error: - resume_errors.append(f"{worker.system_url}: {error}") - return resume_errors - - def pause(self, *, clear_cache: bool) -> None: - """Drain and pause every worker ahead of a weight update. - - Workers that confirmed the pause are resumed if any peer fails. - """ - paused_workers, pause_errors = self._pause_workers(clear_cache=clear_cache) - if pause_errors: - resume_errors = self._resume_workers(paused_workers) - details = "pause/clear failed for " + "; ".join(pause_errors) - if resume_errors: - details += "; resume failed for " + "; ".join(resume_errors) - raise RuntimeError(f"Dynamo pause before weight update failed: {details}") - - def resume(self) -> None: - """Resume every worker after a weight update.""" - resume_errors = self._resume_workers(self._workers) - if resume_errors: - raise RuntimeError( - "Dynamo resume after weight update failed for " - + "; ".join(resume_errors) - ) - - def flush_cache(self) -> bool: - """Drain, clear, and resume every worker using immutable endpoints.""" - paused_workers, pause_errors = self._pause_workers(clear_cache=True) - resume_errors = self._resume_workers(paused_workers) - - if pause_errors or resume_errors: - details = [] - if pause_errors: - details.append("pause/clear failed for " + "; ".join(pause_errors)) - if resume_errors: - details.append("resume failed for " + "; ".join(resume_errors)) - raise RuntimeError( - "Dynamo KV cache invalidation failed: " + "; ".join(details) - ) - return True diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py new file mode 100644 index 00000000000..82f5b897e77 --- /dev/null +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -0,0 +1,114 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Send NeMo RL rollouts through the Dynamo frontend's KV-aware router.""" + +import asyncio +import json +import logging +from typing import Any, Optional + +from nemo_rl.models.generation.dynamo.http_client import ( + async_http_post_json, + format_dynamo_error, +) + +LOGGER = logging.getLogger(__name__) + +# Marks a request as a NeMo RL rollout for the engine (native_engine), which +# then samples it from the generation config. Defined here, not in the engine +# module, so the driver need not import ai-dynamo. +ROLLOUT_ANNOTATION = "nemo_rl:rollout:" + +_HTTP_MAX_ATTEMPTS = 3 +_HTTP_RETRY_DELAY_S = 1.0 +_RETRYABLE_HTTP_STATUS_CODES = {408, 429} + + +def _is_retryable(response: dict[str, Any]) -> bool: + # A timeout is final: the request may still be running, and a retry would + # let one hung rollout hold its slot for several request_timeout_s. + if "transport_error" in response or response.get("json_decode_error") is True: + return True + status = response.get("http_status") + return isinstance(status, int) and ( + status in _RETRYABLE_HTTP_STATUS_CODES or 500 <= status < 600 + ) + + +def rollout_payload( + *, + model_name: str, + prompt_token_ids: list[int], + max_model_len: int, + greedy: bool, + stop_strings: Optional[list[str]], +) -> dict[str, Any]: + """The completion request for one rollout. + + Sampling is the engine's: the annotation tells it to build the request as + ``generate_async`` does, from the generation config. ``ignore_eos`` and a + context-sized ``max_tokens`` keep the frontend from ending the stream on + its own stop rules before the engine reports the result. + """ + rollout = {"greedy": greedy, "stop_strings": stop_strings or None} + return { + "model": model_name, + "prompt": prompt_token_ids, + "max_tokens": max(1, max_model_len - len(prompt_token_ids)), + "ignore_eos": True, + "nvext": { + "annotations": [ROLLOUT_ANNOTATION + json.dumps(rollout)], + "extra_fields": ["engine_data"], + }, + } + + +def parse_rollout_response(response: dict[str, Any], *, url: str) -> dict[str, Any]: + """Return the engine's ``engine_data``, or raise with the frontend's error. + + It holds the rollout result (``request_pipeline.rollout_result_to_wire``) + and the serving worker's index. + """ + if response.get("status") == "error": + raise RuntimeError( + f"Dynamo rollout request to {url} failed: {format_dynamo_error(response)}" + ) + engine_data = (response.get("nvext") or {}).get("engine_data") + if not isinstance(engine_data, dict) or "rollout" not in engine_data: + raise RuntimeError( + f"Dynamo rollout response from {url} carried no rollout engine_data; " + "is every engine NeMo RL's vLLM worker?" + ) + return engine_data + + +async def post_rollout( + url: str, payload: dict[str, Any], timeout_s: float +) -> dict[str, Any]: + """POST one rollout, retrying transient errors; returns its ``engine_data``.""" + response: dict[str, Any] = {} + for attempt in range(1, _HTTP_MAX_ATTEMPTS + 1): + response = await async_http_post_json(url, payload, timeout_s) + if not _is_retryable(response) or attempt == _HTTP_MAX_ATTEMPTS: + break + LOGGER.warning( + "Dynamo rollout attempt %d/%d failed; retrying in %.1fs: %s", + attempt, + _HTTP_MAX_ATTEMPTS, + _HTTP_RETRY_DELAY_S, + format_dynamo_error(response), + ) + await asyncio.sleep(_HTTP_RETRY_DELAY_S) + return parse_rollout_response(response, url=url) diff --git a/nemo_rl/models/generation/dynamo/token_wrapper.py b/nemo_rl/models/generation/dynamo/token_wrapper.py deleted file mode 100644 index eab81fafa35..00000000000 --- a/nemo_rl/models/generation/dynamo/token_wrapper.py +++ /dev/null @@ -1,556 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""OpenAI-compatible token wrapper for NeMo-Gym traffic to Dynamo.""" - -import asyncio -import json -import threading -from contextlib import asynccontextmanager -from copy import deepcopy -from typing import Any, Optional - -from nemo_rl.distributed.virtual_cluster import ( - DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH, - DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, - _get_free_port_local, - _get_node_ip_local, -) -from nemo_rl.models.generation.openai_server_utils import replace_prefix_tokens - -_GYM_TOKEN_METADATA_FIELDS = ( - "prompt_token_ids", - "generation_token_ids", - "generation_log_probs", -) -_TOOL_ARGUMENT_MAPPING_ERROR = "Can only get item pairs from a mapping." - - -def _coerce_token_id_list(value: Any, field_name: str) -> list[int]: - if not isinstance(value, list): - raise ValueError(f"{field_name} must be a list of token IDs.") - try: - return [int(token_id) for token_id in value] - except (TypeError, ValueError) as e: - raise ValueError(f"{field_name} must contain only integer token IDs.") from e - - -def _coerce_logprob_list(value: Any, field_name: str) -> list[float]: - if not isinstance(value, list): - raise ValueError(f"{field_name} must be a list of numeric log probabilities.") - logprobs: list[float] = [] - for index, logprob in enumerate(value): - if not isinstance(logprob, (int, float)) or isinstance(logprob, bool): - raise ValueError(f"{field_name}[{index}] must be numeric.") - logprobs.append(float(logprob)) - return logprobs - - -def _strip_gym_token_metadata(messages: list[Any]) -> list[Any]: - stripped_messages = deepcopy(messages) - for message in stripped_messages: - if isinstance(message, dict): - for field in _GYM_TOKEN_METADATA_FIELDS: - message.pop(field, None) - return stripped_messages - - -def _chat_template_kwargs( - request_body: dict[str, Any], - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], -) -> dict[str, Any]: - kwargs: dict[str, Any] = {} - if tokenizer_chat_template_kwargs is not None: - if not isinstance(tokenizer_chat_template_kwargs, dict): - raise ValueError("tokenizer chat_template_kwargs must be a JSON object.") - kwargs.update(tokenizer_chat_template_kwargs) - - request_kwargs = request_body.get("chat_template_kwargs") - if request_kwargs is not None: - if not isinstance(request_kwargs, dict): - raise ValueError("chat_template_kwargs must be a JSON object.") - kwargs.update(request_kwargs) - - if "reasoning_effort" in request_body: - kwargs["reasoning_effort"] = request_body["reasoning_effort"] - return kwargs - - -def _request_add_generation_prompt(request_body: dict[str, Any]) -> bool: - if "add_generation_prompt" in request_body: - return bool(request_body["add_generation_prompt"]) - return not bool(request_body.get("continue_final_message", False)) - - -def _apply_chat_template( - *, - tokenizer: Any, - request_body: dict[str, Any], - messages: list[Any], - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], - exclude_tools_when_tool_choice_none: bool, - add_generation_prompt: bool, - tokenize: bool, -) -> Any: - tools = request_body.get("tools") - if ( - exclude_tools_when_tool_choice_none - and request_body.get("tool_choice") == "none" - ): - tools = None - - apply_chat_template = type(tokenizer).apply_chat_template - return apply_chat_template( - tokenizer, - messages, - tools=tools, - documents=request_body.get("documents"), - chat_template=request_body.get("chat_template"), - add_generation_prompt=add_generation_prompt, - continue_final_message=bool(request_body.get("continue_final_message", False)), - tokenize=tokenize, - return_tensors=None, - return_dict=False, - **_chat_template_kwargs(request_body, tokenizer_chat_template_kwargs), - ) - - -def _render_prompt_token_ids( - *, - tokenizer: Any, - request_body: dict[str, Any], - messages: list[Any], - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], - exclude_tools_when_tool_choice_none: bool, - add_generation_prompt: bool, -) -> list[int]: - token_ids = _apply_chat_template( - tokenizer=tokenizer, - request_body=request_body, - messages=messages, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=add_generation_prompt, - tokenize=True, - ) - - if isinstance(token_ids, list) and ( - not token_ids or not isinstance(token_ids[0], list) - ): - return _coerce_token_id_list(token_ids, "prompt token IDs") - if isinstance(token_ids, list) and len(token_ids) == 1: - return _coerce_token_id_list(token_ids[0], "prompt token IDs") - raise ValueError( - "Dynamo token wrapper expected chat template rendering to return one " - "list of prompt token IDs." - ) - - -def _render_prompt_token_ids_with_optional_prefix( - *, - tokenizer: Any, - request_body: dict[str, Any], - messages: list[Any], - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], - exclude_tools_when_tool_choice_none: bool, - add_generation_prompt: bool, - assistant_index: int | None, -) -> tuple[list[int], list[int] | None]: - full_prompt_token_ids = _render_prompt_token_ids( - tokenizer=tokenizer, - request_body=request_body, - messages=messages, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=add_generation_prompt, - ) - if assistant_index is None: - return full_prompt_token_ids, None - - template_prefix_token_ids = _render_prompt_token_ids( - tokenizer=tokenizer, - request_body=request_body, - messages=messages[: assistant_index + 1], - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=False, - ) - return full_prompt_token_ids, template_prefix_token_ids - - -def _latest_tokenized_assistant_index(messages: list[Any]) -> Optional[int]: - for index in reversed(range(len(messages))): - message = messages[index] - if not isinstance(message, dict) or message.get("role") != "assistant": - continue - if ( - message.get("prompt_token_ids") is not None - and message.get("generation_token_ids") is not None - ): - return index - return None - - -def _derive_required_prefix_token_ids(messages: list[Any]) -> list[int] | None: - """Return the exact prompt and generation IDs from the latest model turn.""" - assistant_index = _latest_tokenized_assistant_index(messages) - if assistant_index is None: - return None - message = messages[assistant_index] - if not isinstance(message, dict): - return None - return _coerce_token_id_list( - message["prompt_token_ids"], "prompt_token_ids" - ) + _coerce_token_id_list(message["generation_token_ids"], "generation_token_ids") - - -def _normalize_tool_arguments_for_template( - messages: list[Any], *, before_index: int -) -> None: - """Make OpenAI tool calls renderable by model chat templates. - - OpenAI chat messages carry ``function.arguments`` as a JSON string, while - some model templates iterate those arguments as a mapping. Normalize only - the local template copy; the request forwarded to Dynamo retains its - original OpenAI payload. - """ - for message in messages[:before_index]: - if not isinstance(message, dict) or message.get("role") != "assistant": - continue - tool_calls = message.get("tool_calls") - if not isinstance(tool_calls, list): - continue - for tool_call in tool_calls: - if not isinstance(tool_call, dict): - continue - function = tool_call.get("function", tool_call) - if not isinstance(function, dict): - continue - arguments = function.get("arguments") - if not isinstance(arguments, str): - continue - try: - parsed_arguments = json.loads(arguments) - except json.JSONDecodeError: - parsed_arguments = {} - function["arguments"] = ( - parsed_arguments if isinstance(parsed_arguments, dict) else {} - ) - - -def _validate_engine_data( - response_body: dict[str, Any], -) -> tuple[list[int], list[int], list[float]]: - nvext = response_body.get("nvext") - engine_data = nvext.get("engine_data") if isinstance(nvext, dict) else None - if not isinstance(engine_data, dict): - raise ValueError("Dynamo response did not include nvext.engine_data.") - - prompt_token_ids = _coerce_token_id_list( - engine_data.get("prompt_token_ids"), - "nvext.engine_data.prompt_token_ids", - ) - completion_token_ids = _coerce_token_id_list( - engine_data.get("completion_token_ids"), - "nvext.engine_data.completion_token_ids", - ) - completion_logprobs = _coerce_logprob_list( - engine_data.get("completion_logprobs"), - "nvext.engine_data.completion_logprobs", - ) - if len(completion_logprobs) != len(completion_token_ids): - raise ValueError( - "Dynamo response returned " - f"{len(completion_logprobs)} generation log probabilities for " - f"{len(completion_token_ids)} generation token IDs." - ) - return prompt_token_ids, completion_token_ids, completion_logprobs - - -def _inject_gym_token_metadata(response_body: dict[str, Any]) -> None: - """Expose Dynamo engine metadata on the message fields consumed by Gym.""" - ( - prompt_token_ids, - generation_token_ids, - generation_logprobs, - ) = _validate_engine_data(response_body) - choices = response_body.get("choices") - if not isinstance(choices, list) or not choices: - raise ValueError("Dynamo response did not include choices[0].") - choice = choices[0] - if not isinstance(choice, dict): - raise ValueError("Dynamo response choices[0] must be a JSON object.") - message = choice.get("message") - if not isinstance(message, dict): - raise ValueError("Dynamo response choices[0].message must be a JSON object.") - message["prompt_token_ids"] = prompt_token_ids - message["generation_token_ids"] = generation_token_ids - message["generation_log_probs"] = generation_logprobs - - -def prepare_dynamo_chat_completion_request( - request_body: dict[str, Any], - *, - tokenizer: Any, - tokenizer_chat_template_kwargs: Optional[dict[str, Any]] = None, - exclude_tools_when_tool_choice_none: bool, -) -> dict[str, Any]: - """Prepare a NeMo-Gym chat-completion request for Dynamo token input.""" - if request_body.get("stream"): - raise ValueError("Dynamo native token wrapper does not support stream=True.") - - n = request_body.get("n", 1) - if n is not None and int(n) != 1: - raise ValueError("Dynamo native token wrapper currently supports only n=1.") - - messages = request_body.get("messages") - if not isinstance(messages, list): - raise ValueError("Dynamo token wrapper requires chat-completion messages.") - - prepared_body = deepcopy(request_body) - stripped_messages = _strip_gym_token_metadata(messages) - prepared_body["messages"] = stripped_messages - prepared_body.pop("required_prefix_token_ids", None) - - required_prefix_token_ids = _derive_required_prefix_token_ids(messages) - add_generation_prompt = _request_add_generation_prompt(prepared_body) - template_messages = deepcopy(stripped_messages) - assistant_index: int | None = None - if required_prefix_token_ids is not None: - assistant_index = _latest_tokenized_assistant_index(messages) - if assistant_index is None: - raise ValueError( - "Dynamo prefix token metadata must be attached to an assistant message." - ) - - try: - ( - full_prompt_token_ids, - template_prefix_token_ids, - ) = _render_prompt_token_ids_with_optional_prefix( - tokenizer=tokenizer, - request_body=prepared_body, - messages=template_messages, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=add_generation_prompt, - assistant_index=assistant_index, - ) - except TypeError as e: - if str(e) != _TOOL_ARGUMENT_MAPPING_ERROR: - raise - _normalize_tool_arguments_for_template( - template_messages, before_index=len(template_messages) - ) - ( - full_prompt_token_ids, - template_prefix_token_ids, - ) = _render_prompt_token_ids_with_optional_prefix( - tokenizer=tokenizer, - request_body=prepared_body, - messages=template_messages, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=add_generation_prompt, - assistant_index=assistant_index, - ) - - if required_prefix_token_ids is not None: - assert template_prefix_token_ids is not None - full_prompt_token_ids = replace_prefix_tokens( - tokenizer, - model_prefix_token_ids=required_prefix_token_ids, - template_prefix_token_ids=template_prefix_token_ids, - template_token_ids=full_prompt_token_ids, - ) - - nvext = prepared_body.get("nvext") - if nvext is None: - nvext = {} - if not isinstance(nvext, dict): - raise ValueError("nvext must be a JSON object.") - nvext = dict(nvext) - extra_fields = nvext.get("extra_fields", []) - if not isinstance(extra_fields, list): - raise ValueError("nvext.extra_fields must be a JSON list.") - nvext["extra_fields"] = list(dict.fromkeys([*extra_fields, "engine_data"])) - nvext["token_data"] = full_prompt_token_ids - prepared_body["nvext"] = nvext - - return prepared_body - - -class DynamoTokenWrapperServer: - """Small HTTP server that supplies tokenized chat prompts to Dynamo.""" - - def __init__( - self, - *, - dynamo_frontend_base_url: str, - tokenizer: Any, - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], - exclude_tools_when_tool_choice_none: bool, - request_timeout_s: Optional[float], - ) -> None: - self.dynamo_frontend_base_url = dynamo_frontend_base_url - self.tokenizer = tokenizer - self.tokenizer_chat_template_kwargs = tokenizer_chat_template_kwargs - self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none - self.request_timeout_s = request_timeout_s - self.base_url: Optional[str] = None - self.server: Any = None - self.thread: Optional[threading.Thread] = None - self._client_session: Any = None - - def start(self) -> str: - """Start the wrapper in a background uvicorn thread.""" - import aiohttp - import uvicorn - from fastapi import FastAPI, HTTPException, Request - from fastapi.responses import JSONResponse - - @asynccontextmanager - async def lifespan(_: FastAPI): - timeout = ( - aiohttp.ClientTimeout(total=self.request_timeout_s) - if self.request_timeout_s is not None - else aiohttp.ClientTimeout(total=None) - ) - async with aiohttp.ClientSession( - timeout=timeout, - # Explicit: the default TCPConnector(limit=100) would cap the whole - # rollout path at 100 concurrent requests to the frontend. - connector=aiohttp.TCPConnector(limit=0), - ) as session: - self._client_session = session - try: - yield - finally: - self._client_session = None - - app = FastAPI(lifespan=lifespan) - - @app.get("/health") - async def health() -> dict[str, str]: - return { - "status": "ok", - "dynamo_frontend_base_url": self.dynamo_frontend_base_url, - } - - @app.post("/v1/chat/completions") - async def chat_completions(request: Request) -> JSONResponse: - try: - request_body = await request.json() - except json.JSONDecodeError as e: - raise HTTPException(status_code=400, detail="Invalid JSON body.") from e - if not isinstance(request_body, dict): - raise HTTPException( - status_code=400, - detail="Chat completion body must be a JSON object.", - ) - - try: - prepared_body = await asyncio.to_thread( - prepare_dynamo_chat_completion_request, - request_body, - tokenizer=self.tokenizer, - tokenizer_chat_template_kwargs=self.tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=self.exclude_tools_when_tool_choice_none, - ) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - status_code, response_body = await self._forward_chat_completion( - prepared_body, - authorization=request.headers.get("authorization"), - ) - if 200 <= status_code < 300: - try: - _inject_gym_token_metadata(response_body) - except ValueError as e: - return JSONResponse( - content={"error": {"message": str(e)}}, - status_code=502, - ) - return JSONResponse(content=response_body, status_code=status_code) - - node_ip = _get_node_ip_local() - free_port = _get_free_port_local( - DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, - DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH, - ) - self.base_url = f"http://{node_ip}:{free_port}/v1" - - config = uvicorn.Config( - app, - host="0.0.0.0", - port=free_port, - timeout_keep_alive=120, - ) - self.server = uvicorn.Server(config=config) - self.thread = threading.Thread( - target=self.server.run, - name="dynamo-token-wrapper", - daemon=True, - ) - self.thread.start() - return self.base_url - - async def _forward_chat_completion( - self, - request_body: dict[str, Any], - *, - authorization: Optional[str], - ) -> tuple[int, dict[str, Any]]: - import aiohttp - - url = f"{self.dynamo_frontend_base_url.rstrip('/')}/chat/completions" - headers = {"Content-Type": "application/json"} - if authorization: - headers["Authorization"] = authorization - - session = self._client_session - if session is None: - return 503, {"error": {"message": "Dynamo token wrapper is not ready."}} - try: - async with session.post( - url, - json=request_body, - headers=headers, - ) as response: - response_text = await response.text() - if not response_text: - return response.status, {} - try: - response_body = json.loads(response_text) - except json.JSONDecodeError: - response_body = {"raw": response_text} - if not isinstance(response_body, dict): - response_body = {"response": response_body} - return response.status, response_body - except asyncio.TimeoutError: - return 504, {"error": {"message": f"Timed out forwarding to {url}."}} - except aiohttp.ClientError as e: - return 502, { - "error": { - "message": f"Failed to forward request to {url}: {type(e).__name__}: {e}" - } - } - - def shutdown(self) -> None: - """Stop the background uvicorn server.""" - if self.server is not None: - self.server.should_exit = True - if self.thread is not None: - self.thread.join(timeout=10) diff --git a/nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py b/nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py deleted file mode 100644 index 612c75cc7d0..00000000000 --- a/nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Validate a resolved ``dynamo.vllm`` argv without starting an engine.""" - -import dataclasses -import json -import sys - -# Fields NeMo RL sends in each ``update_weights`` round (see refit.py). -_UPDATE_INFO_FIELDS = {"names", "dtype_names", "shapes"} - - -def main() -> None: - if len(sys.argv) != 3: - raise SystemExit( - "usage: validate_dynamo_vllm_args.py '' ''" - ) - - # Parser construction evaluates vLLM device defaults. Image builds and - # unit preflights may run on CPU-only Slurm nodes, where a CUDA wheel leaves - # current_platform unspecified even though no engine will be constructed. - # Match Dynamo frontend's parser-only fallback without changing GPU runs. - import vllm.platforms - - if vllm.platforms.current_platform.device_type == "": - from vllm.platforms.cpu import CpuPlatform - - vllm.platforms.current_platform = CpuPlatform() - - from dynamo.vllm.args import parse_args - from vllm.distributed.weight_transfer.nccl_common import ( - NCCLWeightTransferInitInfo, - ) - from vllm.distributed.weight_transfer.nccl_engine import ( - NCCLWeightTransferUpdateInfo, - ) - - argv = json.loads(sys.argv[1]) - if not isinstance(argv, list) or not all(isinstance(item, str) for item in argv): - raise TypeError("resolved Dynamo argv must be a JSON list of strings") - parse_args(argv) - - # NeMo RL declares the packed geometry in the init info and sends only - # per-round tensor metadata in the update info. Fail before engine startup - # if the installed vLLM weight-transfer API no longer accepts that shape. - geometry = json.loads(sys.argv[2]) - try: - NCCLWeightTransferInitInfo( - master_address="127.0.0.1", - master_port=1, - rank_offset=1, - world_size=2, - packed=True, - packed_buffer_size_bytes=geometry["buffer_size_bytes"], - packed_num_buffers=geometry["num_buffers"], - ) - except TypeError as error: - raise SystemExit( - "vLLM NCCLWeightTransferInitInfo does not accept NeMo RL's packed " - f"init info: {error}. Update the Dynamo refit protocol in refit.py." - ) from error - update_fields = { - field.name for field in dataclasses.fields(NCCLWeightTransferUpdateInfo) - } - if update_fields != _UPDATE_INFO_FIELDS: - raise SystemExit( - "vLLM NCCLWeightTransferUpdateInfo fields changed: engine has " - f"{sorted(update_fields)}, NeMo RL sends {sorted(_UPDATE_INFO_FIELDS)}. " - "Update the Dynamo refit protocol in refit.py." - ) - - -if __name__ == "__main__": - main() diff --git a/nemo_rl/models/generation/dynamo/venv.py b/nemo_rl/models/generation/dynamo/venv.py index 020c923bb34..9a60b194e53 100644 --- a/nemo_rl/models/generation/dynamo/venv.py +++ b/nemo_rl/models/generation/dynamo/venv.py @@ -12,33 +12,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Resolve executables installed by the optional Dynamo environment.""" +"""Resolve the interpreter and service binaries the managed Dynamo runtime launches.""" import os +import shutil from pathlib import Path +from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env +from nemo_rl.utils.venvs import create_local_venv_on_each_node -def get_dynamo_venv_dir() -> Path: - """Return the configured Dynamo virtual-environment directory.""" - configured = os.environ.get("NEMO_RL_DYNAMO_VENV_DIR") - if configured: - return Path(configured).expanduser().resolve() - repo_root = Path(__file__).resolve().parents[4] - return repo_root / "venvs" / "dynamo" +# Every vLLM worker venv carries the locked ai-dynamo release, so the frontend +# runs from the async worker's venv on the same release as every engine. +VLLM_ASYNC_WORKER_FQN = ( + "nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker" +) -def get_dynamo_executable(name: str) -> str: - """Return an executable in the Dynamo environment, failing if absent.""" - executable = get_dynamo_venv_dir() / "bin" / name - if not executable.is_file() or not os.access(executable, os.X_OK): - raise FileNotFoundError( - f"Dynamo executable {executable} is unavailable. Build with " - "BUILD_DYNAMO=1 or run docker/dynamo/install.sh with " - "NEMO_RL_DYNAMO_VENV_DIR set." - ) - return str(executable) +def get_dynamo_python() -> str: + """Return the vLLM async worker's Python, which carries ``ai-dynamo``. + The driver launches ``dynamo.frontend`` with it. A uv-managed venv is + resolved as the workers' own is: an existing one is reused as-is, and a + missing one is built once on each node. Re-syncing it here would change a + venv the engines are already running from. + """ + py_executable = get_actor_python_env(VLLM_ASYNC_WORKER_FQN) + if py_executable.startswith("uv"): + return create_local_venv_on_each_node(py_executable, VLLM_ASYNC_WORKER_FQN) + return py_executable -def get_dynamo_python() -> str: - """Return the validated Python interpreter for ``ai-dynamo``.""" - return get_dynamo_executable("python") + +def get_dynamo_service_executable(name: str) -> str: + """Return the etcd or nats-server binary the runtime starts. + + Looks in ``NEMO_RL_DYNAMO_BIN_DIR`` when it is set, otherwise on ``PATH``. + """ + bin_dir = os.environ.get("NEMO_RL_DYNAMO_BIN_DIR") + if bin_dir: + executable = Path(bin_dir).expanduser() / name + if executable.is_file() and os.access(executable, os.X_OK): + return str(executable) + raise FileNotFoundError( + f"{name} is not an executable file in NEMO_RL_DYNAMO_BIN_DIR={bin_dir}." + ) + found = shutil.which(name) + if found is None: + raise FileNotFoundError( + f"{name} is not on PATH. Install it, or set NEMO_RL_DYNAMO_BIN_DIR to " + "the directory that holds etcd and nats-server." + ) + return found diff --git a/nemo_rl/models/generation/dynamo/worker_pool.py b/nemo_rl/models/generation/dynamo/worker_pool.py deleted file mode 100644 index 1042e542f64..00000000000 --- a/nemo_rl/models/generation/dynamo/worker_pool.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Fixed Ray-managed pool of Dynamo vLLM subprocess owners.""" - -import os -from typing import Any - -import ray -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy - -from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env -from nemo_rl.distributed.virtual_cluster import ( - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH, - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, - DEFAULT_VLLM_PORT_RANGE_LOW, - DEFAULT_VLLM_PORTS_PER_ENGINE, - RayVirtualCluster, -) -from nemo_rl.models.generation.dynamo.dynamo_worker import ( - DynamoGpuReservation, - DynamoVllmWorker, -) - -_WORKER_FQN = "nemo_rl.models.generation.dynamo.dynamo_worker.DynamoVllmWorker" - - -def _vllm_port_for_node_slot(node_slot: int) -> int: - """Return vLLM's node-local scan base with 100 ports of headroom.""" - return DEFAULT_VLLM_PORT_RANGE_LOW + node_slot * DEFAULT_VLLM_PORTS_PER_ENGINE - - -class FixedDynamoWorkerPool: - """Reserve inference GPUs and launch one worker per model-parallel group.""" - - def __init__( - self, - *, - cluster: RayVirtualCluster, - config: dict[str, Any], - namespace: str, - engine_world_size: int, - manager_env: dict[str, str], - startup_timeout_s: float, - ) -> None: - self._cluster = cluster - self._config = config - self._namespace = namespace - self._engine_world_size = engine_world_size - self._manager_env = manager_env - self._startup_timeout_s = startup_timeout_s - self._workers: list[ray.actor.ActorHandle] = [] - self._reservations: list[ray.actor.ActorHandle] = [] - self._cleanup_reservations: list[ray.actor.ActorHandle] = [] - self._reservation_metadata: list[dict[str, Any]] = [] - self._metadata: list[dict[str, Any]] = [] - - @property - def size(self) -> int: - return len(self._workers) - - def is_alive(self) -> bool: - """Return whether every managed vLLM subprocess is still alive.""" - return bool(self._workers) and all( - ray.get([worker.is_alive.remote() for worker in self._workers]) - ) - - def start(self) -> None: - if self._workers: - raise RuntimeError("Managed Dynamo worker pool is already started.") - placement_groups = self._cluster.get_placement_groups() - python_env = get_actor_python_env(_WORKER_FQN) - runtime_env: dict[str, Any] = { - "py_executable": python_env, - "env_vars": dict(os.environ), - } - - group_index = 0 - engine_slots_by_node: dict[str, int] = {} - system_ports_by_node: dict[str, set[int]] = {} - metadata_refs: list[ray.ObjectRef] = [] - for pg_index, placement_group in enumerate(placement_groups): - bundle_count = placement_group.bundle_count - if bundle_count % self._engine_world_size != 0: - raise ValueError( - f"Inference placement group {pg_index} has {bundle_count} GPU " - f"bundles, which is not divisible by engine_world_size=" - f"{self._engine_world_size}." - ) - for start in range(0, bundle_count, self._engine_world_size): - bundle_indices = list(range(start, start + self._engine_world_size)) - reservation_handles = [] - for bundle_index in bundle_indices: - strategy = PlacementGroupSchedulingStrategy( - placement_group=placement_group, - placement_group_bundle_index=bundle_index, - placement_group_capture_child_tasks=True, - ) - reservation_handles.append( - DynamoGpuReservation.options( - num_gpus=1, - runtime_env=runtime_env, - scheduling_strategy=strategy, - ).remote() - ) - self._reservations.extend(reservation_handles) - reservation_metadata = ray.get( - [handle.metadata.remote() for handle in reservation_handles] - ) - self._reservation_metadata.extend(reservation_metadata) - node_ips = {item["node_ip"] for item in reservation_metadata} - if len(node_ips) != 1: - raise RuntimeError( - "A managed Dynamo engine group spans multiple nodes. " - "Multi-node TP/PP is not supported in the fixed-fleet milestone." - ) - cuda_devices = [item["gpu_id"] for item in reservation_metadata] - node_ip = next(iter(node_ips)) - node_slot = engine_slots_by_node.get(node_ip, 0) - engine_slots_by_node[node_ip] = node_slot + 1 - system_ports = system_ports_by_node.setdefault(node_ip, set()) - system_port = ray.get( - reservation_handles[0].select_free_port.remote( - port_range_low=DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, - port_range_high=DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH, - excluded_ports=sorted(system_ports), - ) - ) - system_ports.add(system_port) - vllm_port = _vllm_port_for_node_slot(node_slot) - group_name = f"{self._namespace}-dynamo-vllm-{pg_index}-{group_index}" - leader_strategy = PlacementGroupSchedulingStrategy( - placement_group=placement_group, - placement_group_bundle_index=bundle_indices[0], - placement_group_capture_child_tasks=True, - ) - worker = DynamoVllmWorker.options( - num_gpus=0, - runtime_env={ - **runtime_env, - "env_vars": { - **runtime_env["env_vars"], - "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1", - }, - }, - scheduling_strategy=leader_strategy, - name=group_name, - ).remote( - self._config, - namespace=self._namespace, - group_name=group_name, - cuda_devices=cuda_devices, - system_port=system_port, - vllm_port=vllm_port, - manager_env=self._manager_env, - startup_timeout_s=self._startup_timeout_s, - seed=pg_index * 1024 + group_index, - cleanup_reservation=reservation_handles[0], - ) - self._workers.append(worker) - self._cleanup_reservations.append(reservation_handles[0]) - self._metadata.append({}) - metadata_refs.append(worker.metadata.remote()) - group_index += 1 - - metadata_error: Exception | None = None - for index, metadata_ref in enumerate(metadata_refs): - try: - self._metadata[index] = dict(ray.get(metadata_ref)) - except Exception as error: - if metadata_error is None: - metadata_error = error - if metadata_error is not None: - raise metadata_error - - def refit_workers(self) -> list[dict[str, Any]]: - return [dict(item) for item in self._metadata] - - def validate(self, expected: list[dict[str, Any]]) -> list[dict[str, Any]]: - if not self._workers or not all( - ray.get([w.is_alive.remote() for w in self._workers]) - ): - raise RuntimeError("A Ray-managed Dynamo vLLM worker exited.") - try: - current_reservations = ray.get( - [reservation.metadata.remote() for reservation in self._reservations] - ) - except Exception as exc: - raise RuntimeError( - "A Ray-managed Dynamo GPU reservation actor exited." - ) from exc - if current_reservations != self._reservation_metadata: - raise RuntimeError( - "Ray-managed Dynamo GPU reservation membership changed: " - f"expected={self._reservation_metadata}, current={current_reservations}." - ) - current = ray.get([worker.metadata.remote() for worker in self._workers]) - if current != expected: - raise RuntimeError( - "Ray-managed Dynamo worker membership changed after NCCL collective " - f"initialization: expected={expected}, current={current}." - ) - return [dict(item) for item in current] - - def shutdown(self) -> None: - for worker, reservation in zip( - self._workers, - self._cleanup_reservations, - strict=True, - ): - try: - ray.get(worker.shutdown.remote(), timeout=30) - except Exception: - try: - ray.get( - reservation.cleanup_process_group.remote(), - timeout=15, - ) - except Exception: - pass - for worker in self._workers: - try: - ray.kill(worker, no_restart=True) - except Exception: - pass - for reservation in self._reservations: - try: - ray.kill(reservation, no_restart=True) - except Exception: - pass - self._workers.clear() - self._reservations.clear() - self._cleanup_reservations.clear() - self._reservation_metadata.clear() - self._metadata.clear() diff --git a/nemo_rl/models/generation/interfaces.py b/nemo_rl/models/generation/interfaces.py index 47204458008..a067106d1ef 100644 --- a/nemo_rl/models/generation/interfaces.py +++ b/nemo_rl/models/generation/interfaces.py @@ -289,9 +289,6 @@ def should_use_async_rollouts( return False backend = generation_config.get("backend", "") - if backend == "dynamo": - return True - if backend == "sglang": return bool(generation_config.get("use_async_rollouts", False)) diff --git a/nemo_rl/models/generation/openai_server_utils.py b/nemo_rl/models/generation/openai_server_utils.py index f01cd1f6091..53a757dc04f 100644 --- a/nemo_rl/models/generation/openai_server_utils.py +++ b/nemo_rl/models/generation/openai_server_utils.py @@ -58,13 +58,48 @@ def replace_prefix_tokens( ).token_ids +def _eos_token_id_set(eos_token_id: int | Collection[int]) -> frozenset[int]: + eos_token_ids = ( + frozenset([eos_token_id]) + if isinstance(eos_token_id, int) + else frozenset(eos_token_id) + ) + assert eos_token_ids, "EOS token id collection must not be empty" + return eos_token_ids + + +def derive_prefix_eos_count( + model_prefix_token_ids: list[int], *, eos_token_id: int | Collection[int] +) -> int: + """The EOS count a template render of the prefix's history would hold. + + For hosts that see only the rendered prompt, not the messages (the managed + Dynamo engine receives token ids from its frontend), this stands in for + counting EOS in ``template_prefix_token_ids``. Every EOS in the prefix + closed one message, and a template closes the final assistant turn with one + EOS even when the model was truncated before emitting it. The two counts + agree when the history up to the prefix's last assistant turn is the + history the prefix was generated from (Gym's lineage admission checks this + for captured calls; without capture it is assumed, as on the native server) + and the model emitted no EOS text as ordinary tokens. A render with fewer + EOS than this count fails the splice; an extra EOS before the boundary cuts + early instead. + """ + eos_token_ids = _eos_token_id_set(eos_token_id) + count = sum(tid in eos_token_ids for tid in model_prefix_token_ids) + if not model_prefix_token_ids or model_prefix_token_ids[-1] not in eos_token_ids: + count += 1 + return count + + def splice_prefix_tokens( *, tokenizer: Any, model_prefix_token_ids: list[int], - template_prefix_token_ids: list[int], + template_prefix_token_ids: list[int] | None = None, template_token_ids: list[int], eos_token_id: int | Collection[int] | None = None, + template_prefix_eos_count: int | None = None, ) -> PrefixSplice: """This is a subroutine used inside the OpenAI-compatible Chat Completion server. @@ -132,19 +167,24 @@ def splice_prefix_tokens( toward the boundary, and when the model's prefix ends on one of them that exact id is kept in place of the template's, so the spliced prompt still starts with the model's tokens verbatim. + + Pass exactly one of ``template_prefix_token_ids`` and + ``template_prefix_eos_count``. The prefix render is only ever used for its + EOS count, so a host without the messages passes the count directly (see + :func:`derive_prefix_eos_count`). """ + if (template_prefix_token_ids is None) == (template_prefix_eos_count is None): + raise ValueError( + "Pass exactly one of template_prefix_token_ids and " + "template_prefix_eos_count" + ) if not model_prefix_token_ids: return PrefixSplice(template_token_ids, 0, 0) if eos_token_id is None: eos_token_id = tokenizer.eos_token_id assert eos_token_id is not None, "Tokenizer must have an EOS token ID" - eos_token_ids = ( - frozenset([eos_token_id]) - if isinstance(eos_token_id, int) - else frozenset(eos_token_id) - ) - assert eos_token_ids, "EOS token id collection must not be empty" + eos_token_ids = _eos_token_id_set(eos_token_id) # The model isn't guaranteed to end on EOS (e.g. it hit max_tokens); chat # templates always add one, so cut the model input to just before its EOS. @@ -157,7 +197,11 @@ def splice_prefix_tokens( # templates may strip prior reasoning blocks when re-rendering history; # EOS counting preserves the original generated reasoning tokens without # requiring a customized chat template. - count_needed = sum(tid in eos_token_ids for tid in template_prefix_token_ids) + count_needed = ( + template_prefix_eos_count + if template_prefix_token_ids is None + else sum(tid in eos_token_ids for tid in template_prefix_token_ids) + ) count_seen = 0 template_cut_start = -1 for pos, tid in enumerate(template_token_ids): @@ -171,14 +215,14 @@ def splice_prefix_tokens( message = ( f"EOS token #{count_needed} not found in template_token_ids " f"(only found {count_seen} EOS tokens total)!\n" - f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" - f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" ) + if template_prefix_token_ids is not None: + message += f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" + message += f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" if tokenizer is not None: - message += ( - f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}\n\n" - f"Template repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" - ) + if template_prefix_token_ids is not None: + message += f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}" + message += f"\n\nTemplate repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" raise AssertionError(message) boundary = ( diff --git a/nemo_rl/models/generation/vllm/config.py b/nemo_rl/models/generation/vllm/config.py index 9bb7ca05791..5107f142983 100644 --- a/nemo_rl/models/generation/vllm/config.py +++ b/nemo_rl/models/generation/vllm/config.py @@ -25,6 +25,7 @@ PositiveInt, ) +from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.interfaces import ( GenerationConfig, get_num_routed_experts, @@ -246,6 +247,13 @@ class VllmConfig(GenerationConfig): # ACTOR_ENVIRONMENT_REGISTRY. worker_extension_cls_fqn: NotRequired[str | None] + # Serve these same workers behind Dynamo's frontend (OpenAI API and KV-aware + # router). Every other vLLM setting keeps its meaning. Implies + # vllm_cfg.async_engine, since Dynamo drives the engine's AsyncLLM. + dynamo: NotRequired[bool] + # Optional frontend and registration settings; DynamoCfg holds the defaults. + dynamo_cfg: NotRequired[DynamoCfg] + def validate_vllm_quantization_config(config: VllmConfig) -> None: """Reject quantization options that would otherwise be silently ignored.""" diff --git a/nemo_rl/models/generation/vllm/metric_names.py b/nemo_rl/models/generation/vllm/metric_names.py index 5c369d2566b..98a64634164 100644 --- a/nemo_rl/models/generation/vllm/metric_names.py +++ b/nemo_rl/models/generation/vllm/metric_names.py @@ -31,7 +31,7 @@ # Engine series surfaced per step, beyond the spec-decode family. These are the # family names vLLM registers (vllm/v1/metrics/loggers.py); its reader returns # family names, which prometheus_client strips of ``_total``, so no suffixed -# alias is needed here (unlike the HTTP-scrape path in dynamo/metrics.py). +# alias is needed here. # test_engine_metric_names_match_vllm checks these against vLLM itself. PROMPT_TOKEN_COUNTERS = ("vllm:prompt_tokens",) GENERATION_TOKEN_COUNTERS = ("vllm:generation_tokens",) diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py new file mode 100644 index 00000000000..b57808f0078 --- /dev/null +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -0,0 +1,217 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Turn a finished vLLM rollout into NeMo RL's per-sample generation output. + +Every path that serves a rollout from a NeMo RL vLLM engine goes through here: +the worker's own ``generate_async``, and Dynamo serving (the engine builds the +result, the driver assembles the output). A rollout feature added here, such +as router replay (R3), therefore behaves the same on every path. +""" + +import logging +from dataclasses import dataclass +from typing import Any, Optional, cast + +import torch + +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.models.generation.interfaces import GenerationOutputSpec +from nemo_rl.models.generation.vllm.utils import pad_and_align_routed_expert_indices +from nemo_rl.utils.routed_experts_codec import ( + decode_routed_experts, + encode_routed_experts, + routed_experts_envelope_dtype, +) + +LOGGER = logging.getLogger(__name__) + + +@dataclass +class RolloutResult: + """One rollout's outcome, independent of where its output is assembled.""" + + token_ids: list[int] + # The sampled token's log probability at each position; 0.0 where vLLM + # reported none. + logprobs: list[float] + # The engine stopped at max_tokens. + truncated: bool + # [prompt + generated, num_moe_layers, topk], aligned for router replay. + routed_experts: Optional[torch.Tensor] + # missing/expected/actual route counts; set when router replay is on. + r3_stats: Optional[dict[str, int]] + + @classmethod + def nothing_generated(cls) -> "RolloutResult": + """The result when no new token fits: the prompt alone, untruncated.""" + return cls([], [], False, None, None) + + +def selected_logprobs(completion: Any) -> list[float]: + """The sampled token's log probability at each position of a completion.""" + by_position = completion.logprobs or () + logprobs = [] + for idx, token_id in enumerate(completion.token_ids): + candidates = by_position[idx] if idx < len(by_position) else None + entry = candidates.get(token_id) if candidates else None + logprobs.append(0.0 if entry is None else float(entry.logprob)) + return logprobs + + +def rollout_result( + final_output: Any, + *, + input_length: int, + return_routed_experts: bool, + routed_experts_dtype: torch.dtype, + device: torch.device, + sample_idx: int = 0, +) -> RolloutResult: + """Build the result of a finished rollout from vLLM's final RequestOutput. + + Args: + final_output: The request's last (cumulative) ``RequestOutput``. + input_length: Prompt length in tokens. + return_routed_experts: Router replay is on, so routes are required. + routed_experts_dtype: The dtype router replay consumes. + device: Where to place the routed-experts tensor. + sample_idx: For log messages only. + """ + completion = final_output.outputs[0] + token_ids = list(completion.token_ids) + total_length = input_length + len(token_ids) + # return_stats=True returns the (routes, stats) pair. + routed_experts, r3_stats = cast( + tuple[Optional[torch.Tensor], dict[str, int]], + pad_and_align_routed_expert_indices( + final_output, + completion, + valid_length=total_length, + padded_length=total_length, + device=device, + require_complete_routed_experts=return_routed_experts, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ), + ) + if return_routed_experts and routed_experts is None: + raise RuntimeError( + "vLLM was asked to return routed experts but the generation output " + "did not include routed_experts." + ) + if return_routed_experts and r3_stats["missing_routes"] > 0: + LOGGER.warning( + "R3 router replay fallback: vLLM returned incomplete " + "routed_experts for sample_idx=%d, missing_token_routes=%d, " + "actual_routes=%d, expected_routes=%d. Megatron will use its " + "own router for those missing token routes.", + sample_idx, + r3_stats["missing_routes"], + r3_stats["actual_routes"], + r3_stats["expected_routes"], + ) + return RolloutResult( + token_ids=token_ids, + logprobs=selected_logprobs(completion), + truncated=completion.finish_reason == "length", + routed_experts=routed_experts, + r3_stats=r3_stats if return_routed_experts else None, + ) + + +def rollout_output( + result: RolloutResult, + *, + input_ids: torch.Tensor, + input_length: int, + pad_token_id: int, +) -> BatchedDataDict[GenerationOutputSpec]: + """The single-sample output ``generate_async`` yields for ``result``. + + Args: + result: The rollout's result. + input_ids: The sample's (right-padded) prompt row. + input_length: Prompt length in tokens. + pad_token_id: The tokenizer's pad token, as ``generate_async`` uses it. + """ + device = input_ids.device + generated = result.token_ids + output_length = input_length + len(generated) + output_ids = torch.full( + (output_length,), pad_token_id, dtype=input_ids.dtype, device=device + ) + output_ids[:input_length] = input_ids[:input_length] + output_ids[input_length:] = torch.tensor( + generated, dtype=input_ids.dtype, device=device + ) + logprobs = torch.zeros((1, output_length), dtype=torch.float32, device=device) + logprobs[0, input_length:] = torch.tensor( + result.logprobs, dtype=torch.float32, device=device + ) + output: dict[str, Any] = { + "output_ids": output_ids.unsqueeze(0), + "logprobs": logprobs, + "generation_lengths": torch.tensor( + [len(generated)], dtype=torch.long, device=device + ), + "unpadded_sequence_lengths": torch.tensor( + [output_length], dtype=torch.long, device=device + ), + "truncated": torch.tensor([result.truncated], dtype=torch.bool, device=device), + } + if result.r3_stats is not None: + for key in ("missing_routes", "expected_routes", "actual_routes"): + output[f"r3_routed_experts_{key}"] = torch.tensor( + [result.r3_stats[key]], dtype=torch.long, device=device + ) + if result.routed_experts is not None: + output["routed_experts"] = result.routed_experts.to(device).unsqueeze(0) + return BatchedDataDict[GenerationOutputSpec](output) + + +def rollout_result_to_wire(result: RolloutResult) -> dict[str, Any]: + """JSON-safe form of ``result``, for engines that answer over HTTP.""" + return { + "token_ids": result.token_ids, + "logprobs": result.logprobs, + "truncated": result.truncated, + "routed_experts": ( + None + if result.routed_experts is None + else encode_routed_experts(result.routed_experts) + ), + "r3_stats": result.r3_stats, + } + + +def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: + """Inverse of :func:`rollout_result_to_wire`; routes keep their own dtype.""" + routed_experts = wire["routed_experts"] + if routed_experts is not None: + routed_experts = decode_routed_experts( + routed_experts, routed_experts_envelope_dtype(routed_experts) + ) + if len(wire["logprobs"]) != len(wire["token_ids"]): + raise ValueError( + f"rollout result has {len(wire['logprobs'])} logprobs for " + f"{len(wire['token_ids'])} tokens" + ) + return RolloutResult( + token_ids=list(wire["token_ids"]), + logprobs=list(wire["logprobs"]), + truncated=bool(wire["truncated"]), + routed_experts=routed_experts, + r3_stats=wire["r3_stats"], + ) diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index b77da9c6e6f..2d871819ae7 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -318,6 +318,57 @@ def _as_routed_experts_tensor( return tensor.to(dtype=dtype) +def off_policy_sampling_error( + *, + temperature: Optional[float], + top_p: Optional[float], + top_k: Optional[int], + generation_config: Any, +) -> Optional[str]: + """Why an HTTP rollout request would sample off-policy, or None if it would not. + + Request sampling must match NeMo RL's exactly, or inference runs off-policy and + destabilizes training. Validation rollouts are the one exception: they carry the + validation profile (``val_temperature`` / ``val_top_p``), which is metric-only. + Multi-turn agents issue their own requests, so the serving HTTP endpoint is the + one chokepoint they all pass. + + Args: + temperature: The request's temperature. + top_p: The request's top_p. It must be explicit: vLLM resolves an unset top_p + from the model's generation_config.json, not to 1.0. + top_k: The request's top_k, which must be unset or -1 (disabled), matching + ``BaseVllmGenerationWorker._build_sampling_params``. + generation_config: ``policy.generation``, with the train and validation + sampling parameters. + """ + if top_k not in (None, -1): + return f"Top k sampling parameter must be unset, empty, or -1. Got `{top_k}`" + if top_p is None: + return ( + "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " + "resolved by vLLM from the model's generation_config.json and would " + "bypass the on-policy sampling check." + ) + is_train_sampling = ( + temperature == generation_config["temperature"] + and top_p == generation_config["top_p"] + ) + is_val_sampling = ( + temperature == generation_config["val_temperature"] + and top_p == generation_config["val_top_p"] + ) + if is_train_sampling or is_val_sampling: + return None + return ( + f"request sampling (temperature={temperature}, top_p={top_p}) matches neither " + f"the train sampling params (temperature={generation_config['temperature']}, " + f"top_p={generation_config['top_p']}) nor the validation sampling params " + f"(val_temperature={generation_config['val_temperature']}, " + f"val_top_p={generation_config['val_top_p']})" + ) + + def validate_rollout_prompt(expected: list[int], actual: list[int] | None) -> None: """Reject rollouts generated from a different prompt than the learner's.""" if actual != expected: @@ -496,6 +547,58 @@ def pad_and_align_routed_expert_indices( return (full, stats) if return_stats else full +def chat_routed_experts( + final_request_output: Any, + generation_details: Any, + *, + prompt_token_count: int, + device: torch.device, + logger: Any, + routed_experts_dtype: torch.dtype, +) -> str: + """The routed-experts envelope the chat endpoint attaches to one choice.""" + generation_token_count = len(getattr(generation_details, "token_ids", []) or []) + routed_result = pad_and_align_routed_expert_indices( + final_request_output, + generation_details, + valid_length=prompt_token_count + generation_token_count, + padded_length=prompt_token_count + generation_token_count, + device=device, + require_complete_routed_experts=True, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ) + if not isinstance(routed_result, tuple): + raise RuntimeError( + "Expected routed_experts alignment to return stats for the " + "OpenAI-compatible chat endpoint." + ) + routed_experts, r3_stats = routed_result + if routed_experts is None: + raise RuntimeError( + "vLLM was asked to return routed experts for the " + "OpenAI-compatible chat endpoint but the generation " + "output did not include routed_experts." + ) + if r3_stats["missing_routes"] > 0 and logger is not None: + logger.warning( + "R3 router replay fallback: vLLM returned incomplete " + "routed_experts for chat choice_idx=%d, " + "missing_token_routes=%d, actual_routes=%d, " + "expected_routes=%d. Megatron will use its own router " + "for those missing token routes.", + generation_details.index, + r3_stats["missing_routes"], + r3_stats["actual_routes"], + r3_stats["expected_routes"], + ) + # Base64 envelope instead of .tolist(): nested JSON int lists cost + # ~1s of CPU per serialize/parse hop at long context lengths and get + # re-validated at every gym HTTP hop; a single string passes through + # the gym chain opaquely. + return encode_routed_experts(routed_experts.to(dtype=routed_experts_dtype)) + + def attach_routed_experts_to_chat_response_choices( response: Any, final_request_output: Any, @@ -519,49 +622,14 @@ def attach_routed_experts_to_chat_response_choices( if generation_details is None: continue attached_choice_indices.add(choice.index) - - generation_token_count = len(getattr(generation_details, "token_ids", []) or []) - routed_result = pad_and_align_routed_expert_indices( + choice.message.routed_experts = chat_routed_experts( final_request_output, generation_details, - valid_length=prompt_token_count + generation_token_count, - padded_length=prompt_token_count + generation_token_count, + prompt_token_count=prompt_token_count, device=device, - require_complete_routed_experts=True, - return_stats=True, + logger=logger, routed_experts_dtype=routed_experts_dtype, ) - if not isinstance(routed_result, tuple): - raise RuntimeError( - "Expected routed_experts alignment to return stats for the " - "OpenAI-compatible chat endpoint." - ) - routed_experts, r3_stats = routed_result - if routed_experts is None: - raise RuntimeError( - "vLLM was asked to return routed experts for the " - "OpenAI-compatible chat endpoint but the generation " - "output did not include routed_experts." - ) - if r3_stats["missing_routes"] > 0 and logger is not None: - logger.warning( - "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for chat choice_idx=%d, " - "missing_token_routes=%d, actual_routes=%d, " - "expected_routes=%d. Megatron will use its own router " - "for those missing token routes.", - choice.index, - r3_stats["missing_routes"], - r3_stats["actual_routes"], - r3_stats["expected_routes"], - ) - # Base64 envelope instead of .tolist(): nested JSON int lists cost - # ~1s of CPU per serialize/parse hop at long context lengths and get - # re-validated at every gym HTTP hop; a single string passes through - # the gym chain opaquely. - choice.message.routed_experts = encode_routed_experts( - routed_experts.to(dtype=routed_experts_dtype) - ) if len(attached_choice_indices) != len(choices): missing_choice_indices = sorted( @@ -623,50 +691,9 @@ def attach_token_information_to_chat_response_choices( ) for choice in choices: - generation_details = outputs_by_index[choice.index] - output_token_ids = getattr(generation_details, "token_ids", None) - if output_token_ids is None: - raise RuntimeError( - "vLLM was asked to return token information for the " - "OpenAI-compatible chat endpoint but generation output " - f"choice_idx={choice.index} did not include token_ids." - ) - generation_token_ids = list(output_token_ids) - - generation_logprob_details = getattr(generation_details, "logprobs", None) - if generation_logprob_details is None: - if generation_token_ids: - raise RuntimeError( - "vLLM was asked to return token information for the " - "OpenAI-compatible chat endpoint but generation output " - f"choice_idx={choice.index} did not include logprobs." - ) - generation_log_probs = [] - else: - if len(generation_token_ids) != len(generation_logprob_details): - raise RuntimeError( - "vLLM returned mismatched generation token IDs and log " - "probabilities for the OpenAI-compatible chat endpoint: " - f"choice_idx={choice.index}, " - f"token_count={len(generation_token_ids)}, " - f"logprob_count={len(generation_logprob_details)}." - ) - generation_log_probs = [] - for token_id, position_logprobs in zip( - generation_token_ids, generation_logprob_details - ): - selected_token_logprob = position_logprobs.get(token_id) - if selected_token_logprob is None: - raise RuntimeError( - "vLLM generation log probabilities did not include the " - "selected token while attaching token information to " - "the OpenAI-compatible chat response: " - f"choice_idx={choice.index}, token_id={token_id}." - ) - generation_log_probs.append( - max(float(selected_token_logprob.logprob), VLLM_LOGPROB_FLOOR) - ) - + generation_token_ids, generation_log_probs = chat_generation_tokens( + outputs_by_index[choice.index] + ) choice.message.prompt_token_ids = list(prompt_token_ids) choice.message.generation_token_ids = generation_token_ids choice.message.generation_log_probs = generation_log_probs @@ -674,6 +701,58 @@ def attach_token_information_to_chat_response_choices( return response +def chat_generation_tokens(generation_details: Any) -> tuple[list[int], list[float]]: + """One choice's generated token IDs and their sampled-token log probabilities. + + As the chat endpoint reports them in ``generation_token_ids`` and + ``generation_log_probs``: each log probability is floored at + ``VLLM_LOGPROB_FLOOR``. + """ + choice_index = generation_details.index + output_token_ids = getattr(generation_details, "token_ids", None) + if output_token_ids is None: + raise RuntimeError( + "vLLM was asked to return token information for the " + "OpenAI-compatible chat endpoint but generation output " + f"choice_idx={choice_index} did not include token_ids." + ) + generation_token_ids = list(output_token_ids) + + generation_logprob_details = getattr(generation_details, "logprobs", None) + if generation_logprob_details is None: + if generation_token_ids: + raise RuntimeError( + "vLLM was asked to return token information for the " + "OpenAI-compatible chat endpoint but generation output " + f"choice_idx={choice_index} did not include logprobs." + ) + return generation_token_ids, [] + if len(generation_token_ids) != len(generation_logprob_details): + raise RuntimeError( + "vLLM returned mismatched generation token IDs and log " + "probabilities for the OpenAI-compatible chat endpoint: " + f"choice_idx={choice_index}, " + f"token_count={len(generation_token_ids)}, " + f"logprob_count={len(generation_logprob_details)}." + ) + generation_log_probs = [] + for token_id, position_logprobs in zip( + generation_token_ids, generation_logprob_details + ): + selected_token_logprob = position_logprobs.get(token_id) + if selected_token_logprob is None: + raise RuntimeError( + "vLLM generation log probabilities did not include the " + "selected token while attaching token information to " + "the OpenAI-compatible chat response: " + f"choice_idx={choice_index}, token_id={token_id}." + ) + generation_log_probs.append( + max(float(selected_token_logprob.logprob), VLLM_LOGPROB_FLOOR) + ) + return generation_token_ids, generation_log_probs + + def model_dump_chat_response_with_dynamic_message_fields( response: Any, ) -> dict[str, Any]: diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index eb395b4baca..7dee0203c6f 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -32,12 +32,14 @@ import ray from ray.util.placement_group import PlacementGroup +from nemo_rl.data.multimodal_utils import VLLM_PROMPT_KEYS from nemo_rl.distributed.batched_data_dict import BatchedDataDict, SlicedDataDict from nemo_rl.distributed.named_sharding import NamedSharding from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env from nemo_rl.distributed.refit_watchdog import RefitAborted from nemo_rl.distributed.virtual_cluster import NVLINK_DOMAIN_UNKNOWN, RayVirtualCluster from nemo_rl.distributed.worker_groups import RayWorkerBuilder, RayWorkerGroup +from nemo_rl.models.generation.dynamo.config import dynamo_cfg, dynamo_enabled from nemo_rl.models.generation.fleet_health import ( GenerationFleetHealth, HealthyShardSelector, @@ -47,12 +49,17 @@ GenerationInterface, GenerationOutputSpec, agreed_media_columns, + verify_right_padding, ) from nemo_rl.models.generation.vllm.config import ( REFITTABLE_FP8_KV_CACHE_DTYPES, VllmConfig, validate_vllm_quantization_config, ) +from nemo_rl.models.generation.vllm.request_pipeline import ( + rollout_output, + rollout_result_from_wire, +) from nemo_rl.models.generation.vllm.draft_utils import ( DRAFT_DISABLE_MODULE_SHARING_ENV, draft_module_sharing_disable_required, @@ -82,6 +89,7 @@ from nemo_rl.algorithms.single_controller_utils.config import MasterConfig from nemo_rl.data.captured_media import MediaColumnSpec from nemo_rl.data_plane.interfaces import DataPlaneConfig + from nemo_rl.models.generation.dynamo.managed_runtime import ManagedDynamoRuntime logger = logging.getLogger(__name__) @@ -168,11 +176,47 @@ def _record_vllm_generation_metrics( class VllmGeneration(GenerationInterface): + # Dynamo serving (policy.generation.dynamo): __init__ starts etcd, NATS and + # the frontend; each engine registers itself once its AsyncLLM runs. Class + # defaults, so shutdown() also works when __init__ failed before starting them. + _dynamo_runtime: Optional["ManagedDynamoRuntime"] = None + dynamo_frontend_url: Optional[str] = None + @classmethod def validate_settings(cls, master_config: "MasterConfig") -> None: """Reject pure-config vLLM settings the SC entrypoint cannot honor.""" generation_config = cast(VllmConfig, master_config.policy["generation"]) assert_reload_refit_config_supported(generation_config) + if dynamo_enabled(generation_config): + # Deferred: the NeMo-Gym environment module is heavy and optional. + from nemo_rl.environments.nemo_gym import should_use_nemo_gym + from nemo_rl.models.megatron.router_replay import router_replay_enabled + + if master_config.async_rl.generation_router.enabled: + raise ValueError( + "async_rl.generation_router.enabled=true does not apply with " + "policy.generation.dynamo=true: NeMo-Gym calls the Dynamo " + "frontend, which routes the whole fleet behind one URL. " + "Disable the generation router." + ) + if master_config.async_rl.generation_fleet_health.enabled: + raise ValueError( + "async_rl.generation_fleet_health.enabled=true is not supported " + "with policy.generation.dynamo=true yet: the Dynamo frontend " + "picks engines itself, and a restarted shard does not register " + "with it. Disable fleet health." + ) + if ( + should_use_nemo_gym(master_config) + and router_replay_enabled(master_config.policy) + and not master_config.token_capture.enabled + ): + raise ValueError( + "Router replay with NeMo-Gym and policy.generation.dynamo=true " + "requires token_capture.enabled=true: the engine stages routed " + "experts with the captured call, and NeMo-Gym's Dynamo dialect " + "does not return them in responses." + ) @staticmethod def init_cluster_placement_groups( @@ -262,6 +306,12 @@ def __init__( "See https://github.com/NVIDIA-NeMo/RL/issues/1101 for more details." ) + if dynamo_enabled(self.cfg) and not self.cfg["vllm_cfg"]["async_engine"]: + raise ValueError( + "policy.generation.dynamo=true serves the async vLLM engine; set " + "vllm_cfg.async_engine=true (configure_generation_config does)." + ) + # Validate sampling parameters early to avoid resource allocation with unsupported configs. top_k: int | None = self.cfg["top_k"] if top_k is not None and top_k != -1 and top_k < 1: @@ -429,6 +479,9 @@ def __init__( # and it should not be discovered with getattr at the read site. self._refit_membership: Optional["RefitMembership"] = None + if dynamo_enabled(self.cfg): + self._start_dynamo_services() + if defer_model_load: # Workers only reserved ports — collect URLs immediately and defer # the heavy model loading (and HTTP server start) to load_and_start(). @@ -443,6 +496,7 @@ def __init__( # when the http server is active. self.dp_openai_server_base_urls = self._report_dp_openai_server_base_urls() self.device_uuids = self._report_device_id() + self._register_dynamo_engines() self._step_metrics_snapshot: dict[str | tuple[str, int], float] | None = None @@ -685,6 +739,79 @@ def load_and_start(self) -> None: # Save device UUIDs self.device_uuids = self._report_device_id() + self._register_dynamo_engines() + + @property + def client_base_urls(self) -> list[Optional[str]]: + """Where OpenAI clients such as NeMo-Gym send requests. + + The Dynamo frontend when it serves this fleet (``policy.generation.dynamo``); + otherwise every data-parallel shard's own server, as + ``dp_openai_server_base_urls``. Known before a deferred model load. + """ + if self.dynamo_frontend_url is not None: + return [self.dynamo_frontend_url] + return self.dp_openai_server_base_urls + + def _start_dynamo_services(self) -> None: + """Start the etcd, NATS and frontend this fleet's engines register with.""" + from nemo_rl.models.generation.dynamo.managed_runtime import ( + ManagedDynamoRuntime, + ) + from nemo_rl.models.generation.dynamo.venv import get_dynamo_python + + runtime = ManagedDynamoRuntime( + dynamo_cfg=dynamo_cfg(self.cfg), + model_name=self.cfg["model_name"], + frontend_python=get_dynamo_python(), + ) + runtime.start() + self._dynamo_runtime = runtime + self.dynamo_frontend_url = runtime.frontend_url + + def _register_dynamo_engines(self) -> None: + """Serve every shard's engine through Dynamo; wait until it routes to all.""" + runtime = self._dynamo_runtime + if runtime is None: + return + leader_indices = [ + self.worker_group.get_dp_leader_worker_idx(shard) + for shard in range(self.dp_size) + ] + ray.get( + [ + self.worker_group.workers[idx].start_dynamo_endpoint_async.remote( + namespace=runtime.namespace, + engine_env=runtime.engine_env, + leader_worker_idx=idx, + ) + for idx in leader_indices + ] + ) + runtime.wait_for_engines( + self.dp_size, check_engines=self._check_dynamo_endpoints + ) + + def _check_dynamo_endpoints(self) -> None: + """Raise if a DP leader exited or its Dynamo endpoint stopped serving.""" + leaders = [ + self.worker_group.workers[self.worker_group.get_dp_leader_worker_idx(shard)] + for shard in range(self.dp_size) + ] + try: + errors = ray.get( + [leader.dynamo_endpoint_error.remote() for leader in leaders] + ) + except ray.exceptions.RayActorError as error: + raise RuntimeError( + "A vLLM engine exited while waiting for Dynamo registration." + ) from error + for shard, error in enumerate(errors): + if error is not None: + raise RuntimeError( + f"The Dynamo endpoint of generation shard {shard} stopped " + f"serving: {error}" + ) def _post_init(self): # Choose the appropriate method based on async_engine setting @@ -1379,11 +1506,69 @@ def validate_generate_data(data): return False # Return False for empty case to trigger early return return True + if self._dynamo_serves(data): + if validate_generate_data(data): + yield await self._generate_through_dynamo(data, greedy) + return async for result in self._async_generate_base( data, "generate_async", validate_generate_data, greedy ): yield result + def _dynamo_serves(self, data: BatchedDataDict[GenerationDatumSpec]) -> bool: + """Whether a rollout goes through the Dynamo frontend's KV-aware router. + + Media-backed prompts keep this class's own dispatch: the engine does not + yet take them through the frontend. + """ + return self.dynamo_frontend_url is not None and not ( + VLLM_PROMPT_KEYS & data.keys() + ) + + async def _generate_through_dynamo( + self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool + ) -> tuple[int, BatchedDataDict[GenerationOutputSpec]]: + """One rollout through the frontend, sampled by the engine as natively.""" + # Local import: nemo_rl.experience pulls the rollout stack, which must not + # become a load-time dependency of the generation backend. + from nemo_rl.experience.failures import GenerationUnavailable + from nemo_rl.models.generation.dynamo.rollout_client import ( + post_rollout, + rollout_payload, + ) + + assert data.size == 1, ( + "generate_async is restricted to handle only single samples, " + f"but received batch_size={data.size}. Please handle batching " + "outside this method." + ) + verify_right_padding(data, pad_value=self.cfg["_pad_token_id"]) + input_length = int(data["input_lengths"][0].item()) + input_ids = data["input_ids"][0] + stop_strings = data.get("stop_strings", [None])[0] + payload = rollout_payload( + model_name=self.cfg["model_name"], + prompt_token_ids=input_ids[:input_length].tolist(), + max_model_len=int(self.cfg["vllm_cfg"]["max_model_len"]), + greedy=greedy, + stop_strings=stop_strings, + ) + url = f"{self.dynamo_frontend_url}/completions" + try: + engine_data = await post_rollout( + url, payload, dynamo_cfg(self.cfg).request_timeout_s + ) + except RuntimeError as error: + raise GenerationUnavailable(str(error)) from error + output = rollout_output( + rollout_result_from_wire(engine_data["rollout"]), + input_ids=input_ids, + input_length=input_length, + pad_token_id=self.cfg["_pad_token_id"], + ) + output["gen_leader_worker_idx"] = [int(engine_data["gen_leader_worker_idx"])] + return 0, output + def prepare_for_generation(self, *args: Any, **kwargs: Any) -> bool: """Wake workers up for colocated inference.""" # non-colocated no need to wake up @@ -1448,6 +1633,21 @@ def shutdown(self) -> bool: except Exception as e: print(f"Error during policy shutdown: {e}") return False + finally: + # After the engines: they leave discovery as their processes exit. + if self._dynamo_runtime is not None: + self._dynamo_runtime.shutdown() + self._dynamo_runtime = None + + def __getstate__(self) -> dict[str, Any]: + """Leave the driver-owned Dynamo services behind when shipped to an actor. + + Rollout actors receive this object; a copy keeps the frontend URL and the + worker handles, but the service processes stay with the driver. + """ + state = self.__dict__.copy() + state["_dynamo_runtime"] = None + return state def prepare_refit_info( self, state_dict_info: Optional[dict[str, Any]] diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 9231aa06d22..d61641a91d8 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -49,6 +49,7 @@ ) from nemo_rl.distributed.worker_group_utils import get_nsight_config_if_pattern_matches from nemo_rl.experience.route_assembly import boundary_route_rows +from nemo_rl.models.generation.dynamo.config import dynamo_cfg, dynamo_enabled from nemo_rl.models.generation.interfaces import ( GenerationDatumSpec, GenerationOutputSpec, @@ -64,13 +65,18 @@ resolve_collective_rpc_result, ) from nemo_rl.models.generation.vllm.config import parse_nvfp4_pertoken_rollout +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result, +) from nemo_rl.models.generation.vllm.utils import ( attach_routed_experts_to_chat_response_choices, attach_token_information_to_chat_response_choices, format_prompt_for_vllm_generation, + off_policy_sampling_error, validate_rollout_prompt, model_dump_chat_response_with_dynamic_message_fields, - pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) from nemo_rl.models.generation.vllm.vllm_worker import BaseVllmGenerationWorker @@ -273,6 +279,8 @@ def __init__( self.http_server = None self._engine_loop = None self._http_engine_client = None + # The task serving this engine through Dynamo, when it is registered. + self._dynamo_endpoint_task: Optional[asyncio.Task] = None # Ledger-authoritative token capture (dormant until the # setup_token_capture fan-out runs). The weight @@ -407,6 +415,26 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: stacklevel=1, ) llm_kwargs["compilation_config"] = CompilationConfig(**compilation_config) + if dynamo_enabled(self.cfg): + from nemo_rl.models.generation.dynamo.native_engine import ( + dynamo_kv_events_config, + ) + + # Dynamo's KV-aware router indexes the engine's own KV-cache events. + llm_kwargs.setdefault( + "kv_events_config", + dynamo_kv_events_config( + dynamo_cfg(self.cfg), + prefix_caching=llm_kwargs["enable_prefix_caching"], + ), + ) + # AsyncEngineArgs leaves kv_events_config dicts unconverted. + if isinstance(llm_kwargs.get("kv_events_config"), dict): + from vllm.config import KVEventsConfig + + llm_kwargs["kv_events_config"] = KVEventsConfig( + **llm_kwargs["kv_events_config"] + ) self.llm_async_engine_args = AsyncEngineArgs(**llm_kwargs) self.stat_loggers = ( @@ -414,6 +442,13 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: if self.cfg["vllm_cfg"].get("enable_vllm_metrics_logger", False) else [] ) + if dynamo_enabled(self.cfg): + from nemo_rl.models.generation.dynamo.native_engine import ( + SnapshotStatLogger, + ) + + # Feeds the router's per-engine load signal. + self.stat_loggers.append(SnapshotStatLogger) self.llm = AsyncLLM.from_engine_args( self.llm_async_engine_args, stat_loggers=self.stat_loggers ) @@ -577,6 +612,37 @@ async def post_init_async(self): self._setup_vllm_server() ) + async def start_dynamo_endpoint_async( + self, *, namespace: str, engine_env: dict[str, str], leader_worker_idx: int + ) -> None: + """Register this engine with Dynamo and serve it from this actor's loop. + + The loop is the one that owns the AsyncLLM, so Dynamo requests, refit, + pause and sleep interleave exactly as this worker's own requests do. + ``leader_worker_idx`` is this worker's index in its group; rollouts the + engine serves report it, as ``generate_async`` results do. + """ + from nemo_rl.models.generation.dynamo.native_engine import ( + start_dynamo_endpoint, + ) + + self._dynamo_endpoint_task = start_dynamo_endpoint( + self, + namespace=namespace, + engine_env=engine_env, + leader_worker_idx=leader_worker_idx, + ) + + async def dynamo_endpoint_error(self) -> Optional[str]: + """Why this engine's Dynamo endpoint stopped serving; None while it serves.""" + task = self._dynamo_endpoint_task + if task is None or not task.done(): + return None + if task.cancelled(): + return "cancelled" + error = task.exception() + return "exited" if error is None else repr(error) + async def get_reserved_url(self) -> Optional[str]: """Return the URL from the reserved socket, available before model loading.""" if self._reserved_socket is not None: @@ -691,7 +757,10 @@ def _capture_admission(self, request: Any) -> Any | None: context. The dict itself is never mutated: the admission is the typed, read-only contract that the prefix resolution and ``begin_call`` share. """ - context = getattr(request, "ng_capture", None) + return self._parse_capture_admission(getattr(request, "ng_capture", None)) + + def _parse_capture_admission(self, context: Any) -> Any | None: + """``_capture_admission`` for a raw ``ng_capture`` context (Dynamo's annotation).""" if self.token_capture is None or not context: return None # Deferred: nemo_gym is an optional extra absent in non-gym runs. @@ -1473,46 +1542,19 @@ class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): async def create_chat_completion( request: NeMoRLChatCompletionRequest, raw_request: Request ): - # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params - # Right now we explicitly assert set this to -1. - assert request.top_k in (None, -1), ( - f"Top k sampling parameter must be unset, empty, or -1. Got `{request.top_k}`" - ) - request.top_k = -1 - # The request sampling params need to exactly match those as are set in NeMo RL. # If they do not match, the inference will be off policy and destroy training - # stability. Validation rollouts are the one exception: they are stamped with - # the validation sampling profile (generation.val_temperature / val_top_p), - # which is metric-only and safe to serve — grpo.validate() is the only - # caller that constructs a non-train GenerationSamplingParams. Multi-turn - # agents issue their own requests, so this server-side check is the one - # chokepoint they all pass. - # vLLM resolves an unset top_p from the model's generation_config.json - # (ModelConfig.generation_config defaults to "auto"), NOT to 1.0, so a - # request omitting it would sample off-policy while passing this check. - assert request.top_p is not None, ( - "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " - "resolved by vLLM from the model's generation_config.json and would " - "bypass the on-policy sampling check." - ) - request_top_p = request.top_p - is_train_sampling = ( - request.temperature == generation_config["temperature"] - and request_top_p == generation_config["top_p"] - ) - is_val_sampling = ( - request.temperature == generation_config["val_temperature"] - and request_top_p == generation_config["val_top_p"] - ) - assert is_train_sampling or is_val_sampling, ( - f"request sampling (temperature={request.temperature}, " - f"top_p={request.top_p}) matches neither the train sampling params " - f"(temperature={generation_config['temperature']}, " - f"top_p={generation_config['top_p']}) nor the validation sampling " - f"params (val_temperature={generation_config['val_temperature']}, " - f"val_top_p={generation_config['val_top_p']})" + # stability. grpo.validate() is the only caller that constructs a non-train + # GenerationSamplingParams, and the validation profile it stamps is accepted. + sampling_error = off_policy_sampling_error( + temperature=request.temperature, + top_p=request.top_p, + top_k=request.top_k, + generation_config=generation_config, ) + assert sampling_error is None, sampling_error + # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params + request.top_k = -1 try: generator = await openai_serving_chat.create_chat_completion( @@ -1813,6 +1855,55 @@ async def init_collective_async( ), ) + def _rollout_max_new_tokens(self, input_length: int) -> int: + """Token budget for one rollout prompt; 0 means nothing fits. + + The configured cap, clipped to the context window and to speculative + decoding's lookahead. Shared by ``generate_async`` and the Dynamo engine, + so a rollout gets the same budget whichever path serves it. + """ + max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) + allowed_new_tokens = max( + 0, min(self.cfg["max_new_tokens"], max_model_len - input_length) + ) + spec_cfg = self.cfg.get("vllm_kwargs", {}).get("speculative_config") or {} + spec_lookahead = int(spec_cfg.get("num_speculative_tokens", 0)) + if allowed_new_tokens > 0 and spec_lookahead > 0: + allowed_new_tokens = self._request_max_new_tokens( + configured_max_new_tokens=allowed_new_tokens, + input_length=input_length, + max_model_len=max_model_len, + cap_to_context=False, + spec_lookahead=spec_lookahead, + ) + return allowed_new_tokens + + def _rollout_sampling_params( + self, + *, + max_new_tokens: int, + greedy: bool, + stop_strings: Optional[list[str]], + ) -> Any: + """The SamplingParams for one rollout. + + Shared by ``generate_async`` and Dynamo serving, so a rollout is sampled + the same way whichever path serves it. + + Args: + max_new_tokens: The rollout's budget, from ``_rollout_max_new_tokens``. + greedy: Greedy decoding instead of the configured sampling. + stop_strings: The sample's own stop strings, merged with the + configured ones. + """ + return self._build_sampling_params( + greedy=greedy, + stop_strings=self._merge_stop_strings( + [stop_strings] if stop_strings else None + ), + max_new_tokens=max_new_tokens, + ) + async def generate_async( self, data: BatchedDataDict[GenerationDatumSpec], @@ -1865,231 +1956,49 @@ async def process_single_sample(sample_idx): ): per_sample_stop_strings = batch_specific_stop_strings_list[sample_idx] - final_stop_strings_for_sample = self._merge_stop_strings( - [per_sample_stop_strings] if per_sample_stop_strings else None - ) - - max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) - remaining_ctx = max_model_len - current_input_actual_length - allowed_new_tokens = max(0, min(self.cfg["max_new_tokens"], remaining_ctx)) - - spec_cfg = self.cfg.get("vllm_kwargs", {}).get("speculative_config") or {} - spec_lookahead = int(spec_cfg.get("num_speculative_tokens", 0)) - if allowed_new_tokens > 0 and spec_lookahead > 0: - allowed_new_tokens = self._request_max_new_tokens( - configured_max_new_tokens=allowed_new_tokens, - input_length=current_input_actual_length, - max_model_len=max_model_len, - cap_to_context=False, - spec_lookahead=spec_lookahead, - ) - - # Handle case where no tokens can be generated due to length constraints - if allowed_new_tokens == 0: - # Access the input data directly from the function parameters - input_ids_single_row = input_ids_batch[sample_idx] - - # Create output tensors with just the input (no generated tokens) - output_ids_single_item_batched = input_ids_single_row[ - :current_input_actual_length - ].unsqueeze(0) - - logprobs_single_item = torch.zeros( - (1, current_input_actual_length), - dtype=torch.float32, - device=input_ids_single_row.device, + max_new_tokens = self._rollout_max_new_tokens(current_input_actual_length) + input_ids_single_row = input_ids_batch[sample_idx] + if max_new_tokens == 0: + # No new token fits in the context: return the prompt alone. + result = RolloutResult.nothing_generated() + else: + sampling_params_for_request = self._rollout_sampling_params( + max_new_tokens=max_new_tokens, + greedy=greedy, + stop_strings=per_sample_stop_strings, ) + request_id = str(uuid.uuid4()) + final_request_output = None + async for req_output in self.llm.generate( + prompt=prompt, + sampling_params=sampling_params_for_request, + request_id=request_id, + ): + final_request_output = req_output + if final_request_output is None: + raise RuntimeError(f"No output received for request {request_id}") - generation_lengths_tensor = torch.tensor( - [0], dtype=torch.long, device=input_ids_single_row.device + validate_rollout_prompt( + input_ids_single_row[:current_input_actual_length].tolist(), + final_request_output.prompt_token_ids, ) - - unpadded_sequence_lengths_tensor = torch.tensor( - [current_input_actual_length], - dtype=torch.long, + result = rollout_result( + final_request_output, + input_length=current_input_actual_length, + return_routed_experts=self._return_routed_experts_enabled(), + routed_experts_dtype=self.routed_experts_dtype, device=input_ids_single_row.device, + sample_idx=sample_idx, ) - - # Not truncated since no generation was attempted (length constraint) - truncated_tensor = torch.tensor( - [False], dtype=torch.bool, device=input_ids_single_row.device - ) - - result_batch = BatchedDataDict[GenerationOutputSpec]( - { - "output_ids": output_ids_single_item_batched, - "logprobs": logprobs_single_item, - "generation_lengths": generation_lengths_tensor, - "unpadded_sequence_lengths": unpadded_sequence_lengths_tensor, - "truncated": truncated_tensor, - } - ) - - return (sample_idx, result_batch) - - sampling_params_for_request = self._build_sampling_params( - greedy=greedy, - stop_strings=final_stop_strings_for_sample, - max_new_tokens=allowed_new_tokens, - ) - - request_id = str(uuid.uuid4()) - - # Generate using vLLM async engine - vllm_request_generator = self.llm.generate( - prompt=prompt, - sampling_params=sampling_params_for_request, - request_id=request_id, - ) - - # Get the final result from the generator - final_request_output = None - async for req_output in vllm_request_generator: - final_request_output = req_output - - if final_request_output is None: - raise RuntimeError(f"No output received for request {request_id}") - - validate_rollout_prompt( - input_ids_batch[sample_idx, :current_input_actual_length].tolist(), - final_request_output.prompt_token_ids, - ) - - # Process the output - generation_details = final_request_output.outputs[0] - generated_token_ids = list(generation_details.token_ids) - num_generated_tokens = len(generated_token_ids) - return_routed_experts = self._return_routed_experts_enabled() - - original_input_ids_single_row = input_ids_batch[sample_idx] - final_output_tensor_len = current_input_actual_length + num_generated_tokens - - # Create output_ids tensor for this single item - output_ids_single_item = torch.full( - (final_output_tensor_len,), - self.cfg["_pad_token_id"], - dtype=original_input_ids_single_row.dtype, - device=original_input_ids_single_row.device, - ) - # Copy original input (up to its actual length) - output_ids_single_item[:current_input_actual_length] = ( - original_input_ids_single_row[:current_input_actual_length] - ) - # Add generated tokens after the actual input - output_ids_single_item[ - current_input_actual_length : current_input_actual_length - + num_generated_tokens - ] = torch.tensor( - generated_token_ids, - dtype=original_input_ids_single_row.dtype, - device=original_input_ids_single_row.device, - ) - - # Reshape to (1, seq_len) for BatchedDataDict - output_ids_single_item_batched = output_ids_single_item.unsqueeze(0) - - # Create logprobs tensor for this single item - logprobs_single_item = torch.zeros( - (1, final_output_tensor_len), - dtype=torch.float32, - device=original_input_ids_single_row.device, - ) - if hasattr(generation_details, "logprobs") and generation_details.logprobs: - for idx, logprob_dict_per_token in enumerate( - generation_details.logprobs - ): - if logprob_dict_per_token and idx < len(generated_token_ids): - token_id_at_idx = generated_token_ids[idx] - if token_id_at_idx in logprob_dict_per_token: - logprob_value = logprob_dict_per_token[ - token_id_at_idx - ].logprob - position_in_output_tensor = ( - current_input_actual_length + idx - ) - if position_in_output_tensor < final_output_tensor_len: - logprobs_single_item[0, position_in_output_tensor] = ( - logprob_value - ) - - # Generation lengths - generation_lengths_tensor = torch.tensor( - [num_generated_tokens], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - - # Unpadded sequence lengths (actual_input + actual_generated) - unpadded_total_length = current_input_actual_length + num_generated_tokens - unpadded_sequence_lengths_tensor = torch.tensor( - [unpadded_total_length], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - - # Check if response was truncated (hit max_tokens length limit) - is_truncated = generation_details.finish_reason == "length" - truncated_tensor = torch.tensor( - [is_truncated], - dtype=torch.bool, - device=original_input_ids_single_row.device, - ) - - result_dict = { - "output_ids": output_ids_single_item_batched, - "logprobs": logprobs_single_item, - "generation_lengths": generation_lengths_tensor, - "unpadded_sequence_lengths": unpadded_sequence_lengths_tensor, - "truncated": truncated_tensor, - } - routed_experts, r3_stats = pad_and_align_routed_expert_indices( - final_request_output, - generation_details, - valid_length=unpadded_total_length, - padded_length=final_output_tensor_len, - device=original_input_ids_single_row.device, - require_complete_routed_experts=return_routed_experts, - return_stats=True, - routed_experts_dtype=self.routed_experts_dtype, + return ( + sample_idx, + rollout_output( + result, + input_ids=input_ids_single_row, + input_length=current_input_actual_length, + pad_token_id=self.cfg["_pad_token_id"], + ), ) - if return_routed_experts and routed_experts is None: - raise RuntimeError( - "vLLM was asked to return routed experts but the generation output " - "did not include routed_experts." - ) - if return_routed_experts: - if r3_stats["missing_routes"] > 0: - LOGGER.warning( - "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for sample_idx=%d, missing_token_routes=%d, " - "actual_routes=%d, expected_routes=%d. Megatron will use its " - "own router for those missing token routes.", - sample_idx, - r3_stats["missing_routes"], - r3_stats["actual_routes"], - r3_stats["expected_routes"], - ) - result_dict["r3_routed_experts_missing_routes"] = torch.tensor( - [r3_stats["missing_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - result_dict["r3_routed_experts_expected_routes"] = torch.tensor( - [r3_stats["expected_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - result_dict["r3_routed_experts_actual_routes"] = torch.tensor( - [r3_stats["actual_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - if routed_experts is not None: - result_dict["routed_experts"] = routed_experts.unsqueeze(0) - - result_batch = BatchedDataDict[GenerationOutputSpec](result_dict) - - return (sample_idx, result_batch) # Create tasks for all samples and yield results as they complete sample_tasks = [ diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 2456d8e488f..5e511555b3e 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -201,6 +201,57 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: ) +# Keys of the managed dynamo.vllm backend's dynamo_cfg with no counterpart now that +# Dynamo serves NeMo RL's own vLLM workers, and what replaces each. +_REMOVED_DYNAMO_CFG_KEYS = { + "engine": "Dynamo always serves the vLLM backend's workers", + "control_timeout_s": "refit and pause use the vLLM backend's own controls", + "metrics_include_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", + "metrics_exclude_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", +} + + +def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: + """Fail on the removed ``backend: dynamo`` and its dynamo_cfg keys. + + Dynamo now serves NeMo RL's own vLLM workers: ``backend: vllm`` plus + ``dynamo: true``. One migration, so the backend is reported first and the + dynamo_cfg keys that no longer exist with it. + + Args: + config: The config as the user wrote it, resolved to plain dicts. + """ + policy = config.get("policy") + if not isinstance(policy, dict): + return + generation = policy.get("generation") + if not isinstance(generation, dict): + return + problems = [] + if generation.get("backend") == "dynamo": + problems.append( + "policy.generation.backend='dynamo' was removed: Dynamo now serves NeMo " + "RL's own vLLM workers. Set policy.generation.backend=vllm and " + "policy.generation.dynamo=true; vllm_cfg and vllm_kwargs keep their " + "meaning." + ) + dynamo_cfg = generation.get("dynamo_cfg") + if isinstance(dynamo_cfg, dict): + for key, replacement in _REMOVED_DYNAMO_CFG_KEYS.items(): + if key in dynamo_cfg: + problems.append( + f"policy.generation.dynamo_cfg.{key} was removed: {replacement}." + ) + worker_args = dynamo_cfg.get("worker_args") + if isinstance(worker_args, dict) and "extra_cli_args" in worker_args: + problems.append( + "policy.generation.dynamo_cfg.worker_args.extra_cli_args was " + "removed: engines are configured through vllm_cfg and vllm_kwargs." + ) + if problems: + raise ValueError(" ".join(problems)) + + def check_outdated_config(config: dict[str, Any]) -> None: """Fail fast on config the code no longer accepts, naming the migration to apply. @@ -218,3 +269,4 @@ def check_outdated_config(config: dict[str, Any]) -> None: reject_outdated_metric_name_format(config) reject_outdated_mcore_generation_keys(config) reject_outdated_dynamo_router_reset_states(config) + reject_outdated_dynamo_backend(config) diff --git a/nemo_rl/utils/routed_experts_codec.py b/nemo_rl/utils/routed_experts_codec.py index e7cc41be2c5..511748740c1 100644 --- a/nemo_rl/utils/routed_experts_codec.py +++ b/nemo_rl/utils/routed_experts_codec.py @@ -104,3 +104,11 @@ def decode_routed_experts(payload: Union[str, Any], dtype: torch.dtype) -> torch return torch.empty(shape, dtype=dtype) tensor = torch.frombuffer(raw, dtype=wire_dtype).reshape(shape) return tensor if tensor.dtype == dtype else tensor.to(dtype) + + +def routed_experts_envelope_dtype(payload: str) -> torch.dtype: + """The dtype a base64 envelope was encoded with.""" + parts = payload.split(":", 2) + if len(parts) != 3 or parts[0] != _MAGIC or parts[1] not in _WIRE_TORCH_DTYPES: + raise ValueError("routed_experts payload is not a valid envelope.") + return _WIRE_TORCH_DTYPES[parts[1]] diff --git a/nemo_rl/weight_sync/collective_weight_synchronizer.py b/nemo_rl/weight_sync/collective_weight_synchronizer.py index 6b8fa7d50ad..7ac9366d6ae 100644 --- a/nemo_rl/weight_sync/collective_weight_synchronizer.py +++ b/nemo_rl/weight_sync/collective_weight_synchronizer.py @@ -190,13 +190,13 @@ def _desired_membership( ) -> Optional[RefitMembership]: """The membership to track, or None for a backend that owns no DP worker group. - NOT every GenerationInterface has one. vLLM and TRT-LLM do; Dynamo, Megatron and - SGLang do not, and the interface declares nothing either way -- so reading + NOT every GenerationInterface has one. vLLM and TRT-LLM do; Megatron and SGLang + do not, and the interface declares nothing either way -- so reading ``self._generation.worker_group`` assumes a vLLM shape that this synchronizer is - not entitled to assume. It holds a GenerationInterface, and Dynamo reaches it - through the ordinary non-colocated branch of the factory. + not entitled to assume. - That assumption broke L1_Functional_Tests_Dynamo on the plain grpo.py path: + That assumption once broke L1_Functional_Tests_Dynamo on the plain grpo.py path, + under the since-removed managed Dynamo backend: grpo.py:1747 policy_generation.weight_synchronizer.init_communicator() AttributeError: 'DynamoGeneration' object has no attribute 'worker_group' diff --git a/nemo_rl/weight_sync/factory.py b/nemo_rl/weight_sync/factory.py index 3524b22c559..9a09ee550b2 100644 --- a/nemo_rl/weight_sync/factory.py +++ b/nemo_rl/weight_sync/factory.py @@ -27,7 +27,6 @@ from typing import Any, Optional from nemo_rl.models.generation.constants import ( - DYNAMO_BACKEND, MEGATRON_BACKEND, SGLANG_BACKEND, TRTLLM_BACKEND, @@ -92,7 +91,7 @@ def create_weight_synchronizer( policy: Policy object (ColocatablePolicyInterface). generation: Generation object (GenerationInterface). generation_backend: Name of the generation backend ("vllm", "sglang", - "megatron", or "dynamo"). + "megatron", or "trtllm"). colocated: Whether policy and generation share the same GPUs. train_cluster: RayVirtualCluster for training workers. Required for non-colocated deployments except SGLang, which owns its own group. @@ -114,7 +113,6 @@ def create_weight_synchronizer( VLLM_BACKEND, SGLANG_BACKEND, MEGATRON_BACKEND, - DYNAMO_BACKEND, TRTLLM_BACKEND, } if generation_backend not in _SUPPORTED_BACKENDS: diff --git a/pyproject.toml b/pyproject.toml index 4e2770faaf7..259d712d387 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,6 +177,11 @@ vllm = [ "flashinfer-cubin==0.6.18", "flashinfer-jit-cache==0.6.18", "nvidia-cutlass-dsl[cu13]==4.6.2", + # Dynamo serving (policy.generation.dynamo=true) hosts these vLLM workers behind + # dynamo.frontend through Dynamo's engine-agnostic backend SDK. Only the base + # package: its [vllm] extra pins Dynamo's own vLLM, which NeMo RL never runs. + "ai-dynamo==1.5.1", + "ai-dynamo-runtime==1.5.1", ] sglang = [ # sglang-miles (3003d70f) and sglang-kernel 0.4.5 are built against torch 2.11; @@ -453,10 +458,14 @@ override-dependencies = [ "nvidia-cudnn-frontend==1.25.0 ; sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')", # NCCL 2.30.4 fixes a context-parallel hang on GB200 when GPUs are non-contiguous; # 2.30.7 additionally fixes hangs in the NCCL m2n reshard (). - # Keep NCCL shared with the isolated Dynamo environment across torch upgrades. "nvidia-nccl-cu13==2.30.7", "nvidia-nvshmem-cu13>=3.6.5; sys_platform != 'darwin'", "opencv-python-headless; sys_platform == 'never'", + # ai-dynamo (vllm extra) requires these for its planner and simulators only + # (aisimulate brings jax, flax and tfp-nightly). The backend SDK and frontend + # that Dynamo serving uses import neither. + "aisimulate; sys_platform == 'never'", + "kubernetes; sys_platform == 'never'", "timm<=1.0.22", "fastapi[standard]>=0.133.0,<0.137.0", "nvidia-modelopt[torch]>=0.44.0a0", diff --git a/pyrefly.toml b/pyrefly.toml index ca658115f82..ae201bdbf1d 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -234,16 +234,11 @@ project-includes = [ "nemo_rl/models/generation/dynamo/__init__.py", "nemo_rl/models/generation/dynamo/arguments.py", "nemo_rl/models/generation/dynamo/config.py", - "nemo_rl/models/generation/dynamo/dynamo_generation.py", - "nemo_rl/models/generation/dynamo/dynamo_worker.py", "nemo_rl/models/generation/dynamo/http_client.py", "nemo_rl/models/generation/dynamo/managed_runtime.py", - "nemo_rl/models/generation/dynamo/metrics.py", - "nemo_rl/models/generation/dynamo/refit.py", - "nemo_rl/models/generation/dynamo/token_wrapper.py", - "nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py", + "nemo_rl/models/generation/dynamo/native_engine.py", + "nemo_rl/models/generation/dynamo/rollout_client.py", "nemo_rl/models/generation/dynamo/venv.py", - "nemo_rl/models/generation/dynamo/worker_pool.py", "nemo_rl/models/generation/engine_supervisor.py", "nemo_rl/models/generation/fleet_health.py", "nemo_rl/models/generation/generation_router.py", @@ -280,6 +275,7 @@ project-includes = [ "nemo_rl/models/generation/vllm/quantization/nvfp4_pertoken_config.py", "nemo_rl/models/generation/vllm/quantization/utils.py", "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", + "nemo_rl/models/generation/vllm/request_pipeline.py", "nemo_rl/models/generation/vllm/refit_layout.py", "nemo_rl/models/generation/vllm/refit_loader.py", "nemo_rl/models/generation/vllm/utils.py", diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index e3b08a8b481..225bbb65919 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -33,8 +33,13 @@ run_test() { fi } -run_test fast uv run --no-sync bash ./tests/functional/grpo_dynamo.sh -run_test uv run --no-sync bash ./tests/functional/grpo_dynamo_single_controller.sh +# Dynamo overlays: native SingleController tests rerun with policy.generation.dynamo=true. +# Each keeps the native test's checks and adds check_dynamo_overlay.sh's. +run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh ++policy.generation.dynamo=true +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true ++policy.generation.dynamo=true +run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true +# Router replay with deferred routes: the engine stages each captured call's routed experts. +run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/check_dynamo_overlay.sh b/tests/functional/check_dynamo_overlay.sh new file mode 100755 index 00000000000..ee98e2671a0 --- /dev/null +++ b/tests/functional/check_dynamo_overlay.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# What a native functional test also checks when it runs as the Dynamo overlay +# (++policy.generation.dynamo=true): every engine registered with the managed +# frontend, the frontend routed rollouts to them, the KV router applied the +# engines' KV-cache events (when they publish any: KV routing with prefix +# caching), and shutdown stopped its services. The test's own metric checks hold +# as they do natively. +# +# bash check_dynamo_overlay.sh + +set -euo pipefail + +run_log=$1 + +grep -E "\[Dynamo\] frontend .* routes to [0-9]+ engine" "${run_log}" +routed=$(grep -c 'route request completed request.outcome="success"' "${run_log}" || true) +echo "Requests the Dynamo frontend routed: ${routed}" +if [[ "${routed}" -eq 0 ]]; then + echo "No request went through the Dynamo frontend" >&2 + exit 1 +fi + +# vLLM's KV-event wire format is the one coupling between NeMo RL's vLLM and +# Dynamo's router; a format change shows up here as no applied events. +if grep -q "\[Dynamo\] engine publishes KV events: True" "${run_log}"; then + grep -E "\[Dynamo\] router applied KV events" "${run_log}" + applied=$(grep -oE '\[Dynamo\] router applied KV events: .*"ok": [0-9.]+' "${run_log}" \ + | grep -oE '[0-9.]+$' | tail -1) + if [[ -z "${applied}" || "${applied%.*}" -eq 0 ]]; then + echo "The KV router applied none of the engines' KV-cache events" >&2 + exit 1 + fi +fi + +dynamo_bin_dir=${NEMO_RL_DYNAMO_BIN_DIR:-/opt/dynamo/bin} +if pgrep -f "[d]ynamo.frontend|${dynamo_bin_dir}/[e]tcd|${dynamo_bin_dir}/[n]ats-server"; then + echo "Managed Dynamo services remain after shutdown" >&2 + exit 1 +fi diff --git a/tests/functional/grpo_async_gym_single_controller.sh b/tests/functional/grpo_async_gym_single_controller.sh index 6acd46388cb..a69e17c847f 100755 --- a/tests/functional/grpo_async_gym_single_controller.sh +++ b/tests/functional/grpo_async_gym_single_controller.sh @@ -113,6 +113,10 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then if [[ "$*" == *token_capture.enabled=true* ]]; then # Nonzero only when a finalizer actor ran, i.e. capture really was on. EXTRA_CHECKS+=('max(data["train/finalize/total_ms"]) > 0') + if [[ "$*" == *router_replay.enabled=true* ]]; then + # Every captured row carries routed experts for the policy to replay. + EXTRA_CHECKS+=('min(data["train/finalize/routed_experts_row_coverage"]) == 1') + fi fi # Observed to be between 0.8-1.3 @@ -121,3 +125,7 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then 'max(data["train/reward"]) > 0' \ "${EXTRA_CHECKS[@]}" fi + +if [[ "$*" == *generation.dynamo=true* ]]; then + bash $SCRIPT_DIR/check_dynamo_overlay.sh $RUN_LOG +fi diff --git a/tests/functional/grpo_dp_single_controller.sh b/tests/functional/grpo_dp_single_controller.sh index f20bece801d..de4656dc6cf 100755 --- a/tests/functional/grpo_dp_single_controller.sh +++ b/tests/functional/grpo_dp_single_controller.sh @@ -61,3 +61,7 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then 'min(data["train/probs_ratio_clamped_max"]) > 0.79' \ 'max(data["train/probs_ratio_clamped_max"]) < 1.21' fi + +if [[ "$*" == *generation.dynamo=true* ]]; then + bash $SCRIPT_DIR/check_dynamo_overlay.sh $RUN_LOG +fi diff --git a/tests/functional/grpo_dynamo.sh b/tests/functional/grpo_dynamo.sh deleted file mode 100644 index bce045b9df8..00000000000 --- a/tests/functional/grpo_dynamo.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) -PROJECT_ROOT=$(realpath "${SCRIPT_DIR}/../..") -EXP_NAME=$(basename "$0" .sh) -EXP_DIR=${SCRIPT_DIR}/${EXP_NAME} -LOG_DIR=${EXP_DIR}/logs -RUN_LOG=${EXP_DIR}/run.log -export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} - -rm -rf "${EXP_DIR}" -mkdir -p "${LOG_DIR}" -git config --global --add safe.directory "${PROJECT_ROOT}" - -dynamo_venv_dir=${NEMO_RL_DYNAMO_VENV_DIR:-/opt/dynamo_venv} -dynamo_python=${dynamo_venv_dir}/bin/python -"${dynamo_python}" -c \ - 'import importlib.metadata as m; assert m.version("ai-dynamo") == "1.5.1"; assert m.version("vllm") == "0.28.0"' -"${dynamo_venv_dir}/bin/etcd" --version -"${dynamo_venv_dir}/bin/nats-server" --version - -cd "${PROJECT_ROOT}" -uv run --no-sync coverage run -a \ - --data-file="${PROJECT_ROOT}/tests/.coverage" \ - --source="${PROJECT_ROOT}/nemo_rl" \ - "${PROJECT_ROOT}/examples/run_grpo.py" \ - --config "${PROJECT_ROOT}/examples/configs/grpo_math_1B_dynamo.yaml" \ - policy.model_name=Qwen/Qwen3-0.6B \ - policy.tokenizer.name=Qwen/Qwen3-0.6B \ - logger.log_dir="${LOG_DIR}" \ - 2>&1 | tee "${RUN_LOG}" - -grep -F "Performing policy generation refit" "${RUN_LOG}" -grep -F "Invalidated generation backend KV caches after weight update" "${RUN_LOG}" - -refit_count=$(grep -Fc "Performing policy generation refit" "${RUN_LOG}" || true) -cache_success_count=$(grep -Fc \ - "Invalidated generation backend KV caches after weight update" \ - "${RUN_LOG}" || true) -if [[ "${refit_count}" -eq 0 || "${cache_success_count}" -ne "${refit_count}" ]]; then - echo "Expected one successful cache invalidation per refit; refits=${refit_count}, successes=${cache_success_count}" >&2 - exit 1 -fi -if grep -Fq \ - -e "Failed to invalidate generation backend KV caches" \ - -e "KV cache invalidation not supported or only partially applied" \ - "${RUN_LOG}"; then - echo "The Dynamo run reported a cache invalidation failure" >&2 - exit 1 -fi - -metrics_json=${EXP_DIR}/metrics.json -uv run --no-sync tests/json_dump_tb_logs.py \ - "${LOG_DIR}" \ - --output_path "${metrics_json}" \ - --require-tag-prefix "generation_metrics/" -uv run --no-sync tests/check_metrics.py \ - "${metrics_json}" \ - 'max(data["train/token_mult_prob_error"]) < 1.05' - -if pgrep -f "[d]ynamo.frontend|[d]ynamo.vllm|[V]LLM::EngineCore|${dynamo_venv_dir}/bin/[e]tcd|${dynamo_venv_dir}/bin/[n]ats-server"; then - echo "Managed Dynamo processes remain after GRPO shutdown" >&2 - exit 1 -fi diff --git a/tests/functional/grpo_dynamo_single_controller.sh b/tests/functional/grpo_dynamo_single_controller.sh deleted file mode 100644 index 9e41e463e50..00000000000 --- a/tests/functional/grpo_dynamo_single_controller.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Two-GPU SingleController + TransferQueue run with managed Dynamo generation. - -set -euo pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) -PROJECT_ROOT=$(realpath "${SCRIPT_DIR}/../..") -EXP_NAME=$(basename "$0" .sh) -EXP_DIR=${SCRIPT_DIR}/${EXP_NAME} -LOG_DIR=${EXP_DIR}/logs -RUN_LOG=${EXP_DIR}/run.log -export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} - -rm -rf "${EXP_DIR}" -mkdir -p "${LOG_DIR}" -git config --global --add safe.directory "${PROJECT_ROOT}" - -dynamo_venv_dir=${NEMO_RL_DYNAMO_VENV_DIR:-/opt/dynamo_venv} -"${dynamo_venv_dir}/bin/python" -c \ - 'import importlib.metadata as m; assert m.version("ai-dynamo") == "1.5.1"; assert m.version("vllm") == "0.28.0"' - -cd "${PROJECT_ROOT}" -uv run --no-sync coverage run -a \ - --data-file="${PROJECT_ROOT}/tests/.coverage" \ - --source="${PROJECT_ROOT}/nemo_rl" \ - "${PROJECT_ROOT}/examples/run_grpo_single_controller.py" \ - --config "${PROJECT_ROOT}/examples/configs/grpo_math_1B_dynamo_single_controller.yaml" \ - policy.model_name=Qwen/Qwen3-0.6B \ - policy.tokenizer.name=Qwen/Qwen3-0.6B \ - grpo.num_prompts_per_step=2 \ - grpo.num_generations_per_prompt=4 \ - grpo.seq_logprob_error_threshold=1000 \ - grpo.max_num_steps=3 \ - policy.train_global_batch_size=8 \ - policy.train_micro_batch_size=1 \ - policy.logprob_batch_size=1 \ - policy.generation.max_new_tokens=128 \ - async_rl.sampler.name=in_order \ - async_rl.sampler.max_lookahead_versions=1 \ - async_rl.min_groups_for_streaming_train=2 \ - async_rl.max_inflight_prompts=2 \ - async_rl.max_buffered_rollouts=4 \ - data_plane.backend=simple \ - logger.log_dir="${LOG_DIR}" \ - logger.tensorboard_enabled=true \ - logger.wandb_enabled=false \ - logger.monitor_gpus=false \ - checkpointing.enabled=false \ - "$@" \ - 2>&1 | tee "${RUN_LOG}" - -# Every weight sync drains Dynamo first (pause) and restores it afterwards (resume). -# Dynamo accepts allow_unpaused updates, so a lost drain would not fail the refit. -sync_count=$(grep -Fc "_sync_weights: sync done" "${RUN_LOG}" || true) -pause_count=$(grep -Fc "[RL] Engine paused (mode=wait, clear_cache=True)" "${RUN_LOG}" || true) -resume_count=$(grep -Fc "[RL] Engine resumed" "${RUN_LOG}" || true) -if [[ "${sync_count}" -lt 2 || "${pause_count}" -lt "${sync_count}" || "${resume_count}" -lt "${sync_count}" ]]; then - echo "Expected every weight sync to pause and resume Dynamo; syncs=${sync_count}, pauses=${pause_count}, resumes=${resume_count}" >&2 - exit 1 -fi - -metrics_json=${EXP_DIR}/metrics.json -uv run --no-sync tests/json_dump_tb_logs.py "${LOG_DIR}" --output_path "${metrics_json}" -# The worker starts from dummy weights, so these hold only if every refit landed. -uv run --no-sync tests/check_metrics.py "${metrics_json}" \ - 'max(data["train/token_mult_prob_error"]) < 1.05' \ - 'max(data["train/gen_kl_error"]) < 0.002' - -if pgrep -f "[d]ynamo.frontend|[d]ynamo.vllm|[V]LLM::EngineCore|${dynamo_venv_dir}/bin/[e]tcd|${dynamo_venv_dir}/bin/[n]ats-server"; then - echo "Managed Dynamo processes remain after SingleController shutdown" >&2 - exit 1 -fi diff --git a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh index 2d265e6153b..ee3da965de3 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh @@ -157,3 +157,7 @@ uv run --no-sync tests/check_metrics.py "${JSON_METRICS}" \ 'min(data["train/finalize/media_row_rate"]) == 1' \ 'max(data["train/gen_kl_error"]) < 0.05' \ 'all_finite(data["train/reward"])' + +if [[ "$*" == *generation.dynamo=true* ]]; then + bash "${SCRIPT_DIR}/check_dynamo_overlay.sh" "${RUN_LOG}" +fi diff --git a/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh b/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh index 1dc7edd194a..e9cf7073a7c 100755 --- a/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh +++ b/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh @@ -59,8 +59,8 @@ refit_count=$(grep -c "✅ Ready for refit" "${RUN_LOG}" || true) cache_invalidation_count=$(grep -c \ "✅ Invalidated generation backend KV caches after weight update" \ "${RUN_LOG}" || true) -cache_invalidation_failure_count=$(grep -cE \ - "Failed to invalidate generation backend KV caches|Dynamo KV cache invalidation failed" \ +cache_invalidation_failure_count=$(grep -c \ + "Failed to invalidate generation backend KV caches" \ "${RUN_LOG}" || true) if [[ ${cache_invalidation_failure_count} -ne 0 ]]; then echo "[ERROR] Found ${cache_invalidation_failure_count} cache invalidation failure(s)" diff --git a/tests/unit/algorithms/test_async_utils.py b/tests/unit/algorithms/test_async_utils.py index 4753a784c80..67d3b4334f3 100644 --- a/tests/unit/algorithms/test_async_utils.py +++ b/tests/unit/algorithms/test_async_utils.py @@ -2811,34 +2811,6 @@ def test_resume_after_refit_skips_cache_invalidation_when_recompute_disabled(sel collector.policy_generation.invalidate_kv_cache.assert_not_called() - def test_dynamo_cache_invalidation_failure_is_fatal_and_unblocks_waiters(self): - collector = self.create_local_collector() - collector.master_config.policy["generation"] = {"backend": "dynamo"} - collector.master_config.grpo.async_grpo.recompute_kv_cache_after_weight_updates = True - collector.policy_generation.invalidate_kv_cache = mock.Mock( - side_effect=RuntimeError("pause failed") - ) - collector._refit_pause_cleared.clear() - - with pytest.raises(RuntimeError, match="cache invalidation failed"): - collector.resume_after_refit() - - assert collector._refit_pause_cleared.is_set() - - def test_dynamo_prepare_for_refit_drains_pending_generations(self): - """Dynamo layerwise reload never overlaps an active generation.""" - collector = self.create_local_collector() - collector.master_config.policy["generation"] = { - "backend": "dynamo", - "dynamo_cfg": {}, - } - collector.master_config.grpo.async_grpo.in_flight_weight_updates = True - collector.wait_for_pending_generations = mock.MagicMock() - - collector.prepare_for_refit() - - collector.wait_for_pending_generations.assert_called_once_with() - def test_calculate_target_weights(self): """Test target weight calculation logic.""" buffer = ReplayBuffer.remote( diff --git a/tests/unit/algorithms/test_distillation.py b/tests/unit/algorithms/test_distillation.py index e112f379de6..d82c2044179 100644 --- a/tests/unit/algorithms/test_distillation.py +++ b/tests/unit/algorithms/test_distillation.py @@ -1332,7 +1332,7 @@ class DummyVllmGeneration: def __init__(self, cluster, config, defer_model_load=False): self.cfg = config self.defer_model_load = defer_model_load - self.dp_openai_server_base_urls = ["http://reserved-vllm"] + self.client_base_urls = ["http://reserved-vllm"] # The per-job bearer token the real class generates when the # recipe requires one; Gym's model server must receive it. self.http_server_api_key = "k" diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index 90fd42415ec..7950eefff3e 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -102,7 +102,6 @@ ) from nemo_rl.experience.rollouts import calculate_rewards from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo import DynamoConfig from nemo_rl.models.generation.interfaces import should_use_async_rollouts from nemo_rl.models.generation.megatron import MegatronGeneration from nemo_rl.models.policy.draft_config import Eagle3DraftConfig @@ -2096,7 +2095,6 @@ def test_async_resume_plumbs_frontier_metadata_into_collector( @pytest.mark.parametrize( ("generation_config", "expected"), [ - ({"backend": "dynamo"}, True), ({"backend": "vllm", "vllm_cfg": {"async_engine": False}}, False), ({"backend": "vllm", "vllm_cfg": {"async_engine": True}}, True), ( @@ -2139,13 +2137,11 @@ def test_should_use_nemo_gym_accepts_megatron_always_async(): assert should_use_nemo_gym(master_config) -@pytest.mark.parametrize("backend", ["dynamo", "vllm"]) def test_initial_refit_completes_before_async_collection_starts( mock_grpo_components, - backend, ) -> None: master_config = mock_grpo_components["master_config"] - master_config.policy["generation"]["backend"] = backend + master_config.policy["generation"]["backend"] = "vllm" master_config.policy["generation"]["colocated"]["enabled"] = False master_config.grpo.max_num_steps = 1 master_config.grpo.val_period = 0 @@ -2184,7 +2180,7 @@ def record_refit(*args, **kwargs): def test_async_grpo_awaits_resume_after_refit_failure(mock_grpo_components) -> None: master_config = mock_grpo_components["master_config"] - master_config.policy["generation"]["backend"] = "dynamo" + master_config.policy["generation"]["backend"] = "vllm" master_config.policy["generation"]["colocated"]["enabled"] = False master_config.grpo.max_num_steps = 1 master_config.grpo.val_period = 0 @@ -2216,23 +2212,6 @@ def test_async_grpo_awaits_resume_after_refit_failure(mock_grpo_components) -> N ) -def test_should_use_nemo_gym_requires_dynamo_token_wrapper() -> None: - master_config = MagicMock() - master_config.env = {"should_use_nemo_gym": True} - master_config.policy = { - "generation": { - "backend": "dynamo", - "vllm_cfg": {"expose_http_server": False}, - } - } - - with pytest.raises(AssertionError, match="expose_http_server: true"): - should_use_nemo_gym(master_config) - - master_config.policy["generation"]["vllm_cfg"]["expose_http_server"] = True - assert should_use_nemo_gym(master_config) is True - - @pytest.mark.parametrize( ( "lookahead_complete", @@ -3316,247 +3295,6 @@ def test_noncolocated_inference_requires_explicit_gpus_per_node_single_node( setup(master_config, tokenizer, dataset, None) -def test_dynamo_rejects_colocated_inference_before_setup_side_effects( - mock_grpo_components, -): - from nemo_rl.algorithms.grpo import setup - - master_config = mock_grpo_components["master_config"] - master_config.policy["hf_config_overrides"] = {"rope_theta": 1_000_000.0} - master_config.policy["generation"] = { - "backend": "dynamo", - "dynamo_cfg": { - "engine": "vllm", - "startup_timeout_s": 60, - "request_timeout_s": 60, - "control_timeout_s": 30, - "metrics_include_prefixes": None, - "metrics_exclude_prefixes": None, - "worker_args": { - "tool_call_parser": None, - "reasoning_parser": None, - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": False, - "structural_tag_scope": "auto", - "structural_tag_schema": "auto", - "custom_jinja_template": None, - "endpoint_types": ["chat", "completions"], - "extra_cli_args": [], - }, - "frontend_args": { - "tokenizer": "default", - "tokenizer_cache": False, - "tokenizer_cache_bytes": 1024, - "router_mode": "kv", - "extra_cli_args": [], - }, - }, - "vllm_cfg": { - "async_engine": True, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "expert_parallel_size": 1, - "gpu_memory_utilization": 0.6, - "max_model_len": 512, - "kv_cache_dtype": "auto", - "load_format": "auto", - "precision": "bfloat16", - "enforce_eager": True, - "expose_http_server": False, - "enable_vllm_metrics_logger": True, - "vllm_metrics_logger_interval": 1.0, - "env_vars": {}, - }, - "vllm_kwargs": {}, - "colocated": { - "enabled": True, - "resources": {"gpus_per_node": None, "num_nodes": None}, - }, - } - - master_config.grpo.async_grpo.in_flight_weight_updates = True - with ( - patch("nemo_rl.algorithms.grpo.Logger") as mock_logger, - pytest.raises(ValueError, match="in_flight_weight_updates must be false"), - ): - setup( - master_config, - tokenizer=MagicMock(), - dataset=MagicMock(), - val_dataset=None, - ) - mock_logger.assert_not_called() - - master_config.grpo.async_grpo.in_flight_weight_updates = False - - with ( - patch("nemo_rl.algorithms.grpo.Logger") as mock_logger, - pytest.raises( - ValueError, - match="must be false", - ), - ): - setup( - master_config, - tokenizer=MagicMock(), - dataset=MagicMock(), - val_dataset=None, - ) - - mock_logger.assert_not_called() - assert ( - master_config.policy["generation"]["vllm_kwargs"]["hf_overrides"] - == (master_config.policy["hf_config_overrides"]) - ) - - del master_config.policy["generation"]["vllm_kwargs"] - del master_config.policy["hf_config_overrides"] - with pytest.raises(ValueError, match="must be false"): - setup( - master_config, - tokenizer=MagicMock(), - dataset=MagicMock(), - val_dataset=None, - ) - assert master_config.policy["generation"]["vllm_kwargs"]["hf_overrides"] == {} - - -def test_setup_initializes_noncolocated_dynamo_with_nemo_gym(monkeypatch) -> None: - from nemo_rl.algorithms import grpo as grpo_mod - - repo_root = Path(__file__).resolve().parents[3] - register_omegaconf_resolvers() - config = OmegaConf.to_container( - load_config(repo_root / "examples/configs/grpo_math_1B_dynamo.yaml"), - resolve=True, - ) - config["cluster"].update({"num_nodes": 2, "gpus_per_node": 4, "segment_size": 1}) - generation = config["policy"]["generation"] - generation["vllm_cfg"].update( - { - "tensor_parallel_size": 2, - "pipeline_parallel_size": 2, - "expert_parallel_size": 2, - "expose_http_server": True, - } - ) - generation["colocated"]["resources"] = { - "gpus_per_node": 4, - "num_nodes": 1, - } - config["env"]["should_use_nemo_gym"] = True - tokenizer = MagicMock() - tokenizer.pad_token_id = 0 - tokenizer.eos_token_id = 1 - config["policy"]["generation"] = configure_generation_config(generation, tokenizer) - master_config = MasterConfig.model_validate(config) - - cluster_instances = [] - - class DummyCluster: - def __init__(self, **kwargs): - self.kwargs = kwargs - self.num_gpus_per_node = kwargs["num_gpus_per_node"] - self.get_placement_groups = MagicMock(return_value=[object()]) - cluster_instances.append(self) - - class DummyLoader: - def __init__(self, *_args, **_kwargs): - pass - - def __len__(self): - return 1 - - class DummyCheckpointer: - def get_latest_checkpoint_path(self): - return None - - def load_training_info(self, _path): - return None - - def get_resume_paths(self, _path): - return None, None - - class DummyPolicy: - def print_node_ip_and_gpu_id(self): - pass - - dynamo_init = MagicMock() - - class DummyDynamoGeneration: - weight_synchronizer = None - dp_openai_server_base_urls = ["http://dynamo-wrapper.example/v1"] - frontend_url = "http://dynamo-frontend.example/v1" - - def __init__(self, *, cluster, config, tokenizer, tokenizer_config): - dynamo_init( - cluster=cluster, - config=config, - tokenizer=tokenizer, - tokenizer_config=tokenizer_config, - ) - - synchronizer = MagicMock() - nemo_gym_shard_set = MagicMock() - build_nemo_gym_actors = MagicMock(return_value=nemo_gym_shard_set) - monkeypatch.setattr(grpo_mod, "Logger", lambda *_args, **_kwargs: MagicMock()) - monkeypatch.setattr( - grpo_mod, "CheckpointManager", lambda *_args, **_kwargs: DummyCheckpointer() - ) - monkeypatch.setattr( - grpo_mod, "ClippedPGLossFn", lambda *_args, **_kwargs: MagicMock() - ) - monkeypatch.setattr(grpo_mod, "StatefulDataLoader", DummyLoader) - monkeypatch.setattr( - grpo_mod, - "get_ray_cluster_topology", - lambda: { - "train-node": ("nvlink_domain_train", 0), - "inference-node": ("nvlink_domain_inference", 1), - }, - ) - monkeypatch.setattr(grpo_mod, "RayVirtualCluster", DummyCluster) - monkeypatch.setattr(grpo_mod, "Policy", lambda *_args, **_kwargs: DummyPolicy()) - monkeypatch.setattr(grpo_mod, "DynamoGeneration", DummyDynamoGeneration) - monkeypatch.setattr( - grpo_mod, "create_weight_synchronizer", lambda **_kwargs: synchronizer - ) - monkeypatch.setattr(grpo_mod, "build_nemo_gym_actors", build_nemo_gym_actors) - # The agent-coverage check asks an unsharded job's actor for its entries - # over Ray, which a mock actor cannot answer; the check has its own tests. - monkeypatch.setattr( - grpo_mod, "validate_dataset_agent_coverage", lambda *_args, **_kwargs: None - ) - - dataset = MagicMock() - dataset.__len__.return_value = 2 - result = setup(master_config, tokenizer, dataset, None) - - train_cluster, inference_cluster = cluster_instances - assert train_cluster.kwargs["bundle_ct_per_node_list"] == [4] - assert inference_cluster.kwargs["bundle_ct_per_node_list"] == [4] - assert train_cluster.kwargs["node_resource_constraints"] == [ - {"nvlink_domain_train": 0.001} - ] - assert inference_cluster.kwargs["node_resource_constraints"] is None - assert result[1].dp_openai_server_base_urls == ["http://dynamo-wrapper.example/v1"] - assert result[2] is nemo_gym_shard_set - dynamo_config = dynamo_init.call_args.kwargs["config"] - assert dynamo_init.call_args.kwargs["cluster"] is inference_cluster - assert DynamoConfig.model_validate(dynamo_config).engine_world_size == 4 - synchronizer.init_communicator.assert_called_once_with() - build_nemo_gym_actors.assert_called_once_with( - master_config.env, - base_urls=["http://dynamo-wrapper.example/v1"], - model_name=master_config.policy["model_name"], - tokenizer=tokenizer, - enable_router_replay=False, - use_fastokens=False, - # Only the vLLM backend generates a bearer token for its HTTP server. - policy_api_key=None, - ) - - def test_setup_hands_the_vllm_http_server_api_key_to_nemo_gym(monkeypatch) -> None: """The deferred-vLLM branch is the one path that forwards a real key: the workers enforce it, so Gym's model server must send the same one.""" @@ -3632,7 +3370,7 @@ def offload_after_refit(self): class DummyVllmGeneration: weight_synchronizer = None - dp_openai_server_base_urls = ["http://vllm-worker.example/v1"] + client_base_urls = ["http://vllm-worker.example/v1"] # What the real class generates when the recipe requires a key. http_server_api_key = "per-job-key" @@ -7154,22 +6892,20 @@ def test_needs_hf_refit_handshake(backend, nccl_reshard, colocated, expected): @pytest.mark.parametrize( - ("backend", "nccl_reshard", "offload_policy", "expected"), + ("nccl_reshard", "offload_policy", "expected"), [ - ("vllm", False, False, False), - ("vllm", False, True, True), - ("vllm", True, False, True), - ("dynamo", False, False, True), + (False, False, False), + (False, True, True), + (True, False, True), ], ) def test_noncolocated_refit_synchronizer_selection( - backend, nccl_reshard, offload_policy, expected + nccl_reshard, offload_policy, expected ): from nemo_rl.algorithms import grpo as grpo_mod assert ( grpo_mod._uses_managed_noncolocated_refit( - generation_backend=backend, nccl_reshard_refit_enabled=nccl_reshard, offload_policy_before_refit=offload_policy, ) diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index e5daf577f89..3469ac88884 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -39,7 +39,11 @@ def _digest(label: str) -> str: @pytest.mark.parametrize( ("generation_backend", "expected"), - [("vllm", "vllm_worker"), ("megatron", "megatron_worker")], + [ + ("vllm", "vllm_worker"), + ("megatron", "megatron_worker"), + ("dynamo", "dynamo_worker"), + ], ) def test_external_staging_backend_maps_generation_backend( generation_backend: str, expected: str @@ -572,8 +576,11 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> assert should_use_nemo_gym(config) is True -def test_setup_nemo_gym_config_dynamo_exposes_http_server_via_vllm_cfg() -> None: - """Dynamo serves Gym through the vllm_cfg HTTP server settings.""" +@pytest.mark.parametrize("dynamo", [False, True]) +def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( + dynamo: bool, +) -> None: + """With Dynamo, Gym's policy model calls the frontend in Gym's Dynamo dialect.""" from nemo_rl.algorithms.grpo import MasterConfig from nemo_rl.environments.nemo_gym import ( setup_nemo_gym_config, @@ -584,7 +591,8 @@ def test_setup_nemo_gym_config_dynamo_exposes_http_server_via_vllm_cfg() -> None env={"should_use_nemo_gym": True}, policy={ "generation": { - "backend": "dynamo", + "backend": "vllm", + "dynamo": dynamo, "vllm_cfg": {"async_engine": False, "expose_http_server": False}, } }, @@ -596,3 +604,66 @@ def test_setup_nemo_gym_config_dynamo_exposes_http_server_via_vllm_cfg() -> None assert vllm_cfg["async_engine"] is True assert vllm_cfg["expose_http_server"] is True assert should_use_nemo_gym(config) is True + policy_model = config.env["nemo_gym"].get("policy_model") if dynamo else None + if dynamo: + assert policy_model["responses_api_models"]["vllm_model"] == { + "dynamo_frontend": True, + # The frontend refuses reasoning under both of Gym's keys. + "reasoning_field": "reasoning_content", + } + else: + assert "nemo_gym" not in config.env + + +@pytest.mark.parametrize( + ("vllm_model", "expected"), + [ + ({}, {"enable_thinking": True, "truncate_history_thinking": False}), + ( + {"chat_template_kwargs": {"enable_thinking": False}}, + {"enable_thinking": False, "truncate_history_thinking": False}, + ), + ( + {"extra_body": {"chat_template_kwargs": {"force_nonempty_content": True}}}, + { + "enable_thinking": True, + "truncate_history_thinking": False, + "force_nonempty_content": True, + }, + ), + ], +) +def test_setup_nemo_gym_config_dynamo_sends_the_native_default_template_kwargs( + vllm_model: dict, expected: dict +) -> None: + """The native server's default template kwargs sit under the ones Gym sends.""" + from nemo_rl.algorithms.grpo import MasterConfig + from nemo_rl.environments.nemo_gym import setup_nemo_gym_config + + config = MasterConfig.model_construct( + env={ + "should_use_nemo_gym": True, + "nemo_gym": { + "policy_model": {"responses_api_models": {"vllm_model": vllm_model}} + }, + }, + policy={ + "generation": { + "backend": "vllm", + "dynamo": True, + "vllm_cfg": { + "async_engine": True, + "http_server_serving_chat_kwargs": { + "default_chat_template_kwargs": { + "enable_thinking": True, + "truncate_history_thinking": False, + } + }, + }, + } + }, + ) + + setup_nemo_gym_config(config, tokenizer=None) + + assert vllm_model["chat_template_kwargs"] == expected diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index d47b492fae1..ea0b267beef 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -43,7 +43,7 @@ ) -@pytest.mark.parametrize("backend", ["vllm", "dynamo", "megatron", "trtllm"]) +@pytest.mark.parametrize("backend", ["vllm", "megatron", "trtllm"]) def test_setup_nemo_gym_config_enables_http_server(backend: str) -> None: generation = { "backend": backend, @@ -57,10 +57,9 @@ def test_setup_nemo_gym_config_enables_http_server(backend: str) -> None: setup_nemo_gym_config(config, tokenizer=None) - is_vllm_like = backend in ("vllm", "dynamo") assert generation["vllm_cfg"] == { - "async_engine": is_vllm_like, - "expose_http_server": is_vllm_like, + "async_engine": backend == "vllm", + "expose_http_server": backend == "vllm", } assert generation["mcore_generation_config"] == { "expose_http_server": backend == "megatron" diff --git a/tests/unit/experience/test_rollouts.py b/tests/unit/experience/test_rollouts.py index c87795b5df7..2858b40bba3 100644 --- a/tests/unit/experience/test_rollouts.py +++ b/tests/unit/experience/test_rollouts.py @@ -987,11 +987,6 @@ def fake_rewards(batch, task_to_env): assert [call["dedup"] for call in calls] == [deduplicate_multimodal_data] * 2 -class _DummyDynamoGeneration(_DummySGLangGeneration): - def __init__(self): - self.cfg = {"backend": "dynamo"} - - def test_generate_responses_async_requires_sglang_opt_in(): generation_input_data = BatchedDataDict( { @@ -1037,30 +1032,6 @@ def test_generate_responses_async_allows_sglang_opt_in(): assert gen_metrics["total_generated_tokens"] == 1 -def test_generate_responses_async_allows_dynamo(): - generation_input_data = BatchedDataDict( - { - "input_ids": torch.tensor([[1]]), - "input_lengths": torch.tensor([1], dtype=torch.long), - } - ) - batch = BatchedDataDict({"message_log": [[]]}) - - updated_batch, generated_ids, gen_metrics = asyncio.run( - generate_responses_async( - _DummyDynamoGeneration(), - generation_input_data, - batch, - _DummyTokenizer(), - input_lengths=generation_input_data["input_lengths"], - ) - ) - - assert updated_batch["message_log"][0][-1]["content"] == "ok" - assert generated_ids[0].tolist() == [2] - assert gen_metrics["total_generated_tokens"] == 1 - - @pytest.fixture(scope="function") def rollout_tokenizer(): """Loads the tokenizer for the tests.""" diff --git a/tests/unit/models/generation/test_dynamo_arguments.py b/tests/unit/models/generation/test_dynamo_arguments.py deleted file mode 100644 index d5e992dcdc2..00000000000 --- a/tests/unit/models/generation/test_dynamo_arguments.py +++ /dev/null @@ -1,424 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import warnings - -import pytest -from pydantic import ValidationError - -from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.arguments import ( - build_dynamo_frontend_argv, - build_dynamo_vllm_argv, - build_managed_worker_env, - redact_argv, - redact_environment, -) -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - DynamoConfig, - DynamoWorkerArgs, -) -from nemo_rl.models.generation.interfaces import DisaggConfig - - -def _config(**overrides) -> dict: - config = { - "backend": "dynamo", - "model_name": "Qwen/Qwen3-0.6B", - "dynamo_cfg": _dynamo_cfg(), - "vllm_cfg": { - "async_engine": True, - "tensor_parallel_size": 2, - "pipeline_parallel_size": 1, - "expert_parallel_size": 2, - "gpu_memory_utilization": 0.8, - "max_model_len": 512, - "precision": "bfloat16", - "kv_cache_dtype": "auto", - "load_format": "auto", - "enforce_eager": False, - "expose_http_server": False, - "enable_vllm_metrics_logger": True, - "vllm_metrics_logger_interval": 1.0, - "env_vars": None, - }, - "vllm_kwargs": {}, - "colocated": {"enabled": False}, - } - config.update(overrides) - return config - - -def _dynamo_cfg() -> dict: - return { - "engine": "vllm", - "startup_timeout_s": 600, - "request_timeout_s": 900, - "control_timeout_s": 600, - "metrics_include_prefixes": None, - "metrics_exclude_prefixes": None, - "worker_args": { - "tool_call_parser": None, - "reasoning_parser": None, - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": False, - "structural_tag_scope": "auto", - "structural_tag_schema": "auto", - "custom_jinja_template": None, - "endpoint_types": ["chat", "completions"], - "extra_cli_args": [], - }, - "frontend_args": { - "tokenizer": "default", - "tokenizer_cache": False, - "tokenizer_cache_bytes": 50 * 1024 * 1024, - "router_mode": "kv", - "extra_cli_args": [], - }, - } - - -def _flag_value(argv: list[str], flag: str) -> str: - return argv[argv.index(flag) + 1] - - -def test_config_derives_world_size_and_rejects_removed_public_fields() -> None: - assert DynamoConfig.model_validate(_config()).engine_world_size == 2 - for field in ("engine_world_size", "namespace", "dynamo_python", "etcd_port"): - with pytest.raises(ValidationError, match=field): - DynamoConfig.model_validate( - _config(dynamo_cfg={field: 1 if field.endswith("port") else "x"}) - ) - - -@pytest.mark.parametrize( - ("override", "match"), - [ - ({"vllm_cfg": {}}, "nonempty"), - ({"sglang_cfg": {"foo": 1}}, "sglang_cfg"), - ({"trtllm_cfg": {"foo": 1}}, "trtllm_cfg"), - ({"colocated": {"enabled": True}}, "must be false"), - ({"refit_transport": "nccl_reshard"}, "must be null"), - ({"quant_cfg": "nvfp4"}, "quant_cfg"), - ({"vllm_kwargs": {"speculative_config": {"model": "draft"}}}, "draft"), - ], -) -def test_config_rejects_unsupported_modes(override, match) -> None: - if override.get("vllm_cfg") == {}: - config = _config() - config["vllm_cfg"] = {} - else: - config = _config(**override) - with pytest.raises(ValidationError, match=match): - DynamoConfig.model_validate(config) - - -@pytest.mark.parametrize( - ("vllm_cfg", "match"), - [ - ( - {"tensor_parallel_size": 2, "expert_parallel_size": 3}, - "expert_parallel_size", - ), - ({"precision": "fp8"}, "precision"), - ({"kv_cache_dtype": "fp8"}, "kv_cache_dtype"), - ], -) -def test_config_rejects_unsupported_parallelism_and_precision(vllm_cfg, match) -> None: - config = _config() - config["vllm_cfg"].update(vllm_cfg) - with pytest.raises(ValidationError, match=match): - DynamoConfig.model_validate(config) - - -def test_config_classifies_managed_logprobs_and_runtime_fields() -> None: - config = _config() - config["vllm_cfg"].update( - { - "logprobs_mode": "processed_logprobs", - "cap_max_tokens_to_context": False, - "use_deep_gemm": False, - "num_first_layers_in_bf16": 0, - "num_last_layers_in_bf16": 0, - } - ) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - DynamoConfig.model_validate(config) - assert caught == [] - - config["vllm_cfg"]["logprobs_mode"] = "raw_logprobs" - with pytest.raises(ValidationError, match="processed_logprobs"): - DynamoConfig.model_validate(config) - - -@pytest.mark.parametrize("field", ["skip_tokenizer_init", "cap_max_tokens_to_context"]) -def test_config_warns_only_for_active_unsupported_fields(field) -> None: - config = _config() - config["vllm_cfg"][field] = True - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - DynamoConfig.model_validate(config) - assert [str(warning.message) for warning in caught] == [ - f"policy.generation.vllm_cfg.{field} is ignored by backend='dynamo'" - ] - - -@pytest.mark.parametrize( - ("override", "ignored"), - [ - ({"bad_words": ["foo"]}, ["bad_words"]), - ({"ignore_eos": True}, ["ignore_eos"]), - # The validator receives disaggregation as a DisaggConfig, not a dict. - ({"disaggregation": DisaggConfig(enabled=True)}, ["disaggregation"]), - # Inherited shared-YAML settings the exemplars carry stay quiet. - ({"disaggregation": DisaggConfig(enabled=False)}, []), - ({"ignore_eos": False}, []), - ({"refit_cfg": {"nixl": {}}, "mcore_generation_config": {"x": 1}}, []), - ({"real_quant_export_cpu_offload": True}, []), - ], -) -def test_config_warns_for_ignored_generation_keys(override, ignored) -> None: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - DynamoConfig.model_validate(_config(**override)) - expected = ( - [f"policy.generation keys ignored by backend='dynamo': {ignored}"] - if ignored - else [] - ) - assert [str(warning.message) for warning in caught] == expected - - -def test_config_rejects_a_required_http_server_api_key() -> None: - """The managed Dynamo frontend checks no bearer token, so asking for one - fails loudly instead of leaving the exposed server open; the exemplar's - explicit ``false`` is accepted without an "ignored" warning.""" - config = _config() - config["vllm_cfg"]["http_server_api_key_required"] = True - with pytest.raises(ValidationError, match="http_server_api_key_required"): - DynamoConfig.model_validate(config) - - config["vllm_cfg"]["http_server_api_key_required"] = False - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - DynamoConfig.model_validate(config) - assert caught == [] - - -@pytest.mark.parametrize( - "field", - [ - "data_parallel_size", - "prefill_context_parallel_size", - "decode_context_parallel_size", - ], -) -@pytest.mark.parametrize("source", ["vllm_cfg", "vllm_kwargs"]) -def test_config_rejects_parallel_dimensions_outside_tp_pp(field, source) -> None: - config = _config() - config[source][field] = 2 - - with pytest.raises(ValidationError, match=f"{source}.{field} must be 1"): - DynamoConfig.model_validate(config) - - -def test_config_rejects_more_than_32_stop_strings() -> None: - config = _config(stop_strings=[str(index) for index in range(33)]) - - with pytest.raises(ValidationError, match="stop_strings supports at most 32"): - DynamoConfig.model_validate(config) - - -def test_config_does_not_limit_stop_token_ids() -> None: - config = _config(stop_token_ids=list(range(33))) - - assert DynamoConfig.model_validate(config).model_extra["stop_token_ids"] == list( - range(33) - ) - - -def test_configure_generation_config_selects_dynamo_load_format() -> None: - class Tokenizer: - pad_token_id = 0 - eos_token_id = 1 - - training = _config(stop_token_ids=None) - evaluation = _config(stop_token_ids=None) - del training["vllm_cfg"]["load_format"] - del evaluation["vllm_cfg"]["load_format"] - - assert ( - configure_generation_config(training, Tokenizer())["vllm_cfg"]["load_format"] - == "dummy" - ) - assert ( - configure_generation_config(evaluation, Tokenizer(), is_eval=True)["vllm_cfg"][ - "load_format" - ] - == "auto" - ) - - -def test_worker_argv_translates_structured_fields_and_warns_unclassified() -> None: - config = _dynamo_cfg() - config["worker_args"].update( - {"tool_call_parser": "qwen3_coder", "reasoning_parser": "nemotron_nano"} - ) - cfg = DynamoCfg.model_validate(config) - generation_config = _config() - generation_config["vllm_cfg"]["unclassified_field"] = 1 - with pytest.warns(UserWarning, match="unclassified_field"): - validated = DynamoConfig.model_validate(generation_config) - vllm_cfg = validated.vllm_cfg.model_dump() - argv = build_dynamo_vllm_argv( - model_name="model", - namespace="nemo-rl-1", - seed=7, - vllm_cfg=vllm_cfg, - vllm_kwargs={"max_num_seqs": 16, "hf_overrides": {"rope_theta": 1e6}}, - dynamo_cfg=cfg, - ) - - assert _flag_value(argv, "--model") == "model" - assert _flag_value(argv, "--weight-transfer-config") == '{"backend":"nccl"}' - assert _flag_value(argv, "--dyn-tool-call-parser") == "qwen3_coder" - assert _flag_value(argv, "--dyn-reasoning-parser") == "nemotron_nano" - assert _flag_value(argv, "--max-num-seqs") == "16" - assert _flag_value(argv, "--hf-overrides") == '{"rope_theta":1000000.0}' - assert "--enable-expert-parallel" in argv - - -def test_worker_argv_rejects_replaced_and_managed_options() -> None: - generation_config = _config() - generation_config["vllm_cfg"]["http_server_serving_chat_kwargs"] = { - "tool_parser": "x" - } - with pytest.raises(ValueError, match="worker_args.custom_jinja_template"): - DynamoConfig.model_validate(generation_config) - config = _dynamo_cfg() - config["worker_args"]["extra_cli_args"] = ["--model", "other"] - with pytest.raises(ValueError, match="--model is set by both"): - build_dynamo_vllm_argv( - model_name="model", - namespace="namespace", - seed=0, - vllm_cfg=_config()["vllm_cfg"], - vllm_kwargs={}, - dynamo_cfg=DynamoCfg.model_validate(config), - ) - - -def test_config_accepts_inherited_unused_sections() -> None: - config = _config( - mcore_generation_config={"some_shared_setting": True}, - refit_cfg={"some_shared_setting": True}, - ) - - validated = DynamoConfig.model_validate(config) - - assert validated.model_extra["mcore_generation_config"] == { - "some_shared_setting": True - } - assert validated.model_extra["refit_cfg"] == {"some_shared_setting": True} - - -def test_frontend_argv_and_environment_are_runtime_owned() -> None: - cfg = DynamoCfg.model_validate(_dynamo_cfg()) - argv = build_dynamo_frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg - ) - assert _flag_value(argv, "--router-mode") == "kv" - assert not any(arg.startswith("--router-reset-states") for arg in argv) - assert not any(arg.startswith("--no-router-reset-states") for arg in argv) - - env = build_managed_worker_env( - base_env={ - "DYN_NAMESPACE": "stale", - "ETCD_ENDPOINTS": "http://stale-etcd:2379", - "ETCD_USERNAME": "stale-user", - "NATS_SERVER": "nats://stale-nats:4222", - "NATS_AUTH_TOKEN": "stale-token", - "NCCL_DEBUG": "INFO", - }, - configured_env={"NCCL_IB_DISABLE": "0"}, - manager_env={ - "DYN_NAMESPACE": "owned", - "DYN_SYSTEM_PORT": "4000", - "ETCD_ENDPOINTS": "http://managed-etcd:2379", - "NATS_SERVER": "nats://managed-nats:4222", - }, - ) - assert env["DYN_NAMESPACE"] == "owned" - assert env["DYN_SYSTEM_PORT"] == "4000" - assert env["ETCD_ENDPOINTS"] == "http://managed-etcd:2379" - assert env["NATS_SERVER"] == "nats://managed-nats:4222" - assert "ETCD_USERNAME" not in env - assert "NATS_AUTH_TOKEN" not in env - with pytest.raises(ValueError, match="VLLM_PORT"): - build_managed_worker_env( - base_env={}, - configured_env={"VLLM_PORT": "9999"}, - manager_env={"VLLM_PORT": "7000"}, - ) - - -def test_every_worker_config_field_reaches_argv(monkeypatch) -> None: - from nemo_rl.models.generation.dynamo import arguments - - sources: set[str] = set() - original_add = arguments._ArgvBuilder.add - - def record_source(self, flag, value=None, *, source): - sources.add(source) - return original_add(self, flag, value, source=source) - - monkeypatch.setattr(arguments._ArgvBuilder, "add", record_source) - config = _dynamo_cfg() - config["worker_args"].update( - { - "tool_call_parser": "qwen3_coder", - "reasoning_parser": "nemotron_nano", - "custom_jinja_template": "template", - } - ) - cfg = DynamoCfg.model_validate(config) - build_dynamo_vllm_argv( - model_name="model", - namespace="namespace", - seed=0, - vllm_cfg=_config()["vllm_cfg"], - vllm_kwargs={}, - dynamo_cfg=cfg, - ) - - configured_fields = { - source.rsplit(".", 1)[-1] - for source in sources - if source.startswith("dynamo_cfg.worker_args.") - } - assert set(DynamoWorkerArgs.model_fields) - {"extra_cli_args"} <= configured_fields - - -def test_redaction_hides_credentials() -> None: - assert redact_argv(["worker", "--api-key", "secret"])[2] == "" - assert redact_argv( - ["worker", "--max-num-batched-tokens", "8192", "--stop-token-ids", "1,2"] - ) == ["worker", "--max-num-batched-tokens", "8192", "--stop-token-ids", "1,2"] - assert redact_environment({"HF_TOKEN": "secret", "NCCL_DEBUG": "INFO"}) == { - "HF_TOKEN": "", - "NCCL_DEBUG": "INFO", - } diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py new file mode 100644 index 00000000000..ba6edd103e9 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -0,0 +1,202 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from pydantic import ValidationError + +from nemo_rl.models.generation import configure_generation_config +from nemo_rl.models.generation.dynamo.arguments import ( + build_dynamo_frontend_argv, + redact_argv, + redact_environment, +) +from nemo_rl.models.generation.dynamo.config import ( + DynamoCfg, + dynamo_cfg, + dynamo_chat_template, + dynamo_parsers, + native_default_chat_template_kwargs, +) + + +class _Tokenizer: + pad_token_id = 0 + eos_token_id = 1 + + +def _vllm_config(**overrides) -> dict: + config = { + "backend": "vllm", + "model_name": "Qwen/Qwen3-0.6B", + "stop_token_ids": None, + "stop_strings": None, + "vllm_cfg": {"async_engine": False, "max_model_len": 512}, + } + config.update(overrides) + return config + + +def test_dynamo_true_alone_is_a_complete_config() -> None: + config = configure_generation_config(_vllm_config(dynamo=True), _Tokenizer()) + + # Dynamo serves the AsyncLLM, so the switch selects the async engine, and it + # serves OpenAI clients, which need the engine's tokenizer. + assert config["vllm_cfg"]["async_engine"] is True + assert config["vllm_cfg"]["skip_tokenizer_init"] is False + assert dynamo_cfg(config) == DynamoCfg() + assert dynamo_cfg(config).frontend_args.router_mode == "kv" + + +def test_without_dynamo_the_vllm_config_is_untouched() -> None: + config = configure_generation_config(_vllm_config(), _Tokenizer()) + + assert config["vllm_cfg"]["async_engine"] is False + assert config["vllm_cfg"]["skip_tokenizer_init"] is True + assert "dynamo_cfg" not in config + + +@pytest.mark.parametrize( + ("config", "error"), + [ + # A colocated engine sleeps while the policy trains. + ( + _vllm_config(dynamo=True, colocated={"enabled": True}), + "requires non-colocated generation", + ), + (_vllm_config(dynamo=True, backend="sglang"), "backend=vllm"), + # dynamo_cfg alone would be ignored. + (_vllm_config(dynamo_cfg={}), "dynamo is not true"), + ], +) +def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( + config, error +) -> None: + with pytest.raises(ValueError, match=error): + configure_generation_config(config, _Tokenizer()) + + +def test_dynamo_cfg_rejects_unknown_keys() -> None: + with pytest.raises(ValidationError, match="extra_cli_args"): + dynamo_cfg({"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}) + with pytest.raises(ValidationError, match="duplicates"): + dynamo_cfg({"dynamo_cfg": {"worker_args": {"endpoint_types": ["chat"] * 2}}}) + + +def test_frontend_argv_is_runtime_owned() -> None: + cfg = DynamoCfg.model_validate( + {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} + ) + argv = build_dynamo_frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg + ) + assert argv[argv.index("--router-mode") + 1] == "kv" + # Gym names a token-capture call's session; keep the session on one engine. + assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "1800" + assert argv[-2:] == ["--foo", "1"] + no_affinity = DynamoCfg.model_validate( + {"frontend_args": {"session_affinity_ttl_secs": None}} + ) + assert "--router-session-affinity-ttl-secs" not in build_dynamo_frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=no_affinity + ) + + clashing = DynamoCfg.model_validate( + {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} + ) + with pytest.raises(ValueError, match="--http-port is set by both"): + build_dynamo_frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=clashing + ) + + +def test_redaction_hides_credentials() -> None: + assert redact_argv(["frontend", "--api-key", "secret"])[2] == "" + assert redact_environment({"HF_TOKEN": "secret", "NCCL_DEBUG": "INFO"}) == { + "HF_TOKEN": "", + "NCCL_DEBUG": "INFO", + } + + +def _serving(**serving_chat_kwargs) -> dict: + return _vllm_config( + dynamo=True, + vllm_cfg={ + "async_engine": True, + "max_model_len": 512, + "http_server_serving_chat_kwargs": serving_chat_kwargs, + }, + ) + + +def test_parsers_follow_the_native_server_under_dynamos_names() -> None: + assert dynamo_parsers(_serving(tool_parser="hermes", reasoning_parser="qwen3")) == ( + "hermes", + "qwen3", + ) + # NeMo RL's nano_v3 vLLM plugin is Dynamo's nemotron_v3 parser. + nano_v3 = _serving(reasoning_parser="nano_v3") + nano_v3["vllm_cfg"]["reasoning_parser_plugin"] = "nano_v3_reasoning_parser.py" + assert dynamo_parsers(nano_v3) == (None, "nemotron_v3") + assert dynamo_parsers(_serving(tool_parser="openai")) == ("harmony", None) + # dynamo_cfg names them explicitly. + explicit = _serving(tool_parser="hermes") + explicit["dynamo_cfg"] = {"worker_args": {"tool_call_parser": "qwen3_coder"}} + assert dynamo_parsers(explicit) == ("qwen3_coder", None) + + +def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> None: + config = _serving(tool_parser="my_parser") + config["vllm_cfg"]["tool_parser_plugin"] = "my_parser.py" + + with pytest.raises(ValueError, match="worker_args.tool_call_parser"): + configure_generation_config(config, _Tokenizer()) + + +def test_chat_template_and_default_kwargs_follow_the_native_server() -> None: + config = _serving( + chat_template="templates/chat.jinja", + default_chat_template_kwargs={"enable_thinking": False}, + ) + assert dynamo_chat_template(config) == "templates/chat.jinja" + assert native_default_chat_template_kwargs(config) == {"enable_thinking": False} + # The older spelling the native server also accepts. + legacy = _serving(chat_template_kwargs={"enable_thinking": True}) + assert native_default_chat_template_kwargs(legacy) == {"enable_thinking": True} + assert dynamo_chat_template(_serving()) is None + + +@pytest.mark.parametrize( + ("serving_chat_kwargs", "error"), + [ + ({"chat_template_content_format": "string"}, "no content-format setting"), + ({"enable_auto_tools": False}, "always parses"), + ], +) +def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( + serving_chat_kwargs, error +) -> None: + with pytest.raises(ValueError, match=error): + configure_generation_config(_serving(**serving_chat_kwargs), _Tokenizer()) + # The native defaults are what the frontend does anyway. + configure_generation_config( + _serving(chat_template_content_format="auto", enable_auto_tools=True), + _Tokenizer(), + ) + + +def test_a_bearer_token_is_rejected_because_the_frontend_checks_none() -> None: + config = _vllm_config(dynamo=True) + config["vllm_cfg"]["http_server_api_key_required"] = True + + with pytest.raises(ValueError, match="checks no bearer token"): + configure_generation_config(config, _Tokenizer()) diff --git a/tests/unit/models/generation/test_dynamo_generation.py b/tests/unit/models/generation/test_dynamo_generation.py deleted file mode 100644 index 5b13bdf110d..00000000000 --- a/tests/unit/models/generation/test_dynamo_generation.py +++ /dev/null @@ -1,667 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import pickle -from typing import Any -from unittest.mock import MagicMock - -import pytest -import torch - -from nemo_rl.distributed.batched_data_dict import BatchedDataDict -from nemo_rl.models.generation.dynamo import DynamoGeneration -from nemo_rl.models.generation.dynamo import dynamo_generation as generation_module -from nemo_rl.models.generation.dynamo import metrics as metrics_module -from nemo_rl.models.generation.dynamo import refit as refit_module -from nemo_rl.models.generation.dynamo.config import ( - VLLM_PACKED_BUFFER_SIZE_BYTES, - VLLM_PACKED_NUM_BUFFERS, -) -from nemo_rl.models.generation.dynamo.metrics import ( - DynamoMetricsSampler, - parse_prometheus_metrics, -) -from nemo_rl.models.generation.dynamo.refit import DynamoRefitChannel - - -def _config(*, tp: int = 1, expose_http_server: bool = False) -> dict[str, Any]: - return { - "backend": "dynamo", - "model_name": "Qwen/Qwen3-0.6B", - "max_new_tokens": 16, - "temperature": 1.0, - "top_p": 1.0, - "top_k": None, - "stop_token_ids": None, - "stop_strings": None, - "_pad_token_id": 0, - "colocated": {"enabled": False}, - "dynamo_cfg": { - "engine": "vllm", - "startup_timeout_s": 5, - "request_timeout_s": 30, - "control_timeout_s": 10, - "metrics_include_prefixes": None, - "metrics_exclude_prefixes": None, - "worker_args": { - "tool_call_parser": None, - "reasoning_parser": None, - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": False, - "structural_tag_scope": "auto", - "structural_tag_schema": "auto", - "custom_jinja_template": None, - "endpoint_types": ["chat", "completions"], - "extra_cli_args": [], - }, - "frontend_args": { - "tokenizer": "default", - "tokenizer_cache": False, - "tokenizer_cache_bytes": 50 * 1024 * 1024, - "router_mode": "kv", - "extra_cli_args": [], - }, - }, - "vllm_cfg": { - "async_engine": True, - "tensor_parallel_size": tp, - "pipeline_parallel_size": 1, - "expert_parallel_size": tp, - "gpu_memory_utilization": 0.8, - "precision": "bfloat16", - "kv_cache_dtype": "auto", - "max_model_len": 5, - "load_format": "auto", - "enforce_eager": False, - "expose_http_server": expose_http_server, - "enable_vllm_metrics_logger": True, - "vllm_metrics_logger_interval": 1.0, - "env_vars": None, - }, - "vllm_kwargs": {}, - } - - -def _patch_runtime( - monkeypatch: pytest.MonkeyPatch, - workers: list[dict[str, Any]] | None = None, - calls: list[str] | None = None, -) -> None: - endpoints = workers or [ - {"instance_id": "worker-0", "system_url": "http://10.0.0.2:4000"} - ] - events = calls if calls is not None else [] - - class FakeRuntime: - def __init__(self, *, cluster, config): - events.append("init") - - def start(self): - events.append("start") - - @property - def frontend_url(self): - return "http://10.0.0.1:3000/v1" - - def refit_workers(self): - return [dict(worker) for worker in endpoints] - - def validate_workers(self, expected): - return expected - - def shutdown(self): - events.append("shutdown") - - monkeypatch.setattr(generation_module, "ManagedDynamoRuntime", FakeRuntime) - - -def _data() -> BatchedDataDict: - return BatchedDataDict( - { - "input_ids": torch.tensor([[1, 2, 3, 0]], dtype=torch.long), - "input_lengths": torch.tensor([3], dtype=torch.long), - "stop_strings": [["stop"]], - } - ) - - -def _completion_response(token_ids: list[int]) -> dict[str, Any]: - return { - "choices": [ - { - "finish_reason": "stop", - "logprobs": {"token_logprobs": [-0.25] * len(token_ids)}, - } - ], - "nvext": {"completion_token_ids": token_ids}, - } - - -def test_runtime_start_world_size_sender_geometry_and_shutdown(monkeypatch) -> None: - calls: list[str] = [] - _patch_runtime( - monkeypatch, - workers=[ - {"instance_id": "a", "system_url": "http://10.0.0.2:4000"}, - {"instance_id": "b", "system_url": "http://10.0.0.3:4000"}, - ], - calls=calls, - ) - generation = DynamoGeneration(cluster=object(), config=_config(tp=2)) - - assert calls[:2] == ["init", "start"] - assert generation.frontend_url == "http://10.0.0.1:3000/v1" - assert generation.dp_openai_server_base_urls == [None] - assert generation.get_inference_world_size() == 4 - sender = generation.get_collective_sender_spec() - assert sender.nccl_peer == "vllm" - assert sender.buffer_size_bytes == VLLM_PACKED_BUFFER_SIZE_BYTES - assert sender.num_buffers == VLLM_PACKED_NUM_BUFFERS - assert sender.use_legacy_packing - assert generation.shutdown() - assert generation.shutdown() - assert calls.count("shutdown") == 1 - - -def test_blocking_generate_is_rejected_and_async_generation_uses_http( - monkeypatch, -) -> None: - _patch_runtime(monkeypatch) - requests = [] - - async def fake_post(url, payload, timeout_s): - requests.append((url, payload, timeout_s)) - return _completion_response([8, 9]) - - monkeypatch.setattr(generation_module, "async_http_post_json", fake_post) - generation = DynamoGeneration(cluster=object(), config=_config()) - with pytest.raises(NotImplementedError, match="generate_async"): - generation.generate(_data()) - - async def collect(): - return [item async for item in generation.generate_async(_data())] - - outputs = asyncio.run(collect()) - assert outputs[0][0] == 0 - assert outputs[0][1]["output_ids"].tolist() == [[1, 2, 3, 8, 9]] - assert requests[0][0].endswith("/v1/completions") - assert requests[0][1]["max_tokens"] == 2 - assert requests[0][1]["stop"] == ["stop"] - assert "return_tokens_as_token_ids" not in requests[0][1] - - -@pytest.mark.parametrize("prompt_key", ["vllm_content", "vllm_multi_modal_data"]) -def test_async_generate_rejects_multimodal_prompt_columns( - monkeypatch: pytest.MonkeyPatch, prompt_key: str -) -> None: - _patch_runtime(monkeypatch) - post = MagicMock( - side_effect=AssertionError("Multimodal prompts must not reach HTTP") - ) - monkeypatch.setattr(generation_module, "async_http_post_json", post) - generation = DynamoGeneration(cluster=object(), config=_config()) - data = _data() - data[prompt_key] = ["prompt" if prompt_key == "vllm_content" else {"image": "img"}] - - async def collect() -> list: - return [item async for item in generation.generate_async(data)] - - with pytest.raises(NotImplementedError, match="multimodal"): - asyncio.run(collect()) - post.assert_not_called() - - -def test_prompt_at_context_limit_is_rejected(monkeypatch) -> None: - _patch_runtime(monkeypatch) - generation = DynamoGeneration(cluster=object(), config=_config()) - - with pytest.raises(ValueError, match="prompt length 5 must be less than"): - generation._allowed_new_tokens(5) - - -def test_finish_generation_invalidates_sync_rollout_cache(monkeypatch) -> None: - _patch_runtime(monkeypatch) - generation = DynamoGeneration(cluster=object(), config=_config()) - generation.invalidate_kv_cache = MagicMock(return_value=True) - - assert generation.finish_generation() - generation.invalidate_kv_cache.assert_called_once_with() - - -def test_merged_stop_strings_enforce_dynamo_limit(monkeypatch) -> None: - _patch_runtime(monkeypatch) - config = _config() - config["stop_strings"] = [f"configured-{index}" for index in range(16)] - generation = DynamoGeneration(cluster=object(), config=config) - - assert ( - len( - generation._merge_stop_strings( - [[f"request-{index}" for index in range(16)]] - ) - ) - == 32 - ) - with pytest.raises(ValueError, match="at most 32 stop strings"): - generation._merge_stop_strings([[f"request-{index}" for index in range(17)]]) - - -def test_token_wrapper_is_used_for_nemo_gym(monkeypatch) -> None: - _patch_runtime(monkeypatch) - wrappers = [] - - class FakeWrapper: - def __init__(self, **kwargs): - self.kwargs = kwargs - wrappers.append(self) - - def start(self): - return "http://127.0.0.1:3001/v1" - - def shutdown(self): - pass - - monkeypatch.setattr(generation_module, "DynamoTokenWrapperServer", FakeWrapper) - tokenizer = object() - generation = DynamoGeneration( - cluster=object(), - config=_config(expose_http_server=True), - tokenizer=tokenizer, - tokenizer_config={"chat_template_kwargs": {"enable_thinking": False}}, - ) - assert generation.dp_openai_server_base_urls == ["http://127.0.0.1:3001/v1"] - assert wrappers[0].kwargs["tokenizer"] is tokenizer - - -def test_refit_rank_offsets_update_and_pickled_cache_invalidation(monkeypatch) -> None: - workers = [ - {"instance_id": "a", "system_url": "http://10.0.0.2:4000"}, - {"instance_id": "b", "system_url": "http://10.0.0.3:4000"}, - ] - _patch_runtime(monkeypatch, workers=workers) - init_calls = [] - update_calls = [] - cache_calls = [] - monkeypatch.setattr( - refit_module._post_worker_route, - "remote", - lambda **kwargs: init_calls.append(kwargs) or True, - ) - monkeypatch.setattr( - refit_module._update_worker_weights, - "remote", - lambda **kwargs: update_calls.append(kwargs) or True, - ) - monkeypatch.setattr(refit_module.ray, "get", lambda refs: refs) - - generation = DynamoGeneration(cluster=object(), config=_config(tp=2)) - generation.prepare_refit_info({"weight": (torch.Size([4, 8]), torch.bfloat16)}) - assert generation.init_collective("10.1.0.1", 1500, 7, train_world_size=3) == [ - True, - True, - ] - assert [call["payload"]["init_info"]["rank_offset"] for call in init_calls] == [ - 3, - 5, - ] - assert all(call["timeout_s"] == 10 for call in init_calls) - # vLLM reads packing from the init info only (worker default is unpacked). - for call in init_calls: - init_info = call["payload"]["init_info"] - assert init_info["packed"] is True - assert init_info["packed_buffer_size_bytes"] == 1024**3 - assert init_info["packed_num_buffers"] == 2 - assert generation.update_weights_from_collective() == [True, True] - assert update_calls[0]["update_info"] == { - "names": ["weight"], - "dtype_names": ["bfloat16"], - "shapes": [[4, 8]], - } - assert all(call["timeout_s"] == 10 for call in update_calls) - - restored = pickle.loads(pickle.dumps(generation)) - assert restored.frontend_url == generation.frontend_url - # SingleController prepares refit on the driver, then pickles the - # generation client into its actor, which issues the weight updates. - expected_update_info = update_calls[0]["update_info"] - update_calls.clear() - assert restored.update_weights_from_collective() == [True, True] - assert [call["update_info"] for call in update_calls] == [expected_update_info] * 2 - monkeypatch.setattr( - refit_module._post_worker_route, - "remote", - lambda **kwargs: cache_calls.append(kwargs) or True, - ) - assert restored.invalidate_kv_cache() - assert [call["route"] for call in cache_calls] == [ - "pause_generation", - "pause_generation", - "resume_generation", - "resume_generation", - ] - assert all(call["timeout_s"] == 10 for call in cache_calls) - assert all( - call["payload"] == {"mode": "wait", "clear_cache": True} - for call in cache_calls[:2] - ) - assert restored._managed_runtime is None - - # The metrics sampler starts in the process that reads it: the driver starts - # none, and the SingleController's telemetry pump starts it in the copy. - monkeypatch.setattr(metrics_module, "_http_get_text", lambda *args: None) - assert generation._metrics_sampler is None - assert not any(restored.drain_latest_logger_metrics().values()) - assert restored._metrics_sampler is not None - assert restored._metrics_sampler._workers == restored._refit_channel.workers - assert restored.shutdown() - assert restored._metrics_sampler is None - - -def test_cache_invalidation_resumes_workers_that_paused_before_peer_failure( - monkeypatch, -) -> None: - calls = [] - - def remote(**kwargs): - calls.append(kwargs) - return (kwargs["route"], kwargs["system_url"]) - - def get(ref): - if ref == ("pause_generation", "http://worker-b:4000"): - raise RuntimeError("pause refused") - return True - - monkeypatch.setattr(refit_module._post_worker_route, "remote", remote) - monkeypatch.setattr(refit_module.ray, "get", get) - channel = DynamoRefitChannel( - [ - {"instance_id": "a", "system_url": "http://worker-a:4000"}, - {"instance_id": "b", "system_url": "http://worker-b:4000"}, - ], - engine_world_size=1, - control_timeout_s=10, - ) - - with pytest.raises(RuntimeError, match="pause/clear failed"): - channel.flush_cache() - - assert [(call["route"], call["system_url"]) for call in calls] == [ - ("pause_generation", "http://worker-a:4000"), - ("pause_generation", "http://worker-b:4000"), - ("resume_generation", "http://worker-a:4000"), - ] - - -@pytest.mark.parametrize("clear_cache", [True, False]) -def test_refit_pause_drains_every_worker_and_resume_restores_them( - monkeypatch, clear_cache: bool -) -> None: - workers = [ - {"instance_id": "a", "system_url": "http://10.0.0.2:4000"}, - {"instance_id": "b", "system_url": "http://10.0.0.3:4000"}, - ] - _patch_runtime(monkeypatch, workers=workers) - calls = [] - monkeypatch.setattr( - refit_module._post_worker_route, - "remote", - lambda **kwargs: calls.append(kwargs) or True, - ) - monkeypatch.setattr(refit_module.ray, "get", lambda refs: refs) - - # SingleController drives the pause from its own pickled client copy. - generation = pickle.loads( - pickle.dumps(DynamoGeneration(cluster=object(), config=_config())) - ) - assert generation.pause_generation_for_refit(clear_cache=clear_cache) - assert [(call["route"], call["payload"]) for call in calls] == [ - ("pause_generation", {"mode": "wait", "clear_cache": clear_cache}), - ("pause_generation", {"mode": "wait", "clear_cache": clear_cache}), - ] - - calls.clear() - assert generation.resume_generation_after_refit() - assert [(call["route"], call["system_url"]) for call in calls] == [ - ("resume_generation", "http://10.0.0.2:4000"), - ("resume_generation", "http://10.0.0.3:4000"), - ] - - -def test_failed_pause_resumes_the_workers_that_paused(monkeypatch) -> None: - calls = [] - - def remote(**kwargs): - calls.append(kwargs) - return (kwargs["route"], kwargs["system_url"]) - - def get(ref): - if ref == ("pause_generation", "http://worker-b:4000"): - raise RuntimeError("pause refused") - return True - - monkeypatch.setattr(refit_module._post_worker_route, "remote", remote) - monkeypatch.setattr(refit_module.ray, "get", get) - channel = DynamoRefitChannel( - [ - {"instance_id": "a", "system_url": "http://worker-a:4000"}, - {"instance_id": "b", "system_url": "http://worker-b:4000"}, - ], - engine_world_size=1, - control_timeout_s=10, - ) - - with pytest.raises(RuntimeError, match="pause before weight update failed"): - channel.pause(clear_cache=True) - - assert [(call["route"], call["system_url"]) for call in calls] == [ - ("pause_generation", "http://worker-a:4000"), - ("pause_generation", "http://worker-b:4000"), - ("resume_generation", "http://worker-a:4000"), - ] - - -def test_native_refit_transaction_keeps_cache_mode_external(monkeypatch) -> None: - calls = [] - monkeypatch.setattr( - refit_module, - "http_post_json", - lambda url, payload, timeout_s: calls.append(payload) or {"status": "ok"}, - ) - assert refit_module._update_worker_weights._function( - system_url="http://worker:4000", - update_info={"names": ["weight"]}, - timeout_s=30, - ) - assert [call["engine_rpc"] for call in calls] == [ - "start_weight_update", - "update_weights", - "finish_weight_update", - ] - assert all(call["reset_prefix_cache"] is False for call in calls) - # vLLM's Worker.start_weight_update takes no arguments; Dynamo forwards - # every non-reserved key, so any extra key would raise in the engine. - assert set(calls[0]) == {"engine_rpc", "allow_unpaused", "reset_prefix_cache"} - - -def test_metrics_parser_and_sampler_aliases() -> None: - sampler = DynamoMetricsSampler( - [{"instance_id": "a", "system_url": "http://worker:4000"}], - interval_s=1, - include_prefixes=None, - exclude_prefixes=None, - ) - parsed = parse_prometheus_metrics( - 'vllm:num_requests_running{model_name="model",engine="0"} 3\n' - 'vllm:num_requests_waiting{model_name="model",engine="0"} 2\n' - 'vllm:kv_cache_usage_perc{model_name="model",engine="0"} 0.5\n' - 'vllm:generation_tokens_total{model_name="model",engine="0"} 7\n' - 'vllm:gpu_cache_usage_perc{model_name="model",engine="0"} 0.9\n' - "python_gc_objects_collected_total 10\n", - sampler._include_prefixes, - sampler._exclude_prefixes, - ) - sampler._samples = {name: {0: [value]} for name, value in parsed.items()} - metrics = sampler.snapshot() - assert metrics["inflight_batch_sizes"] == {0: [3.0]} - assert metrics["num_pending_samples"] == {0: [2.0]} - assert metrics["kv_cache_usage_perc"] == {0: [0.5]} - assert metrics["generation_tokens"] == {0: [7.0]} - assert "vllm_gpu_cache_usage_perc" not in parsed - assert "python_gc_objects_collected_total" not in parsed - - # Rollout telemetry drains to each worker's latest sample, which it keeps. - sampler._samples["vllm_generation_tokens_total"][0].append(9.0) - for _ in range(2): - assert sampler.drain_latest()["generation_tokens"] == {0: [9.0]} - - -def test_metrics_http_errors_are_ignored(monkeypatch) -> None: - monkeypatch.setattr( - metrics_module.urllib.request, - "urlopen", - lambda *args, **kwargs: (_ for _ in ()).throw( - metrics_module.urllib.error.URLError("refused") - ), - ) - assert metrics_module._http_get_text("http://worker/metrics", 1) is None - - -def test_completion_parser_rejects_misaligned_logprobs() -> None: - response = _completion_response([8, 9]) - response["choices"][0]["logprobs"]["token_logprobs"] = [-0.1] - with pytest.raises(RuntimeError, match="1 token logprobs for 2"): - generation_module._parse_dynamo_completion_response( - response, request_url="http://dynamo/v1/completions" - ) - - -@pytest.mark.parametrize( - ("response", "expected"), - [ - ({"http_status": 408}, True), - ({"http_status": 429}, True), - ({"http_status": 503}, True), - ({"transport_error": "refused"}, True), - ({"json_decode_error": True}, True), - ({"http_status": 400}, False), - ({"http_status": 404}, False), - ], -) -def test_completion_retry_predicate(response, expected) -> None: - assert generation_module._is_retryable_http_response(response) is expected - - -def test_completion_retry_eventually_succeeds(monkeypatch) -> None: - _patch_runtime(monkeypatch) - responses = iter( - [{"status": "error", "http_status": 503}, _completion_response([8])] - ) - calls = [] - - async def fake_post(*args): - calls.append(args) - return next(responses) - - async def no_sleep(_): - return None - - monkeypatch.setattr(generation_module, "async_http_post_json", fake_post) - monkeypatch.setattr(generation_module.asyncio, "sleep", no_sleep) - generation = DynamoGeneration(cluster=object(), config=_config()) - - token_ids, _, _ = asyncio.run( - generation._post_completion_request( - prompt_token_ids=[1], - greedy=False, - stop_strings=None, - max_new_tokens=1, - ) - ) - - assert token_ids == [8] - assert len(calls) == 2 - - -@pytest.mark.parametrize("status", [400, 503]) -def test_completion_retry_stops_on_nonretryable_or_exhaustion( - monkeypatch, status -) -> None: - _patch_runtime(monkeypatch) - calls = [] - - async def fake_post(*args): - calls.append(args) - return {"status": "error", "http_status": status} - - async def no_sleep(_): - return None - - monkeypatch.setattr(generation_module, "async_http_post_json", fake_post) - monkeypatch.setattr(generation_module.asyncio, "sleep", no_sleep) - generation = DynamoGeneration(cluster=object(), config=_config()) - - with pytest.raises(RuntimeError, match=f"HTTP {status}"): - asyncio.run( - generation._post_completion_request( - prompt_token_ids=[1], - greedy=False, - stop_strings=None, - max_new_tokens=1, - ) - ) - - assert len(calls) == (1 if status == 400 else generation_module._HTTP_MAX_ATTEMPTS) - - -def test_direct_completions_are_not_limited_by_default_thread_pool( - monkeypatch, -) -> None: - _patch_runtime(monkeypatch) - request_count = 40 - entered_count = 0 - all_entered = asyncio.Event() - release = asyncio.Event() - - async def fake_post(*args): - nonlocal entered_count - entered_count += 1 - if entered_count == request_count: - all_entered.set() - await release.wait() - return _completion_response([8]) - - monkeypatch.setattr(generation_module, "async_http_post_json", fake_post) - generation = DynamoGeneration(cluster=object(), config=_config()) - - async def run_requests(): - tasks = [ - asyncio.create_task( - generation._post_completion_request( - prompt_token_ids=[1], - greedy=False, - stop_strings=None, - max_new_tokens=1, - ) - ) - for _ in range(request_count) - ] - await asyncio.wait_for(all_entered.wait(), timeout=1) - release.set() - return await asyncio.gather(*tasks) - - responses = asyncio.run(run_requests()) - - assert entered_count == request_count - assert len(responses) == request_count diff --git a/tests/unit/models/generation/test_dynamo_http_client.py b/tests/unit/models/generation/test_dynamo_http_client.py index 195db9bb55a..d9002bce5c1 100644 --- a/tests/unit/models/generation/test_dynamo_http_client.py +++ b/tests/unit/models/generation/test_dynamo_http_client.py @@ -13,29 +13,12 @@ # limitations under the License. import asyncio -import io -import json -import urllib.error import aiohttp import pytest -from nemo_rl.models.generation.dynamo import dynamo_generation as generation_module from nemo_rl.models.generation.dynamo import http_client - - -class _SyncResponse: - def __init__(self, body: bytes): - self._body = body - - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self) -> bytes: - return self._body +from nemo_rl.models.generation.dynamo.rollout_client import _is_retryable class _AsyncResponse: @@ -72,89 +55,6 @@ def post(self, url, *, json): return self._response -def _sync_post(monkeypatch, result): - def fake_urlopen(request, timeout): - if isinstance(result, BaseException): - raise result - return _SyncResponse(result) - - monkeypatch.setattr(http_client.urllib.request, "urlopen", fake_urlopen) - return http_client.http_post_json("http://worker/route", {"value": 1}, 3) - - -def test_http_post_json_success_preserves_request_contract(monkeypatch) -> None: - captured = {} - - def fake_urlopen(request, timeout): - captured["request"] = request - captured["timeout"] = timeout - return _SyncResponse(b'{"status":"ok"}') - - monkeypatch.setattr(http_client.urllib.request, "urlopen", fake_urlopen) - - response = http_client.http_post_json( - "http://worker/route", {"value": 1}, timeout_s=3 - ) - - request = captured["request"] - assert response == {"status": "ok"} - assert request.full_url == "http://worker/route" - assert request.get_method() == "POST" - assert json.loads(request.data) == {"value": 1} - assert captured["timeout"] == 3 - - -@pytest.mark.parametrize( - ("body", "expected"), - [ - ( - b"not-json", - {"status": "error", "json_decode_error": True, "raw": "not-json"}, - ), - (b"[1, 2]", {"status": "error", "raw": "[1, 2]"}), - ], -) -def test_http_post_json_rejects_invalid_json_shapes( - monkeypatch, body, expected -) -> None: - assert _sync_post(monkeypatch, body) == expected - - -@pytest.mark.parametrize( - ("error", "retryable", "message"), - [ - ( - urllib.error.HTTPError( - "http://worker/route", - 503, - "unavailable", - hdrs=None, - fp=io.BytesIO(b"try later"), - ), - True, - "HTTP 503: try later", - ), - ( - urllib.error.HTTPError( - "http://worker/route", 400, "bad request", hdrs=None, fp=None - ), - False, - "HTTP 400: ", - ), - (urllib.error.URLError("refused"), True, "URLError"), - (TimeoutError("slow"), True, "TimeoutError"), - ], -) -def test_http_errors_round_trip_through_consumers( - monkeypatch, error, retryable, message -) -> None: - response = _sync_post(monkeypatch, error) - - assert response["status"] == "error" - assert generation_module._is_retryable_http_response(response) is retryable - assert message in http_client.format_dynamo_error(response) - - @pytest.mark.parametrize( ("body", "status", "expected", "retryable"), [ @@ -173,7 +73,7 @@ def test_http_errors_round_trip_through_consumers( ), ], ) -def test_async_http_post_json_uses_same_error_contract( +def test_async_http_post_json_error_contract( monkeypatch, body, status, expected, retryable ) -> None: session = _AsyncSession(_AsyncResponse(body, status)) @@ -190,14 +90,22 @@ def test_async_http_post_json_uses_same_error_contract( ) assert response == expected - assert generation_module._is_retryable_http_response(response) is retryable + assert _is_retryable(response) is retryable + if response.get("status") == "error": + assert expected["raw"] in http_client.format_dynamo_error(response) assert session.requests == [("http://worker/route", {"value": 1})] @pytest.mark.parametrize( - "error", [aiohttp.ClientConnectionError("refused"), TimeoutError("slow")] + ("error", "key", "retryable"), + [ + (aiohttp.ClientConnectionError("refused"), "transport_error", True), + # One request_timeout_s bounds a rollout: a timeout is not retried. + (TimeoutError("slow"), "timeout_error", False), + (aiohttp.ServerTimeoutError("slow read"), "timeout_error", False), + ], ) -def test_async_http_transport_errors_are_retryable(monkeypatch, error) -> None: +def test_async_http_transport_errors(monkeypatch, error, key, retryable) -> None: session = _AsyncSession(error=error) monkeypatch.setattr( http_client.aiohttp, @@ -212,5 +120,6 @@ def test_async_http_transport_errors_are_retryable(monkeypatch, error) -> None: ) assert response["status"] == "error" - assert generation_module._is_retryable_http_response(response) - assert "transport_error" in response + assert _is_retryable(response) is retryable + assert key in response + assert response[key] in http_client.format_dynamo_error(response) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index e13cce6aaf2..0b56635518c 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -11,128 +11,31 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import json import signal import subprocess import sys from pathlib import Path -from types import SimpleNamespace from unittest.mock import MagicMock import pytest -from nemo_rl.models.generation.dynamo.dynamo_worker import ( - DynamoGpuReservation, - DynamoVllmWorker, -) +from nemo_rl.models.generation.dynamo import venv as dynamo_venv +from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, _managed_namespace, ) -from nemo_rl.models.generation.dynamo.venv import get_dynamo_venv_dir -from nemo_rl.models.generation.dynamo.worker_pool import ( - FixedDynamoWorkerPool, - _vllm_port_for_node_slot, -) - -class _Cluster: - num_gpus_per_node = 4 - -def test_dynamo_package_directory_does_not_shadow_stdlib_http() -> None: - package_dir = ( - Path(__file__).resolve().parents[4] - / "nemo_rl" - / "models" - / "generation" - / "dynamo" - ) - result = subprocess.run( - [sys.executable, "-c", "import http.client"], - cwd=package_dir, - capture_output=True, - text=True, - check=False, +def _runtime(**frontend_args) -> ManagedDynamoRuntime: + return ManagedDynamoRuntime( + dynamo_cfg=DynamoCfg.model_validate( + {"startup_timeout_s": 5, "frontend_args": frontend_args} + ), + model_name="model", + frontend_python="/venvs/vllm/bin/python", ) - assert result.returncode == 0, result.stderr - - -def _config(*, tp: int = 1) -> dict: - return { - "backend": "dynamo", - "model_name": "model", - "colocated": {"enabled": False}, - "dynamo_cfg": { - "engine": "vllm", - "startup_timeout_s": 5, - "request_timeout_s": 30, - "control_timeout_s": 10, - "metrics_include_prefixes": None, - "metrics_exclude_prefixes": None, - "worker_args": { - "tool_call_parser": None, - "reasoning_parser": None, - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": False, - "structural_tag_scope": "auto", - "structural_tag_schema": "auto", - "custom_jinja_template": None, - "endpoint_types": ["chat", "completions"], - "extra_cli_args": [], - }, - "frontend_args": { - "tokenizer": "default", - "tokenizer_cache": False, - "tokenizer_cache_bytes": 50 * 1024 * 1024, - "router_mode": "kv", - "extra_cli_args": [], - }, - }, - "vllm_cfg": { - "async_engine": True, - "tensor_parallel_size": tp, - "pipeline_parallel_size": 1, - "expert_parallel_size": tp, - "gpu_memory_utilization": 0.8, - "max_model_len": 512, - "precision": "bfloat16", - "kv_cache_dtype": "auto", - "load_format": "auto", - "enforce_eager": False, - "expose_http_server": False, - "enable_vllm_metrics_logger": True, - "vllm_metrics_logger_interval": 1.0, - "env_vars": None, - }, - "vllm_kwargs": {}, - } - - -class _RemoteMethod: - def __init__(self, result): - self._result = result - self.calls = [] - - def remote(self, *args, **kwargs): - self.calls.append((args, kwargs)) - return self._result - - -class _FakeWorker: - def __init__(self, alive=True, metadata=None): - self.is_alive = _RemoteMethod(alive) - self.metadata = _RemoteMethod(metadata or {}) - self.shutdown = _RemoteMethod(True) - - -class _FakeReservation: - def __init__(self, metadata=None, system_port=4000): - self.metadata = _RemoteMethod(metadata or {}) - self.select_free_port = _RemoteMethod(system_port) - self.register_process_group = _RemoteMethod(True) - self.cleanup_process_group = _RemoteMethod(True) class _FakeProcess: @@ -159,35 +62,55 @@ def read(self): return json.dumps(self._payload).encode() +def _started(runtime: ManagedDynamoRuntime) -> ManagedDynamoRuntime: + runtime._started = True + runtime._host = "10.0.0.1" + runtime._frontend_port = 3000 + runtime._namespace = "nemo-rl-test" + runtime._etcd_process = _FakeProcess() + runtime._nats_process = _FakeProcess() + runtime._frontend_process = _FakeProcess() + return runtime + + +def test_dynamo_package_directory_does_not_shadow_stdlib_http() -> None: + package_dir = ( + Path(__file__).resolve().parents[4] + / "nemo_rl" + / "models" + / "generation" + / "dynamo" + ) + result = subprocess.run( + [sys.executable, "-c", "import http.client"], + cwd=package_dir, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + def test_runtime_construction_is_inert_and_namespace_is_driver_owned( monkeypatch, ) -> None: monkeypatch.setenv("SLURM_JOB_ID", "Job/123.4") assert _managed_namespace() == "nemo-rl-job-123-4" - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) + runtime = _runtime() assert runtime._started is False assert runtime._etcd_process is None with pytest.raises(RuntimeError, match="not been started"): _ = runtime.frontend_url - - -def test_runtime_rejects_multinode_engine_group_before_spawning() -> None: - with pytest.raises(ValueError, match="fit on one node"): - ManagedDynamoRuntime(cluster=_Cluster(), config=_config(tp=8)) + with pytest.raises(RuntimeError, match="not been started"): + _ = runtime.engine_env def test_managed_service_and_frontend_environments_are_runtime_owned( monkeypatch, ) -> None: - config = _config() - config["dynamo_cfg"]["frontend_args"].update( - { - "tokenizer": "fastokens", - "tokenizer_cache": True, - "tokenizer_cache_bytes": 4096, - } + runtime = _runtime( + tokenizer="fastokens", tokenizer_cache=True, tokenizer_cache_bytes=4096 ) - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=config) runtime._manager_env = { "DYN_NAMESPACE": "managed", "DYN_DISCOVERY_BACKEND": "etcd", @@ -196,22 +119,18 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( } monkeypatch.setenv("DYN_NAMESPACE", "stale") monkeypatch.setenv("DYN_STALE_SETTING", "remove-me") - monkeypatch.setenv("ETCD_ENDPOINTS", "http://stale-etcd:2379") monkeypatch.setenv("ETCD_STALE_SETTING", "remove-me") - monkeypatch.setenv("NATS_SERVER", "nats://stale-nats:4222") monkeypatch.setenv("NATS_STALE_SETTING", "remove-me") monkeypatch.setenv("NCCL_DEBUG", "INFO") service_env = runtime._service_env() assert service_env["DYN_NAMESPACE"] == "managed" - assert service_env["DYN_DISCOVERY_BACKEND"] == "etcd" - assert "DYN_STALE_SETTING" not in service_env assert service_env["ETCD_ENDPOINTS"] == "http://managed-etcd:2379" assert service_env["NATS_SERVER"] == "nats://managed-nats:4222" + assert "DYN_STALE_SETTING" not in service_env assert "ETCD_STALE_SETTING" not in service_env assert "NATS_STALE_SETTING" not in service_env assert service_env["NCCL_DEBUG"] == "INFO" - assert "ALLOW_NONE_AUTHENTICATION" not in service_env frontend_env = runtime._frontend_env() assert frontend_env["DYN_TOKENIZER"] == "fastokens" @@ -219,99 +138,54 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( assert frontend_env["DYN_TOKENIZER_CACHE_BYTES"] == "4096" -def test_dynamo_venv_uses_explicit_env_or_repository_fallback(monkeypatch) -> None: - monkeypatch.setenv("NEMO_RL_DYNAMO_VENV_DIR", "/custom/dynamo") - assert get_dynamo_venv_dir() == Path("/custom/dynamo") - - monkeypatch.delenv("NEMO_RL_DYNAMO_VENV_DIR") - monkeypatch.setenv("NRL_CONTAINER", "1") - assert get_dynamo_venv_dir().parts[-2:] == ("venvs", "dynamo") - - -def test_vllm_node_local_port_bands_are_deterministic() -> None: - assert [_vllm_port_for_node_slot(slot) for slot in range(3)] == [7000, 7100, 7200] - - -def test_startup_failure_cleans_up_partial_worker_pool(monkeypatch, tmp_path) -> None: - calls = [] - exit_hooks = [] - pool_init_kwargs = {} - - class FailingPool: - def __init__(self, **kwargs): - calls.append("pool-init") - pool_init_kwargs.update(kwargs) - - def start(self): - calls.append("pool-start") - raise RuntimeError("worker failed") - - def shutdown(self): - calls.append("pool-shutdown") - - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) - etcd_dir = tmp_path / "etcd" - nats_dir = tmp_path / "nats" - etcd_dir.mkdir() - nats_dir.mkdir() - ports = iter([1313, 1314, 1315, 3000]) +def test_dynamo_python_resolves_the_vllm_worker_venv_as_workers_do( + monkeypatch, +) -> None: + built = [] monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime._get_node_ip_local", - lambda: "10.0.0.1", + dynamo_venv, "get_actor_python_env", lambda fqn: "uv run --locked --extra vllm" ) + # The workers' resolver: reuses an existing venv, builds a missing one per node. monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime._get_free_port_local", - lambda low, high, **kwargs: next(ports), + dynamo_venv, + "create_local_venv_on_each_node", + lambda py_executable, venv_name: built.append((py_executable, venv_name)) + or "/venvs/vllm/bin/python", ) - def start_etcd(): - calls.append("etcd") - runtime._etcd_process = object() - runtime._etcd_data_dir = str(etcd_dir) - - def start_nats(): - calls.append("nats") - runtime._nats_process = object() - runtime._nats_data_dir = str(nats_dir) + assert dynamo_venv.get_dynamo_python() == "/venvs/vllm/bin/python" + assert built == [ + ("uv run --locked --extra vllm", dynamo_venv.VLLM_ASYNC_WORKER_FQN) + ] - def stop_process(process, label, timeout_s=15): - if process is not None: - calls.append(f"stop-{label}") - monkeypatch.setattr(runtime, "_start_etcd", start_etcd) - monkeypatch.setattr(runtime, "_start_nats", start_nats) - monkeypatch.setattr(runtime, "_stop_process", stop_process) +def test_dynamo_python_passes_an_interpreter_through(monkeypatch) -> None: monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime.atexit.register", - lambda hook: exit_hooks.append(hook), + dynamo_venv, "get_actor_python_env", lambda fqn: "/usr/bin/python3" ) monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime.atexit.unregister", - lambda hook: exit_hooks.remove(hook), - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime.FixedDynamoWorkerPool", - FailingPool, + dynamo_venv, + "create_local_venv_on_each_node", + MagicMock(side_effect=AssertionError("no venv for an interpreter")), ) - with pytest.raises(RuntimeError, match="worker failed"): - runtime.start() - assert calls == [ - "etcd", - "nats", - "pool-init", - "pool-start", - "pool-shutdown", - "stop-NATS", - "stop-etcd", - ] - assert runtime._pool is None - assert runtime._started is False - assert exit_hooks == [] - assert pool_init_kwargs["manager_env"]["DYN_ENABLE_EXPERIMENTAL_PARSERS_V2"] == "1" - assert pool_init_kwargs["manager_env"]["DYN_RL_INIT_WEIGHTS_TIMEOUT_S"] == "10.0" - assert not etcd_dir.exists() - assert not nats_dir.exists() + assert dynamo_venv.get_dynamo_python() == "/usr/bin/python3" + + +def test_dynamo_services_resolve_from_bin_dir_then_path(monkeypatch, tmp_path) -> None: + etcd = tmp_path / "etcd" + etcd.write_text("#!/bin/sh\n") + etcd.chmod(0o755) + monkeypatch.setenv("NEMO_RL_DYNAMO_BIN_DIR", str(tmp_path)) + assert dynamo_venv.get_dynamo_service_executable("etcd") == str(etcd) + with pytest.raises(FileNotFoundError, match="NEMO_RL_DYNAMO_BIN_DIR"): + dynamo_venv.get_dynamo_service_executable("nats-server") + + monkeypatch.delenv("NEMO_RL_DYNAMO_BIN_DIR") + monkeypatch.setenv("PATH", str(tmp_path)) + assert dynamo_venv.get_dynamo_service_executable("etcd") == str(etcd) + with pytest.raises(FileNotFoundError, match="not on PATH"): + dynamo_venv.get_dynamo_service_executable("nats-server") def test_stop_process_escalates_its_process_group(monkeypatch) -> None: @@ -337,144 +211,17 @@ def wait(self, timeout): lambda pid, sig: signals.append((pid, sig)), ) process = EscalatingProcess() - ManagedDynamoRuntime._stop_process(process, "worker", timeout_s=0.01) + ManagedDynamoRuntime._stop_process(process, "frontend", timeout_s=0.01) assert signals == [(process.pid, signal.SIGTERM), (process.pid, signal.SIGKILL)] -def test_reservation_records_and_cleans_registered_process_group(monkeypatch) -> None: - reservation_cls = DynamoGpuReservation.__ray_metadata__.modified_class - reservation = reservation_cls() - signals = [] - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.os.killpg", - lambda pid, sig: signals.append((pid, sig)), - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.time.sleep", lambda _: None - ) - - assert reservation.register_process_group(4321) - assert reservation.cleanup_process_group() - assert reservation.cleanup_process_group() - assert signals == [(4321, signal.SIGTERM), (4321, signal.SIGKILL)] - - -def test_reservation_selects_a_free_nonexcluded_system_port(monkeypatch) -> None: - reservation_cls = DynamoGpuReservation.__ray_metadata__.modified_class - reservation = reservation_cls() - calls = [] - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker._get_free_port_local", - lambda low, high, **kwargs: calls.append((low, high, kwargs)) or 4002, - ) - - assert ( - reservation.select_free_port( - port_range_low=4000, - port_range_high=4003, - excluded_ports=[4001], - ) - == 4002 - ) - assert calls == [ - ( - 4000, - 4003, - {"max_retries": None, "excluded_ports": {4001}}, - ) - ] - - -def test_worker_argument_validation_has_startup_timeout(monkeypatch) -> None: - worker_cls = DynamoVllmWorker.__ray_metadata__.modified_class - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.subprocess.run", - lambda *args, **kwargs: (_ for _ in ()).throw( - subprocess.TimeoutExpired("validator", kwargs["timeout"]) - ), - ) - - with pytest.raises(RuntimeError, match="startup_timeout_s=7"): - worker_cls._validate_argv( - "/opt/dynamo_venv/bin/python", - ["--model", "model"], - {}, - timeout_s=7, - ) - - -def test_worker_registers_process_group_immediately_after_launch(monkeypatch) -> None: - class FakeSocket: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def bind(self, address): - if address[1] == 7000: - raise AssertionError("VLLM_PORT must not be probed") - return None - - process = SimpleNamespace(pid=4321) - reservation = _FakeReservation() - worker_cls = DynamoVllmWorker.__ray_metadata__.modified_class - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker._get_node_ip_local", - lambda: "10.0.0.1", - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.socket.socket", - lambda *args, **kwargs: FakeSocket(), - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.get_dynamo_python", - lambda: "/opt/dynamo_venv/bin/python", - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.get_dynamo_venv_dir", - lambda: Path("/opt/dynamo_venv"), - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.subprocess.Popen", - lambda *args, **kwargs: process, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.ray.get", lambda ref: ref - ) - monkeypatch.setattr(worker_cls, "_validate_argv", MagicMock()) - monkeypatch.setattr(worker_cls, "_wait_for_system_port", MagicMock()) - - worker_cls( - _config(), - namespace="nemo-rl-test", - group_name="worker-0", - cuda_devices=[0], - system_port=4000, - vllm_port=7000, - manager_env={}, - startup_timeout_s=5, - seed=0, - cleanup_reservation=reservation, - ) - - assert reservation.register_process_group.calls == [((4321,), {})] - - def test_shutdown_guards_each_owned_resource_independently(monkeypatch) -> None: - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) + runtime = _runtime() runtime._frontend_process = object() runtime._nats_process = object() runtime._etcd_process = object() calls = [] - class FailingPool: - def shutdown(self): - calls.append("pool") - raise RuntimeError("pool failure") - - runtime._pool = FailingPool() - def stop_process(process, label, timeout_s=15): if process is None: return @@ -486,320 +233,34 @@ def stop_process(process, label, timeout_s=15): runtime.shutdown() runtime.shutdown() - assert calls[:4] == ["frontend", "pool", "NATS", "etcd"] + assert calls == ["frontend", "NATS", "etcd"] assert runtime._frontend_process is None - assert runtime._pool is None assert runtime._nats_process is None assert runtime._etcd_process is None -def test_fixed_pool_detects_worker_membership_change(monkeypatch) -> None: - expected = [{"instance_id": "worker-0", "system_url": "http://10.0.0.1:4000"}] - reservation_metadata = {"node_ip": "10.0.0.1", "gpu_id": 0} - reservation = _FakeReservation(metadata=reservation_metadata) - pool = object.__new__(FixedDynamoWorkerPool) - pool._workers = [_FakeWorker(metadata={"instance_id": "changed"})] - pool._reservations = [reservation] - pool._cleanup_reservations = [reservation] - pool._reservation_metadata = [reservation_metadata] - - def fake_get(refs, **kwargs): - if refs == [True]: - return [True] - if refs == [reservation_metadata]: - return [reservation_metadata] - return [{"instance_id": "changed"}] - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - with pytest.raises(RuntimeError, match="worker membership changed"): - pool.validate(expected) - - -def test_fixed_pool_detects_reservation_membership_change(monkeypatch) -> None: - expected_worker = { - "instance_id": "worker-0", - "system_url": "http://10.0.0.1:4000", - } - expected_reservation = {"node_ip": "10.0.0.1", "gpu_id": 0} - changed_reservation = {"node_ip": "10.0.0.2", "gpu_id": 0} - reservation = _FakeReservation(metadata=changed_reservation) - pool = object.__new__(FixedDynamoWorkerPool) - pool._workers = [_FakeWorker(metadata=expected_worker)] - pool._reservations = [reservation] - pool._cleanup_reservations = [reservation] - pool._reservation_metadata = [expected_reservation] - - def fake_get(refs, **kwargs): - if refs == [True]: - return [True] - if refs == [changed_reservation]: - return [changed_reservation] - return [expected_worker] - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - with pytest.raises(RuntimeError, match="GPU reservation membership changed"): - pool.validate([expected_worker]) - - -def test_fixed_pool_tracks_worker_before_metadata_failure(monkeypatch) -> None: - reservation = _FakeReservation(metadata={"node_ip": "10.0.0.1", "gpu_id": 0}) - worker = _FakeWorker() - - class RemoteFactory: - def __init__(self, actor): - self.actor = actor - - def remote(self, *args, **kwargs): - return self.actor - - class ReservationFactory: - @staticmethod - def options(**kwargs): - return RemoteFactory(reservation) - - class WorkerFactory: - @staticmethod - def options(**kwargs): - return RemoteFactory(worker) - - class Cluster: - @staticmethod - def get_placement_groups(): - return [SimpleNamespace(bundle_count=1)] - - pool = FixedDynamoWorkerPool( - cluster=Cluster(), - config=_config(), - namespace="nemo-rl-test", - engine_world_size=1, - manager_env={}, - startup_timeout_s=5, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.DynamoGpuReservation", - ReservationFactory, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.DynamoVllmWorker", - WorkerFactory, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.get_actor_python_env", - lambda _fqn: "python", - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.PlacementGroupSchedulingStrategy", - lambda **kwargs: kwargs, - ) - - def fake_get(refs, **kwargs): - if refs == [reservation.metadata.remote()]: - return [{"node_ip": "10.0.0.1", "gpu_id": 0}] - if refs == 4000: - return refs - raise RuntimeError("metadata failed") - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - with pytest.raises(RuntimeError, match="metadata failed"): - pool.start() - - assert pool._workers == [worker] - assert pool._cleanup_reservations == [reservation] - assert pool._metadata == [{}] - - -def test_fixed_pool_launches_all_workers_before_waiting_for_model_metadata( +def test_engines_are_ready_once_registered_and_the_model_is_served( monkeypatch, ) -> None: - reservation_objects = [ - _FakeReservation( - metadata={"node_ip": "10.0.0.1", "gpu_id": 0}, system_port=4001 - ), - _FakeReservation( - metadata={"node_ip": "10.0.0.1", "gpu_id": 1}, system_port=4002 - ), - ] - reservations = iter(reservation_objects) - workers = [ - _FakeWorker(metadata={"instance_id": "worker-0"}), - _FakeWorker(metadata={"instance_id": "worker-1"}), - ] - workers_to_launch = iter(workers) - launch_kwargs = [] - - class RemoteFactory: - def __init__(self, actor): - self.actor = actor - - def remote(self, *args, **kwargs): - return self.actor - - class ReservationFactory: - @staticmethod - def options(**kwargs): - return RemoteFactory(next(reservations)) - - class WorkerFactory: - @staticmethod - def options(**kwargs): - worker = next(workers_to_launch) - - class CapturingRemoteFactory(RemoteFactory): - def remote(self, *args, **kwargs): - launch_kwargs.append(kwargs) - return super().remote(*args, **kwargs) - - return CapturingRemoteFactory(worker) - - class Cluster: - @staticmethod - def get_placement_groups(): - return [SimpleNamespace(bundle_count=2)] - - pool = FixedDynamoWorkerPool( - cluster=Cluster(), - config=_config(), - namespace="nemo-rl-test", - engine_world_size=1, - manager_env={}, - startup_timeout_s=5, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.DynamoGpuReservation", - ReservationFactory, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.DynamoVllmWorker", - WorkerFactory, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.get_actor_python_env", - lambda _fqn: "python", - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.PlacementGroupSchedulingStrategy", - lambda **kwargs: kwargs, - ) - - def fake_get(refs, **kwargs): - if isinstance(refs, int): - return refs - if isinstance(refs, list) and refs and "node_ip" in refs[0]: - return refs - assert pool._workers == workers - return refs - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - - pool.start() - - assert pool._metadata == [ - {"instance_id": "worker-0"}, - {"instance_id": "worker-1"}, - ] - assert [kwargs["system_port"] for kwargs in launch_kwargs] == [4001, 4002] - assert [kwargs["vllm_port"] for kwargs in launch_kwargs] == [7000, 7100] - assert [kwargs["cleanup_reservation"] for kwargs in launch_kwargs] == ( - reservation_objects - ) - assert reservation_objects[0].select_free_port.calls == [ - ( - (), - { - "port_range_low": 4000, - "port_range_high": 4100, - "excluded_ports": [], - }, - ) - ] - assert reservation_objects[1].select_free_port.calls == [ - ( - (), - { - "port_range_low": 4000, - "port_range_high": 4100, - "excluded_ports": [4001], - }, - ) - ] - - -def test_fixed_pool_shutdown_releases_workers_and_reservations(monkeypatch) -> None: - pool = object.__new__(FixedDynamoWorkerPool) - worker = _FakeWorker() - reservation = _FakeReservation() - pool._workers = [worker] - pool._reservations = [reservation] - pool._cleanup_reservations = [reservation] - pool._reservation_metadata = [] - pool._metadata = [{"instance_id": "worker-0", "process_pid": 1234}] - killed = [] - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", - lambda refs, **kwargs: [True], - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.kill", - lambda actor, **kwargs: killed.append(actor), - ) - pool.shutdown() - assert killed == [worker, reservation] - assert pool._workers == [] - assert pool._reservations == [] - - -def test_fixed_pool_shutdown_uses_registered_pid_when_worker_dies(monkeypatch) -> None: - worker = _FakeWorker() - reservation = _FakeReservation() - pool = object.__new__(FixedDynamoWorkerPool) - pool._workers = [worker] - pool._reservations = [reservation] - pool._cleanup_reservations = [reservation] - pool._reservation_metadata = [] - pool._metadata = [{}] - shutdown_ref = worker.shutdown.remote() - - def fake_get(ref, **kwargs): - if ref == shutdown_ref: - raise RuntimeError("worker died") - return True - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.kill", - lambda actor, **kwargs: None, - ) - - pool.shutdown() - - assert reservation.cleanup_process_group.calls == [((), {})] - - -def test_frontend_waits_for_model_after_endpoint_registration(monkeypatch) -> None: - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) - runtime._frontend_port = 3000 - runtime._frontend_process = _FakeProcess() - runtime._namespace = "nemo-rl-test" - runtime._pool = SimpleNamespace(is_alive=lambda: True) + runtime = _started(_runtime()) health = { "instances": [ { "namespace": "nemo-rl-test", "component": "backend", - "endpoint": endpoint, - "instance_id": "worker-0", + "endpoint": "generate", + "instance_id": f"engine-{idx}", + } + for idx in range(2) + ] + # Another namespace's engine does not count toward this fleet. + + [ + { + "namespace": "other", + "component": "backend", + "endpoint": "generate", + "instance_id": "foreign", } - for endpoint in ("generate", "rl") ] } responses = iter( @@ -823,44 +284,69 @@ def fake_urlopen(url, timeout): monkeypatch.setattr( "nemo_rl.models.generation.dynamo.managed_runtime.time.sleep", lambda _: None ) - runtime._wait_for_frontend(expected_workers=1) - assert urls == [ - "http://127.0.0.1:3000/health", - "http://127.0.0.1:3000/v1/models", - "http://127.0.0.1:3000/health", - "http://127.0.0.1:3000/v1/models", - ] + runtime.wait_for_engines(2, check_engines=lambda: None) + assert ( + urls + == [ + "http://127.0.0.1:3000/health", + "http://127.0.0.1:3000/v1/models", + ] + * 2 + ) -def test_frontend_wait_fails_immediately_when_worker_exits(monkeypatch) -> None: - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) - runtime._frontend_port = 3000 - runtime._frontend_process = _FakeProcess() - runtime._pool = SimpleNamespace(is_alive=lambda: False) +def test_engine_wait_fails_immediately_with_the_engines_error() -> None: + runtime = _started(_runtime()) - with pytest.raises(RuntimeError, match="worker exited while the frontend"): - runtime._wait_for_frontend(expected_workers=1) + def endpoint_stopped() -> None: + raise RuntimeError("endpoint of generation shard 0 stopped serving") + with pytest.raises(RuntimeError, match="shard 0 stopped serving"): + runtime.wait_for_engines(1, check_engines=endpoint_stopped) -def test_frontend_logs_resolved_tokenizer_environment(monkeypatch, capsys) -> None: - config = _config() - config["dynamo_cfg"]["frontend_args"].update( - {"tokenizer": "fastokens", "tokenizer_cache": True} - ) - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=config) + +def test_frontend_launches_from_the_given_interpreter(monkeypatch, capsys) -> None: + runtime = _runtime(tokenizer="fastokens", tokenizer_cache=True) runtime._frontend_port = 3000 runtime._namespace = "nemo-rl-test" - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime.get_dynamo_python", - lambda: "/opt/dynamo_venv/bin/python", - ) + launched = [] monkeypatch.setattr( "nemo_rl.models.generation.dynamo.managed_runtime.subprocess.Popen", - lambda *args, **kwargs: _FakeProcess(), + lambda command, **kwargs: launched.append(command) or _FakeProcess(), ) runtime._start_frontend() + assert launched[0][:3] == ["/venvs/vllm/bin/python", "-m", "dynamo.frontend"] output = capsys.readouterr().out assert "DYN_TOKENIZER': 'fastokens" in output - assert "" not in output + + +def test_shutdown_logs_the_kv_events_the_router_applied(monkeypatch, capsys) -> None: + runtime = _started(_runtime()) + metrics = ( + "# HELP dynamo_component_kv_cache_events_applied KV events applied\n" + 'dynamo_component_kv_cache_events_applied{status="ok",worker_id="1"} 40\n' + 'dynamo_component_kv_cache_events_applied{status="ok",worker_id="2"} 2\n' + 'dynamo_component_kv_cache_events_applied{status="parent_block_not_found"} 3\n' + 'dynamo_component_requests_total{status="ok"} 9\n' + ) + + class _Metrics(_FakeHttpResponse): + def read(self): + return metrics.encode() + + urls = [] + monkeypatch.setattr( + "urllib.request.urlopen", + lambda url, timeout: urls.append(url) or _Metrics(None), + ) + monkeypatch.setattr(runtime, "_stop_process", lambda *args, **kwargs: None) + + runtime.shutdown() + + assert urls == ["http://127.0.0.1:3000/metrics"] + assert ( + '[Dynamo] router applied KV events: {"ok": 42.0, "parent_block_not_found": 3.0}' + in capsys.readouterr().out + ) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py new file mode 100644 index 00000000000..f6ad9727175 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -0,0 +1,746 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NeMo RL's Dynamo engine adapter over a fake AsyncLLM. + +The host is a real ``VllmAsyncGenerationWorkerImpl`` built without its +constructor, so rollouts are sampled by the native worker's own helpers. +""" + +import asyncio +import base64 +import io +import json +from http import HTTPStatus +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from PIL import Image + +pytestmark = pytest.mark.vllm +pytest.importorskip("dynamo._core") +vllm = pytest.importorskip("vllm") + +from dynamo.llm.exceptions import HttpError, InvalidArgument # noqa: E402 +from vllm.exceptions import QueueOverflowError # noqa: E402 +from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 + +from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.dynamo.config import DynamoWorkerArgs # noqa: E402 +from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 + ROLLOUT_ANNOTATION, +) +from nemo_rl.models.generation.openai_server_utils import ( # noqa: E402 + derive_prefix_eos_count, + splice_prefix_tokens, +) +from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 + rollout_output, + rollout_result, + rollout_result_from_wire, + rollout_result_to_wire, +) +from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 + VllmAsyncGenerationWorkerImpl, +) + +MAX_MODEL_LEN = 16 + + +EOS = 2 +# The fake processor expands each image placeholder to IMAGE_TOKENS image tokens. +IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS = 90, 91, 3 + + +class _Context: + def id(self) -> str: + return "request-0" + + def is_stopped(self) -> bool: + return False + + +class _Tokenizer: + eos_token_id = EOS + + def decode(self, token_ids) -> str: + return "".join(f"<{token_id}>" for token_id in token_ids) + + +def _logprob(value: float): + return SimpleNamespace(logprob=value, rank=1, decoded_token=None) + + +def _completion(token_ids, finish_reason=None, index=0, routed_experts=None): + return SimpleNamespace( + index=index, + token_ids=token_ids, + logprobs=[{t: _logprob(-0.5)} for t in token_ids], + finish_reason=finish_reason, + routed_experts=routed_experts, + ) + + +def _output(prompt, *completions, prompt_routed_experts=None, prompt_logprobs=None): + return SimpleNamespace( + prompt_token_ids=prompt, + num_cached_tokens=0, + prompt_logprobs=prompt_logprobs, + prompt_routed_experts=prompt_routed_experts, + outputs=list(completions), + ) + + +class _FakeRenderer: + """vLLM's processing step: expands each image placeholder, as a VLM processor does.""" + + def __init__(self) -> None: + self.prompts = [] + + async def process_for_engine_async(self, prompt, arrival_time, *, skip_mm_cache): + self.prompts.append((prompt, skip_mm_cache)) + if "multi_modal_data" not in prompt: + return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} + tokens, placeholders = [], [] + for token in prompt["prompt_token_ids"]: + if token == IMAGE_PLACEHOLDER: + placeholders.append(PlaceholderRange(len(tokens), IMAGE_TOKENS)) + tokens.extend([IMAGE_TOKEN] * IMAGE_TOKENS) + else: + tokens.append(token) + return { + "type": "multimodal", + "prompt_token_ids": tokens, + "mm_kwargs": {"image": [object() for _ in placeholders]}, + "mm_placeholders": {"image": placeholders}, + } + + +class _FakeLLM: + """Yields scripted RequestOutputs and records what it was asked to generate.""" + + def __init__(self, outputs=(), *, defaults=None, admission_error=None): + self.outputs = list(outputs) + self.calls = [] + self.admission_error = admission_error + self.tokenizer = None + self.renderer = _FakeRenderer() + self.model_config = SimpleNamespace( + get_diff_sampling_param=lambda: dict(defaults or {}), + multimodal_config=None, + allowed_local_media_path="", + allowed_media_domains=None, + ) + self.vllm_config = SimpleNamespace( + model_config=SimpleNamespace(max_model_len=MAX_MODEL_LEN), + kv_events_config=None, + ) + + def check_admission(self, n, request_id): + if self.admission_error is not None: + raise self.admission_error + + async def generate(self, prompt, sampling_params, request_id): + self.calls.append((prompt, sampling_params, request_id)) + for output in self.outputs: + yield output + + +def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: + host = object.__new__(VllmAsyncGenerationWorkerImpl) + host.cfg = { + "max_new_tokens": 8, + "temperature": 0.7, + "top_p": 0.9, + "val_temperature": 0.0, + "val_top_p": 1.0, + "top_k": None, + "stop_token_ids": [2], + "stop_strings": [""], + "vllm_cfg": {"max_model_len": MAX_MODEL_LEN}, + **cfg, + } + host.SamplingParams = vllm.SamplingParams + host.model_name = "model" + host.routed_experts_dtype = torch.int16 + host.llm = llm + host._capture_media = False + return host + + +def _collect(engine, request) -> list[dict]: + async def run(): + return [chunk async for chunk in engine.generate(request, _Context())] + + return asyncio.run(run()) + + +def _rollout_request(prompt: list[int], **rollout) -> dict: + spec = {"greedy": False, "stop_strings": None, **rollout} + return {"token_ids": prompt, "annotations": [ROLLOUT_ANNOTATION + json.dumps(spec)]} + + +def _openai_request(**fields) -> dict: + return {"token_ids": [1, 2, 3, 4], **fields} + + +def test_rollouts_are_served_like_generate_async() -> None: + final = _output([1, 2, 3], _completion([10, 11, 12], "stop")) + llm = _FakeLLM([final]) + host = _host(llm) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=3) + + (chunk,) = _collect( + engine, _rollout_request([1, 2, 3], greedy=True, stop_strings=[""]) + ) + + ((_, sampling_params, _),) = llm.calls + # The request the native worker's generate_async builds... + assert sampling_params == host._rollout_sampling_params( + max_new_tokens=host._rollout_max_new_tokens(3), + greedy=True, + stop_strings=[""], + ) + assert sorted(sampling_params.stop) == ["", ""] + # ...and the result the shared pipeline builds from its final output. + expected = rollout_result( + final, + input_length=3, + return_routed_experts=False, + routed_experts_dtype=torch.int16, + device=torch.device("cpu"), + ) + assert chunk["token_ids"] == [10, 11, 12] + assert chunk["engine_data"] == { + "rollout": rollout_result_to_wire(expected), + "gen_leader_worker_idx": 3, + } + + +def test_router_replay_routes_reach_the_driver_as_generate_async_returns_them() -> None: + routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) + final = _output( + [1, 2, 3], + _completion([10, 11, 12], "stop", routed_experts=routes[3:]), + prompt_routed_experts=routes[:3], + ) + host = _host(_FakeLLM([final]), vllm_kwargs={"enable_return_routed_experts": True}) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect(engine, _rollout_request([1, 2, 3])) + + input_ids = torch.tensor([1, 2, 3]) + shipped = rollout_output( + rollout_result_from_wire(chunk["engine_data"]["rollout"]), + input_ids=input_ids, + input_length=3, + pad_token_id=0, + ) + native = rollout_output( + rollout_result( + final, + input_length=3, + return_routed_experts=True, + routed_experts_dtype=torch.int16, + device=torch.device("cpu"), + ), + input_ids=input_ids, + input_length=3, + pad_token_id=0, + ) + assert shipped.keys() == native.keys() + assert "routed_experts" in shipped + for key in native: + assert torch.equal(shipped[key], native[key]), key + + +def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect(engine, _rollout_request(list(range(MAX_MODEL_LEN)))) + + assert llm.calls == [] + rollout = chunk["engine_data"]["rollout"] + assert rollout["token_ids"] == [] and rollout["truncated"] is False + + +def test_openai_requests_map_onto_sampling_params_as_vllms_server_does() -> None: + params = native_engine.build_sampling_params( + _openai_request( + sampling_options={"top_k": 5, "seed": 7, "n": 2}, + stop_conditions={ + "max_tokens": 100, + "stop_token_ids": [9], + "stop_token_ids_hidden": [2], + }, + output_options={"logprobs": 2}, + extra_args={ + "sampling_options": { + "allowed_token_ids": [5, 6], + "bad_words_token_ids": [[7]], + } + }, + ), + # The model's generation config, for what the request leaves unset. + {"temperature": 0.6, "top_k": 20, "max_tokens": 9}, + output_room=MAX_MODEL_LEN - 4, + ) + + assert (params.temperature, params.top_k, params.seed, params.n) == (0.6, 5, 7, 2) + assert params.max_tokens == 9 + assert sorted(params.stop_token_ids) == [2, 9] + assert params.logprobs == 2 + assert params.allowed_token_ids == [5, 6] + assert params._bad_words_token_ids == [[7]] + assert params.detokenize is False + + +def test_guided_decoding_maps_onto_structured_outputs() -> None: + params = native_engine.build_sampling_params( + _openai_request( + sampling_options={"guided_decoding": {"json": {"type": "object"}}} + ), + {}, + output_room=MAX_MODEL_LEN - 4, + ) + + assert params.structured_outputs.json == {"type": "object"} + + +def test_best_of_is_rejected_as_by_vllm() -> None: + with pytest.raises(InvalidArgument, match="best_of"): + native_engine.build_sampling_params( + _openai_request(sampling_options={"n": 1, "best_of": 2}), + {}, + output_room=MAX_MODEL_LEN - 4, + ) + + +def test_openai_requests_stream_each_choice_by_index() -> None: + prompt = [1, 2, 3, 4] + llm = _FakeLLM( + [ + _output(prompt, _completion([5], index=0), _completion([7], index=1)), + _output(prompt, _completion([6], "stop", index=0)), + _output(prompt, _completion([8], "length", index=1)), + ] + ) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + chunks = _collect(engine, _openai_request(sampling_options={"n": 2})) + + assert [(c["index"], c["token_ids"]) for c in chunks] == [ + (0, [5]), + (1, [7]), + (0, [6]), + (1, [8]), + ] + assert [c.get("finish_reason") for c in chunks] == [None, None, "stop", "length"] + assert chunks[-1]["completion_usage"]["completion_tokens"] == 4 + + +def test_prompt_logprobs_are_returned_in_engine_data() -> None: + prompt = [1, 2, 3, 4] + llm = _FakeLLM( + [ + # Under DELTA, vLLM sends prompt logprobs once, at the end of prefill, + # and an empty list on every later output. + _output( + prompt, + _completion([5]), + prompt_logprobs=[None, {2: _logprob(-0.25)}, None, None], + ), + _output(prompt, _completion([6], "stop"), prompt_logprobs=[]), + ] + ) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + chunks = _collect(engine, _openai_request(output_options={"prompt_logprobs": 0})) + + assert chunks[-1]["engine_data"]["prompt_logprobs"][1]["2"]["logprob"] == -0.25 + + +def test_queue_limits_reject_with_the_engines_status() -> None: + llm = _FakeLLM(admission_error=QueueOverflowError()) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(HttpError) as raised: + _collect(engine, _openai_request()) + + assert raised.value.code == HTTPStatus.SERVICE_UNAVAILABLE + assert llm.calls == [] + + +def _start_worker_config(monkeypatch, host) -> SimpleNamespace: + """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" + configs = [] + + class _Worker: + def __init__(self, engine, config): + configs.append(config) + + async def run(self): + return None + + monkeypatch.setattr(native_engine, "Worker", _Worker) + monkeypatch.setattr(native_engine, "WorkerConfig", SimpleNamespace) + env = { + "DYN_DISCOVERY_BACKEND": "etcd", + "DYN_REQUEST_PLANE": "tcp", + "DYN_EVENT_PLANE": "nats", + } + for key, value in env.items(): + # Registered here so the variables the engine exports are restored. + monkeypatch.setenv(key, value) + + async def start(): + await native_engine.start_dynamo_endpoint( + host, namespace="ns", engine_env=env, leader_worker_idx=0 + ) + + asyncio.run(start()) + (config,) = configs + return config + + +def test_every_worker_arg_reaches_the_dynamo_worker_config( + monkeypatch, tmp_path +) -> None: + template = tmp_path / "chat.jinja" + template.write_text("{{ messages }}") + worker_args = { + "tool_call_parser": "qwen3_coder", + "reasoning_parser": "nemotron_nano", + "exclude_tools_when_tool_choice_none": True, + "enable_structural_tag": True, + "structural_tag_scope": "always", + "structural_tag_schema": "strict", + "custom_jinja_template": str(template), + "endpoint_types": ["chat"], + } + assert set(worker_args) == set(DynamoWorkerArgs.model_fields) + host = _host(_FakeLLM(), dynamo_cfg={"worker_args": worker_args}) + + config = _start_worker_config(monkeypatch, host) + + assert config.namespace == "ns" + assert config.endpoint_types == "chat" + assert config.structural_tag_mode == "on" + with open(config.custom_jinja_template) as template_file: + assert template_file.read() == "{{ messages }}" + for key in worker_args.keys() - { + "endpoint_types", + "enable_structural_tag", + "custom_jinja_template", + }: + assert getattr(config, key) == worker_args[key], key + + +def test_the_native_servers_chat_settings_reach_the_frontend(monkeypatch) -> None: + host = _host(_FakeLLM()) + host.cfg["vllm_cfg"] = { + "max_model_len": MAX_MODEL_LEN, + "reasoning_parser_plugin": "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", + "http_server_serving_chat_kwargs": { + "tool_parser": "qwen3_coder", + "reasoning_parser": "nano_v3", + # Literal Jinja, which the native server also accepts. + "chat_template": "{{ messages }}\n", + }, + } + + config = _start_worker_config(monkeypatch, host) + + assert (config.tool_call_parser, config.reasoning_parser) == ( + "qwen3_coder", + "nemotron_v3", + ) + with open(config.custom_jinja_template) as template_file: + assert template_file.read() == "{{ messages }}\n" + + +def test_a_parser_dynamo_lacks_fails_registration(monkeypatch) -> None: + host = _host( + _FakeLLM(), dynamo_cfg={"worker_args": {"tool_call_parser": "no_such_parser"}} + ) + + with pytest.raises(ValueError, match="no tool call parser named 'no_such_parser'"): + _start_worker_config(monkeypatch, host) + + +def test_an_endpoint_that_stops_serving_reports_why() -> None: + async def endpoint_errors() -> list: + async def registration_fails() -> None: + raise RuntimeError("etcd unreachable") + + host = SimpleNamespace(_dynamo_endpoint_task=None) + errors = [await VllmAsyncGenerationWorkerImpl.dynamo_endpoint_error(host)] + host._dynamo_endpoint_task = asyncio.get_running_loop().create_task( + registration_fails() + ) + await asyncio.sleep(0) + errors.append(await VllmAsyncGenerationWorkerImpl.dynamo_endpoint_error(host)) + return errors + + # Not registered yet, then the driver's registration wait gets the cause. + assert asyncio.run(endpoint_errors()) == [ + None, + "RuntimeError('etcd unreachable')", + ] + + +def test_a_prompt_that_fills_the_context_is_refused_as_natively() -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match="maximum context length"): + _collect(engine, _openai_request(token_ids=list(range(MAX_MODEL_LEN)))) + + assert llm.calls == [] + + +def _image_url() -> str: + buffer = io.BytesIO() + Image.new("RGB", (8, 8), color="red").save(buffer, format="PNG") + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + + +def test_media_is_loaded_and_processed_as_vllms_chat_parser_does() -> None: + final = _output([1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3], _completion([5], "stop")) + llm = _FakeLLM([final]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + _collect( + engine, + _openai_request( + token_ids=[1, IMAGE_PLACEHOLDER, 3], + multi_modal_data={"image_url": [{"Url": _image_url()}]}, + mm_processor_kwargs={"max_num_tiles": 2}, + ), + ) + + ((prompt, skip_mm_cache),) = llm.renderer.prompts + # The frontend's unexpanded tokens, the decoded image and the processor kwargs. + assert prompt["prompt_token_ids"] == [1, IMAGE_PLACEHOLDER, 3] + (image,) = prompt["multi_modal_data"]["image"] + assert image.size == (8, 8) and image.mode == "RGB" + assert prompt["multi_modal_uuids"] == {"image": [None]} + assert prompt["mm_processor_kwargs"] == {"max_num_tiles": 2} + assert skip_mm_cache is False + # vLLM generates from the processed prompt, and the output cap counts its tokens. + ((engine_input, params, _),) = llm.calls + assert engine_input["prompt_token_ids"] == [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + assert params.max_tokens == MAX_MODEL_LEN - 2 - IMAGE_TOKENS + + +@pytest.mark.parametrize( + ("fields", "error"), + [ + ({"prompt_embeds": "AAAA"}, "Embedding prompts"), + ({"multi_modal_data": {"image_url": [{"UuidOnly": "u"}]}}, "carry their URL"), + ( + { + "multi_modal_data": { + "image_url": [{"Url": "data:image/png;base64,eA=="}] + } + }, + "Cannot load image_url", + ), + ], +) +def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match=error): + _collect(engine, _openai_request(**fields)) + + assert llm.calls == [] + + +# --- NeMo-Gym's Dynamo dialect ------------------------------------------------- + + +def _f32(value: float) -> float: + return float(np.float32(value)) + + +def _gym_request( + *, temperature=0.7, top_p=0.9, prefix=None, sampling=None, **fields +) -> dict: + annotations = [native_engine.GYM_ROLLOUT_ANNOTATION] + if prefix is not None: + annotations.insert( + 0, native_engine.GYM_PREFIX_ANNOTATION_PREFIX + json.dumps(prefix) + ) + sampling_options = {"temperature": temperature, "top_p": top_p, **(sampling or {})} + return { + "token_ids": [1, 3, 4], + # The frontend carries sampling floats as f32. + "sampling_options": { + key: _f32(value) if isinstance(value, float) else value + for key, value in sampling_options.items() + }, + # Gym's token information: logprobs=true, top_logprobs=0. + "output_options": {"logprobs": 0}, + "annotations": annotations, + **fields, + } + + +def test_gym_calls_sample_with_nemo_rls_exact_profile() -> None: + final = _output([1, 3, 4], _completion([10, 11], "stop")) + llm = _FakeLLM([final]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect( + engine, _gym_request(stop_conditions={"max_tokens": 100, "stop": [""]}) + ) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == [1, 3, 4] + # The configured values, not the f32 values the frontend delivered. + assert (params.temperature, params.top_p) == (0.7, 0.9) + assert params.top_k == vllm.SamplingParams(top_k=-1).top_k + assert params.output_kind == vllm.sampling_params.RequestOutputKind.FINAL_ONLY + # Output fits the context, and vLLM stops on the stop strings itself. + assert params.max_tokens == MAX_MODEL_LEN - 3 + assert (params.stop, params.detokenize) == ([""], True) + assert chunk["engine_data"] == { + "prompt_token_ids": [1, 3, 4], + "completion_token_ids": [10, 11], + "completion_logprobs": [-0.5, -0.5], + } + assert (chunk["token_ids"], chunk["finish_reason"]) == ([10, 11], "stop") + assert chunk["log_probs"] == [-0.5, -0.5] + + +def test_gym_validation_calls_sample_with_the_validation_profile() -> None: + llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + _collect(engine, _gym_request(temperature=0.0, top_p=1.0)) + + ((_, params, _),) = llm.calls + assert (params.temperature, params.top_p) == (0.0, 1.0) + + +@pytest.mark.parametrize( + ("request_fields", "error"), + [ + ({"temperature": 0.5}, "matches neither"), + ({"top_p": None}, "top_p must be set explicitly"), + ({"sampling": {"top_k": 5}}, "Top k sampling parameter"), + ({"sampling": {"n": 2}}, "take n=1"), + ], +) +def test_gym_calls_that_would_sample_off_policy_are_refused( + request_fields, error +) -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match=error): + _collect(engine, _gym_request(**request_fields)) + + assert llm.calls == [] + + +def test_gym_echoed_prefix_replaces_the_rendered_history() -> None: + # The model generated [5, 6, EOS]; the frontend re-rendered that turn as [7, EOS]. + prefix, rendered = [1, 5, 6, EOS], [1, 7, EOS, 3, 4] + spliced = splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=prefix, + template_token_ids=rendered, + template_prefix_eos_count=derive_prefix_eos_count(prefix, eos_token_id=EOS), + eos_token_id=EOS, + ).token_ids + assert spliced != rendered + llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect(engine, _gym_request(prefix=prefix, token_ids=rendered)) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == spliced + assert params.max_tokens == MAX_MODEL_LEN - len(spliced) + assert chunk["engine_data"]["prompt_token_ids"] == spliced + + +def test_gym_calls_whose_splice_fills_the_context_are_refused() -> None: + prefix = [1, *range(20, 20 + MAX_MODEL_LEN), EOS] + llm = _FakeLLM() + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match="maximum context length"): + _collect(engine, _gym_request(prefix=prefix, token_ids=[1, 7, EOS, 3])) + + assert llm.calls == [] + + +def test_gym_echoed_prefix_keeps_media_placeholders_on_their_tokens() -> None: + # Turn 1 ran [1, , 3] and generated [5, 6, EOS]; the frontend + # re-rendered that turn as [7, EOS] and appended a turn with a second image. + turn1 = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + prefix = [*turn1, 5, 6, EOS] + rendered = [1, IMAGE_PLACEHOLDER, 3, 7, EOS, IMAGE_PLACEHOLDER, 4] + spliced = [*prefix, *[IMAGE_TOKEN] * IMAGE_TOKENS, 4] + llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect( + engine, + _gym_request( + prefix=prefix, + token_ids=rendered, + multi_modal_data={"image_url": [{"Url": _image_url()}] * 2}, + ), + ) + + ((engine_input, params, _),) = llm.calls + assert engine_input["prompt_token_ids"] == spliced + # The first image sits in the model's own prefix; the second moved with the + # longer generated turn, as the native endpoint remaps them. + assert [ + (span.offset, span.length) for span in engine_input["mm_placeholders"]["image"] + ] == [(1, IMAGE_TOKENS), (len(prefix), IMAGE_TOKENS)] + assert params.max_tokens == MAX_MODEL_LEN - len(spliced) + assert chunk["engine_data"]["prompt_token_ids"] == spliced + + +@pytest.mark.parametrize( + ("router_mode", "prefix_caching", "publishes"), + [("kv", True, True), ("kv", False, False), ("round-robin", True, False)], +) +def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( + router_mode, prefix_caching, publishes +) -> None: + cfg = native_engine.DynamoCfg.model_validate( + {"frontend_args": {"router_mode": router_mode}} + ) + + config = native_engine.dynamo_kv_events_config(cfg, prefix_caching=prefix_caching) + + # vLLM publishes block events only for blocks its prefix cache stores. + assert (config is not None) is publishes + if publishes: + assert config["enable_kv_cache_events"] and config["publisher"] == "zmq" + # vLLM's publisher binds only a wildcard endpoint; Dynamo's source connects. + assert config["endpoint"].startswith("tcp://*:") diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py new file mode 100644 index 00000000000..bb1a00de798 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -0,0 +1,232 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NeMo-Gym token capture through the Dynamo engine adapter. + +The worker's own capture lifecycle stages each call, as on the native chat +endpoint; only Gym's capture core is faked. +""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +pytestmark = [pytest.mark.nemo_gym, pytest.mark.vllm] +pytest.importorskip("dynamo._core") +pytest.importorskip("vllm") + +from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 +from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 + DYNAMO_CAPTURE_ANNOTATION_PREFIX, + DYNAMO_GYM_REQUEST_ANNOTATION, + DYNAMO_PREFIX_ANNOTATION_PREFIX, +) +from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 + +from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 +from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 +from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 +from nemo_rl.utils.routed_experts_codec import decode_routed_experts # noqa: E402 +from tests.unit.models.generation.test_dynamo_native_engine import ( # noqa: E402 + EOS, + IMAGE_PLACEHOLDER, + IMAGE_TOKEN, + IMAGE_TOKENS, + _image_url, + _collect, + _completion, + _FakeLLM, + _gym_request, + _host, + _output, + _Tokenizer, +) + +COORDS = {"staging_key": "rollout-1/c2"} + + +class _Capture: + """Gym's capture core: records begin/complete/fail.""" + + adapter = VLLMCaptureAdapter() + + def __init__(self) -> None: + self.begun: list[tuple] = [] + self.staged: list[dict] = [] + self.failed: list[str] = [] + + def begin_call(self, admission, *, prefix_token_ids, stream): + self.begun.append((admission, prefix_token_ids, stream)) + return SimpleNamespace(admission=admission) + + def complete_call_from_response(self, call, payload, attachments=None): + self.staged.append(payload) + return SimpleNamespace(model_dump=lambda: dict(COORDS)) + + def fail_call(self, call, *, reason) -> None: + self.failed.append(reason) + + +def _capturing_host(llm, **cfg): + host = _host(llm, **cfg) + host.token_capture = _Capture() + host._capture_calls = {} + host._chain_prefix = ChainPrefixCache() + return host + + +def _captured_request(admission: CaptureAdmission, **fields) -> dict: + request = _gym_request(**fields) + # Gym's dynamo_worker handler: no logprobs, the admission in an annotation. + request["output_options"] = {} + request["annotations"] = [ + DYNAMO_CAPTURE_ANNOTATION_PREFIX + admission.model_dump_json(), + *request["annotations"], + ] + return request + + +def test_annotations_are_the_ones_gyms_dynamo_dialect_sends() -> None: + assert native_engine.GYM_ROLLOUT_ANNOTATION == DYNAMO_GYM_REQUEST_ANNOTATION + assert native_engine.GYM_CAPTURE_ANNOTATION_PREFIX == ( + DYNAMO_CAPTURE_ANNOTATION_PREFIX + ) + assert native_engine.GYM_PREFIX_ANNOTATION_PREFIX == ( + DYNAMO_PREFIX_ANNOTATION_PREFIX + ) + + +def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: + # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. + prefix = [1, 5, 6, EOS] + admission = CaptureAdmission( + rollout_id="rollout-1", + model_call_id="c2", + parent_call_id="c1", + prev_len=len(prefix), + mode="token_in", + required_prefix_token_ids=prefix, + parent_chain_hash="0" * 64, + ) + spliced = [*prefix, 3, 4] + routes = np.arange(8 * 2 * 2, dtype=np.int16).reshape(8, 2, 2) + final = _output( + spliced, + _completion([10, 11], "stop", routed_experts=routes[6:]), + prompt_routed_experts=routes[:6], + ) + llm = _FakeLLM([final]) + llm.tokenizer = _Tokenizer() + host = _capturing_host(llm, vllm_kwargs={"enable_return_routed_experts": True}) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect( + engine, _captured_request(admission, token_ids=[1, 7, EOS, 3, 4]) + ) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == spliced + # The sampled tokens' log probabilities are staged; Gym asked for none. + assert params.logprobs == 0 + ((begun_admission, begun_prefix, stream),) = host.token_capture.begun + assert (begun_admission, begun_prefix, stream) == (admission, prefix, False) + (payload,) = host.token_capture.staged + assert payload["prompt_token_ids"] == spliced + message = payload["choices"][0]["message"] + assert message["generation_token_ids"] == [10, 11] + assert message["generation_log_probs"] == [-0.5, -0.5] + # As the native endpoint stages them: the full sequence's routes (none for + # the last token) from the boundary row, the prefix's last token, on. + staged_routes = decode_routed_experts(message["routed_experts"], torch.int16) + expected = torch.full((5, 2, 2), R3_MISSING_ROUTE_SENTINEL, dtype=torch.int16) + expected[:4] = torch.from_numpy(routes[3:7]) + assert torch.equal(staged_routes, expected) + # Only the commit coordinates go back through the frontend. + assert chunk["engine_data"] == {"ng_commit_coords": COORDS} + assert "log_probs" not in chunk + assert host._capture_calls == {} + + +def test_a_call_that_fails_after_admission_fails_its_capture() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + + class _FailingLLM(_FakeLLM): + async def generate(self, prompt, sampling_params, request_id): + raise RuntimeError("engine died") + yield # pragma: no cover + + host = _capturing_host(_FailingLLM()) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + with pytest.raises(RuntimeError, match="engine died"): + _collect(engine, _captured_request(admission)) + + assert host.token_capture.failed == ["engine_error"] + assert host._capture_calls == {} + + +def test_without_capture_installed_a_captured_call_is_served_uncaptured() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) + host = _host(llm) + host.token_capture = None + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect(engine, _captured_request(admission)) + + assert json.loads(json.dumps(chunk["engine_data"]))["completion_token_ids"] == [10] + + +def test_a_captured_media_call_stages_the_media_the_engine_ran_on() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + processed = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + llm = _FakeLLM([_output(processed, _completion([10], "stop"))]) + host = _capturing_host(llm) + host._capture_media = True + captured = CapturedMedia((), None) + snapshots = [] + + def capture_request_media(engine_input, *, admission, splice): + snapshots.append((list(engine_input["prompt_token_ids"]), admission, splice)) + return captured + + host._capture_request_media = capture_request_media + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect( + engine, + _captured_request( + admission, + token_ids=[1, IMAGE_PLACEHOLDER, 3], + multi_modal_data={"image_url": [{"Url": _image_url()}]}, + ), + ) + + # The processor's own pixels, not cache references, as natively (#4124). + ((_, skip_mm_cache),) = llm.renderer.prompts + assert skip_mm_cache is True + # Media is snapshotted from the processed prompt the engine runs. + assert snapshots == [(processed, admission, None)] + (payload,) = host.token_capture.staged + assert payload["prompt_token_ids"] == processed + assert chunk["engine_data"] == {"ng_commit_coords": COORDS} diff --git a/tests/unit/models/generation/test_dynamo_rollout_client.py b/tests/unit/models/generation/test_dynamo_rollout_client.py new file mode 100644 index 00000000000..ac072ad72ad --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_rollout_client.py @@ -0,0 +1,67 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +import pytest + +from nemo_rl.models.generation.dynamo.rollout_client import ( + ROLLOUT_ANNOTATION, + parse_rollout_response, + rollout_payload, +) + + +def test_rollout_payload_leaves_sampling_and_stopping_to_the_engine() -> None: + payload = rollout_payload( + model_name="model", + prompt_token_ids=[5, 6, 7], + max_model_len=10, + greedy=True, + stop_strings=[""], + ) + + assert payload["prompt"] == [5, 6, 7] + # The frontend's own stop rules must never end the stream first. + assert payload["ignore_eos"] is True + assert payload["max_tokens"] == 7 + assert "stop" not in payload and "temperature" not in payload + (annotation,) = payload["nvext"]["annotations"] + assert annotation.startswith(ROLLOUT_ANNOTATION) + assert json.loads(annotation[len(ROLLOUT_ANNOTATION) :]) == { + "greedy": True, + "stop_strings": [""], + } + assert payload["nvext"]["extra_fields"] == ["engine_data"] + + +@pytest.mark.parametrize( + "response, match", + [ + ({"status": "error", "http_status": 400, "raw": "bad"}, "HTTP 400"), + ({"nvext": {}}, "no rollout engine_data"), + ({"nvext": {"engine_data": {"prompt_logprobs": []}}}, "no rollout engine_data"), + ], +) +def test_parse_rollout_response_rejects_unusable_responses(response, match) -> None: + with pytest.raises(RuntimeError, match=match): + parse_rollout_response(response, url="http://frontend/v1/completions") + + +def test_parse_rollout_response_returns_engine_data() -> None: + engine_data = {"rollout": {"token_ids": []}, "gen_leader_worker_idx": 1} + + assert ( + parse_rollout_response({"nvext": {"engine_data": engine_data}}, url="u") + == engine_data + ) diff --git a/tests/unit/models/generation/test_dynamo_token_wrapper.py b/tests/unit/models/generation/test_dynamo_token_wrapper.py deleted file mode 100644 index 53eb020bc2b..00000000000 --- a/tests/unit/models/generation/test_dynamo_token_wrapper.py +++ /dev/null @@ -1,646 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import json - -import pytest -from transformers import AutoTokenizer - -from nemo_rl.models.generation.dynamo.token_wrapper import ( - DynamoTokenWrapperServer, - _inject_gym_token_metadata, - _validate_engine_data, - prepare_dynamo_chat_completion_request, -) - - -class _Tokenizer: - eos_token_id = 2 - eos_token = "" - - def __init__(self) -> None: - self.calls = [] - - def decode(self, token_ids): - return repr(token_ids) - - def encode(self, text, add_special_tokens=False): - assert add_special_tokens is False - token_ids = [] - while text: - if text.startswith(self.eos_token): - token_ids.append(self.eos_token_id) - text = text[len(self.eos_token) :] - elif text.startswith("next"): - token_ids.append(40) - text = text[len("next") :] - elif text.startswith("GEN"): - token_ids.append(99) - text = text[len("GEN") :] - else: - raise AssertionError(f"Unexpected text to encode: {text!r}") - return token_ids - - def apply_chat_template( - self, - conversation, - tools=None, - documents=None, - chat_template=None, - add_generation_prompt=False, - continue_final_message=False, - tokenize=True, - return_tensors=None, - return_dict=False, - **kwargs, - ): - self.calls.append( - { - "tools": tools, - "documents": documents, - "chat_template": chat_template, - "add_generation_prompt": add_generation_prompt, - "continue_final_message": continue_final_message, - "tokenize": tokenize, - "return_tensors": return_tensors, - "return_dict": return_dict, - "kwargs": kwargs, - } - ) - token_ids = [] - rendered = "" - for index, message in enumerate(conversation): - role = message["role"] - content = message.get("content") - for tool_call in message.get("tool_calls", []): - function = tool_call.get("function", tool_call) - arguments = function.get("arguments") - if arguments is not None and not isinstance(arguments, dict): - raise TypeError("Can only get item pairs from a mapping.") - if role == "user" and content == "hello": - token_ids.extend([10]) - rendered += "hello" - elif role == "assistant" and content == "first": - # Model the Nemotron template's context-dependent history - # truncation: the assistant is longer when rendered alone than - # when a later user turn is present. - has_later_user = any( - item["role"] == "user" for item in conversation[index + 1 :] - ) - token_ids.extend( - [3, self.eos_token_id] - if has_later_user - else [300, 301, 302, self.eos_token_id] - ) - rendered += f"first{self.eos_token}" - elif role == "user" and content == "next": - token_ids.extend([40]) - rendered += "next" - elif role == "assistant" and isinstance(content, str): - token_ids.extend([777, self.eos_token_id]) - rendered += f"{content}{self.eos_token}" - else: - token_ids.extend([900]) - rendered += "other" - if add_generation_prompt: - token_ids.extend([99]) - rendered += "GEN" - return token_ids if tokenize else rendered - - -TOOLS = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather for a city.", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - }, - } -] - - -def _tool_conversation() -> list[dict]: - return [ - {"role": "user", "content": "What is the weather in San Francisco?"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"city":"San Francisco"}', - }, - } - ], - }, - {"role": "tool", "content": "Sunny, 20C"}, - {"role": "user", "content": "What about tomorrow?"}, - ] - - -def test_prepare_dynamo_chat_completion_request_first_turn() -> None: - tokenizer = _Tokenizer() - body = { - "model": "dummy-model", - "messages": [{"role": "user", "content": "hello"}], - "nvext": {"extra_fields": ["timing"], "trace": "keep-me"}, - "chat_template_kwargs": { - "enable_thinking": False, - "force_nonempty_content": True, - }, - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs={"enable_thinking": True}, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["messages"] == [{"role": "user", "content": "hello"}] - assert "logprobs" not in prepared - assert "return_tokens_as_token_ids" not in prepared - assert prepared["chat_template_kwargs"] == { - "enable_thinking": False, - "force_nonempty_content": True, - } - assert prepared["nvext"] == { - "extra_fields": ["timing", "engine_data"], - "trace": "keep-me", - "token_data": [10, 99], - } - assert tokenizer.calls[0]["kwargs"] == { - "enable_thinking": False, - "force_nonempty_content": True, - } - - -def test_public_qwen3_tokenizer_first_turn_matches_chat_template( - tiny_qwen3_model_path, -) -> None: - tokenizer = AutoTokenizer.from_pretrained(tiny_qwen3_model_path) - messages = [{"role": "user", "content": "Solve 2 + 2."}] - template_kwargs = {"enable_thinking": False} - expected = tokenizer.apply_chat_template( - messages, - tools=None, - documents=None, - chat_template=None, - add_generation_prompt=True, - continue_final_message=False, - tokenize=True, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - - prepared = prepare_dynamo_chat_completion_request( - {"model": "Qwen/Qwen3-0.6B", "messages": messages}, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs=template_kwargs, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == expected - - -def test_public_qwen3_tool_conversation_matches_plain_chat_template( - tiny_qwen3_model_path, -) -> None: - tokenizer = AutoTokenizer.from_pretrained(tiny_qwen3_model_path) - messages = _tool_conversation() - template_kwargs = {"enable_thinking": False} - expected = tokenizer.apply_chat_template( - messages, - tools=TOOLS, - tokenize=True, - add_generation_prompt=True, - continue_final_message=False, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - - prepared = prepare_dynamo_chat_completion_request( - {"model": "Qwen/Qwen3-0.6B", "messages": messages, "tools": TOOLS}, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs=template_kwargs, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == expected - - -def test_public_qwen3_multiturn_prefix_splice_parity( - tiny_qwen3_model_path, -) -> None: - tokenizer = AutoTokenizer.from_pretrained(tiny_qwen3_model_path) - messages = _tool_conversation() - template_kwargs = {"enable_thinking": False} - expected = tokenizer.apply_chat_template( - messages, - tools=TOOLS, - tokenize=True, - add_generation_prompt=True, - continue_final_message=False, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - assistant_prefix = tokenizer.apply_chat_template( - messages[:2], - tools=TOOLS, - tokenize=True, - add_generation_prompt=False, - continue_final_message=False, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - first_prompt = tokenizer.apply_chat_template( - messages[:1], - tools=TOOLS, - tokenize=True, - add_generation_prompt=True, - continue_final_message=False, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - generation_token_ids = assistant_prefix[len(first_prompt) : -1] - messages[1]["prompt_token_ids"] = first_prompt - messages[1]["generation_token_ids"] = generation_token_ids - messages[1]["generation_log_probs"] = [-0.1] * len(generation_token_ids) - - assistant_eos_index = [ - index - for index, token_id in enumerate(expected) - if token_id == tokenizer.eos_token_id - ][2] - expected_splice = assistant_prefix[:-2] + expected[assistant_eos_index:] - - prepared = prepare_dynamo_chat_completion_request( - {"model": "Qwen/Qwen3-0.6B", "messages": messages, "tools": TOOLS}, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs=template_kwargs, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == expected_splice - - -def test_prepare_dynamo_chat_completion_request_preserves_logprob_fields() -> None: - tokenizer = _Tokenizer() - body = { - "model": "dummy-model", - "messages": [{"role": "user", "content": "hello"}], - "logprobs": True, - "return_tokens_as_token_ids": True, - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["logprobs"] is True - assert prepared["return_tokens_as_token_ids"] is True - assert prepared["nvext"] == { - "extra_fields": ["engine_data"], - "token_data": [10, 99], - } - - -def test_prepare_dynamo_chat_completion_request_preserves_prior_prefix() -> None: - tokenizer = _Tokenizer() - body = { - "model": "dummy-model", - "required_prefix_token_ids": [999], - "messages": [ - {"role": "user", "content": "hello"}, - { - "role": "assistant", - "content": "first", - "prompt_token_ids": [10], - "generation_token_ids": [31, 32, 2], - "generation_log_probs": [-0.1, -0.2, -0.3], - }, - {"role": "user", "content": "next"}, - ], - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == [10, 31, 32, 2, 40, 99] - assert "required_prefix_token_ids" not in prepared - assert "prompt_token_ids" not in prepared["messages"][1] - assert "generation_token_ids" not in prepared["messages"][1] - assert "generation_log_probs" not in prepared["messages"][1] - assert tokenizer.calls[0]["add_generation_prompt"] is True - assert tokenizer.calls[1]["add_generation_prompt"] is False - assert tokenizer.calls[0]["tokenize"] is True - assert tokenizer.calls[1]["tokenize"] is True - - -def test_prepare_dynamo_chat_completion_request_validates_extra_fields() -> None: - body = { - "messages": [{"role": "user", "content": "hello"}], - "nvext": {"extra_fields": ["engine_data", "timing", "engine_data"]}, - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=_Tokenizer(), - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["extra_fields"] == ["engine_data", "timing"] - - body["nvext"]["extra_fields"] = "timing" - with pytest.raises(ValueError, match="extra_fields must be a JSON list"): - prepare_dynamo_chat_completion_request( - body, - tokenizer=_Tokenizer(), - exclude_tools_when_tool_choice_none=True, - ) - - -def test_tool_choice_none_honors_worker_exclusion_setting() -> None: - body = { - "messages": [{"role": "user", "content": "hello"}], - "tools": TOOLS, - "tool_choice": "none", - } - excluded_tokenizer = _Tokenizer() - included_tokenizer = _Tokenizer() - - prepare_dynamo_chat_completion_request( - body, - tokenizer=excluded_tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - prepare_dynamo_chat_completion_request( - body, - tokenizer=included_tokenizer, - exclude_tools_when_tool_choice_none=False, - ) - - assert excluded_tokenizer.calls[0]["tools"] is None - assert included_tokenizer.calls[0]["tools"] == TOOLS - - -def test_prepare_dynamo_chat_completion_request_normalizes_prior_tool_arguments() -> ( - None -): - tokenizer = _Tokenizer() - body = { - "model": "dummy-model", - "messages": [ - {"role": "user", "content": "hello"}, - { - "role": "assistant", - "content": "older tool call", - "tool_calls": [ - { - "type": "function", - "function": { - "name": "shell", - "arguments": '{"command":"pwd"}', - }, - }, - { - "type": "function", - "function": { - "name": "malformed", - "arguments": "not-json", - }, - }, - ], - }, - {"role": "user", "content": "next"}, - { - "role": "assistant", - "content": "first", - "prompt_token_ids": [10, 777, 2, 40], - "generation_token_ids": [31, 32, 2], - "generation_log_probs": [-0.1, -0.2, -0.3], - }, - {"role": "user", "content": "next"}, - ], - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == [10, 777, 2, 40, 31, 32, 2, 40, 99] - assert prepared["messages"][1]["tool_calls"][0]["function"]["arguments"] == ( - '{"command":"pwd"}' - ) - assert prepared["messages"][1]["tool_calls"][1]["function"]["arguments"] == ( - "not-json" - ) - - -def test_prepare_dynamo_chat_completion_request_normalizes_tools_without_prefix() -> ( - None -): - tokenizer = _Tokenizer() - body = { - "messages": [ - {"role": "user", "content": "hello"}, - { - "role": "assistant", - "content": "older tool call", - "tool_calls": [ - { - "type": "function", - "function": { - "name": "shell", - "arguments": '{"command":"pwd"}', - }, - } - ], - }, - {"role": "user", "content": "next"}, - ] - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == [10, 777, 2, 40, 99] - assert prepared["messages"][1]["tool_calls"][0]["function"]["arguments"] == ( - '{"command":"pwd"}' - ) - assert len(tokenizer.calls) == 2 - - -def test_prepare_dynamo_chat_completion_request_rejects_stream() -> None: - with pytest.raises(ValueError, match="stream=True"): - prepare_dynamo_chat_completion_request( - {"messages": [{"role": "user", "content": "hello"}], "stream": True}, - tokenizer=_Tokenizer(), - exclude_tools_when_tool_choice_none=True, - ) - - -def test_prepare_dynamo_chat_completion_request_rejects_multiple_choices() -> None: - with pytest.raises(ValueError, match="only n=1"): - prepare_dynamo_chat_completion_request( - {"messages": [{"role": "user", "content": "hello"}], "n": 2}, - tokenizer=_Tokenizer(), - exclude_tools_when_tool_choice_none=True, - ) - - -def test_validate_engine_data_requires_prompt_completion_and_logprobs() -> None: - _validate_engine_data( - { - "nvext": { - "engine_data": { - "prompt_token_ids": [1, 2], - "completion_token_ids": [3], - "completion_logprobs": [-0.25], - } - } - } - ) - - with pytest.raises(ValueError, match="engine_data"): - _validate_engine_data({"nvext": {}}) - with pytest.raises(ValueError, match="prompt_token_ids"): - _validate_engine_data( - { - "nvext": { - "engine_data": { - "completion_token_ids": [], - "completion_logprobs": [], - } - } - } - ) - with pytest.raises(ValueError, match="completion_token_ids"): - _validate_engine_data( - { - "nvext": { - "engine_data": { - "prompt_token_ids": [], - "completion_logprobs": [], - } - } - } - ) - with pytest.raises(ValueError, match="completion_logprobs"): - _validate_engine_data( - { - "nvext": { - "engine_data": { - "prompt_token_ids": [], - "completion_token_ids": [], - } - } - } - ) - - -def test_inject_gym_token_metadata_validates_and_populates_message() -> None: - response = { - "choices": [ - { - "message": {"role": "assistant", "content": "answer"}, - "logprobs": None, - } - ], - "nvext": { - "engine_data": { - "prompt_token_ids": [1, 2, 3], - "completion_token_ids": [4, 5], - "completion_logprobs": [-0.25, -0.5], - } - }, - } - - _inject_gym_token_metadata(response) - - assert response["choices"][0]["message"] == { - "role": "assistant", - "content": "answer", - "prompt_token_ids": [1, 2, 3], - "generation_token_ids": [4, 5], - "generation_log_probs": [-0.25, -0.5], - } - - response["nvext"]["engine_data"]["completion_logprobs"] = [-0.25] - with pytest.raises(ValueError, match="1 generation log probabilities for 2"): - _inject_gym_token_metadata(response) - - -def test_forward_chat_completion_reuses_loop_bound_session() -> None: - class FakeResponse: - status = 200 - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - return None - - async def text(self): - return json.dumps({"choices": []}) - - class FakeSession: - def __init__(self): - self.calls = [] - - def post(self, url, *, json, headers): - self.calls.append((url, json, headers)) - return FakeResponse() - - server = DynamoTokenWrapperServer( - dynamo_frontend_base_url="http://dynamo/v1", - tokenizer=_Tokenizer(), - tokenizer_chat_template_kwargs=None, - exclude_tools_when_tool_choice_none=True, - request_timeout_s=30, - ) - session = FakeSession() - server._client_session = session - - async def forward_twice(): - await server._forward_chat_completion({}, authorization=None) - await server._forward_chat_completion({}, authorization="Bearer token") - - asyncio.run(forward_twice()) - - assert len(session.calls) == 2 - assert session.calls[1][2]["Authorization"] == "Bearer token" diff --git a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py new file mode 100644 index 00000000000..f729715eede --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Dynamo stays out of NeMo RL's vLLM engines. + +The engines run NeMo RL's own vLLM and use Dynamo only through its +engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only where +it also breaks native vLLM. These guards fail if NeMo RL starts importing +Dynamo's vLLM integration (or any other Dynamo module), or if the SDK itself +starts importing vLLM. +""" + +import ast +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +NEMO_RL_ROOT = Path(__file__).parents[4] / "nemo_rl" + +# Every Dynamo module NeMo RL may import. A new entry needs a reason it does not +# reach Dynamo-owned vLLM code. +DYNAMO_SDK_MODULES = { + "dynamo._core", + "dynamo.common.backend", + "dynamo.common.backend.logprobs", + "dynamo.common.backend.publisher", + "dynamo.llm.exceptions", +} + + +def _dynamo_imports() -> dict[str, set[str]]: + """The Dynamo modules each nemo_rl source file imports.""" + imports: dict[str, set[str]] = {} + for path in NEMO_RL_ROOT.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if isinstance(node, ast.ImportFrom) and node.level == 0: + modules = [node.module or ""] + elif isinstance(node, ast.Import): + modules = [alias.name for alias in node.names] + else: + continue + for module in modules: + if module == "dynamo" or module.startswith("dynamo."): + imports.setdefault(str(path.relative_to(NEMO_RL_ROOT)), set()).add( + module + ) + return imports + + +def test_nemo_rl_imports_only_dynamos_backend_sdk() -> None: + imports = _dynamo_imports() + + assert imports, "the Dynamo adapter's imports were not found" + outside = { + path: sorted(modules - DYNAMO_SDK_MODULES) + for path, modules in imports.items() + if modules - DYNAMO_SDK_MODULES + } + assert not outside, ( + f"nemo_rl imports Dynamo modules beyond its backend SDK: {outside}" + ) + + +@pytest.mark.vllm +def test_dynamos_backend_sdk_imports_no_vllm() -> None: + pytest.importorskip("dynamo._core") + # A fresh interpreter, so modules other tests imported do not count. + code = ( + "import json, sys\n" + + "".join(f"import {module}\n" for module in sorted(DYNAMO_SDK_MODULES)) + + "print(json.dumps(sorted(m for m in sys.modules" + " if m.split('.')[0] == 'vllm' or m.startswith('dynamo.vllm'))))" + ) + loaded = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + + assert json.loads(loaded.stdout.splitlines()[-1]) == [] diff --git a/tests/unit/models/generation/test_openai_server_utils.py b/tests/unit/models/generation/test_openai_server_utils.py index 223be2dc883..425e46442bc 100644 --- a/tests/unit/models/generation/test_openai_server_utils.py +++ b/tests/unit/models/generation/test_openai_server_utils.py @@ -17,6 +17,7 @@ from nemo_rl.models.generation.openai_server_utils import ( PrefixSplice, + derive_prefix_eos_count, replace_prefix_tokens, splice_prefix_tokens, ) @@ -160,6 +161,97 @@ def test_splice_prefix_tokens_without_tokenizer_honors_explicit_eos_ids( ) assert result == expected assert result.token_ids[: len(model_prefix)] == model_prefix + # A host without the messages (managed Dynamo's engine) passes the EOS count + # derived from the prefix instead of the template prefix render. + assert ( + splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=model_prefix, + template_token_ids=template, + eos_token_id=eos_token_id, + template_prefix_eos_count=derive_prefix_eos_count( + model_prefix, eos_token_id=eos_token_id + ), + ) + == expected + ) + with pytest.raises(ValueError, match="exactly one"): + splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=model_prefix, + template_prefix_token_ids=template_prefix, + template_token_ids=template, + eos_token_id=eos_token_id, + template_prefix_eos_count=1, + ) + + +@pytest.mark.parametrize( + "model_name", ["Qwen/Qwen3-0.6B", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"] +) +def test_derived_prefix_eos_count_matches_real_template_renders(model_name): + """Across tool-calling turns with reasoning, the EOS count derived from the + exact prefix equals the count in the template's render of the same history.""" + transformers = pytest.importorskip("transformers") + try: + tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) + except OSError as error: + pytest.skip(f"{model_name} tokenizer unavailable: {error}") + eos = tokenizer.eos_token_id + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + + def render(messages, *, add_generation_prompt): + rendered = tokenizer.apply_chat_template( + messages, + tools=tools, + add_generation_prompt=add_generation_prompt, + tokenize=True, + ) + return list(getattr(rendered, "input_ids", rendered)) + + def generate(text): + return tokenizer.encode(text, add_special_tokens=False) + [eos] + + messages = [ + {"role": "system", "content": "You are a weather assistant."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + replies = [ + "\nI should look it up.\n\n\nChecking the forecast.", + "\nIt is sunny.\n\n\nIt is sunny in Paris.", + ] + follow_ups = [ + {"role": "tool", "content": "sunny, 22C"}, + {"role": "user", "content": "And London?"}, + ] + chain = render(messages, add_generation_prompt=True) + generate(replies[0]) + for reply, follow_up, next_reply in zip(replies, follow_ups, replies[1:] + ["ok"]): + messages = [*messages, {"role": "assistant", "content": reply}, follow_up] + template_prefix = render(messages[:-1], add_generation_prompt=False) + template = render(messages, add_generation_prompt=True) + assert derive_prefix_eos_count(chain, eos_token_id=eos) == sum( + token == eos for token in template_prefix + ) + spliced = splice_prefix_tokens( + tokenizer=tokenizer, + model_prefix_token_ids=chain, + template_prefix_token_ids=template_prefix, + template_token_ids=template, + ).token_ids + chain = spliced + generate(next_reply) def test_replace_prefix_tokens_without_tokenizer_reports_missing_eos_without_decoding(): diff --git a/tests/unit/models/generation/test_swe1_dynamo_config.py b/tests/unit/models/generation/test_swe1_dynamo_config.py index c0a51db633c..8b2053e6103 100644 --- a/tests/unit/models/generation/test_swe1_dynamo_config.py +++ b/tests/unit/models/generation/test_swe1_dynamo_config.py @@ -8,7 +8,7 @@ from nemo_rl.environments.nemo_gym import setup_nemo_gym_config from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.config import DynamoConfig +from nemo_rl.models.generation.dynamo.config import dynamo_cfg from nemo_rl.utils.config import load_config, register_omegaconf_resolvers REPO_ROOT = Path(__file__).resolve().parents[4] @@ -43,9 +43,11 @@ class Tokenizer: setup_nemo_gym_config( SimpleNamespace(policy=config["policy"], env=config["env"]), tokenizer ) - validated = DynamoConfig.model_validate(configured_generation) - assert validated.vllm_cfg.async_engine is True - assert validated.vllm_cfg.expose_http_server is True + vllm_cfg = configured_generation["vllm_cfg"] + assert configured_generation["backend"] == "vllm" + assert configured_generation["dynamo"] is True + assert vllm_cfg["async_engine"] is True + assert vllm_cfg["expose_http_server"] is True assert config["policy"]["model_name"] == ( "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" @@ -63,12 +65,11 @@ class Tokenizer: "gpus_per_node": 8, "num_nodes": 1, } - assert validated.engine_world_size == 4 - assert generation["vllm_cfg"]["expert_parallel_size"] == 4 - assert validated.dynamo_cfg.frontend_args.router_mode == "kv" - assert validated.dynamo_cfg.control_timeout_s == 600 - assert validated.vllm_cfg.enable_vllm_metrics_logger is True - assert validated.vllm_cfg.load_format == "dummy" + assert vllm_cfg["tensor_parallel_size"] * vllm_cfg["pipeline_parallel_size"] == 4 + assert vllm_cfg["expert_parallel_size"] == 4 + assert dynamo_cfg(configured_generation).frontend_args.router_mode == "kv" + assert vllm_cfg["enable_vllm_metrics_logger"] is True + assert vllm_cfg["load_format"] == "dummy" assert config["env"]["nemo_gym"]["config_paths"][-1].endswith( "swe_pivot_single_step_tool_use_with_argument_comparison.yaml" ) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index dffb6b6af15..f3d4b7720c0 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -380,7 +380,7 @@ def test_vllm_generation_rejects_unregistered_worker_extension( cluster._init_placement_groups.assert_not_called() -@pytest.mark.parametrize("backend", ["sglang", "megatron", "trtllm", "dynamo"]) +@pytest.mark.parametrize("backend", ["sglang", "megatron", "trtllm"]) def test_generation_config_rejects_worker_extension_for_other_backends( backend: str, ) -> None: @@ -392,7 +392,7 @@ def test_generation_config_rejects_worker_extension_for_other_backends( configure_generation_config(config, MagicMock()) -@pytest.mark.parametrize("backend", ["vllm", "sglang", "megatron", "trtllm", "dynamo"]) +@pytest.mark.parametrize("backend", ["vllm", "sglang", "megatron", "trtllm"]) @pytest.mark.parametrize("include_null", [False, True]) def test_generation_config_allows_no_worker_extension( backend: str, @@ -2427,6 +2427,63 @@ def test_vllm_validate_settings_accepts_missing_vllm_cfg_as_refit_disabled(): VllmGeneration.validate_settings(master_config) +def _dynamo_master_config( + *, + router=False, + fleet_health=False, + nemo_gym=True, + router_replay=False, + token_capture=False, +): + vllm_config = deepcopy(basic_vllm_test_config) + vllm_config["dynamo"] = True + vllm_config["vllm_cfg"]["async_engine"] = True + vllm_config["vllm_cfg"]["expose_http_server"] = True + return types.SimpleNamespace( + policy={"generation": vllm_config, "router_replay": {"enabled": router_replay}}, + env={"should_use_nemo_gym": nemo_gym}, + async_rl=types.SimpleNamespace( + generation_router=types.SimpleNamespace(enabled=router), + generation_fleet_health=types.SimpleNamespace(enabled=fleet_health), + ), + token_capture=types.SimpleNamespace(enabled=token_capture), + ) + + +def test_vllm_validate_settings_accepts_dynamo_with_nemo_gym(): + VllmGeneration.validate_settings(_dynamo_master_config()) + VllmGeneration.validate_settings( + _dynamo_master_config(router_replay=True, token_capture=True) + ) + VllmGeneration.validate_settings( + _dynamo_master_config(router_replay=True, nemo_gym=False) + ) + + +@pytest.mark.parametrize( + ("settings", "error"), + [ + ({"router": True}, "routes the whole fleet behind one URL"), + ({"fleet_health": True}, "Disable fleet health"), + ({"router_replay": True}, "requires token_capture.enabled=true"), + ], +) +def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error): + with pytest.raises(ValueError, match=error): + VllmGeneration.validate_settings(_dynamo_master_config(**settings)) + + +@pytest.mark.parametrize("dynamo", [False, True]) +def test_vllm_client_base_urls_name_the_dynamo_frontend_when_it_serves(dynamo): + generation = object.__new__(VllmGeneration) + generation.dp_openai_server_base_urls = ["http://a/v1", "http://b/v1"] + generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None + + assert generation.client_base_urls == ( + ["http://frontend/v1"] if dynamo else ["http://a/v1", "http://b/v1"] + ) + + def test_vllm_policy_generation(policy, test_input_data, tokenizer): """Test vLLM policy generation capabilities.""" # Test generation diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py new file mode 100644 index 00000000000..99125fd1c29 --- /dev/null +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -0,0 +1,146 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The rollout result pipeline shared by generate_async and Dynamo serving.""" + +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result, + rollout_result_from_wire, + rollout_result_to_wire, +) +from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL + +LAYERS, TOPK = 2, 2 + + +def _logprob(value: float): + return SimpleNamespace(logprob=value) + + +def _final_output( + generated: list[int], + *, + finish_reason: str = "stop", + prompt_routes: int | None = None, + generated_routes: int | None = None, +): + def routes(count): + if count is None: + return None + return np.arange(count * LAYERS * TOPK, dtype=np.int16).reshape( + count, LAYERS, TOPK + ) + + return SimpleNamespace( + prompt_routed_experts=routes(prompt_routes), + num_cached_tokens=0, + outputs=[ + SimpleNamespace( + token_ids=generated, + # The second token has no reported logprob. + logprobs=[{generated[0]: _logprob(-0.5)}, {}] + + [{t: _logprob(-1.0)} for t in generated[2:]], + finish_reason=finish_reason, + routed_experts=routes(generated_routes), + ) + ], + ) + + +def _result(final_output, *, input_length=3, r3=False) -> RolloutResult: + return rollout_result( + final_output, + input_length=input_length, + return_routed_experts=r3, + routed_experts_dtype=torch.int16, + device=torch.device("cpu"), + ) + + +def test_output_layout_matches_generate_async() -> None: + result = _result(_final_output([7, 8, 9], finish_reason="length")) + output = rollout_output( + result, input_ids=torch.tensor([1, 2, 3, 0, 0]), input_length=3, pad_token_id=0 + ) + + assert output["output_ids"].tolist() == [[1, 2, 3, 7, 8, 9]] + # A position without a reported logprob stays 0.0, aligned with its token. + assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0, -0.5, 0.0, -1.0]] + assert output["generation_lengths"].tolist() == [3] + assert output["unpadded_sequence_lengths"].tolist() == [6] + assert output["truncated"].tolist() == [True] + assert "routed_experts" not in output + + +def test_nothing_generated_returns_the_prompt_untruncated() -> None: + output = rollout_output( + RolloutResult.nothing_generated(), + input_ids=torch.tensor([1, 2, 3, 0]), + input_length=3, + pad_token_id=0, + ) + + assert output["output_ids"].tolist() == [[1, 2, 3]] + assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0]] + assert output["generation_lengths"].tolist() == [0] + assert output["truncated"].tolist() == [False] + + +def test_router_replay_routes_are_aligned_and_counted() -> None: + # 3 prompt + 3 generated tokens need 5 routes; vLLM reported 3 + 1. + result = _result( + _final_output([7, 8, 9], prompt_routes=3, generated_routes=1), r3=True + ) + output = rollout_output( + result, input_ids=torch.tensor([1, 2, 3]), input_length=3, pad_token_id=0 + ) + + routed = output["routed_experts"] + assert routed.shape == (1, 6, LAYERS, TOPK) and routed.dtype == torch.int16 + assert (routed[0, 4:] == R3_MISSING_ROUTE_SENTINEL).all() + assert output["r3_routed_experts_expected_routes"].tolist() == [5] + assert output["r3_routed_experts_actual_routes"].tolist() == [4] + assert output["r3_routed_experts_missing_routes"].tolist() == [1] + + +def test_router_replay_requires_routes() -> None: + with pytest.raises(RuntimeError, match="did not include routed_experts"): + _result(_final_output([7, 8, 9]), r3=True) + + +def test_wire_round_trip_preserves_the_output() -> None: + result = _result( + _final_output([7, 8, 9], prompt_routes=3, generated_routes=3), r3=True + ) + wire = rollout_result_to_wire(result) + input_ids = torch.tensor([1, 2, 3]) + + direct = rollout_output(result, input_ids=input_ids, input_length=3, pad_token_id=0) + shipped = rollout_output( + rollout_result_from_wire(wire), + input_ids=input_ids, + input_length=3, + pad_token_id=0, + ) + + assert direct.keys() == shipped.keys() + for key in direct: + assert torch.equal(direct[key], shipped[key]), key diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index 3414010f089..25b5a2e19ee 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -230,6 +230,7 @@ def _worker_with_capture(sink: _MemorySink): for name in ( "_fetch_chain_prefix", "_capture_admission", + "_parse_capture_admission", "_resolve_admission_prefix", "_enter_request_prefix", ): diff --git a/tests/unit/models/generation/test_vllm_utils.py b/tests/unit/models/generation/test_vllm_utils.py index 7bf7546d9d0..3ce5907bc06 100644 --- a/tests/unit/models/generation/test_vllm_utils.py +++ b/tests/unit/models/generation/test_vllm_utils.py @@ -38,6 +38,7 @@ encode_counter_key, format_prompt_for_vllm_generation, model_dump_chat_response_with_dynamic_message_fields, + off_policy_sampling_error, pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) @@ -1451,3 +1452,29 @@ def test_remap_multimodal_placeholders_rebuilds_real_vllm_placeholder_range(): assert isinstance(moved, PlaceholderRange) assert (moved.offset, moved.length) == (3, 3) assert moved.is_embed is is_embed + + +@pytest.mark.parametrize( + ("temperature", "top_p", "top_k", "error"), + [ + (1.0, 1.0, None, None), # train profile + (0.0, 0.9, -1, None), # validation profile + (0.7, 1.0, None, "matches neither"), + (1.0, None, None, "top_p must be set explicitly"), + (1.0, 1.0, 5, "Top k sampling parameter"), + ], +) +def test_off_policy_sampling_error(temperature, top_p, top_k, error) -> None: + generation_config = { + "temperature": 1.0, + "top_p": 1.0, + "val_temperature": 0.0, + "val_top_p": 0.9, + } + message = off_policy_sampling_error( + temperature=temperature, + top_p=top_p, + top_k=top_k, + generation_config=generation_config, + ) + assert (message is None) if error is None else (error in message) diff --git a/tests/unit/models/policy/test_worker_refit_signatures.py b/tests/unit/models/policy/test_worker_refit_signatures.py index fc1e6200c34..d20bd2a203c 100644 --- a/tests/unit/models/policy/test_worker_refit_signatures.py +++ b/tests/unit/models/policy/test_worker_refit_signatures.py @@ -99,7 +99,6 @@ GENERATION_BACKENDS = [ "interfaces.py", "vllm/vllm_generation.py", - "dynamo/dynamo_generation.py", "trtllm/trtllm_generation.py", "megatron/megatron_generation.py", "sglang/sglang_generation.py", diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index 4f3318e9990..6acf4f133cd 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -171,13 +171,6 @@ def _make_master_config( "kv_cache_management_mode": "recompute", } policy_config["model_name"] = "test-model" - if backend == "dynamo": - # The fields DynamoGeneration.validate_settings reads, at accepted values. - generation_config["dynamo_cfg"] = { - "request_timeout_s": 900.0, - "control_timeout_s": 900.0, - } - generation_config["vllm_cfg"] = {"enable_vllm_metrics_logger": False} master_config = MasterConfig.model_construct( data_plane={ "enabled": dp_enabled, @@ -224,8 +217,6 @@ def _make_master_config( **({} if sampler_cfg is None else {"sampler": sampler_cfg}), ), ) - if backend == "dynamo": - master_config.async_rl.generation_fleet_health.refit_timeout_s = None return master_config @@ -364,51 +355,6 @@ def test_build_generation_passes_sglang_config(): generation.finish_generation.assert_called_once_with() -def test_build_generation_constructs_dynamo_from_normalized_config(): - """DynamoGeneration receives the generation config setup already validated.""" - master_config = _make_master_config(backend="dynamo") - master_config.policy["model_name"] = "Qwen/Qwen3-0.6B" - inference_cluster = MagicMock(name="inference_cluster") - - with patch.object(sc_setup_mod, "DynamoGeneration") as mock_dynamo: - generation, _ = sc_setup_mod._build_generation( - inference_cluster, - master_config, - ) - - mock_dynamo.assert_called_once_with( - cluster=inference_cluster, - config=master_config.policy["generation"], - ) - assert master_config.policy["generation"]["model_name"] == "Qwen/Qwen3-0.6B" - generation.finish_generation.assert_called_once_with() - - -def test_build_generation_rejects_deferred_dynamo_load(): - master_config = _make_master_config(backend="dynamo") - master_config.policy["model_name"] = "Qwen/Qwen3-0.6B" - - with ( - patch.object(sc_setup_mod, "DynamoGeneration") as mock_dynamo, - pytest.raises(AssertionError, match="defer_model_load"), - ): - sc_setup_mod._build_generation( - MagicMock(name="inference_cluster"), - master_config, - defer_model_load=True, - ) - mock_dynamo.assert_not_called() - - -def test_generation_max_seq_len_reads_dynamo_vllm_cfg(): - assert ( - sc_setup_mod._generation_max_seq_len( - {"backend": "dynamo", "vllm_cfg": {"max_model_len": 1024}} - ) - == 1024 - ) - - def test_build_clusters_rejects_unsupported_topology_backend(monkeypatch): """Topology planning reports the supported SC backends instead of KeyError.""" master_config = _make_master_config(colocated=False, backend="trtllm") @@ -432,7 +378,7 @@ def test_build_clusters_rejects_unsupported_topology_backend(monkeypatch): with pytest.raises( ValueError, - match="only supports vllm, sglang, megatron, or dynamo generation; got 'trtllm'", + match="only supports vllm, sglang, or megatron generation; got 'trtllm'", ): sc_setup_mod._build_clusters(master_config) @@ -1664,104 +1610,6 @@ def test_ready_first_sampler_rejects_incompatible_loss_config( patched_factories["setup_response_data"].assert_not_called() patched_factories["_build_clusters"].assert_not_called() - def test_invalid_dynamo_config_fails_before_clusters(self, patched_factories): - mc = _make_master_config(backend="dynamo") - - with ( - patch.object( - sc_setup_mod.DynamoConfig, - "model_validate", - side_effect=ValueError("bad dynamo config"), - ), - pytest.raises(ValueError, match="bad dynamo config"), - ): - setup_single_controller(mc, MagicMock(pad_token_id=0)) - - patched_factories["_build_clusters"].assert_not_called() - patched_factories["_build_generation"].assert_not_called() - - @pytest.mark.parametrize( - ("misconfigure", "match"), - [ - pytest.param( - lambda mc: mc.policy.update(router_replay={"enabled": True}), - "does not return routed experts", - id="router_replay", - ), - pytest.param( - lambda mc: mc.policy.update(is_vlm=True), - "serves text prompts only", - id="vlm", - ), - pytest.param( - lambda mc: setattr( - mc.async_rl.generation_fleet_health, "enabled", True - ), - "Dynamo frontend owns request routing", - id="fleet_health", - ), - # 300.0 is the FleetHealthConfig default, so a Dynamo config that does - # not opt out would otherwise only fail at its first weight sync. - pytest.param( - lambda mc: setattr( - mc.async_rl.generation_fleet_health, "refit_timeout_s", 300.0 - ), - "refit_timeout_s must be null", - id="refit_deadline", - ), - pytest.param( - lambda mc: mc.policy["generation"]["dynamo_cfg"].update( - control_timeout_s=600.0 - ), - "must be >= request_timeout_s", - id="drain_timeout", - ), - pytest.param( - lambda mc: mc.policy["generation"]["vllm_cfg"].update( - enable_vllm_metrics_logger=True - ), - "requires rollout_checkpointing.telemetry_interval_s", - id="unread_metrics_logger", - ), - ], - ) - def test_unsupported_dynamo_settings_fail_before_clusters( - self, patched_factories, misconfigure, match - ): - mc = _make_master_config(backend="dynamo") - misconfigure(mc) - - with ( - patch.object(sc_setup_mod.DynamoConfig, "model_validate") as validate, - pytest.raises(ValueError, match=match), - ): - validate.return_value.model_dump.return_value = mc.policy["generation"] - setup_single_controller(mc, MagicMock(pad_token_id=0)) - - patched_factories["setup_response_data"].assert_not_called() - patched_factories["_build_clusters"].assert_not_called() - patched_factories["_build_generation"].assert_not_called() - - def test_dynamo_generation_config_is_normalized(self, patched_factories): - mc = _make_master_config(backend="dynamo") - mc.policy["hf_config_overrides"] = {"rope_theta": 1.0} - normalized = { - **mc.policy["generation"], - "vllm_kwargs": {"hf_overrides": {"rope_theta": 1.0}}, - } - - with patch.object(sc_setup_mod.DynamoConfig, "model_validate") as validate: - validate.return_value.model_dump.return_value = normalized - actor_args, _ = setup_single_controller(mc, MagicMock(pad_token_id=0)) - - (raw_config,), _ = validate.call_args - assert raw_config["vllm_kwargs"]["hf_overrides"] == {"rope_theta": 1.0} - assert mc.policy["generation"] is normalized - assert actor_args.gen_handle is patched_factories["fake_gen"] - _, sync_kwargs = patched_factories["create_weight_synchronizer"].call_args - assert sync_kwargs["generation_backend"] == "dynamo" - assert sync_kwargs["colocated"] is False - def test_returns_actor_args(self, patched_factories): mc = _make_master_config() tokenizer = MagicMock(pad_token_id=0) @@ -2271,7 +2119,7 @@ def test_nemo_gym_wires_env_handle(self, patched_factories): assert data_kwargs["is_vlm"] is True mock_spinup.assert_called_once_with( mc.env, - base_urls=patched_factories["fake_gen"].dp_openai_server_base_urls, + base_urls=patched_factories["fake_gen"].client_base_urls, model_name="test-model", # Reaches the actor once, at spinup, rather than riding along with every # run_rollouts call. @@ -2384,6 +2232,44 @@ def test_token_capture_always_creates_finalizer_actor_pool( _, setup_kwargs = generation.setup_token_capture.call_args assert setup_kwargs["capture_media"] is with_processor + def test_dynamo_token_capture_tells_gym_to_speak_dynamo(self, patched_factories): + """The same vLLM workers stage the calls; Gym reaches them through the frontend.""" + mc = _make_master_config(backend="vllm") + mc.policy["generation"].update( + { + "model_name": "test-model", + "stop_strings": None, + "stop_token_ids": None, + "top_k": None, + "dynamo": True, + "vllm_cfg": {"async_engine": True, "expose_http_server": True}, + } + ) + mc.token_capture.enabled = True + mc.env = {"should_use_nemo_gym": True, "nemo_gym": {}} + patched_factories["setup_response_data"].return_value = (list(range(8)), None) + + with ( + patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), + patch.object( + sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() + ) as mock_spinup, + patch.object(sc_setup_mod, "validate_dataset_agent_coverage"), + patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), + patch( + "nemo_rl.experience.rollout_reassembler_actor.create_rollout_reassembler_actors", + return_value=[MagicMock()], + ), + ): + setup_single_controller(mc, MagicMock(pad_token_id=9)) + + assert mc.token_capture.generation_backend == "dynamo" + assert mock_spinup.call_args.kwargs["token_capture"]["generation_backend"] == ( + "dynamo" + ) + generation, _ = patched_factories["_build_generation"].return_value + generation.setup_token_capture.assert_called() + def test_nemo_gym_coverage_failure_shuts_down_shards(self, patched_factories): mc = _make_master_config(colocated=False, backend="vllm") mc.policy["generation"]["model_name"] = "test-model" diff --git a/tests/unit/test_dependency_pins.py b/tests/unit/test_dependency_pins.py index 864ee4d40ba..17c14b55aae 100644 --- a/tests/unit/test_dependency_pins.py +++ b/tests/unit/test_dependency_pins.py @@ -144,3 +144,21 @@ def test_torch_split_shims_are_still_needed() -> None: "(nemo_rl/data_plane/adapters/tq_mooncake_checkpoint.py) - drop the " "check outright; the pre-2.13 `names` comparison cannot be restored" ) + + +def test_vllm_extra_carries_base_dynamo_release() -> None: + """Dynamo serving runs NeMo RL's own vLLM workers, so the vllm extra carries the + released Dynamo runtime but never ai-dynamo[vllm], which pins Dynamo's vLLM.""" + extras = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text())["project"][ + "optional-dependencies" + ] + requirements = { + canonicalize_name(req.name): req + for req in (Requirement(dep) for dep in extras["vllm"]) + } + + dynamo, runtime = requirements["ai-dynamo"], requirements["ai-dynamo-runtime"] + assert not dynamo.extras, f"ai-dynamo must not take extras, got {dynamo.extras}" + assert dynamo.specifier == runtime.specifier, ( + "ai-dynamo and ai-dynamo-runtime must pin the same release" + ) diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index dc856d71c83..1e040d3c177 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -380,3 +380,46 @@ def test_dynamo_frontend_args_without_router_reset_states_pass(): def test_generation_without_dynamo_cfg_passes(): check_outdated_config({"policy": {"generation": {"backend": "vllm"}}}) + + +# ============================================================================ +# reject_outdated_dynamo_backend +# ============================================================================ + + +def _generation_config(**generation) -> dict: + return {"policy": {"generation": generation}} + + +def test_dynamo_backend_is_rejected_with_the_migration(): + with pytest.raises( + ValueError, match=r"backend=vllm and policy\.generation\.dynamo=true" + ): + check_outdated_config(_generation_config(backend="dynamo")) + + +@pytest.mark.parametrize( + "dynamo_cfg, removed", + [ + ({"engine": "vllm"}, r"dynamo_cfg\.engine"), + ({"control_timeout_s": 600}, r"dynamo_cfg\.control_timeout_s"), + ({"metrics_include_prefixes": None}, r"dynamo_cfg\.metrics_include_prefixes"), + ({"metrics_exclude_prefixes": None}, r"dynamo_cfg\.metrics_exclude_prefixes"), + ({"worker_args": {"extra_cli_args": []}}, r"worker_args\.extra_cli_args"), + ], +) +def test_removed_dynamo_cfg_keys_are_rejected(dynamo_cfg, removed): + with pytest.raises(ValueError, match=removed): + check_outdated_config( + _generation_config(backend="vllm", dynamo=True, dynamo_cfg=dynamo_cfg) + ) + + +def test_vllm_backend_with_dynamo_passes(): + check_outdated_config( + _generation_config( + backend="vllm", + dynamo=True, + dynamo_cfg={"frontend_args": {"router_mode": "kv"}}, + ) + ) diff --git a/tests/unit/weight_sync/test_reconcile_communicator.py b/tests/unit/weight_sync/test_reconcile_communicator.py index 39cfce24494..e0f3b22e006 100644 --- a/tests/unit/weight_sync/test_reconcile_communicator.py +++ b/tests/unit/weight_sync/test_reconcile_communicator.py @@ -643,10 +643,9 @@ class TestBackendsWithoutAWorkerGroup: """CollectiveWeightSynchronizer holds a GenerationInterface, not a VllmGeneration. Only some implementations own a Ray worker group with DP shards -- vLLM and TRT-LLM do; - Dynamo, Megatron and SGLang do not, and the interface declares nothing either way. Of - those, Dynamo reaches this synchronizer through the ordinary non-colocated branch of - the factory, so reading self._generation.worker_group broke it on the plain grpo.py - path: + Megatron and SGLang do not, and the interface declares nothing either way. Reading + self._generation.worker_group once broke the since-removed managed Dynamo backend + on the plain grpo.py path: grpo.py:1747 policy_generation.weight_synchronizer.init_communicator() AttributeError: 'DynamoGeneration' object has no attribute 'worker_group' diff --git a/tests/unit/weight_sync/test_weight_synchronizer.py b/tests/unit/weight_sync/test_weight_synchronizer.py index 785726ff446..14f4c83ac96 100644 --- a/tests/unit/weight_sync/test_weight_synchronizer.py +++ b/tests/unit/weight_sync/test_weight_synchronizer.py @@ -19,7 +19,6 @@ import pytest from nemo_rl.models.generation.constants import ( - DYNAMO_BACKEND, MEGATRON_BACKEND, SGLANG_BACKEND, VLLM_BACKEND, @@ -1556,17 +1555,6 @@ def test_non_colocated_vllm_nccl_reshard_propagates_refit_memory_release( sync.sync_weights() policy.offload_before_refit.assert_called_once_with() - def test_non_colocated_dynamo_returns_collective(self): - sync = create_weight_synchronizer( - policy=_mock_policy(), - generation=_mock_generation(), - generation_backend=DYNAMO_BACKEND, - colocated=False, - train_cluster=_mock_cluster(), - inference_cluster=_mock_cluster(), - ) - assert isinstance(sync, CollectiveWeightSynchronizer) - def test_non_colocated_sglang_returns_sglang_disaggregated(self): """SGLang owns its own weight-update group, so no clusters are needed.""" policy = _megatron_policy() diff --git a/uv.lock b/uv.lock index 80477156eaa..56bc756bbaf 100644 --- a/uv.lock +++ b/uv.lock @@ -85,12 +85,14 @@ constraints = [ { name = "wheel", specifier = ">=0.46.2" }, ] overrides = [ + { name = "aisimulate", marker = "sys_platform == 'never'" }, { name = "apache-tvm-ffi", specifier = "==0.1.11" }, { name = "av", marker = "sys_platform == 'never'" }, { name = "decord2", marker = "sys_platform == 'never'" }, { name = "deep-ep", git = "https://github.com/deepseek-ai/DeepEP.git?rev=17cfb817bccec3a9c247013360cc550c2bac441e" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.133.0,<0.137.0" }, { name = "instanttensor", marker = "platform_machine == 'x86_64'", specifier = ">=0.1.9" }, + { name = "kubernetes", marker = "sys_platform == 'never'" }, { name = "langchain", specifier = ">=0.3.28" }, { name = "langchain-core", specifier = ">=0.3.80" }, { name = "langchain-core", specifier = ">=1.2.22" }, @@ -221,6 +223,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/15/504337e302d070bd2ff2f39418bb17d2ad73c662c403a4f37cb4e5c5b237/agent_detector-2.0.0-py3-none-any.whl", hash = "sha256:22c6a1c9c23894a12f55aea625f060b8d3330fba774de72d02d5442bb580417f", size = 9352, upload-time = "2026-09-02T14:19:50.971Z" }, ] +[[package]] +name = "ai-dynamo" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ai-dynamo-runtime" }, + { name = "aiohttp" }, + { name = "msgspec" }, + { name = "prometheus-client" }, + { name = "pyzmq" }, + { name = "tqdm" }, + { name = "transformers", version = "5.12.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, + { name = "zstandard" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/19/bc00c468b1d8093abafa7f3ab88362d12cfe06f9be2e65baf0ef03d370d1/ai_dynamo-1.5.1-py3-none-any.whl", hash = "sha256:86d21cb7fbebbf1c3d0933303729b1be222ad0fb4a50b17a03316165d33377b1", size = 3512837, upload-time = "2026-10-07T05:08:09.942Z" }, +] + +[[package]] +name = "ai-dynamo-runtime" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, + { name = "uvloop" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/63/f1cde7bb2d74aebc954251fe40ee093298f0975d6f22c47f1f3c66b11949/ai_dynamo_runtime-1.5.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:88e5580daf32056dbe2da9d4189ddd2f4e4cc2472ddfebdd19412d223ac4a596", size = 65897955, upload-time = "2026-10-07T05:08:55.817Z" }, + { url = "https://files.pythonhosted.org/packages/df/05/777f6f79e8c9359d259cf500ec62fcaf6bd9ce148d3bf47910ec02d996e3/ai_dynamo_runtime-1.5.1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b5297625a4f407b3871e3039a53f49e09102d67039fc1339349802038b84ec3e", size = 67290732, upload-time = "2026-10-07T05:09:45.438Z" }, +] + [[package]] name = "aiobotocore" version = "3.9.1" @@ -345,7 +379,8 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "albucore" }, { name = "numpy" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "pyyaml" }, { name = "scipy" }, ] @@ -409,7 +444,8 @@ dependencies = [ { name = "docstring-parser", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "httpx", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "jiter", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "sniffio", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, ] @@ -431,7 +467,7 @@ dependencies = [ { name = "docstring-parser" }, { name = "httpx2" }, { name = "jiter" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "sniffio" }, { name = "typing-extensions" }, ] @@ -943,7 +979,7 @@ resolution-markers = [ ] dependencies = [ { name = "loguru" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "transformers", version = "5.12.1", source = { registry = "https://pypi.org/simple" } }, ] @@ -963,7 +999,7 @@ resolution-markers = [ dependencies = [ { name = "loguru" }, { name = "psutil" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "transformers", version = "5.12.1", source = { registry = "https://pypi.org/simple" } }, ] @@ -1585,7 +1621,8 @@ version = "0.136.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "starlette" }, { name = "typing-extensions" }, { name = "typing-inspection" }, @@ -1637,7 +1674,8 @@ dependencies = [ { name = "detect-installer" }, { name = "fastar" }, { name = "httpx" }, - { name = "pydantic", extra = ["email"] }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, extra = ["email"], marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, extra = ["email"], marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "rich-toolkit" }, { name = "rignore" }, { name = "sentry-sdk" }, @@ -3034,7 +3072,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "interegular" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/d5/41cd417ba7dfdbbcfe46cebf81fb3dfd7c591b89897560ad05bb410a465d/lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da", size = 40258, upload-time = "2025-08-24T19:37:47.527Z" } @@ -3190,7 +3228,8 @@ dependencies = [ { name = "httpx", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "httpx-sse", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "jsonschema", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "pydantic-settings", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "pyjwt", extra = ["crypto"], marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "python-multipart", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, @@ -3571,7 +3610,8 @@ dependencies = [ { name = "jsonschema" }, { name = "numpy" }, { name = "pillow" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm'" }, { name = "pydantic-extra-types", extra = ["pycountry"], marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra == 'extra-7-nemo-rl-vllm'" }, { name = "requests" }, { name = "tiktoken" }, @@ -3655,7 +3695,8 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "requests" }, @@ -3681,7 +3722,8 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/88/62/64068c0d17a7ba479f4dadfa78b3e48921cde8d20878414e7a4a929e66c0/mlflow_tracing-3.16.1.tar.gz", hash = "sha256:315ba1ace00d43498da2ad2b2bbb751f3198d6f9cd328168dfeb49911df41925", size = 1525331, upload-time = "2026-09-16T22:40:58.167Z" } wheels = [ @@ -3696,7 +3738,7 @@ dependencies = [ { name = "fastapi", extra = ["standard"], marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, { name = "httpx" }, { name = "jmespath" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, { name = "setuptools" }, { name = "starlette" }, { name = "supervisor" }, @@ -4104,8 +4146,10 @@ dependencies = [ { name = "packaging" }, { name = "psutil" }, { name = "pyasn1" }, - { name = "pydantic" }, - { name = "pydantic-core" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic-core", version = "2.46.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic-core", version = "2.46.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "pydot" }, { name = "pyjwt" }, { name = "python-multipart" }, @@ -4351,6 +4395,8 @@ trtllm = [ { name = "transformers", version = "5.5.4", source = { registry = "https://pypi.org/simple" } }, ] vllm = [ + { name = "ai-dynamo" }, + { name = "ai-dynamo-runtime" }, { name = "cuda-python" }, { name = "deep-ep" }, { name = "deep-gemm" }, @@ -4410,6 +4456,8 @@ test = [ [package.metadata] requires-dist = [ { name = "accelerate", specifier = ">=0.26" }, + { name = "ai-dynamo", marker = "extra == 'vllm'", specifier = "==1.5.1" }, + { name = "ai-dynamo-runtime", marker = "extra == 'vllm'", specifier = "==1.5.1" }, { name = "awscrt", specifier = ">=0.35.0" }, { name = "blobfile" }, { name = "causal-conv1d", marker = "extra == 'automodel'", git = "https://github.com/Dao-AILab/causal-conv1d?rev=4f6ae4e26ae5fe8af9372f8d312ab25cc4595223" }, @@ -5293,7 +5341,8 @@ dependencies = [ { name = "omegaconf", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "packaging", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "pulp", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "pyyaml", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "regex", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "rich", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -5368,7 +5417,8 @@ dependencies = [ { name = "packaging", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "protobuf", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "psutil", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "pydantic-settings", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "pyyaml", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "slowapi", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -5533,7 +5583,7 @@ dependencies = [ { name = "distro" }, { name = "httpx" }, { name = "jiter" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "sniffio" }, { name = "tqdm" }, { name = "typing-extensions" }, @@ -5556,7 +5606,8 @@ dependencies = [ { name = "distro", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "httpx", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "jiter", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "sniffio", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "tqdm", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -5577,7 +5628,8 @@ name = "openai-harmony" version = "0.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/92/94/01509d510bebf6606614e51113e5a415ced15b8f34aa98a8bf2539314650/openai_harmony-0.0.4.tar.gz", hash = "sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c", size = 279848, upload-time = "2025-08-09T01:43:24.974Z" } wheels = [ @@ -5880,7 +5932,7 @@ dependencies = [ { name = "jsonschema" }, { name = "outlines-core" }, { name = "pillow" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/0f/95627aba2d038f8ba24c31b689609b153386806a46448abf4c27029b0001/outlines-1.3.3.tar.gz", hash = "sha256:5dcd1b9d261ced1337042d6b1fd2bb4e706a09a0121957bcd9eb331d1ea6940c", size = 2898295, upload-time = "2026-08-06T12:27:14.921Z" } @@ -6430,15 +6482,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/e9/e869bcee87beb89040263c416a8a50204f7f7a83ac11897646c9e71e0daf/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55ccbe27f049743a4caf4f4221b166560d3438d0b1e5ab929e07ae1702a4d6fd", size = 2271038, upload-time = "2025-05-17T17:23:04.872Z" }, ] +[[package]] +name = "pydantic" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "annotated-types", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic-core", version = "2.46.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "typing-inspection", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/6b/69fd5c7194b21ebde0f8637e2a4ddc766ada29d472bfa6a5ca533d79549a/pydantic-2.13.0.tar.gz", hash = "sha256:b89b575b6e670ebf6e7448c01b41b244f471edd276cd0b6fe02e7e7aca320070", size = 843468, upload-time = "2026-04-13T10:51:35.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/d7/c3a52c61f5b7be648e919005820fbac33028c6149994cd64453f49951c17/pydantic-2.13.0-py3-none-any.whl", hash = "sha256:ab0078b90da5f3e2fd2e71e3d9b457ddcb35d0350854fbda93b451e28d56baaf", size = 471872, upload-time = "2026-04-13T10:51:33.343Z" }, +] + +[package.optional-dependencies] +email = [ + { name = "email-validator", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, +] + [[package]] name = "pydantic" version = "2.13.5" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, + { name = "annotated-types", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, + { name = "pydantic-core", version = "2.46.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, + { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, + { name = "typing-inspection", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } wheels = [ @@ -6447,15 +6527,38 @@ wheels = [ [package.optional-dependencies] email = [ - { name = "email-validator" }, + { name = "email-validator", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/0a/9414cddf82eda3976b14048cc0fa8f5b5d1aecb0b22e1dcd2dbfe0e139b1/pydantic_core-2.46.0.tar.gz", hash = "sha256:82d2498c96be47b47e903e1378d1d0f770097ec56ea953322f39936a7cf34977", size = 471441, upload-time = "2026-04-13T09:06:33.813Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/a0/bf9a1ba34537c2ed3872a48195291138fdec8fe26c4009776f00d63cf0c8/pydantic_core-2.46.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c525ecf8a4cdf198327b65030a7d081867ad8e60acb01a7214fff95cf9832d47", size = 1977040, upload-time = "2026-04-13T09:06:16.088Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/5885350203b72e96438eee7f94de0d8f0442f4627237ca8ef75de34db1cd/pydantic_core-2.46.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7897078fe8a13b73623c0955dfb2b3d2c9acb7177aac25144758c9e5a5265aaa", size = 2098522, upload-time = "2026-04-13T09:04:23.239Z" }, + { url = "https://files.pythonhosted.org/packages/a7/53/1958eacbfddc41aadf5ae86dd85041bf054b675f34a2fa76385935f96070/pydantic_core-2.46.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:ee1547a6b8243e73dd10f585555e5a263395e55ce6dea618a078570a1e889aef", size = 2190148, upload-time = "2026-04-13T09:06:56.151Z" }, + { url = "https://files.pythonhosted.org/packages/71/a7/abdb924620b1ac535c690b36ad5b8871f376104090f8842c08625cecf1d3/pydantic_core-2.46.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:004a2081c881abfcc6854a4623da6a09090a0d7c1398a6ae7133ca1256cee70b", size = 2383167, upload-time = "2026-04-13T09:04:52.643Z" }, ] [[package]] name = "pydantic-core" version = "2.46.5" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } wheels = [ @@ -6470,7 +6573,8 @@ name = "pydantic-extra-types" version = "2.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/71/dba38ee2651f84f7842206adbd2233d8bbdb59fb85e9fa14232486a8c471/pydantic_extra_types-2.11.1.tar.gz", hash = "sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049", size = 172002, upload-time = "2026-03-16T08:08:03.92Z" } @@ -6488,7 +6592,8 @@ name = "pydantic-settings" version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] @@ -6994,7 +7099,8 @@ default = [ { name = "opentelemetry-sdk" }, { name = "prometheus-client" }, { name = "py-spy" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "requests" }, { name = "smart-open" }, { name = "virtualenv" }, @@ -7338,7 +7444,7 @@ dependencies = [ { name = "psutil" }, { name = "py-spy" }, { name = "pybase64" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "python-multipart" }, { name = "pyzmq" }, { name = "quack-kernels", version = "0.6.3", source = { registry = "https://pypi.org/simple" } }, @@ -7880,7 +7986,8 @@ dependencies = [ { name = "protobuf" }, { name = "psutil" }, { name = "pwinput" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "pydantic-settings" }, { name = "pyecharts" }, { name = "pyroaring" }, @@ -8055,7 +8162,7 @@ dependencies = [ { name = "protobuf" }, { name = "psutil" }, { name = "pulp" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "pydantic-settings", extra = ["yaml"], marker = "extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "python-multipart" }, { name = "pyzmq" }, @@ -8699,7 +8806,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/31/37/b2010e5c18b25b5016064685dbf858d880bdeb859e2d1b2d5b9cd69230fa/transformer_engine_cu13-2.18.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:d5e9049256d66e1fb1743b5e1ff78e65a580bc09c371f4977ca71cdf7899fca4", size = 278015862, upload-time = "2026-08-11T00:50:33.011Z" }, @@ -8717,7 +8824,7 @@ dependencies = [ { name = "onnx" }, { name = "onnxscript" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "transformer-engine-cu13" }, ] @@ -9017,7 +9124,7 @@ dependencies = [ { name = "psutil" }, { name = "py-cpuinfo" }, { name = "pybase64" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, { name = "pynvvideocodec" }, { name = "python-json-logger" }, { name = "pyyaml" }, @@ -9063,7 +9170,8 @@ dependencies = [ { name = "packaging" }, { name = "platformdirs" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "pyyaml" }, { name = "requests" }, { name = "typing-extensions" }, @@ -9250,7 +9358,8 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, { name = "numpy" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-7-nemo-rl-trtllm' or extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang')" }, { name = "transformers", version = "5.5.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, From 08c50d29e177ec36377554843dfaff8c92c39658 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 24 Oct 2026 17:17:05 -0700 Subject: [PATCH 02/82] fix(dynamo): address the #4905 review Run and CI breaks: - Each engine publishes KV events on its own abstract ipc:// socket, so engines on one node can no longer draw the same port (C1). - Engines get an OS-assigned DYN_SYSTEM_PORT, so a frontend on another node can read a custom chat template (C2). - The Gym capture test skips without nemo_gym (C3). - Gym calls apply the generation config's max_tokens only when the request leaves it unset, as vLLM's server does (C4). - Lint: minimized swe1 recipe (C5) and import order (C6). Behavior: - vLLM client errors reach the client as InvalidArgument with their message, not as a bare 500 (C7). - Non-capture Gym calls return routed experts with R3 on, and Gym#4817 (now a369e28) lifts them, so R3 + Gym no longer needs token capture (C8). - A disconnected capture call fails instead of being staged (C9). - EP > TP is rejected at startup with Dynamo (C10). - The KV-event block size matches vLLM's main-attention kinds (C17). Messages and docs: timeout wording and error (C11, C12), missing etcd/NATS advice (C13), dependency bounds (C19) and ray.sub's port layout (C20). Tests: a Gym-without-capture Dynamo overlay in L1 (C14), the rollout retry loop (C15), media dispatch and failure mapping (C16), hybrid block size (C17) and the Gym engine_data key contract (C18). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/design-docs/dynamo-integration.md | 15 +++- docs/guides/dynamo-generation.md | 14 ++-- ...grpo_math_1B_dynamo_single_controller.yaml | 3 +- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 1 - nemo_rl/distributed/virtual_cluster.py | 3 - nemo_rl/models/generation/__init__.py | 15 +++- nemo_rl/models/generation/dynamo/config.py | 4 +- .../generation/dynamo/managed_runtime.py | 4 +- .../models/generation/dynamo/native_engine.py | 79 ++++++++++++------- .../generation/dynamo/rollout_client.py | 8 ++ nemo_rl/models/generation/dynamo/venv.py | 9 ++- .../models/generation/vllm/vllm_generation.py | 23 +----- ray.sub | 3 +- .../functional/L1_Functional_Tests_Dynamo.sh | 2 + .../models/generation/test_dynamo_config.py | 22 +++++- .../generation/test_dynamo_managed_runtime.py | 8 ++ .../generation/test_dynamo_native_engine.py | 69 +++++++++++++++- .../test_dynamo_native_engine_capture.py | 13 ++- .../generation/test_dynamo_rollout_client.py | 56 +++++++++++-- .../models/generation/test_vllm_generation.py | 65 +++++++++------ 21 files changed, 311 insertions(+), 107 deletions(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 5e2fb03bbab..a369e289f70 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 5e2fb03bbabe89543291cf99c471e3d32effdd83 +Subproject commit a369e289f702c623b077e811eceeb98f2b3b26cf diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 8aa388e138d..0fea1ccc5fd 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -54,9 +54,10 @@ implements its `LLMEngine` over the worker's `AsyncLLM`: publishes no token budget. The engine owns the context check and the output cap, because a prefix splice changes the prompt after the frontend has seen it. - `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ - publisher. It is set only when the engine caches prefixes, and it binds a - wildcard endpoint, because vLLM binds only wildcard endpoints. A vLLM stat - logger publishes the router's per-engine load signal. + publisher. It is set only when the engine caches prefixes. Each engine binds + its own Linux abstract-namespace socket (`ipc://@...`), so engines on one node + need no port and cannot collide. A vLLM stat logger publishes the router's + per-engine load signal. Because the Worker runs on the actor's own loop, the worker's other methods interleave with Dynamo requests exactly as with native requests: @@ -104,3 +105,11 @@ functional overlays fail if the engines published events but none were applied (`tests/functional/check_dynamo_overlay.sh`). `ai-dynamo-runtime` caps pydantic, which pins the vLLM venvs' pydantic version. +Dependency bounds like this one can still block a vLLM bump, and the import +guard does not see them: `ai-dynamo-runtime` 1.5.1 requires +`pydantic>=2.10.6,<=2.13`, and `ai-dynamo` caps `aiohttp<4`, +`prometheus-client<1` and `zstandard<1`. A vLLM release that needs a version +outside one of these bounds does not lock with the pinned `ai-dynamo`. Until a +Dynamo release lifts the bound, override it under +`[tool.uv] override-dependencies` (as `aisimulate` and `kubernetes` are), and +let the vLLM-lane Dynamo unit tests catch a real break. diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 09915d3f276..bc554ec422f 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -27,8 +27,9 @@ independent. With `dynamo: true`: - `vllm_cfg.async_engine` is turned on, because Dynamo drives the engine's - `AsyncLLM`. Every rollout path the sync engine supports also runs on the async - one. + `AsyncLLM`. Settings the async engine does not run, such as data parallelism + inside vLLM (`expert_parallel_size > tensor_parallel_size`), are rejected at + startup. - The driver starts etcd, NATS and `dynamo.frontend`. The frontend runs from the vLLM workers' own venv, so it is the same ai-dynamo release. - Each engine turns on vLLM's KV-cache events for `router_mode: kv` and @@ -50,7 +51,7 @@ shows them all: | Key | Default | Meaning | |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | -| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timed-out rollout is not retried; NeMo-Gym's calls keep Gym's timeouts | +| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies; NeMo-Gym's calls keep Gym's timeouts | | `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | | `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | @@ -102,6 +103,8 @@ chat endpoint does: - Stop strings stop vLLM itself, so the token IDs match the native endpoint's. - The response carries `prompt_token_ids`, `completion_token_ids` and `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. + With router replay on, it also carries the call's `routed_experts`, as the + native endpoint attaches them. - Images, video and audio in the messages are processed as on the native endpoint. The frontend passes each media URL through with one unexpanded placeholder per item. The engine loads the media with vLLM's media connector @@ -145,9 +148,8 @@ Startup rejects these with `dynamo: true`: - `async_rl.generation_router`: the frontend already routes the fleet behind one URL. - `vllm_cfg.http_server_api_key_required`: the frontend checks no bearer token. -- On the SingleController, router replay with Gym but without token capture: - Gym's dialect does not return routed experts in responses. Other entrypoints - do not check this, and such rollouts replay no routes. +- `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data + parallelism inside vLLM, which the async engine Dynamo serves does not. - Parser plugins with no Dynamo parser, and the chat settings above that the frontend has no counterpart for. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 291aac186b6..9ad2b249a95 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -15,7 +15,8 @@ policy: # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: 600 # Timeout for one rollout routed through the frontend (in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS); a timed-out rollout is not retried. + # NRL_VLLM_ASYNC_TIMEOUT_SECONDS). A timeout fails the attempt as a native + # timeout does, and the rollout retry policy applies. request_timeout_s: 900 worker_args: # Dynamo's built-in parser names. null follows the native server's diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml index 010fee9cb49..d12f2536970 100644 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml @@ -82,7 +82,6 @@ policy: sequence_packing: fuse_loss: true generation: - backend: vllm dynamo: true dynamo_cfg: startup_timeout_s: 1800 diff --git a/nemo_rl/distributed/virtual_cluster.py b/nemo_rl/distributed/virtual_cluster.py index 049ddde289f..b3eda15b7ba 100644 --- a/nemo_rl/distributed/virtual_cluster.py +++ b/nemo_rl/distributed/virtual_cluster.py @@ -158,7 +158,6 @@ def uv_py_executable(extras: Sequence[str]) -> str: # 1400-1999 Master address / TCPStore (cluster.master_port_range_low/high) # [3000, 4999) Shared NeMo RL generation range (policy.generation.port_range_low/high) # [3000, 4000) Dynamo frontend HTTP endpoint -# [4000, 4100) Dynamo engine KV-event endpoints (node-local free-port selection) # 5000-5999 NeMo Gym HTTP servers (env.nemo_gym.port_range_low/high) # 6000 NeMo-Skills sandbox Nginx (NEMO_SKILLS_SANDBOX_PORT; ray.sub starts one # sidecar per allocated node, driver included) @@ -179,8 +178,6 @@ def uv_py_executable(extras: Sequence[str]) -> str: DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH = 1400 DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW = 3000 DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH = 4000 -DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW = 4000 -DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH = 4100 DEFAULT_GYM_PORT_RANGE_LOW = 5000 DEFAULT_GYM_PORT_RANGE_HIGH = 5999 # vLLM TP/DP rendezvous ports. Each engine gets PORTS_PER_ENGINE ports starting diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 888015266e3..c24aeddc52c 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -206,9 +206,20 @@ def configure_generation_config( "sleeps while the policy trains, and the Dynamo frontend would " "keep routing requests to it." ) + vllm_cfg = config["vllm_cfg"] + if vllm_cfg["expert_parallel_size"] > vllm_cfg["tensor_parallel_size"]: + # VllmGeneration refuses the async engine with DP inside vLLM. + raise ValueError( + "policy.generation.dynamo=true serves the async vLLM engine, " + "which does not run data parallelism inside vLLM " + "(vllm_cfg.expert_parallel_size > tensor_parallel_size, see " + "https://github.com/NVIDIA-NeMo/RL/issues/1101). Set " + "expert_parallel_size <= tensor_parallel_size, or remove " + "policy.generation.dynamo." + ) if not config["vllm_cfg"]["async_engine"]: - # Dynamo serves the engine's AsyncLLM. The async engine runs every - # rollout path the sync one does, so turning it on is the switch. + # Dynamo serves the engine's AsyncLLM, so turning the async engine + # on is the switch; settings it cannot run are rejected above. print( " [Dynamo] policy.generation.dynamo=true: using the async " "vLLM engine (vllm_cfg.async_engine=true)", diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index d08758ce3ed..7a6a61868d2 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -82,7 +82,9 @@ class DynamoCfg(BaseModel, extra="forbid"): # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: PositiveFloat = 600.0 # Timeout for one NeMo RL rollout routed through the frontend, in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS; a timed-out rollout is not retried. + # NRL_VLLM_ASYNC_TIMEOUT_SECONDS. A timeout fails the attempt as a native one + # does (GenerationUnavailable), and the rollout retry policy re-dispatches it + # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). # NeMo-Gym's own calls keep Gym's timeouts. request_timeout_s: PositiveFloat = 900.0 worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index df75a7d9f12..e4a0fe9507e 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -119,7 +119,9 @@ def engine_env(self) -> dict[str, str]: """The environment each engine's Dynamo worker needs to register.""" if not self._started: raise RuntimeError("Managed Dynamo runtime has not been started") - return dict(self._manager_env) + # An OS-assigned system port makes each engine serve its own model-card + # files (a custom chat template included) to a frontend on another node. + return {**self._manager_env, "DYN_SYSTEM_PORT": "0"} def start(self) -> None: """Start etcd, NATS and the frontend.""" diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index 2eb085d3e5d..39bff56a0fb 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -54,7 +54,7 @@ from dynamo.llm.exceptions import HttpError, InvalidArgument from vllm import envs as vllm_envs from vllm.entrypoints.chat_utils import load_chat_template -from vllm.exceptions import GracefulHTTPError +from vllm.exceptions import GracefulHTTPError, VLLMClientError from vllm.inputs import TokensPrompt from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY from vllm.sampling_params import ( @@ -63,14 +63,10 @@ StructuredOutputsParams, ) from vllm.transformers_utils.processor import get_video_processor_cls_name +from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase from nemo_rl.data.captured_media import MediaCaptureRejected -from nemo_rl.distributed.virtual_cluster import ( - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH, - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, - _get_free_port_local, -) from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, dynamo_cfg, @@ -150,16 +146,15 @@ def dynamo_kv_events_config( """ if cfg.frontend_args.router_mode != "kv" or not prefix_caching: return None - port = _get_free_port_local( - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH - ) return { "enable_kv_cache_events": True, "publisher": "zmq", - # A wildcard host makes vLLM's publisher bind; any other TCP endpoint - # makes it connect, and Dynamo's ZMQ source connects too, so no event - # would flow. kv_event_sources subscribes on 127.0.0.1. - "endpoint": f"tcp://*:{port}", + # A Linux abstract-namespace socket (no port, no file): vLLM binds any + # ipc:// endpoint and this process's Dynamo worker connects to it, so + # engines on one node cannot collide. Random rather than the pid, since + # containers sharing the host network share these names. vLLM rejects + # ipc:// only at data_parallel_rank != 0, which Dynamo rejects at startup. + "endpoint": f"ipc://@nemo-rl-kv-events-{os.urandom(8).hex()}", } @@ -243,13 +238,14 @@ def build_sampling_params( {key: sampling[key] for key in _SAMPLING_KEYS if sampling.get(key) is not None} ) kwargs["n"] = n + # As vLLM's get_max_tokens: the generation config's max_tokens applies only + # when the request leaves max_tokens unset. + requested_max_tokens = stop.get("max_tokens") + if requested_max_tokens is None: + requested_max_tokens = defaults.get("max_tokens") kwargs["max_tokens"] = min( limit - for limit in ( - max(1, output_room), - stop.get("max_tokens"), - defaults.get("max_tokens"), - ) + for limit in (max(1, output_room), requested_max_tokens) if limit is not None ) if stop.get("min_tokens") is not None: @@ -354,7 +350,12 @@ async def start(self, worker_id: int) -> EngineConfig: for group in await self._llm.engine_core.call_utility_async( "get_kv_cache_group_metadata" ): - if group.get("kind") in ("full_attention", "mla_attention"): + # vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them. + if group.get("kind") in ( + KVCacheSpecKind.FULL_ATTENTION, + KVCacheSpecKind.MLA_ATTENTION, + KVCacheSpecKind.SINK_FULL_ATTENTION, + ): block_size = group["block_size"] break SnapshotStatLogger.num_gpu_blocks = cache.num_gpu_blocks or 0 @@ -378,6 +379,17 @@ async def start(self, worker_id: int) -> EngineConfig: async def generate( self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + try: + async for chunk in self._generate(request, context): + yield chunk + except VLLMClientError as error: + # vLLM's request errors are not ValueErrors; the native server + # answers them with a 4xx that carries the message. + raise InvalidArgument(str(error)) from error + + async def _generate( + self, request: GenerateRequest, context: Context ) -> AsyncGenerator[GenerateChunk, None]: if request.get("prompt_embeds"): raise InvalidArgument( @@ -655,6 +667,11 @@ async def _generate_gym( final_output = output if final_output is None: raise RuntimeError(f"No output received for request {context.id()}") + if admission is not None and context.is_stopped(): + # Dynamo aborts the vLLM request on a disconnect but never + # cancels this generator: fail the capture (below), as the + # native endpoint's cancellation does, instead of staging it. + return completion = final_output.outputs[0] chunk: GenerateChunk = { "token_ids": list(completion.token_ids), @@ -771,16 +788,24 @@ def _check_prompt_fits(self, prompt_len: int) -> None: "output tokens." ) - @staticmethod - def _gym_token_data(final_output: Any) -> dict[str, Any]: - """The token arrays Gym's dialect lifts onto the message, as the native endpoint reports them.""" + def _gym_token_data(self, final_output: Any) -> dict[str, Any]: + """The token fields Gym's dialect lifts onto the message, as the native endpoint reports them.""" + completion = final_output.outputs[0] token_data: dict[str, Any] = { "prompt_token_ids": list(final_output.prompt_token_ids), - "completion_token_ids": list(final_output.outputs[0].token_ids), + "completion_token_ids": list(completion.token_ids), } - if final_output.outputs[0].logprobs is not None: - _, token_data["completion_logprobs"] = chat_generation_tokens( - final_output.outputs[0] + if completion.logprobs is not None: + _, token_data["completion_logprobs"] = chat_generation_tokens(completion) + if self._host._return_routed_experts_enabled(): + # Full prompt + generation rows, as the native endpoint attaches them. + token_data["routed_experts"] = chat_routed_experts( + final_output, + completion, + prompt_token_count=len(final_output.prompt_token_ids), + device=torch.device("cpu"), + logger=LOGGER, + routed_experts_dtype=self._host.routed_experts_dtype, ) return token_data @@ -826,7 +851,7 @@ async def kv_event_sources(self) -> list[KvEventSource]: kv_events = self._llm.vllm_config.kv_events_config if kv_events is None or not kv_events.enable_kv_cache_events: return [] - return [ZmqSource(endpoint=kv_events.endpoint.replace("*", "127.0.0.1"))] + return [ZmqSource(endpoint=kv_events.endpoint)] def component_metrics_dp_ranks(self) -> list[int]: return [0] diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py index 82f5b897e77..95beb10d319 100644 --- a/nemo_rl/models/generation/dynamo/rollout_client.py +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -111,4 +111,12 @@ async def post_rollout( format_dynamo_error(response), ) await asyncio.sleep(_HTTP_RETRY_DELAY_S) + if "timeout_error" in response: + # Name the setting, as the native path's timeout message names its own. + raise RuntimeError( + f"Dynamo rollout request to {url} failed: {response['timeout_error']}. " + "For longer sequences, increase " + "policy.generation.dynamo_cfg.request_timeout_s, e.g. to " + f"{int(timeout_s * 2)}." + ) return parse_rollout_response(response, url=url) diff --git a/nemo_rl/models/generation/dynamo/venv.py b/nemo_rl/models/generation/dynamo/venv.py index 9a60b194e53..5708c8d0676 100644 --- a/nemo_rl/models/generation/dynamo/venv.py +++ b/nemo_rl/models/generation/dynamo/venv.py @@ -53,12 +53,15 @@ def get_dynamo_service_executable(name: str) -> str: if executable.is_file() and os.access(executable, os.X_OK): return str(executable) raise FileNotFoundError( - f"{name} is not an executable file in NEMO_RL_DYNAMO_BIN_DIR={bin_dir}." + f"{name} is not an executable file in NEMO_RL_DYNAMO_BIN_DIR={bin_dir}. " + "Build the image with --build-arg BUILD_DYNAMO=1, or install etcd and " + "nats-server there with docker/dynamo/install.sh." ) found = shutil.which(name) if found is None: raise FileNotFoundError( - f"{name} is not on PATH. Install it, or set NEMO_RL_DYNAMO_BIN_DIR to " - "the directory that holds etcd and nats-server." + f"{name} is not on PATH. Install etcd and nats-server with " + "docker/dynamo/install.sh and set NEMO_RL_DYNAMO_BIN_DIR to their " + "directory, or build the image with --build-arg BUILD_DYNAMO=1." ) return found diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 7dee0203c6f..1f6467032e1 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -56,15 +56,15 @@ VllmConfig, validate_vllm_quantization_config, ) -from nemo_rl.models.generation.vllm.request_pipeline import ( - rollout_output, - rollout_result_from_wire, -) from nemo_rl.models.generation.vllm.draft_utils import ( DRAFT_DISABLE_MODULE_SHARING_ENV, draft_module_sharing_disable_required, ) from nemo_rl.models.generation.vllm.metric_names import BATCH_DURATION_KEY +from nemo_rl.models.generation.vllm.request_pipeline import ( + rollout_output, + rollout_result_from_wire, +) from nemo_rl.models.generation.vllm.utils import ( aggregate_spec_decode_counters, assert_reload_refit_config_supported, @@ -188,10 +188,6 @@ def validate_settings(cls, master_config: "MasterConfig") -> None: generation_config = cast(VllmConfig, master_config.policy["generation"]) assert_reload_refit_config_supported(generation_config) if dynamo_enabled(generation_config): - # Deferred: the NeMo-Gym environment module is heavy and optional. - from nemo_rl.environments.nemo_gym import should_use_nemo_gym - from nemo_rl.models.megatron.router_replay import router_replay_enabled - if master_config.async_rl.generation_router.enabled: raise ValueError( "async_rl.generation_router.enabled=true does not apply with " @@ -206,17 +202,6 @@ def validate_settings(cls, master_config: "MasterConfig") -> None: "picks engines itself, and a restarted shard does not register " "with it. Disable fleet health." ) - if ( - should_use_nemo_gym(master_config) - and router_replay_enabled(master_config.policy) - and not master_config.token_capture.enabled - ): - raise ValueError( - "Router replay with NeMo-Gym and policy.generation.dynamo=true " - "requires token_capture.enabled=true: the engine stages routed " - "experts with the captured call, and NeMo-Gym's Dynamo dialect " - "does not return them in responses." - ) @staticmethod def init_cluster_placement_groups( diff --git a/ray.sub b/ray.sub index eb5b03a2a5f..1b9166fef64 100644 --- a/ray.sub +++ b/ray.sub @@ -189,8 +189,7 @@ fi # 1400-1999 Master address / TCPStore (cluster.master_port_range_low/high) # 2000-2999 Ray worker gRPC (min/max-worker-port) # [3000, 4999) Shared NeMo RL generation range (policy.generation.port_range_low/high) -# [3000, 4000) Dynamo frontend/token-wrapper HTTP endpoints -# [4000, 4100) Dynamo worker system endpoints (node-local free-port selection) +# [3000, 4000) Dynamo frontend HTTP endpoint # 5000-5999 NeMo Gym HTTP servers (Gym global config port_range_low/high) # 6000 Sandbox Nginx (NEMO_SKILLS_SANDBOX_PORT) # 6001-6999 Sandbox uWSGI workers (SANDBOX_BASE_PORT) diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index 225bbb65919..23d08060f17 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -37,6 +37,8 @@ run_test() { # Each keeps the native test's checks and adds check_dynamo_overlay.sh's. run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true ++policy.generation.dynamo=true +# Gym's default mode: token arrays come back through the frontend in engine_data. +run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true # Router replay with deferred routes: the engine stages each captured call's routed experts. run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index ba6edd103e9..5de0f0ef08a 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -40,7 +40,12 @@ def _vllm_config(**overrides) -> dict: "model_name": "Qwen/Qwen3-0.6B", "stop_token_ids": None, "stop_strings": None, - "vllm_cfg": {"async_engine": False, "max_model_len": 512}, + "vllm_cfg": { + "async_engine": False, + "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 1, + }, } config.update(overrides) return config @@ -76,6 +81,19 @@ def test_without_dynamo_the_vllm_config_is_untouched() -> None: (_vllm_config(dynamo=True, backend="sglang"), "backend=vllm"), # dynamo_cfg alone would be ignored. (_vllm_config(dynamo_cfg={}), "dynamo is not true"), + # The async engine Dynamo serves runs no data parallelism inside vLLM. + ( + _vllm_config( + dynamo=True, + vllm_cfg={ + "async_engine": False, + "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 2, + }, + ), + "expert_parallel_size <= tensor_parallel_size", + ), ], ) def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( @@ -133,6 +151,8 @@ def _serving(**serving_chat_kwargs) -> dict: vllm_cfg={ "async_engine": True, "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 1, "http_server_serving_chat_kwargs": serving_chat_kwargs, }, ) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 0b56635518c..d9cd8990bd8 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -132,7 +132,15 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( assert "NATS_STALE_SETTING" not in service_env assert service_env["NCCL_DEBUG"] == "INFO" + runtime._started = True + engine_env = runtime.engine_env + assert engine_env["DYN_NAMESPACE"] == "managed" + # Each engine serves its own model card, so a frontend on another node can + # read a chat template that exists only on the engine's node. + assert engine_env["DYN_SYSTEM_PORT"] == "0" + frontend_env = runtime._frontend_env() + assert "DYN_SYSTEM_PORT" not in frontend_env assert frontend_env["DYN_TOKENIZER"] == "fastokens" assert frontend_env["DYN_TOKENIZER_CACHE"] == "1" assert frontend_env["DYN_TOKENIZER_CACHE_BYTES"] == "4096" diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index f6ad9727175..63388e59ade 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -46,6 +46,7 @@ derive_prefix_eos_count, splice_prefix_tokens, ) +from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, @@ -229,7 +230,7 @@ def test_rollouts_are_served_like_generate_async() -> None: } -def test_router_replay_routes_reach_the_driver_as_generate_async_returns_them() -> None: +def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None: routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) final = _output( [1, 2, 3], @@ -265,6 +266,29 @@ def test_router_replay_routes_reach_the_driver_as_generate_async_returns_them() for key in native: assert torch.equal(shipped[key], native[key]), key + # A Gym call without capture carries the routes in engine_data, as the + # native chat endpoint attaches them to the response. + gym_final = _output( + [1, 3, 4], + _completion([10, 11], "stop", routed_experts=routes[3:]), + prompt_routed_experts=routes[:3], + ) + gym_host = _host( + _FakeLLM([gym_final]), vllm_kwargs={"enable_return_routed_experts": True} + ) + gym_engine = native_engine.NativeVllmEngine(gym_host, leader_worker_idx=0) + + (gym_chunk,) = _collect(gym_engine, _gym_request()) + + assert gym_chunk["engine_data"]["routed_experts"] == chat_routed_experts( + gym_final, + gym_final.outputs[0], + prompt_token_count=3, + device=torch.device("cpu"), + logger=native_engine.LOGGER, + routed_experts_dtype=torch.int16, + ) + def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: llm = _FakeLLM() @@ -300,7 +324,13 @@ def test_openai_requests_map_onto_sampling_params_as_vllms_server_does() -> None ) assert (params.temperature, params.top_k, params.seed, params.n) == (0.6, 5, 7, 2) - assert params.max_tokens == 9 + # The request's max_tokens, capped by the context; the generation config's + # max_tokens applies only when the request leaves it unset. + assert params.max_tokens == MAX_MODEL_LEN - 4 + unset = native_engine.build_sampling_params( + _openai_request(), {"max_tokens": 9}, output_room=MAX_MODEL_LEN - 4 + ) + assert unset.max_tokens == 9 assert sorted(params.stop_token_ids) == [2, 9] assert params.logprobs == 2 assert params.allowed_token_ids == [5, 6] @@ -502,6 +532,33 @@ async def registration_fails() -> None: ] +def test_engines_register_the_block_size_kv_events_are_hashed_with() -> None: + llm = _FakeLLM() + llm.vllm_config.cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=64) + llm.vllm_config.scheduler_config = SimpleNamespace( + max_num_seqs=8, max_num_batched_tokens=256 + ) + + async def call_utility_async(method): + assert method == "get_kv_cache_group_metadata" + # A hybrid model: the Mamba group's block size is not the attention one. + return [ + {"kind": "mamba", "block_size": 1056}, + {"kind": "full_attention", "block_size": 528}, + ] + + llm.engine_core = SimpleNamespace(call_utility_async=call_utility_async) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + registration = asyncio.run(engine.start(worker_id=0)).llm + + # The KV router hashes events in blocks of the main-attention group; a + # wrong size would quietly turn KV routing into load-only routing. + assert registration.kv_cache_block_size == 528 + assert registration.total_kv_blocks == 64 + assert registration.context_length == MAX_MODEL_LEN + + def test_a_prompt_that_fills_the_context_is_refused_as_natively() -> None: llm = _FakeLLM() engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) @@ -559,6 +616,9 @@ def test_media_is_loaded_and_processed_as_vllms_chat_parser_does() -> None: }, "Cannot load image_url", ), + # vLLM's own request errors (not ValueErrors) keep their message, as the + # native server's 4xx does, instead of becoming a bare 500. + ({"stop_conditions": {"min_tokens": 10, "max_tokens": 5}}, "min_tokens"), ], ) def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: @@ -742,5 +802,6 @@ def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( assert (config is not None) is publishes if publishes: assert config["enable_kv_cache_events"] and config["publisher"] == "zmq" - # vLLM's publisher binds only a wildcard endpoint; Dynamo's source connects. - assert config["endpoint"].startswith("tcp://*:") + # A per-engine abstract socket, which vLLM's publisher binds and Dynamo's + # source connects to: no port for two engines on a node to collide on. + assert config["endpoint"].startswith("ipc://@") diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py index bb1a00de798..9c509a10fa0 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine_capture.py +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -27,6 +27,8 @@ pytestmark = [pytest.mark.nemo_gym, pytest.mark.vllm] pytest.importorskip("dynamo._core") pytest.importorskip("vllm") +# The vLLM lanes import this module without the nemo_gym extra. +pytest.importorskip("nemo_gym.token_id_capture.staging") from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 @@ -35,6 +37,9 @@ DYNAMO_PREFIX_ANNOTATION_PREFIX, ) from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 +from responses_api_models.vllm_model.app import ( # noqa: E402 + _DYNAMO_ENGINE_DATA_FIELDS, +) from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 @@ -46,12 +51,12 @@ IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS, - _image_url, _collect, _completion, _FakeLLM, _gym_request, _host, + _image_url, _output, _Tokenizer, ) @@ -100,7 +105,7 @@ def _captured_request(admission: CaptureAdmission, **fields) -> dict: return request -def test_annotations_are_the_ones_gyms_dynamo_dialect_sends() -> None: +def test_annotations_and_engine_data_are_the_ones_gyms_dynamo_dialect_uses() -> None: assert native_engine.GYM_ROLLOUT_ANNOTATION == DYNAMO_GYM_REQUEST_ANNOTATION assert native_engine.GYM_CAPTURE_ANNOTATION_PREFIX == ( DYNAMO_CAPTURE_ANNOTATION_PREFIX @@ -108,6 +113,10 @@ def test_annotations_are_the_ones_gyms_dynamo_dialect_sends() -> None: assert native_engine.GYM_PREFIX_ANNOTATION_PREFIX == ( DYNAMO_PREFIX_ANNOTATION_PREFIX ) + # The engine_data keys Gym requires on a call without capture. + engine = native_engine.NativeVllmEngine(_host(_FakeLLM()), leader_worker_idx=0) + final = _output([1, 3, 4], _completion([10], "stop")) + assert engine._gym_token_data(final).keys() == _DYNAMO_ENGINE_DATA_FIELDS.keys() def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: diff --git a/tests/unit/models/generation/test_dynamo_rollout_client.py b/tests/unit/models/generation/test_dynamo_rollout_client.py index ac072ad72ad..1b2b942dc7f 100644 --- a/tests/unit/models/generation/test_dynamo_rollout_client.py +++ b/tests/unit/models/generation/test_dynamo_rollout_client.py @@ -11,13 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import json import pytest +from nemo_rl.models.generation.dynamo import rollout_client from nemo_rl.models.generation.dynamo.rollout_client import ( ROLLOUT_ANNOTATION, parse_rollout_response, + post_rollout, rollout_payload, ) @@ -58,10 +61,51 @@ def test_parse_rollout_response_rejects_unusable_responses(response, match) -> N parse_rollout_response(response, url="http://frontend/v1/completions") -def test_parse_rollout_response_returns_engine_data() -> None: - engine_data = {"rollout": {"token_ids": []}, "gen_leader_worker_idx": 1} +_ENGINE_DATA = {"rollout": {"token_ids": [8]}, "gen_leader_worker_idx": 1} - assert ( - parse_rollout_response({"nvext": {"engine_data": engine_data}}, url="u") - == engine_data - ) + +@pytest.mark.parametrize( + ("responses", "error"), + [ + # A transient failure is retried, and the rollout still succeeds. + ( + [ + {"status": "error", "http_status": 503}, + {"nvext": {"engine_data": _ENGINE_DATA}}, + ], + None, + ), + # A request the frontend refuses is final. + ([{"status": "error", "http_status": 400}], "HTTP 400"), + # A timeout is final, and names the setting that bounds it. + ( + [{"status": "error", "timeout_error": "no response within 1.0s"}], + "no response.*dynamo_cfg.request_timeout_s", + ), + # Retries are bounded. + ( + [{"status": "error", "http_status": 503}] + * rollout_client._HTTP_MAX_ATTEMPTS, + "HTTP 503", + ), + ], +) +def test_post_rollout_retries_only_transient_failures( + monkeypatch, responses, error +) -> None: + calls = [] + + async def fake_post(url, payload, timeout_s): + calls.append(payload) + return responses[len(calls) - 1] + + monkeypatch.setattr(rollout_client, "async_http_post_json", fake_post) + monkeypatch.setattr(rollout_client, "_HTTP_RETRY_DELAY_S", 0.0) + rollout = post_rollout("http://frontend/v1/completions", {}, timeout_s=1.0) + + if error is None: + assert asyncio.run(rollout) == _ENGINE_DATA + else: + with pytest.raises(RuntimeError, match=error): + asyncio.run(rollout) + assert len(calls) == len(responses) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index f3d4b7720c0..195a8bb3911 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2427,61 +2427,78 @@ def test_vllm_validate_settings_accepts_missing_vllm_cfg_as_refit_disabled(): VllmGeneration.validate_settings(master_config) -def _dynamo_master_config( - *, - router=False, - fleet_health=False, - nemo_gym=True, - router_replay=False, - token_capture=False, -): +def _dynamo_master_config(*, router=False, fleet_health=False): vllm_config = deepcopy(basic_vllm_test_config) vllm_config["dynamo"] = True vllm_config["vllm_cfg"]["async_engine"] = True vllm_config["vllm_cfg"]["expose_http_server"] = True return types.SimpleNamespace( - policy={"generation": vllm_config, "router_replay": {"enabled": router_replay}}, - env={"should_use_nemo_gym": nemo_gym}, + policy={"generation": vllm_config}, async_rl=types.SimpleNamespace( generation_router=types.SimpleNamespace(enabled=router), generation_fleet_health=types.SimpleNamespace(enabled=fleet_health), ), - token_capture=types.SimpleNamespace(enabled=token_capture), - ) - - -def test_vllm_validate_settings_accepts_dynamo_with_nemo_gym(): - VllmGeneration.validate_settings(_dynamo_master_config()) - VllmGeneration.validate_settings( - _dynamo_master_config(router_replay=True, token_capture=True) - ) - VllmGeneration.validate_settings( - _dynamo_master_config(router_replay=True, nemo_gym=False) ) @pytest.mark.parametrize( ("settings", "error"), [ + ({}, None), ({"router": True}, "routes the whole fleet behind one URL"), ({"fleet_health": True}, "Disable fleet health"), - ({"router_replay": True}, "requires token_capture.enabled=true"), ], ) def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error): + master_config = _dynamo_master_config(**settings) + if error is None: + VllmGeneration.validate_settings(master_config) + return with pytest.raises(ValueError, match=error): - VllmGeneration.validate_settings(_dynamo_master_config(**settings)) + VllmGeneration.validate_settings(master_config) @pytest.mark.parametrize("dynamo", [False, True]) -def test_vllm_client_base_urls_name_the_dynamo_frontend_when_it_serves(dynamo): +def test_vllm_token_rollouts_and_clients_use_the_dynamo_frontend_when_it_serves( + dynamo, +): generation = object.__new__(VllmGeneration) generation.dp_openai_server_base_urls = ["http://a/v1", "http://b/v1"] generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None + tokens = BatchedDataDict({"input_ids": torch.tensor([[1]])}) + # The frontend request carries token IDs only, so media prompts stay native. + media = BatchedDataDict({"input_ids": torch.tensor([[1]]), "vllm_content": [None]}) assert generation.client_base_urls == ( ["http://frontend/v1"] if dynamo else ["http://a/v1", "http://b/v1"] ) + assert generation._dynamo_serves(tokens) is dynamo + assert not generation._dynamo_serves(media) + + +def test_vllm_dynamo_frontend_failures_are_generation_unavailable(monkeypatch): + from nemo_rl.experience.failures import GenerationUnavailable + from nemo_rl.models.generation.dynamo import rollout_client + + async def frontend_down(url, payload, timeout_s): + raise RuntimeError(f"Dynamo rollout request to {url} failed: HTTP 503") + + monkeypatch.setattr(rollout_client, "post_rollout", frontend_down) + generation = object.__new__(VllmGeneration) + generation.cfg = { + "model_name": "model", + "_pad_token_id": 0, + "vllm_cfg": {"max_model_len": 16}, + } + generation.dynamo_frontend_url = "http://frontend/v1" + data = BatchedDataDict( + {"input_ids": torch.tensor([[5, 6, 0]]), "input_lengths": torch.tensor([2])} + ) + + # An infrastructure failure is re-dispatched; an unmapped RuntimeError would + # count as a DATA failure against the prompt. + with pytest.raises(GenerationUnavailable, match="HTTP 503"): + asyncio.run(anext(generation.generate_async(data))) def test_vllm_policy_generation(policy, test_input_data, tokenizer): From 2ba2caeecdd32f9bac31bb36e7426a02c773efb9 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 24 Oct 2026 17:27:00 -0700 Subject: [PATCH 03/82] style(dynamo): sort the test's chat_routed_experts import Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/models/generation/test_dynamo_native_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index 63388e59ade..836b70c5ee5 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -46,13 +46,13 @@ derive_prefix_eos_count, splice_prefix_tokens, ) -from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, rollout_result_from_wire, rollout_result_to_wire, ) +from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) From 94c0e96e2b4849edc88dd5786335920be4b0fe88 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 26 Oct 2026 14:19:32 -0700 Subject: [PATCH 04/82] build(gym): pin Gym#4817 rebased onto Gym main Gym#4817 (jthomson04/dynamo-frontend-dialect) is rebased onto Gym main and now sits on top of it; its head moves from a369e28 to a5f936c. The only conflict was Gym main's new custom-tool-strategy validator, which sits next to the Dynamo frontend validator; both are kept. Gym's dependencies are unchanged, so uv.lock needs no change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index a369e289f70..a5f936c473d 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit a369e289f702c623b077e811eceeb98f2b3b26cf +Subproject commit a5f936c473dbd477ce4632373eaba884272b7f6a From e1828110d0e2a4e91ef8cf7629924641746583e9 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 26 Oct 2026 15:50:32 -0700 Subject: [PATCH 05/82] fix(dynamo): apply the Gym Dynamo dialect per actor, after shard composition From the #4905 review: - C23: setup_nemo_gym_config wrote the dialect as a top-level policy_model overlay, which a sharded Gym plan rejects as unowned. Setup now stashes it under a NeMo RL key (dynamo_dialect, alongside tokenizer_config), and _build_gym_actor_config applies it to each actor's composed config. A shard's own template kwargs still win over the native defaults. - C24: the local install instructions export NEMO_RL_DYNAMO_BIN_DIR, so the run that follows finds etcd and nats-server. - C25: the two in-function rationale blocks in native_engine.py are one line each; the design doc carries the full reasons. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 11 +++- docs/guides/dynamo-generation.md | 6 +- nemo_rl/environments/nemo_gym.py | 65 ++++++++++++------ nemo_rl/environments/nemo_gym_shards.py | 7 +- .../models/generation/dynamo/native_engine.py | 12 +--- .../test_nemo_gym_token_capture.py | 36 +++++++--- .../unit/environments/test_nemo_gym_utils.py | 66 +++++++++++++++++++ 7 files changed, 158 insertions(+), 45 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 0fea1ccc5fd..843dad2c897 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -52,12 +52,17 @@ implements its `LLMEngine` over the worker's `AsyncLLM`: - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It publishes no token budget. The engine owns the context check and the output - cap, because a prefix splice changes the prompt after the frontend has seen it. + cap, as the native server does. With a budget, the frontend would fill an + omitted `max_tokens` from the prompt it rendered, which a prefix splice then + changes, and would refuse NeMo RL's own rollouts that fill the context. - `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ publisher. It is set only when the engine caches prefixes. Each engine binds its own Linux abstract-namespace socket (`ipc://@...`), so engines on one node - need no port and cannot collide. A vLLM stat logger publishes the router's - per-engine load signal. + need no port and cannot collide. The name is random rather than the pid, + because containers that share the host network share these names. vLLM refuses + `ipc://` only above data-parallel rank 0, and vLLM data parallelism is rejected + at startup with Dynamo. A vLLM stat logger publishes the router's per-engine + load signal. Because the Worker runs on the actor's own loop, the worker's other methods interleave with Dynamo requests exactly as with native requests: diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index bc554ec422f..f1764202897 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -173,10 +173,12 @@ docker buildx build \ . ``` -For a local checkout, install the binaries with: +For a local checkout, install the binaries and keep the variable set for the +runs that follow. The runtime looks for etcd and nats-server there, or else on `PATH`: ```bash -NEMO_RL_DYNAMO_BIN_DIR=$PWD/venvs/dynamo-bin bash docker/dynamo/install.sh +export NEMO_RL_DYNAMO_BIN_DIR="$PWD/venvs/dynamo-bin" +bash docker/dynamo/install.sh ``` ## Run diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 017f5d152dd..3a47fbeb847 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -1897,27 +1897,14 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_token_ids"] = None if dynamo_enabled(generation_config): - # Gym's policy model calls the Dynamo frontend, which takes token data in - # nvext (Gym's Dynamo dialect) instead of vLLM's request extensions. - vllm_model = ( - config.env.setdefault("nemo_gym", {}) - .setdefault("policy_model", {}) - .setdefault("responses_api_models", {}) - .setdefault("vllm_model", {}) - ) - vllm_model["dynamo_frontend"] = True - # The frontend reads `reasoning` as another name for `reasoning_content` and - # refuses an echoed assistant turn that carries both, Gym's default. - vllm_model.setdefault("reasoning_field", "reasoning_content") - # The native server merges these under every request's own template - # kwargs; the frontend has no such default, so Gym sends them. They sit - # under the kwargs Gym sends: vllm_model's own, else extra_body's. - default_kwargs = native_default_chat_template_kwargs(generation_config) - if default_kwargs: - gym_kwargs = vllm_model.get("chat_template_kwargs") or ( - (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} + # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect. + # Each actor applies it to its own composed config (_apply_dynamo_dialect), + # so it never becomes a policy_model overlay that a shard must claim. + config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = { + "default_chat_template_kwargs": native_default_chat_template_kwargs( + generation_config ) - vllm_model["chat_template_kwargs"] = {**default_kwargs, **gym_kwargs} + } # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for @@ -1987,6 +1974,39 @@ def build_nemo_gym_config( ) +def _apply_dynamo_dialect( + nemo_gym_dict: dict[str, Any], default_chat_template_kwargs: Mapping[str, Any] +) -> None: + """Point one actor's policy model at the Dynamo frontend, in Gym's Dynamo dialect. + + The frontend takes token data in nvext instead of vLLM's request extensions. + Applied to the actor's composed config (a shard's overrides included), so + settings the user wrote there win. Copies the policy_model levels it + changes, so the caller's config is not mutated. + """ + policy_model = dict(nemo_gym_dict.get("policy_model") or {}) + models = dict(policy_model.get("responses_api_models") or {}) + vllm_model = dict(models.get("vllm_model") or {}) + vllm_model["dynamo_frontend"] = True + # The frontend reads `reasoning` as another name for `reasoning_content` and + # refuses an echoed assistant turn that carries both, Gym's default. + vllm_model.setdefault("reasoning_field", "reasoning_content") + # The native server merges these under every request's own template + # kwargs; the frontend has no such default, so Gym sends them. They sit + # under the kwargs Gym sends: vllm_model's own, else extra_body's. + if default_chat_template_kwargs: + gym_kwargs = vllm_model.get("chat_template_kwargs") or ( + (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} + ) + vllm_model["chat_template_kwargs"] = { + **default_chat_template_kwargs, + **gym_kwargs, + } + models["vllm_model"] = vllm_model + policy_model["responses_api_models"] = models + nemo_gym_dict["policy_model"] = policy_model + + def _build_gym_actor_config( nemo_gym_dict: dict[str, Any], *, @@ -2015,6 +2035,11 @@ def _build_gym_actor_config( invalid_tool_call_patterns = nemo_gym_dict.pop("invalid_tool_call_patterns", None) thinking_tags = nemo_gym_dict.pop("thinking_tags", None) tokenizer_config = nemo_gym_dict.pop("tokenizer_config", None) + dynamo_dialect = nemo_gym_dict.pop("dynamo_dialect", None) + if dynamo_dialect is not None: + _apply_dynamo_dialect( + nemo_gym_dict, dynamo_dialect["default_chat_template_kwargs"] + ) external_service_readiness = extract_external_service_readiness(nemo_gym_dict) port_range = { key: value diff --git a/nemo_rl/environments/nemo_gym_shards.py b/nemo_rl/environments/nemo_gym_shards.py index d8322a8d23c..b03b9cd9eb6 100644 --- a/nemo_rl/environments/nemo_gym_shards.py +++ b/nemo_rl/environments/nemo_gym_shards.py @@ -38,7 +38,12 @@ # Dict-shaped NeMo RL integration settings that are not Gym server entries. # The actor factory or rollout code consumes these separately. NEMO_RL_DICT_CONFIG_KEYS = frozenset( - {"effort_levels", "external_service_readiness", "tokenizer_config"} + { + "dynamo_dialect", + "effort_levels", + "external_service_readiness", + "tokenizer_config", + } ) # Ray placement-group strategies a shard plan may ask for. STRICT_SPREAD is the diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index 39bff56a0fb..b3221afcf53 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -149,11 +149,7 @@ def dynamo_kv_events_config( return { "enable_kv_cache_events": True, "publisher": "zmq", - # A Linux abstract-namespace socket (no port, no file): vLLM binds any - # ipc:// endpoint and this process's Dynamo worker connects to it, so - # engines on one node cannot collide. Random rather than the pid, since - # containers sharing the host network share these names. vLLM rejects - # ipc:// only at data_parallel_rank != 0, which Dynamo rejects at startup. + # A random abstract-namespace socket per engine (dynamo-integration.md). "endpoint": f"ipc://@nemo-rl-kv-events-{os.urandom(8).hex()}", } @@ -363,11 +359,7 @@ async def start(self, worker_id: int) -> EngineConfig: return EngineConfig( model=model, served_model_name=model, - # No token budget: the engine owns prompt overflow and the output - # cap, as the native server does. A budget would make the frontend - # fill an omitted max_tokens from the prompt it rendered, which a - # NeMo-Gym prefix splice then changes, and refuse NeMo RL's own - # rollouts that fill the context. + # No token budget: the engine owns overflow (dynamo-integration.md). llm=LlmRegistration( context_length=self._max_model_len, kv_cache_block_size=block_size, diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index 3469ac88884..ac3707a315d 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -604,12 +604,11 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( assert vllm_cfg["async_engine"] is True assert vllm_cfg["expose_http_server"] is True assert should_use_nemo_gym(config) is True - policy_model = config.env["nemo_gym"].get("policy_model") if dynamo else None if dynamo: - assert policy_model["responses_api_models"]["vllm_model"] == { - "dynamo_frontend": True, - # The frontend refuses reasoning under both of Gym's keys. - "reasoning_field": "reasoning_content", + # A NeMo RL key each actor applies to its composed config, not a + # policy_model overlay a sharded plan would need an owner for. + assert config.env["nemo_gym"] == { + "dynamo_dialect": {"default_chat_template_kwargs": {}} } else: assert "nemo_gym" not in config.env @@ -633,12 +632,17 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( ), ], ) -def test_setup_nemo_gym_config_dynamo_sends_the_native_default_template_kwargs( +def test_gym_actor_config_dynamo_sends_the_native_default_template_kwargs( vllm_model: dict, expected: dict ) -> None: """The native server's default template kwargs sit under the ones Gym sends.""" from nemo_rl.algorithms.grpo import MasterConfig - from nemo_rl.environments.nemo_gym import setup_nemo_gym_config + from nemo_rl.environments.nemo_gym import ( + build_nemo_gym_config, + setup_nemo_gym_config, + ) + + user_vllm_model = copy.deepcopy(vllm_model) config = MasterConfig.model_construct( env={ @@ -665,5 +669,19 @@ def test_setup_nemo_gym_config_dynamo_sends_the_native_default_template_kwargs( ) setup_nemo_gym_config(config, tokenizer=None) - - assert vllm_model["chat_template_kwargs"] == expected + gym_config = build_nemo_gym_config( + config.env, + base_urls=["http://frontend"], + model_name="test-model", + enable_router_replay=False, + use_fastokens=False, + )["initial_global_config_dict"] + + composed = gym_config["policy_model"]["responses_api_models"]["vllm_model"] + assert composed["dynamo_frontend"] is True + # The frontend refuses reasoning under both of Gym's keys. + assert composed["reasoning_field"] == "reasoning_content" + assert composed["chat_template_kwargs"] == expected + assert "dynamo_dialect" not in gym_config + # The user's overlay is left as written. + assert vllm_model == user_vllm_model diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index ea0b267beef..bea407f3213 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -932,6 +932,72 @@ def test_build_nemo_gym_actors_hands_every_actor_the_policy_api_key( assert all(cfg["policy_api_key"] == "per-job-key" for cfg in cluster.actor_configs) +@pytest.mark.parametrize( + "env_configs", [_env_configs, _shard_env_configs], ids=["unsharded", "sharded"] +) +def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( + detected_uv_dirs, env_configs +): + """With Dynamo, each actor's composed config speaks Gym's Dynamo dialect. + + The dialect is not a policy_model overlay, which a sharded plan would reject + as unowned. A shard's own template kwargs still win over the native defaults. + """ + from nemo_rl.algorithms.grpo import MasterConfig + + env = env_configs() + sharded = "shards" in env["nemo_gym"] + if sharded: + env["nemo_gym"]["shards"][1]["overrides"] = { + "policy_model": { + "responses_api_models": { + "vllm_model": {"chat_template_kwargs": {"enable_thinking": False}} + } + } + } + config = MasterConfig.model_construct( + env=env, + policy={ + "generation": { + "backend": "vllm", + "dynamo": True, + "vllm_cfg": { + "http_server_serving_chat_kwargs": { + "default_chat_template_kwargs": {"enable_thinking": True} + } + }, + } + }, + ) + setup_nemo_gym_config(config, tokenizer=None) + cluster = _FakeGymCluster() + + with _patched_cluster(cluster): + nemo_gym_mod.build_nemo_gym_actors( + config.env, + base_urls=["http://frontend"], + model_name="test-model", + tokenizer=_TOKENIZER, + enable_router_replay=False, + use_fastokens=False, + ) + + thinking = [ + cfg["initial_global_config_dict"]["policy_model"]["responses_api_models"][ + "vllm_model" + ]["chat_template_kwargs"]["enable_thinking"] + for cfg in cluster.actor_configs + ] + # judged, then the tools shard's two replicas with their own override. + assert thinking == ([True, False, False] if sharded else [True]) + for cfg in cluster.actor_configs: + gym_config = cfg["initial_global_config_dict"] + vllm_model = gym_config["policy_model"]["responses_api_models"]["vllm_model"] + assert vllm_model["dynamo_frontend"] is True + assert vllm_model["reasoning_field"] == "reasoning_content" + assert "dynamo_dialect" not in gym_config + + def test_build_nemo_gym_actors_spreads_every_replica_onto_its_own_node( detected_uv_dirs, ): From e0d3789f2d9bce3afd0ec9a1894cd27d87a49a02 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 09:39:37 -0700 Subject: [PATCH 06/82] refactor(vllm): share the per-sample rollout pipeline and chat token helpers Split generate_async's per-sample work into building the result (request_pipeline.rollout_result: sampled logprobs, router-replay route alignment and its fallback warning) and assembling the output tensors (rollout_output). The rollout token budget and SamplingParams become worker methods (_rollout_max_new_tokens, _rollout_sampling_params). vllm/utils gains off_policy_sampling_error (the chat endpoint's on-policy checks, unchanged) and chat_routed_experts / chat_generation_tokens (the loop bodies of the attach_* response helpers). No behaviour change for native vLLM: the new generate_async wiring tests pass against both the old and the new implementation. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/vllm/request_pipeline.py | 177 +++++++++ nemo_rl/models/generation/vllm/utils.py | 243 ++++++++---- .../generation/vllm/vllm_worker_async.py | 359 +++++------------- pyrefly.toml | 1 + .../models/generation/test_vllm_generation.py | 125 ++++++ .../generation/test_vllm_request_pipeline.py | 124 ++++++ .../unit/models/generation/test_vllm_utils.py | 27 ++ 7 files changed, 717 insertions(+), 339 deletions(-) create mode 100644 nemo_rl/models/generation/vllm/request_pipeline.py create mode 100644 tests/unit/models/generation/test_vllm_request_pipeline.py diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py new file mode 100644 index 00000000000..bb77e3b0e27 --- /dev/null +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -0,0 +1,177 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Turn a finished vLLM rollout into NeMo RL's per-sample generation output. + +Building the per-sample result is kept apart from assembling the output tensors, +so a path that builds the result on the engine and assembles the output +elsewhere behaves the same as the worker's own ``generate_async``. A rollout +feature added here, such as router replay (R3), then behaves the same on every +path. +""" + +import logging +from dataclasses import dataclass +from typing import Any, Optional, cast + +import torch + +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.models.generation.interfaces import GenerationOutputSpec +from nemo_rl.models.generation.vllm.utils import pad_and_align_routed_expert_indices + +LOGGER = logging.getLogger(__name__) + + +@dataclass +class RolloutResult: + """One rollout's outcome, independent of where its output is assembled.""" + + token_ids: list[int] + # The sampled token's log probability at each position; 0.0 where vLLM + # reported none. + logprobs: list[float] + # The engine stopped at max_tokens. + truncated: bool + # [prompt + generated, num_moe_layers, topk], aligned for router replay. + routed_experts: Optional[torch.Tensor] + # missing/expected/actual route counts; set when router replay is on. + r3_stats: Optional[dict[str, int]] + + @classmethod + def nothing_generated(cls) -> "RolloutResult": + """The result when no new token fits: the prompt alone, untruncated.""" + return cls([], [], False, None, None) + + +def selected_logprobs(completion: Any) -> list[float]: + """The sampled token's log probability at each position of a completion.""" + by_position = completion.logprobs or () + logprobs = [] + for idx, token_id in enumerate(completion.token_ids): + candidates = by_position[idx] if idx < len(by_position) else None + entry = candidates.get(token_id) if candidates else None + logprobs.append(0.0 if entry is None else float(entry.logprob)) + return logprobs + + +def rollout_result( + final_output: Any, + *, + input_length: int, + return_routed_experts: bool, + routed_experts_dtype: torch.dtype, + device: torch.device, + sample_idx: int = 0, +) -> RolloutResult: + """Build the result of a finished rollout from vLLM's final RequestOutput. + + Args: + final_output: The request's last (cumulative) ``RequestOutput``. + input_length: Prompt length in tokens. + return_routed_experts: Router replay is on, so routes are required. + routed_experts_dtype: The dtype router replay consumes. + device: Where to place the routed-experts tensor. + sample_idx: For log messages only. + """ + completion = final_output.outputs[0] + token_ids = list(completion.token_ids) + total_length = input_length + len(token_ids) + # return_stats=True returns the (routes, stats) pair. + routed_experts, r3_stats = cast( + tuple[Optional[torch.Tensor], dict[str, int]], + pad_and_align_routed_expert_indices( + final_output, + completion, + valid_length=total_length, + padded_length=total_length, + device=device, + require_complete_routed_experts=return_routed_experts, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ), + ) + if return_routed_experts and routed_experts is None: + raise RuntimeError( + "vLLM was asked to return routed experts but the generation output " + "did not include routed_experts." + ) + if return_routed_experts and r3_stats["missing_routes"] > 0: + LOGGER.warning( + "R3 router replay fallback: vLLM returned incomplete " + "routed_experts for sample_idx=%d, missing_token_routes=%d, " + "actual_routes=%d, expected_routes=%d. Megatron will use its " + "own router for those missing token routes.", + sample_idx, + r3_stats["missing_routes"], + r3_stats["actual_routes"], + r3_stats["expected_routes"], + ) + return RolloutResult( + token_ids=token_ids, + logprobs=selected_logprobs(completion), + truncated=completion.finish_reason == "length", + routed_experts=routed_experts, + r3_stats=r3_stats if return_routed_experts else None, + ) + + +def rollout_output( + result: RolloutResult, + *, + input_ids: torch.Tensor, + input_length: int, + pad_token_id: int, +) -> BatchedDataDict[GenerationOutputSpec]: + """The single-sample output ``generate_async`` yields for ``result``. + + Args: + result: The rollout's result. + input_ids: The sample's (right-padded) prompt row. + input_length: Prompt length in tokens. + pad_token_id: The tokenizer's pad token, as ``generate_async`` uses it. + """ + device = input_ids.device + generated = result.token_ids + output_length = input_length + len(generated) + output_ids = torch.full( + (output_length,), pad_token_id, dtype=input_ids.dtype, device=device + ) + output_ids[:input_length] = input_ids[:input_length] + output_ids[input_length:] = torch.tensor( + generated, dtype=input_ids.dtype, device=device + ) + logprobs = torch.zeros((1, output_length), dtype=torch.float32, device=device) + logprobs[0, input_length:] = torch.tensor( + result.logprobs, dtype=torch.float32, device=device + ) + output: dict[str, Any] = { + "output_ids": output_ids.unsqueeze(0), + "logprobs": logprobs, + "generation_lengths": torch.tensor( + [len(generated)], dtype=torch.long, device=device + ), + "unpadded_sequence_lengths": torch.tensor( + [output_length], dtype=torch.long, device=device + ), + "truncated": torch.tensor([result.truncated], dtype=torch.bool, device=device), + } + if result.r3_stats is not None: + for key in ("missing_routes", "expected_routes", "actual_routes"): + output[f"r3_routed_experts_{key}"] = torch.tensor( + [result.r3_stats[key]], dtype=torch.long, device=device + ) + if result.routed_experts is not None: + output["routed_experts"] = result.routed_experts.to(device).unsqueeze(0) + return BatchedDataDict[GenerationOutputSpec](output) diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index b77da9c6e6f..2d871819ae7 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -318,6 +318,57 @@ def _as_routed_experts_tensor( return tensor.to(dtype=dtype) +def off_policy_sampling_error( + *, + temperature: Optional[float], + top_p: Optional[float], + top_k: Optional[int], + generation_config: Any, +) -> Optional[str]: + """Why an HTTP rollout request would sample off-policy, or None if it would not. + + Request sampling must match NeMo RL's exactly, or inference runs off-policy and + destabilizes training. Validation rollouts are the one exception: they carry the + validation profile (``val_temperature`` / ``val_top_p``), which is metric-only. + Multi-turn agents issue their own requests, so the serving HTTP endpoint is the + one chokepoint they all pass. + + Args: + temperature: The request's temperature. + top_p: The request's top_p. It must be explicit: vLLM resolves an unset top_p + from the model's generation_config.json, not to 1.0. + top_k: The request's top_k, which must be unset or -1 (disabled), matching + ``BaseVllmGenerationWorker._build_sampling_params``. + generation_config: ``policy.generation``, with the train and validation + sampling parameters. + """ + if top_k not in (None, -1): + return f"Top k sampling parameter must be unset, empty, or -1. Got `{top_k}`" + if top_p is None: + return ( + "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " + "resolved by vLLM from the model's generation_config.json and would " + "bypass the on-policy sampling check." + ) + is_train_sampling = ( + temperature == generation_config["temperature"] + and top_p == generation_config["top_p"] + ) + is_val_sampling = ( + temperature == generation_config["val_temperature"] + and top_p == generation_config["val_top_p"] + ) + if is_train_sampling or is_val_sampling: + return None + return ( + f"request sampling (temperature={temperature}, top_p={top_p}) matches neither " + f"the train sampling params (temperature={generation_config['temperature']}, " + f"top_p={generation_config['top_p']}) nor the validation sampling params " + f"(val_temperature={generation_config['val_temperature']}, " + f"val_top_p={generation_config['val_top_p']})" + ) + + def validate_rollout_prompt(expected: list[int], actual: list[int] | None) -> None: """Reject rollouts generated from a different prompt than the learner's.""" if actual != expected: @@ -496,6 +547,58 @@ def pad_and_align_routed_expert_indices( return (full, stats) if return_stats else full +def chat_routed_experts( + final_request_output: Any, + generation_details: Any, + *, + prompt_token_count: int, + device: torch.device, + logger: Any, + routed_experts_dtype: torch.dtype, +) -> str: + """The routed-experts envelope the chat endpoint attaches to one choice.""" + generation_token_count = len(getattr(generation_details, "token_ids", []) or []) + routed_result = pad_and_align_routed_expert_indices( + final_request_output, + generation_details, + valid_length=prompt_token_count + generation_token_count, + padded_length=prompt_token_count + generation_token_count, + device=device, + require_complete_routed_experts=True, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ) + if not isinstance(routed_result, tuple): + raise RuntimeError( + "Expected routed_experts alignment to return stats for the " + "OpenAI-compatible chat endpoint." + ) + routed_experts, r3_stats = routed_result + if routed_experts is None: + raise RuntimeError( + "vLLM was asked to return routed experts for the " + "OpenAI-compatible chat endpoint but the generation " + "output did not include routed_experts." + ) + if r3_stats["missing_routes"] > 0 and logger is not None: + logger.warning( + "R3 router replay fallback: vLLM returned incomplete " + "routed_experts for chat choice_idx=%d, " + "missing_token_routes=%d, actual_routes=%d, " + "expected_routes=%d. Megatron will use its own router " + "for those missing token routes.", + generation_details.index, + r3_stats["missing_routes"], + r3_stats["actual_routes"], + r3_stats["expected_routes"], + ) + # Base64 envelope instead of .tolist(): nested JSON int lists cost + # ~1s of CPU per serialize/parse hop at long context lengths and get + # re-validated at every gym HTTP hop; a single string passes through + # the gym chain opaquely. + return encode_routed_experts(routed_experts.to(dtype=routed_experts_dtype)) + + def attach_routed_experts_to_chat_response_choices( response: Any, final_request_output: Any, @@ -519,49 +622,14 @@ def attach_routed_experts_to_chat_response_choices( if generation_details is None: continue attached_choice_indices.add(choice.index) - - generation_token_count = len(getattr(generation_details, "token_ids", []) or []) - routed_result = pad_and_align_routed_expert_indices( + choice.message.routed_experts = chat_routed_experts( final_request_output, generation_details, - valid_length=prompt_token_count + generation_token_count, - padded_length=prompt_token_count + generation_token_count, + prompt_token_count=prompt_token_count, device=device, - require_complete_routed_experts=True, - return_stats=True, + logger=logger, routed_experts_dtype=routed_experts_dtype, ) - if not isinstance(routed_result, tuple): - raise RuntimeError( - "Expected routed_experts alignment to return stats for the " - "OpenAI-compatible chat endpoint." - ) - routed_experts, r3_stats = routed_result - if routed_experts is None: - raise RuntimeError( - "vLLM was asked to return routed experts for the " - "OpenAI-compatible chat endpoint but the generation " - "output did not include routed_experts." - ) - if r3_stats["missing_routes"] > 0 and logger is not None: - logger.warning( - "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for chat choice_idx=%d, " - "missing_token_routes=%d, actual_routes=%d, " - "expected_routes=%d. Megatron will use its own router " - "for those missing token routes.", - choice.index, - r3_stats["missing_routes"], - r3_stats["actual_routes"], - r3_stats["expected_routes"], - ) - # Base64 envelope instead of .tolist(): nested JSON int lists cost - # ~1s of CPU per serialize/parse hop at long context lengths and get - # re-validated at every gym HTTP hop; a single string passes through - # the gym chain opaquely. - choice.message.routed_experts = encode_routed_experts( - routed_experts.to(dtype=routed_experts_dtype) - ) if len(attached_choice_indices) != len(choices): missing_choice_indices = sorted( @@ -623,50 +691,9 @@ def attach_token_information_to_chat_response_choices( ) for choice in choices: - generation_details = outputs_by_index[choice.index] - output_token_ids = getattr(generation_details, "token_ids", None) - if output_token_ids is None: - raise RuntimeError( - "vLLM was asked to return token information for the " - "OpenAI-compatible chat endpoint but generation output " - f"choice_idx={choice.index} did not include token_ids." - ) - generation_token_ids = list(output_token_ids) - - generation_logprob_details = getattr(generation_details, "logprobs", None) - if generation_logprob_details is None: - if generation_token_ids: - raise RuntimeError( - "vLLM was asked to return token information for the " - "OpenAI-compatible chat endpoint but generation output " - f"choice_idx={choice.index} did not include logprobs." - ) - generation_log_probs = [] - else: - if len(generation_token_ids) != len(generation_logprob_details): - raise RuntimeError( - "vLLM returned mismatched generation token IDs and log " - "probabilities for the OpenAI-compatible chat endpoint: " - f"choice_idx={choice.index}, " - f"token_count={len(generation_token_ids)}, " - f"logprob_count={len(generation_logprob_details)}." - ) - generation_log_probs = [] - for token_id, position_logprobs in zip( - generation_token_ids, generation_logprob_details - ): - selected_token_logprob = position_logprobs.get(token_id) - if selected_token_logprob is None: - raise RuntimeError( - "vLLM generation log probabilities did not include the " - "selected token while attaching token information to " - "the OpenAI-compatible chat response: " - f"choice_idx={choice.index}, token_id={token_id}." - ) - generation_log_probs.append( - max(float(selected_token_logprob.logprob), VLLM_LOGPROB_FLOOR) - ) - + generation_token_ids, generation_log_probs = chat_generation_tokens( + outputs_by_index[choice.index] + ) choice.message.prompt_token_ids = list(prompt_token_ids) choice.message.generation_token_ids = generation_token_ids choice.message.generation_log_probs = generation_log_probs @@ -674,6 +701,58 @@ def attach_token_information_to_chat_response_choices( return response +def chat_generation_tokens(generation_details: Any) -> tuple[list[int], list[float]]: + """One choice's generated token IDs and their sampled-token log probabilities. + + As the chat endpoint reports them in ``generation_token_ids`` and + ``generation_log_probs``: each log probability is floored at + ``VLLM_LOGPROB_FLOOR``. + """ + choice_index = generation_details.index + output_token_ids = getattr(generation_details, "token_ids", None) + if output_token_ids is None: + raise RuntimeError( + "vLLM was asked to return token information for the " + "OpenAI-compatible chat endpoint but generation output " + f"choice_idx={choice_index} did not include token_ids." + ) + generation_token_ids = list(output_token_ids) + + generation_logprob_details = getattr(generation_details, "logprobs", None) + if generation_logprob_details is None: + if generation_token_ids: + raise RuntimeError( + "vLLM was asked to return token information for the " + "OpenAI-compatible chat endpoint but generation output " + f"choice_idx={choice_index} did not include logprobs." + ) + return generation_token_ids, [] + if len(generation_token_ids) != len(generation_logprob_details): + raise RuntimeError( + "vLLM returned mismatched generation token IDs and log " + "probabilities for the OpenAI-compatible chat endpoint: " + f"choice_idx={choice_index}, " + f"token_count={len(generation_token_ids)}, " + f"logprob_count={len(generation_logprob_details)}." + ) + generation_log_probs = [] + for token_id, position_logprobs in zip( + generation_token_ids, generation_logprob_details + ): + selected_token_logprob = position_logprobs.get(token_id) + if selected_token_logprob is None: + raise RuntimeError( + "vLLM generation log probabilities did not include the " + "selected token while attaching token information to " + "the OpenAI-compatible chat response: " + f"choice_idx={choice_index}, token_id={token_id}." + ) + generation_log_probs.append( + max(float(selected_token_logprob.logprob), VLLM_LOGPROB_FLOOR) + ) + return generation_token_ids, generation_log_probs + + def model_dump_chat_response_with_dynamic_message_fields( response: Any, ) -> dict[str, Any]: diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 9231aa06d22..2c67cc2a1f8 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -64,13 +64,18 @@ resolve_collective_rpc_result, ) from nemo_rl.models.generation.vllm.config import parse_nvfp4_pertoken_rollout +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result, +) from nemo_rl.models.generation.vllm.utils import ( attach_routed_experts_to_chat_response_choices, attach_token_information_to_chat_response_choices, format_prompt_for_vllm_generation, + off_policy_sampling_error, validate_rollout_prompt, model_dump_chat_response_with_dynamic_message_fields, - pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) from nemo_rl.models.generation.vllm.vllm_worker import BaseVllmGenerationWorker @@ -1473,46 +1478,19 @@ class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): async def create_chat_completion( request: NeMoRLChatCompletionRequest, raw_request: Request ): - # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params - # Right now we explicitly assert set this to -1. - assert request.top_k in (None, -1), ( - f"Top k sampling parameter must be unset, empty, or -1. Got `{request.top_k}`" - ) - request.top_k = -1 - # The request sampling params need to exactly match those as are set in NeMo RL. # If they do not match, the inference will be off policy and destroy training - # stability. Validation rollouts are the one exception: they are stamped with - # the validation sampling profile (generation.val_temperature / val_top_p), - # which is metric-only and safe to serve — grpo.validate() is the only - # caller that constructs a non-train GenerationSamplingParams. Multi-turn - # agents issue their own requests, so this server-side check is the one - # chokepoint they all pass. - # vLLM resolves an unset top_p from the model's generation_config.json - # (ModelConfig.generation_config defaults to "auto"), NOT to 1.0, so a - # request omitting it would sample off-policy while passing this check. - assert request.top_p is not None, ( - "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " - "resolved by vLLM from the model's generation_config.json and would " - "bypass the on-policy sampling check." - ) - request_top_p = request.top_p - is_train_sampling = ( - request.temperature == generation_config["temperature"] - and request_top_p == generation_config["top_p"] - ) - is_val_sampling = ( - request.temperature == generation_config["val_temperature"] - and request_top_p == generation_config["val_top_p"] - ) - assert is_train_sampling or is_val_sampling, ( - f"request sampling (temperature={request.temperature}, " - f"top_p={request.top_p}) matches neither the train sampling params " - f"(temperature={generation_config['temperature']}, " - f"top_p={generation_config['top_p']}) nor the validation sampling " - f"params (val_temperature={generation_config['val_temperature']}, " - f"val_top_p={generation_config['val_top_p']})" + # stability. grpo.validate() is the only caller that constructs a non-train + # GenerationSamplingParams, and the validation profile it stamps is accepted. + sampling_error = off_policy_sampling_error( + temperature=request.temperature, + top_p=request.top_p, + top_k=request.top_k, + generation_config=generation_config, ) + assert sampling_error is None, sampling_error + # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params + request.top_k = -1 try: generator = await openai_serving_chat.create_chat_completion( @@ -1813,6 +1791,55 @@ async def init_collective_async( ), ) + def _rollout_max_new_tokens(self, input_length: int) -> int: + """Token budget for one rollout prompt; 0 means nothing fits. + + The configured cap, clipped to the context window and to speculative + decoding's lookahead. Kept apart from ``generate_async`` so every path that + serves a rollout from this engine gives it the same budget. + """ + max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) + allowed_new_tokens = max( + 0, min(self.cfg["max_new_tokens"], max_model_len - input_length) + ) + spec_cfg = self.cfg.get("vllm_kwargs", {}).get("speculative_config") or {} + spec_lookahead = int(spec_cfg.get("num_speculative_tokens", 0)) + if allowed_new_tokens > 0 and spec_lookahead > 0: + allowed_new_tokens = self._request_max_new_tokens( + configured_max_new_tokens=allowed_new_tokens, + input_length=input_length, + max_model_len=max_model_len, + cap_to_context=False, + spec_lookahead=spec_lookahead, + ) + return allowed_new_tokens + + def _rollout_sampling_params( + self, + *, + max_new_tokens: int, + greedy: bool, + stop_strings: Optional[list[str]], + ) -> Any: + """The SamplingParams for one rollout. + + Kept apart from ``generate_async`` so every path that serves a rollout + from this engine samples it the same way. + + Args: + max_new_tokens: The rollout's budget, from ``_rollout_max_new_tokens``. + greedy: Greedy decoding instead of the configured sampling. + stop_strings: The sample's own stop strings, merged with the + configured ones. + """ + return self._build_sampling_params( + greedy=greedy, + stop_strings=self._merge_stop_strings( + [stop_strings] if stop_strings else None + ), + max_new_tokens=max_new_tokens, + ) + async def generate_async( self, data: BatchedDataDict[GenerationDatumSpec], @@ -1865,231 +1892,49 @@ async def process_single_sample(sample_idx): ): per_sample_stop_strings = batch_specific_stop_strings_list[sample_idx] - final_stop_strings_for_sample = self._merge_stop_strings( - [per_sample_stop_strings] if per_sample_stop_strings else None - ) - - max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) - remaining_ctx = max_model_len - current_input_actual_length - allowed_new_tokens = max(0, min(self.cfg["max_new_tokens"], remaining_ctx)) - - spec_cfg = self.cfg.get("vllm_kwargs", {}).get("speculative_config") or {} - spec_lookahead = int(spec_cfg.get("num_speculative_tokens", 0)) - if allowed_new_tokens > 0 and spec_lookahead > 0: - allowed_new_tokens = self._request_max_new_tokens( - configured_max_new_tokens=allowed_new_tokens, - input_length=current_input_actual_length, - max_model_len=max_model_len, - cap_to_context=False, - spec_lookahead=spec_lookahead, - ) - - # Handle case where no tokens can be generated due to length constraints - if allowed_new_tokens == 0: - # Access the input data directly from the function parameters - input_ids_single_row = input_ids_batch[sample_idx] - - # Create output tensors with just the input (no generated tokens) - output_ids_single_item_batched = input_ids_single_row[ - :current_input_actual_length - ].unsqueeze(0) - - logprobs_single_item = torch.zeros( - (1, current_input_actual_length), - dtype=torch.float32, - device=input_ids_single_row.device, + max_new_tokens = self._rollout_max_new_tokens(current_input_actual_length) + input_ids_single_row = input_ids_batch[sample_idx] + if max_new_tokens == 0: + # No new token fits in the context: return the prompt alone. + result = RolloutResult.nothing_generated() + else: + sampling_params_for_request = self._rollout_sampling_params( + max_new_tokens=max_new_tokens, + greedy=greedy, + stop_strings=per_sample_stop_strings, ) + request_id = str(uuid.uuid4()) + final_request_output = None + async for req_output in self.llm.generate( + prompt=prompt, + sampling_params=sampling_params_for_request, + request_id=request_id, + ): + final_request_output = req_output + if final_request_output is None: + raise RuntimeError(f"No output received for request {request_id}") - generation_lengths_tensor = torch.tensor( - [0], dtype=torch.long, device=input_ids_single_row.device + validate_rollout_prompt( + input_ids_single_row[:current_input_actual_length].tolist(), + final_request_output.prompt_token_ids, ) - - unpadded_sequence_lengths_tensor = torch.tensor( - [current_input_actual_length], - dtype=torch.long, + result = rollout_result( + final_request_output, + input_length=current_input_actual_length, + return_routed_experts=self._return_routed_experts_enabled(), + routed_experts_dtype=self.routed_experts_dtype, device=input_ids_single_row.device, + sample_idx=sample_idx, ) - - # Not truncated since no generation was attempted (length constraint) - truncated_tensor = torch.tensor( - [False], dtype=torch.bool, device=input_ids_single_row.device - ) - - result_batch = BatchedDataDict[GenerationOutputSpec]( - { - "output_ids": output_ids_single_item_batched, - "logprobs": logprobs_single_item, - "generation_lengths": generation_lengths_tensor, - "unpadded_sequence_lengths": unpadded_sequence_lengths_tensor, - "truncated": truncated_tensor, - } - ) - - return (sample_idx, result_batch) - - sampling_params_for_request = self._build_sampling_params( - greedy=greedy, - stop_strings=final_stop_strings_for_sample, - max_new_tokens=allowed_new_tokens, - ) - - request_id = str(uuid.uuid4()) - - # Generate using vLLM async engine - vllm_request_generator = self.llm.generate( - prompt=prompt, - sampling_params=sampling_params_for_request, - request_id=request_id, - ) - - # Get the final result from the generator - final_request_output = None - async for req_output in vllm_request_generator: - final_request_output = req_output - - if final_request_output is None: - raise RuntimeError(f"No output received for request {request_id}") - - validate_rollout_prompt( - input_ids_batch[sample_idx, :current_input_actual_length].tolist(), - final_request_output.prompt_token_ids, - ) - - # Process the output - generation_details = final_request_output.outputs[0] - generated_token_ids = list(generation_details.token_ids) - num_generated_tokens = len(generated_token_ids) - return_routed_experts = self._return_routed_experts_enabled() - - original_input_ids_single_row = input_ids_batch[sample_idx] - final_output_tensor_len = current_input_actual_length + num_generated_tokens - - # Create output_ids tensor for this single item - output_ids_single_item = torch.full( - (final_output_tensor_len,), - self.cfg["_pad_token_id"], - dtype=original_input_ids_single_row.dtype, - device=original_input_ids_single_row.device, - ) - # Copy original input (up to its actual length) - output_ids_single_item[:current_input_actual_length] = ( - original_input_ids_single_row[:current_input_actual_length] - ) - # Add generated tokens after the actual input - output_ids_single_item[ - current_input_actual_length : current_input_actual_length - + num_generated_tokens - ] = torch.tensor( - generated_token_ids, - dtype=original_input_ids_single_row.dtype, - device=original_input_ids_single_row.device, - ) - - # Reshape to (1, seq_len) for BatchedDataDict - output_ids_single_item_batched = output_ids_single_item.unsqueeze(0) - - # Create logprobs tensor for this single item - logprobs_single_item = torch.zeros( - (1, final_output_tensor_len), - dtype=torch.float32, - device=original_input_ids_single_row.device, - ) - if hasattr(generation_details, "logprobs") and generation_details.logprobs: - for idx, logprob_dict_per_token in enumerate( - generation_details.logprobs - ): - if logprob_dict_per_token and idx < len(generated_token_ids): - token_id_at_idx = generated_token_ids[idx] - if token_id_at_idx in logprob_dict_per_token: - logprob_value = logprob_dict_per_token[ - token_id_at_idx - ].logprob - position_in_output_tensor = ( - current_input_actual_length + idx - ) - if position_in_output_tensor < final_output_tensor_len: - logprobs_single_item[0, position_in_output_tensor] = ( - logprob_value - ) - - # Generation lengths - generation_lengths_tensor = torch.tensor( - [num_generated_tokens], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - - # Unpadded sequence lengths (actual_input + actual_generated) - unpadded_total_length = current_input_actual_length + num_generated_tokens - unpadded_sequence_lengths_tensor = torch.tensor( - [unpadded_total_length], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - - # Check if response was truncated (hit max_tokens length limit) - is_truncated = generation_details.finish_reason == "length" - truncated_tensor = torch.tensor( - [is_truncated], - dtype=torch.bool, - device=original_input_ids_single_row.device, - ) - - result_dict = { - "output_ids": output_ids_single_item_batched, - "logprobs": logprobs_single_item, - "generation_lengths": generation_lengths_tensor, - "unpadded_sequence_lengths": unpadded_sequence_lengths_tensor, - "truncated": truncated_tensor, - } - routed_experts, r3_stats = pad_and_align_routed_expert_indices( - final_request_output, - generation_details, - valid_length=unpadded_total_length, - padded_length=final_output_tensor_len, - device=original_input_ids_single_row.device, - require_complete_routed_experts=return_routed_experts, - return_stats=True, - routed_experts_dtype=self.routed_experts_dtype, + return ( + sample_idx, + rollout_output( + result, + input_ids=input_ids_single_row, + input_length=current_input_actual_length, + pad_token_id=self.cfg["_pad_token_id"], + ), ) - if return_routed_experts and routed_experts is None: - raise RuntimeError( - "vLLM was asked to return routed experts but the generation output " - "did not include routed_experts." - ) - if return_routed_experts: - if r3_stats["missing_routes"] > 0: - LOGGER.warning( - "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for sample_idx=%d, missing_token_routes=%d, " - "actual_routes=%d, expected_routes=%d. Megatron will use its " - "own router for those missing token routes.", - sample_idx, - r3_stats["missing_routes"], - r3_stats["actual_routes"], - r3_stats["expected_routes"], - ) - result_dict["r3_routed_experts_missing_routes"] = torch.tensor( - [r3_stats["missing_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - result_dict["r3_routed_experts_expected_routes"] = torch.tensor( - [r3_stats["expected_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - result_dict["r3_routed_experts_actual_routes"] = torch.tensor( - [r3_stats["actual_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - if routed_experts is not None: - result_dict["routed_experts"] = routed_experts.unsqueeze(0) - - result_batch = BatchedDataDict[GenerationOutputSpec](result_dict) - - return (sample_idx, result_batch) # Create tasks for all samples and yield results as they complete sample_tasks = [ diff --git a/pyrefly.toml b/pyrefly.toml index 55d8a74cf27..31acd1641e5 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -282,6 +282,7 @@ project-includes = [ "nemo_rl/models/generation/vllm/quantization/nvfp4_pertoken_config.py", "nemo_rl/models/generation/vllm/quantization/utils.py", "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", + "nemo_rl/models/generation/vllm/request_pipeline.py", "nemo_rl/models/generation/vllm/refit_layout.py", "nemo_rl/models/generation/vllm/refit_loader.py", "nemo_rl/models/generation/vllm/utils.py", diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index e25d808c663..d0533c1a734 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -474,6 +474,131 @@ async def test_async_vllm_worker_gpu_profiling_awaits_collective_rpc( worker.llm.collective_rpc.assert_awaited_once_with("stop_gpu_profiling", args=()) +class _ScriptedAsyncLLM: + """Yields one scripted final RequestOutput and records what it was asked to generate.""" + + def __init__(self, generated: list[int], finish_reason: str = "stop") -> None: + self.generated = generated + self.finish_reason = finish_reason + self.calls: list[tuple[dict[str, Any], dict[str, Any]]] = [] + + async def generate(self, prompt, sampling_params, request_id): + self.calls.append((prompt, sampling_params)) + yield types.SimpleNamespace( + prompt_token_ids=prompt["prompt_token_ids"], + prompt_routed_experts=None, + num_cached_tokens=0, + outputs=[ + types.SimpleNamespace( + token_ids=self.generated, + logprobs=[ + {t: types.SimpleNamespace(logprob=-0.5)} for t in self.generated + ], + finish_reason=self.finish_reason, + routed_experts=None, + ) + ], + ) + + +def _rollout_worker(llm: _ScriptedAsyncLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: + worker = VllmAsyncGenerationWorkerImpl.__new__(VllmAsyncGenerationWorkerImpl) + worker.cfg = { + "max_new_tokens": 8, + "temperature": 0.7, + "top_p": 0.9, + "top_k": None, + "stop_token_ids": [2], + "stop_strings": [""], + "_pad_token_id": 0, + "vllm_cfg": {"async_engine": True, "max_model_len": 16}, + **cfg, + } + # Records the sampling kwargs instead of building vLLM's SamplingParams. + worker.SamplingParams = dict + worker.routed_experts_dtype = torch.int16 + worker.llm = llm + return worker + + +async def _generate_one( + worker: VllmAsyncGenerationWorkerImpl, + prompt: list[int], + stop_strings: list[str] | None = None, +) -> BatchedDataDict: + data = BatchedDataDict( + { + "input_ids": torch.tensor([prompt]), + "input_lengths": torch.tensor([len(prompt)]), + "stop_strings": [stop_strings], + } + ) + ((sample_idx, output),) = [item async for item in worker.generate_async(data)] + assert sample_idx == 0 + return output + + +@pytest.mark.asyncio +async def test_async_vllm_generate_async_samples_and_returns_the_rollout() -> None: + llm = _ScriptedAsyncLLM([10, 11, 2], finish_reason="stop") + worker = _rollout_worker(llm) + + output = await _generate_one(worker, [1, 2, 3, 4, 5], stop_strings=[""]) + + ((prompt, sampling),) = llm.calls + assert prompt == {"prompt_token_ids": [1, 2, 3, 4, 5]} + assert sampling["max_tokens"] == 8 + assert sampling["temperature"] == 0.7 + assert sampling["top_k"] == -1 + assert sorted(sampling["stop"]) == ["", ""] + assert output["output_ids"].tolist() == [[1, 2, 3, 4, 5, 10, 11, 2]] + assert output["logprobs"].tolist() == [[0.0] * 5 + [-0.5] * 3] + assert output["generation_lengths"].tolist() == [3] + assert output["unpadded_sequence_lengths"].tolist() == [8] + assert output["truncated"].tolist() == [False] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("speculative_config", "expected_max_tokens"), + [ + # min(max_new_tokens=8, max_model_len 16 - prompt 12) + (None, 4), + # The drafter reads 2 tokens ahead: 16 - 12 - (2 + 1) + ({"num_speculative_tokens": 2}, 1), + ], +) +async def test_async_vllm_generate_async_caps_the_budget_to_the_context( + speculative_config, expected_max_tokens +) -> None: + llm = _ScriptedAsyncLLM([10], finish_reason="length") + worker = _rollout_worker( + llm, vllm_kwargs={"speculative_config": speculative_config} + ) + + output = await _generate_one(worker, list(range(1, 13))) + + ((_, sampling),) = llm.calls + assert sampling["max_tokens"] == expected_max_tokens + assert output["truncated"].tolist() == [True] + + +@pytest.mark.asyncio +async def test_async_vllm_generate_async_returns_the_prompt_when_nothing_fits() -> None: + llm = _ScriptedAsyncLLM([10]) + worker = _rollout_worker(llm) + prompt = list(range(1, 17)) + + output = await _generate_one(worker, prompt) + + assert llm.calls == [] + assert output["output_ids"].tolist() == [prompt] + assert output["logprobs"].tolist() == [[0.0] * 16] + assert output["generation_lengths"].tolist() == [0] + assert output["unpadded_sequence_lengths"].tolist() == [16] + assert output["truncated"].tolist() == [False] + + @pytest.mark.asyncio async def test_async_vllm_http_client_runs_generation_on_owner_loop() -> None: engine_loop = asyncio.get_running_loop() diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py new file mode 100644 index 00000000000..c00be9dc2c4 --- /dev/null +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -0,0 +1,124 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The rollout result pipeline behind generate_async.""" + +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result, +) +from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL + +LAYERS, TOPK = 2, 2 + + +def _logprob(value: float): + return SimpleNamespace(logprob=value) + + +def _final_output( + generated: list[int], + *, + finish_reason: str = "stop", + prompt_routes: int | None = None, + generated_routes: int | None = None, +): + def routes(count): + if count is None: + return None + return np.arange(count * LAYERS * TOPK, dtype=np.int16).reshape( + count, LAYERS, TOPK + ) + + return SimpleNamespace( + prompt_routed_experts=routes(prompt_routes), + num_cached_tokens=0, + outputs=[ + SimpleNamespace( + token_ids=generated, + # The second token has no reported logprob. + logprobs=[{generated[0]: _logprob(-0.5)}, {}] + + [{t: _logprob(-1.0)} for t in generated[2:]], + finish_reason=finish_reason, + routed_experts=routes(generated_routes), + ) + ], + ) + + +def _result(final_output, *, input_length=3, r3=False) -> RolloutResult: + return rollout_result( + final_output, + input_length=input_length, + return_routed_experts=r3, + routed_experts_dtype=torch.int16, + device=torch.device("cpu"), + ) + + +def test_output_layout_matches_generate_async() -> None: + result = _result(_final_output([7, 8, 9], finish_reason="length")) + output = rollout_output( + result, input_ids=torch.tensor([1, 2, 3, 0, 0]), input_length=3, pad_token_id=0 + ) + + assert output["output_ids"].tolist() == [[1, 2, 3, 7, 8, 9]] + # A position without a reported logprob stays 0.0, aligned with its token. + assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0, -0.5, 0.0, -1.0]] + assert output["generation_lengths"].tolist() == [3] + assert output["unpadded_sequence_lengths"].tolist() == [6] + assert output["truncated"].tolist() == [True] + assert "routed_experts" not in output + + +def test_nothing_generated_returns_the_prompt_untruncated() -> None: + output = rollout_output( + RolloutResult.nothing_generated(), + input_ids=torch.tensor([1, 2, 3, 0]), + input_length=3, + pad_token_id=0, + ) + + assert output["output_ids"].tolist() == [[1, 2, 3]] + assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0]] + assert output["generation_lengths"].tolist() == [0] + assert output["truncated"].tolist() == [False] + + +def test_router_replay_routes_are_aligned_and_counted() -> None: + # 3 prompt + 3 generated tokens need 5 routes; vLLM reported 3 + 1. + result = _result( + _final_output([7, 8, 9], prompt_routes=3, generated_routes=1), r3=True + ) + output = rollout_output( + result, input_ids=torch.tensor([1, 2, 3]), input_length=3, pad_token_id=0 + ) + + routed = output["routed_experts"] + assert routed.shape == (1, 6, LAYERS, TOPK) and routed.dtype == torch.int16 + assert (routed[0, 4:] == R3_MISSING_ROUTE_SENTINEL).all() + assert output["r3_routed_experts_expected_routes"].tolist() == [5] + assert output["r3_routed_experts_actual_routes"].tolist() == [4] + assert output["r3_routed_experts_missing_routes"].tolist() == [1] + + +def test_router_replay_requires_routes() -> None: + with pytest.raises(RuntimeError, match="did not include routed_experts"): + _result(_final_output([7, 8, 9]), r3=True) diff --git a/tests/unit/models/generation/test_vllm_utils.py b/tests/unit/models/generation/test_vllm_utils.py index 7bf7546d9d0..3ce5907bc06 100644 --- a/tests/unit/models/generation/test_vllm_utils.py +++ b/tests/unit/models/generation/test_vllm_utils.py @@ -38,6 +38,7 @@ encode_counter_key, format_prompt_for_vllm_generation, model_dump_chat_response_with_dynamic_message_fields, + off_policy_sampling_error, pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) @@ -1451,3 +1452,29 @@ def test_remap_multimodal_placeholders_rebuilds_real_vllm_placeholder_range(): assert isinstance(moved, PlaceholderRange) assert (moved.offset, moved.length) == (3, 3) assert moved.is_embed is is_embed + + +@pytest.mark.parametrize( + ("temperature", "top_p", "top_k", "error"), + [ + (1.0, 1.0, None, None), # train profile + (0.0, 0.9, -1, None), # validation profile + (0.7, 1.0, None, "matches neither"), + (1.0, None, None, "top_p must be set explicitly"), + (1.0, 1.0, 5, "Top k sampling parameter"), + ], +) +def test_off_policy_sampling_error(temperature, top_p, top_k, error) -> None: + generation_config = { + "temperature": 1.0, + "top_p": 1.0, + "val_temperature": 0.0, + "val_top_p": 0.9, + } + message = off_policy_sampling_error( + temperature=temperature, + top_p=top_p, + top_k=top_k, + generation_config=generation_config, + ) + assert (message is None) if error is None else (error in message) From d217dd89e82b5830b2a830d138f1a0f9fc4bcc09 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 09:54:29 -0700 Subject: [PATCH 07/82] refactor(dynamo): split NeMo-Gym support out to its own PR At review request, NeMo-Gym through the Dynamo frontend moves to a follow-up PR that waits on NVIDIA-NeMo/Gym#4817, so this PR no longer needs a Gym bump: - Gym's submodule is back on main's pin. - The engine's Gym path (_generate_gym, capture staging, prefix splice), the Gym dialect setup, session affinity, client_base_urls and the Gym-only startup checks move to the follow-up. - NeMo-Gym with policy.generation.dynamo=true is refused at startup in should_use_nemo_gym, before any cluster starts. - The Dynamo L1 keeps its SingleController math and reload-refit overlays; the swe1 Dynamo + Gym nightly is disabled until the follow-up. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/design-docs/dynamo-integration.md | 18 +- docs/guides/dynamo-generation.md | 60 +--- ...grpo_math_1B_dynamo_single_controller.yaml | 3 - nemo_rl/algorithms/distillation.py | 2 +- nemo_rl/algorithms/grpo.py | 6 +- .../single_controller_utils/config.py | 4 +- .../single_controller_utils/setup.py | 9 +- nemo_rl/environments/nemo_gym.py | 63 +--- nemo_rl/environments/nemo_gym_shards.py | 7 +- nemo_rl/models/generation/__init__.py | 6 - nemo_rl/models/generation/dynamo/arguments.py | 6 - nemo_rl/models/generation/dynamo/config.py | 22 -- .../generation/dynamo/managed_runtime.py | 4 +- .../models/generation/dynamo/native_engine.py | 292 +----------------- .../models/generation/openai_server_utils.py | 72 +---- .../models/generation/vllm/vllm_generation.py | 37 +-- .../generation/vllm/vllm_worker_async.py | 5 +- .../functional/L1_Functional_Tests_Dynamo.sh | 5 - .../grpo_async_gym_single_controller.sh | 8 - ...lm_token_capture_single_controller_1n2g.sh | 4 - tests/test_suites/disabled.txt | 4 + tests/test_suites/nightly.txt | 1 - tests/unit/algorithms/test_distillation.py | 2 +- tests/unit/algorithms/test_grpo.py | 2 +- .../test_nemo_gym_token_capture.py | 103 +----- .../unit/environments/test_nemo_gym_utils.py | 67 ---- .../models/generation/test_dynamo_config.py | 28 +- .../generation/test_dynamo_native_engine.py | 188 +---------- .../test_dynamo_native_engine_capture.py | 241 --------------- .../generation/test_openai_server_utils.py | 92 ------ .../models/generation/test_vllm_generation.py | 12 +- .../test_vllm_token_capture_hosting.py | 1 - tests/unit/single_controller/test_setup.py | 40 +-- 34 files changed, 84 insertions(+), 1332 deletions(-) delete mode 100644 tests/unit/models/generation/test_dynamo_native_engine_capture.py diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index a5f936c473d..3f6f20593a1 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit a5f936c473dbd477ce4632373eaba884272b7f6a +Subproject commit 3f6f20593a126ca9484793769a51d186aa1486c3 diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 843dad2c897..9c07ccda5c7 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -32,29 +32,17 @@ implements its `LLMEngine` over the worker's `AsyncLLM`: The result travels in `engine_data`, and the driver assembles the same output with `request_pipeline.rollout_output`. - NeMo-Gym's calls also carry an annotation (Gym's Dynamo dialect). The engine - applies the native chat endpoint's rules to them: the on-policy sampling check - (`vllm/utils.off_policy_sampling_error`), the prefix splice - (`openai_server_utils.splice_prefix_tokens`) and the context clamp. A - token-capture call stages through the worker's own capture lifecycle - (`_begin_request_capture` / `_finish_request_capture`), and the token fields - and routed experts come from the native endpoint's helpers - (`chat_generation_tokens`, `chat_routed_experts`). - A request with media carries the frontend's tokens, with one unexpanded placeholder per item, and the media URLs. The adapter loads the media with vLLM's media connector. It then builds the processed prompt with the engine's renderer (`process_for_engine_async`), the same step that follows rendering - on the native endpoint. The splice, the placeholder remap - (`remap_multimodal_placeholders`) and media capture (`_capture_request_media`) - all run on that processed prompt. + on the native endpoint. - `abort` cancels the vLLM request on client disconnect. - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It publishes no token budget. The engine owns the context check and the output - cap, as the native server does. With a budget, the frontend would fill an - omitted `max_tokens` from the prompt it rendered, which a prefix splice then - changes, and would refuse NeMo RL's own rollouts that fill the context. + cap, as the native server does. With a budget, the frontend would refuse NeMo + RL's own rollouts that fill the context. - `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ publisher. It is set only when the engine caches prefixes. Each engine binds its own Linux abstract-namespace socket (`ipc://@...`), so engines on one node diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index f1764202897..b780dcda242 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -51,11 +51,10 @@ shows them all: | Key | Default | Meaning | |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | -| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies; NeMo-Gym's calls keep Gym's timeouts | -| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | +| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies | +| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [Chat settings](#chat-settings) | | `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | -| `frontend_args.session_affinity_ttl_secs` | 1800 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it | | `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | @@ -69,7 +68,7 @@ is set (the image sets it to `/opt/dynamo/bin`), and on `PATH` otherwise. | `generate_async` on text token prompts, router replay (R3) included | Frontend (KV-aware); the engine samples and assembles it exactly as native `generate_async` does | | NeMo RL's own multimodal rollouts | Native shard dispatch. Those rollouts send media to the engine directly rather than as OpenAI content | | `generate_text_async`, which evaluation uses | Native shard dispatch | -| NeMo-Gym's OpenAI requests | Frontend, in Gym's Dynamo dialect; the engine serves them as the native chat endpoint does. See [NeMo-Gym](#nemo-gym) | +| NeMo-Gym's OpenAI requests | Not served yet; see [Not supported](#not-supported) | A rollout through the frontend has the same `GenerationOutputSpec` as the native path: token IDs, log probabilities, truncation, routed experts with @@ -87,67 +86,32 @@ OpenAI clients of the frontend get what vLLM's own OpenAI server applies: for output. The engine, not the frontend, checks the context, so it checks the prompt it actually runs. -## NeMo-Gym - -NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect -(`vllm_model.dynamo_frontend`, from -[NVIDIA-NeMo/Gym#4817](https://github.com/NVIDIA-NeMo/Gym/pull/4817)). Gym marks -every call with an annotation, and the engine serves a marked call as the native -chat endpoint does: - -- Sampling must match `temperature`/`top_p` or `val_temperature`/`val_top_p`, - with `top_k` unset. vLLM samples with the configured values. -- The token prefix Gym echoes replaces the history the frontend rendered. -- Output is clamped to the context, and a prompt that leaves no room is refused - with the native endpoint's "maximum context length" error. -- Stop strings stop vLLM itself, so the token IDs match the native endpoint's. -- The response carries `prompt_token_ids`, `completion_token_ids` and - `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. - With router replay on, it also carries the call's `routed_experts`, as the - native endpoint attaches them. -- Images, video and audio in the messages are processed as on the native - endpoint. The frontend passes each media URL through with one unexpanded - placeholder per item. The engine loads the media with vLLM's media connector - and runs vLLM's multimodal processor on the frontend's tokens. The splice, - placeholder remap and media capture then work on that processed prompt. - -With `token_capture.enabled`, Gym uses its `dynamo_worker` capture backend. The -same vLLM workers stage each call's tokens, log probabilities, routed experts and -processed media from their own process, exactly as the native endpoint does, and -only the commit coordinates travel back through the frontend. The frontend keeps a rollout on -one engine (`session_affinity_ttl_secs`), as the native backend pins each Gym -session to one server. - -The native server's chat settings carry over: +## Chat settings + +The native server's chat settings carry over to the frontend: | Native setting (`vllm_cfg`) | Under Dynamo | |---|---| | `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | | `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | | `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | -| `http_server_serving_chat_kwargs.default_chat_template_kwargs` | Sent by Gym under its own `chat_template_kwargs` | | `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | -Where the behavior differs from the native endpoint: - -- The engine sees the rendered prompt, not the messages. It finds where the - echoed prefix ends by counting the prefix's EOS tokens, not by rendering the - history. -- When a request sets `chat_template_kwargs.force_nonempty_content`, Dynamo - reports a reasoning-only turn as content. +Where the behavior differs from the native endpoint: when a request sets +`chat_template_kwargs.force_nonempty_content`, Dynamo reports a reasoning-only +turn as content. ## Not supported Startup rejects these with `dynamo: true`: +- NeMo-Gym (`env.should_use_nemo_gym`): the frontend does not serve Gym's + calls yet. Run Gym recipes with `dynamo: false`. - `colocated.enabled: true`: a colocated engine sleeps while the policy trains, and the frontend would keep routing to it. The SingleController never runs colocated vLLM; the other entrypoints need non-colocated generation. - `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, and a restarted shard does not register with it yet. -- `async_rl.generation_router`: the frontend already routes the fleet behind one - URL. -- `vllm_cfg.http_server_api_key_required`: the frontend checks no bearer token. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data parallelism inside vLLM, which the async engine Dynamo serves does not. - Parser plugins with no Dynamo parser, and the chat settings above that the @@ -155,8 +119,6 @@ Startup rejects these with `dynamo: true`: - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. -A Gym call with `n > 1` is refused when it arrives. - ## Build the image ai-dynamo is part of the `vllm` extra, so every vLLM worker venv carries it. diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 9ad2b249a95..1b62d39ba8d 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -37,9 +37,6 @@ policy: tokenizer_cache_bytes: 52428800 # kv: route each request to the engine holding most of its prefix. router_mode: kv - # Keep a NeMo-Gym token-capture rollout on one engine while it idles up - # to this many seconds (null disables it). - session_affinity_ttl_secs: 1800 extra_cli_args: [] logger: diff --git a/nemo_rl/algorithms/distillation.py b/nemo_rl/algorithms/distillation.py index f00109256d3..f3a5d883ae2 100644 --- a/nemo_rl/algorithms/distillation.py +++ b/nemo_rl/algorithms/distillation.py @@ -556,7 +556,7 @@ def init_vllm_deferred(): def init_nemo_gym(): shard_set = build_nemo_gym_actors( env_configs, - base_urls=cast(list[str], deferred_vllm.client_base_urls), + base_urls=cast(list[str], deferred_vllm.dp_openai_server_base_urls), model_name=generation_config["model_name"], tokenizer=tokenizer, # Distillation does not configure vLLM for router replay. diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 505b1919261..31ef02d7895 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -1771,8 +1771,8 @@ def init_megatron_generation_task(policy_future): ) vllm_reserve_time = time.perf_counter() - vllm_reserve_t0 print( - f" ✓ Reserved {len(deferred_vllm.client_base_urls)} vLLM server URLs: " - f"{deferred_vllm.client_base_urls}", + f" ✓ Reserved {len(deferred_vllm.dp_openai_server_base_urls)} vLLM server URLs: " + f"{deferred_vllm.dp_openai_server_base_urls}", flush=True, ) @@ -1786,7 +1786,7 @@ def init_vllm_deferred(): def init_nemo_gym(): """Spin up NeMo Gym servers with the pre-assigned vLLM URLs.""" return _spinup_nemo_gym( - deferred_vllm.client_base_urls, + deferred_vllm.dp_openai_server_base_urls, generation_config["model_name"], policy_api_key=deferred_vllm.http_server_api_key, ) diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py index 758e836f402..a6d42371804 100644 --- a/nemo_rl/algorithms/single_controller_utils/config.py +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -679,9 +679,7 @@ class TokenCaptureConfig(BaseModel, extra="allow"): capture_dir: Optional[str] = None # Generation backend hosting token capture. This is derived from # policy.generation.backend during setup; users should not set it separately. - # "dynamo" is the vLLM backend served through Dynamo (policy.generation.dynamo): - # the same workers host capture, and NeMo-Gym reaches them through the frontend. - generation_backend: Optional[Literal["vllm", "megatron", "dynamo"]] = None + generation_backend: Optional[Literal["vllm", "megatron"]] = None # Keep routed_experts out of canonical rows and assemble them on policy # workers from strict staged-fragment plans. defer_routed_experts_to_policy: bool = False diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index d3dc6dd53c2..e3f65134811 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -125,7 +125,6 @@ should_mask_flagged_samples, ) from nemo_rl.models.generation import resolve_generation_class -from nemo_rl.models.generation.dynamo.config import dynamo_enabled from nemo_rl.models.generation.fleet_health import ( FleetHealthPolicy, GenerationFleetHealth, @@ -1370,11 +1369,7 @@ def setup_single_controller( # Fill the derived ledger-hosting fields (see TokenCaptureConfig): a # per-run control-plane bearer token, the process-shared capture # directory used by every Gym worker, and the capture-host backend. - token_capture_cfg.generation_backend = ( - "dynamo" - if dynamo_enabled(generation_config) - else generation_config["backend"] - ) + token_capture_cfg.generation_backend = generation_config["backend"] if token_capture_cfg.control_auth_token is None: # Deferred import: only needed on the capture path. import secrets @@ -1861,7 +1856,7 @@ def _build_trainer_then_megatron_generation() -> tuple[ defer_model_load=True, ) defer_generation_model_load = True - gym_base_urls = generation.client_base_urls + gym_base_urls = generation.dp_openai_server_base_urls # Before the Gym task is built, so Gym can be handed the router's single URL. # These two statements are the only failable ones related to the port holder's creation. try: diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 5ab370dbe65..435e35198f8 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -98,10 +98,7 @@ RolloutDataFailure, http_status_is_infra, ) -from nemo_rl.models.generation.dynamo.config import ( - dynamo_enabled, - native_default_chat_template_kwargs, -) +from nemo_rl.models.generation.dynamo.config import dynamo_enabled from nemo_rl.models.generation.interfaces import ( resolve_routed_experts_dtype_name_for_model, should_use_async_rollouts, @@ -304,6 +301,12 @@ def should_use_nemo_gym(master_config: NemoGymCompatibleConfig) -> bool: return False generation_config = master_config.policy["generation"] + if dynamo_enabled(generation_config): + raise ValueError( + "policy.generation.dynamo=true does not serve NeMo-Gym rollouts yet. " + "Set policy.generation.dynamo=false to run this NeMo-Gym config on " + "NeMo RL's vLLM servers." + ) assert should_use_async_rollouts(generation_config), ( "❌ Error: In order to use NeMo-Gym, you must use a generation " "backend with `async_engine: true`!" @@ -489,11 +492,9 @@ def _external_staging_backend(token_capture: Dict[str, Any]) -> str: return "vllm_worker" if generation_backend == "megatron": return "megatron_worker" - if generation_backend == "dynamo": - return "dynamo_worker" raise ValueError( "token_capture.enabled requires setup-derived generation_backend to be " - f"'vllm', 'megatron' or 'dynamo'; got {generation_backend!r}" + f"'vllm' or 'megatron'; got {generation_backend!r}" ) @@ -2056,16 +2057,6 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_strings"] = None generation_config["stop_token_ids"] = None - if dynamo_enabled(generation_config): - # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect. - # Each actor applies it to its own composed config (_apply_dynamo_dialect), - # so it never becomes a policy_model overlay that a shard must claim. - config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = { - "default_chat_template_kwargs": native_default_chat_template_kwargs( - generation_config - ) - } - # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for # multi-turn multimodal postprocessing). @@ -2134,39 +2125,6 @@ def build_nemo_gym_config( ) -def _apply_dynamo_dialect( - nemo_gym_dict: dict[str, Any], default_chat_template_kwargs: Mapping[str, Any] -) -> None: - """Point one actor's policy model at the Dynamo frontend, in Gym's Dynamo dialect. - - The frontend takes token data in nvext instead of vLLM's request extensions. - Applied to the actor's composed config (a shard's overrides included), so - settings the user wrote there win. Copies the policy_model levels it - changes, so the caller's config is not mutated. - """ - policy_model = dict(nemo_gym_dict.get("policy_model") or {}) - models = dict(policy_model.get("responses_api_models") or {}) - vllm_model = dict(models.get("vllm_model") or {}) - vllm_model["dynamo_frontend"] = True - # The frontend reads `reasoning` as another name for `reasoning_content` and - # refuses an echoed assistant turn that carries both, Gym's default. - vllm_model.setdefault("reasoning_field", "reasoning_content") - # The native server merges these under every request's own template - # kwargs; the frontend has no such default, so Gym sends them. They sit - # under the kwargs Gym sends: vllm_model's own, else extra_body's. - if default_chat_template_kwargs: - gym_kwargs = vllm_model.get("chat_template_kwargs") or ( - (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} - ) - vllm_model["chat_template_kwargs"] = { - **default_chat_template_kwargs, - **gym_kwargs, - } - models["vllm_model"] = vllm_model - policy_model["responses_api_models"] = models - nemo_gym_dict["policy_model"] = policy_model - - def _build_gym_actor_config( nemo_gym_dict: dict[str, Any], *, @@ -2195,11 +2153,6 @@ def _build_gym_actor_config( invalid_tool_call_patterns = nemo_gym_dict.pop("invalid_tool_call_patterns", None) thinking_tags = nemo_gym_dict.pop("thinking_tags", None) tokenizer_config = nemo_gym_dict.pop("tokenizer_config", None) - dynamo_dialect = nemo_gym_dict.pop("dynamo_dialect", None) - if dynamo_dialect is not None: - _apply_dynamo_dialect( - nemo_gym_dict, dynamo_dialect["default_chat_template_kwargs"] - ) external_service_readiness = extract_external_service_readiness(nemo_gym_dict) port_range = { key: value diff --git a/nemo_rl/environments/nemo_gym_shards.py b/nemo_rl/environments/nemo_gym_shards.py index b03b9cd9eb6..d8322a8d23c 100644 --- a/nemo_rl/environments/nemo_gym_shards.py +++ b/nemo_rl/environments/nemo_gym_shards.py @@ -38,12 +38,7 @@ # Dict-shaped NeMo RL integration settings that are not Gym server entries. # The actor factory or rollout code consumes these separately. NEMO_RL_DICT_CONFIG_KEYS = frozenset( - { - "dynamo_dialect", - "effort_levels", - "external_service_readiness", - "tokenizer_config", - } + {"effort_levels", "external_service_readiness", "tokenizer_config"} ) # Ray placement-group strategies a shard plan may ask for. STRICT_SPREAD is the diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 8f4ea47ce9f..c8547bbce41 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -191,12 +191,6 @@ def configure_generation_config( dynamo_cfg(config) dynamo_parsers(config) check_dynamo_serving_chat_kwargs(config) - if config["vllm_cfg"].get("http_server_api_key_required"): - raise ValueError( - "policy.generation.vllm_cfg.http_server_api_key_required is not " - "supported with policy.generation.dynamo=true: NeMo-Gym calls the " - "Dynamo frontend, which checks no bearer token." - ) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( diff --git a/nemo_rl/models/generation/dynamo/arguments.py b/nemo_rl/models/generation/dynamo/arguments.py index c08d9873200..e5a6abebee5 100644 --- a/nemo_rl/models/generation/dynamo/arguments.py +++ b/nemo_rl/models/generation/dynamo/arguments.py @@ -111,12 +111,6 @@ def build_dynamo_frontend_argv( dynamo_cfg.frontend_args.router_mode, source="dynamo_cfg.frontend_args.router_mode", ) - if dynamo_cfg.frontend_args.session_affinity_ttl_secs is not None: - builder.add( - "--router-session-affinity-ttl-secs", - dynamo_cfg.frontend_args.session_affinity_ttl_secs, - source="dynamo_cfg.frontend_args.session_affinity_ttl_secs", - ) builder.add_raw( dynamo_cfg.frontend_args.extra_cli_args, source="dynamo_cfg.frontend_args.extra_cli_args", diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 7a6a61868d2..b6ae598b1a1 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -67,12 +67,6 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "least-loaded", "device-aware-weighted", ] = "kv" - # Idle TTL in seconds for keeping one session on one engine, or None to - # disable it. NeMo-Gym names the session on token-capture calls only (its - # rollout id), as the native backend pins each Gym session to one server; the - # engine splices a call's staged prefix after routing, so the KV router alone - # cannot see where that prefix is cached. - session_affinity_ttl_secs: PositiveInt | None = 1800 extra_cli_args: list[str] = Field(default_factory=list) @@ -85,7 +79,6 @@ class DynamoCfg(BaseModel, extra="forbid"): # NRL_VLLM_ASYNC_TIMEOUT_SECONDS. A timeout fails the attempt as a native one # does (GenerationUnavailable), and the rollout retry policy re-dispatches it # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). - # NeMo-Gym's own calls keep Gym's timeouts. request_timeout_s: PositiveFloat = 900.0 worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) @@ -196,21 +189,6 @@ def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: return serving.get("chat_template") -def native_default_chat_template_kwargs( - generation_config: Mapping[str, Any], -) -> dict[str, Any]: - """The template kwargs the native server merges under every request's own. - - ``default_chat_template_kwargs``, or the older ``chat_template_kwargs`` - spelling, of ``vllm_cfg.http_server_serving_chat_kwargs``. - """ - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - kwargs = serving.get("default_chat_template_kwargs") - if kwargs is None: - kwargs = serving.get("chat_template_kwargs") - return dict(kwargs or {}) - - def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" return bool(generation_config.get("dynamo")) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index e4a0fe9507e..ad9ad9b7659 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -167,8 +167,8 @@ def allocate_port(*, low: int, high: int) -> int: "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. - # Its v2 parser path preserves engine_data, which NeMo-Gym needs for - # exact token IDs and log probabilities. + # Its v2 parser path preserves engine_data, which carries every NeMo + # RL rollout's result. "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index b3221afcf53..ff31f98dfca 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -33,7 +33,6 @@ from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional -import numpy as np import torch from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names from dynamo.common.backend import ( @@ -66,7 +65,6 @@ from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase -from nemo_rl.data.captured_media import MediaCaptureRejected from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, dynamo_cfg, @@ -78,22 +76,12 @@ DYNAMO_ENDPOINT, ) from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION -from nemo_rl.models.generation.openai_server_utils import ( - derive_prefix_eos_count, - splice_prefix_tokens, -) from nemo_rl.models.generation.vllm.request_pipeline import ( RolloutResult, rollout_result, rollout_result_to_wire, ) -from nemo_rl.models.generation.vllm.utils import ( - chat_generation_tokens, - chat_routed_experts, - off_policy_sampling_error, - remap_multimodal_placeholders, - validate_rollout_prompt, -) +from nemo_rl.models.generation.vllm.utils import validate_rollout_prompt if TYPE_CHECKING: from nemo_rl.models.generation.vllm.vllm_worker_async import ( @@ -102,16 +90,6 @@ LOGGER = logging.getLogger(__name__) -# Annotations NeMo-Gym's Dynamo dialect (vllm_model with dynamo_frontend, Gym#4817) -# sets on its requests; Dynamo forwards nvext.annotations to the engine verbatim. -GYM_ROLLOUT_ANNOTATION = "nemo_gym:rollout" -# A token-capture call's admission JSON. -GYM_CAPTURE_ANNOTATION_PREFIX = "ng_capture:" -# Without capture: the last echoed assistant turn's prompt + generation token -# IDs, which the native server reads from the messages instead. -GYM_PREFIX_ANNOTATION_PREFIX = "ng_prefix:" -_GYM_SAMPLING_KEYS = ("temperature", "top_p", "val_temperature", "val_top_p") - # The vLLM modality each key of Dynamo's multi_modal_data holds. _DYNAMO_MEDIA_MODALITIES = { "image_url": "image", @@ -172,11 +150,6 @@ def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: return None if rollout is None else json.loads(rollout) -def _as_f32(value: Optional[float]) -> Optional[float]: - """Dynamo's frontend carries sampling floats as f32 (0.7 arrives as 0.69999998).""" - return None if value is None else float(np.float32(value)) - - def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. @@ -325,11 +298,6 @@ def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int self._llm = host.llm self._max_model_len = self._llm.vllm_config.model_config.max_model_len self._defaults = self._llm.model_config.get_diff_sampling_param() - # The train and validation sampling NeMo-Gym rollouts must match, at the - # f32 precision the frontend delivers. - self._gym_sampling_profile = { - key: _as_f32(host.cfg[key]) for key in _GYM_SAMPLING_KEYS - } @classmethod async def from_args(cls, argv=None): @@ -393,11 +361,6 @@ async def _generate( async for chunk in self._generate_rollout(request, rollout, context): yield chunk return - annotations = request.get("annotations") or () - if GYM_ROLLOUT_ANNOTATION in annotations: - async for chunk in self._generate_gym(request, context): - yield chunk - return engine_input = await self._engine_input(request) prompt_len = len(engine_input["prompt_token_ids"]) self._check_prompt_fits(prompt_len) @@ -511,191 +474,6 @@ async def _generate_rollout( chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) yield chunk - async def _generate_gym( - self, request: GenerateRequest, context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - """One NeMo-Gym model call, served as the native chat endpoint serves it. - - The request was rendered by the frontend. As on the native server, it must - sample on-policy, its echoed or staged prefix replaces the rendered - history, output is clamped to the context, and a token-capture call stages - its delta from this process. Tokens, log probabilities and routed experts - come from vLLM's final output through the native endpoint's helpers. One - chunk carries the whole completion and, in ``engine_data``, the token - arrays Gym's dialect reads, or a captured call's commit coordinates. - """ - host = self._host - sampling: dict[str, Any] = request.get("sampling_options") or {} - sampling_error = off_policy_sampling_error( - temperature=_as_f32(sampling.get("temperature")), - top_p=_as_f32(sampling.get("top_p")), - top_k=sampling.get("top_k"), - generation_config=self._gym_sampling_profile, - ) - if sampling_error is not None: - raise InvalidArgument(sampling_error) - if (sampling.get("n") or 1) != 1: - raise InvalidArgument( - "NeMo-Gym calls through Dynamo take n=1: engine_data carries one " - "choice's tokens." - ) - # The profile's own values, so vLLM samples exactly as natively. - validation = ( - _as_f32(sampling["temperature"]) - != self._gym_sampling_profile["temperature"] - or _as_f32(sampling["top_p"]) != self._gym_sampling_profile["top_p"] - ) - temperature, top_p = ( - (host.cfg["val_temperature"], host.cfg["val_top_p"]) - if validation - else (host.cfg["temperature"], host.cfg["top_p"]) - ) - - capture_context = _annotation_value(request, GYM_CAPTURE_ANNOTATION_PREFIX) - admission = ( - None - if capture_context is None - else host._parse_capture_admission(json.loads(capture_context)) - ) - prefix_token_ids = None - if admission is not None: - if admission.mode == "token_in": - # A blocking data-plane read; keep it off the engine loop. - prefix_token_ids = await asyncio.to_thread( - host._resolve_admission_prefix, admission - ) - elif ( - echoed := _annotation_value(request, GYM_PREFIX_ANNOTATION_PREFIX) - ) is not None: - prefix_token_ids = json.loads(echoed) - # As the native endpoint does: a captured call reads the processor's own - # pixels, not cache references (#4124). - engine_input = await self._engine_input( - request, skip_mm_cache=admission is not None and host._capture_media - ) - template_token_ids = engine_input["prompt_token_ids"] - self._check_prompt_fits(len(template_token_ids)) - token_ids, splice = template_token_ids, None - if prefix_token_ids is not None: - # The frontend rendered the whole conversation; replace its history - # with the exact tokens the model generated for it. The engine sees no - # messages, so the prefix's EOS count stands in for the native - # endpoint's render of the history. - tokenizer = self._llm.tokenizer - splice = splice_prefix_tokens( - tokenizer=tokenizer, - model_prefix_token_ids=prefix_token_ids, - template_token_ids=template_token_ids, - template_prefix_eos_count=derive_prefix_eos_count( - prefix_token_ids, eos_token_id=tokenizer.eos_token_id - ), - ) - token_ids = splice.token_ids - self._check_prompt_fits(len(token_ids)) - try: - # Off the loop: a retained-media check reads staged calls. - media = await asyncio.to_thread( - host._capture_request_media, - engine_input, - admission=admission, - splice=splice, - ) - except MediaCaptureRejected as error: - raise InvalidArgument(f"{error} ({error.code})") from error - if ( - media is None - and splice is not None - and (placeholders := engine_input.get("mm_placeholders")) - ): - # Media capture already moved them to spliced coordinates. - engine_input["mm_placeholders"] = remap_multimodal_placeholders( - template_token_ids=template_token_ids, - final_token_ids=token_ids, - mm_placeholders=placeholders, - splice=splice, - ) - engine_input["prompt_token_ids"] = token_ids - - stop = (request.get("stop_conditions") or {}).get("stop") - overrides: dict[str, Any] = { - "temperature": temperature, - "top_p": top_p, - # As the native endpoint does: Gym requests leave top_k to NeMo RL. - "top_k": -1, - } - if stop: - # vLLM stops on them as natively, so the token arrays match; the - # frontend then cuts the text at the same place. - overrides.update(stop=stop, detokenize=True) - if admission is not None: - # A captured call stages the sampled tokens' log probabilities. - overrides["logprobs"] = 0 - sampling_params = build_sampling_params( - request, - self._defaults, - output_room=self._max_model_len - len(token_ids), - overrides=overrides, - output_kind=RequestOutputKind.FINAL_ONLY, - ) - try: - self._llm.check_admission(1, context.id()) - except GracefulHTTPError as error: - raise HttpError(int(error.http_status), error.message) from error - - if admission is not None: - host._begin_request_capture( - request, - token_ids, - admission=admission, - prefix_token_ids=prefix_token_ids, - media=media, - ) - staged = False - try: - final_output = None - async for output in self._llm.generate( - engine_input, sampling_params, context.id() - ): - final_output = output - if final_output is None: - raise RuntimeError(f"No output received for request {context.id()}") - if admission is not None and context.is_stopped(): - # Dynamo aborts the vLLM request on a disconnect but never - # cancels this generator: fail the capture (below), as the - # native endpoint's cancellation does, instead of staging it. - return - completion = final_output.outputs[0] - chunk: GenerateChunk = { - "token_ids": list(completion.token_ids), - "index": 0, - "finish_reason": completion.finish_reason, - "completion_usage": _usage(final_output, len(completion.token_ids)), - } - if admission is not None: - chunk["engine_data"] = { - "ng_commit_coords": await self._stage_gym_call( - request, final_output - ) - } - staged = True - else: - chunk["engine_data"] = self._gym_token_data(final_output) - log_probs, top_logprobs = extract_from_completion_output( - completion, 0, tokenizer=self._llm.tokenizer, include_bytes=True - ) - if log_probs is not None: - chunk["log_probs"] = log_probs - chunk["top_logprobs"] = top_logprobs - yield chunk - finally: - if admission is not None and not staged: - host._abort_request_capture( - request, - reason=( - "client_disconnect" if context.is_stopped() else "engine_error" - ), - ) - async def _load_media( self, request: GenerateRequest ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: @@ -748,9 +526,7 @@ async def _load_media( mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) return mm_data, mm_uuids - async def _engine_input( - self, request: GenerateRequest, *, skip_mm_cache: bool = False - ) -> dict[str, Any]: + async def _engine_input(self, request: GenerateRequest) -> dict[str, Any]: """The processed vLLM prompt for the request, as the native chat endpoint builds it. The frontend's tokens carry one unexpanded placeholder per media item; @@ -763,15 +539,13 @@ async def _engine_input( prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media if request.get("mm_processor_kwargs"): prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] - return await self._llm.renderer.process_for_engine_async( - prompt, time.time(), skip_mm_cache=skip_mm_cache - ) + return await self._llm.renderer.process_for_engine_async(prompt, time.time()) def _check_prompt_fits(self, prompt_len: int) -> None: """Refuse a prompt that leaves no room for output, as the native endpoint does. - NeMo-Gym reads "context length" in a 400 as a length refusal rather than an - engine failure. + OpenAI clients read "context length" in a 400 as a length refusal rather + than an engine failure. """ if prompt_len >= self._max_model_len: raise InvalidArgument( @@ -780,62 +554,6 @@ def _check_prompt_fits(self, prompt_len: int) -> None: "output tokens." ) - def _gym_token_data(self, final_output: Any) -> dict[str, Any]: - """The token fields Gym's dialect lifts onto the message, as the native endpoint reports them.""" - completion = final_output.outputs[0] - token_data: dict[str, Any] = { - "prompt_token_ids": list(final_output.prompt_token_ids), - "completion_token_ids": list(completion.token_ids), - } - if completion.logprobs is not None: - _, token_data["completion_logprobs"] = chat_generation_tokens(completion) - if self._host._return_routed_experts_enabled(): - # Full prompt + generation rows, as the native endpoint attaches them. - token_data["routed_experts"] = chat_routed_experts( - final_output, - completion, - prompt_token_count=len(final_output.prompt_token_ids), - device=torch.device("cpu"), - logger=LOGGER, - routed_experts_dtype=self._host.routed_experts_dtype, - ) - return token_data - - async def _stage_gym_call(self, request: GenerateRequest, final_output: Any) -> Any: - """Stage a captured call from this process; return its commit coordinates.""" - host = self._host - completion = final_output.outputs[0] - generation_token_ids, generation_log_probs = chat_generation_tokens(completion) - message: dict[str, Any] = { - "generation_token_ids": generation_token_ids, - "generation_log_probs": generation_log_probs, - } - if host._return_routed_experts_enabled(): - message["routed_experts"] = chat_routed_experts( - final_output, - completion, - prompt_token_count=len(final_output.prompt_token_ids), - device=torch.device("cpu"), - logger=LOGGER, - routed_experts_dtype=host.routed_experts_dtype, - ) - # The native endpoint's response shape; the prompt is the one recorded at - # begin_call. - content = { - "choices": [ - { - "index": 0, - "finish_reason": completion.finish_reason, - "message": message, - } - ] - } - # The sink write is a blocking data-plane round trip. - content = await asyncio.to_thread( - host._finish_request_capture, request, content - ) - return content["ng_commit_coords"] - async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) diff --git a/nemo_rl/models/generation/openai_server_utils.py b/nemo_rl/models/generation/openai_server_utils.py index 53a757dc04f..f01cd1f6091 100644 --- a/nemo_rl/models/generation/openai_server_utils.py +++ b/nemo_rl/models/generation/openai_server_utils.py @@ -58,48 +58,13 @@ def replace_prefix_tokens( ).token_ids -def _eos_token_id_set(eos_token_id: int | Collection[int]) -> frozenset[int]: - eos_token_ids = ( - frozenset([eos_token_id]) - if isinstance(eos_token_id, int) - else frozenset(eos_token_id) - ) - assert eos_token_ids, "EOS token id collection must not be empty" - return eos_token_ids - - -def derive_prefix_eos_count( - model_prefix_token_ids: list[int], *, eos_token_id: int | Collection[int] -) -> int: - """The EOS count a template render of the prefix's history would hold. - - For hosts that see only the rendered prompt, not the messages (the managed - Dynamo engine receives token ids from its frontend), this stands in for - counting EOS in ``template_prefix_token_ids``. Every EOS in the prefix - closed one message, and a template closes the final assistant turn with one - EOS even when the model was truncated before emitting it. The two counts - agree when the history up to the prefix's last assistant turn is the - history the prefix was generated from (Gym's lineage admission checks this - for captured calls; without capture it is assumed, as on the native server) - and the model emitted no EOS text as ordinary tokens. A render with fewer - EOS than this count fails the splice; an extra EOS before the boundary cuts - early instead. - """ - eos_token_ids = _eos_token_id_set(eos_token_id) - count = sum(tid in eos_token_ids for tid in model_prefix_token_ids) - if not model_prefix_token_ids or model_prefix_token_ids[-1] not in eos_token_ids: - count += 1 - return count - - def splice_prefix_tokens( *, tokenizer: Any, model_prefix_token_ids: list[int], - template_prefix_token_ids: list[int] | None = None, + template_prefix_token_ids: list[int], template_token_ids: list[int], eos_token_id: int | Collection[int] | None = None, - template_prefix_eos_count: int | None = None, ) -> PrefixSplice: """This is a subroutine used inside the OpenAI-compatible Chat Completion server. @@ -167,24 +132,19 @@ def splice_prefix_tokens( toward the boundary, and when the model's prefix ends on one of them that exact id is kept in place of the template's, so the spliced prompt still starts with the model's tokens verbatim. - - Pass exactly one of ``template_prefix_token_ids`` and - ``template_prefix_eos_count``. The prefix render is only ever used for its - EOS count, so a host without the messages passes the count directly (see - :func:`derive_prefix_eos_count`). """ - if (template_prefix_token_ids is None) == (template_prefix_eos_count is None): - raise ValueError( - "Pass exactly one of template_prefix_token_ids and " - "template_prefix_eos_count" - ) if not model_prefix_token_ids: return PrefixSplice(template_token_ids, 0, 0) if eos_token_id is None: eos_token_id = tokenizer.eos_token_id assert eos_token_id is not None, "Tokenizer must have an EOS token ID" - eos_token_ids = _eos_token_id_set(eos_token_id) + eos_token_ids = ( + frozenset([eos_token_id]) + if isinstance(eos_token_id, int) + else frozenset(eos_token_id) + ) + assert eos_token_ids, "EOS token id collection must not be empty" # The model isn't guaranteed to end on EOS (e.g. it hit max_tokens); chat # templates always add one, so cut the model input to just before its EOS. @@ -197,11 +157,7 @@ def splice_prefix_tokens( # templates may strip prior reasoning blocks when re-rendering history; # EOS counting preserves the original generated reasoning tokens without # requiring a customized chat template. - count_needed = ( - template_prefix_eos_count - if template_prefix_token_ids is None - else sum(tid in eos_token_ids for tid in template_prefix_token_ids) - ) + count_needed = sum(tid in eos_token_ids for tid in template_prefix_token_ids) count_seen = 0 template_cut_start = -1 for pos, tid in enumerate(template_token_ids): @@ -215,14 +171,14 @@ def splice_prefix_tokens( message = ( f"EOS token #{count_needed} not found in template_token_ids " f"(only found {count_seen} EOS tokens total)!\n" + f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" + f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" ) - if template_prefix_token_ids is not None: - message += f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" - message += f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" if tokenizer is not None: - if template_prefix_token_ids is not None: - message += f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}" - message += f"\n\nTemplate repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" + message += ( + f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}\n\n" + f"Template repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" + ) raise AssertionError(message) boundary = ( diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 18fc37208e6..42eb7ccceed 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -191,21 +191,16 @@ def validate_settings(cls, master_config: "MasterConfig") -> None: """Reject pure-config vLLM settings the SC entrypoint cannot honor.""" generation_config = cast(VllmConfig, master_config.policy["generation"]) assert_reload_refit_config_supported(generation_config) - if dynamo_enabled(generation_config): - if master_config.async_rl.generation_router.enabled: - raise ValueError( - "async_rl.generation_router.enabled=true does not apply with " - "policy.generation.dynamo=true: NeMo-Gym calls the Dynamo " - "frontend, which routes the whole fleet behind one URL. " - "Disable the generation router." - ) - if master_config.async_rl.generation_fleet_health.enabled: - raise ValueError( - "async_rl.generation_fleet_health.enabled=true is not supported " - "with policy.generation.dynamo=true yet: the Dynamo frontend " - "picks engines itself, and a restarted shard does not register " - "with it. Disable fleet health." - ) + if ( + dynamo_enabled(generation_config) + and master_config.async_rl.generation_fleet_health.enabled + ): + raise ValueError( + "async_rl.generation_fleet_health.enabled=true is not supported " + "with policy.generation.dynamo=true yet: the Dynamo frontend " + "picks engines itself, and a restarted shard does not register " + "with it. Disable fleet health." + ) @staticmethod def init_cluster_placement_groups( @@ -722,18 +717,6 @@ def load_and_start(self) -> None: self.device_uuids = self._report_device_id() self._register_dynamo_engines() - @property - def client_base_urls(self) -> list[Optional[str]]: - """Where OpenAI clients such as NeMo-Gym send requests. - - The Dynamo frontend when it serves this fleet (``policy.generation.dynamo``); - otherwise every data-parallel shard's own server, as - ``dp_openai_server_base_urls``. Known before a deferred model load. - """ - if self.dynamo_frontend_url is not None: - return [self.dynamo_frontend_url] - return self.dp_openai_server_base_urls - def _start_dynamo_services(self) -> None: """Start the etcd, NATS and frontend this fleet's engines register with.""" from nemo_rl.models.generation.dynamo.managed_runtime import ( diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 1434af1c1d7..4b26762926a 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -757,10 +757,7 @@ def _capture_admission(self, request: Any) -> Any | None: context. The dict itself is never mutated: the admission is the typed, read-only contract that the prefix resolution and ``begin_call`` share. """ - return self._parse_capture_admission(getattr(request, "ng_capture", None)) - - def _parse_capture_admission(self, context: Any) -> Any | None: - """``_capture_admission`` for a raw ``ng_capture`` context (Dynamo's annotation).""" + context = getattr(request, "ng_capture", None) if self.token_capture is None or not context: return None # Deferred: nemo_gym is an optional extra absent in non-gym runs. diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index 23d08060f17..8f7e689228b 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -36,12 +36,7 @@ run_test() { # Dynamo overlays: native SingleController tests rerun with policy.generation.dynamo=true. # Each keeps the native test's checks and adds check_dynamo_overlay.sh's. run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh ++policy.generation.dynamo=true -run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true ++policy.generation.dynamo=true -# Gym's default mode: token arrays come back through the frontend in engine_data. -run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true -# Router replay with deferred routes: the engine stages each captured call's routed experts. -run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/grpo_async_gym_single_controller.sh b/tests/functional/grpo_async_gym_single_controller.sh index a69e17c847f..6acd46388cb 100755 --- a/tests/functional/grpo_async_gym_single_controller.sh +++ b/tests/functional/grpo_async_gym_single_controller.sh @@ -113,10 +113,6 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then if [[ "$*" == *token_capture.enabled=true* ]]; then # Nonzero only when a finalizer actor ran, i.e. capture really was on. EXTRA_CHECKS+=('max(data["train/finalize/total_ms"]) > 0') - if [[ "$*" == *router_replay.enabled=true* ]]; then - # Every captured row carries routed experts for the policy to replay. - EXTRA_CHECKS+=('min(data["train/finalize/routed_experts_row_coverage"]) == 1') - fi fi # Observed to be between 0.8-1.3 @@ -125,7 +121,3 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then 'max(data["train/reward"]) > 0' \ "${EXTRA_CHECKS[@]}" fi - -if [[ "$*" == *generation.dynamo=true* ]]; then - bash $SCRIPT_DIR/check_dynamo_overlay.sh $RUN_LOG -fi diff --git a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh index ee3da965de3..2d265e6153b 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh @@ -157,7 +157,3 @@ uv run --no-sync tests/check_metrics.py "${JSON_METRICS}" \ 'min(data["train/finalize/media_row_rate"]) == 1' \ 'max(data["train/gen_kl_error"]) < 0.05' \ 'all_finite(data["train/reward"])' - -if [[ "$*" == *generation.dynamo=true* ]]; then - bash "${SCRIPT_DIR}/check_dynamo_overlay.sh" "${RUN_LOG}" -fi diff --git a/tests/test_suites/disabled.txt b/tests/test_suites/disabled.txt index 2d43c913cc5..83b5213a584 100644 --- a/tests/test_suites/disabled.txt +++ b/tests/test_suites/disabled.txt @@ -102,3 +102,7 @@ research/template_project/tests/test_suites/llm/single_update_1n8g.sh # Full Flow-GRPO recipe requires an externally served GenRM OCR judge. research/flow_grpo/tests/test_suites/flow_grpo_qwen_image_ocr.sh + +# A Dynamo + NeMo-Gym recipe: policy.generation.dynamo does not serve NeMo-Gym +# rollouts yet, and setup refuses the pairing. +tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index b1e4b880346..fad27b6913d 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -275,7 +275,6 @@ tests/test_suites/llm/grpo-qwen3-30ba3b-2n8g-megatron_fused_linear_logprobs.sh tests/test_suites/llm/sft-gpt-oss-20b-1n8g-automodel-ep8.sh # Nemotron 3 Nano 30B A3B Base BF16 tests -tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-automodel-ep8.sh tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-automodel-lora.sh diff --git a/tests/unit/algorithms/test_distillation.py b/tests/unit/algorithms/test_distillation.py index f1c87daa2b4..8a34019eac8 100644 --- a/tests/unit/algorithms/test_distillation.py +++ b/tests/unit/algorithms/test_distillation.py @@ -1332,7 +1332,7 @@ class DummyVllmGeneration: def __init__(self, cluster, config, defer_model_load=False): self.cfg = config self.defer_model_load = defer_model_load - self.client_base_urls = ["http://reserved-vllm"] + self.dp_openai_server_base_urls = ["http://reserved-vllm"] # The per-job bearer token the real class generates when the # recipe requires one; Gym's model server must receive it. self.http_server_api_key = "k" diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index 1578e27c9a7..c161e7289d2 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -4408,7 +4408,7 @@ def offload_after_refit(self): class DummyVllmGeneration: weight_synchronizer = None - client_base_urls = ["http://vllm-worker.example/v1"] + dp_openai_server_base_urls = ["http://vllm-worker.example/v1"] # What the real class generates when the recipe requires a key. http_server_api_key = "per-job-key" diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index dfa4a991168..7fc9e3c9737 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -39,11 +39,7 @@ def _digest(label: str) -> str: @pytest.mark.parametrize( ("generation_backend", "expected"), - [ - ("vllm", "vllm_worker"), - ("megatron", "megatron_worker"), - ("dynamo", "dynamo_worker"), - ], + [("vllm", "vllm_worker"), ("megatron", "megatron_worker")], ) def test_external_staging_backend_maps_generation_backend( generation_backend: str, expected: str @@ -577,11 +573,8 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> assert should_use_nemo_gym(config) is True -@pytest.mark.parametrize("dynamo", [False, True]) -def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( - dynamo: bool, -) -> None: - """With Dynamo, Gym's policy model calls the frontend in Gym's Dynamo dialect.""" +def test_setup_nemo_gym_config_refuses_dynamo_serving() -> None: + """Dynamo does not serve NeMo-Gym's calls yet, so the pairing fails at startup.""" from nemo_rl.algorithms.grpo import MasterConfig from nemo_rl.environments.nemo_gym import ( setup_nemo_gym_config, @@ -594,7 +587,7 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( policy={ "generation": { "backend": "vllm", - "dynamo": dynamo, + "dynamo": True, "vllm_cfg": {"async_engine": False, "expose_http_server": False}, } }, @@ -602,89 +595,5 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( setup_nemo_gym_config(config, tokenizer=None) - vllm_cfg = config.policy["generation"]["vllm_cfg"] - assert vllm_cfg["async_engine"] is True - assert vllm_cfg["expose_http_server"] is True - assert should_use_nemo_gym(config) is True - if dynamo: - # A NeMo RL key each actor applies to its composed config, not a - # policy_model overlay a sharded plan would need an owner for. - assert config.env["nemo_gym"] == { - "dynamo_dialect": {"default_chat_template_kwargs": {}} - } - else: - assert "nemo_gym" not in config.env - - -@pytest.mark.parametrize( - ("vllm_model", "expected"), - [ - ({}, {"enable_thinking": True, "truncate_history_thinking": False}), - ( - {"chat_template_kwargs": {"enable_thinking": False}}, - {"enable_thinking": False, "truncate_history_thinking": False}, - ), - ( - {"extra_body": {"chat_template_kwargs": {"force_nonempty_content": True}}}, - { - "enable_thinking": True, - "truncate_history_thinking": False, - "force_nonempty_content": True, - }, - ), - ], -) -def test_gym_actor_config_dynamo_sends_the_native_default_template_kwargs( - vllm_model: dict, expected: dict -) -> None: - """The native server's default template kwargs sit under the ones Gym sends.""" - from nemo_rl.algorithms.grpo import MasterConfig - from nemo_rl.environments.nemo_gym import ( - build_nemo_gym_config, - setup_nemo_gym_config, - ) - - user_vllm_model = copy.deepcopy(vllm_model) - - config = MasterConfig.model_construct( - env={ - "should_use_nemo_gym": True, - "nemo_gym": { - "policy_model": {"responses_api_models": {"vllm_model": vllm_model}} - }, - }, - data={}, - policy={ - "generation": { - "backend": "vllm", - "dynamo": True, - "vllm_cfg": { - "async_engine": True, - "http_server_serving_chat_kwargs": { - "default_chat_template_kwargs": { - "enable_thinking": True, - "truncate_history_thinking": False, - } - }, - }, - } - }, - ) - - setup_nemo_gym_config(config, tokenizer=None) - gym_config = build_nemo_gym_config( - config.env, - base_urls=["http://frontend"], - model_name="test-model", - enable_router_replay=False, - use_fastokens=False, - )["initial_global_config_dict"] - - composed = gym_config["policy_model"]["responses_api_models"]["vllm_model"] - assert composed["dynamo_frontend"] is True - # The frontend refuses reasoning under both of Gym's keys. - assert composed["reasoning_field"] == "reasoning_content" - assert composed["chat_template_kwargs"] == expected - assert "dynamo_dialect" not in gym_config - # The user's overlay is left as written. - assert vllm_model == user_vllm_model + with pytest.raises(ValueError, match="does not serve NeMo-Gym rollouts yet"): + should_use_nemo_gym(config) diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index c13270ccdb8..990a4603c44 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -1016,73 +1016,6 @@ def test_build_nemo_gym_actors_hands_every_actor_the_policy_api_key( assert all(cfg["policy_api_key"] == "per-job-key" for cfg in cluster.actor_configs) -@pytest.mark.parametrize( - "env_configs", [_env_configs, _shard_env_configs], ids=["unsharded", "sharded"] -) -def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( - detected_uv_dirs, env_configs -): - """With Dynamo, each actor's composed config speaks Gym's Dynamo dialect. - - The dialect is not a policy_model overlay, which a sharded plan would reject - as unowned. A shard's own template kwargs still win over the native defaults. - """ - from nemo_rl.algorithms.grpo import MasterConfig - - env = env_configs() - sharded = "shards" in env["nemo_gym"] - if sharded: - env["nemo_gym"]["shards"][1]["overrides"] = { - "policy_model": { - "responses_api_models": { - "vllm_model": {"chat_template_kwargs": {"enable_thinking": False}} - } - } - } - config = MasterConfig.model_construct( - env=env, - data={}, - policy={ - "generation": { - "backend": "vllm", - "dynamo": True, - "vllm_cfg": { - "http_server_serving_chat_kwargs": { - "default_chat_template_kwargs": {"enable_thinking": True} - } - }, - } - }, - ) - setup_nemo_gym_config(config, tokenizer=None) - cluster = _FakeGymCluster() - - with _patched_cluster(cluster): - nemo_gym_mod.build_nemo_gym_actors( - config.env, - base_urls=["http://frontend"], - model_name="test-model", - tokenizer=_TOKENIZER, - enable_router_replay=False, - use_fastokens=False, - ) - - thinking = [ - cfg["initial_global_config_dict"]["policy_model"]["responses_api_models"][ - "vllm_model" - ]["chat_template_kwargs"]["enable_thinking"] - for cfg in cluster.actor_configs - ] - # judged, then the tools shard's two replicas with their own override. - assert thinking == ([True, False, False] if sharded else [True]) - for cfg in cluster.actor_configs: - gym_config = cfg["initial_global_config_dict"] - vllm_model = gym_config["policy_model"]["responses_api_models"]["vllm_model"] - assert vllm_model["dynamo_frontend"] is True - assert vllm_model["reasoning_field"] == "reasoning_content" - assert "dynamo_dialect" not in gym_config - - def test_build_nemo_gym_actors_spreads_every_replica_onto_its_own_node( detected_uv_dirs, ): diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 5de0f0ef08a..289f8b55bf7 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -25,7 +25,6 @@ dynamo_cfg, dynamo_chat_template, dynamo_parsers, - native_default_chat_template_kwargs, ) @@ -118,15 +117,7 @@ def test_frontend_argv_is_runtime_owned() -> None: host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg ) assert argv[argv.index("--router-mode") + 1] == "kv" - # Gym names a token-capture call's session; keep the session on one engine. - assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "1800" assert argv[-2:] == ["--foo", "1"] - no_affinity = DynamoCfg.model_validate( - {"frontend_args": {"session_affinity_ttl_secs": None}} - ) - assert "--router-session-affinity-ttl-secs" not in build_dynamo_frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=no_affinity - ) clashing = DynamoCfg.model_validate( {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} @@ -182,16 +173,9 @@ def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> Non configure_generation_config(config, _Tokenizer()) -def test_chat_template_and_default_kwargs_follow_the_native_server() -> None: - config = _serving( - chat_template="templates/chat.jinja", - default_chat_template_kwargs={"enable_thinking": False}, - ) +def test_chat_template_follows_the_native_server() -> None: + config = _serving(chat_template="templates/chat.jinja") assert dynamo_chat_template(config) == "templates/chat.jinja" - assert native_default_chat_template_kwargs(config) == {"enable_thinking": False} - # The older spelling the native server also accepts. - legacy = _serving(chat_template_kwargs={"enable_thinking": True}) - assert native_default_chat_template_kwargs(legacy) == {"enable_thinking": True} assert dynamo_chat_template(_serving()) is None @@ -212,11 +196,3 @@ def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( _serving(chat_template_content_format="auto", enable_auto_tools=True), _Tokenizer(), ) - - -def test_a_bearer_token_is_rejected_because_the_frontend_checks_none() -> None: - config = _vllm_config(dynamo=True) - config["vllm_cfg"]["http_server_api_key_required"] = True - - with pytest.raises(ValueError, match="checks no bearer token"): - configure_generation_config(config, _Tokenizer()) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index 836b70c5ee5..f12a5937bb7 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -42,17 +42,12 @@ from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) -from nemo_rl.models.generation.openai_server_utils import ( # noqa: E402 - derive_prefix_eos_count, - splice_prefix_tokens, -) from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, rollout_result_from_wire, rollout_result_to_wire, ) -from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) @@ -110,7 +105,9 @@ class _FakeRenderer: def __init__(self) -> None: self.prompts = [] - async def process_for_engine_async(self, prompt, arrival_time, *, skip_mm_cache): + async def process_for_engine_async( + self, prompt, arrival_time, *, skip_mm_cache=False + ): self.prompts.append((prompt, skip_mm_cache)) if "multi_modal_data" not in prompt: return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} @@ -230,7 +227,7 @@ def test_rollouts_are_served_like_generate_async() -> None: } -def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None: +def test_router_replay_routes_reach_rollouts_as_natively() -> None: routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) final = _output( [1, 2, 3], @@ -266,29 +263,6 @@ def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None for key in native: assert torch.equal(shipped[key], native[key]), key - # A Gym call without capture carries the routes in engine_data, as the - # native chat endpoint attaches them to the response. - gym_final = _output( - [1, 3, 4], - _completion([10, 11], "stop", routed_experts=routes[3:]), - prompt_routed_experts=routes[:3], - ) - gym_host = _host( - _FakeLLM([gym_final]), vllm_kwargs={"enable_return_routed_experts": True} - ) - gym_engine = native_engine.NativeVllmEngine(gym_host, leader_worker_idx=0) - - (gym_chunk,) = _collect(gym_engine, _gym_request()) - - assert gym_chunk["engine_data"]["routed_experts"] == chat_routed_experts( - gym_final, - gym_final.outputs[0], - prompt_token_count=3, - device=torch.device("cpu"), - logger=native_engine.LOGGER, - routed_experts_dtype=torch.int16, - ) - def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: llm = _FakeLLM() @@ -631,160 +605,6 @@ def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: assert llm.calls == [] -# --- NeMo-Gym's Dynamo dialect ------------------------------------------------- - - -def _f32(value: float) -> float: - return float(np.float32(value)) - - -def _gym_request( - *, temperature=0.7, top_p=0.9, prefix=None, sampling=None, **fields -) -> dict: - annotations = [native_engine.GYM_ROLLOUT_ANNOTATION] - if prefix is not None: - annotations.insert( - 0, native_engine.GYM_PREFIX_ANNOTATION_PREFIX + json.dumps(prefix) - ) - sampling_options = {"temperature": temperature, "top_p": top_p, **(sampling or {})} - return { - "token_ids": [1, 3, 4], - # The frontend carries sampling floats as f32. - "sampling_options": { - key: _f32(value) if isinstance(value, float) else value - for key, value in sampling_options.items() - }, - # Gym's token information: logprobs=true, top_logprobs=0. - "output_options": {"logprobs": 0}, - "annotations": annotations, - **fields, - } - - -def test_gym_calls_sample_with_nemo_rls_exact_profile() -> None: - final = _output([1, 3, 4], _completion([10, 11], "stop")) - llm = _FakeLLM([final]) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - (chunk,) = _collect( - engine, _gym_request(stop_conditions={"max_tokens": 100, "stop": [""]}) - ) - - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == [1, 3, 4] - # The configured values, not the f32 values the frontend delivered. - assert (params.temperature, params.top_p) == (0.7, 0.9) - assert params.top_k == vllm.SamplingParams(top_k=-1).top_k - assert params.output_kind == vllm.sampling_params.RequestOutputKind.FINAL_ONLY - # Output fits the context, and vLLM stops on the stop strings itself. - assert params.max_tokens == MAX_MODEL_LEN - 3 - assert (params.stop, params.detokenize) == ([""], True) - assert chunk["engine_data"] == { - "prompt_token_ids": [1, 3, 4], - "completion_token_ids": [10, 11], - "completion_logprobs": [-0.5, -0.5], - } - assert (chunk["token_ids"], chunk["finish_reason"]) == ([10, 11], "stop") - assert chunk["log_probs"] == [-0.5, -0.5] - - -def test_gym_validation_calls_sample_with_the_validation_profile() -> None: - llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - _collect(engine, _gym_request(temperature=0.0, top_p=1.0)) - - ((_, params, _),) = llm.calls - assert (params.temperature, params.top_p) == (0.0, 1.0) - - -@pytest.mark.parametrize( - ("request_fields", "error"), - [ - ({"temperature": 0.5}, "matches neither"), - ({"top_p": None}, "top_p must be set explicitly"), - ({"sampling": {"top_k": 5}}, "Top k sampling parameter"), - ({"sampling": {"n": 2}}, "take n=1"), - ], -) -def test_gym_calls_that_would_sample_off_policy_are_refused( - request_fields, error -) -> None: - llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match=error): - _collect(engine, _gym_request(**request_fields)) - - assert llm.calls == [] - - -def test_gym_echoed_prefix_replaces_the_rendered_history() -> None: - # The model generated [5, 6, EOS]; the frontend re-rendered that turn as [7, EOS]. - prefix, rendered = [1, 5, 6, EOS], [1, 7, EOS, 3, 4] - spliced = splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=prefix, - template_token_ids=rendered, - template_prefix_eos_count=derive_prefix_eos_count(prefix, eos_token_id=EOS), - eos_token_id=EOS, - ).token_ids - assert spliced != rendered - llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) - llm.tokenizer = _Tokenizer() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - (chunk,) = _collect(engine, _gym_request(prefix=prefix, token_ids=rendered)) - - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == spliced - assert params.max_tokens == MAX_MODEL_LEN - len(spliced) - assert chunk["engine_data"]["prompt_token_ids"] == spliced - - -def test_gym_calls_whose_splice_fills_the_context_are_refused() -> None: - prefix = [1, *range(20, 20 + MAX_MODEL_LEN), EOS] - llm = _FakeLLM() - llm.tokenizer = _Tokenizer() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match="maximum context length"): - _collect(engine, _gym_request(prefix=prefix, token_ids=[1, 7, EOS, 3])) - - assert llm.calls == [] - - -def test_gym_echoed_prefix_keeps_media_placeholders_on_their_tokens() -> None: - # Turn 1 ran [1, , 3] and generated [5, 6, EOS]; the frontend - # re-rendered that turn as [7, EOS] and appended a turn with a second image. - turn1 = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] - prefix = [*turn1, 5, 6, EOS] - rendered = [1, IMAGE_PLACEHOLDER, 3, 7, EOS, IMAGE_PLACEHOLDER, 4] - spliced = [*prefix, *[IMAGE_TOKEN] * IMAGE_TOKENS, 4] - llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) - llm.tokenizer = _Tokenizer() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - (chunk,) = _collect( - engine, - _gym_request( - prefix=prefix, - token_ids=rendered, - multi_modal_data={"image_url": [{"Url": _image_url()}] * 2}, - ), - ) - - ((engine_input, params, _),) = llm.calls - assert engine_input["prompt_token_ids"] == spliced - # The first image sits in the model's own prefix; the second moved with the - # longer generated turn, as the native endpoint remaps them. - assert [ - (span.offset, span.length) for span in engine_input["mm_placeholders"]["image"] - ] == [(1, IMAGE_TOKENS), (len(prefix), IMAGE_TOKENS)] - assert params.max_tokens == MAX_MODEL_LEN - len(spliced) - assert chunk["engine_data"]["prompt_token_ids"] == spliced - - @pytest.mark.parametrize( ("router_mode", "prefix_caching", "publishes"), [("kv", True, True), ("kv", False, False), ("round-robin", True, False)], diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py deleted file mode 100644 index 9c509a10fa0..00000000000 --- a/tests/unit/models/generation/test_dynamo_native_engine_capture.py +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""NeMo-Gym token capture through the Dynamo engine adapter. - -The worker's own capture lifecycle stages each call, as on the native chat -endpoint; only Gym's capture core is faked. -""" - -import json -from types import SimpleNamespace - -import numpy as np -import pytest -import torch - -pytestmark = [pytest.mark.nemo_gym, pytest.mark.vllm] -pytest.importorskip("dynamo._core") -pytest.importorskip("vllm") -# The vLLM lanes import this module without the nemo_gym extra. -pytest.importorskip("nemo_gym.token_id_capture.staging") - -from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 -from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 - DYNAMO_CAPTURE_ANNOTATION_PREFIX, - DYNAMO_GYM_REQUEST_ANNOTATION, - DYNAMO_PREFIX_ANNOTATION_PREFIX, -) -from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 -from responses_api_models.vllm_model.app import ( # noqa: E402 - _DYNAMO_ENGINE_DATA_FIELDS, -) - -from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 -from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 -from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 -from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 -from nemo_rl.utils.routed_experts_codec import decode_routed_experts # noqa: E402 -from tests.unit.models.generation.test_dynamo_native_engine import ( # noqa: E402 - EOS, - IMAGE_PLACEHOLDER, - IMAGE_TOKEN, - IMAGE_TOKENS, - _collect, - _completion, - _FakeLLM, - _gym_request, - _host, - _image_url, - _output, - _Tokenizer, -) - -COORDS = {"staging_key": "rollout-1/c2"} - - -class _Capture: - """Gym's capture core: records begin/complete/fail.""" - - adapter = VLLMCaptureAdapter() - - def __init__(self) -> None: - self.begun: list[tuple] = [] - self.staged: list[dict] = [] - self.failed: list[str] = [] - - def begin_call(self, admission, *, prefix_token_ids, stream): - self.begun.append((admission, prefix_token_ids, stream)) - return SimpleNamespace(admission=admission) - - def complete_call_from_response(self, call, payload, attachments=None): - self.staged.append(payload) - return SimpleNamespace(model_dump=lambda: dict(COORDS)) - - def fail_call(self, call, *, reason) -> None: - self.failed.append(reason) - - -def _capturing_host(llm, **cfg): - host = _host(llm, **cfg) - host.token_capture = _Capture() - host._capture_calls = {} - host._chain_prefix = ChainPrefixCache() - return host - - -def _captured_request(admission: CaptureAdmission, **fields) -> dict: - request = _gym_request(**fields) - # Gym's dynamo_worker handler: no logprobs, the admission in an annotation. - request["output_options"] = {} - request["annotations"] = [ - DYNAMO_CAPTURE_ANNOTATION_PREFIX + admission.model_dump_json(), - *request["annotations"], - ] - return request - - -def test_annotations_and_engine_data_are_the_ones_gyms_dynamo_dialect_uses() -> None: - assert native_engine.GYM_ROLLOUT_ANNOTATION == DYNAMO_GYM_REQUEST_ANNOTATION - assert native_engine.GYM_CAPTURE_ANNOTATION_PREFIX == ( - DYNAMO_CAPTURE_ANNOTATION_PREFIX - ) - assert native_engine.GYM_PREFIX_ANNOTATION_PREFIX == ( - DYNAMO_PREFIX_ANNOTATION_PREFIX - ) - # The engine_data keys Gym requires on a call without capture. - engine = native_engine.NativeVllmEngine(_host(_FakeLLM()), leader_worker_idx=0) - final = _output([1, 3, 4], _completion([10], "stop")) - assert engine._gym_token_data(final).keys() == _DYNAMO_ENGINE_DATA_FIELDS.keys() - - -def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: - # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. - prefix = [1, 5, 6, EOS] - admission = CaptureAdmission( - rollout_id="rollout-1", - model_call_id="c2", - parent_call_id="c1", - prev_len=len(prefix), - mode="token_in", - required_prefix_token_ids=prefix, - parent_chain_hash="0" * 64, - ) - spliced = [*prefix, 3, 4] - routes = np.arange(8 * 2 * 2, dtype=np.int16).reshape(8, 2, 2) - final = _output( - spliced, - _completion([10, 11], "stop", routed_experts=routes[6:]), - prompt_routed_experts=routes[:6], - ) - llm = _FakeLLM([final]) - llm.tokenizer = _Tokenizer() - host = _capturing_host(llm, vllm_kwargs={"enable_return_routed_experts": True}) - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - - (chunk,) = _collect( - engine, _captured_request(admission, token_ids=[1, 7, EOS, 3, 4]) - ) - - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == spliced - # The sampled tokens' log probabilities are staged; Gym asked for none. - assert params.logprobs == 0 - ((begun_admission, begun_prefix, stream),) = host.token_capture.begun - assert (begun_admission, begun_prefix, stream) == (admission, prefix, False) - (payload,) = host.token_capture.staged - assert payload["prompt_token_ids"] == spliced - message = payload["choices"][0]["message"] - assert message["generation_token_ids"] == [10, 11] - assert message["generation_log_probs"] == [-0.5, -0.5] - # As the native endpoint stages them: the full sequence's routes (none for - # the last token) from the boundary row, the prefix's last token, on. - staged_routes = decode_routed_experts(message["routed_experts"], torch.int16) - expected = torch.full((5, 2, 2), R3_MISSING_ROUTE_SENTINEL, dtype=torch.int16) - expected[:4] = torch.from_numpy(routes[3:7]) - assert torch.equal(staged_routes, expected) - # Only the commit coordinates go back through the frontend. - assert chunk["engine_data"] == {"ng_commit_coords": COORDS} - assert "log_probs" not in chunk - assert host._capture_calls == {} - - -def test_a_call_that_fails_after_admission_fails_its_capture() -> None: - admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" - ) - - class _FailingLLM(_FakeLLM): - async def generate(self, prompt, sampling_params, request_id): - raise RuntimeError("engine died") - yield # pragma: no cover - - host = _capturing_host(_FailingLLM()) - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - - with pytest.raises(RuntimeError, match="engine died"): - _collect(engine, _captured_request(admission)) - - assert host.token_capture.failed == ["engine_error"] - assert host._capture_calls == {} - - -def test_without_capture_installed_a_captured_call_is_served_uncaptured() -> None: - admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" - ) - llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) - host = _host(llm) - host.token_capture = None - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - - (chunk,) = _collect(engine, _captured_request(admission)) - - assert json.loads(json.dumps(chunk["engine_data"]))["completion_token_ids"] == [10] - - -def test_a_captured_media_call_stages_the_media_the_engine_ran_on() -> None: - admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" - ) - processed = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] - llm = _FakeLLM([_output(processed, _completion([10], "stop"))]) - host = _capturing_host(llm) - host._capture_media = True - captured = CapturedMedia((), None) - snapshots = [] - - def capture_request_media(engine_input, *, admission, splice): - snapshots.append((list(engine_input["prompt_token_ids"]), admission, splice)) - return captured - - host._capture_request_media = capture_request_media - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - - (chunk,) = _collect( - engine, - _captured_request( - admission, - token_ids=[1, IMAGE_PLACEHOLDER, 3], - multi_modal_data={"image_url": [{"Url": _image_url()}]}, - ), - ) - - # The processor's own pixels, not cache references, as natively (#4124). - ((_, skip_mm_cache),) = llm.renderer.prompts - assert skip_mm_cache is True - # Media is snapshotted from the processed prompt the engine runs. - assert snapshots == [(processed, admission, None)] - (payload,) = host.token_capture.staged - assert payload["prompt_token_ids"] == processed - assert chunk["engine_data"] == {"ng_commit_coords": COORDS} diff --git a/tests/unit/models/generation/test_openai_server_utils.py b/tests/unit/models/generation/test_openai_server_utils.py index 425e46442bc..223be2dc883 100644 --- a/tests/unit/models/generation/test_openai_server_utils.py +++ b/tests/unit/models/generation/test_openai_server_utils.py @@ -17,7 +17,6 @@ from nemo_rl.models.generation.openai_server_utils import ( PrefixSplice, - derive_prefix_eos_count, replace_prefix_tokens, splice_prefix_tokens, ) @@ -161,97 +160,6 @@ def test_splice_prefix_tokens_without_tokenizer_honors_explicit_eos_ids( ) assert result == expected assert result.token_ids[: len(model_prefix)] == model_prefix - # A host without the messages (managed Dynamo's engine) passes the EOS count - # derived from the prefix instead of the template prefix render. - assert ( - splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=model_prefix, - template_token_ids=template, - eos_token_id=eos_token_id, - template_prefix_eos_count=derive_prefix_eos_count( - model_prefix, eos_token_id=eos_token_id - ), - ) - == expected - ) - with pytest.raises(ValueError, match="exactly one"): - splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=model_prefix, - template_prefix_token_ids=template_prefix, - template_token_ids=template, - eos_token_id=eos_token_id, - template_prefix_eos_count=1, - ) - - -@pytest.mark.parametrize( - "model_name", ["Qwen/Qwen3-0.6B", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"] -) -def test_derived_prefix_eos_count_matches_real_template_renders(model_name): - """Across tool-calling turns with reasoning, the EOS count derived from the - exact prefix equals the count in the template's render of the same history.""" - transformers = pytest.importorskip("transformers") - try: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - except OSError as error: - pytest.skip(f"{model_name} tokenizer unavailable: {error}") - eos = tokenizer.eos_token_id - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Weather for a city.", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - }, - } - ] - - def render(messages, *, add_generation_prompt): - rendered = tokenizer.apply_chat_template( - messages, - tools=tools, - add_generation_prompt=add_generation_prompt, - tokenize=True, - ) - return list(getattr(rendered, "input_ids", rendered)) - - def generate(text): - return tokenizer.encode(text, add_special_tokens=False) + [eos] - - messages = [ - {"role": "system", "content": "You are a weather assistant."}, - {"role": "user", "content": "Weather in Paris?"}, - ] - replies = [ - "\nI should look it up.\n\n\nChecking the forecast.", - "\nIt is sunny.\n\n\nIt is sunny in Paris.", - ] - follow_ups = [ - {"role": "tool", "content": "sunny, 22C"}, - {"role": "user", "content": "And London?"}, - ] - chain = render(messages, add_generation_prompt=True) + generate(replies[0]) - for reply, follow_up, next_reply in zip(replies, follow_ups, replies[1:] + ["ok"]): - messages = [*messages, {"role": "assistant", "content": reply}, follow_up] - template_prefix = render(messages[:-1], add_generation_prompt=False) - template = render(messages, add_generation_prompt=True) - assert derive_prefix_eos_count(chain, eos_token_id=eos) == sum( - token == eos for token in template_prefix - ) - spliced = splice_prefix_tokens( - tokenizer=tokenizer, - model_prefix_token_ids=chain, - template_prefix_token_ids=template_prefix, - template_token_ids=template, - ).token_ids - chain = spliced + generate(next_reply) def test_replace_prefix_tokens_without_tokenizer_reports_missing_eos_without_decoding(): diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 8e342ae8274..7cf184b311b 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2571,7 +2571,7 @@ def test_vllm_validate_settings_accepts_missing_vllm_cfg_as_refit_disabled(): VllmGeneration.validate_settings(master_config) -def _dynamo_master_config(*, router=False, fleet_health=False): +def _dynamo_master_config(*, fleet_health=False): vllm_config = deepcopy(basic_vllm_test_config) vllm_config["dynamo"] = True vllm_config["vllm_cfg"]["async_engine"] = True @@ -2579,7 +2579,6 @@ def _dynamo_master_config(*, router=False, fleet_health=False): return types.SimpleNamespace( policy={"generation": vllm_config}, async_rl=types.SimpleNamespace( - generation_router=types.SimpleNamespace(enabled=router), generation_fleet_health=types.SimpleNamespace(enabled=fleet_health), ), ) @@ -2589,7 +2588,6 @@ def _dynamo_master_config(*, router=False, fleet_health=False): ("settings", "error"), [ ({}, None), - ({"router": True}, "routes the whole fleet behind one URL"), ({"fleet_health": True}, "Disable fleet health"), ], ) @@ -2603,19 +2601,13 @@ def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error @pytest.mark.parametrize("dynamo", [False, True]) -def test_vllm_token_rollouts_and_clients_use_the_dynamo_frontend_when_it_serves( - dynamo, -): +def test_vllm_token_rollouts_use_the_dynamo_frontend_when_it_serves(dynamo): generation = object.__new__(VllmGeneration) - generation.dp_openai_server_base_urls = ["http://a/v1", "http://b/v1"] generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None tokens = BatchedDataDict({"input_ids": torch.tensor([[1]])}) # The frontend request carries token IDs only, so media prompts stay native. media = BatchedDataDict({"input_ids": torch.tensor([[1]]), "vllm_content": [None]}) - assert generation.client_base_urls == ( - ["http://frontend/v1"] if dynamo else ["http://a/v1", "http://b/v1"] - ) assert generation._dynamo_serves(tokens) is dynamo assert not generation._dynamo_serves(media) diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index 25b5a2e19ee..3414010f089 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -230,7 +230,6 @@ def _worker_with_capture(sink: _MemorySink): for name in ( "_fetch_chain_prefix", "_capture_admission", - "_parse_capture_admission", "_resolve_admission_prefix", "_enter_request_prefix", ): diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index 4328d05736b..d81d66628d0 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -2222,7 +2222,7 @@ def test_nemo_gym_wires_env_handle(self, patched_factories): assert data_kwargs["is_vlm"] is True mock_spinup.assert_called_once_with( mc.env, - base_urls=patched_factories["fake_gen"].client_base_urls, + base_urls=patched_factories["fake_gen"].dp_openai_server_base_urls, model_name="test-model", # Reaches the actor once, at spinup, rather than riding along with every # run_rollouts call. @@ -2375,44 +2375,6 @@ def test_token_capture_always_creates_finalizer_actor_pool( _, setup_kwargs = generation.setup_token_capture.call_args assert setup_kwargs["capture_media"] is with_processor - def test_dynamo_token_capture_tells_gym_to_speak_dynamo(self, patched_factories): - """The same vLLM workers stage the calls; Gym reaches them through the frontend.""" - mc = _make_master_config(backend="vllm") - mc.policy["generation"].update( - { - "model_name": "test-model", - "stop_strings": None, - "stop_token_ids": None, - "top_k": None, - "dynamo": True, - "vllm_cfg": {"async_engine": True, "expose_http_server": True}, - } - ) - mc.token_capture.enabled = True - mc.env = {"should_use_nemo_gym": True, "nemo_gym": {}} - patched_factories["setup_response_data"].return_value = (list(range(8)), None) - - with ( - patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), - patch.object( - sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() - ) as mock_spinup, - patch.object(sc_setup_mod, "validate_dataset_agent_coverage"), - patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), - patch( - "nemo_rl.experience.rollout_reassembler_actor.create_rollout_reassembler_actors", - return_value=[MagicMock()], - ), - ): - setup_single_controller(mc, MagicMock(pad_token_id=9)) - - assert mc.token_capture.generation_backend == "dynamo" - assert mock_spinup.call_args.kwargs["token_capture"]["generation_backend"] == ( - "dynamo" - ) - generation, _ = patched_factories["_build_generation"].return_value - generation.setup_token_capture.assert_called() - def test_nemo_gym_coverage_failure_shuts_down_shards(self, patched_factories): mc = _make_master_config(colocated=False, backend="vllm") mc.policy["generation"]["model_name"] = "test-model" From 3f45420d1fb678904471fb5f553bca10eb5901bd Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 09:56:40 -0700 Subject: [PATCH 08/82] feat(dynamo): serve NeMo-Gym through the Dynamo frontend Restores NeMo-Gym support under policy.generation.dynamo=true, split out of the Dynamo core PR at review request: - Gym's policy model calls the frontend in Gym's Dynamo dialect (NVIDIA-NeMo/Gym#4817); the engine serves those calls as the native chat endpoint does: on-policy sampling check, echoed-prefix splice, context clamp, token fields and routed experts in engine_data. - Token capture stages from the engine process (dynamo_worker backend), with frontend session affinity keeping a rollout on one engine. - The Dynamo L1 regains its three Gym overlays, and the swe1 Dynamo + Gym nightly is re-enabled. - Gym is pinned to Gym#4817's rebased head b0e681c; uv.lock records Gym's updated requirements (no resolved versions change). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/design-docs/dynamo-integration.md | 18 +- docs/guides/dynamo-generation.md | 60 +++- ...grpo_math_1B_dynamo_single_controller.yaml | 3 + nemo_rl/algorithms/distillation.py | 2 +- nemo_rl/algorithms/grpo.py | 6 +- .../single_controller_utils/config.py | 4 +- .../single_controller_utils/setup.py | 9 +- nemo_rl/environments/nemo_gym.py | 63 +++- nemo_rl/environments/nemo_gym_shards.py | 7 +- nemo_rl/models/generation/__init__.py | 6 + nemo_rl/models/generation/dynamo/arguments.py | 6 + nemo_rl/models/generation/dynamo/config.py | 22 ++ .../generation/dynamo/managed_runtime.py | 4 +- .../models/generation/dynamo/native_engine.py | 292 +++++++++++++++++- .../models/generation/openai_server_utils.py | 72 ++++- .../models/generation/vllm/vllm_generation.py | 37 ++- .../generation/vllm/vllm_worker_async.py | 5 +- .../functional/L1_Functional_Tests_Dynamo.sh | 5 + .../grpo_async_gym_single_controller.sh | 8 + ...lm_token_capture_single_controller_1n2g.sh | 4 + tests/test_suites/disabled.txt | 4 - tests/test_suites/nightly.txt | 1 + tests/unit/algorithms/test_distillation.py | 2 +- tests/unit/algorithms/test_grpo.py | 2 +- .../test_nemo_gym_token_capture.py | 103 +++++- .../unit/environments/test_nemo_gym_utils.py | 67 ++++ .../models/generation/test_dynamo_config.py | 28 +- .../generation/test_dynamo_native_engine.py | 188 ++++++++++- .../test_dynamo_native_engine_capture.py | 241 +++++++++++++++ .../generation/test_openai_server_utils.py | 92 ++++++ .../models/generation/test_vllm_generation.py | 12 +- .../test_vllm_token_capture_hosting.py | 1 + tests/unit/single_controller/test_setup.py | 40 ++- uv.lock | 4 +- 35 files changed, 1334 insertions(+), 86 deletions(-) create mode 100644 tests/unit/models/generation/test_dynamo_native_engine_capture.py diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 3f6f20593a1..b0e681cf43f 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 3f6f20593a126ca9484793769a51d186aa1486c3 +Subproject commit b0e681cf43f20882a328fc964ac08680681e2909 diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 9c07ccda5c7..843dad2c897 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -32,17 +32,29 @@ implements its `LLMEngine` over the worker's `AsyncLLM`: The result travels in `engine_data`, and the driver assembles the same output with `request_pipeline.rollout_output`. + NeMo-Gym's calls also carry an annotation (Gym's Dynamo dialect). The engine + applies the native chat endpoint's rules to them: the on-policy sampling check + (`vllm/utils.off_policy_sampling_error`), the prefix splice + (`openai_server_utils.splice_prefix_tokens`) and the context clamp. A + token-capture call stages through the worker's own capture lifecycle + (`_begin_request_capture` / `_finish_request_capture`), and the token fields + and routed experts come from the native endpoint's helpers + (`chat_generation_tokens`, `chat_routed_experts`). + A request with media carries the frontend's tokens, with one unexpanded placeholder per item, and the media URLs. The adapter loads the media with vLLM's media connector. It then builds the processed prompt with the engine's renderer (`process_for_engine_async`), the same step that follows rendering - on the native endpoint. + on the native endpoint. The splice, the placeholder remap + (`remap_multimodal_placeholders`) and media capture (`_capture_request_media`) + all run on that processed prompt. - `abort` cancels the vLLM request on client disconnect. - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It publishes no token budget. The engine owns the context check and the output - cap, as the native server does. With a budget, the frontend would refuse NeMo - RL's own rollouts that fill the context. + cap, as the native server does. With a budget, the frontend would fill an + omitted `max_tokens` from the prompt it rendered, which a prefix splice then + changes, and would refuse NeMo RL's own rollouts that fill the context. - `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ publisher. It is set only when the engine caches prefixes. Each engine binds its own Linux abstract-namespace socket (`ipc://@...`), so engines on one node diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index b780dcda242..f1764202897 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -51,10 +51,11 @@ shows them all: | Key | Default | Meaning | |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | -| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies | -| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [Chat settings](#chat-settings) | +| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies; NeMo-Gym's calls keep Gym's timeouts | +| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | | `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | +| `frontend_args.session_affinity_ttl_secs` | 1800 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it | | `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | @@ -68,7 +69,7 @@ is set (the image sets it to `/opt/dynamo/bin`), and on `PATH` otherwise. | `generate_async` on text token prompts, router replay (R3) included | Frontend (KV-aware); the engine samples and assembles it exactly as native `generate_async` does | | NeMo RL's own multimodal rollouts | Native shard dispatch. Those rollouts send media to the engine directly rather than as OpenAI content | | `generate_text_async`, which evaluation uses | Native shard dispatch | -| NeMo-Gym's OpenAI requests | Not served yet; see [Not supported](#not-supported) | +| NeMo-Gym's OpenAI requests | Frontend, in Gym's Dynamo dialect; the engine serves them as the native chat endpoint does. See [NeMo-Gym](#nemo-gym) | A rollout through the frontend has the same `GenerationOutputSpec` as the native path: token IDs, log probabilities, truncation, routed experts with @@ -86,32 +87,67 @@ OpenAI clients of the frontend get what vLLM's own OpenAI server applies: for output. The engine, not the frontend, checks the context, so it checks the prompt it actually runs. -## Chat settings - -The native server's chat settings carry over to the frontend: +## NeMo-Gym + +NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect +(`vllm_model.dynamo_frontend`, from +[NVIDIA-NeMo/Gym#4817](https://github.com/NVIDIA-NeMo/Gym/pull/4817)). Gym marks +every call with an annotation, and the engine serves a marked call as the native +chat endpoint does: + +- Sampling must match `temperature`/`top_p` or `val_temperature`/`val_top_p`, + with `top_k` unset. vLLM samples with the configured values. +- The token prefix Gym echoes replaces the history the frontend rendered. +- Output is clamped to the context, and a prompt that leaves no room is refused + with the native endpoint's "maximum context length" error. +- Stop strings stop vLLM itself, so the token IDs match the native endpoint's. +- The response carries `prompt_token_ids`, `completion_token_ids` and + `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. + With router replay on, it also carries the call's `routed_experts`, as the + native endpoint attaches them. +- Images, video and audio in the messages are processed as on the native + endpoint. The frontend passes each media URL through with one unexpanded + placeholder per item. The engine loads the media with vLLM's media connector + and runs vLLM's multimodal processor on the frontend's tokens. The splice, + placeholder remap and media capture then work on that processed prompt. + +With `token_capture.enabled`, Gym uses its `dynamo_worker` capture backend. The +same vLLM workers stage each call's tokens, log probabilities, routed experts and +processed media from their own process, exactly as the native endpoint does, and +only the commit coordinates travel back through the frontend. The frontend keeps a rollout on +one engine (`session_affinity_ttl_secs`), as the native backend pins each Gym +session to one server. + +The native server's chat settings carry over: | Native setting (`vllm_cfg`) | Under Dynamo | |---|---| | `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | | `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | | `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | +| `http_server_serving_chat_kwargs.default_chat_template_kwargs` | Sent by Gym under its own `chat_template_kwargs` | | `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | -Where the behavior differs from the native endpoint: when a request sets -`chat_template_kwargs.force_nonempty_content`, Dynamo reports a reasoning-only -turn as content. +Where the behavior differs from the native endpoint: + +- The engine sees the rendered prompt, not the messages. It finds where the + echoed prefix ends by counting the prefix's EOS tokens, not by rendering the + history. +- When a request sets `chat_template_kwargs.force_nonempty_content`, Dynamo + reports a reasoning-only turn as content. ## Not supported Startup rejects these with `dynamo: true`: -- NeMo-Gym (`env.should_use_nemo_gym`): the frontend does not serve Gym's - calls yet. Run Gym recipes with `dynamo: false`. - `colocated.enabled: true`: a colocated engine sleeps while the policy trains, and the frontend would keep routing to it. The SingleController never runs colocated vLLM; the other entrypoints need non-colocated generation. - `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, and a restarted shard does not register with it yet. +- `async_rl.generation_router`: the frontend already routes the fleet behind one + URL. +- `vllm_cfg.http_server_api_key_required`: the frontend checks no bearer token. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data parallelism inside vLLM, which the async engine Dynamo serves does not. - Parser plugins with no Dynamo parser, and the chat settings above that the @@ -119,6 +155,8 @@ Startup rejects these with `dynamo: true`: - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. +A Gym call with `n > 1` is refused when it arrives. + ## Build the image ai-dynamo is part of the `vllm` extra, so every vLLM worker venv carries it. diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 1b62d39ba8d..9ad2b249a95 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -37,6 +37,9 @@ policy: tokenizer_cache_bytes: 52428800 # kv: route each request to the engine holding most of its prefix. router_mode: kv + # Keep a NeMo-Gym token-capture rollout on one engine while it idles up + # to this many seconds (null disables it). + session_affinity_ttl_secs: 1800 extra_cli_args: [] logger: diff --git a/nemo_rl/algorithms/distillation.py b/nemo_rl/algorithms/distillation.py index f3a5d883ae2..f00109256d3 100644 --- a/nemo_rl/algorithms/distillation.py +++ b/nemo_rl/algorithms/distillation.py @@ -556,7 +556,7 @@ def init_vllm_deferred(): def init_nemo_gym(): shard_set = build_nemo_gym_actors( env_configs, - base_urls=cast(list[str], deferred_vllm.dp_openai_server_base_urls), + base_urls=cast(list[str], deferred_vllm.client_base_urls), model_name=generation_config["model_name"], tokenizer=tokenizer, # Distillation does not configure vLLM for router replay. diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 31ef02d7895..505b1919261 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -1771,8 +1771,8 @@ def init_megatron_generation_task(policy_future): ) vllm_reserve_time = time.perf_counter() - vllm_reserve_t0 print( - f" ✓ Reserved {len(deferred_vllm.dp_openai_server_base_urls)} vLLM server URLs: " - f"{deferred_vllm.dp_openai_server_base_urls}", + f" ✓ Reserved {len(deferred_vllm.client_base_urls)} vLLM server URLs: " + f"{deferred_vllm.client_base_urls}", flush=True, ) @@ -1786,7 +1786,7 @@ def init_vllm_deferred(): def init_nemo_gym(): """Spin up NeMo Gym servers with the pre-assigned vLLM URLs.""" return _spinup_nemo_gym( - deferred_vllm.dp_openai_server_base_urls, + deferred_vllm.client_base_urls, generation_config["model_name"], policy_api_key=deferred_vllm.http_server_api_key, ) diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py index a6d42371804..758e836f402 100644 --- a/nemo_rl/algorithms/single_controller_utils/config.py +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -679,7 +679,9 @@ class TokenCaptureConfig(BaseModel, extra="allow"): capture_dir: Optional[str] = None # Generation backend hosting token capture. This is derived from # policy.generation.backend during setup; users should not set it separately. - generation_backend: Optional[Literal["vllm", "megatron"]] = None + # "dynamo" is the vLLM backend served through Dynamo (policy.generation.dynamo): + # the same workers host capture, and NeMo-Gym reaches them through the frontend. + generation_backend: Optional[Literal["vllm", "megatron", "dynamo"]] = None # Keep routed_experts out of canonical rows and assemble them on policy # workers from strict staged-fragment plans. defer_routed_experts_to_policy: bool = False diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index e3f65134811..d3dc6dd53c2 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -125,6 +125,7 @@ should_mask_flagged_samples, ) from nemo_rl.models.generation import resolve_generation_class +from nemo_rl.models.generation.dynamo.config import dynamo_enabled from nemo_rl.models.generation.fleet_health import ( FleetHealthPolicy, GenerationFleetHealth, @@ -1369,7 +1370,11 @@ def setup_single_controller( # Fill the derived ledger-hosting fields (see TokenCaptureConfig): a # per-run control-plane bearer token, the process-shared capture # directory used by every Gym worker, and the capture-host backend. - token_capture_cfg.generation_backend = generation_config["backend"] + token_capture_cfg.generation_backend = ( + "dynamo" + if dynamo_enabled(generation_config) + else generation_config["backend"] + ) if token_capture_cfg.control_auth_token is None: # Deferred import: only needed on the capture path. import secrets @@ -1856,7 +1861,7 @@ def _build_trainer_then_megatron_generation() -> tuple[ defer_model_load=True, ) defer_generation_model_load = True - gym_base_urls = generation.dp_openai_server_base_urls + gym_base_urls = generation.client_base_urls # Before the Gym task is built, so Gym can be handed the router's single URL. # These two statements are the only failable ones related to the port holder's creation. try: diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 435e35198f8..5ab370dbe65 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -98,7 +98,10 @@ RolloutDataFailure, http_status_is_infra, ) -from nemo_rl.models.generation.dynamo.config import dynamo_enabled +from nemo_rl.models.generation.dynamo.config import ( + dynamo_enabled, + native_default_chat_template_kwargs, +) from nemo_rl.models.generation.interfaces import ( resolve_routed_experts_dtype_name_for_model, should_use_async_rollouts, @@ -301,12 +304,6 @@ def should_use_nemo_gym(master_config: NemoGymCompatibleConfig) -> bool: return False generation_config = master_config.policy["generation"] - if dynamo_enabled(generation_config): - raise ValueError( - "policy.generation.dynamo=true does not serve NeMo-Gym rollouts yet. " - "Set policy.generation.dynamo=false to run this NeMo-Gym config on " - "NeMo RL's vLLM servers." - ) assert should_use_async_rollouts(generation_config), ( "❌ Error: In order to use NeMo-Gym, you must use a generation " "backend with `async_engine: true`!" @@ -492,9 +489,11 @@ def _external_staging_backend(token_capture: Dict[str, Any]) -> str: return "vllm_worker" if generation_backend == "megatron": return "megatron_worker" + if generation_backend == "dynamo": + return "dynamo_worker" raise ValueError( "token_capture.enabled requires setup-derived generation_backend to be " - f"'vllm' or 'megatron'; got {generation_backend!r}" + f"'vllm', 'megatron' or 'dynamo'; got {generation_backend!r}" ) @@ -2057,6 +2056,16 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_strings"] = None generation_config["stop_token_ids"] = None + if dynamo_enabled(generation_config): + # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect. + # Each actor applies it to its own composed config (_apply_dynamo_dialect), + # so it never becomes a policy_model overlay that a shard must claim. + config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = { + "default_chat_template_kwargs": native_default_chat_template_kwargs( + generation_config + ) + } + # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for # multi-turn multimodal postprocessing). @@ -2125,6 +2134,39 @@ def build_nemo_gym_config( ) +def _apply_dynamo_dialect( + nemo_gym_dict: dict[str, Any], default_chat_template_kwargs: Mapping[str, Any] +) -> None: + """Point one actor's policy model at the Dynamo frontend, in Gym's Dynamo dialect. + + The frontend takes token data in nvext instead of vLLM's request extensions. + Applied to the actor's composed config (a shard's overrides included), so + settings the user wrote there win. Copies the policy_model levels it + changes, so the caller's config is not mutated. + """ + policy_model = dict(nemo_gym_dict.get("policy_model") or {}) + models = dict(policy_model.get("responses_api_models") or {}) + vllm_model = dict(models.get("vllm_model") or {}) + vllm_model["dynamo_frontend"] = True + # The frontend reads `reasoning` as another name for `reasoning_content` and + # refuses an echoed assistant turn that carries both, Gym's default. + vllm_model.setdefault("reasoning_field", "reasoning_content") + # The native server merges these under every request's own template + # kwargs; the frontend has no such default, so Gym sends them. They sit + # under the kwargs Gym sends: vllm_model's own, else extra_body's. + if default_chat_template_kwargs: + gym_kwargs = vllm_model.get("chat_template_kwargs") or ( + (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} + ) + vllm_model["chat_template_kwargs"] = { + **default_chat_template_kwargs, + **gym_kwargs, + } + models["vllm_model"] = vllm_model + policy_model["responses_api_models"] = models + nemo_gym_dict["policy_model"] = policy_model + + def _build_gym_actor_config( nemo_gym_dict: dict[str, Any], *, @@ -2153,6 +2195,11 @@ def _build_gym_actor_config( invalid_tool_call_patterns = nemo_gym_dict.pop("invalid_tool_call_patterns", None) thinking_tags = nemo_gym_dict.pop("thinking_tags", None) tokenizer_config = nemo_gym_dict.pop("tokenizer_config", None) + dynamo_dialect = nemo_gym_dict.pop("dynamo_dialect", None) + if dynamo_dialect is not None: + _apply_dynamo_dialect( + nemo_gym_dict, dynamo_dialect["default_chat_template_kwargs"] + ) external_service_readiness = extract_external_service_readiness(nemo_gym_dict) port_range = { key: value diff --git a/nemo_rl/environments/nemo_gym_shards.py b/nemo_rl/environments/nemo_gym_shards.py index d8322a8d23c..b03b9cd9eb6 100644 --- a/nemo_rl/environments/nemo_gym_shards.py +++ b/nemo_rl/environments/nemo_gym_shards.py @@ -38,7 +38,12 @@ # Dict-shaped NeMo RL integration settings that are not Gym server entries. # The actor factory or rollout code consumes these separately. NEMO_RL_DICT_CONFIG_KEYS = frozenset( - {"effort_levels", "external_service_readiness", "tokenizer_config"} + { + "dynamo_dialect", + "effort_levels", + "external_service_readiness", + "tokenizer_config", + } ) # Ray placement-group strategies a shard plan may ask for. STRICT_SPREAD is the diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index c8547bbce41..8f4ea47ce9f 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -191,6 +191,12 @@ def configure_generation_config( dynamo_cfg(config) dynamo_parsers(config) check_dynamo_serving_chat_kwargs(config) + if config["vllm_cfg"].get("http_server_api_key_required"): + raise ValueError( + "policy.generation.vllm_cfg.http_server_api_key_required is not " + "supported with policy.generation.dynamo=true: NeMo-Gym calls the " + "Dynamo frontend, which checks no bearer token." + ) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( diff --git a/nemo_rl/models/generation/dynamo/arguments.py b/nemo_rl/models/generation/dynamo/arguments.py index e5a6abebee5..c08d9873200 100644 --- a/nemo_rl/models/generation/dynamo/arguments.py +++ b/nemo_rl/models/generation/dynamo/arguments.py @@ -111,6 +111,12 @@ def build_dynamo_frontend_argv( dynamo_cfg.frontend_args.router_mode, source="dynamo_cfg.frontend_args.router_mode", ) + if dynamo_cfg.frontend_args.session_affinity_ttl_secs is not None: + builder.add( + "--router-session-affinity-ttl-secs", + dynamo_cfg.frontend_args.session_affinity_ttl_secs, + source="dynamo_cfg.frontend_args.session_affinity_ttl_secs", + ) builder.add_raw( dynamo_cfg.frontend_args.extra_cli_args, source="dynamo_cfg.frontend_args.extra_cli_args", diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index b6ae598b1a1..7a6a61868d2 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -67,6 +67,12 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "least-loaded", "device-aware-weighted", ] = "kv" + # Idle TTL in seconds for keeping one session on one engine, or None to + # disable it. NeMo-Gym names the session on token-capture calls only (its + # rollout id), as the native backend pins each Gym session to one server; the + # engine splices a call's staged prefix after routing, so the KV router alone + # cannot see where that prefix is cached. + session_affinity_ttl_secs: PositiveInt | None = 1800 extra_cli_args: list[str] = Field(default_factory=list) @@ -79,6 +85,7 @@ class DynamoCfg(BaseModel, extra="forbid"): # NRL_VLLM_ASYNC_TIMEOUT_SECONDS. A timeout fails the attempt as a native one # does (GenerationUnavailable), and the rollout retry policy re-dispatches it # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). + # NeMo-Gym's own calls keep Gym's timeouts. request_timeout_s: PositiveFloat = 900.0 worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) @@ -189,6 +196,21 @@ def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: return serving.get("chat_template") +def native_default_chat_template_kwargs( + generation_config: Mapping[str, Any], +) -> dict[str, Any]: + """The template kwargs the native server merges under every request's own. + + ``default_chat_template_kwargs``, or the older ``chat_template_kwargs`` + spelling, of ``vllm_cfg.http_server_serving_chat_kwargs``. + """ + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + kwargs = serving.get("default_chat_template_kwargs") + if kwargs is None: + kwargs = serving.get("chat_template_kwargs") + return dict(kwargs or {}) + + def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" return bool(generation_config.get("dynamo")) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index ad9ad9b7659..e4a0fe9507e 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -167,8 +167,8 @@ def allocate_port(*, low: int, high: int) -> int: "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. - # Its v2 parser path preserves engine_data, which carries every NeMo - # RL rollout's result. + # Its v2 parser path preserves engine_data, which NeMo-Gym needs for + # exact token IDs and log probabilities. "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index ff31f98dfca..b3221afcf53 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -33,6 +33,7 @@ from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional +import numpy as np import torch from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names from dynamo.common.backend import ( @@ -65,6 +66,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase +from nemo_rl.data.captured_media import MediaCaptureRejected from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, dynamo_cfg, @@ -76,12 +78,22 @@ DYNAMO_ENDPOINT, ) from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION +from nemo_rl.models.generation.openai_server_utils import ( + derive_prefix_eos_count, + splice_prefix_tokens, +) from nemo_rl.models.generation.vllm.request_pipeline import ( RolloutResult, rollout_result, rollout_result_to_wire, ) -from nemo_rl.models.generation.vllm.utils import validate_rollout_prompt +from nemo_rl.models.generation.vllm.utils import ( + chat_generation_tokens, + chat_routed_experts, + off_policy_sampling_error, + remap_multimodal_placeholders, + validate_rollout_prompt, +) if TYPE_CHECKING: from nemo_rl.models.generation.vllm.vllm_worker_async import ( @@ -90,6 +102,16 @@ LOGGER = logging.getLogger(__name__) +# Annotations NeMo-Gym's Dynamo dialect (vllm_model with dynamo_frontend, Gym#4817) +# sets on its requests; Dynamo forwards nvext.annotations to the engine verbatim. +GYM_ROLLOUT_ANNOTATION = "nemo_gym:rollout" +# A token-capture call's admission JSON. +GYM_CAPTURE_ANNOTATION_PREFIX = "ng_capture:" +# Without capture: the last echoed assistant turn's prompt + generation token +# IDs, which the native server reads from the messages instead. +GYM_PREFIX_ANNOTATION_PREFIX = "ng_prefix:" +_GYM_SAMPLING_KEYS = ("temperature", "top_p", "val_temperature", "val_top_p") + # The vLLM modality each key of Dynamo's multi_modal_data holds. _DYNAMO_MEDIA_MODALITIES = { "image_url": "image", @@ -150,6 +172,11 @@ def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: return None if rollout is None else json.loads(rollout) +def _as_f32(value: Optional[float]) -> Optional[float]: + """Dynamo's frontend carries sampling floats as f32 (0.7 arrives as 0.69999998).""" + return None if value is None else float(np.float32(value)) + + def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. @@ -298,6 +325,11 @@ def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int self._llm = host.llm self._max_model_len = self._llm.vllm_config.model_config.max_model_len self._defaults = self._llm.model_config.get_diff_sampling_param() + # The train and validation sampling NeMo-Gym rollouts must match, at the + # f32 precision the frontend delivers. + self._gym_sampling_profile = { + key: _as_f32(host.cfg[key]) for key in _GYM_SAMPLING_KEYS + } @classmethod async def from_args(cls, argv=None): @@ -361,6 +393,11 @@ async def _generate( async for chunk in self._generate_rollout(request, rollout, context): yield chunk return + annotations = request.get("annotations") or () + if GYM_ROLLOUT_ANNOTATION in annotations: + async for chunk in self._generate_gym(request, context): + yield chunk + return engine_input = await self._engine_input(request) prompt_len = len(engine_input["prompt_token_ids"]) self._check_prompt_fits(prompt_len) @@ -474,6 +511,191 @@ async def _generate_rollout( chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) yield chunk + async def _generate_gym( + self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + """One NeMo-Gym model call, served as the native chat endpoint serves it. + + The request was rendered by the frontend. As on the native server, it must + sample on-policy, its echoed or staged prefix replaces the rendered + history, output is clamped to the context, and a token-capture call stages + its delta from this process. Tokens, log probabilities and routed experts + come from vLLM's final output through the native endpoint's helpers. One + chunk carries the whole completion and, in ``engine_data``, the token + arrays Gym's dialect reads, or a captured call's commit coordinates. + """ + host = self._host + sampling: dict[str, Any] = request.get("sampling_options") or {} + sampling_error = off_policy_sampling_error( + temperature=_as_f32(sampling.get("temperature")), + top_p=_as_f32(sampling.get("top_p")), + top_k=sampling.get("top_k"), + generation_config=self._gym_sampling_profile, + ) + if sampling_error is not None: + raise InvalidArgument(sampling_error) + if (sampling.get("n") or 1) != 1: + raise InvalidArgument( + "NeMo-Gym calls through Dynamo take n=1: engine_data carries one " + "choice's tokens." + ) + # The profile's own values, so vLLM samples exactly as natively. + validation = ( + _as_f32(sampling["temperature"]) + != self._gym_sampling_profile["temperature"] + or _as_f32(sampling["top_p"]) != self._gym_sampling_profile["top_p"] + ) + temperature, top_p = ( + (host.cfg["val_temperature"], host.cfg["val_top_p"]) + if validation + else (host.cfg["temperature"], host.cfg["top_p"]) + ) + + capture_context = _annotation_value(request, GYM_CAPTURE_ANNOTATION_PREFIX) + admission = ( + None + if capture_context is None + else host._parse_capture_admission(json.loads(capture_context)) + ) + prefix_token_ids = None + if admission is not None: + if admission.mode == "token_in": + # A blocking data-plane read; keep it off the engine loop. + prefix_token_ids = await asyncio.to_thread( + host._resolve_admission_prefix, admission + ) + elif ( + echoed := _annotation_value(request, GYM_PREFIX_ANNOTATION_PREFIX) + ) is not None: + prefix_token_ids = json.loads(echoed) + # As the native endpoint does: a captured call reads the processor's own + # pixels, not cache references (#4124). + engine_input = await self._engine_input( + request, skip_mm_cache=admission is not None and host._capture_media + ) + template_token_ids = engine_input["prompt_token_ids"] + self._check_prompt_fits(len(template_token_ids)) + token_ids, splice = template_token_ids, None + if prefix_token_ids is not None: + # The frontend rendered the whole conversation; replace its history + # with the exact tokens the model generated for it. The engine sees no + # messages, so the prefix's EOS count stands in for the native + # endpoint's render of the history. + tokenizer = self._llm.tokenizer + splice = splice_prefix_tokens( + tokenizer=tokenizer, + model_prefix_token_ids=prefix_token_ids, + template_token_ids=template_token_ids, + template_prefix_eos_count=derive_prefix_eos_count( + prefix_token_ids, eos_token_id=tokenizer.eos_token_id + ), + ) + token_ids = splice.token_ids + self._check_prompt_fits(len(token_ids)) + try: + # Off the loop: a retained-media check reads staged calls. + media = await asyncio.to_thread( + host._capture_request_media, + engine_input, + admission=admission, + splice=splice, + ) + except MediaCaptureRejected as error: + raise InvalidArgument(f"{error} ({error.code})") from error + if ( + media is None + and splice is not None + and (placeholders := engine_input.get("mm_placeholders")) + ): + # Media capture already moved them to spliced coordinates. + engine_input["mm_placeholders"] = remap_multimodal_placeholders( + template_token_ids=template_token_ids, + final_token_ids=token_ids, + mm_placeholders=placeholders, + splice=splice, + ) + engine_input["prompt_token_ids"] = token_ids + + stop = (request.get("stop_conditions") or {}).get("stop") + overrides: dict[str, Any] = { + "temperature": temperature, + "top_p": top_p, + # As the native endpoint does: Gym requests leave top_k to NeMo RL. + "top_k": -1, + } + if stop: + # vLLM stops on them as natively, so the token arrays match; the + # frontend then cuts the text at the same place. + overrides.update(stop=stop, detokenize=True) + if admission is not None: + # A captured call stages the sampled tokens' log probabilities. + overrides["logprobs"] = 0 + sampling_params = build_sampling_params( + request, + self._defaults, + output_room=self._max_model_len - len(token_ids), + overrides=overrides, + output_kind=RequestOutputKind.FINAL_ONLY, + ) + try: + self._llm.check_admission(1, context.id()) + except GracefulHTTPError as error: + raise HttpError(int(error.http_status), error.message) from error + + if admission is not None: + host._begin_request_capture( + request, + token_ids, + admission=admission, + prefix_token_ids=prefix_token_ids, + media=media, + ) + staged = False + try: + final_output = None + async for output in self._llm.generate( + engine_input, sampling_params, context.id() + ): + final_output = output + if final_output is None: + raise RuntimeError(f"No output received for request {context.id()}") + if admission is not None and context.is_stopped(): + # Dynamo aborts the vLLM request on a disconnect but never + # cancels this generator: fail the capture (below), as the + # native endpoint's cancellation does, instead of staging it. + return + completion = final_output.outputs[0] + chunk: GenerateChunk = { + "token_ids": list(completion.token_ids), + "index": 0, + "finish_reason": completion.finish_reason, + "completion_usage": _usage(final_output, len(completion.token_ids)), + } + if admission is not None: + chunk["engine_data"] = { + "ng_commit_coords": await self._stage_gym_call( + request, final_output + ) + } + staged = True + else: + chunk["engine_data"] = self._gym_token_data(final_output) + log_probs, top_logprobs = extract_from_completion_output( + completion, 0, tokenizer=self._llm.tokenizer, include_bytes=True + ) + if log_probs is not None: + chunk["log_probs"] = log_probs + chunk["top_logprobs"] = top_logprobs + yield chunk + finally: + if admission is not None and not staged: + host._abort_request_capture( + request, + reason=( + "client_disconnect" if context.is_stopped() else "engine_error" + ), + ) + async def _load_media( self, request: GenerateRequest ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: @@ -526,7 +748,9 @@ async def _load_media( mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) return mm_data, mm_uuids - async def _engine_input(self, request: GenerateRequest) -> dict[str, Any]: + async def _engine_input( + self, request: GenerateRequest, *, skip_mm_cache: bool = False + ) -> dict[str, Any]: """The processed vLLM prompt for the request, as the native chat endpoint builds it. The frontend's tokens carry one unexpanded placeholder per media item; @@ -539,13 +763,15 @@ async def _engine_input(self, request: GenerateRequest) -> dict[str, Any]: prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media if request.get("mm_processor_kwargs"): prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] - return await self._llm.renderer.process_for_engine_async(prompt, time.time()) + return await self._llm.renderer.process_for_engine_async( + prompt, time.time(), skip_mm_cache=skip_mm_cache + ) def _check_prompt_fits(self, prompt_len: int) -> None: """Refuse a prompt that leaves no room for output, as the native endpoint does. - OpenAI clients read "context length" in a 400 as a length refusal rather - than an engine failure. + NeMo-Gym reads "context length" in a 400 as a length refusal rather than an + engine failure. """ if prompt_len >= self._max_model_len: raise InvalidArgument( @@ -554,6 +780,62 @@ def _check_prompt_fits(self, prompt_len: int) -> None: "output tokens." ) + def _gym_token_data(self, final_output: Any) -> dict[str, Any]: + """The token fields Gym's dialect lifts onto the message, as the native endpoint reports them.""" + completion = final_output.outputs[0] + token_data: dict[str, Any] = { + "prompt_token_ids": list(final_output.prompt_token_ids), + "completion_token_ids": list(completion.token_ids), + } + if completion.logprobs is not None: + _, token_data["completion_logprobs"] = chat_generation_tokens(completion) + if self._host._return_routed_experts_enabled(): + # Full prompt + generation rows, as the native endpoint attaches them. + token_data["routed_experts"] = chat_routed_experts( + final_output, + completion, + prompt_token_count=len(final_output.prompt_token_ids), + device=torch.device("cpu"), + logger=LOGGER, + routed_experts_dtype=self._host.routed_experts_dtype, + ) + return token_data + + async def _stage_gym_call(self, request: GenerateRequest, final_output: Any) -> Any: + """Stage a captured call from this process; return its commit coordinates.""" + host = self._host + completion = final_output.outputs[0] + generation_token_ids, generation_log_probs = chat_generation_tokens(completion) + message: dict[str, Any] = { + "generation_token_ids": generation_token_ids, + "generation_log_probs": generation_log_probs, + } + if host._return_routed_experts_enabled(): + message["routed_experts"] = chat_routed_experts( + final_output, + completion, + prompt_token_count=len(final_output.prompt_token_ids), + device=torch.device("cpu"), + logger=LOGGER, + routed_experts_dtype=host.routed_experts_dtype, + ) + # The native endpoint's response shape; the prompt is the one recorded at + # begin_call. + content = { + "choices": [ + { + "index": 0, + "finish_reason": completion.finish_reason, + "message": message, + } + ] + } + # The sink write is a blocking data-plane round trip. + content = await asyncio.to_thread( + host._finish_request_capture, request, content + ) + return content["ng_commit_coords"] + async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) diff --git a/nemo_rl/models/generation/openai_server_utils.py b/nemo_rl/models/generation/openai_server_utils.py index f01cd1f6091..53a757dc04f 100644 --- a/nemo_rl/models/generation/openai_server_utils.py +++ b/nemo_rl/models/generation/openai_server_utils.py @@ -58,13 +58,48 @@ def replace_prefix_tokens( ).token_ids +def _eos_token_id_set(eos_token_id: int | Collection[int]) -> frozenset[int]: + eos_token_ids = ( + frozenset([eos_token_id]) + if isinstance(eos_token_id, int) + else frozenset(eos_token_id) + ) + assert eos_token_ids, "EOS token id collection must not be empty" + return eos_token_ids + + +def derive_prefix_eos_count( + model_prefix_token_ids: list[int], *, eos_token_id: int | Collection[int] +) -> int: + """The EOS count a template render of the prefix's history would hold. + + For hosts that see only the rendered prompt, not the messages (the managed + Dynamo engine receives token ids from its frontend), this stands in for + counting EOS in ``template_prefix_token_ids``. Every EOS in the prefix + closed one message, and a template closes the final assistant turn with one + EOS even when the model was truncated before emitting it. The two counts + agree when the history up to the prefix's last assistant turn is the + history the prefix was generated from (Gym's lineage admission checks this + for captured calls; without capture it is assumed, as on the native server) + and the model emitted no EOS text as ordinary tokens. A render with fewer + EOS than this count fails the splice; an extra EOS before the boundary cuts + early instead. + """ + eos_token_ids = _eos_token_id_set(eos_token_id) + count = sum(tid in eos_token_ids for tid in model_prefix_token_ids) + if not model_prefix_token_ids or model_prefix_token_ids[-1] not in eos_token_ids: + count += 1 + return count + + def splice_prefix_tokens( *, tokenizer: Any, model_prefix_token_ids: list[int], - template_prefix_token_ids: list[int], + template_prefix_token_ids: list[int] | None = None, template_token_ids: list[int], eos_token_id: int | Collection[int] | None = None, + template_prefix_eos_count: int | None = None, ) -> PrefixSplice: """This is a subroutine used inside the OpenAI-compatible Chat Completion server. @@ -132,19 +167,24 @@ def splice_prefix_tokens( toward the boundary, and when the model's prefix ends on one of them that exact id is kept in place of the template's, so the spliced prompt still starts with the model's tokens verbatim. + + Pass exactly one of ``template_prefix_token_ids`` and + ``template_prefix_eos_count``. The prefix render is only ever used for its + EOS count, so a host without the messages passes the count directly (see + :func:`derive_prefix_eos_count`). """ + if (template_prefix_token_ids is None) == (template_prefix_eos_count is None): + raise ValueError( + "Pass exactly one of template_prefix_token_ids and " + "template_prefix_eos_count" + ) if not model_prefix_token_ids: return PrefixSplice(template_token_ids, 0, 0) if eos_token_id is None: eos_token_id = tokenizer.eos_token_id assert eos_token_id is not None, "Tokenizer must have an EOS token ID" - eos_token_ids = ( - frozenset([eos_token_id]) - if isinstance(eos_token_id, int) - else frozenset(eos_token_id) - ) - assert eos_token_ids, "EOS token id collection must not be empty" + eos_token_ids = _eos_token_id_set(eos_token_id) # The model isn't guaranteed to end on EOS (e.g. it hit max_tokens); chat # templates always add one, so cut the model input to just before its EOS. @@ -157,7 +197,11 @@ def splice_prefix_tokens( # templates may strip prior reasoning blocks when re-rendering history; # EOS counting preserves the original generated reasoning tokens without # requiring a customized chat template. - count_needed = sum(tid in eos_token_ids for tid in template_prefix_token_ids) + count_needed = ( + template_prefix_eos_count + if template_prefix_token_ids is None + else sum(tid in eos_token_ids for tid in template_prefix_token_ids) + ) count_seen = 0 template_cut_start = -1 for pos, tid in enumerate(template_token_ids): @@ -171,14 +215,14 @@ def splice_prefix_tokens( message = ( f"EOS token #{count_needed} not found in template_token_ids " f"(only found {count_seen} EOS tokens total)!\n" - f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" - f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" ) + if template_prefix_token_ids is not None: + message += f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" + message += f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" if tokenizer is not None: - message += ( - f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}\n\n" - f"Template repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" - ) + if template_prefix_token_ids is not None: + message += f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}" + message += f"\n\nTemplate repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" raise AssertionError(message) boundary = ( diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 42eb7ccceed..18fc37208e6 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -191,16 +191,21 @@ def validate_settings(cls, master_config: "MasterConfig") -> None: """Reject pure-config vLLM settings the SC entrypoint cannot honor.""" generation_config = cast(VllmConfig, master_config.policy["generation"]) assert_reload_refit_config_supported(generation_config) - if ( - dynamo_enabled(generation_config) - and master_config.async_rl.generation_fleet_health.enabled - ): - raise ValueError( - "async_rl.generation_fleet_health.enabled=true is not supported " - "with policy.generation.dynamo=true yet: the Dynamo frontend " - "picks engines itself, and a restarted shard does not register " - "with it. Disable fleet health." - ) + if dynamo_enabled(generation_config): + if master_config.async_rl.generation_router.enabled: + raise ValueError( + "async_rl.generation_router.enabled=true does not apply with " + "policy.generation.dynamo=true: NeMo-Gym calls the Dynamo " + "frontend, which routes the whole fleet behind one URL. " + "Disable the generation router." + ) + if master_config.async_rl.generation_fleet_health.enabled: + raise ValueError( + "async_rl.generation_fleet_health.enabled=true is not supported " + "with policy.generation.dynamo=true yet: the Dynamo frontend " + "picks engines itself, and a restarted shard does not register " + "with it. Disable fleet health." + ) @staticmethod def init_cluster_placement_groups( @@ -717,6 +722,18 @@ def load_and_start(self) -> None: self.device_uuids = self._report_device_id() self._register_dynamo_engines() + @property + def client_base_urls(self) -> list[Optional[str]]: + """Where OpenAI clients such as NeMo-Gym send requests. + + The Dynamo frontend when it serves this fleet (``policy.generation.dynamo``); + otherwise every data-parallel shard's own server, as + ``dp_openai_server_base_urls``. Known before a deferred model load. + """ + if self.dynamo_frontend_url is not None: + return [self.dynamo_frontend_url] + return self.dp_openai_server_base_urls + def _start_dynamo_services(self) -> None: """Start the etcd, NATS and frontend this fleet's engines register with.""" from nemo_rl.models.generation.dynamo.managed_runtime import ( diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 4b26762926a..1434af1c1d7 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -757,7 +757,10 @@ def _capture_admission(self, request: Any) -> Any | None: context. The dict itself is never mutated: the admission is the typed, read-only contract that the prefix resolution and ``begin_call`` share. """ - context = getattr(request, "ng_capture", None) + return self._parse_capture_admission(getattr(request, "ng_capture", None)) + + def _parse_capture_admission(self, context: Any) -> Any | None: + """``_capture_admission`` for a raw ``ng_capture`` context (Dynamo's annotation).""" if self.token_capture is None or not context: return None # Deferred: nemo_gym is an optional extra absent in non-gym runs. diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index 8f7e689228b..23d08060f17 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -36,7 +36,12 @@ run_test() { # Dynamo overlays: native SingleController tests rerun with policy.generation.dynamo=true. # Each keeps the native test's checks and adds check_dynamo_overlay.sh's. run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh ++policy.generation.dynamo=true +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true ++policy.generation.dynamo=true +# Gym's default mode: token arrays come back through the frontend in engine_data. +run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true +# Router replay with deferred routes: the engine stages each captured call's routed experts. +run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/grpo_async_gym_single_controller.sh b/tests/functional/grpo_async_gym_single_controller.sh index 6acd46388cb..a69e17c847f 100755 --- a/tests/functional/grpo_async_gym_single_controller.sh +++ b/tests/functional/grpo_async_gym_single_controller.sh @@ -113,6 +113,10 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then if [[ "$*" == *token_capture.enabled=true* ]]; then # Nonzero only when a finalizer actor ran, i.e. capture really was on. EXTRA_CHECKS+=('max(data["train/finalize/total_ms"]) > 0') + if [[ "$*" == *router_replay.enabled=true* ]]; then + # Every captured row carries routed experts for the policy to replay. + EXTRA_CHECKS+=('min(data["train/finalize/routed_experts_row_coverage"]) == 1') + fi fi # Observed to be between 0.8-1.3 @@ -121,3 +125,7 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then 'max(data["train/reward"]) > 0' \ "${EXTRA_CHECKS[@]}" fi + +if [[ "$*" == *generation.dynamo=true* ]]; then + bash $SCRIPT_DIR/check_dynamo_overlay.sh $RUN_LOG +fi diff --git a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh index 2d265e6153b..ee3da965de3 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh @@ -157,3 +157,7 @@ uv run --no-sync tests/check_metrics.py "${JSON_METRICS}" \ 'min(data["train/finalize/media_row_rate"]) == 1' \ 'max(data["train/gen_kl_error"]) < 0.05' \ 'all_finite(data["train/reward"])' + +if [[ "$*" == *generation.dynamo=true* ]]; then + bash "${SCRIPT_DIR}/check_dynamo_overlay.sh" "${RUN_LOG}" +fi diff --git a/tests/test_suites/disabled.txt b/tests/test_suites/disabled.txt index 83b5213a584..2d43c913cc5 100644 --- a/tests/test_suites/disabled.txt +++ b/tests/test_suites/disabled.txt @@ -102,7 +102,3 @@ research/template_project/tests/test_suites/llm/single_update_1n8g.sh # Full Flow-GRPO recipe requires an externally served GenRM OCR judge. research/flow_grpo/tests/test_suites/flow_grpo_qwen_image_ocr.sh - -# A Dynamo + NeMo-Gym recipe: policy.generation.dynamo does not serve NeMo-Gym -# rollouts yet, and setup refuses the pairing. -tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index fad27b6913d..b1e4b880346 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -275,6 +275,7 @@ tests/test_suites/llm/grpo-qwen3-30ba3b-2n8g-megatron_fused_linear_logprobs.sh tests/test_suites/llm/sft-gpt-oss-20b-1n8g-automodel-ep8.sh # Nemotron 3 Nano 30B A3B Base BF16 tests +tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-automodel-ep8.sh tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-automodel-lora.sh diff --git a/tests/unit/algorithms/test_distillation.py b/tests/unit/algorithms/test_distillation.py index 8a34019eac8..f1c87daa2b4 100644 --- a/tests/unit/algorithms/test_distillation.py +++ b/tests/unit/algorithms/test_distillation.py @@ -1332,7 +1332,7 @@ class DummyVllmGeneration: def __init__(self, cluster, config, defer_model_load=False): self.cfg = config self.defer_model_load = defer_model_load - self.dp_openai_server_base_urls = ["http://reserved-vllm"] + self.client_base_urls = ["http://reserved-vllm"] # The per-job bearer token the real class generates when the # recipe requires one; Gym's model server must receive it. self.http_server_api_key = "k" diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index c161e7289d2..1578e27c9a7 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -4408,7 +4408,7 @@ def offload_after_refit(self): class DummyVllmGeneration: weight_synchronizer = None - dp_openai_server_base_urls = ["http://vllm-worker.example/v1"] + client_base_urls = ["http://vllm-worker.example/v1"] # What the real class generates when the recipe requires a key. http_server_api_key = "per-job-key" diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index 7fc9e3c9737..dfa4a991168 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -39,7 +39,11 @@ def _digest(label: str) -> str: @pytest.mark.parametrize( ("generation_backend", "expected"), - [("vllm", "vllm_worker"), ("megatron", "megatron_worker")], + [ + ("vllm", "vllm_worker"), + ("megatron", "megatron_worker"), + ("dynamo", "dynamo_worker"), + ], ) def test_external_staging_backend_maps_generation_backend( generation_backend: str, expected: str @@ -573,8 +577,11 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> assert should_use_nemo_gym(config) is True -def test_setup_nemo_gym_config_refuses_dynamo_serving() -> None: - """Dynamo does not serve NeMo-Gym's calls yet, so the pairing fails at startup.""" +@pytest.mark.parametrize("dynamo", [False, True]) +def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( + dynamo: bool, +) -> None: + """With Dynamo, Gym's policy model calls the frontend in Gym's Dynamo dialect.""" from nemo_rl.algorithms.grpo import MasterConfig from nemo_rl.environments.nemo_gym import ( setup_nemo_gym_config, @@ -587,7 +594,7 @@ def test_setup_nemo_gym_config_refuses_dynamo_serving() -> None: policy={ "generation": { "backend": "vllm", - "dynamo": True, + "dynamo": dynamo, "vllm_cfg": {"async_engine": False, "expose_http_server": False}, } }, @@ -595,5 +602,89 @@ def test_setup_nemo_gym_config_refuses_dynamo_serving() -> None: setup_nemo_gym_config(config, tokenizer=None) - with pytest.raises(ValueError, match="does not serve NeMo-Gym rollouts yet"): - should_use_nemo_gym(config) + vllm_cfg = config.policy["generation"]["vllm_cfg"] + assert vllm_cfg["async_engine"] is True + assert vllm_cfg["expose_http_server"] is True + assert should_use_nemo_gym(config) is True + if dynamo: + # A NeMo RL key each actor applies to its composed config, not a + # policy_model overlay a sharded plan would need an owner for. + assert config.env["nemo_gym"] == { + "dynamo_dialect": {"default_chat_template_kwargs": {}} + } + else: + assert "nemo_gym" not in config.env + + +@pytest.mark.parametrize( + ("vllm_model", "expected"), + [ + ({}, {"enable_thinking": True, "truncate_history_thinking": False}), + ( + {"chat_template_kwargs": {"enable_thinking": False}}, + {"enable_thinking": False, "truncate_history_thinking": False}, + ), + ( + {"extra_body": {"chat_template_kwargs": {"force_nonempty_content": True}}}, + { + "enable_thinking": True, + "truncate_history_thinking": False, + "force_nonempty_content": True, + }, + ), + ], +) +def test_gym_actor_config_dynamo_sends_the_native_default_template_kwargs( + vllm_model: dict, expected: dict +) -> None: + """The native server's default template kwargs sit under the ones Gym sends.""" + from nemo_rl.algorithms.grpo import MasterConfig + from nemo_rl.environments.nemo_gym import ( + build_nemo_gym_config, + setup_nemo_gym_config, + ) + + user_vllm_model = copy.deepcopy(vllm_model) + + config = MasterConfig.model_construct( + env={ + "should_use_nemo_gym": True, + "nemo_gym": { + "policy_model": {"responses_api_models": {"vllm_model": vllm_model}} + }, + }, + data={}, + policy={ + "generation": { + "backend": "vllm", + "dynamo": True, + "vllm_cfg": { + "async_engine": True, + "http_server_serving_chat_kwargs": { + "default_chat_template_kwargs": { + "enable_thinking": True, + "truncate_history_thinking": False, + } + }, + }, + } + }, + ) + + setup_nemo_gym_config(config, tokenizer=None) + gym_config = build_nemo_gym_config( + config.env, + base_urls=["http://frontend"], + model_name="test-model", + enable_router_replay=False, + use_fastokens=False, + )["initial_global_config_dict"] + + composed = gym_config["policy_model"]["responses_api_models"]["vllm_model"] + assert composed["dynamo_frontend"] is True + # The frontend refuses reasoning under both of Gym's keys. + assert composed["reasoning_field"] == "reasoning_content" + assert composed["chat_template_kwargs"] == expected + assert "dynamo_dialect" not in gym_config + # The user's overlay is left as written. + assert vllm_model == user_vllm_model diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index 990a4603c44..c13270ccdb8 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -1016,6 +1016,73 @@ def test_build_nemo_gym_actors_hands_every_actor_the_policy_api_key( assert all(cfg["policy_api_key"] == "per-job-key" for cfg in cluster.actor_configs) +@pytest.mark.parametrize( + "env_configs", [_env_configs, _shard_env_configs], ids=["unsharded", "sharded"] +) +def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( + detected_uv_dirs, env_configs +): + """With Dynamo, each actor's composed config speaks Gym's Dynamo dialect. + + The dialect is not a policy_model overlay, which a sharded plan would reject + as unowned. A shard's own template kwargs still win over the native defaults. + """ + from nemo_rl.algorithms.grpo import MasterConfig + + env = env_configs() + sharded = "shards" in env["nemo_gym"] + if sharded: + env["nemo_gym"]["shards"][1]["overrides"] = { + "policy_model": { + "responses_api_models": { + "vllm_model": {"chat_template_kwargs": {"enable_thinking": False}} + } + } + } + config = MasterConfig.model_construct( + env=env, + data={}, + policy={ + "generation": { + "backend": "vllm", + "dynamo": True, + "vllm_cfg": { + "http_server_serving_chat_kwargs": { + "default_chat_template_kwargs": {"enable_thinking": True} + } + }, + } + }, + ) + setup_nemo_gym_config(config, tokenizer=None) + cluster = _FakeGymCluster() + + with _patched_cluster(cluster): + nemo_gym_mod.build_nemo_gym_actors( + config.env, + base_urls=["http://frontend"], + model_name="test-model", + tokenizer=_TOKENIZER, + enable_router_replay=False, + use_fastokens=False, + ) + + thinking = [ + cfg["initial_global_config_dict"]["policy_model"]["responses_api_models"][ + "vllm_model" + ]["chat_template_kwargs"]["enable_thinking"] + for cfg in cluster.actor_configs + ] + # judged, then the tools shard's two replicas with their own override. + assert thinking == ([True, False, False] if sharded else [True]) + for cfg in cluster.actor_configs: + gym_config = cfg["initial_global_config_dict"] + vllm_model = gym_config["policy_model"]["responses_api_models"]["vllm_model"] + assert vllm_model["dynamo_frontend"] is True + assert vllm_model["reasoning_field"] == "reasoning_content" + assert "dynamo_dialect" not in gym_config + + def test_build_nemo_gym_actors_spreads_every_replica_onto_its_own_node( detected_uv_dirs, ): diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 289f8b55bf7..5de0f0ef08a 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -25,6 +25,7 @@ dynamo_cfg, dynamo_chat_template, dynamo_parsers, + native_default_chat_template_kwargs, ) @@ -117,7 +118,15 @@ def test_frontend_argv_is_runtime_owned() -> None: host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg ) assert argv[argv.index("--router-mode") + 1] == "kv" + # Gym names a token-capture call's session; keep the session on one engine. + assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "1800" assert argv[-2:] == ["--foo", "1"] + no_affinity = DynamoCfg.model_validate( + {"frontend_args": {"session_affinity_ttl_secs": None}} + ) + assert "--router-session-affinity-ttl-secs" not in build_dynamo_frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=no_affinity + ) clashing = DynamoCfg.model_validate( {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} @@ -173,9 +182,16 @@ def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> Non configure_generation_config(config, _Tokenizer()) -def test_chat_template_follows_the_native_server() -> None: - config = _serving(chat_template="templates/chat.jinja") +def test_chat_template_and_default_kwargs_follow_the_native_server() -> None: + config = _serving( + chat_template="templates/chat.jinja", + default_chat_template_kwargs={"enable_thinking": False}, + ) assert dynamo_chat_template(config) == "templates/chat.jinja" + assert native_default_chat_template_kwargs(config) == {"enable_thinking": False} + # The older spelling the native server also accepts. + legacy = _serving(chat_template_kwargs={"enable_thinking": True}) + assert native_default_chat_template_kwargs(legacy) == {"enable_thinking": True} assert dynamo_chat_template(_serving()) is None @@ -196,3 +212,11 @@ def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( _serving(chat_template_content_format="auto", enable_auto_tools=True), _Tokenizer(), ) + + +def test_a_bearer_token_is_rejected_because_the_frontend_checks_none() -> None: + config = _vllm_config(dynamo=True) + config["vllm_cfg"]["http_server_api_key_required"] = True + + with pytest.raises(ValueError, match="checks no bearer token"): + configure_generation_config(config, _Tokenizer()) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index f12a5937bb7..836b70c5ee5 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -42,12 +42,17 @@ from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) +from nemo_rl.models.generation.openai_server_utils import ( # noqa: E402 + derive_prefix_eos_count, + splice_prefix_tokens, +) from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, rollout_result_from_wire, rollout_result_to_wire, ) +from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) @@ -105,9 +110,7 @@ class _FakeRenderer: def __init__(self) -> None: self.prompts = [] - async def process_for_engine_async( - self, prompt, arrival_time, *, skip_mm_cache=False - ): + async def process_for_engine_async(self, prompt, arrival_time, *, skip_mm_cache): self.prompts.append((prompt, skip_mm_cache)) if "multi_modal_data" not in prompt: return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} @@ -227,7 +230,7 @@ def test_rollouts_are_served_like_generate_async() -> None: } -def test_router_replay_routes_reach_rollouts_as_natively() -> None: +def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None: routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) final = _output( [1, 2, 3], @@ -263,6 +266,29 @@ def test_router_replay_routes_reach_rollouts_as_natively() -> None: for key in native: assert torch.equal(shipped[key], native[key]), key + # A Gym call without capture carries the routes in engine_data, as the + # native chat endpoint attaches them to the response. + gym_final = _output( + [1, 3, 4], + _completion([10, 11], "stop", routed_experts=routes[3:]), + prompt_routed_experts=routes[:3], + ) + gym_host = _host( + _FakeLLM([gym_final]), vllm_kwargs={"enable_return_routed_experts": True} + ) + gym_engine = native_engine.NativeVllmEngine(gym_host, leader_worker_idx=0) + + (gym_chunk,) = _collect(gym_engine, _gym_request()) + + assert gym_chunk["engine_data"]["routed_experts"] == chat_routed_experts( + gym_final, + gym_final.outputs[0], + prompt_token_count=3, + device=torch.device("cpu"), + logger=native_engine.LOGGER, + routed_experts_dtype=torch.int16, + ) + def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: llm = _FakeLLM() @@ -605,6 +631,160 @@ def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: assert llm.calls == [] +# --- NeMo-Gym's Dynamo dialect ------------------------------------------------- + + +def _f32(value: float) -> float: + return float(np.float32(value)) + + +def _gym_request( + *, temperature=0.7, top_p=0.9, prefix=None, sampling=None, **fields +) -> dict: + annotations = [native_engine.GYM_ROLLOUT_ANNOTATION] + if prefix is not None: + annotations.insert( + 0, native_engine.GYM_PREFIX_ANNOTATION_PREFIX + json.dumps(prefix) + ) + sampling_options = {"temperature": temperature, "top_p": top_p, **(sampling or {})} + return { + "token_ids": [1, 3, 4], + # The frontend carries sampling floats as f32. + "sampling_options": { + key: _f32(value) if isinstance(value, float) else value + for key, value in sampling_options.items() + }, + # Gym's token information: logprobs=true, top_logprobs=0. + "output_options": {"logprobs": 0}, + "annotations": annotations, + **fields, + } + + +def test_gym_calls_sample_with_nemo_rls_exact_profile() -> None: + final = _output([1, 3, 4], _completion([10, 11], "stop")) + llm = _FakeLLM([final]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect( + engine, _gym_request(stop_conditions={"max_tokens": 100, "stop": [""]}) + ) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == [1, 3, 4] + # The configured values, not the f32 values the frontend delivered. + assert (params.temperature, params.top_p) == (0.7, 0.9) + assert params.top_k == vllm.SamplingParams(top_k=-1).top_k + assert params.output_kind == vllm.sampling_params.RequestOutputKind.FINAL_ONLY + # Output fits the context, and vLLM stops on the stop strings itself. + assert params.max_tokens == MAX_MODEL_LEN - 3 + assert (params.stop, params.detokenize) == ([""], True) + assert chunk["engine_data"] == { + "prompt_token_ids": [1, 3, 4], + "completion_token_ids": [10, 11], + "completion_logprobs": [-0.5, -0.5], + } + assert (chunk["token_ids"], chunk["finish_reason"]) == ([10, 11], "stop") + assert chunk["log_probs"] == [-0.5, -0.5] + + +def test_gym_validation_calls_sample_with_the_validation_profile() -> None: + llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + _collect(engine, _gym_request(temperature=0.0, top_p=1.0)) + + ((_, params, _),) = llm.calls + assert (params.temperature, params.top_p) == (0.0, 1.0) + + +@pytest.mark.parametrize( + ("request_fields", "error"), + [ + ({"temperature": 0.5}, "matches neither"), + ({"top_p": None}, "top_p must be set explicitly"), + ({"sampling": {"top_k": 5}}, "Top k sampling parameter"), + ({"sampling": {"n": 2}}, "take n=1"), + ], +) +def test_gym_calls_that_would_sample_off_policy_are_refused( + request_fields, error +) -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match=error): + _collect(engine, _gym_request(**request_fields)) + + assert llm.calls == [] + + +def test_gym_echoed_prefix_replaces_the_rendered_history() -> None: + # The model generated [5, 6, EOS]; the frontend re-rendered that turn as [7, EOS]. + prefix, rendered = [1, 5, 6, EOS], [1, 7, EOS, 3, 4] + spliced = splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=prefix, + template_token_ids=rendered, + template_prefix_eos_count=derive_prefix_eos_count(prefix, eos_token_id=EOS), + eos_token_id=EOS, + ).token_ids + assert spliced != rendered + llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect(engine, _gym_request(prefix=prefix, token_ids=rendered)) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == spliced + assert params.max_tokens == MAX_MODEL_LEN - len(spliced) + assert chunk["engine_data"]["prompt_token_ids"] == spliced + + +def test_gym_calls_whose_splice_fills_the_context_are_refused() -> None: + prefix = [1, *range(20, 20 + MAX_MODEL_LEN), EOS] + llm = _FakeLLM() + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match="maximum context length"): + _collect(engine, _gym_request(prefix=prefix, token_ids=[1, 7, EOS, 3])) + + assert llm.calls == [] + + +def test_gym_echoed_prefix_keeps_media_placeholders_on_their_tokens() -> None: + # Turn 1 ran [1, , 3] and generated [5, 6, EOS]; the frontend + # re-rendered that turn as [7, EOS] and appended a turn with a second image. + turn1 = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + prefix = [*turn1, 5, 6, EOS] + rendered = [1, IMAGE_PLACEHOLDER, 3, 7, EOS, IMAGE_PLACEHOLDER, 4] + spliced = [*prefix, *[IMAGE_TOKEN] * IMAGE_TOKENS, 4] + llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect( + engine, + _gym_request( + prefix=prefix, + token_ids=rendered, + multi_modal_data={"image_url": [{"Url": _image_url()}] * 2}, + ), + ) + + ((engine_input, params, _),) = llm.calls + assert engine_input["prompt_token_ids"] == spliced + # The first image sits in the model's own prefix; the second moved with the + # longer generated turn, as the native endpoint remaps them. + assert [ + (span.offset, span.length) for span in engine_input["mm_placeholders"]["image"] + ] == [(1, IMAGE_TOKENS), (len(prefix), IMAGE_TOKENS)] + assert params.max_tokens == MAX_MODEL_LEN - len(spliced) + assert chunk["engine_data"]["prompt_token_ids"] == spliced + + @pytest.mark.parametrize( ("router_mode", "prefix_caching", "publishes"), [("kv", True, True), ("kv", False, False), ("round-robin", True, False)], diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py new file mode 100644 index 00000000000..9c509a10fa0 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -0,0 +1,241 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NeMo-Gym token capture through the Dynamo engine adapter. + +The worker's own capture lifecycle stages each call, as on the native chat +endpoint; only Gym's capture core is faked. +""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +pytestmark = [pytest.mark.nemo_gym, pytest.mark.vllm] +pytest.importorskip("dynamo._core") +pytest.importorskip("vllm") +# The vLLM lanes import this module without the nemo_gym extra. +pytest.importorskip("nemo_gym.token_id_capture.staging") + +from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 +from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 + DYNAMO_CAPTURE_ANNOTATION_PREFIX, + DYNAMO_GYM_REQUEST_ANNOTATION, + DYNAMO_PREFIX_ANNOTATION_PREFIX, +) +from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 +from responses_api_models.vllm_model.app import ( # noqa: E402 + _DYNAMO_ENGINE_DATA_FIELDS, +) + +from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 +from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 +from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 +from nemo_rl.utils.routed_experts_codec import decode_routed_experts # noqa: E402 +from tests.unit.models.generation.test_dynamo_native_engine import ( # noqa: E402 + EOS, + IMAGE_PLACEHOLDER, + IMAGE_TOKEN, + IMAGE_TOKENS, + _collect, + _completion, + _FakeLLM, + _gym_request, + _host, + _image_url, + _output, + _Tokenizer, +) + +COORDS = {"staging_key": "rollout-1/c2"} + + +class _Capture: + """Gym's capture core: records begin/complete/fail.""" + + adapter = VLLMCaptureAdapter() + + def __init__(self) -> None: + self.begun: list[tuple] = [] + self.staged: list[dict] = [] + self.failed: list[str] = [] + + def begin_call(self, admission, *, prefix_token_ids, stream): + self.begun.append((admission, prefix_token_ids, stream)) + return SimpleNamespace(admission=admission) + + def complete_call_from_response(self, call, payload, attachments=None): + self.staged.append(payload) + return SimpleNamespace(model_dump=lambda: dict(COORDS)) + + def fail_call(self, call, *, reason) -> None: + self.failed.append(reason) + + +def _capturing_host(llm, **cfg): + host = _host(llm, **cfg) + host.token_capture = _Capture() + host._capture_calls = {} + host._chain_prefix = ChainPrefixCache() + return host + + +def _captured_request(admission: CaptureAdmission, **fields) -> dict: + request = _gym_request(**fields) + # Gym's dynamo_worker handler: no logprobs, the admission in an annotation. + request["output_options"] = {} + request["annotations"] = [ + DYNAMO_CAPTURE_ANNOTATION_PREFIX + admission.model_dump_json(), + *request["annotations"], + ] + return request + + +def test_annotations_and_engine_data_are_the_ones_gyms_dynamo_dialect_uses() -> None: + assert native_engine.GYM_ROLLOUT_ANNOTATION == DYNAMO_GYM_REQUEST_ANNOTATION + assert native_engine.GYM_CAPTURE_ANNOTATION_PREFIX == ( + DYNAMO_CAPTURE_ANNOTATION_PREFIX + ) + assert native_engine.GYM_PREFIX_ANNOTATION_PREFIX == ( + DYNAMO_PREFIX_ANNOTATION_PREFIX + ) + # The engine_data keys Gym requires on a call without capture. + engine = native_engine.NativeVllmEngine(_host(_FakeLLM()), leader_worker_idx=0) + final = _output([1, 3, 4], _completion([10], "stop")) + assert engine._gym_token_data(final).keys() == _DYNAMO_ENGINE_DATA_FIELDS.keys() + + +def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: + # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. + prefix = [1, 5, 6, EOS] + admission = CaptureAdmission( + rollout_id="rollout-1", + model_call_id="c2", + parent_call_id="c1", + prev_len=len(prefix), + mode="token_in", + required_prefix_token_ids=prefix, + parent_chain_hash="0" * 64, + ) + spliced = [*prefix, 3, 4] + routes = np.arange(8 * 2 * 2, dtype=np.int16).reshape(8, 2, 2) + final = _output( + spliced, + _completion([10, 11], "stop", routed_experts=routes[6:]), + prompt_routed_experts=routes[:6], + ) + llm = _FakeLLM([final]) + llm.tokenizer = _Tokenizer() + host = _capturing_host(llm, vllm_kwargs={"enable_return_routed_experts": True}) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect( + engine, _captured_request(admission, token_ids=[1, 7, EOS, 3, 4]) + ) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == spliced + # The sampled tokens' log probabilities are staged; Gym asked for none. + assert params.logprobs == 0 + ((begun_admission, begun_prefix, stream),) = host.token_capture.begun + assert (begun_admission, begun_prefix, stream) == (admission, prefix, False) + (payload,) = host.token_capture.staged + assert payload["prompt_token_ids"] == spliced + message = payload["choices"][0]["message"] + assert message["generation_token_ids"] == [10, 11] + assert message["generation_log_probs"] == [-0.5, -0.5] + # As the native endpoint stages them: the full sequence's routes (none for + # the last token) from the boundary row, the prefix's last token, on. + staged_routes = decode_routed_experts(message["routed_experts"], torch.int16) + expected = torch.full((5, 2, 2), R3_MISSING_ROUTE_SENTINEL, dtype=torch.int16) + expected[:4] = torch.from_numpy(routes[3:7]) + assert torch.equal(staged_routes, expected) + # Only the commit coordinates go back through the frontend. + assert chunk["engine_data"] == {"ng_commit_coords": COORDS} + assert "log_probs" not in chunk + assert host._capture_calls == {} + + +def test_a_call_that_fails_after_admission_fails_its_capture() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + + class _FailingLLM(_FakeLLM): + async def generate(self, prompt, sampling_params, request_id): + raise RuntimeError("engine died") + yield # pragma: no cover + + host = _capturing_host(_FailingLLM()) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + with pytest.raises(RuntimeError, match="engine died"): + _collect(engine, _captured_request(admission)) + + assert host.token_capture.failed == ["engine_error"] + assert host._capture_calls == {} + + +def test_without_capture_installed_a_captured_call_is_served_uncaptured() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) + host = _host(llm) + host.token_capture = None + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect(engine, _captured_request(admission)) + + assert json.loads(json.dumps(chunk["engine_data"]))["completion_token_ids"] == [10] + + +def test_a_captured_media_call_stages_the_media_the_engine_ran_on() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + processed = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + llm = _FakeLLM([_output(processed, _completion([10], "stop"))]) + host = _capturing_host(llm) + host._capture_media = True + captured = CapturedMedia((), None) + snapshots = [] + + def capture_request_media(engine_input, *, admission, splice): + snapshots.append((list(engine_input["prompt_token_ids"]), admission, splice)) + return captured + + host._capture_request_media = capture_request_media + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect( + engine, + _captured_request( + admission, + token_ids=[1, IMAGE_PLACEHOLDER, 3], + multi_modal_data={"image_url": [{"Url": _image_url()}]}, + ), + ) + + # The processor's own pixels, not cache references, as natively (#4124). + ((_, skip_mm_cache),) = llm.renderer.prompts + assert skip_mm_cache is True + # Media is snapshotted from the processed prompt the engine runs. + assert snapshots == [(processed, admission, None)] + (payload,) = host.token_capture.staged + assert payload["prompt_token_ids"] == processed + assert chunk["engine_data"] == {"ng_commit_coords": COORDS} diff --git a/tests/unit/models/generation/test_openai_server_utils.py b/tests/unit/models/generation/test_openai_server_utils.py index 223be2dc883..425e46442bc 100644 --- a/tests/unit/models/generation/test_openai_server_utils.py +++ b/tests/unit/models/generation/test_openai_server_utils.py @@ -17,6 +17,7 @@ from nemo_rl.models.generation.openai_server_utils import ( PrefixSplice, + derive_prefix_eos_count, replace_prefix_tokens, splice_prefix_tokens, ) @@ -160,6 +161,97 @@ def test_splice_prefix_tokens_without_tokenizer_honors_explicit_eos_ids( ) assert result == expected assert result.token_ids[: len(model_prefix)] == model_prefix + # A host without the messages (managed Dynamo's engine) passes the EOS count + # derived from the prefix instead of the template prefix render. + assert ( + splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=model_prefix, + template_token_ids=template, + eos_token_id=eos_token_id, + template_prefix_eos_count=derive_prefix_eos_count( + model_prefix, eos_token_id=eos_token_id + ), + ) + == expected + ) + with pytest.raises(ValueError, match="exactly one"): + splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=model_prefix, + template_prefix_token_ids=template_prefix, + template_token_ids=template, + eos_token_id=eos_token_id, + template_prefix_eos_count=1, + ) + + +@pytest.mark.parametrize( + "model_name", ["Qwen/Qwen3-0.6B", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"] +) +def test_derived_prefix_eos_count_matches_real_template_renders(model_name): + """Across tool-calling turns with reasoning, the EOS count derived from the + exact prefix equals the count in the template's render of the same history.""" + transformers = pytest.importorskip("transformers") + try: + tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) + except OSError as error: + pytest.skip(f"{model_name} tokenizer unavailable: {error}") + eos = tokenizer.eos_token_id + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + + def render(messages, *, add_generation_prompt): + rendered = tokenizer.apply_chat_template( + messages, + tools=tools, + add_generation_prompt=add_generation_prompt, + tokenize=True, + ) + return list(getattr(rendered, "input_ids", rendered)) + + def generate(text): + return tokenizer.encode(text, add_special_tokens=False) + [eos] + + messages = [ + {"role": "system", "content": "You are a weather assistant."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + replies = [ + "\nI should look it up.\n\n\nChecking the forecast.", + "\nIt is sunny.\n\n\nIt is sunny in Paris.", + ] + follow_ups = [ + {"role": "tool", "content": "sunny, 22C"}, + {"role": "user", "content": "And London?"}, + ] + chain = render(messages, add_generation_prompt=True) + generate(replies[0]) + for reply, follow_up, next_reply in zip(replies, follow_ups, replies[1:] + ["ok"]): + messages = [*messages, {"role": "assistant", "content": reply}, follow_up] + template_prefix = render(messages[:-1], add_generation_prompt=False) + template = render(messages, add_generation_prompt=True) + assert derive_prefix_eos_count(chain, eos_token_id=eos) == sum( + token == eos for token in template_prefix + ) + spliced = splice_prefix_tokens( + tokenizer=tokenizer, + model_prefix_token_ids=chain, + template_prefix_token_ids=template_prefix, + template_token_ids=template, + ).token_ids + chain = spliced + generate(next_reply) def test_replace_prefix_tokens_without_tokenizer_reports_missing_eos_without_decoding(): diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 7cf184b311b..8e342ae8274 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2571,7 +2571,7 @@ def test_vllm_validate_settings_accepts_missing_vllm_cfg_as_refit_disabled(): VllmGeneration.validate_settings(master_config) -def _dynamo_master_config(*, fleet_health=False): +def _dynamo_master_config(*, router=False, fleet_health=False): vllm_config = deepcopy(basic_vllm_test_config) vllm_config["dynamo"] = True vllm_config["vllm_cfg"]["async_engine"] = True @@ -2579,6 +2579,7 @@ def _dynamo_master_config(*, fleet_health=False): return types.SimpleNamespace( policy={"generation": vllm_config}, async_rl=types.SimpleNamespace( + generation_router=types.SimpleNamespace(enabled=router), generation_fleet_health=types.SimpleNamespace(enabled=fleet_health), ), ) @@ -2588,6 +2589,7 @@ def _dynamo_master_config(*, fleet_health=False): ("settings", "error"), [ ({}, None), + ({"router": True}, "routes the whole fleet behind one URL"), ({"fleet_health": True}, "Disable fleet health"), ], ) @@ -2601,13 +2603,19 @@ def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error @pytest.mark.parametrize("dynamo", [False, True]) -def test_vllm_token_rollouts_use_the_dynamo_frontend_when_it_serves(dynamo): +def test_vllm_token_rollouts_and_clients_use_the_dynamo_frontend_when_it_serves( + dynamo, +): generation = object.__new__(VllmGeneration) + generation.dp_openai_server_base_urls = ["http://a/v1", "http://b/v1"] generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None tokens = BatchedDataDict({"input_ids": torch.tensor([[1]])}) # The frontend request carries token IDs only, so media prompts stay native. media = BatchedDataDict({"input_ids": torch.tensor([[1]]), "vllm_content": [None]}) + assert generation.client_base_urls == ( + ["http://frontend/v1"] if dynamo else ["http://a/v1", "http://b/v1"] + ) assert generation._dynamo_serves(tokens) is dynamo assert not generation._dynamo_serves(media) diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index 3414010f089..25b5a2e19ee 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -230,6 +230,7 @@ def _worker_with_capture(sink: _MemorySink): for name in ( "_fetch_chain_prefix", "_capture_admission", + "_parse_capture_admission", "_resolve_admission_prefix", "_enter_request_prefix", ): diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index d81d66628d0..4328d05736b 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -2222,7 +2222,7 @@ def test_nemo_gym_wires_env_handle(self, patched_factories): assert data_kwargs["is_vlm"] is True mock_spinup.assert_called_once_with( mc.env, - base_urls=patched_factories["fake_gen"].dp_openai_server_base_urls, + base_urls=patched_factories["fake_gen"].client_base_urls, model_name="test-model", # Reaches the actor once, at spinup, rather than riding along with every # run_rollouts call. @@ -2375,6 +2375,44 @@ def test_token_capture_always_creates_finalizer_actor_pool( _, setup_kwargs = generation.setup_token_capture.call_args assert setup_kwargs["capture_media"] is with_processor + def test_dynamo_token_capture_tells_gym_to_speak_dynamo(self, patched_factories): + """The same vLLM workers stage the calls; Gym reaches them through the frontend.""" + mc = _make_master_config(backend="vllm") + mc.policy["generation"].update( + { + "model_name": "test-model", + "stop_strings": None, + "stop_token_ids": None, + "top_k": None, + "dynamo": True, + "vllm_cfg": {"async_engine": True, "expose_http_server": True}, + } + ) + mc.token_capture.enabled = True + mc.env = {"should_use_nemo_gym": True, "nemo_gym": {}} + patched_factories["setup_response_data"].return_value = (list(range(8)), None) + + with ( + patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), + patch.object( + sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() + ) as mock_spinup, + patch.object(sc_setup_mod, "validate_dataset_agent_coverage"), + patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), + patch( + "nemo_rl.experience.rollout_reassembler_actor.create_rollout_reassembler_actors", + return_value=[MagicMock()], + ), + ): + setup_single_controller(mc, MagicMock(pad_token_id=9)) + + assert mc.token_capture.generation_backend == "dynamo" + assert mock_spinup.call_args.kwargs["token_capture"]["generation_backend"] == ( + "dynamo" + ) + generation, _ = patched_factories["_build_generation"].return_value + generation.setup_token_capture.assert_called() + def test_nemo_gym_coverage_failure_shuts_down_shards(self, patched_factories): mc = _make_master_config(colocated=False, backend="vllm") mc.policy["generation"]["model_name"] = "test-model" diff --git a/uv.lock b/uv.lock index 56bc756bbaf..76b3e8897be 100644 --- a/uv.lock +++ b/uv.lock @@ -4174,7 +4174,7 @@ requires-dist = [ { name = "daytona", marker = "extra == 'sandbox'", specifier = ">=0.179.0" }, { name = "defusedxml", marker = "extra == 'dev'", specifier = ">=0.7.1" }, { name = "devtools" }, - { name = "e2b", marker = "extra == 'sandbox'", specifier = ">=2.36.0,<3.0.0" }, + { name = "e2b", marker = "extra == 'sandbox'", specifier = ">=2.46.0,<3.0.0" }, { name = "fastapi", specifier = "!=0.142.*" }, { name = "flashinfer-python", marker = "extra == 'vllm'", specifier = "==0.6.18" }, { name = "fonttools", specifier = ">=4.60.2" }, @@ -4185,7 +4185,7 @@ requires-dist = [ { name = "itsdangerous" }, { name = "jinja2", marker = "extra == 'ray'", specifier = ">=3.1" }, { name = "jsonschema", specifier = ">=4.18,<5" }, - { name = "mcp", specifier = ">=1.28.1,<2" }, + { name = "mcp", specifier = ">=1.28.1,<3" }, { name = "mlflow", marker = "extra == 'mlflow'", specifier = ">=3.16.1" }, { name = "mlflow-skinny", marker = "extra == 'mlflow'", specifier = ">=3.16.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8.0" }, From 39285257758a395e6955dbdee9599e447a63108f Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 13:47:47 -0700 Subject: [PATCH 09/82] refactor(vllm): serve every rollout through one worker method Address review on this PR: - _serve_rollout runs a rollout's whole sequence (budget, nothing-fits, SamplingParams, generate, prompt check, result), so every host that serves a rollout from this engine calls one method instead of copying the steps. It replaces _rollout_sampling_params. - generate_async asserts a single sample, so it no longer fans out per-sample tasks: it formats the prompt, calls _serve_rollout and yields the output. vLLM still aborts the request on cancellation. - rollout_output drops pad_token_id, whose fill never reached the output. - off_policy_sampling_error documents that vLLM fills unset top_p/top_k from generation_config.json, so callers must pin top_k=-1. - chat_routed_experts drops a dead isinstance check. - Tests: a route test for the on-policy assert and the top_k pin; the budget tests fold into one parametrization that covers a full context with speculative decoding; the wiring test asserts the exact output keys and a missing logprob; the R3 pipeline test uses vLLM 0.29's route shape. Two pipeline tests the wiring tests now cover are removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/vllm/request_pipeline.py | 36 ++--- nemo_rl/models/generation/vllm/utils.py | 37 ++--- .../generation/vllm/vllm_worker_async.py | 151 +++++++----------- .../models/generation/test_vllm_generation.py | 78 ++++++--- .../generation/test_vllm_request_pipeline.py | 40 +---- 5 files changed, 146 insertions(+), 196 deletions(-) diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index bb77e3b0e27..9e5bea46416 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -14,11 +14,11 @@ """Turn a finished vLLM rollout into NeMo RL's per-sample generation output. -Building the per-sample result is kept apart from assembling the output tensors, -so a path that builds the result on the engine and assembles the output -elsewhere behaves the same as the worker's own ``generate_async``. A rollout -feature added here, such as router replay (R3), then behaves the same on every -path. +The worker's ``_serve_rollout`` builds the result; ``rollout_output`` assembles +the output tensors from it. The two are apart so a host that builds the result +on the engine and assembles the output elsewhere returns what ``generate_async`` +returns. A rollout feature added here, such as router replay (R3), then behaves +the same on every path. """ import logging @@ -55,7 +55,7 @@ def nothing_generated(cls) -> "RolloutResult": return cls([], [], False, None, None) -def selected_logprobs(completion: Any) -> list[float]: +def _selected_logprobs(completion: Any) -> list[float]: """The sampled token's log probability at each position of a completion.""" by_position = completion.logprobs or () logprobs = [] @@ -73,7 +73,6 @@ def rollout_result( return_routed_experts: bool, routed_experts_dtype: torch.dtype, device: torch.device, - sample_idx: int = 0, ) -> RolloutResult: """Build the result of a finished rollout from vLLM's final RequestOutput. @@ -83,7 +82,6 @@ def rollout_result( return_routed_experts: Router replay is on, so routes are required. routed_experts_dtype: The dtype router replay consumes. device: Where to place the routed-experts tensor. - sample_idx: For log messages only. """ completion = final_output.outputs[0] token_ids = list(completion.token_ids) @@ -110,17 +108,16 @@ def rollout_result( if return_routed_experts and r3_stats["missing_routes"] > 0: LOGGER.warning( "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for sample_idx=%d, missing_token_routes=%d, " - "actual_routes=%d, expected_routes=%d. Megatron will use its " - "own router for those missing token routes.", - sample_idx, + "routed_experts, missing_token_routes=%d, actual_routes=%d, " + "expected_routes=%d. Megatron will use its own router for those " + "missing token routes.", r3_stats["missing_routes"], r3_stats["actual_routes"], r3_stats["expected_routes"], ) return RolloutResult( token_ids=token_ids, - logprobs=selected_logprobs(completion), + logprobs=_selected_logprobs(completion), truncated=completion.finish_reason == "length", routed_experts=routed_experts, r3_stats=r3_stats if return_routed_experts else None, @@ -132,7 +129,6 @@ def rollout_output( *, input_ids: torch.Tensor, input_length: int, - pad_token_id: int, ) -> BatchedDataDict[GenerationOutputSpec]: """The single-sample output ``generate_async`` yields for ``result``. @@ -140,17 +136,15 @@ def rollout_output( result: The rollout's result. input_ids: The sample's (right-padded) prompt row. input_length: Prompt length in tokens. - pad_token_id: The tokenizer's pad token, as ``generate_async`` uses it. """ device = input_ids.device generated = result.token_ids output_length = input_length + len(generated) - output_ids = torch.full( - (output_length,), pad_token_id, dtype=input_ids.dtype, device=device - ) - output_ids[:input_length] = input_ids[:input_length] - output_ids[input_length:] = torch.tensor( - generated, dtype=input_ids.dtype, device=device + output_ids = torch.cat( + ( + input_ids[:input_length], + torch.tensor(generated, dtype=input_ids.dtype, device=device), + ) ) logprobs = torch.zeros((1, output_length), dtype=torch.float32, device=device) logprobs[0, input_length:] = torch.tensor( diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index 2d871819ae7..4a6e41e5c5f 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -15,7 +15,7 @@ from collections import defaultdict from collections.abc import Mapping from dataclasses import is_dataclass, replace -from typing import Any, Optional +from typing import Any, Optional, cast import torch @@ -336,9 +336,12 @@ def off_policy_sampling_error( Args: temperature: The request's temperature. top_p: The request's top_p. It must be explicit: vLLM resolves an unset top_p - from the model's generation_config.json, not to 1.0. + from the model's generation_config.json (``ModelConfig.generation_config`` + defaults to ``"auto"``), not necessarily 1.0. top_k: The request's top_k, which must be unset or -1 (disabled), matching - ``BaseVllmGenerationWorker._build_sampling_params``. + ``BaseVllmGenerationWorker._build_sampling_params``. vLLM resolves an + unset top_k from generation_config.json too, so None means on-policy only + once the caller sends the request with top_k=-1. generation_config: ``policy.generation``, with the train and validation sampling parameters. """ @@ -558,22 +561,20 @@ def chat_routed_experts( ) -> str: """The routed-experts envelope the chat endpoint attaches to one choice.""" generation_token_count = len(getattr(generation_details, "token_ids", []) or []) - routed_result = pad_and_align_routed_expert_indices( - final_request_output, - generation_details, - valid_length=prompt_token_count + generation_token_count, - padded_length=prompt_token_count + generation_token_count, - device=device, - require_complete_routed_experts=True, - return_stats=True, - routed_experts_dtype=routed_experts_dtype, + # return_stats=True returns the (routes, stats) pair. + routed_experts, r3_stats = cast( + tuple[Optional[torch.Tensor], dict[str, int]], + pad_and_align_routed_expert_indices( + final_request_output, + generation_details, + valid_length=prompt_token_count + generation_token_count, + padded_length=prompt_token_count + generation_token_count, + device=device, + require_complete_routed_experts=True, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ), ) - if not isinstance(routed_result, tuple): - raise RuntimeError( - "Expected routed_experts alignment to return stats for the " - "OpenAI-compatible chat endpoint." - ) - routed_experts, r3_stats = routed_result if routed_experts is None: raise RuntimeError( "vLLM was asked to return routed experts for the " diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 2c67cc2a1f8..e47a99245d5 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -1795,8 +1795,7 @@ def _rollout_max_new_tokens(self, input_length: int) -> int: """Token budget for one rollout prompt; 0 means nothing fits. The configured cap, clipped to the context window and to speculative - decoding's lookahead. Kept apart from ``generate_async`` so every path that - serves a rollout from this engine gives it the same budget. + decoding's lookahead. """ max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) allowed_new_tokens = max( @@ -1814,45 +1813,73 @@ def _rollout_max_new_tokens(self, input_length: int) -> int: ) return allowed_new_tokens - def _rollout_sampling_params( + async def _serve_rollout( self, + prompt: Any, *, - max_new_tokens: int, + prompt_token_ids: list[int], greedy: bool, stop_strings: Optional[list[str]], - ) -> Any: - """The SamplingParams for one rollout. + request_id: str, + device: torch.device, + ) -> tuple[RolloutResult, Optional[Any]]: + """Serve one rollout from this engine: its budget, sampling, generation and result. - Kept apart from ``generate_async`` so every path that serves a rollout - from this engine samples it the same way. + Every path that serves a rollout from this engine calls this, so a rollout + step added here runs on all of them. Args: - max_new_tokens: The rollout's budget, from ``_rollout_max_new_tokens``. + prompt: The vLLM prompt to generate from. + prompt_token_ids: The learner's prompt tokens, which vLLM must have run. greedy: Greedy decoding instead of the configured sampling. - stop_strings: The sample's own stop strings, merged with the + stop_strings: The rollout's own stop strings, merged with the configured ones. + request_id: The vLLM request ID. + device: Where to place the routed-experts tensor. + + Returns: + The result, and vLLM's final RequestOutput (None when no new token fits). """ - return self._build_sampling_params( + max_new_tokens = self._rollout_max_new_tokens(len(prompt_token_ids)) + if max_new_tokens == 0: + return RolloutResult.nothing_generated(), None + sampling_params = self._build_sampling_params( greedy=greedy, stop_strings=self._merge_stop_strings( [stop_strings] if stop_strings else None ), max_new_tokens=max_new_tokens, ) + final_output = None + async for output in self.llm.generate( + prompt=prompt, sampling_params=sampling_params, request_id=request_id + ): + final_output = output + if final_output is None: + raise RuntimeError(f"No output received for request {request_id}") + validate_rollout_prompt(prompt_token_ids, final_output.prompt_token_ids) + result = rollout_result( + final_output, + input_length=len(prompt_token_ids), + return_routed_experts=self._return_routed_experts_enabled(), + routed_experts_dtype=self.routed_experts_dtype, + device=device, + ) + return result, final_output async def generate_async( self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False, ) -> AsyncGenerator[tuple[int, BatchedDataDict[GenerationOutputSpec]], None]: - """Generate a batch of data using vLLM's AsyncLLMEngine, yielding results as they are ready. + """Generate one sample with vLLM's AsyncLLM, yielding it when it is ready. Args: - data: BatchedDataDict with input_ids and input_lengths - greedy: Whether to use greedy decoding instead of sampling + data: BatchedDataDict with input_ids and input_lengths for one sample. + greedy: Whether to use greedy decoding instead of sampling. Yields: - Tuple of (original_index, BatchedDataDict conforming to GenerationOutputSpec for the single sequence) + (0, the sample's output conforming to GenerationOutputSpec). """ if not self.cfg["vllm_cfg"]["async_engine"]: raise RuntimeError( @@ -1865,92 +1892,26 @@ async def generate_async( verify_right_padding(data, pad_value=self.cfg["_pad_token_id"]) - input_ids_batch = data["input_ids"] - input_lengths_batch = data["input_lengths"] - batch_size = input_ids_batch.shape[0] - - # Ensure generate_async only receives single samples (batch_size = 1) + batch_size = data["input_ids"].shape[0] assert batch_size == 1, ( f"generate_async is restricted to handle only single samples, " f"but received batch_size={batch_size}. Please handle batching outside this method." ) - batch_specific_stop_strings_list = data.get( - "stop_strings", [[] for _ in range(batch_size)] + input_ids = data["input_ids"][0] + input_length = int(data["input_lengths"][0].item()) + prompt = self._tokenize_prompt_with_bos( + format_prompt_for_vllm_generation(data, 0) ) - - # Create tasks for each sample in the batch - async def process_single_sample(sample_idx): - """Process a single sample and return the result.""" - current_input_actual_length = input_lengths_batch[sample_idx].item() - prompt = format_prompt_for_vllm_generation(data, sample_idx) - prompt = self._tokenize_prompt_with_bos(prompt) - - per_sample_stop_strings = None - if batch_specific_stop_strings_list and sample_idx < len( - batch_specific_stop_strings_list - ): - per_sample_stop_strings = batch_specific_stop_strings_list[sample_idx] - - max_new_tokens = self._rollout_max_new_tokens(current_input_actual_length) - input_ids_single_row = input_ids_batch[sample_idx] - if max_new_tokens == 0: - # No new token fits in the context: return the prompt alone. - result = RolloutResult.nothing_generated() - else: - sampling_params_for_request = self._rollout_sampling_params( - max_new_tokens=max_new_tokens, - greedy=greedy, - stop_strings=per_sample_stop_strings, - ) - request_id = str(uuid.uuid4()) - final_request_output = None - async for req_output in self.llm.generate( - prompt=prompt, - sampling_params=sampling_params_for_request, - request_id=request_id, - ): - final_request_output = req_output - if final_request_output is None: - raise RuntimeError(f"No output received for request {request_id}") - - validate_rollout_prompt( - input_ids_single_row[:current_input_actual_length].tolist(), - final_request_output.prompt_token_ids, - ) - result = rollout_result( - final_request_output, - input_length=current_input_actual_length, - return_routed_experts=self._return_routed_experts_enabled(), - routed_experts_dtype=self.routed_experts_dtype, - device=input_ids_single_row.device, - sample_idx=sample_idx, - ) - return ( - sample_idx, - rollout_output( - result, - input_ids=input_ids_single_row, - input_length=current_input_actual_length, - pad_token_id=self.cfg["_pad_token_id"], - ), - ) - - # Create tasks for all samples and yield results as they complete - sample_tasks = [ - asyncio.create_task(process_single_sample(i)) for i in range(batch_size) - ] - - # Yield results as they become available - try: - for completed_task in asyncio.as_completed(sample_tasks): - result = await completed_task - yield result - finally: - for task in sample_tasks: - if not task.done(): - task.cancel() - await asyncio.gather(*sample_tasks, return_exceptions=True) + result, _ = await self._serve_rollout( + prompt, + prompt_token_ids=input_ids[:input_length].tolist(), + greedy=greedy, + stop_strings=(data.get("stop_strings") or [None])[0], + request_id=str(uuid.uuid4()), + device=input_ids.device, + ) + yield 0, rollout_output(result, input_ids=input_ids, input_length=input_length) async def generate_text_async( self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index d0533c1a734..a79543ce11e 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -491,8 +491,10 @@ async def generate(self, prompt, sampling_params, request_id): outputs=[ types.SimpleNamespace( token_ids=self.generated, + # The second token has no reported logprob. logprobs=[ - {t: types.SimpleNamespace(logprob=-0.5)} for t in self.generated + {} if i == 1 else {t: types.SimpleNamespace(logprob=-0.5)} + for i, t in enumerate(self.generated) ], finish_reason=self.finish_reason, routed_experts=None, @@ -552,51 +554,56 @@ async def test_async_vllm_generate_async_samples_and_returns_the_rollout() -> No assert sampling["top_k"] == -1 assert sorted(sampling["stop"]) == ["", ""] assert output["output_ids"].tolist() == [[1, 2, 3, 4, 5, 10, 11, 2]] - assert output["logprobs"].tolist() == [[0.0] * 5 + [-0.5] * 3] + # A position without a reported logprob stays 0.0, aligned with its token. + assert output["logprobs"].tolist() == [[0.0] * 5 + [-0.5, 0.0, -0.5]] assert output["generation_lengths"].tolist() == [3] assert output["unpadded_sequence_lengths"].tolist() == [8] assert output["truncated"].tolist() == [False] + # No router replay keys when R3 is off. + assert set(output) == { + "output_ids", + "logprobs", + "generation_lengths", + "unpadded_sequence_lengths", + "truncated", + } @pytest.mark.asyncio @pytest.mark.parametrize( - ("speculative_config", "expected_max_tokens"), + ("prompt_length", "speculative_config", "expected_max_tokens"), [ # min(max_new_tokens=8, max_model_len 16 - prompt 12) - (None, 4), + (12, None, 4), # The drafter reads 2 tokens ahead: 16 - 12 - (2 + 1) - ({"num_speculative_tokens": 2}, 1), + (12, {"num_speculative_tokens": 2}, 1), + # Nothing fits: the prompt comes back without calling vLLM. + (16, None, 0), + # The spec-decode clamp floors at 1 token; a full context must not reach it. + (16, {"num_speculative_tokens": 2}, 0), ], ) -async def test_async_vllm_generate_async_caps_the_budget_to_the_context( - speculative_config, expected_max_tokens +async def test_async_vllm_generate_async_budget( + prompt_length, speculative_config, expected_max_tokens ) -> None: llm = _ScriptedAsyncLLM([10], finish_reason="length") worker = _rollout_worker( llm, vllm_kwargs={"speculative_config": speculative_config} ) - - output = await _generate_one(worker, list(range(1, 13))) - - ((_, sampling),) = llm.calls - assert sampling["max_tokens"] == expected_max_tokens - assert output["truncated"].tolist() == [True] - - -@pytest.mark.asyncio -async def test_async_vllm_generate_async_returns_the_prompt_when_nothing_fits() -> None: - llm = _ScriptedAsyncLLM([10]) - worker = _rollout_worker(llm) - prompt = list(range(1, 17)) + prompt = list(range(1, prompt_length + 1)) output = await _generate_one(worker, prompt) - assert llm.calls == [] - assert output["output_ids"].tolist() == [prompt] - assert output["logprobs"].tolist() == [[0.0] * 16] - assert output["generation_lengths"].tolist() == [0] - assert output["unpadded_sequence_lengths"].tolist() == [16] - assert output["truncated"].tolist() == [False] + if expected_max_tokens == 0: + assert llm.calls == [] + assert output["output_ids"].tolist() == [prompt] + assert output["logprobs"].tolist() == [[0.0] * prompt_length] + assert output["generation_lengths"].tolist() == [0] + assert output["truncated"].tolist() == [False] + else: + ((_, sampling),) = llm.calls + assert sampling["max_tokens"] == expected_max_tokens + assert output["truncated"].tolist() == [True] @pytest.mark.asyncio @@ -1430,6 +1437,25 @@ async def create_chat_completion(_request, _raw_request): assert error["param"] == "messages" +@pytest.mark.asyncio +async def test_chat_handler_rejects_off_policy_sampling_and_pins_top_k(monkeypatch): + """The route asserts on-policy sampling, then hands vLLM top_k=-1.""" + _, serving_chat, chat_handler = _setup_fake_vllm_chat_handler(monkeypatch) + serving_chat.create_chat_completion = AsyncMock(return_value=iter(())) + + off_policy = _fake_chat_request() + off_policy.temperature = 0.5 + with pytest.raises(AssertionError, match="matches neither"): + await chat_handler(off_policy, MagicMock()) + serving_chat.create_chat_completion.assert_not_awaited() + + request = _fake_chat_request() + request.top_k = None + await chat_handler(request, MagicMock()) + serving_chat.create_chat_completion.assert_awaited_once() + assert request.top_k == -1 + + @pytest.mark.asyncio @pytest.mark.parametrize("logprobs", [False, True]) async def test_chat_completion_cancelled_on_client_disconnect(monkeypatch, logprobs): diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py index c00be9dc2c4..4b84739f04a 100644 --- a/tests/unit/models/generation/test_vllm_request_pipeline.py +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -73,43 +73,11 @@ def _result(final_output, *, input_length=3, r3=False) -> RolloutResult: ) -def test_output_layout_matches_generate_async() -> None: - result = _result(_final_output([7, 8, 9], finish_reason="length")) - output = rollout_output( - result, input_ids=torch.tensor([1, 2, 3, 0, 0]), input_length=3, pad_token_id=0 - ) - - assert output["output_ids"].tolist() == [[1, 2, 3, 7, 8, 9]] - # A position without a reported logprob stays 0.0, aligned with its token. - assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0, -0.5, 0.0, -1.0]] - assert output["generation_lengths"].tolist() == [3] - assert output["unpadded_sequence_lengths"].tolist() == [6] - assert output["truncated"].tolist() == [True] - assert "routed_experts" not in output - - -def test_nothing_generated_returns_the_prompt_untruncated() -> None: - output = rollout_output( - RolloutResult.nothing_generated(), - input_ids=torch.tensor([1, 2, 3, 0]), - input_length=3, - pad_token_id=0, - ) - - assert output["output_ids"].tolist() == [[1, 2, 3]] - assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0]] - assert output["generation_lengths"].tolist() == [0] - assert output["truncated"].tolist() == [False] - - def test_router_replay_routes_are_aligned_and_counted() -> None: - # 3 prompt + 3 generated tokens need 5 routes; vLLM reported 3 + 1. - result = _result( - _final_output([7, 8, 9], prompt_routes=3, generated_routes=1), r3=True - ) - output = rollout_output( - result, input_ids=torch.tensor([1, 2, 3]), input_length=3, pad_token_id=0 - ) + # 3 prompt + 3 generated tokens need 5 routes; vLLM reports prompt and + # generated routes together on the completion, here only 4 of them. + result = _result(_final_output([7, 8, 9], generated_routes=4), r3=True) + output = rollout_output(result, input_ids=torch.tensor([1, 2, 3]), input_length=3) routed = output["routed_experts"] assert routed.shape == (1, 6, LAYERS, TOPK) and routed.dtype == torch.int16 From 29d200478c0ae36f53c36a09165e8a908f3855da Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 14:54:41 -0700 Subject: [PATCH 10/82] fix: adopt Gym's capture_key rename and config paths for the Gym bump The Gym re-pin crosses Gym#4128 (rollout_id -> capture_key, staging schema 3) and Gym#4858 (config-path aliases removed): - Data-plane test fixtures build records and receipts with capture_key and STAGING_SCHEMA_VERSION; the captured-media lineage case uses capture_key. - NeMo RL reads Gym's capture models by capture_key (admission, snapshot, staged record, receipt) instead of the deprecated alias. NeMo RL's own TQ column and tag names keep rollout_id. - The QA distillation recipe lists verifiers_agent.yaml, the target of the removed acereason-math.yaml alias. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- examples/modelopt/qa_distillation_nano3_megatron.yaml | 2 +- nemo_rl/data_plane/tq_token_sink.py | 6 +++--- nemo_rl/environments/nemo_gym.py | 2 +- nemo_rl/experience/rollout_reassembler.py | 4 ++-- nemo_rl/models/generation/vllm/vllm_worker_async.py | 4 ++-- tests/unit/data/test_captured_media.py | 2 +- tests/unit/data_plane/test_rollout_reassembler.py | 8 ++++---- tests/unit/data_plane/token_capture_test_fixtures.py | 7 ++++--- .../generation/test_dynamo_native_engine_capture.py | 8 ++++---- 9 files changed, 22 insertions(+), 21 deletions(-) diff --git a/examples/modelopt/qa_distillation_nano3_megatron.yaml b/examples/modelopt/qa_distillation_nano3_megatron.yaml index bfe75b2844d..a8cfbb6b9de 100644 --- a/examples/modelopt/qa_distillation_nano3_megatron.yaml +++ b/examples/modelopt/qa_distillation_nano3_megatron.yaml @@ -158,7 +158,7 @@ env: nemo_gym: config_paths: - responses_api_models/vllm_model/configs/vllm_model_for_training.yaml - - responses_api_agents/verifiers_agent/configs/acereason-math.yaml + - responses_api_agents/verifiers_agent/configs/verifiers_agent.yaml # Use legacy agent orchestration. verifiers_agent_environment_server: diff --git a/nemo_rl/data_plane/tq_token_sink.py b/nemo_rl/data_plane/tq_token_sink.py index db8dc76654e..d9699842cfb 100644 --- a/nemo_rl/data_plane/tq_token_sink.py +++ b/nemo_rl/data_plane/tq_token_sink.py @@ -499,7 +499,7 @@ def stage( "extras_digest_version": torch.tensor( [record.extras_digest_version], dtype=torch.int64 ), - "rollout_id_utf8": _bytes_tensor(record.rollout_id.encode("utf-8")), + "rollout_id_utf8": _bytes_tensor(record.capture_key.encode("utf-8")), "model_call_id_utf8": _bytes_tensor( record.model_call_id.encode("utf-8") ), @@ -596,7 +596,7 @@ def stage( field_dict.update(media_columns) tags = [ { - "rollout_id": record.rollout_id, + "rollout_id": record.capture_key, "model_call_id": record.model_call_id, "parent_call_id": record.parent_call_id, "prev_len": record.prev_len, @@ -1092,7 +1092,7 @@ def _optional_digest(name: str, present_name: str) -> str | None: schema_version=int(_row_leaf(row, "schema_version")[0].item()), digest_version=int(_row_leaf(row, "digest_version")[0].item()), extras_digest_version=int(_row_leaf(row, "extras_digest_version")[0].item()), - rollout_id=_row_text(row, "rollout_id_utf8"), + capture_key=_row_text(row, "rollout_id_utf8"), model_call_id=_row_text(row, "model_call_id_utf8"), parent_call_id=parent_call_id, mode=mode, diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 5ab370dbe65..f94bf9509d4 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -1493,7 +1493,7 @@ def _assemble_receipt( elif terminal_record is None: failure_reason = selection_reason or "missing_terminal_row" return { - "rollout_id": rollout_id, + "capture_key": rollout_id, "reward": reward, "terminal_model_call_id": ( terminal_record.get("model_call_id") diff --git a/nemo_rl/experience/rollout_reassembler.py b/nemo_rl/experience/rollout_reassembler.py index ccb7ac112d0..4f62d6ec5c6 100644 --- a/nemo_rl/experience/rollout_reassembler.py +++ b/nemo_rl/experience/rollout_reassembler.py @@ -341,8 +341,8 @@ def rejected(reason: str, staging_keys: list[str]) -> FinalizedRollout: except (TypeError, ValueError) as error: return rejected(f"invalid_receipt:{error}", []) staging_keys = [record.staging_key for record in parsed.manifest] - if parsed.rollout_id != rollout_id: - return rejected(f"identity_mismatch:{parsed.rollout_id}", staging_keys) + if parsed.capture_key != rollout_id: + return rejected(f"identity_mismatch:{parsed.capture_key}", staging_keys) if parsed.failure_reason is not None: return rejected(f"rollout_failed:{parsed.failure_reason}", staging_keys) if parsed.capture_poisoned: diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index b537e5961c9..733677a9056 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -846,7 +846,7 @@ def _capture_request_media( raise MediaCaptureRejected("Cycle in retained media chain") visited.add(parent) call = source.fetch_for_finalization( - [staging_key(admission.rollout_id, parent)], + [staging_key(admission.capture_key, parent)], include_route_fragments=False, )[0] calls.append(call) @@ -860,7 +860,7 @@ def _capture_request_media( for call in calls: snapshot = call.snapshot if ( - snapshot.rollout_id != admission.rollout_id + snapshot.capture_key != admission.capture_key or snapshot.parent_call_id != parent or snapshot.prev_len != length or snapshot.chain_hash diff --git a/tests/unit/data/test_captured_media.py b/tests/unit/data/test_captured_media.py index f0c68cfcb9c..238f95adb85 100644 --- a/tests/unit/data/test_captured_media.py +++ b/tests/unit/data/test_captured_media.py @@ -1077,7 +1077,7 @@ def test_worker_bounds_cyclic_inline_chain_lookup(retained_call): @pytest.mark.parametrize( "field,value", [ - ("rollout_id", "other"), + ("capture_key", "other"), ("parent_call_id", "other"), ("prev_len", 1), ("chain_hash", "invalid"), diff --git a/tests/unit/data_plane/test_rollout_reassembler.py b/tests/unit/data_plane/test_rollout_reassembler.py index 8d871629899..bb59c4d8942 100644 --- a/tests/unit/data_plane/test_rollout_reassembler.py +++ b/tests/unit/data_plane/test_rollout_reassembler.py @@ -228,7 +228,7 @@ def test_finalize_group_publishes_n_rows_with_placeholder(tq_client, partitions) receipt, expected = _stage_fixture( tq_client, "worked_example", rollout_id=f"{group_id}_g0" ) - receipt["rollout_id"] = f"{group_id}_g0" + receipt["capture_key"] = f"{group_id}_g0" # Mark this receipt's terminal as heuristically selected so the group # metric sees a mixed declared/heuristic population. receipt["terminal_selection"] = "heuristic" @@ -297,7 +297,7 @@ def test_finalize_group_skips_unset_terminal_selection(tq_client, partitions): receipt, expected = _stage_fixture( tq_client, "worked_example", rollout_id=f"{group_id}_g0" ) - receipt["rollout_id"] = f"{group_id}_g0" + receipt["capture_key"] = f"{group_id}_g0" assert receipt["terminal_selection"] == "declared" # A manifest that never parsed ran no attribution stage, so the receipt # carries terminal_selection=None (Gym #2823, pinned 9fc05c0f) rather than a method. @@ -355,7 +355,7 @@ def test_finalize_group_maps_physical_attempt_to_stable_canonical_id( "worked_example", rollout_id=physical_id, ) - receipt["rollout_id"] = physical_id + receipt["capture_key"] = physical_id finalized = _finalizer(tq_client).finalize_group( group_id, @@ -479,7 +479,7 @@ def _record_with_routes(record: StagedCallRecord, routes: list) -> StagedCallRec schema_version=record.schema_version, digest_version=record.digest_version, extras_digest_version=record.extras_digest_version, - rollout_id=record.rollout_id, + capture_key=record.capture_key, model_call_id=record.model_call_id, parent_call_id=record.parent_call_id, mode=record.mode, diff --git a/tests/unit/data_plane/token_capture_test_fixtures.py b/tests/unit/data_plane/token_capture_test_fixtures.py index 7fc92162c1c..c4edbbd7550 100644 --- a/tests/unit/data_plane/token_capture_test_fixtures.py +++ b/tests/unit/data_plane/token_capture_test_fixtures.py @@ -6,6 +6,7 @@ from __future__ import annotations from nemo_gym.token_id_capture.staging.digest import ( + STAGING_SCHEMA_VERSION, compute_chain_hash, compute_extras_digest, compute_staging_digest, @@ -47,7 +48,7 @@ def _record( cum_len = prev_len + delta_len extras_digest = compute_extras_digest(None) values = { - "rollout_id": rollout_id, + "capture_key": rollout_id, "model_call_id": model_call_id, "parent_call_id": parent_call_id, "mode": mode, @@ -66,7 +67,7 @@ def _record( return StagedCallRecord( **values, digest=compute_staging_digest( - schema_version=2, + schema_version=STAGING_SCHEMA_VERSION, digest_version=2, extras_digest_version=1, **{key: value for key, value in values.items() if key != "extras"}, @@ -136,7 +137,7 @@ def build_fixture_artifacts( ) ) receipt = RolloutReceipt( - rollout_id=rollout_id, + capture_key=rollout_id, terminal_model_call_id=records[-1].model_call_id, manifest=[_manifest(record) for record in records], terminal_selection="declared", diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py index 9c509a10fa0..d572f4b68cf 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine_capture.py +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -123,7 +123,7 @@ def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. prefix = [1, 5, 6, EOS] admission = CaptureAdmission( - rollout_id="rollout-1", + capture_key="rollout-1", model_call_id="c2", parent_call_id="c1", prev_len=len(prefix), @@ -172,7 +172,7 @@ def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: def test_a_call_that_fails_after_admission_fails_its_capture() -> None: admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" + capture_key="rollout-1", model_call_id="c1", mode="text" ) class _FailingLLM(_FakeLLM): @@ -192,7 +192,7 @@ async def generate(self, prompt, sampling_params, request_id): def test_without_capture_installed_a_captured_call_is_served_uncaptured() -> None: admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" + capture_key="rollout-1", model_call_id="c1", mode="text" ) llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) host = _host(llm) @@ -206,7 +206,7 @@ def test_without_capture_installed_a_captured_call_is_served_uncaptured() -> Non def test_a_captured_media_call_stages_the_media_the_engine_ran_on() -> None: admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" + capture_key="rollout-1", model_call_id="c1", mode="text" ) processed = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] llm = _FakeLLM([_output(processed, _completion([10], "stop"))]) From 46317dca5de16d5145dfa9c9920f26cc8d1cf1f0 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:11:41 -0700 Subject: [PATCH 11/82] fix(dynamo): address the #4979 review and slim the NeMo-Gym path Review fixes: - Default frontend session affinity to 300 s (Dynamo caps the table at 65,536 sessions and answers 529 past it), bounded by the frontend's max. - Never stage a call vLLM ended with finish_reason "error" (raise first, as vLLM's chat server does); decode special tokens as the request asks when vLLM matches stop strings, so engine and frontend stop at the same place. - Read Gym's NG_COMMIT_COORDS_FIELD instead of copying the key. - Raise the frontend HTTP body and request-plane message limits for inline Gym media. - Reject vllm_model.extra_body.bad_words at setup (the frontend 400s it). - Tests for the disconnect branch, the media-placeholder guard and the error finish; the dialect test reads fields back through Gym's VLLMModelConfig. Docs list the template-kwargs and 4-stop limits. - Keep the receipt dict's rollout_id key: NeMo RL's recovery and manager identity checks read it, and Gym still accepts the alias. - swe1 drops request_timeout_s (nothing reads it for Gym calls). Simplifications: - splice_prefix_tokens(template_prefix_token_ids=None) replaces derive_prefix_eos_count and template_prefix_eos_count. - One _gym_message builds the native message fields for both the staged and the engine_data paths; no separate staged flag (staging pops the capture). - The dialect key stores the template kwargs directly. - Fold duplicate and circular tests into parametrized cases; drop the unused Omni Dynamo hook and repeated guide text. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 11 +- docs/guides/dynamo-generation.md | 28 ++-- ...grpo_math_1B_dynamo_single_controller.yaml | 9 +- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 1 - .../nemo_gym/run_distillation_nemo_gym.py | 2 +- examples/nemo_gym/run_grpo_nemo_gym.py | 2 +- nemo_rl/environments/nemo_gym.py | 32 +++-- nemo_rl/models/generation/dynamo/config.py | 10 +- .../generation/dynamo/managed_runtime.py | 5 + .../models/generation/dynamo/native_engine.py | 129 ++++++++--------- .../models/generation/openai_server_utils.py | 84 ++++------- ...lm_token_capture_single_controller_1n2g.sh | 4 - .../test_nemo_gym_token_capture.py | 87 +++++------- .../unit/environments/test_nemo_gym_utils.py | 21 ++- .../models/generation/test_dynamo_config.py | 2 +- .../generation/test_dynamo_native_engine.py | 134 ++++++------------ .../test_dynamo_native_engine_capture.py | 114 +++++++++++---- .../generation/test_openai_server_utils.py | 38 ++--- tests/unit/single_controller/test_setup.py | 7 +- 19 files changed, 336 insertions(+), 384 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 843dad2c897..942cd8fec15 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -33,13 +33,10 @@ implements its `LLMEngine` over the worker's `AsyncLLM`: output with `request_pipeline.rollout_output`. NeMo-Gym's calls also carry an annotation (Gym's Dynamo dialect). The engine - applies the native chat endpoint's rules to them: the on-policy sampling check - (`vllm/utils.off_policy_sampling_error`), the prefix splice - (`openai_server_utils.splice_prefix_tokens`) and the context clamp. A - token-capture call stages through the worker's own capture lifecycle - (`_begin_request_capture` / `_finish_request_capture`), and the token fields - and routed experts come from the native endpoint's helpers - (`chat_generation_tokens`, `chat_routed_experts`). + applies the native chat endpoint's rules to them (on-policy sampling, the + prefix splice, the context clamp), stages a token-capture call through the + worker's own capture lifecycle, and takes the token fields and routed experts + from the native endpoint's helpers. A request with media carries the frontend's tokens, with one unexpanded placeholder per item, and the media URLs. The adapter loads the media with diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index f1764202897..894ab5ba667 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -55,7 +55,7 @@ shows them all: | `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | | `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | -| `frontend_args.session_affinity_ttl_secs` | 1800 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it | +| `frontend_args.session_affinity_ttl_secs` | 300 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it. Dynamo keeps at most 65,536 sessions and answers a new one past that with HTTP 529 | | `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | @@ -96,27 +96,20 @@ every call with an annotation, and the engine serves a marked call as the native chat endpoint does: - Sampling must match `temperature`/`top_p` or `val_temperature`/`val_top_p`, - with `top_k` unset. vLLM samples with the configured values. + with `top_k` unset and `n` 1. vLLM samples with the configured values. - The token prefix Gym echoes replaces the history the frontend rendered. -- Output is clamped to the context, and a prompt that leaves no room is refused - with the native endpoint's "maximum context length" error. - Stop strings stop vLLM itself, so the token IDs match the native endpoint's. - The response carries `prompt_token_ids`, `completion_token_ids` and `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. With router replay on, it also carries the call's `routed_experts`, as the native endpoint attaches them. -- Images, video and audio in the messages are processed as on the native - endpoint. The frontend passes each media URL through with one unexpanded - placeholder per item. The engine loads the media with vLLM's media connector - and runs vLLM's multimodal processor on the frontend's tokens. The splice, - placeholder remap and media capture then work on that processed prompt. +- Images, video and audio in the messages work as on the native endpoint, + token capture included. With `token_capture.enabled`, Gym uses its `dynamo_worker` capture backend. The same vLLM workers stage each call's tokens, log probabilities, routed experts and processed media from their own process, exactly as the native endpoint does, and -only the commit coordinates travel back through the frontend. The frontend keeps a rollout on -one engine (`session_affinity_ttl_secs`), as the native backend pins each Gym -session to one server. +only the commit coordinates travel back through the frontend. The native server's chat settings carry over: @@ -135,6 +128,11 @@ Where the behavior differs from the native endpoint: history. - When a request sets `chat_template_kwargs.force_nonempty_content`, Dynamo reports a reasoning-only turn as content. +- `default_chat_template_kwargs` sit under the template kwargs set in + `env.nemo_gym` only. A key that a Gym YAML also sets loses to the default, and + template kwargs sent through `extra_body` from a Gym YAML or a row's + `metadata.extra_body` are dropped. +- The frontend refuses more than 4 `stop` strings or `stop_token_ids` (HTTP 500). ## Not supported @@ -148,6 +146,10 @@ Startup rejects these with `dynamo: true`: - `async_rl.generation_router`: the frontend already routes the fleet behind one URL. - `vllm_cfg.http_server_api_key_required`: the frontend checks no bearer token. +- `extra_body.bad_words` on the policy model + (`env.nemo_gym.policy_model.responses_api_models.vllm_model`): the frontend + answers every request carrying it with HTTP 400. Startup checks only this + overlay, not Gym's own YAMLs. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data parallelism inside vLLM, which the async engine Dynamo serves does not. - Parser plugins with no Dynamo parser, and the chat settings above that the @@ -155,8 +157,6 @@ Startup rejects these with `dynamo: true`: - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. -A Gym call with `n > 1` is refused when it arrives. - ## Build the image ai-dynamo is part of the `vllm` extra, so every vLLM worker venv carries it. diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 9ad2b249a95..d38b9c949a2 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -14,9 +14,10 @@ policy: dynamo_cfg: # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: 600 - # Timeout for one rollout routed through the frontend (in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS). A timeout fails the attempt as a native - # timeout does, and the rollout retry policy applies. + # Timeout for one NeMo RL rollout routed through the frontend (in place + # of NRL_VLLM_ASYNC_TIMEOUT_SECONDS). A timeout fails the attempt as a + # native timeout does, and the rollout retry policy applies. NeMo-Gym's + # own calls keep Gym's timeouts. request_timeout_s: 900 worker_args: # Dynamo's built-in parser names. null follows the native server's @@ -39,7 +40,7 @@ policy: router_mode: kv # Keep a NeMo-Gym token-capture rollout on one engine while it idles up # to this many seconds (null disables it). - session_affinity_ttl_secs: 1800 + session_affinity_ttl_secs: 300 extra_cli_args: [] logger: diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml index d12f2536970..02153f83ba6 100644 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml @@ -85,7 +85,6 @@ policy: dynamo: true dynamo_cfg: startup_timeout_s: 1800 - request_timeout_s: 7200 frontend_args: tokenizer: fastokens tokenizer_cache: true diff --git a/examples/nemo_gym/run_distillation_nemo_gym.py b/examples/nemo_gym/run_distillation_nemo_gym.py index c5b590deee5..95b7a010e2c 100644 --- a/examples/nemo_gym/run_distillation_nemo_gym.py +++ b/examples/nemo_gym/run_distillation_nemo_gym.py @@ -111,7 +111,7 @@ def main() -> None: # We assert here since this is right after the final config has been materialized. assert should_use_nemo_gym(config) - # NeMo-Gym environment needs to get dp_openai_server_base_urls from + # NeMo-Gym environment needs to get client_base_urls from # student_generation, so we don't setup env here. print("\n▶ Setting up data...") train_dataset, val_dataset = setup_response_data( diff --git a/examples/nemo_gym/run_grpo_nemo_gym.py b/examples/nemo_gym/run_grpo_nemo_gym.py index 906a0c1c53a..cebd79ce345 100644 --- a/examples/nemo_gym/run_grpo_nemo_gym.py +++ b/examples/nemo_gym/run_grpo_nemo_gym.py @@ -211,7 +211,7 @@ def main() -> None: config.grpo.val_period > 0 or config.grpo.val_at_start or config.grpo.val_at_end ) - # NeMo-Gym environment needs to get dp_openai_server_base_urls from policy_generation, so we don't setup env here. + # NeMo-Gym environment needs to get client_base_urls from policy_generation, so we don't setup env here. with rl_init_timer.time("data"): print("\n▶ Setting up data...") data_tokenizer = processor if processor is not None else tokenizer diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index f94bf9509d4..720a8e6af8c 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -1493,7 +1493,7 @@ def _assemble_receipt( elif terminal_record is None: failure_reason = selection_reason or "missing_terminal_row" return { - "capture_key": rollout_id, + "rollout_id": rollout_id, "reward": reward, "terminal_model_call_id": ( terminal_record.get("model_call_id") @@ -2057,14 +2057,13 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_token_ids"] = None if dynamo_enabled(generation_config): - # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect. - # Each actor applies it to its own composed config (_apply_dynamo_dialect), - # so it never becomes a policy_model overlay that a shard must claim. - config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = { - "default_chat_template_kwargs": native_default_chat_template_kwargs( - generation_config - ) - } + # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect, + # with the native server's default template kwargs. Each actor applies it + # to its own composed config (_apply_dynamo_dialect), so it never becomes + # a policy_model overlay that a shard must claim. + config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = ( + native_default_chat_template_kwargs(generation_config) + ) # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for @@ -2141,12 +2140,19 @@ def _apply_dynamo_dialect( The frontend takes token data in nvext instead of vLLM's request extensions. Applied to the actor's composed config (a shard's overrides included), so - settings the user wrote there win. Copies the policy_model levels it - changes, so the caller's config is not mutated. + settings the user wrote there win. Sets ``policy_model`` on ``nemo_gym_dict`` + in place, so pass a per-actor copy; the nested policy_model levels are + copied, so dicts shared with the user's config are not mutated. """ policy_model = dict(nemo_gym_dict.get("policy_model") or {}) models = dict(policy_model.get("responses_api_models") or {}) vllm_model = dict(models.get("vllm_model") or {}) + if "bad_words" in (vllm_model.get("extra_body") or {}): + raise ValueError( + "vllm_model.extra_body.bad_words is not supported with " + "policy.generation.dynamo=true: the Dynamo frontend answers every " + "request carrying it with HTTP 400." + ) vllm_model["dynamo_frontend"] = True # The frontend reads `reasoning` as another name for `reasoning_content` and # refuses an echoed assistant turn that carries both, Gym's default. @@ -2197,9 +2203,7 @@ def _build_gym_actor_config( tokenizer_config = nemo_gym_dict.pop("tokenizer_config", None) dynamo_dialect = nemo_gym_dict.pop("dynamo_dialect", None) if dynamo_dialect is not None: - _apply_dynamo_dialect( - nemo_gym_dict, dynamo_dialect["default_chat_template_kwargs"] - ) + _apply_dynamo_dialect(nemo_gym_dict, dynamo_dialect) external_service_readiness = extract_external_service_readiness(nemo_gym_dict) port_range = { key: value diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 7a6a61868d2..fb65516f1b1 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -19,7 +19,7 @@ every field below carries its default. """ -from typing import Any, Literal, Mapping +from typing import Annotated, Any, Literal, Mapping from pydantic import BaseModel, Field, PositiveFloat, PositiveInt, model_validator @@ -69,10 +69,12 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): ] = "kv" # Idle TTL in seconds for keeping one session on one engine, or None to # disable it. NeMo-Gym names the session on token-capture calls only (its - # rollout id), as the native backend pins each Gym session to one server; the + # capture key), as the native backend pins each Gym session to one server; the # engine splices a call's staged prefix after routing, so the KV router alone - # cannot see where that prefix is cached. - session_affinity_ttl_secs: PositiveInt | None = 1800 + # cannot see where that prefix is cached. Dynamo keeps at most 65,536 sessions + # and answers a new one past that with HTTP 529, so keep (captured rollouts + # started per second) x TTL below 65,536. The bound is the frontend's own. + session_affinity_ttl_secs: Annotated[int, Field(gt=0, le=31_536_000)] | None = 300 extra_cli_args: list[str] = Field(default_factory=list) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index e4a0fe9507e..42a862c0785 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -172,6 +172,11 @@ def allocate_port(*, low: int, high: int) -> int: "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", + # NeMo-Gym sends media inline: raise the frontend's 45 MB HTTP body + # limit, and the 32 MiB request-plane limit above it (the engine + # request carries the same media plus token IDs). + "DYN_HTTP_BODY_LIMIT_MB": "512", + "DYN_TCP_MAX_MESSAGE_SIZE": str(1 << 30), # A canary request would stall behind a refit pause; engine liveness # is the Ray actor's, as for every other vLLM worker. "DYN_HEALTH_CHECK_ENABLED": "false", diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index 42f7694c6cc..a55291407db 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -78,10 +78,7 @@ DYNAMO_ENDPOINT, ) from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION -from nemo_rl.models.generation.openai_server_utils import ( - derive_prefix_eos_count, - splice_prefix_tokens, -) +from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens from nemo_rl.models.generation.vllm.request_pipeline import rollout_result_to_wire from nemo_rl.models.generation.vllm.utils import ( chat_generation_tokens, @@ -106,6 +103,12 @@ # IDs, which the native server reads from the messages instead. GYM_PREFIX_ANNOTATION_PREFIX = "ng_prefix:" _GYM_SAMPLING_KEYS = ("temperature", "top_p", "val_temperature", "val_top_p") +# The engine_data name Gym's dialect reads for each native message field. +_GYM_ENGINE_DATA_KEYS = { + "generation_token_ids": "completion_token_ids", + "generation_log_probs": "completion_logprobs", + "routed_experts": "routed_experts", +} # The vLLM modality each key of Dynamo's multi_modal_data holds. _DYNAMO_MEDIA_MODALITIES = { @@ -500,9 +503,11 @@ async def _generate_gym( """ host = self._host sampling: dict[str, Any] = request.get("sampling_options") or {} + temperature = _as_f32(sampling.get("temperature")) + top_p = _as_f32(sampling.get("top_p")) sampling_error = off_policy_sampling_error( - temperature=_as_f32(sampling.get("temperature")), - top_p=_as_f32(sampling.get("top_p")), + temperature=temperature, + top_p=top_p, top_k=sampling.get("top_k"), generation_config=self._gym_sampling_profile, ) @@ -513,23 +518,17 @@ async def _generate_gym( "NeMo-Gym calls through Dynamo take n=1: engine_data carries one " "choice's tokens." ) - # The profile's own values, so vLLM samples exactly as natively. - validation = ( - _as_f32(sampling["temperature"]) - != self._gym_sampling_profile["temperature"] - or _as_f32(sampling["top_p"]) != self._gym_sampling_profile["top_p"] - ) - temperature, top_p = ( - (host.cfg["val_temperature"], host.cfg["val_top_p"]) - if validation - else (host.cfg["temperature"], host.cfg["top_p"]) - ) + # The train or validation profile's own values, so vLLM samples exactly + # as natively. + profile = self._gym_sampling_profile + if (temperature, top_p) == (profile["temperature"], profile["top_p"]): + temperature, top_p = host.cfg["temperature"], host.cfg["top_p"] + else: + temperature, top_p = host.cfg["val_temperature"], host.cfg["val_top_p"] capture_context = _annotation_value(request, GYM_CAPTURE_ANNOTATION_PREFIX) - admission = ( - None - if capture_context is None - else host._parse_capture_admission(json.loads(capture_context)) + admission = host._parse_capture_admission( + None if capture_context is None else json.loads(capture_context) ) prefix_token_ids = None if admission is not None: @@ -553,16 +552,13 @@ async def _generate_gym( if prefix_token_ids is not None: # The frontend rendered the whole conversation; replace its history # with the exact tokens the model generated for it. The engine sees no - # messages, so the prefix's EOS count stands in for the native - # endpoint's render of the history. - tokenizer = self._llm.tokenizer + # messages, so the prefix stands in for the native endpoint's render + # of the history. splice = splice_prefix_tokens( - tokenizer=tokenizer, + tokenizer=self._llm.tokenizer, model_prefix_token_ids=prefix_token_ids, + template_prefix_token_ids=None, template_token_ids=template_token_ids, - template_prefix_eos_count=derive_prefix_eos_count( - prefix_token_ids, eos_token_id=tokenizer.eos_token_id - ), ) token_ids = splice.token_ids self._check_prompt_fits(len(token_ids)) @@ -599,8 +595,14 @@ async def _generate_gym( } if stop: # vLLM stops on them as natively, so the token arrays match; the - # frontend then cuts the text at the same place. - overrides.update(stop=stop, detokenize=True) + # frontend then cuts the text at the same place, so both must decode + # special tokens alike (the frontend defaults this off for some parsers). + skip = (request.get("output_options") or {}).get("skip_special_tokens") + overrides.update( + stop=stop, + detokenize=True, + skip_special_tokens=True if skip is None else skip, + ) if admission is not None: # A captured call stages the sampled tokens' log probabilities. overrides["logprobs"] = 0 @@ -624,7 +626,6 @@ async def _generate_gym( prefix_token_ids=prefix_token_ids, media=media, ) - staged = False try: final_output = None async for output in self._llm.generate( @@ -639,6 +640,9 @@ async def _generate_gym( # native endpoint's cancellation does, instead of staging it. return completion = final_output.outputs[0] + if completion.finish_reason == "error": + # As vLLM's chat server does (_raise_if_error): never stage a failed call. + raise RuntimeError(f"Generation failed for request {context.id()}") chunk: GenerateChunk = { "token_ids": list(completion.token_ids), "index": 0, @@ -646,14 +650,12 @@ async def _generate_gym( "completion_usage": _usage(final_output, len(completion.token_ids)), } if admission is not None: - chunk["engine_data"] = { - "ng_commit_coords": await self._stage_gym_call( - request, final_output - ) - } - staged = True + chunk["engine_data"] = await self._stage_gym_call(request, final_output) else: - chunk["engine_data"] = self._gym_token_data(final_output) + engine_data = {"prompt_token_ids": list(final_output.prompt_token_ids)} + for field, value in self._gym_message(final_output).items(): + engine_data[_GYM_ENGINE_DATA_KEYS[field]] = value + chunk["engine_data"] = engine_data log_probs, top_logprobs = extract_from_completion_output( completion, 0, tokenizer=self._llm.tokenizer, include_bytes=True ) @@ -662,7 +664,8 @@ async def _generate_gym( chunk["top_logprobs"] = top_logprobs yield chunk finally: - if admission is not None and not staged: + # A no-op once the call was staged: staging pops its capture state. + if admission is not None: host._abort_request_capture( request, reason=( @@ -754,18 +757,15 @@ def _check_prompt_fits(self, prompt_len: int) -> None: "output tokens." ) - def _gym_token_data(self, final_output: Any) -> dict[str, Any]: - """The token fields Gym's dialect lifts onto the message, as the native endpoint reports them.""" + def _gym_message(self, final_output: Any) -> dict[str, Any]: + """The token fields the native endpoint puts on the call's message.""" completion = final_output.outputs[0] - token_data: dict[str, Any] = { - "prompt_token_ids": list(final_output.prompt_token_ids), - "completion_token_ids": list(completion.token_ids), - } + message: dict[str, Any] = {"generation_token_ids": list(completion.token_ids)} if completion.logprobs is not None: - _, token_data["completion_logprobs"] = chat_generation_tokens(completion) + _, message["generation_log_probs"] = chat_generation_tokens(completion) if self._host._return_routed_experts_enabled(): # Full prompt + generation rows, as the native endpoint attaches them. - token_data["routed_experts"] = chat_routed_experts( + message["routed_experts"] = chat_routed_experts( final_output, completion, prompt_token_count=len(final_output.prompt_token_ids), @@ -773,42 +773,31 @@ def _gym_token_data(self, final_output: Any) -> dict[str, Any]: logger=LOGGER, routed_experts_dtype=self._host.routed_experts_dtype, ) - return token_data + return message + + async def _stage_gym_call( + self, request: GenerateRequest, final_output: Any + ) -> dict[str, Any]: + """Stage a captured call from this process; return the engine_data carrying its commit coordinates.""" + # Deferred: only captured calls reach here, and capture already imports nemo_gym. + from nemo_gym.token_id_capture import NG_COMMIT_COORDS_FIELD - async def _stage_gym_call(self, request: GenerateRequest, final_output: Any) -> Any: - """Stage a captured call from this process; return its commit coordinates.""" - host = self._host - completion = final_output.outputs[0] - generation_token_ids, generation_log_probs = chat_generation_tokens(completion) - message: dict[str, Any] = { - "generation_token_ids": generation_token_ids, - "generation_log_probs": generation_log_probs, - } - if host._return_routed_experts_enabled(): - message["routed_experts"] = chat_routed_experts( - final_output, - completion, - prompt_token_count=len(final_output.prompt_token_ids), - device=torch.device("cpu"), - logger=LOGGER, - routed_experts_dtype=host.routed_experts_dtype, - ) # The native endpoint's response shape; the prompt is the one recorded at # begin_call. content = { "choices": [ { "index": 0, - "finish_reason": completion.finish_reason, - "message": message, + "finish_reason": final_output.outputs[0].finish_reason, + "message": self._gym_message(final_output), } ] } # The sink write is a blocking data-plane round trip. content = await asyncio.to_thread( - host._finish_request_capture, request, content + self._host._finish_request_capture, request, content ) - return content["ng_commit_coords"] + return {NG_COMMIT_COORDS_FIELD: content[NG_COMMIT_COORDS_FIELD]} async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) diff --git a/nemo_rl/models/generation/openai_server_utils.py b/nemo_rl/models/generation/openai_server_utils.py index 53a757dc04f..cd6b49ad2e1 100644 --- a/nemo_rl/models/generation/openai_server_utils.py +++ b/nemo_rl/models/generation/openai_server_utils.py @@ -58,48 +58,13 @@ def replace_prefix_tokens( ).token_ids -def _eos_token_id_set(eos_token_id: int | Collection[int]) -> frozenset[int]: - eos_token_ids = ( - frozenset([eos_token_id]) - if isinstance(eos_token_id, int) - else frozenset(eos_token_id) - ) - assert eos_token_ids, "EOS token id collection must not be empty" - return eos_token_ids - - -def derive_prefix_eos_count( - model_prefix_token_ids: list[int], *, eos_token_id: int | Collection[int] -) -> int: - """The EOS count a template render of the prefix's history would hold. - - For hosts that see only the rendered prompt, not the messages (the managed - Dynamo engine receives token ids from its frontend), this stands in for - counting EOS in ``template_prefix_token_ids``. Every EOS in the prefix - closed one message, and a template closes the final assistant turn with one - EOS even when the model was truncated before emitting it. The two counts - agree when the history up to the prefix's last assistant turn is the - history the prefix was generated from (Gym's lineage admission checks this - for captured calls; without capture it is assumed, as on the native server) - and the model emitted no EOS text as ordinary tokens. A render with fewer - EOS than this count fails the splice; an extra EOS before the boundary cuts - early instead. - """ - eos_token_ids = _eos_token_id_set(eos_token_id) - count = sum(tid in eos_token_ids for tid in model_prefix_token_ids) - if not model_prefix_token_ids or model_prefix_token_ids[-1] not in eos_token_ids: - count += 1 - return count - - def splice_prefix_tokens( *, tokenizer: Any, model_prefix_token_ids: list[int], - template_prefix_token_ids: list[int] | None = None, + template_prefix_token_ids: list[int] | None, template_token_ids: list[int], eos_token_id: int | Collection[int] | None = None, - template_prefix_eos_count: int | None = None, ) -> PrefixSplice: """This is a subroutine used inside the OpenAI-compatible Chat Completion server. @@ -168,23 +133,26 @@ def splice_prefix_tokens( exact id is kept in place of the template's, so the spliced prompt still starts with the model's tokens verbatim. - Pass exactly one of ``template_prefix_token_ids`` and - ``template_prefix_eos_count``. The prefix render is only ever used for its - EOS count, so a host without the messages passes the count directly (see - :func:`derive_prefix_eos_count`). + A host that sees only the rendered prompt, not the messages (the Dynamo + engine), passes ``template_prefix_token_ids=None``. The prefix render is only + used for its EOS count, and every EOS in the model's prefix closed one + message, plus the one the template adds to a final assistant turn the model + did not close. The counts agree when the history up to that turn is the one + the prefix was generated from and the model emitted no EOS text as ordinary + tokens. """ - if (template_prefix_token_ids is None) == (template_prefix_eos_count is None): - raise ValueError( - "Pass exactly one of template_prefix_token_ids and " - "template_prefix_eos_count" - ) if not model_prefix_token_ids: return PrefixSplice(template_token_ids, 0, 0) if eos_token_id is None: eos_token_id = tokenizer.eos_token_id assert eos_token_id is not None, "Tokenizer must have an EOS token ID" - eos_token_ids = _eos_token_id_set(eos_token_id) + eos_token_ids = ( + frozenset([eos_token_id]) + if isinstance(eos_token_id, int) + else frozenset(eos_token_id) + ) + assert eos_token_ids, "EOS token id collection must not be empty" # The model isn't guaranteed to end on EOS (e.g. it hit max_tokens); chat # templates always add one, so cut the model input to just before its EOS. @@ -192,16 +160,18 @@ def splice_prefix_tokens( model_ended_on_eos = model_prefix_token_ids[-1] in eos_token_ids if model_ended_on_eos: model_cut_end -= 1 + if template_prefix_token_ids is None: + template_prefix_token_ids = ( + model_prefix_token_ids + if model_ended_on_eos + else [*model_prefix_token_ids, next(iter(eos_token_ids))] + ) # Locate the turn boundary by EOS count rather than token position. Qwen3 # templates may strip prior reasoning blocks when re-rendering history; # EOS counting preserves the original generated reasoning tokens without # requiring a customized chat template. - count_needed = ( - template_prefix_eos_count - if template_prefix_token_ids is None - else sum(tid in eos_token_ids for tid in template_prefix_token_ids) - ) + count_needed = sum(tid in eos_token_ids for tid in template_prefix_token_ids) count_seen = 0 template_cut_start = -1 for pos, tid in enumerate(template_token_ids): @@ -215,14 +185,14 @@ def splice_prefix_tokens( message = ( f"EOS token #{count_needed} not found in template_token_ids " f"(only found {count_seen} EOS tokens total)!\n" + f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" + f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" ) - if template_prefix_token_ids is not None: - message += f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" - message += f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" if tokenizer is not None: - if template_prefix_token_ids is not None: - message += f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}" - message += f"\n\nTemplate repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" + message += ( + f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}\n\n" + f"Template repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" + ) raise AssertionError(message) boundary = ( diff --git a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh index ee3da965de3..2d265e6153b 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh @@ -157,7 +157,3 @@ uv run --no-sync tests/check_metrics.py "${JSON_METRICS}" \ 'min(data["train/finalize/media_row_rate"]) == 1' \ 'max(data["train/gen_kl_error"]) < 0.05' \ 'all_finite(data["train/reward"])' - -if [[ "$*" == *generation.dynamo=true* ]]; then - bash "${SCRIPT_DIR}/check_dynamo_overlay.sh" "${RUN_LOG}" -fi diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index dfa4a991168..f976130314f 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -577,45 +577,6 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> assert should_use_nemo_gym(config) is True -@pytest.mark.parametrize("dynamo", [False, True]) -def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( - dynamo: bool, -) -> None: - """With Dynamo, Gym's policy model calls the frontend in Gym's Dynamo dialect.""" - from nemo_rl.algorithms.grpo import MasterConfig - from nemo_rl.environments.nemo_gym import ( - setup_nemo_gym_config, - should_use_nemo_gym, - ) - - config = MasterConfig.model_construct( - env={"should_use_nemo_gym": True}, - data={}, - policy={ - "generation": { - "backend": "vllm", - "dynamo": dynamo, - "vllm_cfg": {"async_engine": False, "expose_http_server": False}, - } - }, - ) - - setup_nemo_gym_config(config, tokenizer=None) - - vllm_cfg = config.policy["generation"]["vllm_cfg"] - assert vllm_cfg["async_engine"] is True - assert vllm_cfg["expose_http_server"] is True - assert should_use_nemo_gym(config) is True - if dynamo: - # A NeMo RL key each actor applies to its composed config, not a - # policy_model overlay a sharded plan would need an owner for. - assert config.env["nemo_gym"] == { - "dynamo_dialect": {"default_chat_template_kwargs": {}} - } - else: - assert "nemo_gym" not in config.env - - @pytest.mark.parametrize( ("vllm_model", "expected"), [ @@ -632,16 +593,21 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( "force_nonempty_content": True, }, ), + # The frontend answers every request carrying bad_words with a 400. + ({"extra_body": {"bad_words": ["x"]}}, None), ], ) def test_gym_actor_config_dynamo_sends_the_native_default_template_kwargs( - vllm_model: dict, expected: dict + vllm_model: dict, expected: dict | None ) -> None: """The native server's default template kwargs sit under the ones Gym sends.""" + from responses_api_models.vllm_model.app import VLLMModelConfig + from nemo_rl.algorithms.grpo import MasterConfig from nemo_rl.environments.nemo_gym import ( build_nemo_gym_config, setup_nemo_gym_config, + should_use_nemo_gym, ) user_vllm_model = copy.deepcopy(vllm_model) @@ -672,19 +638,40 @@ def test_gym_actor_config_dynamo_sends_the_native_default_template_kwargs( ) setup_nemo_gym_config(config, tokenizer=None) - gym_config = build_nemo_gym_config( - config.env, - base_urls=["http://frontend"], - model_name="test-model", - enable_router_replay=False, - use_fastokens=False, - )["initial_global_config_dict"] + assert should_use_nemo_gym(config) is True + def build() -> dict: + return build_nemo_gym_config( + config.env, + base_urls=["http://frontend"], + model_name="test-model", + enable_router_replay=False, + use_fastokens=False, + )["initial_global_config_dict"] + + if expected is None: + with pytest.raises(ValueError, match="bad_words"): + build() + return + gym_config = build() composed = gym_config["policy_model"]["responses_api_models"]["vllm_model"] - assert composed["dynamo_frontend"] is True + # Read back through Gym's own config, which ignores keys it does not know. + gym_model = VLLMModelConfig( + host="localhost", + port=8000, + name="policy_model", + entrypoint="app.py", + base_url="http://frontend", + api_key="dummy_key", + model="test-model", + return_token_id_information=True, + uses_reasoning_parser=True, + **composed, + ) + assert gym_model.dynamo_frontend is True # The frontend refuses reasoning under both of Gym's keys. - assert composed["reasoning_field"] == "reasoning_content" - assert composed["chat_template_kwargs"] == expected + assert gym_model.reasoning_field == "reasoning_content" + assert gym_model.chat_template_kwargs == expected assert "dynamo_dialect" not in gym_config # The user's overlay is left as written. assert vllm_model == user_vllm_model diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index c13270ccdb8..33f8048f03d 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -1016,11 +1016,8 @@ def test_build_nemo_gym_actors_hands_every_actor_the_policy_api_key( assert all(cfg["policy_api_key"] == "per-job-key" for cfg in cluster.actor_configs) -@pytest.mark.parametrize( - "env_configs", [_env_configs, _shard_env_configs], ids=["unsharded", "sharded"] -) def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( - detected_uv_dirs, env_configs + detected_uv_dirs, ): """With Dynamo, each actor's composed config speaks Gym's Dynamo dialect. @@ -1029,16 +1026,14 @@ def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( """ from nemo_rl.algorithms.grpo import MasterConfig - env = env_configs() - sharded = "shards" in env["nemo_gym"] - if sharded: - env["nemo_gym"]["shards"][1]["overrides"] = { - "policy_model": { - "responses_api_models": { - "vllm_model": {"chat_template_kwargs": {"enable_thinking": False}} - } + env = _shard_env_configs() + env["nemo_gym"]["shards"][1]["overrides"] = { + "policy_model": { + "responses_api_models": { + "vllm_model": {"chat_template_kwargs": {"enable_thinking": False}} } } + } config = MasterConfig.model_construct( env=env, data={}, @@ -1074,7 +1069,7 @@ def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( for cfg in cluster.actor_configs ] # judged, then the tools shard's two replicas with their own override. - assert thinking == ([True, False, False] if sharded else [True]) + assert thinking == [True, False, False] for cfg in cluster.actor_configs: gym_config = cfg["initial_global_config_dict"] vllm_model = gym_config["policy_model"]["responses_api_models"]["vllm_model"] diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 5de0f0ef08a..ddff145ca05 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -119,7 +119,7 @@ def test_frontend_argv_is_runtime_owned() -> None: ) assert argv[argv.index("--router-mode") + 1] == "kv" # Gym names a token-capture call's session; keep the session on one engine. - assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "1800" + assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "300" assert argv[-2:] == ["--foo", "1"] no_affinity = DynamoCfg.model_validate( {"frontend_args": {"session_affinity_ttl_secs": None}} diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index bda6b3d47c5..81a216b655d 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -42,17 +42,12 @@ from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) -from nemo_rl.models.generation.openai_server_utils import ( # noqa: E402 - derive_prefix_eos_count, - splice_prefix_tokens, -) from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, rollout_result_from_wire, rollout_result_to_wire, ) -from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) @@ -178,12 +173,15 @@ def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: host.routed_experts_dtype = torch.int16 host.llm = llm host._capture_media = False + host.token_capture = None return host -def _collect(engine, request) -> list[dict]: +def _collect(engine, request, context=None) -> list[dict]: async def run(): - return [chunk async for chunk in engine.generate(request, _Context())] + return [ + chunk async for chunk in engine.generate(request, context or _Context()) + ] return asyncio.run(run()) @@ -227,7 +225,7 @@ def test_rollouts_are_served_like_generate_async() -> None: } -def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None: +def test_router_replay_routes_reach_rollouts_as_natively() -> None: routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) final = _output( [1, 2, 3], @@ -261,29 +259,6 @@ def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None for key in native: assert torch.equal(shipped[key], native[key]), key - # A Gym call without capture carries the routes in engine_data, as the - # native chat endpoint attaches them to the response. - gym_final = _output( - [1, 3, 4], - _completion([10, 11], "stop", routed_experts=routes[3:]), - prompt_routed_experts=routes[:3], - ) - gym_host = _host( - _FakeLLM([gym_final]), vllm_kwargs={"enable_return_routed_experts": True} - ) - gym_engine = native_engine.NativeVllmEngine(gym_host, leader_worker_idx=0) - - (gym_chunk,) = _collect(gym_engine, _gym_request()) - - assert gym_chunk["engine_data"]["routed_experts"] == chat_routed_experts( - gym_final, - gym_final.outputs[0], - prompt_token_count=3, - device=torch.device("cpu"), - logger=native_engine.LOGGER, - routed_experts_dtype=torch.int16, - ) - def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: llm = _FakeLLM() @@ -629,10 +604,6 @@ def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: # --- NeMo-Gym's Dynamo dialect ------------------------------------------------- -def _f32(value: float) -> float: - return float(np.float32(value)) - - def _gym_request( *, temperature=0.7, top_p=0.9, prefix=None, sampling=None, **fields ) -> dict: @@ -646,7 +617,7 @@ def _gym_request( "token_ids": [1, 3, 4], # The frontend carries sampling floats as f32. "sampling_options": { - key: _f32(value) if isinstance(value, float) else value + key: native_engine._as_f32(value) if isinstance(value, float) else value for key, value in sampling_options.items() }, # Gym's token information: logprobs=true, top_logprobs=0. @@ -656,24 +627,47 @@ def _gym_request( } -def test_gym_calls_sample_with_nemo_rls_exact_profile() -> None: +@pytest.mark.parametrize( + ("temperature", "top_p", "skip_special_tokens"), + [ + # The train profile; the frontend left skip_special_tokens unset. + (0.7, 0.9, None), + # The validation profile; the frontend decodes special tokens. + (0.0, 1.0, False), + ], +) +def test_gym_calls_sample_with_nemo_rls_exact_profile( + temperature, top_p, skip_special_tokens +) -> None: final = _output([1, 3, 4], _completion([10, 11], "stop")) llm = _FakeLLM([final]) engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + output_options = {"logprobs": 0} + if skip_special_tokens is not None: + output_options["skip_special_tokens"] = skip_special_tokens (chunk,) = _collect( - engine, _gym_request(stop_conditions={"max_tokens": 100, "stop": [""]}) + engine, + _gym_request( + temperature=temperature, + top_p=top_p, + stop_conditions={"max_tokens": 100, "stop": [""]}, + output_options=output_options, + ), ) ((prompt, params, _),) = llm.calls assert prompt["prompt_token_ids"] == [1, 3, 4] # The configured values, not the f32 values the frontend delivered. - assert (params.temperature, params.top_p) == (0.7, 0.9) - assert params.top_k == vllm.SamplingParams(top_k=-1).top_k + assert (params.temperature, params.top_p) == (temperature, top_p) + # top_k unset, as vLLM normalizes it for this temperature. + assert params.top_k == vllm.SamplingParams(temperature=temperature, top_k=-1).top_k assert params.output_kind == vllm.sampling_params.RequestOutputKind.FINAL_ONLY - # Output fits the context, and vLLM stops on the stop strings itself. + # Output fits the context, and vLLM stops on the stop strings itself, + # decoding special tokens as the frontend does. assert params.max_tokens == MAX_MODEL_LEN - 3 assert (params.stop, params.detokenize) == ([""], True) + assert params.skip_special_tokens is (skip_special_tokens is not False) assert chunk["engine_data"] == { "prompt_token_ids": [1, 3, 4], "completion_token_ids": [10, 11], @@ -683,16 +677,6 @@ def test_gym_calls_sample_with_nemo_rls_exact_profile() -> None: assert chunk["log_probs"] == [-0.5, -0.5] -def test_gym_validation_calls_sample_with_the_validation_profile() -> None: - llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - _collect(engine, _gym_request(temperature=0.0, top_p=1.0)) - - ((_, params, _),) = llm.calls - assert (params.temperature, params.top_p) == (0.0, 1.0) - - @pytest.mark.parametrize( ("request_fields", "error"), [ @@ -700,12 +684,19 @@ def test_gym_validation_calls_sample_with_the_validation_profile() -> None: ({"top_p": None}, "top_p must be set explicitly"), ({"sampling": {"top_k": 5}}, "Top k sampling parameter"), ({"sampling": {"n": 2}}, "take n=1"), + # The echoed prefix, spliced in, leaves no room for output. + ( + { + "prefix": [1, *range(20, 20 + MAX_MODEL_LEN), EOS], + "token_ids": [1, 7, EOS, 3], + }, + "maximum context length", + ), ], ) -def test_gym_calls_that_would_sample_off_policy_are_refused( - request_fields, error -) -> None: +def test_gym_calls_the_engine_cannot_serve_are_refused(request_fields, error) -> None: llm = _FakeLLM() + llm.tokenizer = _Tokenizer() engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) with pytest.raises(InvalidArgument, match=error): @@ -714,41 +705,6 @@ def test_gym_calls_that_would_sample_off_policy_are_refused( assert llm.calls == [] -def test_gym_echoed_prefix_replaces_the_rendered_history() -> None: - # The model generated [5, 6, EOS]; the frontend re-rendered that turn as [7, EOS]. - prefix, rendered = [1, 5, 6, EOS], [1, 7, EOS, 3, 4] - spliced = splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=prefix, - template_token_ids=rendered, - template_prefix_eos_count=derive_prefix_eos_count(prefix, eos_token_id=EOS), - eos_token_id=EOS, - ).token_ids - assert spliced != rendered - llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) - llm.tokenizer = _Tokenizer() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - (chunk,) = _collect(engine, _gym_request(prefix=prefix, token_ids=rendered)) - - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == spliced - assert params.max_tokens == MAX_MODEL_LEN - len(spliced) - assert chunk["engine_data"]["prompt_token_ids"] == spliced - - -def test_gym_calls_whose_splice_fills_the_context_are_refused() -> None: - prefix = [1, *range(20, 20 + MAX_MODEL_LEN), EOS] - llm = _FakeLLM() - llm.tokenizer = _Tokenizer() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match="maximum context length"): - _collect(engine, _gym_request(prefix=prefix, token_ids=[1, 7, EOS, 3])) - - assert llm.calls == [] - - def test_gym_echoed_prefix_keeps_media_placeholders_on_their_tokens() -> None: # Turn 1 ran [1, , 3] and generated [5, 6, EOS]; the frontend # re-rendered that turn as [7, EOS] and appended a turn with a second image. diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py index d572f4b68cf..882ca397461 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine_capture.py +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -14,10 +14,10 @@ """NeMo-Gym token capture through the Dynamo engine adapter. The worker's own capture lifecycle stages each call, as on the native chat -endpoint; only Gym's capture core is faked. +endpoint; only Gym's capture core, and in the media tests the worker's media +snapshot, are faked. """ -import json from types import SimpleNamespace import numpy as np @@ -30,6 +30,7 @@ # The vLLM lanes import this module without the nemo_gym extra. pytest.importorskip("nemo_gym.token_id_capture.staging") +from nemo_gym.token_id_capture import NG_COMMIT_COORDS_FIELD # noqa: E402 from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 DYNAMO_CAPTURE_ANNOTATION_PREFIX, @@ -40,6 +41,7 @@ from responses_api_models.vllm_model.app import ( # noqa: E402 _DYNAMO_ENGINE_DATA_FIELDS, ) +from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 @@ -52,6 +54,7 @@ IMAGE_TOKEN, IMAGE_TOKENS, _collect, + _Context, _completion, _FakeLLM, _gym_request, @@ -114,9 +117,10 @@ def test_annotations_and_engine_data_are_the_ones_gyms_dynamo_dialect_uses() -> DYNAMO_PREFIX_ANNOTATION_PREFIX ) # The engine_data keys Gym requires on a call without capture. - engine = native_engine.NativeVllmEngine(_host(_FakeLLM()), leader_worker_idx=0) - final = _output([1, 3, 4], _completion([10], "stop")) - assert engine._gym_token_data(final).keys() == _DYNAMO_ENGINE_DATA_FIELDS.keys() + llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + (chunk,) = _collect(engine, _gym_request()) + assert chunk["engine_data"].keys() == _DYNAMO_ENGINE_DATA_FIELDS.keys() def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: @@ -165,43 +169,103 @@ def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: expected[:4] = torch.from_numpy(routes[3:7]) assert torch.equal(staged_routes, expected) # Only the commit coordinates go back through the frontend. - assert chunk["engine_data"] == {"ng_commit_coords": COORDS} + assert chunk["engine_data"] == {NG_COMMIT_COORDS_FIELD: COORDS} assert "log_probs" not in chunk assert host._capture_calls == {} -def test_a_call_that_fails_after_admission_fails_its_capture() -> None: +class _FailingLLM(_FakeLLM): + async def generate(self, prompt, sampling_params, request_id): + raise RuntimeError("engine died") + yield # pragma: no cover + + +class _Disconnected(_Context): + def is_stopped(self) -> bool: + return True + + +@pytest.mark.parametrize( + ("llm", "context", "error", "reason"), + [ + (_FailingLLM(), _Context(), "engine died", "engine_error"), + # As vLLM's chat server: a call vLLM ended with an error is never staged. + ( + _FakeLLM([_output([1, 3, 4], _completion([10], "error"))]), + _Context(), + "Generation failed", + "engine_error", + ), + # Dynamo aborts the vLLM request on a disconnect but lets the generator finish. + ( + _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]), + _Disconnected(), + None, + "client_disconnect", + ), + ], + ids=["engine_error", "finished_with_error", "client_disconnect"], +) +def test_a_call_that_does_not_finish_cleanly_fails_its_capture( + llm, context, error, reason +) -> None: admission = CaptureAdmission( capture_key="rollout-1", model_call_id="c1", mode="text" ) - - class _FailingLLM(_FakeLLM): - async def generate(self, prompt, sampling_params, request_id): - raise RuntimeError("engine died") - yield # pragma: no cover - - host = _capturing_host(_FailingLLM()) + host = _capturing_host(llm) engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - with pytest.raises(RuntimeError, match="engine died"): - _collect(engine, _captured_request(admission)) + if error is None: + assert _collect(engine, _captured_request(admission), context) == [] + else: + with pytest.raises(RuntimeError, match=error): + _collect(engine, _captured_request(admission), context) - assert host.token_capture.failed == ["engine_error"] + assert host.token_capture.staged == [] + assert host.token_capture.failed == [reason] assert host._capture_calls == {} -def test_without_capture_installed_a_captured_call_is_served_uncaptured() -> None: +def test_a_captured_media_call_keeps_the_placeholders_capture_moved() -> None: + # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. + prefix = [1, 5, 6, EOS] admission = CaptureAdmission( - capture_key="rollout-1", model_call_id="c1", mode="text" + capture_key="rollout-1", + model_call_id="c2", + parent_call_id="c1", + prev_len=len(prefix), + mode="token_in", + required_prefix_token_ids=prefix, + parent_chain_hash="0" * 64, ) - llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) - host = _host(llm) - host.token_capture = None + spliced = [*prefix, *[IMAGE_TOKEN] * IMAGE_TOKENS, 4] + llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) + llm.tokenizer = _Tokenizer() + host = _capturing_host(llm) + host._capture_media = True + moved = {"image": [PlaceholderRange(len(prefix), IMAGE_TOKENS)]} + + def capture_request_media(engine_input, *, admission, splice): + # As capture_processed_media, which moves the placeholders itself. + engine_input["mm_placeholders"] = moved + return CapturedMedia((), None) + + host._capture_request_media = capture_request_media engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - (chunk,) = _collect(engine, _captured_request(admission)) + _collect( + engine, + _captured_request( + admission, + token_ids=[1, 7, EOS, IMAGE_PLACEHOLDER, 4], + multi_modal_data={"image_url": [{"Url": _image_url()}]}, + ), + ) - assert json.loads(json.dumps(chunk["engine_data"]))["completion_token_ids"] == [10] + ((engine_input, _, _),) = llm.calls + assert engine_input["prompt_token_ids"] == spliced + # Not remapped a second time. + assert engine_input["mm_placeholders"] == moved def test_a_captured_media_call_stages_the_media_the_engine_ran_on() -> None: @@ -238,4 +302,4 @@ def capture_request_media(engine_input, *, admission, splice): assert snapshots == [(processed, admission, None)] (payload,) = host.token_capture.staged assert payload["prompt_token_ids"] == processed - assert chunk["engine_data"] == {"ng_commit_coords": COORDS} + assert chunk["engine_data"] == {NG_COMMIT_COORDS_FIELD: COORDS} diff --git a/tests/unit/models/generation/test_openai_server_utils.py b/tests/unit/models/generation/test_openai_server_utils.py index 425e46442bc..7fafb1ea04d 100644 --- a/tests/unit/models/generation/test_openai_server_utils.py +++ b/tests/unit/models/generation/test_openai_server_utils.py @@ -17,7 +17,6 @@ from nemo_rl.models.generation.openai_server_utils import ( PrefixSplice, - derive_prefix_eos_count, replace_prefix_tokens, splice_prefix_tokens, ) @@ -161,37 +160,27 @@ def test_splice_prefix_tokens_without_tokenizer_honors_explicit_eos_ids( ) assert result == expected assert result.token_ids[: len(model_prefix)] == model_prefix - # A host without the messages (managed Dynamo's engine) passes the EOS count - # derived from the prefix instead of the template prefix render. + # A host without the messages (the Dynamo engine) passes no template prefix. assert ( splice_prefix_tokens( tokenizer=None, model_prefix_token_ids=model_prefix, + template_prefix_token_ids=None, template_token_ids=template, eos_token_id=eos_token_id, - template_prefix_eos_count=derive_prefix_eos_count( - model_prefix, eos_token_id=eos_token_id - ), ) == expected ) - with pytest.raises(ValueError, match="exactly one"): - splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=model_prefix, - template_prefix_token_ids=template_prefix, - template_token_ids=template, - eos_token_id=eos_token_id, - template_prefix_eos_count=1, - ) @pytest.mark.parametrize( "model_name", ["Qwen/Qwen3-0.6B", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"] ) -def test_derived_prefix_eos_count_matches_real_template_renders(model_name): - """Across tool-calling turns with reasoning, the EOS count derived from the - exact prefix equals the count in the template's render of the same history.""" +def test_splicing_without_the_template_prefix_matches_real_template_renders( + model_name, +): + """Across tool-calling turns with reasoning, splicing without the template's + render of the history (as the Dynamo engine does) matches splicing with it.""" transformers = pytest.importorskip("transformers") try: tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) @@ -242,16 +231,19 @@ def generate(text): messages = [*messages, {"role": "assistant", "content": reply}, follow_up] template_prefix = render(messages[:-1], add_generation_prompt=False) template = render(messages, add_generation_prompt=True) - assert derive_prefix_eos_count(chain, eos_token_id=eos) == sum( - token == eos for token in template_prefix - ) spliced = splice_prefix_tokens( tokenizer=tokenizer, model_prefix_token_ids=chain, template_prefix_token_ids=template_prefix, template_token_ids=template, - ).token_ids - chain = spliced + generate(next_reply) + ) + assert spliced == splice_prefix_tokens( + tokenizer=tokenizer, + model_prefix_token_ids=chain, + template_prefix_token_ids=None, + template_token_ids=template, + ) + chain = spliced.token_ids + generate(next_reply) def test_replace_prefix_tokens_without_tokenizer_reports_missing_eos_without_decoding(): diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index 4328d05736b..095a13923e4 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -2396,7 +2396,7 @@ def test_dynamo_token_capture_tells_gym_to_speak_dynamo(self, patched_factories) patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), patch.object( sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() - ) as mock_spinup, + ), patch.object(sc_setup_mod, "validate_dataset_agent_coverage"), patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), patch( @@ -2407,11 +2407,6 @@ def test_dynamo_token_capture_tells_gym_to_speak_dynamo(self, patched_factories) setup_single_controller(mc, MagicMock(pad_token_id=9)) assert mc.token_capture.generation_backend == "dynamo" - assert mock_spinup.call_args.kwargs["token_capture"]["generation_backend"] == ( - "dynamo" - ) - generation, _ = patched_factories["_build_generation"].return_value - generation.setup_token_capture.assert_called() def test_nemo_gym_coverage_failure_shuts_down_shards(self, patched_factories): mc = _make_master_config(colocated=False, backend="vllm") From a15ba5dcb3d4280d2bd658fd939566defee301f6 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:12:41 -0700 Subject: [PATCH 12/82] build: pin Gym to Gym#4817's head a7be5c5 (dynamo_worker pairing check only on dialect servers) Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index b0e681cf43f..a7be5c5f3a4 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit b0e681cf43f20882a328fc964ac08680681e2909 +Subproject commit a7be5c5f3a4d0957c9e84a8f7face90e30b7171e From ed3ee353804675bde437e8aeb90dab131f8f88e9 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:26:12 -0700 Subject: [PATCH 13/82] refactor(dynamo): serve only NeMo RL rollouts; general OpenAI path moves to #4979 After the NeMo-Gym split nothing in this PR calls the engine's general OpenAI path, so it moves to #4979 with Gym, its only client: - NativeVllmEngine serves requests carrying ROLLOUT_ANNOTATION and refuses the rest. build_sampling_params, guided decoding, media loading, prompt logprobs, queue-limit mapping and the parser/chat-template registration go, and engines register endpoint_types="completions". - dynamo_cfg.worker_args (parsers, chat template, structural tags, endpoint types) is removed and rejected at startup with the migration. - DYN_ENABLE_EXPERIMENTAL_PARSERS_V2 goes: only Dynamo's chat path reads it, and /v1/completions returns engine_data via nvext.extra_fields. - The engine keeps skip_tokenizer_init as native chooses it, so rollouts skip detokenization as they do natively. #4979 restores all of this by reverting this commit. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 28 +- docs/guides/dynamo-generation.md | 33 +- ...grpo_math_1B_dynamo_single_controller.yaml | 13 - nemo_rl/models/generation/__init__.py | 14 +- nemo_rl/models/generation/dynamo/__init__.py | 2 - nemo_rl/models/generation/dynamo/config.py | 134 +------ .../generation/dynamo/managed_runtime.py | 6 +- .../models/generation/dynamo/native_engine.py | 370 +----------------- nemo_rl/utils/outdated_config_checks.py | 10 +- .../models/generation/test_dynamo_config.py | 81 +--- .../generation/test_dynamo_native_engine.py | 308 +-------------- .../unit/utils/test_outdated_config_checks.py | 3 +- 12 files changed, 50 insertions(+), 952 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 9c07ccda5c7..2f1acd26d4e 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -20,23 +20,12 @@ which is the loop that owns the `AsyncLLM`. vLLM. The adapter, `nemo_rl/models/generation/dynamo/native_engine.py`, implements its `LLMEngine` over the worker's `AsyncLLM`: -- `generate` maps Dynamo's token request onto vLLM `SamplingParams` the way - vLLM's OpenAI server does, and streams token deltas back. Rollouts that NeMo RL - itself sends carry an annotation. For those, the engine uses the same code as - native `generate_async`: - - the worker's `_rollout_max_new_tokens` and `_rollout_sampling_params` build - the request; - - `request_pipeline.rollout_result` builds the result, routed experts - included. - - The result travels in `engine_data`, and the driver assembles the same - output with `request_pipeline.rollout_output`. - - A request with media carries the frontend's tokens, with one unexpanded - placeholder per item, and the media URLs. The adapter loads the media with - vLLM's media connector. It then builds the processed prompt with the engine's - renderer (`process_for_engine_async`), the same step that follows rendering - on the native endpoint. +- `generate` serves only the rollouts NeMo RL itself sends, which carry an + annotation, and refuses other requests. The worker's own `_serve_rollout` runs + each rollout exactly as native `generate_async` does, and + `request_pipeline.rollout_result` builds its result, routed experts included. + The result travels in `engine_data`, and the driver assembles the same output + with `request_pipeline.rollout_output`. - `abort` cancels the vLLM request on client disconnect. - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It @@ -86,10 +75,9 @@ imports `dynamo.common.backend`, `dynamo.llm.exceptions` and `dynamo._core`. What a Dynamo upgrade has to re-check: - the `dynamo.common.backend` contract (`LLMEngine`, `Worker`, `WorkerConfig`, - publisher and logprob helpers); + publisher); - the request fields of Dynamo's preprocessed request; -- the frontend's `nvext` extensions and annotation forwarding; -- the parser names the native settings map to (`dynamo/config.py`). +- the frontend's `nvext` extensions and annotation forwarding. The one vLLM↔Dynamo wire coupling is vLLM's KV-event msgpack, which Dynamo's router decodes. A change there degrades routing, not serving. The managed diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index b780dcda242..b1a543a52bf 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -52,8 +52,6 @@ shows them all: |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | | `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies | -| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [Chat settings](#chat-settings) | -| `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | | `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | @@ -75,31 +73,7 @@ native path: token IDs, log probabilities, truncation, routed experts with their `r3_routed_experts_*` counts, and `gen_leader_worker_idx`. Both paths build it with `nemo_rl/models/generation/vllm/request_pipeline.py`. -OpenAI clients of the frontend get what vLLM's own OpenAI server applies: -- the model's generation-config sampling defaults; -- `n > 1`; -- guided decoding (`guided_*`, `response_format`); -- `bad_words_token_ids` and `allowed_token_ids`; -- prompt log probabilities (`nvext.extra_fields: ["prompt_logprobs"]`); -- vLLM's queue limits, rejected with HTTP 503; -- a 400 naming the "maximum context length" for a prompt that leaves no room - for output. The engine, not the frontend, checks the context, so it checks - the prompt it actually runs. - -## Chat settings - -The native server's chat settings carry over to the frontend: - -| Native setting (`vllm_cfg`) | Under Dynamo | -|---|---| -| `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | -| `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | -| `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | -| `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | - -Where the behavior differs from the native endpoint: when a request sets -`chat_template_kwargs.force_nonempty_content`, Dynamo reports a reasoning-only -turn as content. +The engines serve only these rollouts; the frontend refuses other requests. ## Not supported @@ -114,8 +88,6 @@ Startup rejects these with `dynamo: true`: and a restarted shard does not register with it yet. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data parallelism inside vLLM, which the async engine Dynamo serves does not. -- Parser plugins with no Dynamo parser, and the chat settings above that the - frontend has no counterpart for. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. @@ -168,5 +140,6 @@ rejects it with the migration: - `control_timeout_s`: refit and pause use the vLLM backend's controls; - `metrics_include_prefixes` / `metrics_exclude_prefixes`: use `vllm_cfg.enable_vllm_metrics_logger`; - - `worker_args.extra_cli_args`: engines are configured through `vllm_cfg` and + - `worker_args`: the engines serve only NeMo RL's rollouts, which need no chat + parsers or templates; engines are configured through `vllm_cfg` and `vllm_kwargs`. diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 1b62d39ba8d..1ac1a92abaf 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -18,19 +18,6 @@ policy: # NRL_VLLM_ASYNC_TIMEOUT_SECONDS). A timeout fails the attempt as a native # timeout does, and the rollout retry policy applies. request_timeout_s: 900 - worker_args: - # Dynamo's built-in parser names. null follows the native server's - # vllm_cfg.http_server_serving_chat_kwargs (tool_parser, reasoning_parser). - tool_call_parser: null - reasoning_parser: null - exclude_tools_when_tool_choice_none: false - enable_structural_tag: false - structural_tag_scope: auto - structural_tag_schema: auto - # A chat template that replaces the model's own. null follows - # vllm_cfg.http_server_serving_chat_kwargs.chat_template. - custom_jinja_template: null - endpoint_types: [chat, completions] frontend_args: tokenizer: default tokenizer_cache: false diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index c8547bbce41..7d17e9e8942 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -17,12 +17,7 @@ from transformers import PreTrainedTokenizerBase -from nemo_rl.models.generation.dynamo.config import ( - check_dynamo_serving_chat_kwargs, - dynamo_cfg, - dynamo_enabled, - dynamo_parsers, -) +from nemo_rl.models.generation.dynamo.config import dynamo_cfg, dynamo_enabled from nemo_rl.models.generation.interfaces import GenerationConfig, GenerationInterface from nemo_rl.models.generation.trtllm.config import ( TrtllmConfig, @@ -186,11 +181,8 @@ def configure_generation_config( # vision parameters; freezing alone does not exclude them from refit. config["vllm_cfg"].setdefault("load_format", "auto" if is_eval else "dummy") if dynamo_enabled(config): - # Validate before any Ray allocation; dynamo_parsers also rejects - # parser plugins that Dynamo's Rust frontend cannot run. + # Validate before any Ray allocation. dynamo_cfg(config) - dynamo_parsers(config) - check_dynamo_serving_chat_kwargs(config) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( @@ -274,8 +266,6 @@ def configure_generation_config( is_eval or config["stop_strings"] is not None or config["vllm_cfg"].get("expose_http_server", None) - # Dynamo serves OpenAI clients as the exposed server does. - or dynamo_enabled(config) ): config["vllm_cfg"]["skip_tokenizer_init"] = False else: diff --git a/nemo_rl/models/generation/dynamo/__init__.py b/nemo_rl/models/generation/dynamo/__init__.py index 40376929afa..427c1ede100 100644 --- a/nemo_rl/models/generation/dynamo/__init__.py +++ b/nemo_rl/models/generation/dynamo/__init__.py @@ -14,7 +14,6 @@ from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, DynamoFrontendArgs, - DynamoWorkerArgs, dynamo_cfg, dynamo_enabled, ) @@ -22,7 +21,6 @@ __all__ = [ "DynamoCfg", "DynamoFrontendArgs", - "DynamoWorkerArgs", "dynamo_cfg", "dynamo_enabled", ] diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index b6ae598b1a1..af592bc0fd5 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -21,33 +21,7 @@ from typing import Any, Literal, Mapping -from pydantic import BaseModel, Field, PositiveFloat, PositiveInt, model_validator - - -class DynamoWorkerArgs(BaseModel, extra="forbid"): - """How each vLLM engine registers with Dynamo (``WorkerConfig`` fields).""" - - # Dynamo's built-in parser names (dynamo-parsers). None follows the native - # server's vllm_cfg.http_server_serving_chat_kwargs (see dynamo_parsers). - tool_call_parser: str | None = None - reasoning_parser: str | None = None - # False, as vLLM's OpenAIServingChat (the native server) defaults it. - exclude_tools_when_tool_choice_none: bool = False - enable_structural_tag: bool = False - structural_tag_scope: Literal["auto", "always"] = "auto" - structural_tag_schema: Literal["auto", "strict"] = "auto" - # A Jinja chat template file that replaces the model's own. None follows - # vllm_cfg.http_server_serving_chat_kwargs.chat_template. - custom_jinja_template: str | None = None - endpoint_types: list[Literal["chat", "completions"]] = ["chat", "completions"] - - @model_validator(mode="after") - def _validate_endpoint_types(self) -> "DynamoWorkerArgs": - if not self.endpoint_types: - raise ValueError("endpoint_types must contain at least one endpoint") - if len(self.endpoint_types) != len(set(self.endpoint_types)): - raise ValueError("endpoint_types must not contain duplicates") - return self +from pydantic import BaseModel, Field, PositiveFloat, PositiveInt class DynamoFrontendArgs(BaseModel, extra="forbid"): @@ -80,115 +54,9 @@ class DynamoCfg(BaseModel, extra="forbid"): # does (GenerationUnavailable), and the rollout retry policy re-dispatches it # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). request_timeout_s: PositiveFloat = 900.0 - worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) -# vLLM parser names whose Dynamo built-in parser goes by another name. nano_v3 is -# NeMo RL's vLLM plugin: DeepSeek-R1 reasoning parsing, with the whole output as -# content when chat_template_kwargs.enable_thinking is false, as Dynamo's -# nemotron_v3 parser does. -_DYNAMO_REASONING_PARSER_NAMES = {"nano_v3": "nemotron_v3"} -_DYNAMO_TOOL_PARSER_NAMES = {"openai": "harmony"} - - -def _dynamo_parser( - configured: str | None, - native: str | None, - plugin: str | None, - renames: Mapping[str, str], - *, - kind: str, - worker_arg: str, -) -> str | None: - if configured is not None or native is None: - return configured - if native in renames: - return renames[native] - if plugin: - raise ValueError( - f"policy.generation.vllm_cfg.{kind}_parser_plugin loads a Python parser " - "into vLLM's OpenAI server, which Dynamo's Rust frontend cannot run. Set " - f"policy.generation.dynamo_cfg.worker_args.{worker_arg} to the Dynamo " - f"built-in parser for {native!r}." - ) - return native - - -def dynamo_parsers( - generation_config: Mapping[str, Any], -) -> tuple[str | None, str | None]: - """The ``(tool_call_parser, reasoning_parser)`` Dynamo's frontend applies. - - ``dynamo_cfg.worker_args`` names them explicitly. Otherwise they follow the - native server's ``vllm_cfg.http_server_serving_chat_kwargs``, renamed where - Dynamo calls the same parser differently. A parser plugin without such a - rename has no Dynamo equivalent and is rejected. - """ - worker_args = dynamo_cfg(generation_config).worker_args - vllm_cfg = generation_config["vllm_cfg"] - serving = vllm_cfg.get("http_server_serving_chat_kwargs") or {} - tool_call_parser = _dynamo_parser( - worker_args.tool_call_parser, - serving.get("tool_parser"), - vllm_cfg.get("tool_parser_plugin"), - _DYNAMO_TOOL_PARSER_NAMES, - kind="tool", - worker_arg="tool_call_parser", - ) - reasoning_parser = _dynamo_parser( - worker_args.reasoning_parser, - serving.get("reasoning_parser"), - vllm_cfg.get("reasoning_parser_plugin"), - _DYNAMO_REASONING_PARSER_NAMES, - kind="reasoning", - worker_arg="reasoning_parser", - ) - return tool_call_parser, reasoning_parser - - -def check_dynamo_serving_chat_kwargs(generation_config: Mapping[str, Any]) -> None: - """Reject native chat-server settings Dynamo's frontend has no counterpart for. - - The frontend renders templates and parses tool calls itself: it always - parses ``tool_choice="auto"`` calls when a tool parser is set, and has no - content-format setting. The native defaults (``enable_auto_tools: true``, - ``chat_template_content_format: auto``) match it; other values would - silently change what the policy sees. - """ - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - content_format = serving.get("chat_template_content_format") - if content_format not in (None, "auto"): - raise ValueError( - "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." - f"chat_template_content_format={content_format!r} is not supported " - "with policy.generation.dynamo=true: Dynamo's frontend renders chat " - "templates itself and has no content-format setting. Remove it or " - "set it to 'auto'." - ) - if serving.get("enable_auto_tools") is False: - raise ValueError( - "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." - "enable_auto_tools=false is not supported with " - "policy.generation.dynamo=true: Dynamo's frontend always parses " - "tool_choice='auto' calls when a tool parser is set. Remove it, or " - "remove the tool parser." - ) - - -def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: - """The chat template the frontend renders with: a file path or literal Jinja. - - ``dynamo_cfg.worker_args.custom_jinja_template``, else the native server's - ``http_server_serving_chat_kwargs.chat_template``; None uses the model's own. - """ - configured = dynamo_cfg(generation_config).worker_args.custom_jinja_template - if configured is not None: - return configured - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - return serving.get("chat_template") - - def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" return bool(generation_config.get("dynamo")) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index ad9ad9b7659..e53c0aef8cc 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -120,7 +120,7 @@ def engine_env(self) -> dict[str, str]: if not self._started: raise RuntimeError("Managed Dynamo runtime has not been started") # An OS-assigned system port makes each engine serve its own model-card - # files (a custom chat template included) to a frontend on another node. + # files to a frontend on another node. return {**self._manager_env, "DYN_SYSTEM_PORT": "0"} def start(self) -> None: @@ -166,10 +166,6 @@ def allocate_port(*, low: int, high: int) -> int: "NATS_SERVER": f"nats://{self._host}:{self._nats_port}", "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", - # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. - # Its v2 parser path preserves engine_data, which carries every NeMo - # RL rollout's result. - "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", # A canary request would stall behind a refit pause; engine liveness diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index f6c877b49c1..e95f535bc25 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -23,18 +23,14 @@ from __future__ import annotations import asyncio -import functools -import hashlib import json import logging import os -import tempfile -import time from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional import torch -from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names +from dynamo._core import Context from dynamo.common.backend import ( EngineConfig, GenerateChunk, @@ -44,33 +40,13 @@ Worker, WorkerConfig, ) -from dynamo.common.backend.logprobs import ( - extract_from_completion_output, - extract_prompt_logprobs_from_completion_output, - parse_logprob_options, -) from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource -from dynamo.llm.exceptions import HttpError, InvalidArgument -from vllm import envs as vllm_envs -from vllm.entrypoints.chat_utils import load_chat_template -from vllm.exceptions import GracefulHTTPError, VLLMClientError +from dynamo.llm.exceptions import InvalidArgument from vllm.inputs import TokensPrompt -from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY -from vllm.sampling_params import ( - RequestOutputKind, - SamplingParams, - StructuredOutputsParams, -) -from vllm.transformers_utils.processor import get_video_processor_cls_name from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - dynamo_cfg, - dynamo_chat_template, - dynamo_parsers, -) +from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.dynamo.managed_runtime import ( DYNAMO_COMPONENT, DYNAMO_ENDPOINT, @@ -85,29 +61,6 @@ LOGGER = logging.getLogger(__name__) -# The vLLM modality each key of Dynamo's multi_modal_data holds. -_DYNAMO_MEDIA_MODALITIES = { - "image_url": "image", - "video_url": "video", - "audio_url": "audio", -} - -# Filled from the model's generation config when a request leaves them unset, -# as vLLM's OpenAI server fills them. -_GENERATION_CONFIG_KEYS = ( - "repetition_penalty", - "temperature", - "top_k", - "top_p", - "min_p", -) -_SAMPLING_KEYS = ( - *_GENERATION_CONFIG_KEYS, - "seed", - "presence_penalty", - "frequency_penalty", -) - def dynamo_kv_events_config( cfg: DynamoCfg, *, prefix_caching: bool @@ -145,111 +98,6 @@ def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: return None if rollout is None else json.loads(rollout) -def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: - """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. - - Dynamo sets at most one constraint. The decoding backend is the engine's - own (``structured_outputs_config``), as for the native server. - """ - structural_tag = guided_decoding.get("structural_tag") - if isinstance(structural_tag, dict): - structural_tag = json.dumps(structural_tag) - return StructuredOutputsParams( - json=guided_decoding.get("json"), - regex=guided_decoding.get("regex"), - choice=guided_decoding.get("choice"), - grammar=guided_decoding.get("grammar"), - whitespace_pattern=guided_decoding.get("whitespace_pattern"), - structural_tag=structural_tag, - ) - - -def build_sampling_params( - request: GenerateRequest, - defaults: dict[str, Any], - *, - output_room: int, - overrides: Optional[dict[str, Any]] = None, - output_kind: RequestOutputKind = RequestOutputKind.DELTA, -) -> SamplingParams: - """Map an OpenAI-shaped Dynamo request onto vLLM SamplingParams. - - As vLLM's OpenAI server builds them: the model's generation-config - ``defaults`` for what the request leaves unset, and ``max_tokens`` capped - by the context. Stop strings are applied by Dynamo's Rust detokenizer, so - only token-level stops reach vLLM unless ``overrides`` sets them. - - Args: - request: Dynamo's preprocessed request. - defaults: ``ModelConfig.get_diff_sampling_param()``. - output_room: The context left after the prompt the engine runs, which - media expansion or a prefix splice can make longer than the - request's ``token_ids``. - overrides: SamplingParams fields that replace the request's. - output_kind: How vLLM reports outputs. - """ - sampling = request.get("sampling_options") or {} - passthrough = (request.get("extra_args") or {}).get("sampling_options") or {} - stop = request.get("stop_conditions") or {} - n = sampling.get("n") or 1 - if sampling.get("best_of") not in (None, n): - raise InvalidArgument("best_of is not supported by vLLM's V1 engine") - - kwargs: dict[str, Any] = { - key: defaults[key] for key in _GENERATION_CONFIG_KEYS if key in defaults - } - kwargs.update( - {key: sampling[key] for key in _SAMPLING_KEYS if sampling.get(key) is not None} - ) - kwargs["n"] = n - # As vLLM's get_max_tokens: the generation config's max_tokens applies only - # when the request leaves max_tokens unset. - requested_max_tokens = stop.get("max_tokens") - if requested_max_tokens is None: - requested_max_tokens = defaults.get("max_tokens") - kwargs["max_tokens"] = min( - limit - for limit in (max(1, output_room), requested_max_tokens) - if limit is not None - ) - if stop.get("min_tokens") is not None: - kwargs["min_tokens"] = stop["min_tokens"] - if stop.get("ignore_eos") is not None: - kwargs["ignore_eos"] = stop["ignore_eos"] - stop_ids = set(stop.get("stop_token_ids") or ()) | set( - stop.get("stop_token_ids_hidden") or () - ) - if stop_ids: - kwargs["stop_token_ids"] = sorted(stop_ids) - if sampling.get("guided_decoding"): - kwargs["structured_outputs"] = structured_outputs(sampling["guided_decoding"]) - for key in ("allowed_token_ids", "logprob_token_ids"): - if passthrough.get(key) is not None: - kwargs[key] = passthrough[key] - - logprobs, prompt_logprobs = parse_logprob_options( - request.get("output_options") or {} - ) - # Explicit logprob_token_ids replace top-k selection; vLLM rejects both. - if logprobs is not None and "logprob_token_ids" not in kwargs: - kwargs["logprobs"] = logprobs - if prompt_logprobs is not None: - kwargs["prompt_logprobs"] = prompt_logprobs - kwargs["detokenize"] = False - kwargs.update(overrides or {}) - params = SamplingParams(**kwargs, output_kind=output_kind) - if passthrough.get("bad_words_token_ids") is not None: - # vLLM only tokenizes bad_words strings into this field; a token-in - # client sends the token IDs themselves. - if not hasattr(params, "_bad_words_token_ids"): - raise RuntimeError( - "vLLM SamplingParams has no _bad_words_token_ids; update the " - "bad_words_token_ids passthrough for this vLLM" - ) - params._bad_words_token_ids = passthrough["bad_words_token_ids"] - return params - - class SnapshotStatLogger(StatLoggerBase): """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" @@ -292,7 +140,6 @@ def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int self._leader_worker_idx = leader_worker_idx self._llm = host.llm self._max_model_len = self._llm.vllm_config.model_config.max_model_len - self._defaults = self._llm.model_config.get_diff_sampling_param() @classmethod async def from_args(cls, argv=None): @@ -335,84 +182,14 @@ async def start(self, worker_id: int) -> EngineConfig: async def generate( self, request: GenerateRequest, context: Context ) -> AsyncGenerator[GenerateChunk, None]: - try: - async for chunk in self._generate(request, context): - yield chunk - except VLLMClientError as error: - # vLLM's request errors are not ValueErrors; the native server - # answers them with a 4xx that carries the message. - raise InvalidArgument(str(error)) from error - - async def _generate( - self, request: GenerateRequest, context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - if request.get("prompt_embeds"): + rollout = _rollout_request(request) + if rollout is None: raise InvalidArgument( - "Embedding prompts are not served through Dynamo by NeMo RL's " - "vLLM engines." + "NeMo RL's vLLM engines serve only NeMo RL rollouts through Dynamo " + f"(requests annotated {ROLLOUT_ANNOTATION!r})." ) - rollout = _rollout_request(request) - if rollout is not None: - async for chunk in self._generate_rollout(request, rollout, context): - yield chunk - return - engine_input = await self._engine_input(request) - prompt_len = len(engine_input["prompt_token_ids"]) - self._check_prompt_fits(prompt_len) - sampling_params = build_sampling_params( - request, self._defaults, output_room=self._max_model_len - prompt_len - ) - try: - # vLLM's queue limits (max_num_queued_reqs / max_num_queued_tokens), - # checked before every request as the native server checks them. - self._llm.check_admission(sampling_params.n, context.id()) - except GracefulHTTPError as error: - raise HttpError(int(error.http_status), error.message) from error - logprobs_requested, prompt_logprobs_requested = parse_logprob_options( - request.get("output_options") or {} - ) - tokenizer = self._llm.tokenizer - completion_tokens: dict[int, int] = {} - prompt_output = None - async for output in self._llm.generate( - engine_input, sampling_params, context.id() - ): - # Under DELTA, vLLM sends every prompt logprob once, at the end of - # prefill; later outputs carry an empty list. - if getattr(output, "prompt_logprobs", None): - prompt_output = output - for completion in output.outputs: - token_ids = list(completion.token_ids) - completion_tokens[completion.index] = completion_tokens.get( - completion.index, 0 - ) + len(token_ids) - if not token_ids and completion.finish_reason is None: - continue - chunk: GenerateChunk = { - "token_ids": token_ids, - "index": completion.index, - } - log_probs, top_logprobs = extract_from_completion_output( - completion, 0, tokenizer=tokenizer, include_bytes=True - ) - if log_probs is not None: - chunk["log_probs"] = log_probs - if logprobs_requested: - chunk["top_logprobs"] = top_logprobs - if completion.finish_reason is not None: - chunk["finish_reason"] = completion.finish_reason - chunk["completion_usage"] = _usage( - output, sum(completion_tokens.values()) - ) - if prompt_logprobs_requested is not None and prompt_output: - prompt_logprobs = ( - extract_prompt_logprobs_from_completion_output( - prompt_output, tokenizer=tokenizer - ) - ) - if prompt_logprobs is not None: - chunk["engine_data"] = {"prompt_logprobs": prompt_logprobs} - yield chunk + async for chunk in self._generate_rollout(request, rollout, context): + yield chunk async def _generate_rollout( self, request: GenerateRequest, rollout: dict[str, Any], context: Context @@ -448,86 +225,6 @@ async def _generate_rollout( chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) yield chunk - async def _load_media( - self, request: GenerateRequest - ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: - """The request's media and their UUIDs by modality, as vLLM's chat parser loads them. - - Dynamo passes each media URL through (data: URIs included); the engine - fetches and decodes it with vLLM's own media connector and settings. - """ - media = request.get("multi_modal_data") - if not media: - return None - model_config = self._llm.model_config - mm_config = model_config.multimodal_config - connector = MEDIA_CONNECTOR_REGISTRY.load( - vllm_envs.VLLM_MEDIA_CONNECTOR, - media_io_kwargs=request.get("media_io_kwargs") - or (mm_config.media_io_kwargs if mm_config else None), - allowed_local_media_path=model_config.allowed_local_media_path, - allowed_media_domains=model_config.allowed_media_domains, - ) - fetch = { - "image": connector.fetch_image_async, - "audio": connector.fetch_audio_async, - } - if "video_url" in media: - fetch["video"] = functools.partial( - connector.fetch_video_async, - video_processor=get_video_processor_cls_name(model_config), - ) - uuids = request.get("multi_modal_uuids") or {} - mm_data: dict[str, list[Any]] = {} - mm_uuids: dict[str, list[Any]] = {} - for key, items in media.items(): - modality = _DYNAMO_MEDIA_MODALITIES.get(key) - if modality is None: - raise InvalidArgument(f"Unsupported media type {key!r}") - if any("Url" not in item for item in items): - raise InvalidArgument( - f"{key} items must carry their URL: NeMo RL's vLLM engines load " - "media themselves" - ) - try: - mm_data[modality] = list( - await asyncio.gather( - *(fetch[modality](item["Url"]) for item in items) - ) - ) - except ValueError as error: - raise InvalidArgument(f"Cannot load {key}: {error}") from error - mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) - return mm_data, mm_uuids - - async def _engine_input(self, request: GenerateRequest) -> dict[str, Any]: - """The processed vLLM prompt for the request, as the native chat endpoint builds it. - - The frontend's tokens carry one unexpanded placeholder per media item; - vLLM's processor expands them and processes the media, exactly as after - the native endpoint renders and tokenizes the messages. - """ - prompt = TokensPrompt(prompt_token_ids=list(request["token_ids"])) - media = await self._load_media(request) - if media is not None: - prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media - if request.get("mm_processor_kwargs"): - prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] - return await self._llm.renderer.process_for_engine_async(prompt, time.time()) - - def _check_prompt_fits(self, prompt_len: int) -> None: - """Refuse a prompt that leaves no room for output, as the native endpoint does. - - OpenAI clients read "context length" in a 400 as a length refusal rather - than an engine failure. - """ - if prompt_len >= self._max_model_len: - raise InvalidArgument( - f"Prompt length ({prompt_len}) fills or exceeds this model's " - f"maximum context length ({self._max_model_len}). No room for " - "output tokens." - ) - async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) @@ -558,37 +255,6 @@ def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: } -def _check_parser_name(name: Optional[str], known: list[str], field: str) -> None: - if name is not None and name not in known: - raise ValueError( - f"Dynamo has no {field.replace('_', ' ')} named {name!r}; set " - f"policy.generation.dynamo_cfg.worker_args.{field} to one of " - f"{sorted(known)}." - ) - - -def _chat_template_file(chat_template: Optional[str]) -> Optional[str]: - """A file holding ``chat_template``, which Dynamo reads by path. - - Resolved as the native server resolves it (vLLM's ``load_chat_template``): - a file, a vLLM built-in template name, or literal Jinja. Named by content, - so every engine on a node, and every restart, shares one file. - """ - if chat_template is None: - return None - content = load_chat_template(chat_template) - digest = hashlib.sha256(content.encode()).hexdigest()[:16] - path = os.path.join(tempfile.gettempdir(), f"nemo_rl_chat_template_{digest}.jinja") - if not os.path.exists(path): - # Written aside and renamed, so a concurrent reader never sees a part. - with tempfile.NamedTemporaryFile( - "w", dir=os.path.dirname(path), suffix=".jinja", delete=False - ) as template_file: - template_file.write(content) - os.replace(template_file.name, path) - return path - - def start_dynamo_endpoint( host: "VllmAsyncGenerationWorkerImpl", *, @@ -608,31 +274,17 @@ def start_dynamo_endpoint( publishes = kv_events is not None and kv_events.enable_kv_cache_events # The Dynamo functional overlays check the router applied these. print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) - worker_args = dynamo_cfg(host.cfg).worker_args - tool_call_parser, reasoning_parser = dynamo_parsers(host.cfg) - _check_parser_name(tool_call_parser, get_tool_parser_names(), "tool_call_parser") - _check_parser_name( - reasoning_parser, get_reasoning_parser_names(), "reasoning_parser" - ) config = WorkerConfig( namespace=namespace, component=DYNAMO_COMPONENT, endpoint=DYNAMO_ENDPOINT, model_name=host.model_name, served_model_name=host.model_name, - endpoint_types=",".join(worker_args.endpoint_types), + # NeMo RL's rollouts go to /v1/completions only. + endpoint_types="completions", discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], request_plane=engine_env["DYN_REQUEST_PLANE"], event_plane=engine_env["DYN_EVENT_PLANE"], - custom_jinja_template=_chat_template_file(dynamo_chat_template(host.cfg)), - tool_call_parser=tool_call_parser, - reasoning_parser=reasoning_parser, - exclude_tools_when_tool_choice_none=( - worker_args.exclude_tools_when_tool_choice_none - ), - structural_tag_mode="on" if worker_args.enable_structural_tag else "off", - structural_tag_scope=worker_args.structural_tag_scope, - structural_tag_schema=worker_args.structural_tag_schema, ) return asyncio.get_running_loop().create_task( Worker(NativeVllmEngine(host, leader_worker_idx), config).run() diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 5e511555b3e..8591d1d1a17 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -208,6 +208,10 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: "control_timeout_s": "refit and pause use the vLLM backend's own controls", "metrics_include_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", "metrics_exclude_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", + "worker_args": ( + "Dynamo serves only NeMo RL's own rollouts, which need no chat parsers or " + "templates; engines are configured through vllm_cfg and vllm_kwargs" + ), } @@ -242,12 +246,6 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: problems.append( f"policy.generation.dynamo_cfg.{key} was removed: {replacement}." ) - worker_args = dynamo_cfg.get("worker_args") - if isinstance(worker_args, dict) and "extra_cli_args" in worker_args: - problems.append( - "policy.generation.dynamo_cfg.worker_args.extra_cli_args was " - "removed: engines are configured through vllm_cfg and vllm_kwargs." - ) if problems: raise ValueError(" ".join(problems)) diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 289f8b55bf7..a14f09e95f9 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -20,12 +20,7 @@ redact_argv, redact_environment, ) -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - dynamo_cfg, - dynamo_chat_template, - dynamo_parsers, -) +from nemo_rl.models.generation.dynamo.config import DynamoCfg, dynamo_cfg class _Tokenizer: @@ -53,10 +48,10 @@ def _vllm_config(**overrides) -> dict: def test_dynamo_true_alone_is_a_complete_config() -> None: config = configure_generation_config(_vllm_config(dynamo=True), _Tokenizer()) - # Dynamo serves the AsyncLLM, so the switch selects the async engine, and it - # serves OpenAI clients, which need the engine's tokenizer. + # Dynamo serves the AsyncLLM, so the switch selects the async engine. + # Rollouts read token IDs only, so the engine skips its tokenizer as natively. assert config["vllm_cfg"]["async_engine"] is True - assert config["vllm_cfg"]["skip_tokenizer_init"] is False + assert config["vllm_cfg"]["skip_tokenizer_init"] is True assert dynamo_cfg(config) == DynamoCfg() assert dynamo_cfg(config).frontend_args.router_mode == "kv" @@ -103,10 +98,8 @@ def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( def test_dynamo_cfg_rejects_unknown_keys() -> None: - with pytest.raises(ValidationError, match="extra_cli_args"): - dynamo_cfg({"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}) - with pytest.raises(ValidationError, match="duplicates"): - dynamo_cfg({"dynamo_cfg": {"worker_args": {"endpoint_types": ["chat"] * 2}}}) + with pytest.raises(ValidationError, match="worker_args"): + dynamo_cfg({"dynamo_cfg": {"worker_args": {}}}) def test_frontend_argv_is_runtime_owned() -> None: @@ -134,65 +127,3 @@ def test_redaction_hides_credentials() -> None: "HF_TOKEN": "", "NCCL_DEBUG": "INFO", } - - -def _serving(**serving_chat_kwargs) -> dict: - return _vllm_config( - dynamo=True, - vllm_cfg={ - "async_engine": True, - "max_model_len": 512, - "tensor_parallel_size": 1, - "expert_parallel_size": 1, - "http_server_serving_chat_kwargs": serving_chat_kwargs, - }, - ) - - -def test_parsers_follow_the_native_server_under_dynamos_names() -> None: - assert dynamo_parsers(_serving(tool_parser="hermes", reasoning_parser="qwen3")) == ( - "hermes", - "qwen3", - ) - # NeMo RL's nano_v3 vLLM plugin is Dynamo's nemotron_v3 parser. - nano_v3 = _serving(reasoning_parser="nano_v3") - nano_v3["vllm_cfg"]["reasoning_parser_plugin"] = "nano_v3_reasoning_parser.py" - assert dynamo_parsers(nano_v3) == (None, "nemotron_v3") - assert dynamo_parsers(_serving(tool_parser="openai")) == ("harmony", None) - # dynamo_cfg names them explicitly. - explicit = _serving(tool_parser="hermes") - explicit["dynamo_cfg"] = {"worker_args": {"tool_call_parser": "qwen3_coder"}} - assert dynamo_parsers(explicit) == ("qwen3_coder", None) - - -def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> None: - config = _serving(tool_parser="my_parser") - config["vllm_cfg"]["tool_parser_plugin"] = "my_parser.py" - - with pytest.raises(ValueError, match="worker_args.tool_call_parser"): - configure_generation_config(config, _Tokenizer()) - - -def test_chat_template_follows_the_native_server() -> None: - config = _serving(chat_template="templates/chat.jinja") - assert dynamo_chat_template(config) == "templates/chat.jinja" - assert dynamo_chat_template(_serving()) is None - - -@pytest.mark.parametrize( - ("serving_chat_kwargs", "error"), - [ - ({"chat_template_content_format": "string"}, "no content-format setting"), - ({"enable_auto_tools": False}, "always parses"), - ], -) -def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( - serving_chat_kwargs, error -) -> None: - with pytest.raises(ValueError, match=error): - configure_generation_config(_serving(**serving_chat_kwargs), _Tokenizer()) - # The native defaults are what the frontend does anyway. - configure_generation_config( - _serving(chat_template_content_format="auto", enable_auto_tools=True), - _Tokenizer(), - ) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index eca97ef67a7..ef2fbd01d83 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -18,27 +18,20 @@ """ import asyncio -import base64 -import io import json -from http import HTTPStatus from types import SimpleNamespace import numpy as np import pytest import torch -from PIL import Image pytestmark = pytest.mark.vllm pytest.importorskip("dynamo._core") vllm = pytest.importorskip("vllm") -from dynamo.llm.exceptions import HttpError, InvalidArgument # noqa: E402 -from vllm.exceptions import QueueOverflowError # noqa: E402 -from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 +from dynamo.llm.exceptions import InvalidArgument # noqa: E402 from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 -from nemo_rl.models.generation.dynamo.config import DynamoWorkerArgs # noqa: E402 from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) @@ -56,8 +49,6 @@ EOS = 2 -# The fake processor expands each image placeholder to IMAGE_TOKENS image tokens. -IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS = 90, 91, 3 class _Context: @@ -99,57 +90,18 @@ def _output(prompt, *completions, prompt_routed_experts=None, prompt_logprobs=No ) -class _FakeRenderer: - """vLLM's processing step: expands each image placeholder, as a VLM processor does.""" - - def __init__(self) -> None: - self.prompts = [] - - async def process_for_engine_async( - self, prompt, arrival_time, *, skip_mm_cache=False - ): - self.prompts.append((prompt, skip_mm_cache)) - if "multi_modal_data" not in prompt: - return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} - tokens, placeholders = [], [] - for token in prompt["prompt_token_ids"]: - if token == IMAGE_PLACEHOLDER: - placeholders.append(PlaceholderRange(len(tokens), IMAGE_TOKENS)) - tokens.extend([IMAGE_TOKEN] * IMAGE_TOKENS) - else: - tokens.append(token) - return { - "type": "multimodal", - "prompt_token_ids": tokens, - "mm_kwargs": {"image": [object() for _ in placeholders]}, - "mm_placeholders": {"image": placeholders}, - } - - class _FakeLLM: """Yields scripted RequestOutputs and records what it was asked to generate.""" - def __init__(self, outputs=(), *, defaults=None, admission_error=None): + def __init__(self, outputs=()): self.outputs = list(outputs) self.calls = [] - self.admission_error = admission_error self.tokenizer = None - self.renderer = _FakeRenderer() - self.model_config = SimpleNamespace( - get_diff_sampling_param=lambda: dict(defaults or {}), - multimodal_config=None, - allowed_local_media_path="", - allowed_media_domains=None, - ) self.vllm_config = SimpleNamespace( model_config=SimpleNamespace(max_model_len=MAX_MODEL_LEN), kv_events_config=None, ) - def check_admission(self, n, request_id): - if self.admission_error is not None: - raise self.admission_error - async def generate(self, prompt, sampling_params, request_id): self.calls.append((prompt, sampling_params, request_id)) for output in self.outputs: @@ -190,10 +142,6 @@ def _rollout_request(prompt: list[int], **rollout) -> dict: return {"token_ids": prompt, "annotations": [ROLLOUT_ANNOTATION + json.dumps(spec)]} -def _openai_request(**fields) -> dict: - return {"token_ids": [1, 2, 3, 4], **fields} - - def test_rollouts_are_served_like_generate_async() -> None: final = _output([1, 2, 3], _completion([10, 11, 12], "stop")) llm = _FakeLLM([final]) @@ -270,119 +218,6 @@ def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: assert rollout["token_ids"] == [] and rollout["truncated"] is False -def test_openai_requests_map_onto_sampling_params_as_vllms_server_does() -> None: - params = native_engine.build_sampling_params( - _openai_request( - sampling_options={"top_k": 5, "seed": 7, "n": 2}, - stop_conditions={ - "max_tokens": 100, - "stop_token_ids": [9], - "stop_token_ids_hidden": [2], - }, - output_options={"logprobs": 2}, - extra_args={ - "sampling_options": { - "allowed_token_ids": [5, 6], - "bad_words_token_ids": [[7]], - } - }, - ), - # The model's generation config, for what the request leaves unset. - {"temperature": 0.6, "top_k": 20, "max_tokens": 9}, - output_room=MAX_MODEL_LEN - 4, - ) - - assert (params.temperature, params.top_k, params.seed, params.n) == (0.6, 5, 7, 2) - # The request's max_tokens, capped by the context; the generation config's - # max_tokens applies only when the request leaves it unset. - assert params.max_tokens == MAX_MODEL_LEN - 4 - unset = native_engine.build_sampling_params( - _openai_request(), {"max_tokens": 9}, output_room=MAX_MODEL_LEN - 4 - ) - assert unset.max_tokens == 9 - assert sorted(params.stop_token_ids) == [2, 9] - assert params.logprobs == 2 - assert params.allowed_token_ids == [5, 6] - assert params._bad_words_token_ids == [[7]] - assert params.detokenize is False - - -def test_guided_decoding_maps_onto_structured_outputs() -> None: - params = native_engine.build_sampling_params( - _openai_request( - sampling_options={"guided_decoding": {"json": {"type": "object"}}} - ), - {}, - output_room=MAX_MODEL_LEN - 4, - ) - - assert params.structured_outputs.json == {"type": "object"} - - -def test_best_of_is_rejected_as_by_vllm() -> None: - with pytest.raises(InvalidArgument, match="best_of"): - native_engine.build_sampling_params( - _openai_request(sampling_options={"n": 1, "best_of": 2}), - {}, - output_room=MAX_MODEL_LEN - 4, - ) - - -def test_openai_requests_stream_each_choice_by_index() -> None: - prompt = [1, 2, 3, 4] - llm = _FakeLLM( - [ - _output(prompt, _completion([5], index=0), _completion([7], index=1)), - _output(prompt, _completion([6], "stop", index=0)), - _output(prompt, _completion([8], "length", index=1)), - ] - ) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - chunks = _collect(engine, _openai_request(sampling_options={"n": 2})) - - assert [(c["index"], c["token_ids"]) for c in chunks] == [ - (0, [5]), - (1, [7]), - (0, [6]), - (1, [8]), - ] - assert [c.get("finish_reason") for c in chunks] == [None, None, "stop", "length"] - assert chunks[-1]["completion_usage"]["completion_tokens"] == 4 - - -def test_prompt_logprobs_are_returned_in_engine_data() -> None: - prompt = [1, 2, 3, 4] - llm = _FakeLLM( - [ - # Under DELTA, vLLM sends prompt logprobs once, at the end of prefill, - # and an empty list on every later output. - _output( - prompt, - _completion([5]), - prompt_logprobs=[None, {2: _logprob(-0.25)}, None, None], - ), - _output(prompt, _completion([6], "stop"), prompt_logprobs=[]), - ] - ) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - chunks = _collect(engine, _openai_request(output_options={"prompt_logprobs": 0})) - - assert chunks[-1]["engine_data"]["prompt_logprobs"][1]["2"]["logprob"] == -0.25 - - -def test_queue_limits_reject_with_the_engines_status() -> None: - llm = _FakeLLM(admission_error=QueueOverflowError()) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(HttpError) as raised: - _collect(engine, _openai_request()) - - assert raised.value.code == HTTPStatus.SERVICE_UNAVAILABLE - assert llm.calls == [] - - def _start_worker_config(monkeypatch, host) -> SimpleNamespace: """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" configs = [] @@ -415,69 +250,22 @@ async def start(): return config -def test_every_worker_arg_reaches_the_dynamo_worker_config( - monkeypatch, tmp_path -) -> None: - template = tmp_path / "chat.jinja" - template.write_text("{{ messages }}") - worker_args = { - "tool_call_parser": "qwen3_coder", - "reasoning_parser": "nemotron_nano", - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": True, - "structural_tag_scope": "always", - "structural_tag_schema": "strict", - "custom_jinja_template": str(template), - "endpoint_types": ["chat"], - } - assert set(worker_args) == set(DynamoWorkerArgs.model_fields) - host = _host(_FakeLLM(), dynamo_cfg={"worker_args": worker_args}) - - config = _start_worker_config(monkeypatch, host) +def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: + config = _start_worker_config(monkeypatch, _host(_FakeLLM())) assert config.namespace == "ns" - assert config.endpoint_types == "chat" - assert config.structural_tag_mode == "on" - with open(config.custom_jinja_template) as template_file: - assert template_file.read() == "{{ messages }}" - for key in worker_args.keys() - { - "endpoint_types", - "enable_structural_tag", - "custom_jinja_template", - }: - assert getattr(config, key) == worker_args[key], key - - -def test_the_native_servers_chat_settings_reach_the_frontend(monkeypatch) -> None: - host = _host(_FakeLLM()) - host.cfg["vllm_cfg"] = { - "max_model_len": MAX_MODEL_LEN, - "reasoning_parser_plugin": "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", - "http_server_serving_chat_kwargs": { - "tool_parser": "qwen3_coder", - "reasoning_parser": "nano_v3", - # Literal Jinja, which the native server also accepts. - "chat_template": "{{ messages }}\n", - }, - } - - config = _start_worker_config(monkeypatch, host) + # NeMo RL's rollouts go to /v1/completions; no chat surface is registered. + assert config.endpoint_types == "completions" - assert (config.tool_call_parser, config.reasoning_parser) == ( - "qwen3_coder", - "nemotron_v3", - ) - with open(config.custom_jinja_template) as template_file: - assert template_file.read() == "{{ messages }}\n" +def test_requests_other_than_rollouts_are_refused() -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) -def test_a_parser_dynamo_lacks_fails_registration(monkeypatch) -> None: - host = _host( - _FakeLLM(), dynamo_cfg={"worker_args": {"tool_call_parser": "no_such_parser"}} - ) + with pytest.raises(InvalidArgument, match="only NeMo RL rollouts"): + _collect(engine, {"token_ids": [1, 2, 3, 4]}) - with pytest.raises(ValueError, match="no tool call parser named 'no_such_parser'"): - _start_worker_config(monkeypatch, host) + assert llm.calls == [] def test_an_endpoint_that_stops_serving_reports_why() -> None: @@ -528,78 +316,6 @@ async def call_utility_async(method): assert registration.context_length == MAX_MODEL_LEN -def test_a_prompt_that_fills_the_context_is_refused_as_natively() -> None: - llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match="maximum context length"): - _collect(engine, _openai_request(token_ids=list(range(MAX_MODEL_LEN)))) - - assert llm.calls == [] - - -def _image_url() -> str: - buffer = io.BytesIO() - Image.new("RGB", (8, 8), color="red").save(buffer, format="PNG") - return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() - - -def test_media_is_loaded_and_processed_as_vllms_chat_parser_does() -> None: - final = _output([1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3], _completion([5], "stop")) - llm = _FakeLLM([final]) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - _collect( - engine, - _openai_request( - token_ids=[1, IMAGE_PLACEHOLDER, 3], - multi_modal_data={"image_url": [{"Url": _image_url()}]}, - mm_processor_kwargs={"max_num_tiles": 2}, - ), - ) - - ((prompt, skip_mm_cache),) = llm.renderer.prompts - # The frontend's unexpanded tokens, the decoded image and the processor kwargs. - assert prompt["prompt_token_ids"] == [1, IMAGE_PLACEHOLDER, 3] - (image,) = prompt["multi_modal_data"]["image"] - assert image.size == (8, 8) and image.mode == "RGB" - assert prompt["multi_modal_uuids"] == {"image": [None]} - assert prompt["mm_processor_kwargs"] == {"max_num_tiles": 2} - assert skip_mm_cache is False - # vLLM generates from the processed prompt, and the output cap counts its tokens. - ((engine_input, params, _),) = llm.calls - assert engine_input["prompt_token_ids"] == [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] - assert params.max_tokens == MAX_MODEL_LEN - 2 - IMAGE_TOKENS - - -@pytest.mark.parametrize( - ("fields", "error"), - [ - ({"prompt_embeds": "AAAA"}, "Embedding prompts"), - ({"multi_modal_data": {"image_url": [{"UuidOnly": "u"}]}}, "carry their URL"), - ( - { - "multi_modal_data": { - "image_url": [{"Url": "data:image/png;base64,eA=="}] - } - }, - "Cannot load image_url", - ), - # vLLM's own request errors (not ValueErrors) keep their message, as the - # native server's 4xx does, instead of becoming a bare 500. - ({"stop_conditions": {"min_tokens": 10, "max_tokens": 5}}, "min_tokens"), - ], -) -def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: - llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match=error): - _collect(engine, _openai_request(**fields)) - - assert llm.calls == [] - - @pytest.mark.parametrize( ("router_mode", "prefix_caching", "publishes"), [("kv", True, True), ("kv", False, False), ("round-robin", True, False)], diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 1e040d3c177..2119afcebbb 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -405,7 +405,8 @@ def test_dynamo_backend_is_rejected_with_the_migration(): ({"control_timeout_s": 600}, r"dynamo_cfg\.control_timeout_s"), ({"metrics_include_prefixes": None}, r"dynamo_cfg\.metrics_include_prefixes"), ({"metrics_exclude_prefixes": None}, r"dynamo_cfg\.metrics_exclude_prefixes"), - ({"worker_args": {"extra_cli_args": []}}, r"worker_args\.extra_cli_args"), + ({"worker_args": {"extra_cli_args": []}}, r"dynamo_cfg\.worker_args was"), + ({"worker_args": {"reasoning_parser": None}}, r"dynamo_cfg\.worker_args was"), ], ) def test_removed_dynamo_cfg_keys_are_rejected(dynamo_cfg, removed): From 83cab88b5fdb4ba86b2c865aab5b017f7fdc77d6 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:32:35 -0700 Subject: [PATCH 14/82] fix(dynamo): address round-2 review (C26, C28-C31, C33, C34, C36, C37, C39-C45) - C26: dynamo_cfg.request_timeout_s defaults to None. It bounded the whole generation, which native never does, so long rollouts were dropped and re-sampled; on the SingleController native.generation_timeout_s bounds both paths. - C28: router_mode no longer accepts "direct", which needs a worker ID that rollouts never carry. - C29: the EP > TP rejection gives the real reason: vLLM refuses the per-engine ipc:// KV-event endpoint above DP rank 0. - C30: http_server_api_key_required is rejected again: the frontend serves the same engines on 0.0.0.0 with no bearer check. - C31: kv_event_sources connects a wildcard endpoint to 127.0.0.1. - C33: one driver test checks the request sent on success, the assembled output, and the 503 -> GenerationUnavailable mapping. - C34: check_dynamo_overlay.sh requires published KV events, and each failure prints its reason. - C36: frontend_args.tokenizer* are removed (rollouts send token IDs, so the frontend never tokenizes) and rejected at startup. - C37/C39: docs say non-colocated and non-Gym recipes, and state the adapter's own vLLM calls. - C40-C45: real WorkerConfig in the registration test, ai-dynamo pytest plugins disabled, http_status_is_infra reused, num_gpu_blocks read from vllm_config, DYN_SDK_DISABLE_ANSI_LOGGING set in the actor env, Gym-only test fakes removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 19 +++--- docs/guides/dynamo-generation.md | 23 ++++--- docs/index.md | 2 +- ...grpo_math_1B_dynamo_single_controller.yaml | 11 ++- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 3 - nemo_rl/models/generation/__init__.py | 22 +++--- nemo_rl/models/generation/dynamo/config.py | 18 ++--- .../models/generation/dynamo/http_client.py | 2 +- .../generation/dynamo/managed_runtime.py | 20 +----- .../models/generation/dynamo/native_engine.py | 15 +++-- .../generation/dynamo/rollout_client.py | 11 ++- .../models/generation/vllm/vllm_generation.py | 4 ++ nemo_rl/utils/outdated_config_checks.py | 14 ++++ pyproject.toml | 4 +- tests/functional/check_dynamo_overlay.sh | 41 ++++++------ .../models/generation/test_dynamo_config.py | 23 ++++++- .../generation/test_dynamo_http_client.py | 3 +- .../generation/test_dynamo_managed_runtime.py | 19 ++---- .../generation/test_dynamo_native_engine.py | 19 +----- .../generation/test_dynamo_rollout_client.py | 26 ------- .../models/generation/test_vllm_generation.py | 67 ++++++++++++++----- .../unit/utils/test_outdated_config_checks.py | 4 ++ 22 files changed, 190 insertions(+), 180 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 2f1acd26d4e..4b7e95277e7 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -6,8 +6,13 @@ Dynamo's frontend and KV-aware router. Two requirements shape the design: - A vLLM bump must never be blocked by Dynamo. - Dynamo CI coverage must stay consistent with native vLLM's. -Both hold only if the engine process contains no Dynamo-owned vLLM code. Then a -vLLM bump can break Dynamo serving only where it also breaks native vLLM. +Both hold only if the engine process contains no Dynamo-owned vLLM code. A vLLM +bump can then break Dynamo serving only in NeMo RL's own adapter +(`native_engine.py`), which calls a few vLLM APIs native NeMo RL does not: the +`get_kv_cache_group_metadata` engine utility and `KVCacheSpecKind` at +registration, and `StatLoggerBase` for the router's load signal. A renamed import +fails the vLLM-lane unit tests at collection, and the Dynamo functional overlays +run registration. ## Engine hosting @@ -85,12 +90,8 @@ runtime logs the router's applied-event counts at shutdown, and the Dynamo functional overlays fail if the engines published events but none were applied (`tests/functional/check_dynamo_overlay.sh`). -`ai-dynamo-runtime` caps pydantic, which pins the vLLM venvs' pydantic version. -Dependency bounds like this one can still block a vLLM bump, and the import -guard does not see them: `ai-dynamo-runtime` 1.5.1 requires -`pydantic>=2.10.6,<=2.13`, and `ai-dynamo` caps `aiohttp<4`, -`prometheus-client<1` and `zstandard<1`. A vLLM release that needs a version -outside one of these bounds does not lock with the pinned `ai-dynamo`. Until a -Dynamo release lifts the bound, override it under +ai-dynamo's own dependency bounds (`ai-dynamo-runtime` caps pydantic, for +example) can still block a vLLM bump, and the import guard does not see them. +Until a Dynamo release lifts a bound, override it under `[tool.uv] override-dependencies` (as `aisimulate` and `kubernetes` are), and let the vLLM-lane Dynamo unit tests catch a real break. diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index b1a543a52bf..7acefc88d9b 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -1,6 +1,6 @@ # Dynamo serving for vLLM generation -Any vLLM generation config can serve its rollouts through +Any non-colocated vLLM generation config can serve its rollouts through [Dynamo](https://github.com/ai-dynamo/dynamo)'s OpenAI frontend and KV-aware router. Add one line: @@ -27,9 +27,8 @@ independent. With `dynamo: true`: - `vllm_cfg.async_engine` is turned on, because Dynamo drives the engine's - `AsyncLLM`. Settings the async engine does not run, such as data parallelism - inside vLLM (`expert_parallel_size > tensor_parallel_size`), are rejected at - startup. + `AsyncLLM`. Settings Dynamo does not serve are rejected at startup; see + [Not supported](#not-supported). - The driver starts etcd, NATS and `dynamo.frontend`. The frontend runs from the vLLM workers' own venv, so it is the same ai-dynamo release. - Each engine turns on vLLM's KV-cache events for `router_mode: kv` and @@ -51,9 +50,8 @@ shows them all: | Key | Default | Meaning | |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | -| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies | +| `request_timeout_s` | null | Deadline for one NeMo RL rollout through the frontend. null waits as long as generation takes, as native does; on the SingleController, `async_rl.rollout_failure.native.generation_timeout_s` bounds both paths | | `frontend_args.router_mode` | `kv` | Frontend routing policy | -| `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | The etcd and nats-server binaries are found in `NEMO_RL_DYNAMO_BIN_DIR` when it @@ -87,7 +85,12 @@ Startup rejects these with `dynamo: true`: - `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, and a restarted shard does not register with it yet. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data - parallelism inside vLLM, which the async engine Dynamo serves does not. + parallelism inside vLLM, which Dynamo does not serve yet. Above data-parallel + rank 0, vLLM rejects each engine's `ipc://` KV-event endpoint. +- `vllm_cfg.http_server_api_key_required`: the frontend serves the same engines + on 0.0.0.0 and checks no bearer token. +- `frontend_args.router_mode: direct`: it needs a worker ID on every request, and + NeMo RL's rollouts name none. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. @@ -117,7 +120,7 @@ bash docker/dynamo/install.sh ## Run -Use any non-colocated vLLM recipe and add the switch: +Use any non-colocated vLLM recipe that does not use NeMo-Gym, and add the switch: ```bash uv run examples/run_grpo_single_controller.py \ @@ -142,4 +145,6 @@ rejects it with the migration: `vllm_cfg.enable_vllm_metrics_logger`; - `worker_args`: the engines serve only NeMo RL's rollouts, which need no chat parsers or templates; engines are configured through `vllm_cfg` and - `vllm_kwargs`. + `vllm_kwargs`; + - `frontend_args.tokenizer`, `tokenizer_cache` and `tokenizer_cache_bytes`: + rollouts send token IDs, which the frontend never tokenizes. diff --git a/docs/index.md b/docs/index.md index 9cfda03a275..5f9dba49154 100644 --- a/docs/index.md +++ b/docs/index.md @@ -83,7 +83,7 @@ Configure and launch NeMo RL on multi-node Slurm or Kubernetes clusters for dist :link: guides/dynamo-generation :link-type: doc -Serve any vLLM generation config through Dynamo's frontend and KV-aware router with `dynamo: true`. +Serve non-colocated vLLM generation through Dynamo's frontend and KV-aware router with `dynamo: true`. ::: :::: diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 1ac1a92abaf..a1d89235d1e 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -14,14 +14,11 @@ policy: dynamo_cfg: # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: 600 - # Timeout for one rollout routed through the frontend (in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS). A timeout fails the attempt as a native - # timeout does, and the rollout retry policy applies. - request_timeout_s: 900 + # Deadline for one rollout through the frontend. null waits as long as + # generation takes, as native generate_async does; on the SingleController, + # async_rl.rollout_failure.native.generation_timeout_s bounds both paths. + request_timeout_s: null frontend_args: - tokenizer: default - tokenizer_cache: false - tokenizer_cache_bytes: 52428800 # kv: route each request to the engine holding most of its prefix. router_mode: kv extra_cli_args: [] diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml index d12f2536970..72877ceb232 100644 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml @@ -87,9 +87,6 @@ policy: startup_timeout_s: 1800 request_timeout_s: 7200 frontend_args: - tokenizer: fastokens - tokenizer_cache: true - tokenizer_cache_bytes: 4294967296 router_mode: kv extra_cli_args: [] vllm_cfg: diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 7d17e9e8942..08d28438400 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -183,6 +183,13 @@ def configure_generation_config( if dynamo_enabled(config): # Validate before any Ray allocation. dynamo_cfg(config) + if config["vllm_cfg"].get("http_server_api_key_required"): + raise ValueError( + "policy.generation.vllm_cfg.http_server_api_key_required is not " + "supported with policy.generation.dynamo=true: the Dynamo " + "frontend serves the same engines on 0.0.0.0 and checks no " + "bearer token." + ) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( @@ -193,18 +200,17 @@ def configure_generation_config( ) vllm_cfg = config["vllm_cfg"] if vllm_cfg["expert_parallel_size"] > vllm_cfg["tensor_parallel_size"]: - # VllmGeneration refuses the async engine with DP inside vLLM. + # Data parallelism inside vLLM: above DP rank 0, vLLM rejects the + # per-engine ipc:// KV-event endpoint (dynamo_kv_events_config). raise ValueError( - "policy.generation.dynamo=true serves the async vLLM engine, " - "which does not run data parallelism inside vLLM " - "(vllm_cfg.expert_parallel_size > tensor_parallel_size, see " - "https://github.com/NVIDIA-NeMo/RL/issues/1101). Set " - "expert_parallel_size <= tensor_parallel_size, or remove " - "policy.generation.dynamo." + "policy.generation.dynamo=true does not serve data parallelism " + "inside vLLM yet (vllm_cfg.expert_parallel_size > " + "tensor_parallel_size). Set expert_parallel_size <= " + "tensor_parallel_size, or remove policy.generation.dynamo." ) if not config["vllm_cfg"]["async_engine"]: # Dynamo serves the engine's AsyncLLM, so turning the async engine - # on is the switch; settings it cannot run are rejected above. + # on is the switch; settings Dynamo cannot serve are rejected above. print( " [Dynamo] policy.generation.dynamo=true: using the async " "vLLM engine (vllm_cfg.async_engine=true)", diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index af592bc0fd5..824280c3edf 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -21,23 +21,20 @@ from typing import Any, Literal, Mapping -from pydantic import BaseModel, Field, PositiveFloat, PositiveInt +from pydantic import BaseModel, Field, PositiveFloat class DynamoFrontendArgs(BaseModel, extra="forbid"): """Arguments for the managed ``dynamo.frontend`` process.""" - tokenizer: Literal["default", "fastokens"] = "default" - tokenizer_cache: bool = False - tokenizer_cache_bytes: PositiveInt = 50 * 1024**2 # "kv" routes each request to the engine holding most of its prefix, fed by - # the engines' vLLM KV-cache events. + # the engines' vLLM KV-cache events. No "direct": it needs a worker ID on + # every request, and NeMo RL's rollouts name none. router_mode: Literal[ "round-robin", "random", "power-of-two", "kv", - "direct", "least-loaded", "device-aware-weighted", ] = "kv" @@ -49,11 +46,10 @@ class DynamoCfg(BaseModel, extra="forbid"): # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: PositiveFloat = 600.0 - # Timeout for one NeMo RL rollout routed through the frontend, in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS. A timeout fails the attempt as a native one - # does (GenerationUnavailable), and the rollout retry policy re-dispatches it - # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). - request_timeout_s: PositiveFloat = 900.0 + # Deadline for one NeMo RL rollout through the frontend. None waits as long + # as generation takes, as native generate_async does; on the SingleController, + # async_rl.rollout_failure.native.generation_timeout_s bounds both paths. + request_timeout_s: PositiveFloat | None = None frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) diff --git a/nemo_rl/models/generation/dynamo/http_client.py b/nemo_rl/models/generation/dynamo/http_client.py index b7d81f3de15..77d318f84a4 100644 --- a/nemo_rl/models/generation/dynamo/http_client.py +++ b/nemo_rl/models/generation/dynamo/http_client.py @@ -36,7 +36,7 @@ def _decode_json_object(body: bytes) -> dict[str, Any]: async def async_http_post_json( - url: str, payload: dict[str, Any], timeout_s: float + url: str, payload: dict[str, Any], timeout_s: float | None ) -> dict[str, Any]: """POST JSON without blocking the rollout actor event loop.""" timeout = aiohttp.ClientTimeout(total=timeout_s) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index e53c0aef8cc..d17cf2d7763 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -216,15 +216,6 @@ def _service_env(self) -> dict[str, str]: env.update(self._manager_env) return env - def _frontend_env(self) -> dict[str, str]: - env = self._service_env() - frontend_args = self._dynamo_cfg.frontend_args - env["DYN_TOKENIZER"] = frontend_args.tokenizer - if frontend_args.tokenizer_cache: - env["DYN_TOKENIZER_CACHE"] = "1" - env["DYN_TOKENIZER_CACHE_BYTES"] = str(frontend_args.tokenizer_cache_bytes) - return env - def _start_etcd(self) -> None: self._etcd_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_etcd_") peer_url = f"http://{self._host}:{self._etcd_peer_port}" @@ -312,19 +303,10 @@ def _start_frontend(self) -> None: print( f" [Dynamo] launching frontend argv={redact_argv(command)!r}", flush=True ) - frontend_env = self._frontend_env() + frontend_env = self._service_env() dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) frontend_env["VIRTUAL_ENV"] = dynamo_venv frontend_env["UV_PROJECT_ENVIRONMENT"] = dynamo_venv - tokenizer_env = { - key: value - for key, value in frontend_env.items() - if key.startswith("DYN_TOKENIZER") - } - print( - f" [Dynamo] frontend tokenizer environment={tokenizer_env!r}", - flush=True, - ) self._frontend_process = subprocess.Popen( command, env=frontend_env, start_new_session=True ) diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index e95f535bc25..b422d21c11e 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -101,21 +101,23 @@ def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: class SnapshotStatLogger(StatLoggerBase): """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" + # Set by Dynamo after vLLM built the logger, so shared through the class. publisher: Optional[Any] = None - num_gpu_blocks: int = 0 def __init__(self, vllm_config: Any, engine_index: int = 0) -> None: self.dp_rank = engine_index + # vLLM sizes the KV cache before it builds its stat loggers. + self.num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0 def record(self, scheduler_stats, iteration_stats, *args, **kwargs) -> None: cls = type(self) if scheduler_stats is None or cls.publisher is None: return - total = max(1, cls.num_gpu_blocks) + total = max(1, self.num_gpu_blocks) usage = scheduler_stats.kv_cache_usage hit_rate = None - prefix = getattr(scheduler_stats, "prefix_cache_stats", None) - if prefix is not None and prefix.queries > 0: + prefix = scheduler_stats.prefix_cache_stats + if prefix.queries > 0: hit_rate = prefix.hits / prefix.queries cls.publisher.publish( self.dp_rank, @@ -164,7 +166,6 @@ async def start(self, worker_id: int) -> EngineConfig: ): block_size = group["block_size"] break - SnapshotStatLogger.num_gpu_blocks = cache.num_gpu_blocks or 0 model = self._host.model_name return EngineConfig( model=model, @@ -232,7 +233,9 @@ async def kv_event_sources(self) -> list[KvEventSource]: kv_events = self._llm.vllm_config.kv_events_config if kv_events is None or not kv_events.enable_kv_cache_events: return [] - return [ZmqSource(endpoint=kv_events.endpoint)] + # A user's own kv_events_config can keep vLLM's wildcard default + # (tcp://*:5557): vLLM binds it, but a subscriber cannot connect to "*". + return [ZmqSource(endpoint=kv_events.endpoint.replace("*", "127.0.0.1"))] def component_metrics_dp_ranks(self) -> list[int]: return [0] diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py index 95beb10d319..3fe35a7f7fe 100644 --- a/nemo_rl/models/generation/dynamo/rollout_client.py +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -19,6 +19,7 @@ import logging from typing import Any, Optional +from nemo_rl.experience.failures import http_status_is_infra from nemo_rl.models.generation.dynamo.http_client import ( async_http_post_json, format_dynamo_error, @@ -33,7 +34,6 @@ _HTTP_MAX_ATTEMPTS = 3 _HTTP_RETRY_DELAY_S = 1.0 -_RETRYABLE_HTTP_STATUS_CODES = {408, 429} def _is_retryable(response: dict[str, Any]) -> bool: @@ -42,9 +42,7 @@ def _is_retryable(response: dict[str, Any]) -> bool: if "transport_error" in response or response.get("json_decode_error") is True: return True status = response.get("http_status") - return isinstance(status, int) and ( - status in _RETRYABLE_HTTP_STATUS_CODES or 500 <= status < 600 - ) + return isinstance(status, int) and http_status_is_infra(status) def rollout_payload( @@ -95,7 +93,7 @@ def parse_rollout_response(response: dict[str, Any], *, url: str) -> dict[str, A async def post_rollout( - url: str, payload: dict[str, Any], timeout_s: float + url: str, payload: dict[str, Any], timeout_s: float | None ) -> dict[str, Any]: """POST one rollout, retrying transient errors; returns its ``engine_data``.""" response: dict[str, Any] = {} @@ -116,7 +114,6 @@ async def post_rollout( raise RuntimeError( f"Dynamo rollout request to {url} failed: {response['timeout_error']}. " "For longer sequences, increase " - "policy.generation.dynamo_cfg.request_timeout_s, e.g. to " - f"{int(timeout_s * 2)}." + "policy.generation.dynamo_cfg.request_timeout_s, or set it to null." ) return parse_rollout_response(response, url=url) diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 018ac34fc24..3389737cce4 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -411,6 +411,10 @@ def __init__( # RayWorkerGroup(env_vars=...) below) is inherited by them. if draft_module_sharing_disable_required(self.cfg): env_vars[DRAFT_DISABLE_MODULE_SHARING_ENV] = "1" + if dynamo_enabled(self.cfg): + # dynamo._core fixes its log format when the engine first imports it, + # before engine_env is applied, so it must be in the actor's env. + env_vars["DYN_SDK_DISABLE_ANSI_LOGGING"] = "1" # Check if we need parallelism-aware worker group creation if self.model_parallel_size > 1: diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 8591d1d1a17..28c8b30f1e9 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -213,6 +213,11 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: "templates; engines are configured through vllm_cfg and vllm_kwargs" ), } +_REMOVED_DYNAMO_FRONTEND_KEYS = ( + "tokenizer", + "tokenizer_cache", + "tokenizer_cache_bytes", +) def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: @@ -246,6 +251,15 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: problems.append( f"policy.generation.dynamo_cfg.{key} was removed: {replacement}." ) + frontend_args = dynamo_cfg.get("frontend_args") + if isinstance(frontend_args, dict): + for key in _REMOVED_DYNAMO_FRONTEND_KEYS: + if key in frontend_args: + problems.append( + f"policy.generation.dynamo_cfg.frontend_args.{key} was " + "removed: NeMo RL's rollouts send token IDs, which the " + "frontend never tokenizes." + ) if problems: raise ValueError(" ".join(problems)) diff --git a/pyproject.toml b/pyproject.toml index 259d712d387..a514464e7d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -708,7 +708,9 @@ exclude = ''' ''' [tool.pytest.ini_options] -addopts = "--durations=100 -s -rA -x" +# ai-dynamo (vllm extra) registers pytest plugins that skip test_sglang_*, +# test_trtllm_* and test_triton_* files when those frameworks are absent. +addopts = "--durations=100 -s -rA -x -p no:vllm_tests -p no:sglang_tests -p no:trtllm_tests -p no:triton_tests" testpaths = ["tests"] python_files = "test_*.py" markers = [ diff --git a/tests/functional/check_dynamo_overlay.sh b/tests/functional/check_dynamo_overlay.sh index ee98e2671a0..42a2b381e41 100755 --- a/tests/functional/check_dynamo_overlay.sh +++ b/tests/functional/check_dynamo_overlay.sh @@ -15,10 +15,11 @@ # What a native functional test also checks when it runs as the Dynamo overlay # (++policy.generation.dynamo=true): every engine registered with the managed -# frontend, the frontend routed rollouts to them, the KV router applied the -# engines' KV-cache events (when they publish any: KV routing with prefix -# caching), and shutdown stopped its services. The test's own metric checks hold -# as they do natively. +# frontend, the frontend routed rollouts to them, the engines published KV-cache +# events and the KV router applied them, and shutdown stopped its services. The +# test's own metric checks hold as they do natively. The KV check needs the KV +# router with vLLM prefix caching (the default on sm>=80); an overlay that turns +# either off cannot use this check as is. # # bash check_dynamo_overlay.sh @@ -26,28 +27,28 @@ set -euo pipefail run_log=$1 -grep -E "\[Dynamo\] frontend .* routes to [0-9]+ engine" "${run_log}" +fail() { + echo "$1" >&2 + exit 1 +} + +grep -E "\[Dynamo\] frontend .* routes to [0-9]+ engine" "${run_log}" \ + || fail "No engine registered with the Dynamo frontend" routed=$(grep -c 'route request completed request.outcome="success"' "${run_log}" || true) echo "Requests the Dynamo frontend routed: ${routed}" -if [[ "${routed}" -eq 0 ]]; then - echo "No request went through the Dynamo frontend" >&2 - exit 1 -fi +[[ "${routed}" -gt 0 ]] || fail "No request went through the Dynamo frontend" # vLLM's KV-event wire format is the one coupling between NeMo RL's vLLM and # Dynamo's router; a format change shows up here as no applied events. -if grep -q "\[Dynamo\] engine publishes KV events: True" "${run_log}"; then - grep -E "\[Dynamo\] router applied KV events" "${run_log}" - applied=$(grep -oE '\[Dynamo\] router applied KV events: .*"ok": [0-9.]+' "${run_log}" \ - | grep -oE '[0-9.]+$' | tail -1) - if [[ -z "${applied}" || "${applied%.*}" -eq 0 ]]; then - echo "The KV router applied none of the engines' KV-cache events" >&2 - exit 1 - fi -fi +grep -q "\[Dynamo\] engine publishes KV events: True" "${run_log}" \ + || fail "No engine published KV-cache events for the KV router" +applied=$(grep -oE '\[Dynamo\] router applied KV events: .*"ok": [0-9.]+' "${run_log}" \ + | grep -oE '[0-9.]+$' | tail -1 || true) +echo "KV events the router applied: ${applied:-0}" +[[ "${applied:-0}" =~ ^[0-9]+ && "${applied%%.*}" -gt 0 ]] \ + || fail "The KV router applied none of the engines' KV-cache events" dynamo_bin_dir=${NEMO_RL_DYNAMO_BIN_DIR:-/opt/dynamo/bin} if pgrep -f "[d]ynamo.frontend|${dynamo_bin_dir}/[e]tcd|${dynamo_bin_dir}/[n]ats-server"; then - echo "Managed Dynamo services remain after shutdown" >&2 - exit 1 + fail "Managed Dynamo services remain after shutdown" fi diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index a14f09e95f9..0624535b30c 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -75,7 +75,7 @@ def test_without_dynamo_the_vllm_config_is_untouched() -> None: (_vllm_config(dynamo=True, backend="sglang"), "backend=vllm"), # dynamo_cfg alone would be ignored. (_vllm_config(dynamo_cfg={}), "dynamo is not true"), - # The async engine Dynamo serves runs no data parallelism inside vLLM. + # Dynamo does not serve data parallelism inside vLLM yet. ( _vllm_config( dynamo=True, @@ -88,6 +88,27 @@ def test_without_dynamo_the_vllm_config_is_untouched() -> None: ), "expert_parallel_size <= tensor_parallel_size", ), + # The frontend serves the same engines and checks no bearer token. + ( + _vllm_config( + dynamo=True, + vllm_cfg={ + "async_engine": True, + "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 1, + "http_server_api_key_required": True, + }, + ), + "checks no bearer token", + ), + # Direct routing needs a worker ID that rollouts never carry. + ( + _vllm_config( + dynamo=True, dynamo_cfg={"frontend_args": {"router_mode": "direct"}} + ), + "router_mode", + ), ], ) def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( diff --git a/tests/unit/models/generation/test_dynamo_http_client.py b/tests/unit/models/generation/test_dynamo_http_client.py index d9002bce5c1..df8eabd9344 100644 --- a/tests/unit/models/generation/test_dynamo_http_client.py +++ b/tests/unit/models/generation/test_dynamo_http_client.py @@ -83,9 +83,10 @@ def test_async_http_post_json_error_contract( lambda **kwargs: session, ) + # No deadline is the default (dynamo_cfg.request_timeout_s: null). response = asyncio.run( http_client.async_http_post_json( - "http://worker/route", {"value": 1}, timeout_s=3 + "http://worker/route", {"value": 1}, timeout_s=None ) ) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index d9cd8990bd8..524416b344b 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -108,9 +108,7 @@ def test_runtime_construction_is_inert_and_namespace_is_driver_owned( def test_managed_service_and_frontend_environments_are_runtime_owned( monkeypatch, ) -> None: - runtime = _runtime( - tokenizer="fastokens", tokenizer_cache=True, tokenizer_cache_bytes=4096 - ) + runtime = _runtime() runtime._manager_env = { "DYN_NAMESPACE": "managed", "DYN_DISCOVERY_BACKEND": "etcd", @@ -136,14 +134,9 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( engine_env = runtime.engine_env assert engine_env["DYN_NAMESPACE"] == "managed" # Each engine serves its own model card, so a frontend on another node can - # read a chat template that exists only on the engine's node. + # read model files that exist only on the engine's node. assert engine_env["DYN_SYSTEM_PORT"] == "0" - - frontend_env = runtime._frontend_env() - assert "DYN_SYSTEM_PORT" not in frontend_env - assert frontend_env["DYN_TOKENIZER"] == "fastokens" - assert frontend_env["DYN_TOKENIZER_CACHE"] == "1" - assert frontend_env["DYN_TOKENIZER_CACHE_BYTES"] == "4096" + assert "DYN_SYSTEM_PORT" not in service_env def test_dynamo_python_resolves_the_vllm_worker_venv_as_workers_do( @@ -313,8 +306,8 @@ def endpoint_stopped() -> None: runtime.wait_for_engines(1, check_engines=endpoint_stopped) -def test_frontend_launches_from_the_given_interpreter(monkeypatch, capsys) -> None: - runtime = _runtime(tokenizer="fastokens", tokenizer_cache=True) +def test_frontend_launches_from_the_given_interpreter(monkeypatch) -> None: + runtime = _runtime() runtime._frontend_port = 3000 runtime._namespace = "nemo-rl-test" launched = [] @@ -326,8 +319,6 @@ def test_frontend_launches_from_the_given_interpreter(monkeypatch, capsys) -> No runtime._start_frontend() assert launched[0][:3] == ["/venvs/vllm/bin/python", "-m", "dynamo.frontend"] - output = capsys.readouterr().out - assert "DYN_TOKENIZER': 'fastokens" in output def test_shutdown_logs_the_kv_events_the_router_applied(monkeypatch, capsys) -> None: diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index ef2fbd01d83..d1dda2bfeb4 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -48,23 +48,10 @@ MAX_MODEL_LEN = 16 -EOS = 2 - - class _Context: def id(self) -> str: return "request-0" - def is_stopped(self) -> bool: - return False - - -class _Tokenizer: - eos_token_id = EOS - - def decode(self, token_ids) -> str: - return "".join(f"<{token_id}>" for token_id in token_ids) - def _logprob(value: float): return SimpleNamespace(logprob=value, rank=1, decoded_token=None) @@ -114,8 +101,6 @@ def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: "max_new_tokens": 8, "temperature": 0.7, "top_p": 0.9, - "val_temperature": 0.0, - "val_top_p": 1.0, "top_k": None, "stop_token_ids": [2], "stop_strings": [""], @@ -126,7 +111,6 @@ def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: host.model_name = "model" host.routed_experts_dtype = torch.int16 host.llm = llm - host._capture_media = False return host @@ -218,7 +202,7 @@ def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: assert rollout["token_ids"] == [] and rollout["truncated"] is False -def _start_worker_config(monkeypatch, host) -> SimpleNamespace: +def _start_worker_config(monkeypatch, host) -> native_engine.WorkerConfig: """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" configs = [] @@ -230,7 +214,6 @@ async def run(self): return None monkeypatch.setattr(native_engine, "Worker", _Worker) - monkeypatch.setattr(native_engine, "WorkerConfig", SimpleNamespace) env = { "DYN_DISCOVERY_BACKEND": "etcd", "DYN_REQUEST_PLANE": "tcp", diff --git a/tests/unit/models/generation/test_dynamo_rollout_client.py b/tests/unit/models/generation/test_dynamo_rollout_client.py index 1b2b942dc7f..138b4120017 100644 --- a/tests/unit/models/generation/test_dynamo_rollout_client.py +++ b/tests/unit/models/generation/test_dynamo_rollout_client.py @@ -12,42 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import asyncio -import json import pytest from nemo_rl.models.generation.dynamo import rollout_client from nemo_rl.models.generation.dynamo.rollout_client import ( - ROLLOUT_ANNOTATION, parse_rollout_response, post_rollout, - rollout_payload, ) -def test_rollout_payload_leaves_sampling_and_stopping_to_the_engine() -> None: - payload = rollout_payload( - model_name="model", - prompt_token_ids=[5, 6, 7], - max_model_len=10, - greedy=True, - stop_strings=[""], - ) - - assert payload["prompt"] == [5, 6, 7] - # The frontend's own stop rules must never end the stream first. - assert payload["ignore_eos"] is True - assert payload["max_tokens"] == 7 - assert "stop" not in payload and "temperature" not in payload - (annotation,) = payload["nvext"]["annotations"] - assert annotation.startswith(ROLLOUT_ANNOTATION) - assert json.loads(annotation[len(ROLLOUT_ANNOTATION) :]) == { - "greedy": True, - "stop_strings": [""], - } - assert payload["nvext"]["extra_fields"] == ["engine_data"] - - @pytest.mark.parametrize( "response, match", [ diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 4693fbdac91..1de81eac1df 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2626,26 +2626,30 @@ def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error VllmGeneration.validate_settings(master_config) -@pytest.mark.parametrize("dynamo", [False, True]) -def test_vllm_token_rollouts_use_the_dynamo_frontend_when_it_serves(dynamo): - generation = object.__new__(VllmGeneration) - generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None - tokens = BatchedDataDict({"input_ids": torch.tensor([[1]])}) - # The frontend request carries token IDs only, so media prompts stay native. - media = BatchedDataDict({"input_ids": torch.tensor([[1]]), "vllm_content": [None]}) - - assert generation._dynamo_serves(tokens) is dynamo - assert not generation._dynamo_serves(media) - - -def test_vllm_dynamo_frontend_failures_are_generation_unavailable(monkeypatch): +def test_vllm_dynamo_rollouts_go_through_the_frontend(monkeypatch): from nemo_rl.experience.failures import GenerationUnavailable from nemo_rl.models.generation.dynamo import rollout_client + from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result_to_wire, + ) - async def frontend_down(url, payload, timeout_s): - raise RuntimeError(f"Dynamo rollout request to {url} failed: HTTP 503") + result = RolloutResult([7, 8], [-0.5, -0.25], False, None, None) + responses = [ + {"rollout": rollout_result_to_wire(result), "gen_leader_worker_idx": 3}, + RuntimeError("Dynamo rollout request to frontend failed: HTTP 503"), + ] + requests = [] - monkeypatch.setattr(rollout_client, "post_rollout", frontend_down) + async def frontend(url, payload, timeout_s): + requests.append((url, payload, timeout_s)) + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + monkeypatch.setattr(rollout_client, "post_rollout", frontend) generation = object.__new__(VllmGeneration) generation.cfg = { "model_name": "model", @@ -2654,9 +2658,36 @@ async def frontend_down(url, payload, timeout_s): } generation.dynamo_frontend_url = "http://frontend/v1" data = BatchedDataDict( - {"input_ids": torch.tensor([[5, 6, 0]]), "input_lengths": torch.tensor([2])} + { + "input_ids": torch.tensor([[5, 6, 0]]), + "input_lengths": torch.tensor([2]), + "stop_strings": [[""]], + } ) - + # The frontend request carries token IDs only, so media prompts stay native. + media = BatchedDataDict({**data, "vllm_content": [None]}) + assert generation._dynamo_serves(data) and not generation._dynamo_serves(media) + + _, output = asyncio.run(anext(generation.generate_async(data, greedy=True))) + + ((url, payload, timeout_s),) = requests + assert (url, timeout_s) == ("http://frontend/v1/completions", None) + # The unpadded prompt; the engine samples it from the generation config, and + # the frontend's own stop rules never end the stream before the engine does. + assert payload["prompt"] == [5, 6] + assert payload["ignore_eos"] is True and payload["max_tokens"] == 14 + assert "stop" not in payload and "temperature" not in payload + (annotation,) = payload["nvext"]["annotations"] + assert json.loads(annotation.removeprefix(rollout_client.ROLLOUT_ANNOTATION)) == { + "greedy": True, + "stop_strings": [""], + } + assert payload["nvext"]["extra_fields"] == ["engine_data"] + # The driver returns what native dispatch returns. + native = rollout_output(result, input_ids=data["input_ids"][0], input_length=2) + assert output["gen_leader_worker_idx"] == [3] + for key in native: + assert torch.equal(output[key], native[key]), key # An infrastructure failure is re-dispatched; an unmapped RuntimeError would # count as a DATA failure against the prompt. with pytest.raises(GenerationUnavailable, match="HTTP 503"): diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 2119afcebbb..4539d34b951 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -407,6 +407,10 @@ def test_dynamo_backend_is_rejected_with_the_migration(): ({"metrics_exclude_prefixes": None}, r"dynamo_cfg\.metrics_exclude_prefixes"), ({"worker_args": {"extra_cli_args": []}}, r"dynamo_cfg\.worker_args was"), ({"worker_args": {"reasoning_parser": None}}, r"dynamo_cfg\.worker_args was"), + ( + {"frontend_args": {"tokenizer_cache": False}}, + r"frontend_args\.tokenizer_cache was", + ), ], ) def test_removed_dynamo_cfg_keys_are_rejected(dynamo_cfg, removed): From 1e73994f851d9d67d3f70fea3fef9948b56ac4b6 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:42:09 -0700 Subject: [PATCH 15/82] refactor(dynamo): debloat the Dynamo serving path No behaviour change: - rollout_client absorbs http_client (its only user), and the HTTP tests fold into one parametrized test. - venv.py goes: the frontend runs from worker_group.py_executable, the venv the engines already use, and etcd/nats-server are found with shutil.which in managed_runtime. - arguments.py goes: managed_runtime builds the fixed frontend argv and still rejects extra_cli_args that repeat a managed flag. Nothing logged carries credentials, so the redaction helpers go too. - decode_routed_experts(dtype=None) keeps the envelope's dtype, replacing routed_experts_envelope_dtype; the wire codec drops an impossible check. - managed_runtime: an unreachable namespace check, the atexit flag and the per-port allocation boilerplate go. - The swe1 Dynamo + NeMo-Gym recipe, its driver and its config test move to #4979 with the Gym path that can run them. - Tests: on/off config cases folded (with a check that the exemplar spells out exactly the DynamoCfg defaults), outdated-config cases parametrized, the redundant pin test and a GRPO helper's test removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 20 +- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 181 ------------------ nemo_rl/algorithms/grpo.py | 15 +- nemo_rl/models/generation/dynamo/__init__.py | 13 -- nemo_rl/models/generation/dynamo/arguments.py | 144 -------------- .../models/generation/dynamo/http_client.py | 77 -------- .../generation/dynamo/managed_runtime.py | 116 ++++++----- .../generation/dynamo/rollout_client.py | 58 +++++- nemo_rl/models/generation/dynamo/venv.py | 67 ------- .../generation/vllm/request_pipeline.py | 10 +- .../models/generation/vllm/vllm_generation.py | 5 +- .../generation/vllm/vllm_worker_async.py | 22 +-- nemo_rl/utils/routed_experts_codec.py | 18 +- pyrefly.toml | 3 - tests/test_suites/disabled.txt | 4 - ...nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh | 73 ------- tests/unit/L0_Unit_Tests_Vllm_1.sh | 2 +- tests/unit/L0_Unit_Tests_Vllm_2.sh | 2 +- tests/unit/L0_Unit_Tests_Vllm_3.sh | 2 +- tests/unit/algorithms/test_grpo.py | 49 +---- .../models/generation/test_dynamo_config.py | 54 ++---- .../generation/test_dynamo_http_client.py | 126 ------------ .../generation/test_dynamo_managed_runtime.py | 47 +---- .../generation/test_dynamo_rollout_client.py | 64 ++++++- .../generation/test_swe1_dynamo_config.py | 125 ------------ tests/unit/test_dependency_pins.py | 18 -- .../unit/utils/test_outdated_config_checks.py | 57 +++--- 27 files changed, 268 insertions(+), 1104 deletions(-) delete mode 100644 examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml delete mode 100644 nemo_rl/models/generation/dynamo/arguments.py delete mode 100644 nemo_rl/models/generation/dynamo/http_client.py delete mode 100644 nemo_rl/models/generation/dynamo/venv.py delete mode 100755 tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh delete mode 100644 tests/unit/models/generation/test_dynamo_http_client.py delete mode 100644 tests/unit/models/generation/test_swe1_dynamo_config.py diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 7acefc88d9b..52d65c6cfa3 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -133,18 +133,8 @@ is the same run with every `dynamo_cfg` default spelled out. ## Migrating from `backend: dynamo` -`backend: dynamo` launched Dynamo's own vLLM engine and was removed. Startup -rejects it with the migration: - -- Set `backend: vllm` and `dynamo: true`. `vllm_cfg` and `vllm_kwargs` keep their - meaning, and every vLLM feature is now available. -- Remove these from `dynamo_cfg`: - - `engine`; - - `control_timeout_s`: refit and pause use the vLLM backend's controls; - - `metrics_include_prefixes` / `metrics_exclude_prefixes`: use - `vllm_cfg.enable_vllm_metrics_logger`; - - `worker_args`: the engines serve only NeMo RL's rollouts, which need no chat - parsers or templates; engines are configured through `vllm_cfg` and - `vllm_kwargs`; - - `frontend_args.tokenizer`, `tokenizer_cache` and `tokenizer_cache_bytes`: - rollouts send token IDs, which the frontend never tokenizes. +`backend: dynamo` launched Dynamo's own vLLM engine and was removed. Set +`backend: vllm` and `dynamo: true`; `vllm_cfg` and `vllm_kwargs` keep their +meaning. Startup rejects the old backend and each removed `dynamo_cfg` key +(`engine`, `control_timeout_s`, `metrics_*_prefixes`, `worker_args`, +`frontend_args.tokenizer*`) with what replaces it. diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml deleted file mode 100644 index 72877ceb232..00000000000 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ /dev/null @@ -1,181 +0,0 @@ -defaults: ../../../nemo_gym/grpo_nanov3.yaml -cluster: - num_nodes: 3 - segment_size: 1 -checkpointing: - enabled: false - checkpoint_dir: results/nemotron-3-nano-swe-dynamo - save_period: 3 - keep_top_k: 1 - checkpoint_must_save_by: 00:03:30:00 -grpo: - num_prompts_per_step: 2 - num_generations_per_prompt: 2 - val_num_generations_per_prompt: 2 - max_num_epochs: 4 - max_num_steps: 4 - advantage_clip_low: -100 - advantage_clip_high: 100 - val_period: 10000 - overlong_filtering: false - invalid_tool_call_advantage: -5.0 - malformed_thinking_advantage: -5.0 - reward_shaping: - overlong_buffer_length: 128 - overlong_buffer_penalty: 1 - max_response_length: ${policy.max_total_sequence_length} - stop_properly_penalty_coef: null - reward_scaling: - source_min: 0.0 - source_max: 1.0 - target_min: 0.0 - target_max: 1.0 - async_grpo: - enabled: true - in_flight_weight_updates: false - recompute_kv_cache_after_weight_updates: true -loss_fn: - truncated_importance_sampling_ratio: 5 - truncated_importance_sampling_ratio_min: 0.2 - truncated_importance_sampling_type: tis - force_on_policy_ratio: true - use_kl_in_reward: false -policy: - model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 - tokenizer: - name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 - chat_template_kwargs: - enable_thinking: true - hf_config_overrides: {} - train_global_batch_size: 4 - generation_batch_size: 2 - max_total_sequence_length: 196608 - logprob_chunk_size: 1024 - megatron_cfg: - empty_unused_memory_level: 2 - tensor_model_parallel_size: 4 - pipeline_model_parallel_size: 1 - context_parallel_size: 2 - moe_flex_dispatcher_backend: hybridep - moe_hybridep_num_sms: 32 - use_gloo_process_groups: false - use_fused_weighted_squared_relu: true - do_not_average_loss: true - cp_normalize: true - calculate_per_token_loss: true - scale_loss_by_dp_cp_size: false - mtp_loss_scaling_factor: 0.0 - mtp_use_repeated_layer: true - mtp_num_layers: 0 - mtp_detach_heads: true - scheduler: - override_opt_param_scheduler: true - distributed_data_parallel_config: - overlap_grad_reduce: false - fp8_cfg: - enabled: false - fp8: e4m3 - fp8_recipe: mxfp8 - fp8_param: false - checkpoint: - ckpt_assume_constant_structure: true - sequence_packing: - fuse_loss: true - generation: - dynamo: true - dynamo_cfg: - startup_timeout_s: 1800 - request_timeout_s: 7200 - frontend_args: - router_mode: kv - extra_cli_args: [] - vllm_cfg: - expert_parallel_size: 4 - precision: bfloat16 - gpu_memory_utilization: 0.85 - enable_vllm_metrics_logger: true - vllm_metrics_logger_interval: 0.5 - env_vars: null - vllm_kwargs: - attention_backend: FLASH_ATTN - moe_backend: triton - compilation_config: - cudagraph_capture_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - pass_config: - fuse_allreduce_rms: false - colocated: - enabled: false - resources: - gpus_per_node: 8 - num_nodes: 1 -data: - train: - data_path: ${oc.env:HF_HOME}/superv3_data/swe1/train-split.jsonl - validation: - data_path: ${oc.env:HF_HOME}/superv3_data/swe1/val-split.jsonl -env: - nemo_gym: - skip_venv_if_present: true - num_gpu_nodes: 0 - invalid_tool_call_patterns: - - - - - - - - - thinking_tags: - - - - - config_paths: - - responses_api_models/vllm_model/configs/vllm_model_for_training.yaml - - resources_servers/single_step_tool_use_with_argument_comparison/configs/swe_pivot_single_step_tool_use_with_argument_comparison.yaml - # Drop inherited wrappers for environments omitted from this recipe. - math_with_judge_environment_server: null - code_gen_environment_server: null - workplace_assistant_environment_server: null - mcqa_environment_server: null - instruction_following_environment_server: null - structured_outputs_environment_server: null - swe_pivot_single_step_tool_use_with_argument_comparison_environment_server: - environment_servers: - legacy_agent: - entrypoint: app.py - agent_server: - type: responses_api_agents - name: swe_pivot_single_step_tool_use_with_argument_comparison_agent - policy_model: - responses_api_models: - vllm_model: - chat_template_kwargs: - force_nonempty_content: true - single_step_tool_use_with_argument_comparison_swe: - responses_api_agents: - tool_simulation_agent: - entrypoint: app.py - resources_server: - type: resources_servers - name: swe_pivot_single_step_tool_use_with_argument_comparison_resources_server - model_server: - type: responses_api_models - name: policy_model - single_step_tool_use_with_argument_comparison_swe_environment_server: - environment_servers: - legacy_agent: - entrypoint: app.py - agent_server: - type: responses_api_agents - name: single_step_tool_use_with_argument_comparison_swe - use_absolute_ip: true -logger: - log_dir: results/nemotron-3-nano-swe-dynamo - wandb_enabled: true - tensorboard_enabled: true - wandb: - project: nemo-rl - name: grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1 diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 31ef02d7895..6f4e7afe4b5 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -635,14 +635,6 @@ def _needs_hf_refit_handshake( return not (nccl_reshard_refit_enabled and not colocated_inference) -def _uses_managed_noncolocated_refit( - nccl_reshard_refit_enabled: bool, - offload_policy_before_refit: bool, -) -> bool: - """Whether non-colocated setup must attach a weight synchronizer.""" - return nccl_reshard_refit_enabled or offload_policy_before_refit - - def _shutdown_completed_nemo_gym_startup( future: Future[tuple[NemoGymShardSet, float]] | None, ) -> None: @@ -1935,11 +1927,8 @@ def init_trtllm(): and checkpoint_engine_config is None ): t0 = time.perf_counter() - # init collective - if _uses_managed_noncolocated_refit( - nccl_reshard_refit_enabled=nccl_reshard_refit_enabled, - offload_policy_before_refit=offload_policy_before_refit, - ): + # init collective; these refits need an attached weight synchronizer. + if nccl_reshard_refit_enabled or offload_policy_before_refit: policy_generation.weight_synchronizer = create_weight_synchronizer( policy=policy, generation=policy_generation, diff --git a/nemo_rl/models/generation/dynamo/__init__.py b/nemo_rl/models/generation/dynamo/__init__.py index 427c1ede100..4fc25d0d3c9 100644 --- a/nemo_rl/models/generation/dynamo/__init__.py +++ b/nemo_rl/models/generation/dynamo/__init__.py @@ -11,16 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - DynamoFrontendArgs, - dynamo_cfg, - dynamo_enabled, -) - -__all__ = [ - "DynamoCfg", - "DynamoFrontendArgs", - "dynamo_cfg", - "dynamo_enabled", -] diff --git a/nemo_rl/models/generation/dynamo/arguments.py b/nemo_rl/models/generation/dynamo/arguments.py deleted file mode 100644 index e5a6abebee5..00000000000 --- a/nemo_rl/models/generation/dynamo/arguments.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Argument construction and log redaction for the managed Dynamo frontend.""" - -import json -from collections.abc import Mapping, Sequence -from typing import Any - -from nemo_rl.models.generation.dynamo.config import DynamoCfg - -_CREDENTIAL_FLAGS = { - "--access-token", - "--api-key", - "--apikey", - "--auth-token", - "--hf-token", - "--password", - "--secret", - "--token", -} - - -def _normalise_flag(flag: str) -> str: - flag = flag.split("=", 1)[0].replace("_", "-") - if flag.startswith("--no-"): - return "--" + flag[5:] - return flag - - -def _serialise_value(value: Any) -> str: - if isinstance(value, (dict, list, tuple)): - return json.dumps(value, separators=(",", ":"), sort_keys=True) - return str(value) - - -class _ArgvBuilder: - def __init__(self) -> None: - self.argv: list[str] = [] - self.sources: dict[str, str] = {} - - def add(self, flag: str, value: Any = None, *, source: str) -> None: - normalised = _normalise_flag(flag) - prior = self.sources.get(normalised) - if prior is not None: - raise ValueError( - f"Dynamo frontend option {normalised} is set by both {prior} and {source}." - ) - self.sources[normalised] = source - if isinstance(value, bool): - self.argv.append(flag if value else f"--no-{flag.removeprefix('--')}") - else: - self.argv.append(flag) - if value is not None: - self.argv.append(_serialise_value(value)) - - def add_raw(self, args: Sequence[str], *, source: str) -> None: - if not args: - return - current_flag: str | None = None - current_has_value = False - for token in args: - if token.startswith("--"): - current_flag = token - current_has_value = "=" in token - normalised = _normalise_flag(token) - if normalised in self.sources: - raise ValueError( - f"Dynamo frontend option {normalised} is set by both " - f"{self.sources[normalised]} and {source}." - ) - self.sources[normalised] = source - elif current_flag is None or current_has_value: - raise ValueError( - f"{source} contains invalid positional argument {token!r}; " - "values must immediately follow one --option." - ) - else: - current_has_value = True - self.argv.extend(args) - - -def build_dynamo_frontend_argv( - *, - host: str, - port: int, - namespace: str, - dynamo_cfg: DynamoCfg, -) -> list[str]: - """Build the managed ``dynamo.frontend`` argument list.""" - builder = _ArgvBuilder() - builder.add("--http-host", host, source="managed runtime") - builder.add("--http-port", port, source="managed runtime") - builder.add("--namespace-prefix", namespace, source="managed runtime") - builder.add("--discovery-backend", "etcd", source="managed runtime") - builder.add("--request-plane", "tcp", source="managed runtime") - builder.add("--event-plane", "nats", source="managed runtime") - builder.add( - "--router-mode", - dynamo_cfg.frontend_args.router_mode, - source="dynamo_cfg.frontend_args.router_mode", - ) - builder.add_raw( - dynamo_cfg.frontend_args.extra_cli_args, - source="dynamo_cfg.frontend_args.extra_cli_args", - ) - return builder.argv - - -def redact_argv(argv: Sequence[str]) -> list[str]: - """Redact values following explicit credential options for safe logging.""" - redacted = list(argv) - for idx, token in enumerate(redacted): - is_sensitive = token.startswith("--") and ( - _normalise_flag(token.lower()) in _CREDENTIAL_FLAGS - ) - if is_sensitive and "=" in token: - redacted[idx] = token.split("=", 1)[0] + "=" - elif is_sensitive and idx + 1 < len(redacted): - if not redacted[idx + 1].startswith("--"): - redacted[idx + 1] = "" - return redacted - - -def redact_environment(env: Mapping[str, str]) -> dict[str, str]: - """Redact credential-like environment values before logging.""" - sensitive_fragments = ("token", "password", "secret", "api_key", "apikey") - return { - key: "" - if any(part in key.lower() for part in sensitive_fragments) - else value - for key, value in env.items() - } diff --git a/nemo_rl/models/generation/dynamo/http_client.py b/nemo_rl/models/generation/dynamo/http_client.py deleted file mode 100644 index 77d318f84a4..00000000000 --- a/nemo_rl/models/generation/dynamo/http_client.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""HTTP helpers for requests to the managed Dynamo frontend.""" - -import json -from typing import Any - -import aiohttp - - -def _decode_json_object(body: bytes) -> dict[str, Any]: - """Decode an HTTP response body using the shared Dynamo error contract.""" - try: - decoded = json.loads(body) - except json.JSONDecodeError: - return { - "status": "error", - "json_decode_error": True, - "raw": body.decode("utf-8", "replace"), - } - if not isinstance(decoded, dict): - return {"status": "error", "raw": repr(decoded)} - return decoded - - -async def async_http_post_json( - url: str, payload: dict[str, Any], timeout_s: float | None -) -> dict[str, Any]: - """POST JSON without blocking the rollout actor event loop.""" - timeout = aiohttp.ClientTimeout(total=timeout_s) - try: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.post(url, json=payload) as response: - body = await response.read() - if response.status >= 400: - return { - "status": "error", - "http_status": response.status, - "raw": body.decode("utf-8", "replace"), - } - except TimeoutError: - # Before ClientError: aiohttp's ServerTimeoutError is both. - return { - "status": "error", - "timeout_error": f"no response within {timeout_s}s", - } - except aiohttp.ClientError as error: - return { - "status": "error", - "transport_error": f"{type(error).__name__}: {error}", - } - return _decode_json_object(body) - - -def format_dynamo_error(response: dict[str, Any]) -> str: - """Format an error object returned by :func:`async_http_post_json`.""" - if "http_status" in response: - return f"HTTP {response['http_status']}: {response.get('raw', '')}" - if "transport_error" in response: - return str(response["transport_error"]) - if "timeout_error" in response: - return str(response["timeout_error"]) - if "raw" in response: - return str(response["raw"]) - return str(response) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index d17cf2d7763..273d595c014 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -41,19 +41,61 @@ _get_free_port_local, _get_node_ip_local, ) -from nemo_rl.models.generation.dynamo.arguments import ( - build_dynamo_frontend_argv, - redact_argv, - redact_environment, -) from nemo_rl.models.generation.dynamo.config import DynamoCfg -from nemo_rl.models.generation.dynamo.venv import get_dynamo_service_executable # The engines register on this component and endpoint (see native_engine). DYNAMO_COMPONENT = "backend" DYNAMO_ENDPOINT = "generate" LOGGER = logging.getLogger(__name__) + + +def frontend_argv( + *, host: str, port: int, namespace: str, dynamo_cfg: DynamoCfg +) -> list[str]: + """The ``dynamo.frontend`` arguments; ``extra_cli_args`` may not repeat one.""" + argv = [ + "--http-host", + host, + "--http-port", + str(port), + "--namespace-prefix", + namespace, + "--discovery-backend", + "etcd", + "--request-plane", + "tcp", + "--event-plane", + "nats", + "--router-mode", + dynamo_cfg.frontend_args.router_mode, + ] + extra = dynamo_cfg.frontend_args.extra_cli_args + for token in extra: + flag = token.split("=", 1)[0].replace("_", "-") + if flag.startswith("--") and flag in argv: + raise ValueError( + f"Dynamo frontend option {flag} is set by both the managed runtime " + "and dynamo_cfg.frontend_args.extra_cli_args." + ) + return argv + list(extra) + + +def _service_executable(name: str) -> str: + """The etcd or nats-server binary: in NEMO_RL_DYNAMO_BIN_DIR when set, else on PATH.""" + bin_dir = os.environ.get("NEMO_RL_DYNAMO_BIN_DIR") + found = shutil.which(name, path=os.path.expanduser(bin_dir) if bin_dir else None) + if found is None: + where = f"NEMO_RL_DYNAMO_BIN_DIR={bin_dir}" if bin_dir else "PATH" + raise FileNotFoundError( + f"{name} is not an executable on {where}. Build the image with " + "--build-arg BUILD_DYNAMO=1, or install etcd and nats-server with " + "docker/dynamo/install.sh and set NEMO_RL_DYNAMO_BIN_DIR to their " + "directory." + ) + return found + + # The KV router's applied-event counter (Dynamo's kvrouter.KV_CACHE_EVENTS_APPLIED), # one sample per label set. _KV_EVENTS_APPLIED = re.compile( @@ -65,10 +107,7 @@ def _managed_namespace() -> str: raw = f"nemo-rl-{os.environ.get('SLURM_JOB_ID', os.getpid())}" - namespace = re.sub(r"[^a-zA-Z0-9_-]+", "-", str(raw)).strip("-_").lower() - if not namespace: - raise ValueError(f"Could not derive a valid Dynamo namespace from {raw!r}.") - return namespace + return re.sub(r"[^a-zA-Z0-9_-]+", "-", raw).strip("-_").lower() class ManagedDynamoRuntime: @@ -96,7 +135,6 @@ def __init__( self._frontend_port = 0 self._manager_env: dict[str, str] = {} self._started = False - self._atexit_registered = False self._etcd_process: subprocess.Popen | None = None self._nats_process: subprocess.Popen | None = None self._frontend_process: subprocess.Popen | None = None @@ -131,35 +169,21 @@ def start(self) -> None: # exception that escapes setup. Keep an interpreter-exit fallback in # addition to the normal explicit shutdown path. atexit.register(self.shutdown) - self._atexit_registered = True self._host = _get_node_ip_local() - used_ports: set[int] = set() - - def allocate_port(*, low: int, high: int) -> int: - port = _get_free_port_local( - low, - high, - max_retries=None, - excluded_ports=used_ports, - ) - used_ports.add(port) - return port - - self._etcd_port = allocate_port( - low=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, - high=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, - ) - self._etcd_peer_port = allocate_port( - low=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, - high=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, + control = ( + DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, + DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, ) - self._nats_port = allocate_port( - low=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, - high=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, - ) - self._frontend_port = allocate_port( - low=DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, - high=DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH, + http = (DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH) + ports: list[int] = [] + for low, high in (control, control, control, http): + ports.append( + _get_free_port_local( + low, high, max_retries=None, excluded_ports=set(ports) + ) + ) + self._etcd_port, self._etcd_peer_port, self._nats_port, self._frontend_port = ( + ports ) self._manager_env = { "ETCD_ENDPOINTS": f"http://{self._host}:{self._etcd_port}", @@ -174,7 +198,7 @@ def allocate_port(*, low: int, high: int) -> int: "DYN_SDK_DISABLE_ANSI_LOGGING": "1", } print( - f" [Dynamo] managed environment={redact_environment(self._manager_env)!r}", + f" [Dynamo] managed environment={self._manager_env!r}", flush=True, ) try: @@ -220,7 +244,7 @@ def _start_etcd(self) -> None: self._etcd_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_etcd_") peer_url = f"http://{self._host}:{self._etcd_peer_port}" command = [ - get_dynamo_service_executable("etcd"), + _service_executable("etcd"), "--listen-client-urls", f"http://0.0.0.0:{self._etcd_port}", "--advertise-client-urls", @@ -260,7 +284,7 @@ def _start_nats(self) -> None: self._nats_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_nats_") self._nats_process = subprocess.Popen( [ - get_dynamo_service_executable("nats-server"), + _service_executable("nats-server"), "-js", "-sd", self._nats_data_dir, @@ -288,7 +312,7 @@ def _wait_for_port( raise RuntimeError(f"{label} did not open port {port}.") def _start_frontend(self) -> None: - argv = build_dynamo_frontend_argv( + argv = frontend_argv( host="0.0.0.0", port=self._frontend_port, namespace=self._namespace, @@ -300,9 +324,7 @@ def _start_frontend(self) -> None: "dynamo.frontend", *argv, ] - print( - f" [Dynamo] launching frontend argv={redact_argv(command)!r}", flush=True - ) + print(f" [Dynamo] launching frontend argv={command!r}", flush=True) frontend_env = self._service_env() dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) frontend_env["VIRTUAL_ENV"] = dynamo_venv @@ -435,7 +457,5 @@ def shutdown(self) -> None: if self._nats_data_dir is not None: shutil.rmtree(self._nats_data_dir, ignore_errors=True) self._nats_data_dir = None - if self._atexit_registered: - atexit.unregister(self.shutdown) - self._atexit_registered = False + atexit.unregister(self.shutdown) self._started = False diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py index 3fe35a7f7fe..5e1346f4b2c 100644 --- a/nemo_rl/models/generation/dynamo/rollout_client.py +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -19,11 +19,9 @@ import logging from typing import Any, Optional +import aiohttp + from nemo_rl.experience.failures import http_status_is_infra -from nemo_rl.models.generation.dynamo.http_client import ( - async_http_post_json, - format_dynamo_error, -) LOGGER = logging.getLogger(__name__) @@ -36,6 +34,52 @@ _HTTP_RETRY_DELAY_S = 1.0 +async def _post_json( + url: str, payload: dict[str, Any], timeout_s: float | None +) -> dict[str, Any]: + """POST JSON; a failure comes back as a dict with ``"status": "error"``.""" + try: + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=timeout_s) + ) as session: + async with session.post(url, json=payload) as response: + body = await response.read() + if response.status >= 400: + return { + "status": "error", + "http_status": response.status, + "raw": body.decode("utf-8", "replace"), + } + except TimeoutError: + # Before ClientError: aiohttp's ServerTimeoutError is both. + return {"status": "error", "timeout_error": f"no response within {timeout_s}s"} + except aiohttp.ClientError as error: + return { + "status": "error", + "transport_error": f"{type(error).__name__}: {error}", + } + try: + decoded = json.loads(body) + except json.JSONDecodeError: + return { + "status": "error", + "json_decode_error": True, + "raw": body.decode("utf-8", "replace"), + } + if not isinstance(decoded, dict): + return {"status": "error", "raw": repr(decoded)} + return decoded + + +def _format_error(response: dict[str, Any]) -> str: + if "http_status" in response: + return f"HTTP {response['http_status']}: {response.get('raw', '')}" + for key in ("transport_error", "timeout_error", "raw"): + if key in response: + return str(response[key]) + return str(response) + + def _is_retryable(response: dict[str, Any]) -> bool: # A timeout is final: the request may still be running, and a retry would # let one hung rollout hold its slot for several request_timeout_s. @@ -81,7 +125,7 @@ def parse_rollout_response(response: dict[str, Any], *, url: str) -> dict[str, A """ if response.get("status") == "error": raise RuntimeError( - f"Dynamo rollout request to {url} failed: {format_dynamo_error(response)}" + f"Dynamo rollout request to {url} failed: {_format_error(response)}" ) engine_data = (response.get("nvext") or {}).get("engine_data") if not isinstance(engine_data, dict) or "rollout" not in engine_data: @@ -98,7 +142,7 @@ async def post_rollout( """POST one rollout, retrying transient errors; returns its ``engine_data``.""" response: dict[str, Any] = {} for attempt in range(1, _HTTP_MAX_ATTEMPTS + 1): - response = await async_http_post_json(url, payload, timeout_s) + response = await _post_json(url, payload, timeout_s) if not _is_retryable(response) or attempt == _HTTP_MAX_ATTEMPTS: break LOGGER.warning( @@ -106,7 +150,7 @@ async def post_rollout( attempt, _HTTP_MAX_ATTEMPTS, _HTTP_RETRY_DELAY_S, - format_dynamo_error(response), + _format_error(response), ) await asyncio.sleep(_HTTP_RETRY_DELAY_S) if "timeout_error" in response: diff --git a/nemo_rl/models/generation/dynamo/venv.py b/nemo_rl/models/generation/dynamo/venv.py deleted file mode 100644 index 5708c8d0676..00000000000 --- a/nemo_rl/models/generation/dynamo/venv.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Resolve the interpreter and service binaries the managed Dynamo runtime launches.""" - -import os -import shutil -from pathlib import Path - -from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env -from nemo_rl.utils.venvs import create_local_venv_on_each_node - -# Every vLLM worker venv carries the locked ai-dynamo release, so the frontend -# runs from the async worker's venv on the same release as every engine. -VLLM_ASYNC_WORKER_FQN = ( - "nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker" -) - - -def get_dynamo_python() -> str: - """Return the vLLM async worker's Python, which carries ``ai-dynamo``. - - The driver launches ``dynamo.frontend`` with it. A uv-managed venv is - resolved as the workers' own is: an existing one is reused as-is, and a - missing one is built once on each node. Re-syncing it here would change a - venv the engines are already running from. - """ - py_executable = get_actor_python_env(VLLM_ASYNC_WORKER_FQN) - if py_executable.startswith("uv"): - return create_local_venv_on_each_node(py_executable, VLLM_ASYNC_WORKER_FQN) - return py_executable - - -def get_dynamo_service_executable(name: str) -> str: - """Return the etcd or nats-server binary the runtime starts. - - Looks in ``NEMO_RL_DYNAMO_BIN_DIR`` when it is set, otherwise on ``PATH``. - """ - bin_dir = os.environ.get("NEMO_RL_DYNAMO_BIN_DIR") - if bin_dir: - executable = Path(bin_dir).expanduser() / name - if executable.is_file() and os.access(executable, os.X_OK): - return str(executable) - raise FileNotFoundError( - f"{name} is not an executable file in NEMO_RL_DYNAMO_BIN_DIR={bin_dir}. " - "Build the image with --build-arg BUILD_DYNAMO=1, or install etcd and " - "nats-server there with docker/dynamo/install.sh." - ) - found = shutil.which(name) - if found is None: - raise FileNotFoundError( - f"{name} is not on PATH. Install etcd and nats-server with " - "docker/dynamo/install.sh and set NEMO_RL_DYNAMO_BIN_DIR to their " - "directory, or build the image with --build-arg BUILD_DYNAMO=1." - ) - return found diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index 1e7c0c418a3..66276fa5d62 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -33,7 +33,6 @@ from nemo_rl.utils.routed_experts_codec import ( decode_routed_experts, encode_routed_experts, - routed_experts_envelope_dtype, ) LOGGER = logging.getLogger(__name__) @@ -195,14 +194,7 @@ def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: """Inverse of :func:`rollout_result_to_wire`; routes keep their own dtype.""" routed_experts = wire["routed_experts"] if routed_experts is not None: - routed_experts = decode_routed_experts( - routed_experts, routed_experts_envelope_dtype(routed_experts) - ) - if len(wire["logprobs"]) != len(wire["token_ids"]): - raise ValueError( - f"rollout result has {len(wire['logprobs'])} logprobs for " - f"{len(wire['token_ids'])} tokens" - ) + routed_experts = decode_routed_experts(routed_experts, dtype=None) return RolloutResult( token_ids=list(wire["token_ids"]), logprobs=list(wire["logprobs"]), diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 3389737cce4..12227bd21df 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -726,12 +726,13 @@ def _start_dynamo_services(self) -> None: from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, ) - from nemo_rl.models.generation.dynamo.venv import get_dynamo_python runtime = ManagedDynamoRuntime( dynamo_cfg=dynamo_cfg(self.cfg), model_name=self.cfg["model_name"], - frontend_python=get_dynamo_python(), + # The workers' venv carries ai-dynamo, so the frontend runs the same + # release as every engine. + frontend_python=self.worker_group.py_executable, ) runtime.start() self._dynamo_runtime = runtime diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 35820c8bfe9..40990504ec6 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -415,12 +415,19 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: stacklevel=1, ) llm_kwargs["compilation_config"] = CompilationConfig(**compilation_config) + self.stat_loggers = ( + [PrometheusStatLogger] + if self.cfg["vllm_cfg"].get("enable_vllm_metrics_logger", False) + else [] + ) if dynamo_enabled(self.cfg): from nemo_rl.models.generation.dynamo.native_engine import ( + SnapshotStatLogger, dynamo_kv_events_config, ) - # Dynamo's KV-aware router indexes the engine's own KV-cache events. + # Dynamo's KV-aware router indexes the engine's own KV-cache events; + # the stat logger feeds its per-engine load signal. llm_kwargs.setdefault( "kv_events_config", dynamo_kv_events_config( @@ -428,6 +435,7 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: prefix_caching=llm_kwargs["enable_prefix_caching"], ), ) + self.stat_loggers.append(SnapshotStatLogger) # AsyncEngineArgs leaves kv_events_config dicts unconverted. if isinstance(llm_kwargs.get("kv_events_config"), dict): from vllm.config import KVEventsConfig @@ -437,18 +445,6 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: ) self.llm_async_engine_args = AsyncEngineArgs(**llm_kwargs) - self.stat_loggers = ( - [PrometheusStatLogger] - if self.cfg["vllm_cfg"].get("enable_vllm_metrics_logger", False) - else [] - ) - if dynamo_enabled(self.cfg): - from nemo_rl.models.generation.dynamo.native_engine import ( - SnapshotStatLogger, - ) - - # Feeds the router's per-engine load signal. - self.stat_loggers.append(SnapshotStatLogger) self.llm = AsyncLLM.from_engine_args( self.llm_async_engine_args, stat_loggers=self.stat_loggers ) diff --git a/nemo_rl/utils/routed_experts_codec.py b/nemo_rl/utils/routed_experts_codec.py index cb05d06b814..53544004c73 100644 --- a/nemo_rl/utils/routed_experts_codec.py +++ b/nemo_rl/utils/routed_experts_codec.py @@ -29,7 +29,7 @@ """ import base64 -from typing import Any, Union +from typing import Any, Optional, Union import torch @@ -67,11 +67,13 @@ def encode_routed_experts(routed_experts: torch.Tensor) -> str: return f"{_MAGIC}:{dtype_name}:{tokens}x{num_layers}x{topk}:{data}" -def decode_routed_experts(payload: Union[str, Any], dtype: torch.dtype) -> torch.Tensor: +def decode_routed_experts( + payload: Union[str, Any], dtype: Optional[torch.dtype] +) -> torch.Tensor: """Decode routed experts into a tensor of the requested dtype. Accepts the base64 envelope produced by ``encode_routed_experts`` or the - legacy nested-list format. + legacy nested-list format. ``dtype=None`` keeps the envelope's own dtype. """ if not isinstance(payload, str): return torch.as_tensor(payload, dtype=dtype) @@ -100,20 +102,14 @@ def decode_routed_experts(payload: Union[str, Any], dtype: torch.dtype) -> torch f"routed_experts envelope has {len(raw) // wire_dtype.itemsize} " f"elements, expected {expected} for shape {shape}." ) + if dtype is None: + dtype = wire_dtype if expected == 0: return torch.empty(shape, dtype=dtype) tensor = torch.frombuffer(raw, dtype=wire_dtype).reshape(shape) return tensor if tensor.dtype == dtype else tensor.to(dtype) -def routed_experts_envelope_dtype(payload: str) -> torch.dtype: - """The dtype a base64 envelope was encoded with.""" - parts = payload.split(":", 2) - if len(parts) != 3 or parts[0] != _MAGIC or parts[1] not in _WIRE_TORCH_DTYPES: - raise ValueError("routed_experts payload is not a valid envelope.") - return _WIRE_TORCH_DTYPES[parts[1]] - - def slice_routed_experts( routed_experts: torch.Tensor, start: int, end: int ) -> torch.Tensor: diff --git a/pyrefly.toml b/pyrefly.toml index 06a05c446bd..1845d9072e8 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -233,13 +233,10 @@ project-includes = [ "nemo_rl/models/generation/__init__.py", "nemo_rl/models/generation/constants.py", "nemo_rl/models/generation/dynamo/__init__.py", - "nemo_rl/models/generation/dynamo/arguments.py", "nemo_rl/models/generation/dynamo/config.py", - "nemo_rl/models/generation/dynamo/http_client.py", "nemo_rl/models/generation/dynamo/managed_runtime.py", "nemo_rl/models/generation/dynamo/native_engine.py", "nemo_rl/models/generation/dynamo/rollout_client.py", - "nemo_rl/models/generation/dynamo/venv.py", "nemo_rl/models/generation/engine_supervisor.py", "nemo_rl/models/generation/fleet_health.py", "nemo_rl/models/generation/generation_router.py", diff --git a/tests/test_suites/disabled.txt b/tests/test_suites/disabled.txt index 83b5213a584..2d43c913cc5 100644 --- a/tests/test_suites/disabled.txt +++ b/tests/test_suites/disabled.txt @@ -102,7 +102,3 @@ research/template_project/tests/test_suites/llm/single_update_1n8g.sh # Full Flow-GRPO recipe requires an externally served GenRM OCR judge. research/flow_grpo/tests/test_suites/flow_grpo_qwen_image_ocr.sh - -# A Dynamo + NeMo-Gym recipe: policy.generation.dynamo does not serve NeMo-Gym -# rollouts yet, and setup refuses the pairing. -tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh diff --git a/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh b/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh deleted file mode 100755 index e9cf7073a7c..00000000000 --- a/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) -source "${SCRIPT_DIR}/common.env" - -# ===== BEGIN CONFIG ===== -NUM_NODES=3 -GPUS_PER_NODE=8 -STEPS_PER_RUN=4 -MAX_STEPS=4 -NUM_RUNS=1 -NUM_MINUTES=240 -USE_GYM_CONTAINER=true -# ===== END CONFIG ===== - -exit_if_max_steps_reached - -cd "${PROJECT_ROOT}" -uv run examples/nemo_gym/run_grpo_nemo_gym.py \ - --config "${CONFIG_PATH}" \ - grpo.max_num_steps="${MAX_STEPS}" \ - logger.log_dir="${LOG_DIR}" \ - logger.wandb_enabled=true \ - logger.wandb.project=nemo-rl \ - logger.wandb.name="${EXP_NAME}" \ - logger.tensorboard_enabled=true \ - "$@" \ - 2>&1 | tee "${RUN_LOG}" - -uv run tests/json_dump_tb_logs.py "${LOG_DIR}" \ - --output_path "${JSON_METRICS}" \ - --require-tag-prefix generation_metrics/ - -last_step=$(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' "${JSON_METRICS}") -if [[ ${last_step} -lt ${MAX_STEPS} ]]; then - echo "[ERROR] Expected step ${MAX_STEPS}, but the last train/loss step is ${last_step}" - exit 1 -fi - -uv run tests/check_metrics.py "${JSON_METRICS}" \ - 'median(data["train/token_mult_prob_error"]) < 1.1' \ - "data['train/token_mult_prob_error']['${MAX_STEPS}'] < 1.1" \ - 'mean(data["train/gen_kl_error"]) < 0.02' - -refit_count=$(grep -c "✅ Ready for refit" "${RUN_LOG}" || true) -cache_invalidation_count=$(grep -c \ - "✅ Invalidated generation backend KV caches after weight update" \ - "${RUN_LOG}" || true) -cache_invalidation_failure_count=$(grep -c \ - "Failed to invalidate generation backend KV caches" \ - "${RUN_LOG}" || true) -if [[ ${cache_invalidation_failure_count} -ne 0 ]]; then - echo "[ERROR] Found ${cache_invalidation_failure_count} cache invalidation failure(s)" - exit 1 -fi -if [[ ${refit_count} -ne ${cache_invalidation_count} ]]; then - echo "[ERROR] Expected one cache invalidation per refit, but found " \ - "${cache_invalidation_count} invalidations for ${refit_count} refits" - exit 1 -fi diff --git a/tests/unit/L0_Unit_Tests_Vllm_1.sh b/tests/unit/L0_Unit_Tests_Vllm_1.sh index 163caa97126..2f2badbcd7e 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_1.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_1.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/L0_Unit_Tests_Vllm_2.sh b/tests/unit/L0_Unit_Tests_Vllm_2.sh index 211b1086e0f..e44a28f472a 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_2.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_2.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/L0_Unit_Tests_Vllm_3.sh b/tests/unit/L0_Unit_Tests_Vllm_3.sh index 362044f0c49..3b5300a828b 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_3.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_3.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --extra modelopt bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --extra modelopt bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index c161e7289d2..75be2faa00e 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -14,7 +14,6 @@ import math import os -import warnings from concurrent.futures import Future from contextlib import ExitStack, contextmanager from pathlib import Path @@ -2633,7 +2632,6 @@ def test_initial_refit_completes_before_async_collection_starts( mock_grpo_components, ) -> None: master_config = mock_grpo_components["master_config"] - master_config.policy["generation"]["backend"] = "vllm" master_config.policy["generation"]["colocated"]["enabled"] = False master_config.grpo.max_num_steps = 1 master_config.grpo.val_period = 0 @@ -2672,7 +2670,6 @@ def record_refit(*args, **kwargs): def test_async_grpo_awaits_resume_after_refit_failure(mock_grpo_components) -> None: master_config = mock_grpo_components["master_config"] - master_config.policy["generation"]["backend"] = "vllm" master_config.policy["generation"]["colocated"]["enabled"] = False master_config.grpo.max_num_steps = 1 master_config.grpo.val_period = 0 @@ -4731,10 +4728,7 @@ def init_collective(self, *_args, **_kwargs): assert master_config.grpo.skip_reference_policy_logprobs_calculation is True -@pytest.mark.parametrize("configured_gym_log_dir", [None, "logs/nemo_gym"]) -def test_setup_starts_nemo_gym_for_trtllm( - monkeypatch, mock_grpo_components, configured_gym_log_dir -): +def test_setup_starts_nemo_gym_for_trtllm(monkeypatch, mock_grpo_components): """Guard the TRT-LLM NeMo-Gym startup path in shared GRPO setup.""" from nemo_rl.algorithms import grpo as grpo_mod @@ -4839,8 +4833,6 @@ def prepare_refit_info(self, _state): }, } master_config.env = {"should_use_nemo_gym": True} - if configured_gym_log_dir is not None: - master_config.env["nemo_gym"] = {"nemo_gym_log_dir": configured_gym_log_dir} master_config.loss_fn = ClippedPGLossConfig(reference_policy_kl_penalty=0.0) master_config.grpo.val_period = 0 master_config.grpo.batch_multiplier = 1 @@ -4851,26 +4843,13 @@ def prepare_refit_info(self, _state): dataset = MagicMock() dataset.__len__ = MagicMock(return_value=1) tokenizer = MagicMock() - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = grpo_mod.setup(master_config, tokenizer, dataset, None) + result = grpo_mod.setup(master_config, tokenizer, dataset, None) assert result[2] is nemo_gym_shard_set + # Gym server logs go under the run's log directory. assert master_config.env["nemo_gym"]["nemo_gym_log_dir"] == ( "/tmp/grpo-test-results/nemo_gym" ) - log_dir_warnings = [ - str(warning.message) - for warning in caught - if "nemo_gym_log_dir" in str(warning.message) - ] - if configured_gym_log_dir is None: - assert log_dir_warnings == [] - else: - # A configured log dir is still replaced, but no longer silently. - assert len(log_dir_warnings) == 1 - assert "'logs/nemo_gym'" in log_dir_warnings[0] - assert "'/tmp/grpo-test-results/nemo_gym'" in log_dir_warnings[0] build_nemo_gym_actors.assert_called_once_with( master_config.env, base_urls=["http://trtllm.example/v1"], @@ -8382,28 +8361,6 @@ def test_needs_hf_refit_handshake(backend, nccl_reshard, colocated, expected): assert _needs_hf_refit_handshake(backend, nccl_reshard, colocated) is expected -@pytest.mark.parametrize( - ("nccl_reshard", "offload_policy", "expected"), - [ - (False, False, False), - (False, True, True), - (True, False, True), - ], -) -def test_noncolocated_refit_synchronizer_selection( - nccl_reshard, offload_policy, expected -): - from nemo_rl.algorithms import grpo as grpo_mod - - assert ( - grpo_mod._uses_managed_noncolocated_refit( - nccl_reshard_refit_enabled=nccl_reshard, - offload_policy_before_refit=offload_policy, - ) - is expected - ) - - def test_grpo_train_shuts_down_environments_after_failure(): task_to_env = {"nemo_gym": MagicMock()} val_task_to_env = task_to_env diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 0624535b30c..e552d545578 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -11,16 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path + import pytest -from pydantic import ValidationError +import yaml from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.arguments import ( - build_dynamo_frontend_argv, - redact_argv, - redact_environment, -) from nemo_rl.models.generation.dynamo.config import DynamoCfg, dynamo_cfg +from nemo_rl.models.generation.dynamo.managed_runtime import frontend_argv class _Tokenizer: @@ -45,23 +43,24 @@ def _vllm_config(**overrides) -> dict: return config -def test_dynamo_true_alone_is_a_complete_config() -> None: - config = configure_generation_config(_vllm_config(dynamo=True), _Tokenizer()) - - # Dynamo serves the AsyncLLM, so the switch selects the async engine. - # Rollouts read token IDs only, so the engine skips its tokenizer as natively. - assert config["vllm_cfg"]["async_engine"] is True - assert config["vllm_cfg"]["skip_tokenizer_init"] is True - assert dynamo_cfg(config) == DynamoCfg() - assert dynamo_cfg(config).frontend_args.router_mode == "kv" +_EXEMPLAR = ( + Path(__file__).resolve().parents[4] + / "examples/configs/grpo_math_1B_dynamo_single_controller.yaml" +) -def test_without_dynamo_the_vllm_config_is_untouched() -> None: - config = configure_generation_config(_vllm_config(), _Tokenizer()) +@pytest.mark.parametrize("dynamo", [False, True]) +def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: + config = configure_generation_config(_vllm_config(dynamo=dynamo), _Tokenizer()) - assert config["vllm_cfg"]["async_engine"] is False + # Dynamo serves the AsyncLLM, so the switch selects the async engine. + # Rollouts read token IDs only, so the engine skips its tokenizer as natively. + assert config["vllm_cfg"]["async_engine"] is dynamo assert config["vllm_cfg"]["skip_tokenizer_init"] is True assert "dynamo_cfg" not in config + # The exemplar spells out exactly the defaults. + exemplar = yaml.safe_load(_EXEMPLAR.read_text())["policy"]["generation"] + assert dynamo_cfg(config) == DynamoCfg() == dynamo_cfg(exemplar) @pytest.mark.parametrize( @@ -118,18 +117,11 @@ def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( configure_generation_config(config, _Tokenizer()) -def test_dynamo_cfg_rejects_unknown_keys() -> None: - with pytest.raises(ValidationError, match="worker_args"): - dynamo_cfg({"dynamo_cfg": {"worker_args": {}}}) - - def test_frontend_argv_is_runtime_owned() -> None: cfg = DynamoCfg.model_validate( {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} ) - argv = build_dynamo_frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg - ) + argv = frontend_argv(host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg) assert argv[argv.index("--router-mode") + 1] == "kv" assert argv[-2:] == ["--foo", "1"] @@ -137,14 +129,6 @@ def test_frontend_argv_is_runtime_owned() -> None: {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} ) with pytest.raises(ValueError, match="--http-port is set by both"): - build_dynamo_frontend_argv( + frontend_argv( host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=clashing ) - - -def test_redaction_hides_credentials() -> None: - assert redact_argv(["frontend", "--api-key", "secret"])[2] == "" - assert redact_environment({"HF_TOKEN": "secret", "NCCL_DEBUG": "INFO"}) == { - "HF_TOKEN": "", - "NCCL_DEBUG": "INFO", - } diff --git a/tests/unit/models/generation/test_dynamo_http_client.py b/tests/unit/models/generation/test_dynamo_http_client.py deleted file mode 100644 index df8eabd9344..00000000000 --- a/tests/unit/models/generation/test_dynamo_http_client.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio - -import aiohttp -import pytest - -from nemo_rl.models.generation.dynamo import http_client -from nemo_rl.models.generation.dynamo.rollout_client import _is_retryable - - -class _AsyncResponse: - def __init__(self, body: bytes, status: int = 200): - self._body = body - self.status = status - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - return None - - async def read(self) -> bytes: - return self._body - - -class _AsyncSession: - def __init__(self, response=None, error=None): - self._response = response - self._error = error - self.requests = [] - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - return None - - def post(self, url, *, json): - self.requests.append((url, json)) - if self._error is not None: - raise self._error - return self._response - - -@pytest.mark.parametrize( - ("body", "status", "expected", "retryable"), - [ - (b'{"status":"ok"}', 200, {"status": "ok"}, False), - ( - b"not-json", - 200, - {"status": "error", "json_decode_error": True, "raw": "not-json"}, - True, - ), - ( - b"busy", - 503, - {"status": "error", "http_status": 503, "raw": "busy"}, - True, - ), - ], -) -def test_async_http_post_json_error_contract( - monkeypatch, body, status, expected, retryable -) -> None: - session = _AsyncSession(_AsyncResponse(body, status)) - monkeypatch.setattr( - http_client.aiohttp, - "ClientSession", - lambda **kwargs: session, - ) - - # No deadline is the default (dynamo_cfg.request_timeout_s: null). - response = asyncio.run( - http_client.async_http_post_json( - "http://worker/route", {"value": 1}, timeout_s=None - ) - ) - - assert response == expected - assert _is_retryable(response) is retryable - if response.get("status") == "error": - assert expected["raw"] in http_client.format_dynamo_error(response) - assert session.requests == [("http://worker/route", {"value": 1})] - - -@pytest.mark.parametrize( - ("error", "key", "retryable"), - [ - (aiohttp.ClientConnectionError("refused"), "transport_error", True), - # One request_timeout_s bounds a rollout: a timeout is not retried. - (TimeoutError("slow"), "timeout_error", False), - (aiohttp.ServerTimeoutError("slow read"), "timeout_error", False), - ], -) -def test_async_http_transport_errors(monkeypatch, error, key, retryable) -> None: - session = _AsyncSession(error=error) - monkeypatch.setattr( - http_client.aiohttp, - "ClientSession", - lambda **kwargs: session, - ) - - response = asyncio.run( - http_client.async_http_post_json( - "http://worker/route", {"value": 1}, timeout_s=3 - ) - ) - - assert response["status"] == "error" - assert _is_retryable(response) is retryable - assert key in response - assert response[key] in http_client.format_dynamo_error(response) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 524416b344b..2c5c4dc3541 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -16,15 +16,14 @@ import subprocess import sys from pathlib import Path -from unittest.mock import MagicMock import pytest -from nemo_rl.models.generation.dynamo import venv as dynamo_venv from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, _managed_namespace, + _service_executable, ) @@ -139,54 +138,20 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( assert "DYN_SYSTEM_PORT" not in service_env -def test_dynamo_python_resolves_the_vllm_worker_venv_as_workers_do( - monkeypatch, -) -> None: - built = [] - monkeypatch.setattr( - dynamo_venv, "get_actor_python_env", lambda fqn: "uv run --locked --extra vllm" - ) - # The workers' resolver: reuses an existing venv, builds a missing one per node. - monkeypatch.setattr( - dynamo_venv, - "create_local_venv_on_each_node", - lambda py_executable, venv_name: built.append((py_executable, venv_name)) - or "/venvs/vllm/bin/python", - ) - - assert dynamo_venv.get_dynamo_python() == "/venvs/vllm/bin/python" - assert built == [ - ("uv run --locked --extra vllm", dynamo_venv.VLLM_ASYNC_WORKER_FQN) - ] - - -def test_dynamo_python_passes_an_interpreter_through(monkeypatch) -> None: - monkeypatch.setattr( - dynamo_venv, "get_actor_python_env", lambda fqn: "/usr/bin/python3" - ) - monkeypatch.setattr( - dynamo_venv, - "create_local_venv_on_each_node", - MagicMock(side_effect=AssertionError("no venv for an interpreter")), - ) - - assert dynamo_venv.get_dynamo_python() == "/usr/bin/python3" - - def test_dynamo_services_resolve_from_bin_dir_then_path(monkeypatch, tmp_path) -> None: etcd = tmp_path / "etcd" etcd.write_text("#!/bin/sh\n") etcd.chmod(0o755) monkeypatch.setenv("NEMO_RL_DYNAMO_BIN_DIR", str(tmp_path)) - assert dynamo_venv.get_dynamo_service_executable("etcd") == str(etcd) + assert _service_executable("etcd") == str(etcd) with pytest.raises(FileNotFoundError, match="NEMO_RL_DYNAMO_BIN_DIR"): - dynamo_venv.get_dynamo_service_executable("nats-server") + _service_executable("nats-server") monkeypatch.delenv("NEMO_RL_DYNAMO_BIN_DIR") monkeypatch.setenv("PATH", str(tmp_path)) - assert dynamo_venv.get_dynamo_service_executable("etcd") == str(etcd) - with pytest.raises(FileNotFoundError, match="not on PATH"): - dynamo_venv.get_dynamo_service_executable("nats-server") + assert _service_executable("etcd") == str(etcd) + with pytest.raises(FileNotFoundError, match="on PATH"): + _service_executable("nats-server") def test_stop_process_escalates_its_process_group(monkeypatch) -> None: diff --git a/tests/unit/models/generation/test_dynamo_rollout_client.py b/tests/unit/models/generation/test_dynamo_rollout_client.py index 138b4120017..6aeab3a07a6 100644 --- a/tests/unit/models/generation/test_dynamo_rollout_client.py +++ b/tests/unit/models/generation/test_dynamo_rollout_client.py @@ -13,6 +13,7 @@ # limitations under the License. import asyncio +import aiohttp import pytest from nemo_rl.models.generation.dynamo import rollout_client @@ -22,6 +23,67 @@ ) +class _Response: + def __init__(self, status: int, body: bytes): + self.status = status + self._body = body + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + async def read(self) -> bytes: + return self._body + + +class _Session: + def __init__(self, reply): + self._reply = reply + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + def post(self, url, *, json): + if isinstance(self._reply, Exception): + raise self._reply + return _Response(*self._reply) + + +@pytest.mark.parametrize( + ("reply", "error", "retryable"), + [ + ((200, b'{"status":"ok"}'), None, False), + ((200, b"not-json"), "not-json", True), + ((503, b"busy"), "HTTP 503: busy", True), + (aiohttp.ClientConnectionError("refused"), "refused", True), + # One request_timeout_s bounds a rollout: a timeout is not retried. + (TimeoutError("slow"), "no response within", False), + (aiohttp.ServerTimeoutError("slow read"), "no response within", False), + ], +) +def test_post_json_reports_failures_as_error_dicts( + monkeypatch, reply, error, retryable +) -> None: + monkeypatch.setattr( + rollout_client.aiohttp, "ClientSession", lambda **kwargs: _Session(reply) + ) + + # No deadline is the default (dynamo_cfg.request_timeout_s: null). + response = asyncio.run( + rollout_client._post_json("http://frontend/v1/completions", {}, timeout_s=None) + ) + + assert (response.get("status") == "error") is (error is not None) + if error is not None: + assert error in rollout_client._format_error(response) + assert rollout_client._is_retryable(response) is retryable + + @pytest.mark.parametrize( "response, match", [ @@ -73,7 +135,7 @@ async def fake_post(url, payload, timeout_s): calls.append(payload) return responses[len(calls) - 1] - monkeypatch.setattr(rollout_client, "async_http_post_json", fake_post) + monkeypatch.setattr(rollout_client, "_post_json", fake_post) monkeypatch.setattr(rollout_client, "_HTTP_RETRY_DELAY_S", 0.0) rollout = post_rollout("http://frontend/v1/completions", {}, timeout_s=1.0) diff --git a/tests/unit/models/generation/test_swe1_dynamo_config.py b/tests/unit/models/generation/test_swe1_dynamo_config.py deleted file mode 100644 index 9bd87f972b1..00000000000 --- a/tests/unit/models/generation/test_swe1_dynamo_config.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. - -from pathlib import Path -from types import SimpleNamespace - -import pytest -from omegaconf import OmegaConf - -from nemo_rl.environments.nemo_gym import setup_nemo_gym_config -from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.config import dynamo_cfg -from nemo_rl.utils.config import load_config, register_omegaconf_resolvers - -REPO_ROOT = Path(__file__).resolve().parents[4] -RECIPE = ( - REPO_ROOT / "examples/configs/recipes/llm/" - "grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml" -) -DRIVER = ( - REPO_ROOT / "tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh" -) - - -def _load_recipe() -> dict: - register_omegaconf_resolvers() - return OmegaConf.to_container(load_config(RECIPE), resolve=True) - - -def test_public_swe_recipe_has_supported_topology_and_telemetry( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("HF_HOME", "/hf-home") - config = _load_recipe() - generation = config["policy"]["generation"] - - class Tokenizer: - pad_token_id = 0 - eos_token_id = 1 - - tokenizer = Tokenizer() - configured_generation = configure_generation_config(generation, tokenizer) - config["policy"]["generation"] = configured_generation - setup_nemo_gym_config( - SimpleNamespace( - policy=config["policy"], env=config["env"], data=config["data"] - ), - tokenizer, - ) - vllm_cfg = configured_generation["vllm_cfg"] - assert configured_generation["backend"] == "vllm" - assert configured_generation["dynamo"] is True - assert vllm_cfg["async_engine"] is True - assert vllm_cfg["expose_http_server"] is True - - assert config["policy"]["model_name"] == ( - "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" - ) - assert config["cluster"]["gpus_per_node"] == 8 - assert config["cluster"]["num_nodes"] == 3 - assert config["cluster"]["segment_size"] == 1 - assert config["grpo"]["max_num_steps"] == 4 - assert config["grpo"]["async_grpo"]["enabled"] is True - assert config["grpo"]["async_grpo"]["in_flight_weight_updates"] is False - assert ( - config["grpo"]["async_grpo"]["recompute_kv_cache_after_weight_updates"] is True - ) - assert generation["colocated"]["resources"] == { - "gpus_per_node": 8, - "num_nodes": 1, - } - assert vllm_cfg["tensor_parallel_size"] * vllm_cfg["pipeline_parallel_size"] == 4 - assert vllm_cfg["expert_parallel_size"] == 4 - assert dynamo_cfg(configured_generation).frontend_args.router_mode == "kv" - assert vllm_cfg["enable_vllm_metrics_logger"] is True - assert vllm_cfg["load_format"] == "dummy" - assert config["env"]["nemo_gym"]["config_paths"][-1].endswith( - "swe_pivot_single_step_tool_use_with_argument_comparison.yaml" - ) - assert ( - config["env"]["nemo_gym"]["policy_model"]["responses_api_models"]["vllm_model"][ - "chat_template_kwargs" - ]["force_nonempty_content"] - is True - ) - assert ( - config["env"]["nemo_gym"]["single_step_tool_use_with_argument_comparison_swe"][ - "responses_api_agents" - ]["tool_simulation_agent"]["resources_server"]["name"] - == "swe_pivot_single_step_tool_use_with_argument_comparison_resources_server" - ) - assert config["logger"]["wandb_enabled"] is True - assert config["logger"]["tensorboard_enabled"] is True - assert config["logger"]["wandb"]["project"] == "nemo-rl" - assert config["data"]["train"]["data_path"].endswith( - "/superv3_data/swe1/train-split.jsonl" - ) - assert config["data"]["validation"]["data_path"].endswith( - "/superv3_data/swe1/val-split.jsonl" - ) - - -def test_recipe_and_driver_have_no_removed_swe_modes() -> None: - recipe_text = RECIPE.read_text(encoding="utf-8") - text = recipe_text + DRIVER.read_text(encoding="utf-8") - for forbidden in ( - "/lustre/", - "/path/to", - "/home/", - "jthomson", - "openhands", - "container_formatter", - "SIF_", - "effort_levels", - "hsg_r2", - "USES_SANDBOX", - ): - assert forbidden.lower() not in text.lower() - assert "load_format:" not in recipe_text - assert "--require-tag-prefix generation_metrics/" in text - assert "project: nemo-rl" in text - assert "Expected step ${MAX_STEPS}" in text - assert 'median(data["train/token_mult_prob_error"]) < 1.1' in text - assert "data['train/token_mult_prob_error']['${MAX_STEPS}'] < 1.1" in text - assert 'mean(data["train/gen_kl_error"]) < 0.02' in text - assert "Expected one cache invalidation per refit" in text diff --git a/tests/unit/test_dependency_pins.py b/tests/unit/test_dependency_pins.py index 17c14b55aae..864ee4d40ba 100644 --- a/tests/unit/test_dependency_pins.py +++ b/tests/unit/test_dependency_pins.py @@ -144,21 +144,3 @@ def test_torch_split_shims_are_still_needed() -> None: "(nemo_rl/data_plane/adapters/tq_mooncake_checkpoint.py) - drop the " "check outright; the pre-2.13 `names` comparison cannot be restored" ) - - -def test_vllm_extra_carries_base_dynamo_release() -> None: - """Dynamo serving runs NeMo RL's own vLLM workers, so the vllm extra carries the - released Dynamo runtime but never ai-dynamo[vllm], which pins Dynamo's vLLM.""" - extras = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text())["project"][ - "optional-dependencies" - ] - requirements = { - canonicalize_name(req.name): req - for req in (Requirement(dep) for dep in extras["vllm"]) - } - - dynamo, runtime = requirements["ai-dynamo"], requirements["ai-dynamo-runtime"] - assert not dynamo.extras, f"ai-dynamo must not take extras, got {dynamo.extras}" - assert dynamo.specifier == runtime.specifier, ( - "ai-dynamo and ai-dynamo-runtime must pin the same release" - ) diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 4539d34b951..13757622a81 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -378,8 +378,15 @@ def test_dynamo_frontend_args_without_router_reset_states_pass(): check_outdated_config(_dynamo_frontend_config({"router_mode": "kv"})) -def test_generation_without_dynamo_cfg_passes(): - check_outdated_config({"policy": {"generation": {"backend": "vllm"}}}) +@pytest.mark.parametrize( + "generation", + [ + {"backend": "vllm"}, + {"backend": "vllm", "dynamo": True, "dynamo_cfg": {"frontend_args": {}}}, + ], +) +def test_generation_without_removed_dynamo_keys_passes(generation): + check_outdated_config({"policy": {"generation": generation}}) # ============================================================================ @@ -391,40 +398,32 @@ def _generation_config(**generation) -> dict: return {"policy": {"generation": generation}} -def test_dynamo_backend_is_rejected_with_the_migration(): - with pytest.raises( - ValueError, match=r"backend=vllm and policy\.generation\.dynamo=true" - ): - check_outdated_config(_generation_config(backend="dynamo")) - - @pytest.mark.parametrize( - "dynamo_cfg, removed", + "generation, removed", [ - ({"engine": "vllm"}, r"dynamo_cfg\.engine"), - ({"control_timeout_s": 600}, r"dynamo_cfg\.control_timeout_s"), - ({"metrics_include_prefixes": None}, r"dynamo_cfg\.metrics_include_prefixes"), - ({"metrics_exclude_prefixes": None}, r"dynamo_cfg\.metrics_exclude_prefixes"), - ({"worker_args": {"extra_cli_args": []}}, r"dynamo_cfg\.worker_args was"), - ({"worker_args": {"reasoning_parser": None}}, r"dynamo_cfg\.worker_args was"), + ({"backend": "dynamo"}, r"backend=vllm and policy\.generation\.dynamo=true"), + ({"dynamo_cfg": {"engine": "vllm"}}, r"dynamo_cfg\.engine"), + ({"dynamo_cfg": {"control_timeout_s": 600}}, r"dynamo_cfg\.control_timeout_s"), + ( + {"dynamo_cfg": {"metrics_include_prefixes": None}}, + r"dynamo_cfg\.metrics_include_prefixes", + ), ( - {"frontend_args": {"tokenizer_cache": False}}, + {"dynamo_cfg": {"metrics_exclude_prefixes": None}}, + r"dynamo_cfg\.metrics_exclude_prefixes", + ), + ( + {"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}, + r"dynamo_cfg\.worker_args was", + ), + ( + {"dynamo_cfg": {"frontend_args": {"tokenizer_cache": False}}}, r"frontend_args\.tokenizer_cache was", ), ], ) -def test_removed_dynamo_cfg_keys_are_rejected(dynamo_cfg, removed): +def test_removed_dynamo_config_is_rejected(generation, removed): with pytest.raises(ValueError, match=removed): check_outdated_config( - _generation_config(backend="vllm", dynamo=True, dynamo_cfg=dynamo_cfg) + _generation_config(**{"backend": "vllm", "dynamo": True, **generation}) ) - - -def test_vllm_backend_with_dynamo_passes(): - check_outdated_config( - _generation_config( - backend="vllm", - dynamo=True, - dynamo_cfg={"frontend_args": {"router_mode": "kv"}}, - ) - ) From 4aedf28e550b203bdbc5104268e6a7e9b2cfce54 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:58:56 -0700 Subject: [PATCH 16/82] Revert "refactor(dynamo): serve only NeMo RL rollouts; general OpenAI path moves to #4979" This reverts commit ed3ee35380 on #4979's side of the stack: #4979 keeps the general OpenAI path (parsers, chat templates, build_sampling_params, media, prompt logprobs, PARSERS_V2, the skip_tokenizer_init override) that #4905 now leaves out. Conflicts with #4905's later commits keep those commits' changes: request_timeout_s defaults to None, frontend tokenizer knobs stay removed, arguments.py and redaction stay deleted, the api-key rejection keeps its Gym-free reason, and dynamo/__init__.py stays header-only. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 28 +- docs/guides/dynamo-generation.md | 32 +- ...grpo_math_1B_dynamo_single_controller.yaml | 13 + nemo_rl/models/generation/__init__.py | 14 +- nemo_rl/models/generation/dynamo/config.py | 134 ++++++- .../generation/dynamo/managed_runtime.py | 6 +- .../models/generation/dynamo/native_engine.py | 370 +++++++++++++++++- nemo_rl/utils/outdated_config_checks.py | 10 +- .../models/generation/test_dynamo_config.py | 83 +++- .../generation/test_dynamo_native_engine.py | 311 ++++++++++++++- .../unit/utils/test_outdated_config_checks.py | 2 +- 11 files changed, 957 insertions(+), 46 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 4b7e95277e7..8d8384d989c 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -25,12 +25,23 @@ which is the loop that owns the `AsyncLLM`. vLLM. The adapter, `nemo_rl/models/generation/dynamo/native_engine.py`, implements its `LLMEngine` over the worker's `AsyncLLM`: -- `generate` serves only the rollouts NeMo RL itself sends, which carry an - annotation, and refuses other requests. The worker's own `_serve_rollout` runs - each rollout exactly as native `generate_async` does, and - `request_pipeline.rollout_result` builds its result, routed experts included. - The result travels in `engine_data`, and the driver assembles the same output - with `request_pipeline.rollout_output`. +- `generate` maps Dynamo's token request onto vLLM `SamplingParams` the way + vLLM's OpenAI server does, and streams token deltas back. Rollouts that NeMo RL + itself sends carry an annotation. For those, the engine uses the same code as + native `generate_async`: + - the worker's `_rollout_max_new_tokens` and `_rollout_sampling_params` build + the request; + - `request_pipeline.rollout_result` builds the result, routed experts + included. + + The result travels in `engine_data`, and the driver assembles the same + output with `request_pipeline.rollout_output`. + + A request with media carries the frontend's tokens, with one unexpanded + placeholder per item, and the media URLs. The adapter loads the media with + vLLM's media connector. It then builds the processed prompt with the engine's + renderer (`process_for_engine_async`), the same step that follows rendering + on the native endpoint. - `abort` cancels the vLLM request on client disconnect. - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It @@ -80,9 +91,10 @@ imports `dynamo.common.backend`, `dynamo.llm.exceptions` and `dynamo._core`. What a Dynamo upgrade has to re-check: - the `dynamo.common.backend` contract (`LLMEngine`, `Worker`, `WorkerConfig`, - publisher); + publisher and logprob helpers); - the request fields of Dynamo's preprocessed request; -- the frontend's `nvext` extensions and annotation forwarding. +- the frontend's `nvext` extensions and annotation forwarding; +- the parser names the native settings map to (`dynamo/config.py`). The one vLLM↔Dynamo wire coupling is vLLM's KV-event msgpack, which Dynamo's router decodes. A change there degrades routing, not serving. The managed diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 52d65c6cfa3..3e9372e660e 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -51,6 +51,8 @@ shows them all: |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | | `request_timeout_s` | null | Deadline for one NeMo RL rollout through the frontend. null waits as long as generation takes, as native does; on the SingleController, `async_rl.rollout_failure.native.generation_timeout_s` bounds both paths | +| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [Chat settings](#chat-settings) | +| `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | @@ -71,7 +73,31 @@ native path: token IDs, log probabilities, truncation, routed experts with their `r3_routed_experts_*` counts, and `gen_leader_worker_idx`. Both paths build it with `nemo_rl/models/generation/vllm/request_pipeline.py`. -The engines serve only these rollouts; the frontend refuses other requests. +OpenAI clients of the frontend get what vLLM's own OpenAI server applies: +- the model's generation-config sampling defaults; +- `n > 1`; +- guided decoding (`guided_*`, `response_format`); +- `bad_words_token_ids` and `allowed_token_ids`; +- prompt log probabilities (`nvext.extra_fields: ["prompt_logprobs"]`); +- vLLM's queue limits, rejected with HTTP 503; +- a 400 naming the "maximum context length" for a prompt that leaves no room + for output. The engine, not the frontend, checks the context, so it checks + the prompt it actually runs. + +## Chat settings + +The native server's chat settings carry over to the frontend: + +| Native setting (`vllm_cfg`) | Under Dynamo | +|---|---| +| `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | +| `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | +| `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | +| `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | + +Where the behavior differs from the native endpoint: when a request sets +`chat_template_kwargs.force_nonempty_content`, Dynamo reports a reasoning-only +turn as content. ## Not supported @@ -91,6 +117,8 @@ Startup rejects these with `dynamo: true`: on 0.0.0.0 and checks no bearer token. - `frontend_args.router_mode: direct`: it needs a worker ID on every request, and NeMo RL's rollouts name none. +- Parser plugins with no Dynamo parser, and the chat settings above that the + frontend has no counterpart for. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. @@ -136,5 +164,5 @@ is the same run with every `dynamo_cfg` default spelled out. `backend: dynamo` launched Dynamo's own vLLM engine and was removed. Set `backend: vllm` and `dynamo: true`; `vllm_cfg` and `vllm_kwargs` keep their meaning. Startup rejects the old backend and each removed `dynamo_cfg` key -(`engine`, `control_timeout_s`, `metrics_*_prefixes`, `worker_args`, +(`engine`, `control_timeout_s`, `metrics_*_prefixes`, `worker_args.extra_cli_args`, `frontend_args.tokenizer*`) with what replaces it. diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index a1d89235d1e..00f66367d10 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -18,6 +18,19 @@ policy: # generation takes, as native generate_async does; on the SingleController, # async_rl.rollout_failure.native.generation_timeout_s bounds both paths. request_timeout_s: null + worker_args: + # Dynamo's built-in parser names. null follows the native server's + # vllm_cfg.http_server_serving_chat_kwargs (tool_parser, reasoning_parser). + tool_call_parser: null + reasoning_parser: null + exclude_tools_when_tool_choice_none: false + enable_structural_tag: false + structural_tag_scope: auto + structural_tag_schema: auto + # A chat template that replaces the model's own. null follows + # vllm_cfg.http_server_serving_chat_kwargs.chat_template. + custom_jinja_template: null + endpoint_types: [chat, completions] frontend_args: # kv: route each request to the engine holding most of its prefix. router_mode: kv diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 08d28438400..3f42409b38a 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -17,7 +17,12 @@ from transformers import PreTrainedTokenizerBase -from nemo_rl.models.generation.dynamo.config import dynamo_cfg, dynamo_enabled +from nemo_rl.models.generation.dynamo.config import ( + check_dynamo_serving_chat_kwargs, + dynamo_cfg, + dynamo_enabled, + dynamo_parsers, +) from nemo_rl.models.generation.interfaces import GenerationConfig, GenerationInterface from nemo_rl.models.generation.trtllm.config import ( TrtllmConfig, @@ -181,8 +186,11 @@ def configure_generation_config( # vision parameters; freezing alone does not exclude them from refit. config["vllm_cfg"].setdefault("load_format", "auto" if is_eval else "dummy") if dynamo_enabled(config): - # Validate before any Ray allocation. + # Validate before any Ray allocation; dynamo_parsers also rejects + # parser plugins that Dynamo's Rust frontend cannot run. dynamo_cfg(config) + dynamo_parsers(config) + check_dynamo_serving_chat_kwargs(config) if config["vllm_cfg"].get("http_server_api_key_required"): raise ValueError( "policy.generation.vllm_cfg.http_server_api_key_required is not " @@ -272,6 +280,8 @@ def configure_generation_config( is_eval or config["stop_strings"] is not None or config["vllm_cfg"].get("expose_http_server", None) + # Dynamo serves OpenAI clients as the exposed server does. + or dynamo_enabled(config) ): config["vllm_cfg"]["skip_tokenizer_init"] = False else: diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 824280c3edf..c587d4d24cd 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -21,7 +21,33 @@ from typing import Any, Literal, Mapping -from pydantic import BaseModel, Field, PositiveFloat +from pydantic import BaseModel, Field, PositiveFloat, model_validator + + +class DynamoWorkerArgs(BaseModel, extra="forbid"): + """How each vLLM engine registers with Dynamo (``WorkerConfig`` fields).""" + + # Dynamo's built-in parser names (dynamo-parsers). None follows the native + # server's vllm_cfg.http_server_serving_chat_kwargs (see dynamo_parsers). + tool_call_parser: str | None = None + reasoning_parser: str | None = None + # False, as vLLM's OpenAIServingChat (the native server) defaults it. + exclude_tools_when_tool_choice_none: bool = False + enable_structural_tag: bool = False + structural_tag_scope: Literal["auto", "always"] = "auto" + structural_tag_schema: Literal["auto", "strict"] = "auto" + # A Jinja chat template file that replaces the model's own. None follows + # vllm_cfg.http_server_serving_chat_kwargs.chat_template. + custom_jinja_template: str | None = None + endpoint_types: list[Literal["chat", "completions"]] = ["chat", "completions"] + + @model_validator(mode="after") + def _validate_endpoint_types(self) -> "DynamoWorkerArgs": + if not self.endpoint_types: + raise ValueError("endpoint_types must contain at least one endpoint") + if len(self.endpoint_types) != len(set(self.endpoint_types)): + raise ValueError("endpoint_types must not contain duplicates") + return self class DynamoFrontendArgs(BaseModel, extra="forbid"): @@ -50,9 +76,115 @@ class DynamoCfg(BaseModel, extra="forbid"): # as generation takes, as native generate_async does; on the SingleController, # async_rl.rollout_failure.native.generation_timeout_s bounds both paths. request_timeout_s: PositiveFloat | None = None + worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) +# vLLM parser names whose Dynamo built-in parser goes by another name. nano_v3 is +# NeMo RL's vLLM plugin: DeepSeek-R1 reasoning parsing, with the whole output as +# content when chat_template_kwargs.enable_thinking is false, as Dynamo's +# nemotron_v3 parser does. +_DYNAMO_REASONING_PARSER_NAMES = {"nano_v3": "nemotron_v3"} +_DYNAMO_TOOL_PARSER_NAMES = {"openai": "harmony"} + + +def _dynamo_parser( + configured: str | None, + native: str | None, + plugin: str | None, + renames: Mapping[str, str], + *, + kind: str, + worker_arg: str, +) -> str | None: + if configured is not None or native is None: + return configured + if native in renames: + return renames[native] + if plugin: + raise ValueError( + f"policy.generation.vllm_cfg.{kind}_parser_plugin loads a Python parser " + "into vLLM's OpenAI server, which Dynamo's Rust frontend cannot run. Set " + f"policy.generation.dynamo_cfg.worker_args.{worker_arg} to the Dynamo " + f"built-in parser for {native!r}." + ) + return native + + +def dynamo_parsers( + generation_config: Mapping[str, Any], +) -> tuple[str | None, str | None]: + """The ``(tool_call_parser, reasoning_parser)`` Dynamo's frontend applies. + + ``dynamo_cfg.worker_args`` names them explicitly. Otherwise they follow the + native server's ``vllm_cfg.http_server_serving_chat_kwargs``, renamed where + Dynamo calls the same parser differently. A parser plugin without such a + rename has no Dynamo equivalent and is rejected. + """ + worker_args = dynamo_cfg(generation_config).worker_args + vllm_cfg = generation_config["vllm_cfg"] + serving = vllm_cfg.get("http_server_serving_chat_kwargs") or {} + tool_call_parser = _dynamo_parser( + worker_args.tool_call_parser, + serving.get("tool_parser"), + vllm_cfg.get("tool_parser_plugin"), + _DYNAMO_TOOL_PARSER_NAMES, + kind="tool", + worker_arg="tool_call_parser", + ) + reasoning_parser = _dynamo_parser( + worker_args.reasoning_parser, + serving.get("reasoning_parser"), + vllm_cfg.get("reasoning_parser_plugin"), + _DYNAMO_REASONING_PARSER_NAMES, + kind="reasoning", + worker_arg="reasoning_parser", + ) + return tool_call_parser, reasoning_parser + + +def check_dynamo_serving_chat_kwargs(generation_config: Mapping[str, Any]) -> None: + """Reject native chat-server settings Dynamo's frontend has no counterpart for. + + The frontend renders templates and parses tool calls itself: it always + parses ``tool_choice="auto"`` calls when a tool parser is set, and has no + content-format setting. The native defaults (``enable_auto_tools: true``, + ``chat_template_content_format: auto``) match it; other values would + silently change what the policy sees. + """ + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + content_format = serving.get("chat_template_content_format") + if content_format not in (None, "auto"): + raise ValueError( + "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." + f"chat_template_content_format={content_format!r} is not supported " + "with policy.generation.dynamo=true: Dynamo's frontend renders chat " + "templates itself and has no content-format setting. Remove it or " + "set it to 'auto'." + ) + if serving.get("enable_auto_tools") is False: + raise ValueError( + "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." + "enable_auto_tools=false is not supported with " + "policy.generation.dynamo=true: Dynamo's frontend always parses " + "tool_choice='auto' calls when a tool parser is set. Remove it, or " + "remove the tool parser." + ) + + +def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: + """The chat template the frontend renders with: a file path or literal Jinja. + + ``dynamo_cfg.worker_args.custom_jinja_template``, else the native server's + ``http_server_serving_chat_kwargs.chat_template``; None uses the model's own. + """ + configured = dynamo_cfg(generation_config).worker_args.custom_jinja_template + if configured is not None: + return configured + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + return serving.get("chat_template") + + def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" return bool(generation_config.get("dynamo")) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index 273d595c014..ed499e54512 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -158,7 +158,7 @@ def engine_env(self) -> dict[str, str]: if not self._started: raise RuntimeError("Managed Dynamo runtime has not been started") # An OS-assigned system port makes each engine serve its own model-card - # files to a frontend on another node. + # files (a custom chat template included) to a frontend on another node. return {**self._manager_env, "DYN_SYSTEM_PORT": "0"} def start(self) -> None: @@ -190,6 +190,10 @@ def start(self) -> None: "NATS_SERVER": f"nats://{self._host}:{self._nats_port}", "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", + # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. + # Its v2 parser path preserves engine_data, which carries every NeMo + # RL rollout's result. + "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", # A canary request would stall behind a refit pause; engine liveness diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index b422d21c11e..6c80cfc80a4 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -23,14 +23,18 @@ from __future__ import annotations import asyncio +import functools +import hashlib import json import logging import os +import tempfile +import time from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional import torch -from dynamo._core import Context +from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names from dynamo.common.backend import ( EngineConfig, GenerateChunk, @@ -40,13 +44,33 @@ Worker, WorkerConfig, ) +from dynamo.common.backend.logprobs import ( + extract_from_completion_output, + extract_prompt_logprobs_from_completion_output, + parse_logprob_options, +) from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource -from dynamo.llm.exceptions import InvalidArgument +from dynamo.llm.exceptions import HttpError, InvalidArgument +from vllm import envs as vllm_envs +from vllm.entrypoints.chat_utils import load_chat_template +from vllm.exceptions import GracefulHTTPError, VLLMClientError from vllm.inputs import TokensPrompt +from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY +from vllm.sampling_params import ( + RequestOutputKind, + SamplingParams, + StructuredOutputsParams, +) +from vllm.transformers_utils.processor import get_video_processor_cls_name from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase -from nemo_rl.models.generation.dynamo.config import DynamoCfg +from nemo_rl.models.generation.dynamo.config import ( + DynamoCfg, + dynamo_cfg, + dynamo_chat_template, + dynamo_parsers, +) from nemo_rl.models.generation.dynamo.managed_runtime import ( DYNAMO_COMPONENT, DYNAMO_ENDPOINT, @@ -61,6 +85,29 @@ LOGGER = logging.getLogger(__name__) +# The vLLM modality each key of Dynamo's multi_modal_data holds. +_DYNAMO_MEDIA_MODALITIES = { + "image_url": "image", + "video_url": "video", + "audio_url": "audio", +} + +# Filled from the model's generation config when a request leaves them unset, +# as vLLM's OpenAI server fills them. +_GENERATION_CONFIG_KEYS = ( + "repetition_penalty", + "temperature", + "top_k", + "top_p", + "min_p", +) +_SAMPLING_KEYS = ( + *_GENERATION_CONFIG_KEYS, + "seed", + "presence_penalty", + "frequency_penalty", +) + def dynamo_kv_events_config( cfg: DynamoCfg, *, prefix_caching: bool @@ -98,6 +145,111 @@ def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: return None if rollout is None else json.loads(rollout) +def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: + """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. + + Dynamo sets at most one constraint. The decoding backend is the engine's + own (``structured_outputs_config``), as for the native server. + """ + structural_tag = guided_decoding.get("structural_tag") + if isinstance(structural_tag, dict): + structural_tag = json.dumps(structural_tag) + return StructuredOutputsParams( + json=guided_decoding.get("json"), + regex=guided_decoding.get("regex"), + choice=guided_decoding.get("choice"), + grammar=guided_decoding.get("grammar"), + whitespace_pattern=guided_decoding.get("whitespace_pattern"), + structural_tag=structural_tag, + ) + + +def build_sampling_params( + request: GenerateRequest, + defaults: dict[str, Any], + *, + output_room: int, + overrides: Optional[dict[str, Any]] = None, + output_kind: RequestOutputKind = RequestOutputKind.DELTA, +) -> SamplingParams: + """Map an OpenAI-shaped Dynamo request onto vLLM SamplingParams. + + As vLLM's OpenAI server builds them: the model's generation-config + ``defaults`` for what the request leaves unset, and ``max_tokens`` capped + by the context. Stop strings are applied by Dynamo's Rust detokenizer, so + only token-level stops reach vLLM unless ``overrides`` sets them. + + Args: + request: Dynamo's preprocessed request. + defaults: ``ModelConfig.get_diff_sampling_param()``. + output_room: The context left after the prompt the engine runs, which + media expansion or a prefix splice can make longer than the + request's ``token_ids``. + overrides: SamplingParams fields that replace the request's. + output_kind: How vLLM reports outputs. + """ + sampling = request.get("sampling_options") or {} + passthrough = (request.get("extra_args") or {}).get("sampling_options") or {} + stop = request.get("stop_conditions") or {} + n = sampling.get("n") or 1 + if sampling.get("best_of") not in (None, n): + raise InvalidArgument("best_of is not supported by vLLM's V1 engine") + + kwargs: dict[str, Any] = { + key: defaults[key] for key in _GENERATION_CONFIG_KEYS if key in defaults + } + kwargs.update( + {key: sampling[key] for key in _SAMPLING_KEYS if sampling.get(key) is not None} + ) + kwargs["n"] = n + # As vLLM's get_max_tokens: the generation config's max_tokens applies only + # when the request leaves max_tokens unset. + requested_max_tokens = stop.get("max_tokens") + if requested_max_tokens is None: + requested_max_tokens = defaults.get("max_tokens") + kwargs["max_tokens"] = min( + limit + for limit in (max(1, output_room), requested_max_tokens) + if limit is not None + ) + if stop.get("min_tokens") is not None: + kwargs["min_tokens"] = stop["min_tokens"] + if stop.get("ignore_eos") is not None: + kwargs["ignore_eos"] = stop["ignore_eos"] + stop_ids = set(stop.get("stop_token_ids") or ()) | set( + stop.get("stop_token_ids_hidden") or () + ) + if stop_ids: + kwargs["stop_token_ids"] = sorted(stop_ids) + if sampling.get("guided_decoding"): + kwargs["structured_outputs"] = structured_outputs(sampling["guided_decoding"]) + for key in ("allowed_token_ids", "logprob_token_ids"): + if passthrough.get(key) is not None: + kwargs[key] = passthrough[key] + + logprobs, prompt_logprobs = parse_logprob_options( + request.get("output_options") or {} + ) + # Explicit logprob_token_ids replace top-k selection; vLLM rejects both. + if logprobs is not None and "logprob_token_ids" not in kwargs: + kwargs["logprobs"] = logprobs + if prompt_logprobs is not None: + kwargs["prompt_logprobs"] = prompt_logprobs + kwargs["detokenize"] = False + kwargs.update(overrides or {}) + params = SamplingParams(**kwargs, output_kind=output_kind) + if passthrough.get("bad_words_token_ids") is not None: + # vLLM only tokenizes bad_words strings into this field; a token-in + # client sends the token IDs themselves. + if not hasattr(params, "_bad_words_token_ids"): + raise RuntimeError( + "vLLM SamplingParams has no _bad_words_token_ids; update the " + "bad_words_token_ids passthrough for this vLLM" + ) + params._bad_words_token_ids = passthrough["bad_words_token_ids"] + return params + + class SnapshotStatLogger(StatLoggerBase): """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" @@ -142,6 +294,7 @@ def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int self._leader_worker_idx = leader_worker_idx self._llm = host.llm self._max_model_len = self._llm.vllm_config.model_config.max_model_len + self._defaults = self._llm.model_config.get_diff_sampling_param() @classmethod async def from_args(cls, argv=None): @@ -183,14 +336,84 @@ async def start(self, worker_id: int) -> EngineConfig: async def generate( self, request: GenerateRequest, context: Context ) -> AsyncGenerator[GenerateChunk, None]: - rollout = _rollout_request(request) - if rollout is None: + try: + async for chunk in self._generate(request, context): + yield chunk + except VLLMClientError as error: + # vLLM's request errors are not ValueErrors; the native server + # answers them with a 4xx that carries the message. + raise InvalidArgument(str(error)) from error + + async def _generate( + self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + if request.get("prompt_embeds"): raise InvalidArgument( - "NeMo RL's vLLM engines serve only NeMo RL rollouts through Dynamo " - f"(requests annotated {ROLLOUT_ANNOTATION!r})." + "Embedding prompts are not served through Dynamo by NeMo RL's " + "vLLM engines." ) - async for chunk in self._generate_rollout(request, rollout, context): - yield chunk + rollout = _rollout_request(request) + if rollout is not None: + async for chunk in self._generate_rollout(request, rollout, context): + yield chunk + return + engine_input = await self._engine_input(request) + prompt_len = len(engine_input["prompt_token_ids"]) + self._check_prompt_fits(prompt_len) + sampling_params = build_sampling_params( + request, self._defaults, output_room=self._max_model_len - prompt_len + ) + try: + # vLLM's queue limits (max_num_queued_reqs / max_num_queued_tokens), + # checked before every request as the native server checks them. + self._llm.check_admission(sampling_params.n, context.id()) + except GracefulHTTPError as error: + raise HttpError(int(error.http_status), error.message) from error + logprobs_requested, prompt_logprobs_requested = parse_logprob_options( + request.get("output_options") or {} + ) + tokenizer = self._llm.tokenizer + completion_tokens: dict[int, int] = {} + prompt_output = None + async for output in self._llm.generate( + engine_input, sampling_params, context.id() + ): + # Under DELTA, vLLM sends every prompt logprob once, at the end of + # prefill; later outputs carry an empty list. + if getattr(output, "prompt_logprobs", None): + prompt_output = output + for completion in output.outputs: + token_ids = list(completion.token_ids) + completion_tokens[completion.index] = completion_tokens.get( + completion.index, 0 + ) + len(token_ids) + if not token_ids and completion.finish_reason is None: + continue + chunk: GenerateChunk = { + "token_ids": token_ids, + "index": completion.index, + } + log_probs, top_logprobs = extract_from_completion_output( + completion, 0, tokenizer=tokenizer, include_bytes=True + ) + if log_probs is not None: + chunk["log_probs"] = log_probs + if logprobs_requested: + chunk["top_logprobs"] = top_logprobs + if completion.finish_reason is not None: + chunk["finish_reason"] = completion.finish_reason + chunk["completion_usage"] = _usage( + output, sum(completion_tokens.values()) + ) + if prompt_logprobs_requested is not None and prompt_output: + prompt_logprobs = ( + extract_prompt_logprobs_from_completion_output( + prompt_output, tokenizer=tokenizer + ) + ) + if prompt_logprobs is not None: + chunk["engine_data"] = {"prompt_logprobs": prompt_logprobs} + yield chunk async def _generate_rollout( self, request: GenerateRequest, rollout: dict[str, Any], context: Context @@ -226,6 +449,86 @@ async def _generate_rollout( chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) yield chunk + async def _load_media( + self, request: GenerateRequest + ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: + """The request's media and their UUIDs by modality, as vLLM's chat parser loads them. + + Dynamo passes each media URL through (data: URIs included); the engine + fetches and decodes it with vLLM's own media connector and settings. + """ + media = request.get("multi_modal_data") + if not media: + return None + model_config = self._llm.model_config + mm_config = model_config.multimodal_config + connector = MEDIA_CONNECTOR_REGISTRY.load( + vllm_envs.VLLM_MEDIA_CONNECTOR, + media_io_kwargs=request.get("media_io_kwargs") + or (mm_config.media_io_kwargs if mm_config else None), + allowed_local_media_path=model_config.allowed_local_media_path, + allowed_media_domains=model_config.allowed_media_domains, + ) + fetch = { + "image": connector.fetch_image_async, + "audio": connector.fetch_audio_async, + } + if "video_url" in media: + fetch["video"] = functools.partial( + connector.fetch_video_async, + video_processor=get_video_processor_cls_name(model_config), + ) + uuids = request.get("multi_modal_uuids") or {} + mm_data: dict[str, list[Any]] = {} + mm_uuids: dict[str, list[Any]] = {} + for key, items in media.items(): + modality = _DYNAMO_MEDIA_MODALITIES.get(key) + if modality is None: + raise InvalidArgument(f"Unsupported media type {key!r}") + if any("Url" not in item for item in items): + raise InvalidArgument( + f"{key} items must carry their URL: NeMo RL's vLLM engines load " + "media themselves" + ) + try: + mm_data[modality] = list( + await asyncio.gather( + *(fetch[modality](item["Url"]) for item in items) + ) + ) + except ValueError as error: + raise InvalidArgument(f"Cannot load {key}: {error}") from error + mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) + return mm_data, mm_uuids + + async def _engine_input(self, request: GenerateRequest) -> dict[str, Any]: + """The processed vLLM prompt for the request, as the native chat endpoint builds it. + + The frontend's tokens carry one unexpanded placeholder per media item; + vLLM's processor expands them and processes the media, exactly as after + the native endpoint renders and tokenizes the messages. + """ + prompt = TokensPrompt(prompt_token_ids=list(request["token_ids"])) + media = await self._load_media(request) + if media is not None: + prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media + if request.get("mm_processor_kwargs"): + prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] + return await self._llm.renderer.process_for_engine_async(prompt, time.time()) + + def _check_prompt_fits(self, prompt_len: int) -> None: + """Refuse a prompt that leaves no room for output, as the native endpoint does. + + OpenAI clients read "context length" in a 400 as a length refusal rather + than an engine failure. + """ + if prompt_len >= self._max_model_len: + raise InvalidArgument( + f"Prompt length ({prompt_len}) fills or exceeds this model's " + f"maximum context length ({self._max_model_len}). No room for " + "output tokens." + ) + async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) @@ -258,6 +561,37 @@ def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: } +def _check_parser_name(name: Optional[str], known: list[str], field: str) -> None: + if name is not None and name not in known: + raise ValueError( + f"Dynamo has no {field.replace('_', ' ')} named {name!r}; set " + f"policy.generation.dynamo_cfg.worker_args.{field} to one of " + f"{sorted(known)}." + ) + + +def _chat_template_file(chat_template: Optional[str]) -> Optional[str]: + """A file holding ``chat_template``, which Dynamo reads by path. + + Resolved as the native server resolves it (vLLM's ``load_chat_template``): + a file, a vLLM built-in template name, or literal Jinja. Named by content, + so every engine on a node, and every restart, shares one file. + """ + if chat_template is None: + return None + content = load_chat_template(chat_template) + digest = hashlib.sha256(content.encode()).hexdigest()[:16] + path = os.path.join(tempfile.gettempdir(), f"nemo_rl_chat_template_{digest}.jinja") + if not os.path.exists(path): + # Written aside and renamed, so a concurrent reader never sees a part. + with tempfile.NamedTemporaryFile( + "w", dir=os.path.dirname(path), suffix=".jinja", delete=False + ) as template_file: + template_file.write(content) + os.replace(template_file.name, path) + return path + + def start_dynamo_endpoint( host: "VllmAsyncGenerationWorkerImpl", *, @@ -277,17 +611,31 @@ def start_dynamo_endpoint( publishes = kv_events is not None and kv_events.enable_kv_cache_events # The Dynamo functional overlays check the router applied these. print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) + worker_args = dynamo_cfg(host.cfg).worker_args + tool_call_parser, reasoning_parser = dynamo_parsers(host.cfg) + _check_parser_name(tool_call_parser, get_tool_parser_names(), "tool_call_parser") + _check_parser_name( + reasoning_parser, get_reasoning_parser_names(), "reasoning_parser" + ) config = WorkerConfig( namespace=namespace, component=DYNAMO_COMPONENT, endpoint=DYNAMO_ENDPOINT, model_name=host.model_name, served_model_name=host.model_name, - # NeMo RL's rollouts go to /v1/completions only. - endpoint_types="completions", + endpoint_types=",".join(worker_args.endpoint_types), discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], request_plane=engine_env["DYN_REQUEST_PLANE"], event_plane=engine_env["DYN_EVENT_PLANE"], + custom_jinja_template=_chat_template_file(dynamo_chat_template(host.cfg)), + tool_call_parser=tool_call_parser, + reasoning_parser=reasoning_parser, + exclude_tools_when_tool_choice_none=( + worker_args.exclude_tools_when_tool_choice_none + ), + structural_tag_mode="on" if worker_args.enable_structural_tag else "off", + structural_tag_scope=worker_args.structural_tag_scope, + structural_tag_schema=worker_args.structural_tag_schema, ) return asyncio.get_running_loop().create_task( Worker(NativeVllmEngine(host, leader_worker_idx), config).run() diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 28c8b30f1e9..ccd3447652f 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -208,10 +208,6 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: "control_timeout_s": "refit and pause use the vLLM backend's own controls", "metrics_include_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", "metrics_exclude_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", - "worker_args": ( - "Dynamo serves only NeMo RL's own rollouts, which need no chat parsers or " - "templates; engines are configured through vllm_cfg and vllm_kwargs" - ), } _REMOVED_DYNAMO_FRONTEND_KEYS = ( "tokenizer", @@ -260,6 +256,12 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: "removed: NeMo RL's rollouts send token IDs, which the " "frontend never tokenizes." ) + worker_args = dynamo_cfg.get("worker_args") + if isinstance(worker_args, dict) and "extra_cli_args" in worker_args: + problems.append( + "policy.generation.dynamo_cfg.worker_args.extra_cli_args was " + "removed: engines are configured through vllm_cfg and vllm_kwargs." + ) if problems: raise ValueError(" ".join(problems)) diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index e552d545578..866aeb17f4f 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -15,9 +15,15 @@ import pytest import yaml +from pydantic import ValidationError from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.config import DynamoCfg, dynamo_cfg +from nemo_rl.models.generation.dynamo.config import ( + DynamoCfg, + dynamo_cfg, + dynamo_chat_template, + dynamo_parsers, +) from nemo_rl.models.generation.dynamo.managed_runtime import frontend_argv @@ -53,10 +59,10 @@ def _vllm_config(**overrides) -> dict: def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: config = configure_generation_config(_vllm_config(dynamo=dynamo), _Tokenizer()) - # Dynamo serves the AsyncLLM, so the switch selects the async engine. - # Rollouts read token IDs only, so the engine skips its tokenizer as natively. + # Dynamo serves the AsyncLLM, so the switch selects the async engine, and it + # serves OpenAI clients, which need the engine's tokenizer. assert config["vllm_cfg"]["async_engine"] is dynamo - assert config["vllm_cfg"]["skip_tokenizer_init"] is True + assert config["vllm_cfg"]["skip_tokenizer_init"] is (not dynamo) assert "dynamo_cfg" not in config # The exemplar spells out exactly the defaults. exemplar = yaml.safe_load(_EXEMPLAR.read_text())["policy"]["generation"] @@ -117,6 +123,13 @@ def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( configure_generation_config(config, _Tokenizer()) +def test_dynamo_cfg_rejects_unknown_keys() -> None: + with pytest.raises(ValidationError, match="extra_cli_args"): + dynamo_cfg({"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}) + with pytest.raises(ValidationError, match="duplicates"): + dynamo_cfg({"dynamo_cfg": {"worker_args": {"endpoint_types": ["chat"] * 2}}}) + + def test_frontend_argv_is_runtime_owned() -> None: cfg = DynamoCfg.model_validate( {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} @@ -132,3 +145,65 @@ def test_frontend_argv_is_runtime_owned() -> None: frontend_argv( host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=clashing ) + + +def _serving(**serving_chat_kwargs) -> dict: + return _vllm_config( + dynamo=True, + vllm_cfg={ + "async_engine": True, + "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 1, + "http_server_serving_chat_kwargs": serving_chat_kwargs, + }, + ) + + +def test_parsers_follow_the_native_server_under_dynamos_names() -> None: + assert dynamo_parsers(_serving(tool_parser="hermes", reasoning_parser="qwen3")) == ( + "hermes", + "qwen3", + ) + # NeMo RL's nano_v3 vLLM plugin is Dynamo's nemotron_v3 parser. + nano_v3 = _serving(reasoning_parser="nano_v3") + nano_v3["vllm_cfg"]["reasoning_parser_plugin"] = "nano_v3_reasoning_parser.py" + assert dynamo_parsers(nano_v3) == (None, "nemotron_v3") + assert dynamo_parsers(_serving(tool_parser="openai")) == ("harmony", None) + # dynamo_cfg names them explicitly. + explicit = _serving(tool_parser="hermes") + explicit["dynamo_cfg"] = {"worker_args": {"tool_call_parser": "qwen3_coder"}} + assert dynamo_parsers(explicit) == ("qwen3_coder", None) + + +def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> None: + config = _serving(tool_parser="my_parser") + config["vllm_cfg"]["tool_parser_plugin"] = "my_parser.py" + + with pytest.raises(ValueError, match="worker_args.tool_call_parser"): + configure_generation_config(config, _Tokenizer()) + + +def test_chat_template_follows_the_native_server() -> None: + config = _serving(chat_template="templates/chat.jinja") + assert dynamo_chat_template(config) == "templates/chat.jinja" + assert dynamo_chat_template(_serving()) is None + + +@pytest.mark.parametrize( + ("serving_chat_kwargs", "error"), + [ + ({"chat_template_content_format": "string"}, "no content-format setting"), + ({"enable_auto_tools": False}, "always parses"), + ], +) +def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( + serving_chat_kwargs, error +) -> None: + with pytest.raises(ValueError, match=error): + configure_generation_config(_serving(**serving_chat_kwargs), _Tokenizer()) + # The native defaults are what the frontend does anyway. + configure_generation_config( + _serving(chat_template_content_format="auto", enable_auto_tools=True), + _Tokenizer(), + ) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index d1dda2bfeb4..66b441facf9 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -18,20 +18,27 @@ """ import asyncio +import base64 +import io import json +from http import HTTPStatus from types import SimpleNamespace import numpy as np import pytest import torch +from PIL import Image pytestmark = pytest.mark.vllm pytest.importorskip("dynamo._core") vllm = pytest.importorskip("vllm") -from dynamo.llm.exceptions import InvalidArgument # noqa: E402 +from dynamo.llm.exceptions import HttpError, InvalidArgument # noqa: E402 +from vllm.exceptions import QueueOverflowError # noqa: E402 +from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.dynamo.config import DynamoWorkerArgs # noqa: E402 from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) @@ -48,6 +55,11 @@ MAX_MODEL_LEN = 16 +EOS = 2 +# The fake processor expands each image placeholder to IMAGE_TOKENS image tokens. +IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS = 90, 91, 3 + + class _Context: def id(self) -> str: return "request-0" @@ -77,18 +89,57 @@ def _output(prompt, *completions, prompt_routed_experts=None, prompt_logprobs=No ) +class _FakeRenderer: + """vLLM's processing step: expands each image placeholder, as a VLM processor does.""" + + def __init__(self) -> None: + self.prompts = [] + + async def process_for_engine_async( + self, prompt, arrival_time, *, skip_mm_cache=False + ): + self.prompts.append((prompt, skip_mm_cache)) + if "multi_modal_data" not in prompt: + return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} + tokens, placeholders = [], [] + for token in prompt["prompt_token_ids"]: + if token == IMAGE_PLACEHOLDER: + placeholders.append(PlaceholderRange(len(tokens), IMAGE_TOKENS)) + tokens.extend([IMAGE_TOKEN] * IMAGE_TOKENS) + else: + tokens.append(token) + return { + "type": "multimodal", + "prompt_token_ids": tokens, + "mm_kwargs": {"image": [object() for _ in placeholders]}, + "mm_placeholders": {"image": placeholders}, + } + + class _FakeLLM: """Yields scripted RequestOutputs and records what it was asked to generate.""" - def __init__(self, outputs=()): + def __init__(self, outputs=(), *, defaults=None, admission_error=None): self.outputs = list(outputs) self.calls = [] + self.admission_error = admission_error self.tokenizer = None + self.renderer = _FakeRenderer() + self.model_config = SimpleNamespace( + get_diff_sampling_param=lambda: dict(defaults or {}), + multimodal_config=None, + allowed_local_media_path="", + allowed_media_domains=None, + ) self.vllm_config = SimpleNamespace( model_config=SimpleNamespace(max_model_len=MAX_MODEL_LEN), kv_events_config=None, ) + def check_admission(self, n, request_id): + if self.admission_error is not None: + raise self.admission_error + async def generate(self, prompt, sampling_params, request_id): self.calls.append((prompt, sampling_params, request_id)) for output in self.outputs: @@ -126,6 +177,10 @@ def _rollout_request(prompt: list[int], **rollout) -> dict: return {"token_ids": prompt, "annotations": [ROLLOUT_ANNOTATION + json.dumps(spec)]} +def _openai_request(**fields) -> dict: + return {"token_ids": [1, 2, 3, 4], **fields} + + def test_rollouts_are_served_like_generate_async() -> None: final = _output([1, 2, 3], _completion([10, 11, 12], "stop")) llm = _FakeLLM([final]) @@ -202,6 +257,119 @@ def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: assert rollout["token_ids"] == [] and rollout["truncated"] is False +def test_openai_requests_map_onto_sampling_params_as_vllms_server_does() -> None: + params = native_engine.build_sampling_params( + _openai_request( + sampling_options={"top_k": 5, "seed": 7, "n": 2}, + stop_conditions={ + "max_tokens": 100, + "stop_token_ids": [9], + "stop_token_ids_hidden": [2], + }, + output_options={"logprobs": 2}, + extra_args={ + "sampling_options": { + "allowed_token_ids": [5, 6], + "bad_words_token_ids": [[7]], + } + }, + ), + # The model's generation config, for what the request leaves unset. + {"temperature": 0.6, "top_k": 20, "max_tokens": 9}, + output_room=MAX_MODEL_LEN - 4, + ) + + assert (params.temperature, params.top_k, params.seed, params.n) == (0.6, 5, 7, 2) + # The request's max_tokens, capped by the context; the generation config's + # max_tokens applies only when the request leaves it unset. + assert params.max_tokens == MAX_MODEL_LEN - 4 + unset = native_engine.build_sampling_params( + _openai_request(), {"max_tokens": 9}, output_room=MAX_MODEL_LEN - 4 + ) + assert unset.max_tokens == 9 + assert sorted(params.stop_token_ids) == [2, 9] + assert params.logprobs == 2 + assert params.allowed_token_ids == [5, 6] + assert params._bad_words_token_ids == [[7]] + assert params.detokenize is False + + +def test_guided_decoding_maps_onto_structured_outputs() -> None: + params = native_engine.build_sampling_params( + _openai_request( + sampling_options={"guided_decoding": {"json": {"type": "object"}}} + ), + {}, + output_room=MAX_MODEL_LEN - 4, + ) + + assert params.structured_outputs.json == {"type": "object"} + + +def test_best_of_is_rejected_as_by_vllm() -> None: + with pytest.raises(InvalidArgument, match="best_of"): + native_engine.build_sampling_params( + _openai_request(sampling_options={"n": 1, "best_of": 2}), + {}, + output_room=MAX_MODEL_LEN - 4, + ) + + +def test_openai_requests_stream_each_choice_by_index() -> None: + prompt = [1, 2, 3, 4] + llm = _FakeLLM( + [ + _output(prompt, _completion([5], index=0), _completion([7], index=1)), + _output(prompt, _completion([6], "stop", index=0)), + _output(prompt, _completion([8], "length", index=1)), + ] + ) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + chunks = _collect(engine, _openai_request(sampling_options={"n": 2})) + + assert [(c["index"], c["token_ids"]) for c in chunks] == [ + (0, [5]), + (1, [7]), + (0, [6]), + (1, [8]), + ] + assert [c.get("finish_reason") for c in chunks] == [None, None, "stop", "length"] + assert chunks[-1]["completion_usage"]["completion_tokens"] == 4 + + +def test_prompt_logprobs_are_returned_in_engine_data() -> None: + prompt = [1, 2, 3, 4] + llm = _FakeLLM( + [ + # Under DELTA, vLLM sends prompt logprobs once, at the end of prefill, + # and an empty list on every later output. + _output( + prompt, + _completion([5]), + prompt_logprobs=[None, {2: _logprob(-0.25)}, None, None], + ), + _output(prompt, _completion([6], "stop"), prompt_logprobs=[]), + ] + ) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + chunks = _collect(engine, _openai_request(output_options={"prompt_logprobs": 0})) + + assert chunks[-1]["engine_data"]["prompt_logprobs"][1]["2"]["logprob"] == -0.25 + + +def test_queue_limits_reject_with_the_engines_status() -> None: + llm = _FakeLLM(admission_error=QueueOverflowError()) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(HttpError) as raised: + _collect(engine, _openai_request()) + + assert raised.value.code == HTTPStatus.SERVICE_UNAVAILABLE + assert llm.calls == [] + + def _start_worker_config(monkeypatch, host) -> native_engine.WorkerConfig: """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" configs = [] @@ -233,22 +401,69 @@ async def start(): return config -def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: - config = _start_worker_config(monkeypatch, _host(_FakeLLM())) +def test_every_worker_arg_reaches_the_dynamo_worker_config( + monkeypatch, tmp_path +) -> None: + template = tmp_path / "chat.jinja" + template.write_text("{{ messages }}") + worker_args = { + "tool_call_parser": "qwen3_coder", + "reasoning_parser": "nemotron_nano", + "exclude_tools_when_tool_choice_none": True, + "enable_structural_tag": True, + "structural_tag_scope": "always", + "structural_tag_schema": "strict", + "custom_jinja_template": str(template), + "endpoint_types": ["chat"], + } + assert set(worker_args) == set(DynamoWorkerArgs.model_fields) + host = _host(_FakeLLM(), dynamo_cfg={"worker_args": worker_args}) + + config = _start_worker_config(monkeypatch, host) assert config.namespace == "ns" - # NeMo RL's rollouts go to /v1/completions; no chat surface is registered. - assert config.endpoint_types == "completions" + assert config.endpoint_types == "chat" + assert config.structural_tag_mode == "on" + with open(config.custom_jinja_template) as template_file: + assert template_file.read() == "{{ messages }}" + for key in worker_args.keys() - { + "endpoint_types", + "enable_structural_tag", + "custom_jinja_template", + }: + assert getattr(config, key) == worker_args[key], key + + +def test_the_native_servers_chat_settings_reach_the_frontend(monkeypatch) -> None: + host = _host(_FakeLLM()) + host.cfg["vllm_cfg"] = { + "max_model_len": MAX_MODEL_LEN, + "reasoning_parser_plugin": "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", + "http_server_serving_chat_kwargs": { + "tool_parser": "qwen3_coder", + "reasoning_parser": "nano_v3", + # Literal Jinja, which the native server also accepts. + "chat_template": "{{ messages }}\n", + }, + } + config = _start_worker_config(monkeypatch, host) -def test_requests_other_than_rollouts_are_refused() -> None: - llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + assert (config.tool_call_parser, config.reasoning_parser) == ( + "qwen3_coder", + "nemotron_v3", + ) + with open(config.custom_jinja_template) as template_file: + assert template_file.read() == "{{ messages }}\n" - with pytest.raises(InvalidArgument, match="only NeMo RL rollouts"): - _collect(engine, {"token_ids": [1, 2, 3, 4]}) - assert llm.calls == [] +def test_a_parser_dynamo_lacks_fails_registration(monkeypatch) -> None: + host = _host( + _FakeLLM(), dynamo_cfg={"worker_args": {"tool_call_parser": "no_such_parser"}} + ) + + with pytest.raises(ValueError, match="no tool call parser named 'no_such_parser'"): + _start_worker_config(monkeypatch, host) def test_an_endpoint_that_stops_serving_reports_why() -> None: @@ -299,6 +514,78 @@ async def call_utility_async(method): assert registration.context_length == MAX_MODEL_LEN +def test_a_prompt_that_fills_the_context_is_refused_as_natively() -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match="maximum context length"): + _collect(engine, _openai_request(token_ids=list(range(MAX_MODEL_LEN)))) + + assert llm.calls == [] + + +def _image_url() -> str: + buffer = io.BytesIO() + Image.new("RGB", (8, 8), color="red").save(buffer, format="PNG") + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + + +def test_media_is_loaded_and_processed_as_vllms_chat_parser_does() -> None: + final = _output([1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3], _completion([5], "stop")) + llm = _FakeLLM([final]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + _collect( + engine, + _openai_request( + token_ids=[1, IMAGE_PLACEHOLDER, 3], + multi_modal_data={"image_url": [{"Url": _image_url()}]}, + mm_processor_kwargs={"max_num_tiles": 2}, + ), + ) + + ((prompt, skip_mm_cache),) = llm.renderer.prompts + # The frontend's unexpanded tokens, the decoded image and the processor kwargs. + assert prompt["prompt_token_ids"] == [1, IMAGE_PLACEHOLDER, 3] + (image,) = prompt["multi_modal_data"]["image"] + assert image.size == (8, 8) and image.mode == "RGB" + assert prompt["multi_modal_uuids"] == {"image": [None]} + assert prompt["mm_processor_kwargs"] == {"max_num_tiles": 2} + assert skip_mm_cache is False + # vLLM generates from the processed prompt, and the output cap counts its tokens. + ((engine_input, params, _),) = llm.calls + assert engine_input["prompt_token_ids"] == [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + assert params.max_tokens == MAX_MODEL_LEN - 2 - IMAGE_TOKENS + + +@pytest.mark.parametrize( + ("fields", "error"), + [ + ({"prompt_embeds": "AAAA"}, "Embedding prompts"), + ({"multi_modal_data": {"image_url": [{"UuidOnly": "u"}]}}, "carry their URL"), + ( + { + "multi_modal_data": { + "image_url": [{"Url": "data:image/png;base64,eA=="}] + } + }, + "Cannot load image_url", + ), + # vLLM's own request errors (not ValueErrors) keep their message, as the + # native server's 4xx does, instead of becoming a bare 500. + ({"stop_conditions": {"min_tokens": 10, "max_tokens": 5}}, "min_tokens"), + ], +) +def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match=error): + _collect(engine, _openai_request(**fields)) + + assert llm.calls == [] + + @pytest.mark.parametrize( ("router_mode", "prefix_caching", "publishes"), [("kv", True, True), ("kv", False, False), ("round-robin", True, False)], diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 13757622a81..616535ec4e1 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -414,7 +414,7 @@ def _generation_config(**generation) -> dict: ), ( {"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}, - r"dynamo_cfg\.worker_args was", + r"worker_args\.extra_cli_args", ), ( {"dynamo_cfg": {"frontend_args": {"tokenizer_cache": False}}}, From 580fe7068e1cdcf6df6990f4d3ed6c78c595b379 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 1 Nov 2026 20:16:50 -0800 Subject: [PATCH 17/82] fix(dynamo): reject score centering at startup policy.generation.top_logprobs > 0 needs the sampler top-k columns staged with each captured call, and the Dynamo engine's capture does not stage them, so every call would fail capture mid-run. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 2 ++ nemo_rl/models/generation/__init__.py | 8 ++++++++ tests/unit/models/generation/test_dynamo_config.py | 2 ++ 3 files changed, 12 insertions(+) diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 9881cc25afb..4588d09e125 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -152,6 +152,8 @@ Startup rejects these with `dynamo: true`: rank 0, vLLM rejects each engine's `ipc://` KV-event endpoint. - `vllm_cfg.http_server_api_key_required`: the frontend serves the same engines on 0.0.0.0 and checks no bearer token. +- `top_logprobs > 0` (`loss_fn.score_centering`): the engine's token capture + does not stage sampler top-k log-probs yet. - `frontend_args.router_mode: direct`: it needs a worker ID on every request, and NeMo RL's rollouts name none. - Parser plugins with no Dynamo parser, and the chat settings above that the diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 3f42409b38a..57b3a0c8b1b 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -198,6 +198,14 @@ def configure_generation_config( "frontend serves the same engines on 0.0.0.0 and checks no " "bearer token." ) + if config.get("top_logprobs"): + raise ValueError( + "policy.generation.top_logprobs > 0 (loss_fn.score_centering) is " + "not supported with policy.generation.dynamo=true yet: the " + "Dynamo engine's token capture does not stage sampler top-k " + "log-probs. Set top_logprobs to 0 and turn off " + "loss_fn.score_centering, or remove policy.generation.dynamo." + ) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index e891f0e94ad..77440d9a9a7 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -108,6 +108,8 @@ def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: ), "checks no bearer token", ), + # Dynamo's token capture stages no sampler top-k for score centering. + (_vllm_config(dynamo=True, top_logprobs=4), "sampler top-k"), # Direct routing needs a worker ID that rollouts never carry. ( _vllm_config( From e798c868c4cbbf24f0e42540858f05d3aa33eca7 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 3 Nov 2026 09:48:36 -0800 Subject: [PATCH 18/82] refactor(vllm): address review nits - check_sampling_param replaces off_policy_sampling_error and raises ValueError itself, with the same messages in the same order. It raises rather than asserts so a server that answers an off-policy request with a client error can catch it; the native chat handler still fails with a 500, as its assert did. - Shorten the route-encoding and sampling-profile comments to one line. - The pipeline docstrings name their actual callers and what the final RequestOutput is for. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/vllm/request_pipeline.py | 5 +--- nemo_rl/models/generation/vllm/utils.py | 25 +++++++++++-------- .../generation/vllm/vllm_worker_async.py | 14 +++-------- .../unit/models/generation/test_vllm_utils.py | 23 ++++++++++------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index 9e5bea46416..62b7332f377 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -15,10 +15,7 @@ """Turn a finished vLLM rollout into NeMo RL's per-sample generation output. The worker's ``_serve_rollout`` builds the result; ``rollout_output`` assembles -the output tensors from it. The two are apart so a host that builds the result -on the engine and assembles the output elsewhere returns what ``generate_async`` -returns. A rollout feature added here, such as router replay (R3), then behaves -the same on every path. +its output tensors for ``generate_async``. """ import logging diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index 99b7fa93c05..6ca3c3cdf0d 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -321,14 +321,14 @@ def _as_routed_experts_tensor( return tensor.to(dtype=dtype) -def off_policy_sampling_error( +def check_sampling_param( *, temperature: Optional[float], top_p: Optional[float], top_k: Optional[int], generation_config: Any, -) -> Optional[str]: - """Why an HTTP rollout request would sample off-policy, or None if it would not. +) -> None: + """Reject an HTTP rollout request that would sample off-policy. Request sampling must match NeMo RL's exactly, or inference runs off-policy and destabilizes training. Validation rollouts are the one exception: they carry the @@ -347,11 +347,17 @@ def off_policy_sampling_error( once the caller sends the request with top_k=-1. generation_config: ``policy.generation``, with the train and validation sampling parameters. + + Raises: + ValueError: The request's top_k is set, its top_p is unset, or its + temperature and top_p match neither sampling profile. """ if top_k not in (None, -1): - return f"Top k sampling parameter must be unset, empty, or -1. Got `{top_k}`" + raise ValueError( + f"Top k sampling parameter must be unset, empty, or -1. Got `{top_k}`" + ) if top_p is None: - return ( + raise ValueError( "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " "resolved by vLLM from the model's generation_config.json and would " "bypass the on-policy sampling check." @@ -365,8 +371,8 @@ def off_policy_sampling_error( and top_p == generation_config["val_top_p"] ) if is_train_sampling or is_val_sampling: - return None - return ( + return + raise ValueError( f"request sampling (temperature={temperature}, top_p={top_p}) matches neither " f"the train sampling params (temperature={generation_config['temperature']}, " f"top_p={generation_config['top_p']}) nor the validation sampling params " @@ -596,10 +602,7 @@ def chat_routed_experts( r3_stats["actual_routes"], r3_stats["expected_routes"], ) - # Base64 envelope instead of .tolist(): nested JSON int lists cost - # ~1s of CPU per serialize/parse hop at long context lengths and get - # re-validated at every gym HTTP hop; a single string passes through - # the gym chain opaquely. + # Keep routes opaque across Gym HTTP hops to avoid nested-list validation. return encode_routed_experts(routed_experts.to(dtype=routed_experts_dtype)) diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 05b5d215ae6..fb342cba95b 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -77,7 +77,7 @@ attach_token_information_to_chat_response_choices, attach_sampler_topk_logprobs_to_chat_response_choices, format_prompt_for_vllm_generation, - off_policy_sampling_error, + check_sampling_param, validate_rollout_prompt, model_dump_chat_response_with_dynamic_message_fields, remap_multimodal_placeholders, @@ -1635,17 +1635,13 @@ class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): async def create_chat_completion( request: NeMoRLChatCompletionRequest, raw_request: Request ): - # The request sampling params need to exactly match those as are set in NeMo RL. - # If they do not match, the inference will be off policy and destroy training - # stability. grpo.validate() is the only caller that constructs a non-train - # GenerationSamplingParams, and the validation profile it stamps is accepted. - sampling_error = off_policy_sampling_error( + # Accept train and validation profiles; reject off-policy requests. + check_sampling_param( temperature=request.temperature, top_p=request.top_p, top_k=request.top_k, generation_config=generation_config, ) - assert sampling_error is None, sampling_error # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params request.top_k = -1 @@ -2007,9 +2003,6 @@ async def _serve_rollout( ) -> tuple[RolloutResult, Optional[Any]]: """Serve one rollout from this engine: its budget, sampling, generation and result. - Every path that serves a rollout from this engine calls this, so a rollout - step added here runs on all of them. - Args: prompt: The vLLM prompt to generate from. prompt_token_ids: The learner's prompt tokens, which vLLM must have run. @@ -2021,6 +2014,7 @@ async def _serve_rollout( Returns: The result, and vLLM's final RequestOutput (None when no new token fits). + The final output supplies finish and usage metadata to Dynamo in PR #4905. """ max_new_tokens = self._rollout_max_new_tokens(len(prompt_token_ids)) if max_new_tokens == 0: diff --git a/tests/unit/models/generation/test_vllm_utils.py b/tests/unit/models/generation/test_vllm_utils.py index dcc72dc02ba..240cc592e01 100644 --- a/tests/unit/models/generation/test_vllm_utils.py +++ b/tests/unit/models/generation/test_vllm_utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import math from dataclasses import dataclass from types import SimpleNamespace @@ -39,7 +40,7 @@ encode_counter_key, format_prompt_for_vllm_generation, model_dump_chat_response_with_dynamic_message_fields, - off_policy_sampling_error, + check_sampling_param, pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) @@ -1531,17 +1532,21 @@ def test_remap_multimodal_placeholders_rebuilds_real_vllm_placeholder_range(): (1.0, 1.0, 5, "Top k sampling parameter"), ], ) -def test_off_policy_sampling_error(temperature, top_p, top_k, error) -> None: +def test_check_sampling_param(temperature, top_p, top_k, error) -> None: generation_config = { "temperature": 1.0, "top_p": 1.0, "val_temperature": 0.0, "val_top_p": 0.9, } - message = off_policy_sampling_error( - temperature=temperature, - top_p=top_p, - top_k=top_k, - generation_config=generation_config, - ) - assert (message is None) if error is None else (error in message) + with ( + contextlib.nullcontext() + if error is None + else pytest.raises(ValueError, match=error) + ): + check_sampling_param( + temperature=temperature, + top_p=top_p, + top_k=top_k, + generation_config=generation_config, + ) From d67311fc3c3e5bdea4d0c4005ca241b8164010a1 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 3 Nov 2026 11:41:16 -0800 Subject: [PATCH 19/82] style(vllm): sort check_sampling_param import Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/models/generation/test_vllm_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models/generation/test_vllm_utils.py b/tests/unit/models/generation/test_vllm_utils.py index 240cc592e01..ad1bfdb3803 100644 --- a/tests/unit/models/generation/test_vllm_utils.py +++ b/tests/unit/models/generation/test_vllm_utils.py @@ -35,12 +35,12 @@ attach_routed_experts_to_chat_response_choices, attach_sampler_topk_logprobs_to_chat_response_choices, attach_token_information_to_chat_response_choices, + check_sampling_param, compute_engine_step_metrics, compute_spec_decode_metrics, encode_counter_key, format_prompt_for_vllm_generation, model_dump_chat_response_with_dynamic_message_fields, - check_sampling_param, pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) From 2583db9ab51aadc86695519e7d3c299d49ec2236 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 3 Nov 2026 11:42:06 -0800 Subject: [PATCH 20/82] style(dynamo): sort capture-test imports Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../unit/models/generation/test_dynamo_native_engine_capture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py index 882ca397461..5415f251326 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine_capture.py +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -54,8 +54,8 @@ IMAGE_TOKEN, IMAGE_TOKENS, _collect, - _Context, _completion, + _Context, _FakeLLM, _gym_request, _host, From d15b6f40bc8c5100932cefad4e36c15dfb3f73e1 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 3 Nov 2026 20:55:21 -0800 Subject: [PATCH 21/82] test(vllm): expect ValueError from the chat handler's sampling check check_sampling_param raises ValueError since the review rename. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/models/generation/test_vllm_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 6ffcbf52e14..fb9f6c6ff11 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1499,13 +1499,13 @@ async def create_chat_completion(_request, _raw_request): @pytest.mark.asyncio async def test_chat_handler_rejects_off_policy_sampling_and_pins_top_k(monkeypatch): - """The route asserts on-policy sampling, then hands vLLM top_k=-1.""" + """The route rejects off-policy sampling, then hands vLLM top_k=-1.""" _, serving_chat, chat_handler = _setup_fake_vllm_chat_handler(monkeypatch) serving_chat.create_chat_completion = AsyncMock(return_value=iter(())) off_policy = _fake_chat_request() off_policy.temperature = 0.5 - with pytest.raises(AssertionError, match="matches neither"): + with pytest.raises(ValueError, match="matches neither"): await chat_handler(off_policy, MagicMock()) serving_chat.create_chat_completion.assert_not_awaited() From f28b4d67c62bfb0fc1dfd8472c932cb21eca46c8 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 4 Nov 2026 09:29:14 -0800 Subject: [PATCH 22/82] fix(dynamo): stage native route tensors from Gym capture Main's #4345 makes capture stage routed experts as native tensors and drop any other form. The Dynamo Gym capture path still passed the wire envelope, so captured calls lost their routes. Pass encode_for_wire=False there, as the native chat handler does; uncaptured calls keep the envelope in engine_data. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/models/generation/dynamo/native_engine.py | 11 ++++++++--- .../generation/test_dynamo_native_engine_capture.py | 11 +++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index 4052a3875d0..2723784c3f0 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -655,7 +655,9 @@ async def _generate_gym( chunk["engine_data"] = await self._stage_gym_call(request, final_output) else: engine_data = {"prompt_token_ids": list(final_output.prompt_token_ids)} - for field, value in self._gym_message(final_output).items(): + for field, value in self._gym_message( + final_output, encode_for_wire=True + ).items(): engine_data[_GYM_ENGINE_DATA_KEYS[field]] = value chunk["engine_data"] = engine_data log_probs, top_logprobs = extract_from_completion_output( @@ -759,7 +761,9 @@ def _check_prompt_fits(self, prompt_len: int) -> None: "output tokens." ) - def _gym_message(self, final_output: Any) -> dict[str, Any]: + def _gym_message( + self, final_output: Any, *, encode_for_wire: bool + ) -> dict[str, Any]: """The token fields the native endpoint puts on the call's message.""" completion = final_output.outputs[0] message: dict[str, Any] = {"generation_token_ids": list(completion.token_ids)} @@ -774,6 +778,7 @@ def _gym_message(self, final_output: Any) -> dict[str, Any]: device=torch.device("cpu"), logger=LOGGER, routed_experts_dtype=self._host.routed_experts_dtype, + encode_for_wire=encode_for_wire, ) return message @@ -791,7 +796,7 @@ async def _stage_gym_call( { "index": 0, "finish_reason": final_output.outputs[0].finish_reason, - "message": self._gym_message(final_output), + "message": self._gym_message(final_output, encode_for_wire=False), } ] } diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py index 5415f251326..a2f8990fab1 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine_capture.py +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -47,7 +47,9 @@ from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 -from nemo_rl.utils.routed_experts_codec import decode_routed_experts # noqa: E402 +from nemo_rl.utils.routed_experts_codec import ( # noqa: E402 + routed_experts_tensor_metadata, +) from tests.unit.models.generation.test_dynamo_native_engine import ( # noqa: E402 EOS, IMAGE_PLACEHOLDER, @@ -75,6 +77,7 @@ class _Capture: def __init__(self) -> None: self.begun: list[tuple] = [] self.staged: list[dict] = [] + self.attachments: list[dict | None] = [] self.failed: list[str] = [] def begin_call(self, admission, *, prefix_token_ids, stream): @@ -83,6 +86,7 @@ def begin_call(self, admission, *, prefix_token_ids, stream): def complete_call_from_response(self, call, payload, attachments=None): self.staged.append(payload) + self.attachments.append(attachments) return SimpleNamespace(model_dump=lambda: dict(COORDS)) def fail_call(self, call, *, reason) -> None: @@ -93,6 +97,7 @@ def _capturing_host(llm, **cfg): host = _host(llm, **cfg) host.token_capture = _Capture() host._capture_calls = {} + host._capture_top_logprobs = 0 host._chain_prefix = ChainPrefixCache() return host @@ -164,10 +169,12 @@ def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: assert message["generation_log_probs"] == [-0.5, -0.5] # As the native endpoint stages them: the full sequence's routes (none for # the last token) from the boundary row, the prefix's last token, on. - staged_routes = decode_routed_experts(message["routed_experts"], torch.int16) + (attachments,) = host.token_capture.attachments + staged_routes = attachments["routed_experts"] expected = torch.full((5, 2, 2), R3_MISSING_ROUTE_SENTINEL, dtype=torch.int16) expected[:4] = torch.from_numpy(routes[3:7]) assert torch.equal(staged_routes, expected) + assert message["routed_experts"] == routed_experts_tensor_metadata(staged_routes) # Only the commit coordinates go back through the frontend. assert chunk["engine_data"] == {NG_COMMIT_COORDS_FIELD: COORDS} assert "log_probs" not in chunk From 0b7d3785ef2cb57661fdf58f28f84f5532328b18 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 4 Nov 2026 09:35:59 -0800 Subject: [PATCH 23/82] test(data-plane): bind _parse_capture_admission on the route-tensor fake worker #4979 splits admission parsing out of _capture_admission so the Dynamo engine can parse its annotation; the fake worker from main's #4345 test needs both bound. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/data_plane/test_capture_route_tensors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/data_plane/test_capture_route_tensors.py b/tests/unit/data_plane/test_capture_route_tensors.py index c8212f93559..5b51dfba9cf 100644 --- a/tests/unit/data_plane/test_capture_route_tensors.py +++ b/tests/unit/data_plane/test_capture_route_tensors.py @@ -123,6 +123,9 @@ def test_served_capture_bypasses_codec_through_route_consumers( worker._capture_admission = ( VllmAsyncGenerationWorkerImpl._capture_admission.__get__(worker) ) + worker._parse_capture_admission = ( + VllmAsyncGenerationWorkerImpl._parse_capture_admission.__get__(worker) + ) worker._delta_align_routed_experts = ( VllmAsyncGenerationWorkerImpl._delta_align_routed_experts ) From c4fe2bd540cb1be02daf82f6f2961a80ab4f3d59 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 5 Nov 2026 10:19:44 -0800 Subject: [PATCH 24/82] fix(dynamo): accept enable_auto_tools=false when no tool parser is set Without a tool parser Dynamo's frontend parses no tool calls, so turning auto tools off changes nothing there. Single-turn recipes such as the openmathreasoning Gym config set exactly this and were rejected with only policy.generation.dynamo=true added. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/models/generation/dynamo/config.py | 6 +++++- tests/unit/models/generation/test_dynamo_config.py | 12 +++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 7233cec4835..0bb39711d86 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -171,7 +171,11 @@ def check_dynamo_serving_chat_kwargs(generation_config: Mapping[str, Any]) -> No "templates itself and has no content-format setting. Remove it or " "set it to 'auto'." ) - if serving.get("enable_auto_tools") is False: + # Without a tool parser the frontend parses no tool calls, so turning auto + # tools off changes nothing there (single-turn recipes set exactly this). + if serving.get("enable_auto_tools") is False and ( + dynamo_parsers(generation_config)[0] is not None + ): raise ValueError( "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." "enable_auto_tools=false is not supported with " diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 77440d9a9a7..46a23b994c8 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -212,7 +212,7 @@ def test_chat_template_and_default_kwargs_follow_the_native_server() -> None: ("serving_chat_kwargs", "error"), [ ({"chat_template_content_format": "string"}, "no content-format setting"), - ({"enable_auto_tools": False}, "always parses"), + ({"enable_auto_tools": False, "tool_parser": "hermes"}, "always parses"), ], ) def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( @@ -220,11 +220,13 @@ def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( ) -> None: with pytest.raises(ValueError, match=error): configure_generation_config(_serving(**serving_chat_kwargs), _Tokenizer()) - # The native defaults are what the frontend does anyway. - configure_generation_config( + # The native defaults are what the frontend does anyway, and with no tool + # parser there are no tool calls for auto tools to change. + for accepted in ( _serving(chat_template_content_format="auto", enable_auto_tools=True), - _Tokenizer(), - ) + _serving(enable_auto_tools=False, tool_parser=None), + ): + configure_generation_config(accepted, _Tokenizer()) def test_a_bearer_token_is_rejected_because_the_frontend_checks_none() -> None: From ee813c3ba1381d962959031c6c9f1ed65d11542f Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 5 Nov 2026 20:13:40 -0800 Subject: [PATCH 25/82] fix(dynamo): address round-3 review - Count only stored-block KV events: every prefix-cache reset applies a payload-free "cleared" event, which hid a router that decoded no block event (C46). - Register the engine with a weight-free snapshot from Dynamo's fetch_model(ignore_weights=True): the engine loads dummy weights and gets its real ones by refit, so no node downloads the checkpoint (C47). - Send a RuntimeError from _serve_rollout back as InvalidArgument, so the driver sees the cause and does not regenerate it twice more (C48). - Ship non-finite log probabilities as strings, since Dynamo's frontend carries engine_data as strict JSON (C49). - Name Dynamo's main-attention kinds as a module constant (C55). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/dynamo/managed_runtime.py | 9 ++-- .../models/generation/dynamo/native_engine.py | 43 ++++++++++++------- .../generation/vllm/request_pipeline.py | 7 ++- .../generation/vllm/vllm_worker_async.py | 12 ++++++ 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index 273d595c014..ee4f0383836 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -97,9 +97,12 @@ def _service_executable(name: str) -> str: # The KV router's applied-event counter (Dynamo's kvrouter.KV_CACHE_EVENTS_APPLIED), -# one sample per label set. +# one sample per label set. Stored-block events only: every prefix-cache reset +# emits a payload-free "cleared" event, which applies even when the router +# decodes none of vLLM's block events. _KV_EVENTS_APPLIED = re.compile( - r'^\w*kv_cache_events_applied(?:_total)?\{(?=[^}]*status="(?P[^"]*)")' + r'^\w*kv_cache_events_applied(?:_total)?\{(?=[^}]*event_type="stored")' + r'(?=[^}]*status="(?P[^"]*)")' r"[^}]*\}\s+(?P\S+)$", re.MULTILINE, ) @@ -406,7 +409,7 @@ def _assert_services_alive(self) -> None: ) def _log_router_kv_events(self) -> None: - """Log how many engine KV-cache events the KV router applied, by status. + """Log how many stored-block KV events the KV router applied, by status. Read from the frontend's metrics before it stops. A KV router that never applied one routed on load alone. diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index b422d21c11e..23937f93354 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -61,6 +61,15 @@ LOGGER = logging.getLogger(__name__) +# vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them +# (dynamo/vllm/cache_info.py MAIN_ATTENTION_KV_CACHE_KINDS, which NeMo RL may +# not import); test_main_attention_kinds_match_dynamos checks the copy. +_MAIN_ATTENTION_KINDS = ( + KVCacheSpecKind.FULL_ATTENTION, + KVCacheSpecKind.MLA_ATTENTION, + KVCacheSpecKind.SINK_FULL_ATTENTION, +) + def dynamo_kv_events_config( cfg: DynamoCfg, *, prefix_caching: bool @@ -158,12 +167,7 @@ async def start(self, worker_id: int) -> EngineConfig: for group in await self._llm.engine_core.call_utility_async( "get_kv_cache_group_metadata" ): - # vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them. - if group.get("kind") in ( - KVCacheSpecKind.FULL_ATTENTION, - KVCacheSpecKind.MLA_ATTENTION, - KVCacheSpecKind.SINK_FULL_ATTENTION, - ): + if group.get("kind") in _MAIN_ATTENTION_KINDS: block_size = group["block_size"] break model = self._host.model_name @@ -201,14 +205,19 @@ async def _generate_rollout( the driver assembles the same output ``generate_async`` returns. """ prompt_token_ids = list(request["token_ids"]) - result, final_output = await self._host._serve_rollout( - TokensPrompt(prompt_token_ids=prompt_token_ids), - prompt_token_ids=prompt_token_ids, - greedy=rollout["greedy"], - stop_strings=rollout["stop_strings"], - request_id=context.id(), - device=torch.device("cpu"), - ) + try: + result, final_output = await self._host._serve_rollout( + TokensPrompt(prompt_token_ids=prompt_token_ids), + prompt_token_ids=prompt_token_ids, + greedy=rollout["greedy"], + stop_strings=rollout["stop_strings"], + request_id=context.id(), + device=torch.device("cpu"), + ) + except RuntimeError as error: + # A 5xx reaches the driver as "Internal server error" and is re-sent; + # a 4xx keeps the cause, and the rollout retry policy still applies. + raise InvalidArgument(f"{type(error).__name__}: {error}") from error chunk: GenerateChunk = { "token_ids": result.token_ids, "index": 0, @@ -261,12 +270,16 @@ def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: def start_dynamo_endpoint( host: "VllmAsyncGenerationWorkerImpl", *, + model_path: str, namespace: str, engine_env: dict[str, str], leader_worker_idx: int, ) -> asyncio.Task: """Register ``host``'s engine with Dynamo and serve it on the running loop. + ``model_path`` is a local directory with the model's config and tokenizer + files; the engine is served under ``host.model_name``. + The Rust Worker serves until the process gets SIGTERM. It installs a chained handler, so Ray's actor handler still runs and the actor exits as usual; discovery drops the engine with its process. @@ -281,7 +294,7 @@ def start_dynamo_endpoint( namespace=namespace, component=DYNAMO_COMPONENT, endpoint=DYNAMO_ENDPOINT, - model_name=host.model_name, + model_name=model_path, served_model_name=host.model_name, # NeMo RL's rollouts go to /v1/completions only. endpoint_types="completions", diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index e42904e2714..107a2529a49 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -19,6 +19,7 @@ """ import logging +import math from dataclasses import dataclass from typing import Any, Optional, cast @@ -176,7 +177,9 @@ def rollout_result_to_wire(result: RolloutResult) -> dict[str, Any]: """JSON-safe form of ``result``, for engines that answer over HTTP.""" return { "token_ids": result.token_ids, - "logprobs": result.logprobs, + # Strict JSON has no NaN or infinity, and Dynamo's frontend turns them + # into null; send them as the strings float() reads back. + "logprobs": [lp if math.isfinite(lp) else str(lp) for lp in result.logprobs], "truncated": result.truncated, "routed_experts": ( None @@ -194,7 +197,7 @@ def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: routed_experts = decode_routed_experts(routed_experts, dtype=None) return RolloutResult( token_ids=list(wire["token_ids"]), - logprobs=list(wire["logprobs"]), + logprobs=[float(lp) for lp in wire["logprobs"]], truncated=bool(wire["truncated"]), routed_experts=routed_experts, r3_stats=wire["r3_stats"], diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 388f50b1b4d..9562dab8232 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -16,6 +16,7 @@ import copy import gc import logging +import os import secrets import threading import time @@ -659,12 +660,23 @@ async def start_dynamo_endpoint_async( ``leader_worker_idx`` is this worker's index in its group; rollouts the engine serves report it, as ``generate_async`` results do. """ + # ai-dynamo is optional: import it only for engines served through Dynamo. + from dynamo._core import fetch_model + from nemo_rl.models.generation.dynamo.native_engine import ( start_dynamo_endpoint, ) + # Dynamo fetches a Hub ID's whole checkpoint before it registers; the engine's + # weights come from refit, so register the config and tokenizer files only. + model_path = ( + self.model_name + if os.path.exists(self.model_name) + else await fetch_model(self.model_name, ignore_weights=True) + ) self._dynamo_endpoint_task = start_dynamo_endpoint( self, + model_path=model_path, namespace=namespace, engine_env=engine_env, leader_worker_idx=leader_worker_idx, From 614fb0a66bc32f9b83f7760b007bd9a647653e4b Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 5 Nov 2026 20:15:10 -0800 Subject: [PATCH 26/82] test(dynamo): cover the round-3 gaps; trim stale docstrings - KV-event log test uses Dynamo's real event_type label and a cleared sample (C46); registration test checks the weight-free model path (C47). - New tests: an engine error keeps its cause (C48), non-finite logprobs cross strict JSON (C49), a cancelled rollout aborts its vLLM request (C50), the main-attention kinds match Dynamo's (C55). - Two engines get different KV-event sockets (C51); the engine wait fails fast on a dead frontend too (C52). - Drop the exemplar-equals-defaults check (C53); build DynamoCfg with keyword arguments (C54). - Docstrings: drop the removed DynamoGeneration history (C56), and say a vLLM bump can break Dynamo only in NeMo RL's adapter (C57). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/models/generation/interfaces.py | 3 +- .../collective_weight_synchronizer.py | 13 +--- .../models/generation/test_dynamo_config.py | 29 ++++---- .../generation/test_dynamo_managed_runtime.py | 38 +++++++--- .../generation/test_dynamo_native_engine.py | 72 +++++++++++++++++-- .../generation/test_dynamo_vllm_decoupling.py | 8 +-- .../generation/test_vllm_request_pipeline.py | 15 +++- .../test_reconcile_communicator.py | 9 +-- 8 files changed, 131 insertions(+), 56 deletions(-) diff --git a/nemo_rl/models/generation/interfaces.py b/nemo_rl/models/generation/interfaces.py index f1ea20b88cf..c5a1f35f6a0 100644 --- a/nemo_rl/models/generation/interfaces.py +++ b/nemo_rl/models/generation/interfaces.py @@ -597,8 +597,7 @@ def shard_liveness_ref(self, shard_idx: int) -> ray.ObjectRef: Which Ray worker leads shard N depends on how shards are laid out across workers, which is the backend's business. Asking for it by shard index keeps that here rather than in the control loop, where the same arithmetic had a second copy that - also assumed every backend has a ``worker_group`` -- an assumption that has already - broken a lane once (``'DynamoGeneration' object has no attribute 'worker_group'``). + also assumed every backend has a ``worker_group``. """ raise NotImplementedError diff --git a/nemo_rl/weight_sync/collective_weight_synchronizer.py b/nemo_rl/weight_sync/collective_weight_synchronizer.py index 7ac9366d6ae..252c5ec22d4 100644 --- a/nemo_rl/weight_sync/collective_weight_synchronizer.py +++ b/nemo_rl/weight_sync/collective_weight_synchronizer.py @@ -190,16 +190,9 @@ def _desired_membership( ) -> Optional[RefitMembership]: """The membership to track, or None for a backend that owns no DP worker group. - NOT every GenerationInterface has one. vLLM and TRT-LLM do; Megatron and SGLang - do not, and the interface declares nothing either way -- so reading - ``self._generation.worker_group`` assumes a vLLM shape that this synchronizer is - not entitled to assume. - - That assumption once broke L1_Functional_Tests_Dynamo on the plain grpo.py path, - under the since-removed managed Dynamo backend: - - grpo.py:1747 policy_generation.weight_synchronizer.init_communicator() - AttributeError: 'DynamoGeneration' object has no attribute 'worker_group' + NOT every GenerationInterface has one, and the interface declares nothing either + way -- so reading ``self._generation.worker_group`` assumes a vLLM shape that + this synchronizer is not entitled to assume. None means "this backend has no shards to track", which is the same thing an unrecorded membership already meant: every reconcile falls through to "nothing to diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 078adf6a1b2..baae91f105e 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -11,13 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path - import pytest -import yaml from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.config import DynamoCfg, dynamo_cfg +from nemo_rl.models.generation.dynamo.config import ( + DynamoCfg, + DynamoFrontendArgs, + dynamo_cfg, +) from nemo_rl.models.generation.dynamo.managed_runtime import frontend_argv @@ -45,12 +46,6 @@ def _vllm_config(**overrides) -> dict: return config -_EXEMPLAR = ( - Path(__file__).resolve().parents[4] - / "examples/configs/grpo_math_1B_dynamo_single_controller.yaml" -) - - @pytest.mark.parametrize("dynamo", [False, True]) def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: config = configure_generation_config(_vllm_config(dynamo=dynamo), _Tokenizer()) @@ -60,9 +55,7 @@ def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: assert config["vllm_cfg"]["async_engine"] is dynamo assert config["vllm_cfg"]["skip_tokenizer_init"] is True assert "dynamo_cfg" not in config - # The exemplar spells out exactly the defaults. - exemplar = yaml.safe_load(_EXEMPLAR.read_text())["policy"]["generation"] - assert dynamo_cfg(config) == DynamoCfg() == dynamo_cfg(exemplar) + assert dynamo_cfg(config) == DynamoCfg() @pytest.mark.parametrize( @@ -120,15 +113,17 @@ def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( def test_frontend_argv_is_runtime_owned() -> None: - cfg = DynamoCfg.model_validate( - {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} + cfg = DynamoCfg( + frontend_args=DynamoFrontendArgs( + router_mode="kv", extra_cli_args=["--foo", "1"] + ) ) argv = frontend_argv(host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg) assert argv[argv.index("--router-mode") + 1] == "kv" assert argv[-2:] == ["--foo", "1"] - clashing = DynamoCfg.model_validate( - {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} + clashing = DynamoCfg( + frontend_args=DynamoFrontendArgs(extra_cli_args=["--http-port", "9"]) ) with pytest.raises(ValueError, match="--http-port is set by both"): frontend_argv( diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 2c5c4dc3541..c6dde7fcf51 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -19,7 +19,7 @@ import pytest -from nemo_rl.models.generation.dynamo.config import DynamoCfg +from nemo_rl.models.generation.dynamo.config import DynamoCfg, DynamoFrontendArgs from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, _managed_namespace, @@ -29,8 +29,8 @@ def _runtime(**frontend_args) -> ManagedDynamoRuntime: return ManagedDynamoRuntime( - dynamo_cfg=DynamoCfg.model_validate( - {"startup_timeout_s": 5, "frontend_args": frontend_args} + dynamo_cfg=DynamoCfg( + startup_timeout_s=5, frontend_args=DynamoFrontendArgs(**frontend_args) ), model_name="model", frontend_python="/venvs/vllm/bin/python", @@ -261,14 +261,28 @@ def fake_urlopen(url, timeout): ) -def test_engine_wait_fails_immediately_with_the_engines_error() -> None: +@pytest.mark.parametrize( + ("stopped", "error"), + [ + ("engine", "shard 0 stopped serving"), + # A bad frontend_args.extra_cli_args flag makes the frontend exit at once. + ("frontend", r"frontend is not alive \(code=2\)"), + ], +) +def test_engine_wait_fails_immediately_when_an_engine_or_service_stops( + stopped, error +) -> None: runtime = _started(_runtime()) + if stopped == "frontend": + runtime._frontend_process.poll = lambda: 2 + runtime._frontend_process.returncode = 2 - def endpoint_stopped() -> None: - raise RuntimeError("endpoint of generation shard 0 stopped serving") + def check_engines() -> None: + if stopped == "engine": + raise RuntimeError("endpoint of generation shard 0 stopped serving") - with pytest.raises(RuntimeError, match="shard 0 stopped serving"): - runtime.wait_for_engines(1, check_engines=endpoint_stopped) + with pytest.raises(RuntimeError, match=error): + runtime.wait_for_engines(1, check_engines=check_engines) def test_frontend_launches_from_the_given_interpreter(monkeypatch) -> None: @@ -290,9 +304,11 @@ def test_shutdown_logs_the_kv_events_the_router_applied(monkeypatch, capsys) -> runtime = _started(_runtime()) metrics = ( "# HELP dynamo_component_kv_cache_events_applied KV events applied\n" - 'dynamo_component_kv_cache_events_applied{status="ok",worker_id="1"} 40\n' - 'dynamo_component_kv_cache_events_applied{status="ok",worker_id="2"} 2\n' - 'dynamo_component_kv_cache_events_applied{status="parent_block_not_found"} 3\n' + 'dynamo_component_kv_cache_events_applied{event_type="stored",status="ok",worker_id="1"} 40\n' + 'dynamo_component_kv_cache_events_applied{event_type="stored",status="ok",worker_id="2"} 2\n' + 'dynamo_component_kv_cache_events_applied{event_type="stored",status="parent_block_not_found"} 3\n' + # Every prefix-cache reset applies one; it says nothing about block events. + 'dynamo_component_kv_cache_events_applied{event_type="cleared",status="ok",worker_id="1"} 5\n' 'dynamo_component_requests_total{status="ok"} 9\n' ) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index d1dda2bfeb4..0ef89990af2 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -17,8 +17,11 @@ constructor, so rollouts are sampled by the native worker's own helpers. """ +import ast import asyncio +import importlib.util import json +from pathlib import Path from types import SimpleNamespace import numpy as np @@ -32,6 +35,10 @@ from dynamo.llm.exceptions import InvalidArgument # noqa: E402 from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.dynamo.config import ( # noqa: E402 + DynamoCfg, + DynamoFrontendArgs, +) from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) @@ -156,6 +163,24 @@ def test_rollouts_are_served_like_generate_async() -> None: } +def test_a_cancelled_rollout_aborts_the_vllm_request_it_runs() -> None: + llm = _FakeLLM([_output([1, 2, 3], _completion([10], "stop"))]) + aborted = [] + + async def abort(request_id): + aborted.append(request_id) + + llm.abort = abort + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + _collect(engine, _rollout_request([1, 2, 3])) + # Dynamo's worker calls abort() when the frontend cancels a request. + asyncio.run(engine.abort(_Context())) + + ((_, _, request_id),) = llm.calls + assert aborted == [request_id] + + def test_router_replay_routes_reach_rollouts_as_natively() -> None: routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) final = _output( @@ -225,7 +250,11 @@ async def run(self): async def start(): await native_engine.start_dynamo_endpoint( - host, namespace="ns", engine_env=env, leader_worker_idx=0 + host, + model_path="/snapshots/model", + namespace="ns", + engine_env=env, + leader_worker_idx=0, ) asyncio.run(start()) @@ -237,6 +266,11 @@ def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: config = _start_worker_config(monkeypatch, _host(_FakeLLM())) assert config.namespace == "ns" + # Registered from the weight-free snapshot, served under the model's name. + assert (config.model_name, config.served_model_name) == ( + "/snapshots/model", + "model", + ) # NeMo RL's rollouts go to /v1/completions; no chat surface is registered. assert config.endpoint_types == "completions" @@ -251,6 +285,20 @@ def test_requests_other_than_rollouts_are_refused() -> None: assert llm.calls == [] +def test_an_engine_error_reaches_the_driver_with_its_cause() -> None: + host = _host(_FakeLLM()) + + async def serve_rollout(*args, **kwargs): + raise RuntimeError("the generation output did not include routed_experts") + + host._serve_rollout = serve_rollout + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + # A 5xx would reach the driver as "Internal server error". + with pytest.raises(InvalidArgument, match="RuntimeError: .*routed_experts"): + _collect(engine, _rollout_request([1, 2, 3])) + + def test_an_endpoint_that_stops_serving_reports_why() -> None: async def endpoint_errors() -> list: async def registration_fails() -> None: @@ -306,9 +354,7 @@ async def call_utility_async(method): def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( router_mode, prefix_caching, publishes ) -> None: - cfg = native_engine.DynamoCfg.model_validate( - {"frontend_args": {"router_mode": router_mode}} - ) + cfg = DynamoCfg(frontend_args=DynamoFrontendArgs(router_mode=router_mode)) config = native_engine.dynamo_kv_events_config(cfg, prefix_caching=prefix_caching) @@ -319,3 +365,21 @@ def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( # A per-engine abstract socket, which vLLM's publisher binds and Dynamo's # source connects to: no port for two engines on a node to collide on. assert config["endpoint"].startswith("ipc://@") + other = native_engine.dynamo_kv_events_config(cfg, prefix_caching=True) + assert other["endpoint"] != config["endpoint"] + + +def test_main_attention_kinds_match_dynamos() -> None: + # Read from source: importing dynamo.vllm would import Dynamo's vLLM integration. + spec = importlib.util.find_spec("dynamo.vllm.cache_info") + tree = ast.parse(Path(spec.origin).read_text()) + (dynamos,) = [ + ast.literal_eval(node.value) + for node in tree.body + if isinstance(node, ast.Assign) + and any( + getattr(t, "id", None) == "MAIN_ATTENTION_KV_CACHE_KINDS" + for t in node.targets + ) + ] + assert {kind.value for kind in native_engine._MAIN_ATTENTION_KINDS} == dynamos diff --git a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py index f729715eede..62ff7e4b7c2 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py +++ b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py @@ -14,10 +14,10 @@ """Dynamo stays out of NeMo RL's vLLM engines. The engines run NeMo RL's own vLLM and use Dynamo only through its -engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only where -it also breaks native vLLM. These guards fail if NeMo RL starts importing -Dynamo's vLLM integration (or any other Dynamo module), or if the SDK itself -starts importing vLLM. +engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only in +NeMo RL's own adapter (``dynamo/native_engine.py``). These guards fail if +NeMo RL starts importing Dynamo's vLLM integration (or any other Dynamo +module), or if the SDK itself starts importing vLLM. """ import ast diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py index f26ee51a733..cc9b7cf0725 100644 --- a/tests/unit/models/generation/test_vllm_request_pipeline.py +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -13,6 +13,8 @@ # limitations under the License. """The rollout result pipeline behind generate_async.""" +import json +import math from types import SimpleNamespace import numpy as np @@ -98,7 +100,8 @@ def test_wire_round_trip_preserves_the_output() -> None: result = _result( _final_output([7, 8, 9], prompt_routes=3, generated_routes=3), r3=True ) - wire = rollout_result_to_wire(result) + # The wire crosses Dynamo's frontend as strict JSON. + wire = json.loads(json.dumps(rollout_result_to_wire(result), allow_nan=False)) input_ids = torch.tensor([1, 2, 3]) direct = rollout_output(result, input_ids=input_ids, input_length=3) @@ -111,3 +114,13 @@ def test_wire_round_trip_preserves_the_output() -> None: assert direct.keys() == shipped.keys() for key in direct: assert torch.equal(direct[key], shipped[key]), key + + +def test_wire_carries_non_finite_logprobs_as_strict_json() -> None: + result = RolloutResult( + [7, 8, 9], [-0.5, float("-inf"), float("nan")], False, None, None + ) + # Dynamo's transport is strict JSON: non-finite floats would arrive as null. + wire = json.loads(json.dumps(rollout_result_to_wire(result), allow_nan=False)) + logprobs = rollout_result_from_wire(wire).logprobs + assert logprobs[:2] == [-0.5, float("-inf")] and math.isnan(logprobs[2]) diff --git a/tests/unit/weight_sync/test_reconcile_communicator.py b/tests/unit/weight_sync/test_reconcile_communicator.py index 1253e5aab46..f87e59e8cc5 100644 --- a/tests/unit/weight_sync/test_reconcile_communicator.py +++ b/tests/unit/weight_sync/test_reconcile_communicator.py @@ -655,13 +655,8 @@ def test_an_unconfigured_deadline_still_bounds_the_wait(self): class TestBackendsWithoutAWorkerGroup: """CollectiveWeightSynchronizer holds a GenerationInterface, not a VllmGeneration. - Only some implementations own a Ray worker group with DP shards -- vLLM and TRT-LLM do; - Megatron and SGLang do not, and the interface declares nothing either way. Reading - self._generation.worker_group once broke the since-removed managed Dynamo backend - on the plain grpo.py path: - - grpo.py:1747 policy_generation.weight_synchronizer.init_communicator() - AttributeError: 'DynamoGeneration' object has no attribute 'worker_group' + Not every implementation owns a Ray worker group with DP shards, and the interface + declares nothing either way. Membership tracking is only meaningful where shards exist, so a backend without one records None and every reconcile falls through to "nothing to do" -- which is exactly From f75aa87ccec5da9c0792cd4992ef8b29b30ed808 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 6 Nov 2026 10:33:03 -0800 Subject: [PATCH 27/82] feat(dynamo): pin every NeMo-Gym session to one engine Re-pin Gym to a83f5c1d: Gym#4817's a7be5c5 plus "name the Dynamo session on every chat call". Gym now sends x-dynamo-session-id on every chat call through the Dynamo frontend, keyed by the rollout, else the Gym session, not only on token-capture calls. Native pins each Gym session to one server whether or not capture is on; without capture, Dynamo spread a rollout's turns over engines, and the engine's echoed-prefix splice hides where that prefix is cached from the KV router. Every rollout now counts against the frontend's 65,536 sessions, so the documented bound is rollouts started per second x TTL. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/guides/dynamo-generation.md | 7 ++++++- .../grpo_math_1B_dynamo_single_controller.yaml | 2 +- nemo_rl/models/generation/dynamo/config.py | 13 +++++++------ tests/unit/models/generation/test_dynamo_config.py | 2 +- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index a7be5c5f3a4..a83f5c1dea8 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit a7be5c5f3a4d0957c9e84a8f7face90e30b7171e +Subproject commit a83f5c1dea81c7ce4df1efd52025dbf8a2fc4215 diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 4588d09e125..43f2aa27e49 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -54,7 +54,7 @@ shows them all: | `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | | `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | -| `frontend_args.session_affinity_ttl_secs` | 300 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it. Dynamo keeps at most 65,536 sessions and answers a new one past that with HTTP 529 | +| `frontend_args.session_affinity_ttl_secs` | 300 | Keeps a NeMo-Gym rollout on one engine while it is idle for up to this many seconds; null disables it. Dynamo keeps at most 65,536 sessions and answers a new one past that with HTTP 529 | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | The etcd and nats-server binaries are found in `NEMO_RL_DYNAMO_BIN_DIR` when it @@ -109,6 +109,11 @@ same vLLM workers stage each call's tokens, log probabilities, routed experts an processed media from their own process, exactly as the native endpoint does, and only the commit coordinates travel back through the frontend. +Gym names each call's rollout, and the frontend keeps a rollout on one engine +(`session_affinity_ttl_secs`), as the native backend pins each Gym session to +one server. The engine splices the echoed prefix after the frontend has routed +the call, so the KV router alone cannot see where that prefix is cached. + The native server's chat settings carry over: | Native setting (`vllm_cfg`) | Under Dynamo | diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 2c27bf7f89c..e2e587b3936 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -35,7 +35,7 @@ policy: frontend_args: # kv: route each request to the engine holding most of its prefix. router_mode: kv - # Keep a NeMo-Gym token-capture rollout on one engine while it idles up + # Keep a NeMo-Gym rollout on one engine while it idles up # to this many seconds (null disables it). session_affinity_ttl_secs: 300 extra_cli_args: [] diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 0bb39711d86..ab716edaf41 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -65,12 +65,13 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "device-aware-weighted", ] = "kv" # Idle TTL in seconds for keeping one session on one engine, or None to - # disable it. NeMo-Gym names the session on token-capture calls only (its - # capture key), as the native backend pins each Gym session to one server; the - # engine splices a call's staged prefix after routing, so the KV router alone - # cannot see where that prefix is cached. Dynamo keeps at most 65,536 sessions - # and answers a new one past that with HTTP 529, so keep (captured rollouts - # started per second) x TTL below 65,536. The bound is the frontend's own. + # disable it. NeMo-Gym names the session on every call (its rollout, else its + # own session), as the native backend pins each Gym session to one server; + # the engine splices a call's echoed or staged prefix after routing, so the + # KV router alone cannot see where that prefix is cached. Dynamo keeps at + # most 65,536 sessions and answers a new one past that with HTTP 529, so keep + # (rollouts started per second) x TTL below 65,536. The bound is the + # frontend's own. session_affinity_ttl_secs: Annotated[int, Field(gt=0, le=31_536_000)] | None = 300 extra_cli_args: list[str] = Field(default_factory=list) diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 46a23b994c8..d27f553e58a 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -139,7 +139,7 @@ def test_frontend_argv_is_runtime_owned() -> None: ) argv = frontend_argv(host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg) assert argv[argv.index("--router-mode") + 1] == "kv" - # Gym names a token-capture call's session; keep the session on one engine. + # Gym names every call's session; keep the session on one engine. assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "300" assert argv[-2:] == ["--foo", "1"] no_affinity = DynamoCfg.model_validate( From 776aaa36d0f01ebc8f37382adbbab5158aab84f3 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 6 Nov 2026 10:40:38 -0800 Subject: [PATCH 28/82] feat(dynamo): tokenize Gym chat with fastokens in the frontend when vLLM does NRL_USE_FASTOKENS (mirrored to VLLM_USE_FASTOKENS) makes native vLLM tokenize NeMo-Gym chat with fastokens, but the Dynamo frontend, which tokenizes that chat itself, kept Dynamo's default Hugging Face backend. The managed runtime now passes --tokenizer fastokens to the frontend whenever vLLM's switch is on. DYN_TOKENIZER in the driver's environment could not do this: the runtime drops inherited DYN_* variables. vllm_uses_fastokens() reads vLLM's switch after the NeMo RL mirror. Dynamo falls back to Hugging Face, with a warning, for a tokenizer fastokens cannot load, as NeMo RL's own patch does. The removed frontend_args.tokenizer* keys now point at NRL_USE_FASTOKENS. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 1 + .../generation/dynamo/managed_runtime.py | 21 +++++++++-- .../models/generation/vllm/vllm_generation.py | 3 +- nemo_rl/utils/fastokens.py | 10 ++++++ nemo_rl/utils/outdated_config_checks.py | 4 +-- .../models/generation/test_dynamo_config.py | 36 ++++++++++++++----- .../generation/test_dynamo_managed_runtime.py | 1 + tests/unit/utils/test_fastokens.py | 19 ++++++++++ 8 files changed, 81 insertions(+), 14 deletions(-) diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 43f2aa27e49..085f3ce6cda 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -113,6 +113,7 @@ Gym names each call's rollout, and the frontend keeps a rollout on one engine (`session_affinity_ttl_secs`), as the native backend pins each Gym session to one server. The engine splices the echoed prefix after the frontend has routed the call, so the KV router alone cannot see where that prefix is cached. +The frontend tokenizes Gym chat with fastokens when vLLM does (`NRL_USE_FASTOKENS=1`). The native server's chat settings carry over: diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index 75a02dbba63..fd3239a3f37 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -51,9 +51,13 @@ def frontend_argv( - *, host: str, port: int, namespace: str, dynamo_cfg: DynamoCfg + *, host: str, port: int, namespace: str, dynamo_cfg: DynamoCfg, fastokens: bool ) -> list[str]: - """The ``dynamo.frontend`` arguments; ``extra_cli_args`` may not repeat one.""" + """The ``dynamo.frontend`` arguments; ``extra_cli_args`` may not repeat one. + + ``fastokens`` selects the fastokens tokenizer: the frontend tokenizes NeMo-Gym + chat, which native vLLM tokenizes with fastokens when vLLM's switch is on. + """ argv = [ "--http-host", host, @@ -75,6 +79,8 @@ def frontend_argv( "--router-session-affinity-ttl-secs", str(dynamo_cfg.frontend_args.session_affinity_ttl_secs), ] + if fastokens: + argv += ["--tokenizer", "fastokens"] extra = dynamo_cfg.frontend_args.extra_cli_args for token in extra: flag = token.split("=", 1)[0].replace("_", "-") @@ -119,7 +125,12 @@ class ManagedDynamoRuntime: """Own etcd, NATS and the frontend that NeMo RL's vLLM engines register with.""" def __init__( - self, *, dynamo_cfg: DynamoCfg, model_name: str, frontend_python: str + self, + *, + dynamo_cfg: DynamoCfg, + model_name: str, + frontend_python: str, + fastokens: bool, ) -> None: """Describe the services; nothing starts until :meth:`start`. @@ -128,10 +139,13 @@ def __init__( model_name: The served model; the frontend is ready once it lists it. frontend_python: An interpreter whose environment has ``ai-dynamo``: the vLLM workers' own. + fastokens: Whether the frontend tokenizes with fastokens, as the vLLM + workers do. """ self._dynamo_cfg = dynamo_cfg self._model_name = model_name self._frontend_python = frontend_python + self._fastokens = fastokens self._namespace = _managed_namespace() self._host = "" self._etcd_port = 0 @@ -331,6 +345,7 @@ def _start_frontend(self) -> None: port=self._frontend_port, namespace=self._namespace, dynamo_cfg=self._dynamo_cfg, + fastokens=self._fastokens, ) command = [ self._frontend_python, diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index bc7b5e27eb7..8c8bd4c2731 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -81,7 +81,7 @@ from nemo_rl.telemetry.metrics import record_rl_metrics, warn_once from nemo_rl.telemetry.setup import get_telemetry_handle from nemo_rl.telemetry.span_groups import RLSpanGroup -from nemo_rl.utils.fastokens import normalize_fastokens_env +from nemo_rl.utils.fastokens import normalize_fastokens_env, vllm_uses_fastokens from nemo_rl.utils.multimodal_payload_metrics import ( collect_multimodal_payload_metrics, collect_sharded_multimodal_payload_metrics, @@ -770,6 +770,7 @@ def _start_dynamo_services(self) -> None: # The workers' venv carries ai-dynamo, so the frontend runs the same # release as every engine. frontend_python=self.worker_group.py_executable, + fastokens=vllm_uses_fastokens(), ) runtime.start() self._dynamo_runtime = runtime diff --git a/nemo_rl/utils/fastokens.py b/nemo_rl/utils/fastokens.py index 3efe24bb361..d37160b3bf0 100644 --- a/nemo_rl/utils/fastokens.py +++ b/nemo_rl/utils/fastokens.py @@ -48,6 +48,16 @@ def normalize_fastokens_env() -> None: os.environ[_VLLM_ENV_VAR] = "1" if nrl_value == "1" else "0" +def vllm_uses_fastokens() -> bool: + """Whether vLLM tokenizes with fastokens, once ``NRL_USE_FASTOKENS`` is mirrored. + + For another tokenizer serving the same requests, such as the Dynamo + frontend's, to follow vLLM's switch. + """ + normalize_fastokens_env() + return os.environ.get(_VLLM_ENV_VAR) == "1" + + def maybe_patch_fastokens(enabled: bool) -> None: """Apply the fastokens monkey-patch when enabled. diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 6d75c8bc19e..dda08c320b2 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -253,8 +253,8 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: if key in frontend_args: problems.append( f"policy.generation.dynamo_cfg.frontend_args.{key} was " - "removed: NeMo RL's rollouts send token IDs, which the " - "frontend never tokenizes." + "removed: the frontend tokenizes with fastokens when " + "vLLM does (NRL_USE_FASTOKENS)." ) worker_args = dynamo_cfg.get("worker_args") if isinstance(worker_args, dict) and "extra_cli_args" in worker_args: diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index d27f553e58a..8b22992ebb1 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -137,25 +137,45 @@ def test_frontend_argv_is_runtime_owned() -> None: cfg = DynamoCfg.model_validate( {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} ) - argv = frontend_argv(host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg) + argv = frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg, fastokens=False + ) assert argv[argv.index("--router-mode") + 1] == "kv" # Gym names every call's session; keep the session on one engine. assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "300" + assert "--tokenizer" not in argv assert argv[-2:] == ["--foo", "1"] no_affinity = DynamoCfg.model_validate( {"frontend_args": {"session_affinity_ttl_secs": None}} ) assert "--router-session-affinity-ttl-secs" not in frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=no_affinity + host="0.0.0.0", + port=3001, + namespace="nemo-rl", + dynamo_cfg=no_affinity, + fastokens=False, ) - - clashing = DynamoCfg.model_validate( - {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} + # The frontend tokenizes Gym chat as vLLM does: with fastokens when vLLM's is on. + fast = frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg, fastokens=True ) - with pytest.raises(ValueError, match="--http-port is set by both"): - frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=clashing + assert fast[fast.index("--tokenizer") + 1] == "fastokens" + + for extra, fastokens, flag in ( + (["--http-port", "9"], False, "--http-port"), + (["--tokenizer", "default"], True, "--tokenizer"), + ): + clashing = DynamoCfg.model_validate( + {"frontend_args": {"extra_cli_args": extra}} ) + with pytest.raises(ValueError, match=f"{flag} is set by both"): + frontend_argv( + host="0.0.0.0", + port=3001, + namespace="nemo-rl", + dynamo_cfg=clashing, + fastokens=fastokens, + ) def _serving(**serving_chat_kwargs) -> dict: diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 2c5c4dc3541..eca3833f36a 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -34,6 +34,7 @@ def _runtime(**frontend_args) -> ManagedDynamoRuntime: ), model_name="model", frontend_python="/venvs/vllm/bin/python", + fastokens=False, ) diff --git a/tests/unit/utils/test_fastokens.py b/tests/unit/utils/test_fastokens.py index d1ce8d602f2..ab6ac8bafa8 100644 --- a/tests/unit/utils/test_fastokens.py +++ b/tests/unit/utils/test_fastokens.py @@ -157,6 +157,25 @@ def test_nrl_env_overrides_explicit_vllm_env(monkeypatch): assert os.environ["VLLM_USE_FASTOKENS"] == "1" +@pytest.mark.parametrize( + ("nrl", "vllm", "expected"), + [ + (None, None, False), + (None, "1", True), + ("1", None, True), + ("0", "1", False), + ], +) +def test_vllm_uses_fastokens_follows_the_mirrored_switch( + monkeypatch, nrl, vllm, expected +): + for name, value in (("NRL_USE_FASTOKENS", nrl), ("VLLM_USE_FASTOKENS", vllm)): + if value is not None: + monkeypatch.setenv(name, value) + + assert fastokens_util.vllm_uses_fastokens() is expected + + def test_missing_package_is_non_fatal(monkeypatch): # A ``None`` entry in sys.modules makes ``import fastokens`` raise ImportError. monkeypatch.setitem(sys.modules, "fastokens", None) From a1659e9d673f74a6470f842cace6c7fed0ef4f31 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 6 Nov 2026 10:49:42 -0800 Subject: [PATCH 29/82] revert: drop the frontend fastokens flag and the every-call session pin Reverts 776aaa36d0 (feat(dynamo): tokenize Gym chat with fastokens in the frontend when vLLM does) and f75aa87cce (feat(dynamo): pin every NeMo-Gym session to one engine), including the Gym re-pin, which goes back to a7be5c5. The tree is c4fe2bd540's again. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/guides/dynamo-generation.md | 8 +--- ...grpo_math_1B_dynamo_single_controller.yaml | 2 +- nemo_rl/models/generation/dynamo/config.py | 13 +++---- .../generation/dynamo/managed_runtime.py | 21 ++-------- .../models/generation/vllm/vllm_generation.py | 3 +- nemo_rl/utils/fastokens.py | 10 ----- nemo_rl/utils/outdated_config_checks.py | 4 +- .../models/generation/test_dynamo_config.py | 38 +++++-------------- .../generation/test_dynamo_managed_runtime.py | 1 - tests/unit/utils/test_fastokens.py | 19 ---------- 11 files changed, 24 insertions(+), 97 deletions(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index a83f5c1dea8..a7be5c5f3a4 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit a83f5c1dea81c7ce4df1efd52025dbf8a2fc4215 +Subproject commit a7be5c5f3a4d0957c9e84a8f7face90e30b7171e diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 085f3ce6cda..4588d09e125 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -54,7 +54,7 @@ shows them all: | `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | | `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | -| `frontend_args.session_affinity_ttl_secs` | 300 | Keeps a NeMo-Gym rollout on one engine while it is idle for up to this many seconds; null disables it. Dynamo keeps at most 65,536 sessions and answers a new one past that with HTTP 529 | +| `frontend_args.session_affinity_ttl_secs` | 300 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it. Dynamo keeps at most 65,536 sessions and answers a new one past that with HTTP 529 | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | The etcd and nats-server binaries are found in `NEMO_RL_DYNAMO_BIN_DIR` when it @@ -109,12 +109,6 @@ same vLLM workers stage each call's tokens, log probabilities, routed experts an processed media from their own process, exactly as the native endpoint does, and only the commit coordinates travel back through the frontend. -Gym names each call's rollout, and the frontend keeps a rollout on one engine -(`session_affinity_ttl_secs`), as the native backend pins each Gym session to -one server. The engine splices the echoed prefix after the frontend has routed -the call, so the KV router alone cannot see where that prefix is cached. -The frontend tokenizes Gym chat with fastokens when vLLM does (`NRL_USE_FASTOKENS=1`). - The native server's chat settings carry over: | Native setting (`vllm_cfg`) | Under Dynamo | diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index e2e587b3936..2c27bf7f89c 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -35,7 +35,7 @@ policy: frontend_args: # kv: route each request to the engine holding most of its prefix. router_mode: kv - # Keep a NeMo-Gym rollout on one engine while it idles up + # Keep a NeMo-Gym token-capture rollout on one engine while it idles up # to this many seconds (null disables it). session_affinity_ttl_secs: 300 extra_cli_args: [] diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index ab716edaf41..0bb39711d86 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -65,13 +65,12 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "device-aware-weighted", ] = "kv" # Idle TTL in seconds for keeping one session on one engine, or None to - # disable it. NeMo-Gym names the session on every call (its rollout, else its - # own session), as the native backend pins each Gym session to one server; - # the engine splices a call's echoed or staged prefix after routing, so the - # KV router alone cannot see where that prefix is cached. Dynamo keeps at - # most 65,536 sessions and answers a new one past that with HTTP 529, so keep - # (rollouts started per second) x TTL below 65,536. The bound is the - # frontend's own. + # disable it. NeMo-Gym names the session on token-capture calls only (its + # capture key), as the native backend pins each Gym session to one server; the + # engine splices a call's staged prefix after routing, so the KV router alone + # cannot see where that prefix is cached. Dynamo keeps at most 65,536 sessions + # and answers a new one past that with HTTP 529, so keep (captured rollouts + # started per second) x TTL below 65,536. The bound is the frontend's own. session_affinity_ttl_secs: Annotated[int, Field(gt=0, le=31_536_000)] | None = 300 extra_cli_args: list[str] = Field(default_factory=list) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index fd3239a3f37..75a02dbba63 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -51,13 +51,9 @@ def frontend_argv( - *, host: str, port: int, namespace: str, dynamo_cfg: DynamoCfg, fastokens: bool + *, host: str, port: int, namespace: str, dynamo_cfg: DynamoCfg ) -> list[str]: - """The ``dynamo.frontend`` arguments; ``extra_cli_args`` may not repeat one. - - ``fastokens`` selects the fastokens tokenizer: the frontend tokenizes NeMo-Gym - chat, which native vLLM tokenizes with fastokens when vLLM's switch is on. - """ + """The ``dynamo.frontend`` arguments; ``extra_cli_args`` may not repeat one.""" argv = [ "--http-host", host, @@ -79,8 +75,6 @@ def frontend_argv( "--router-session-affinity-ttl-secs", str(dynamo_cfg.frontend_args.session_affinity_ttl_secs), ] - if fastokens: - argv += ["--tokenizer", "fastokens"] extra = dynamo_cfg.frontend_args.extra_cli_args for token in extra: flag = token.split("=", 1)[0].replace("_", "-") @@ -125,12 +119,7 @@ class ManagedDynamoRuntime: """Own etcd, NATS and the frontend that NeMo RL's vLLM engines register with.""" def __init__( - self, - *, - dynamo_cfg: DynamoCfg, - model_name: str, - frontend_python: str, - fastokens: bool, + self, *, dynamo_cfg: DynamoCfg, model_name: str, frontend_python: str ) -> None: """Describe the services; nothing starts until :meth:`start`. @@ -139,13 +128,10 @@ def __init__( model_name: The served model; the frontend is ready once it lists it. frontend_python: An interpreter whose environment has ``ai-dynamo``: the vLLM workers' own. - fastokens: Whether the frontend tokenizes with fastokens, as the vLLM - workers do. """ self._dynamo_cfg = dynamo_cfg self._model_name = model_name self._frontend_python = frontend_python - self._fastokens = fastokens self._namespace = _managed_namespace() self._host = "" self._etcd_port = 0 @@ -345,7 +331,6 @@ def _start_frontend(self) -> None: port=self._frontend_port, namespace=self._namespace, dynamo_cfg=self._dynamo_cfg, - fastokens=self._fastokens, ) command = [ self._frontend_python, diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 8c8bd4c2731..bc7b5e27eb7 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -81,7 +81,7 @@ from nemo_rl.telemetry.metrics import record_rl_metrics, warn_once from nemo_rl.telemetry.setup import get_telemetry_handle from nemo_rl.telemetry.span_groups import RLSpanGroup -from nemo_rl.utils.fastokens import normalize_fastokens_env, vllm_uses_fastokens +from nemo_rl.utils.fastokens import normalize_fastokens_env from nemo_rl.utils.multimodal_payload_metrics import ( collect_multimodal_payload_metrics, collect_sharded_multimodal_payload_metrics, @@ -770,7 +770,6 @@ def _start_dynamo_services(self) -> None: # The workers' venv carries ai-dynamo, so the frontend runs the same # release as every engine. frontend_python=self.worker_group.py_executable, - fastokens=vllm_uses_fastokens(), ) runtime.start() self._dynamo_runtime = runtime diff --git a/nemo_rl/utils/fastokens.py b/nemo_rl/utils/fastokens.py index d37160b3bf0..3efe24bb361 100644 --- a/nemo_rl/utils/fastokens.py +++ b/nemo_rl/utils/fastokens.py @@ -48,16 +48,6 @@ def normalize_fastokens_env() -> None: os.environ[_VLLM_ENV_VAR] = "1" if nrl_value == "1" else "0" -def vllm_uses_fastokens() -> bool: - """Whether vLLM tokenizes with fastokens, once ``NRL_USE_FASTOKENS`` is mirrored. - - For another tokenizer serving the same requests, such as the Dynamo - frontend's, to follow vLLM's switch. - """ - normalize_fastokens_env() - return os.environ.get(_VLLM_ENV_VAR) == "1" - - def maybe_patch_fastokens(enabled: bool) -> None: """Apply the fastokens monkey-patch when enabled. diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index dda08c320b2..6d75c8bc19e 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -253,8 +253,8 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: if key in frontend_args: problems.append( f"policy.generation.dynamo_cfg.frontend_args.{key} was " - "removed: the frontend tokenizes with fastokens when " - "vLLM does (NRL_USE_FASTOKENS)." + "removed: NeMo RL's rollouts send token IDs, which the " + "frontend never tokenizes." ) worker_args = dynamo_cfg.get("worker_args") if isinstance(worker_args, dict) and "extra_cli_args" in worker_args: diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 8b22992ebb1..46a23b994c8 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -137,45 +137,25 @@ def test_frontend_argv_is_runtime_owned() -> None: cfg = DynamoCfg.model_validate( {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} ) - argv = frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg, fastokens=False - ) + argv = frontend_argv(host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg) assert argv[argv.index("--router-mode") + 1] == "kv" - # Gym names every call's session; keep the session on one engine. + # Gym names a token-capture call's session; keep the session on one engine. assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "300" - assert "--tokenizer" not in argv assert argv[-2:] == ["--foo", "1"] no_affinity = DynamoCfg.model_validate( {"frontend_args": {"session_affinity_ttl_secs": None}} ) assert "--router-session-affinity-ttl-secs" not in frontend_argv( - host="0.0.0.0", - port=3001, - namespace="nemo-rl", - dynamo_cfg=no_affinity, - fastokens=False, - ) - # The frontend tokenizes Gym chat as vLLM does: with fastokens when vLLM's is on. - fast = frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg, fastokens=True + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=no_affinity ) - assert fast[fast.index("--tokenizer") + 1] == "fastokens" - for extra, fastokens, flag in ( - (["--http-port", "9"], False, "--http-port"), - (["--tokenizer", "default"], True, "--tokenizer"), - ): - clashing = DynamoCfg.model_validate( - {"frontend_args": {"extra_cli_args": extra}} + clashing = DynamoCfg.model_validate( + {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} + ) + with pytest.raises(ValueError, match="--http-port is set by both"): + frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=clashing ) - with pytest.raises(ValueError, match=f"{flag} is set by both"): - frontend_argv( - host="0.0.0.0", - port=3001, - namespace="nemo-rl", - dynamo_cfg=clashing, - fastokens=fastokens, - ) def _serving(**serving_chat_kwargs) -> dict: diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index eca3833f36a..2c5c4dc3541 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -34,7 +34,6 @@ def _runtime(**frontend_args) -> ManagedDynamoRuntime: ), model_name="model", frontend_python="/venvs/vllm/bin/python", - fastokens=False, ) diff --git a/tests/unit/utils/test_fastokens.py b/tests/unit/utils/test_fastokens.py index ab6ac8bafa8..d1ce8d602f2 100644 --- a/tests/unit/utils/test_fastokens.py +++ b/tests/unit/utils/test_fastokens.py @@ -157,25 +157,6 @@ def test_nrl_env_overrides_explicit_vllm_env(monkeypatch): assert os.environ["VLLM_USE_FASTOKENS"] == "1" -@pytest.mark.parametrize( - ("nrl", "vllm", "expected"), - [ - (None, None, False), - (None, "1", True), - ("1", None, True), - ("0", "1", False), - ], -) -def test_vllm_uses_fastokens_follows_the_mirrored_switch( - monkeypatch, nrl, vllm, expected -): - for name, value in (("NRL_USE_FASTOKENS", nrl), ("VLLM_USE_FASTOKENS", vllm)): - if value is not None: - monkeypatch.setenv(name, value) - - assert fastokens_util.vllm_uses_fastokens() is expected - - def test_missing_package_is_non_fatal(monkeypatch): # A ``None`` entry in sys.modules makes ``import fastokens`` raise ImportError. monkeypatch.setitem(sys.modules, "fastokens", None) From f774780817edfd57f1dd3d450ac83ceb93113e22 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 6 Nov 2026 18:10:33 -0800 Subject: [PATCH 30/82] refactor(dynamo): split the engine adapter into a shared Dynamo side and a vLLM backend C61. `dynamo/engine_adapter.py` holds `DynamoEngineAdapter`, the Dynamo side written once for every generation backend: request kinds as handlers keyed by their annotation, the error mapping, and `start_dynamo_endpoint` (completions only). A backend implements `serve_rollout`, `start`, `abort` and `kv_event_sources`. `NativeVllmEngine` becomes `DynamoVllmAdapter` in `dynamo/vllm_adapter.py`. `RolloutResult` and its wire form move from `vllm/request_pipeline.py` to the backend-neutral `rollout_result.py`, so the shared side imports no backend; the decoupling test checks that. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/dynamo/engine_adapter.py | 205 ++++++++++++++++++ .../generation/dynamo/managed_runtime.py | 4 +- .../generation/dynamo/rollout_client.py | 4 +- .../{native_engine.py => vllm_adapter.py} | 178 ++++----------- nemo_rl/models/generation/rollout_result.py | 82 +++++++ .../generation/vllm/request_pipeline.py | 59 +---- .../models/generation/vllm/vllm_generation.py | 6 +- .../generation/vllm/vllm_worker_async.py | 17 +- pyrefly.toml | 4 +- ..._engine.py => test_dynamo_vllm_adapter.py} | 43 ++-- .../generation/test_dynamo_vllm_decoupling.py | 56 +++-- .../models/generation/test_vllm_generation.py | 4 +- .../generation/test_vllm_request_pipeline.py | 8 +- 13 files changed, 418 insertions(+), 252 deletions(-) create mode 100644 nemo_rl/models/generation/dynamo/engine_adapter.py rename nemo_rl/models/generation/dynamo/{native_engine.py => vllm_adapter.py} (51%) create mode 100644 nemo_rl/models/generation/rollout_result.py rename tests/unit/models/generation/{test_dynamo_native_engine.py => test_dynamo_vllm_adapter.py} (90%) diff --git a/nemo_rl/models/generation/dynamo/engine_adapter.py b/nemo_rl/models/generation/dynamo/engine_adapter.py new file mode 100644 index 00000000000..9d21a0383fc --- /dev/null +++ b/nemo_rl/models/generation/dynamo/engine_adapter.py @@ -0,0 +1,205 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serve a NeMo RL generation worker's own engine through Dynamo's backend SDK. + +``DynamoEngineAdapter`` is the Dynamo side, written once for every backend: the +request kinds served, how errors reach the driver, and the endpoint's +registration. A backend implements four methods over its worker's engine +(``serve_rollout``, ``start``, ``abort`` and ``kv_event_sources``); +``vllm_adapter.DynamoVllmAdapter`` is the vLLM one. The engine only ever receives +token IDs, on ``/v1/completions``. This module imports no generation backend. +""" + +from __future__ import annotations + +import asyncio +import json +import os +from abc import abstractmethod +from collections.abc import AsyncGenerator, Callable +from dataclasses import dataclass +from typing import Any, Optional + +from dynamo._core import Context +from dynamo.common.backend import ( + EngineConfig, + GenerateChunk, + GenerateRequest, + LLMEngine, + Worker, + WorkerConfig, +) +from dynamo.common.backend.publisher import KvEventSource +from dynamo.llm.exceptions import InvalidArgument + +from nemo_rl.models.generation.dynamo.managed_runtime import ( + DYNAMO_COMPONENT, + DYNAMO_ENDPOINT, +) +from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION +from nemo_rl.models.generation.rollout_result import ( + RolloutResult, + rollout_result_to_wire, +) + +# Serves one request kind: the request, its annotation's value, and the context. +RequestHandler = Callable[ + [GenerateRequest, str, Context], AsyncGenerator[GenerateChunk, None] +] + + +@dataclass(frozen=True) +class ServedRollout: + """A backend's answer to one rollout.""" + + result: RolloutResult + # The OpenAI finish reason the frontend reports. + finish_reason: str + # The OpenAI completion usage; None when no new token fit. + usage: Optional[dict[str, Any]] + + +def _annotation_value(request: GenerateRequest, prefix: str) -> Optional[str]: + """The value of the first ```` annotation, or None.""" + for annotation in request.get("annotations") or (): + if annotation.startswith(prefix): + return annotation[len(prefix) :] + return None + + +class DynamoEngineAdapter(LLMEngine): + """Serves one NeMo RL generation worker through Dynamo. The worker owns the engine.""" + + def __init__(self, *, leader_worker_idx: int): + # The worker's index in its group; rollouts report it, as + # generate_async results do. + self._leader_worker_idx = leader_worker_idx + + @classmethod + async def from_args(cls, argv=None): + raise NotImplementedError("constructed in-process by the generation worker") + + def _handlers(self) -> dict[str, RequestHandler]: + """The request kinds served, keyed by the annotation prefix that marks them.""" + return {ROLLOUT_ANNOTATION: self._generate_rollout} + + async def generate( + self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + handlers = self._handlers() + for prefix, handler in handlers.items(): + value = _annotation_value(request, prefix) + if value is not None: + break + else: + raise InvalidArgument( + "NeMo RL's engines serve only NeMo RL rollouts through Dynamo " + f"(requests annotated {', '.join(map(repr, handlers))})." + ) + try: + async for chunk in handler(request, value, context): + yield chunk + except RuntimeError as error: + # A 5xx reaches the driver as "Internal server error" and is re-sent; + # a 4xx keeps the cause, and the rollout retry policy still applies. + raise InvalidArgument(f"{type(error).__name__}: {error}") from error + + async def _generate_rollout( + self, request: GenerateRequest, rollout_spec: str, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + """One NeMo RL rollout, served by the backend's ``serve_rollout``. + + One chunk carries the result, routed experts included, in ``engine_data``; + the driver assembles the same output ``generate_async`` returns. + """ + rollout = json.loads(rollout_spec) + served = await self.serve_rollout( + list(request["token_ids"]), + greedy=rollout["greedy"], + stop_strings=rollout["stop_strings"], + request_id=context.id(), + ) + chunk: GenerateChunk = { + "token_ids": served.result.token_ids, + "index": 0, + "finish_reason": served.finish_reason, + "engine_data": { + "rollout": rollout_result_to_wire(served.result), + "gen_leader_worker_idx": self._leader_worker_idx, + }, + } + if served.usage is not None: + chunk["completion_usage"] = served.usage + yield chunk + + @abstractmethod + async def serve_rollout( + self, + prompt_token_ids: list[int], + *, + greedy: bool, + stop_strings: Optional[list[str]], + request_id: str, + ) -> ServedRollout: + """Serve one rollout exactly as the worker's native ``generate_async`` does.""" + + @abstractmethod + async def start(self, worker_id: int) -> EngineConfig: + """Register the engine's context length, KV block size and capacity.""" + + @abstractmethod + async def abort(self, context: Context) -> None: + """Cancel the request ``context`` names; Dynamo calls it on disconnect.""" + + @abstractmethod + async def kv_event_sources(self) -> list[KvEventSource]: + """Where the engine publishes the KV-cache events Dynamo's KV router indexes.""" + + async def cleanup(self) -> None: + """Nothing to release: the worker owns its engine and shuts it down.""" + + +def start_dynamo_endpoint( + adapter: DynamoEngineAdapter, + *, + model_path: str, + served_model_name: str, + namespace: str, + engine_env: dict[str, str], +) -> asyncio.Task: + """Register ``adapter``'s engine with Dynamo and serve it on the running loop. + + ``model_path`` is a local directory with the model's config and tokenizer + files; the engine is served under ``served_model_name``. + + The Rust Worker serves until the process gets SIGTERM. It installs a chained + handler, so Ray's actor handler still runs and the actor exits as usual; + discovery drops the engine with its process. + """ + # The Rust runtime reads these when the first Worker is constructed. + os.environ.update(engine_env) + config = WorkerConfig( + namespace=namespace, + component=DYNAMO_COMPONENT, + endpoint=DYNAMO_ENDPOINT, + model_name=model_path, + served_model_name=served_model_name, + # NeMo RL's rollouts go to /v1/completions only. + endpoint_types="completions", + discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], + request_plane=engine_env["DYN_REQUEST_PLANE"], + event_plane=engine_env["DYN_EVENT_PLANE"], + ) + return asyncio.get_running_loop().create_task(Worker(adapter, config).run()) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index ee4f0383836..be438e817b0 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -15,7 +15,7 @@ """Driver-owned Dynamo services: etcd, NATS and the OpenAI frontend. The vLLM engines are NeMo RL's own workers; each registers itself with these -services (see ``native_engine``), so the runtime only waits to see them. +services (see ``engine_adapter``), so the runtime only waits to see them. """ import atexit @@ -43,7 +43,7 @@ ) from nemo_rl.models.generation.dynamo.config import DynamoCfg -# The engines register on this component and endpoint (see native_engine). +# The engines register on this component and endpoint (see engine_adapter). DYNAMO_COMPONENT = "backend" DYNAMO_ENDPOINT = "generate" diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py index 5e1346f4b2c..2683e6525d2 100644 --- a/nemo_rl/models/generation/dynamo/rollout_client.py +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -25,7 +25,7 @@ LOGGER = logging.getLogger(__name__) -# Marks a request as a NeMo RL rollout for the engine (native_engine), which +# Marks a request as a NeMo RL rollout for the engine (engine_adapter), which # then samples it from the generation config. Defined here, not in the engine # module, so the driver need not import ai-dynamo. ROLLOUT_ANNOTATION = "nemo_rl:rollout:" @@ -120,7 +120,7 @@ def rollout_payload( def parse_rollout_response(response: dict[str, Any], *, url: str) -> dict[str, Any]: """Return the engine's ``engine_data``, or raise with the frontend's error. - It holds the rollout result (``request_pipeline.rollout_result_to_wire``) + It holds the rollout result (``rollout_result.rollout_result_to_wire``) and the serving worker's index. """ if response.get("status") == "error": diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/vllm_adapter.py similarity index 51% rename from nemo_rl/models/generation/dynamo/native_engine.py rename to nemo_rl/models/generation/dynamo/vllm_adapter.py index 23937f93354..9c55df7d666 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm_adapter.py @@ -14,53 +14,36 @@ """Serve a NeMo RL vLLM worker's own AsyncLLM through Dynamo's backend SDK. -The engine process holds no Dynamo-owned vLLM code. This adapter implements -``dynamo.common.backend.LLMEngine``, which imports no vLLM, over the AsyncLLM -that ``VllmAsyncGenerationWorkerImpl`` owns, so refit, pause, sleep, -quantization and speculative decoding are that worker's own. +The engine process holds no Dynamo-owned vLLM code. ``DynamoVllmAdapter`` is the +vLLM backend of ``engine_adapter.DynamoEngineAdapter``, over the AsyncLLM that +``VllmAsyncGenerationWorkerImpl`` owns, so refit, pause, sleep, quantization and +speculative decoding are that worker's own. """ from __future__ import annotations -import asyncio -import json -import logging import os -from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional import torch from dynamo._core import Context -from dynamo.common.backend import ( - EngineConfig, - GenerateChunk, - GenerateRequest, - LLMEngine, - LlmRegistration, - Worker, - WorkerConfig, -) +from dynamo.common.backend import EngineConfig, LlmRegistration from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource -from dynamo.llm.exceptions import InvalidArgument from vllm.inputs import TokensPrompt from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase from nemo_rl.models.generation.dynamo.config import DynamoCfg -from nemo_rl.models.generation.dynamo.managed_runtime import ( - DYNAMO_COMPONENT, - DYNAMO_ENDPOINT, +from nemo_rl.models.generation.dynamo.engine_adapter import ( + DynamoEngineAdapter, + ServedRollout, ) -from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION -from nemo_rl.models.generation.vllm.request_pipeline import rollout_result_to_wire if TYPE_CHECKING: from nemo_rl.models.generation.vllm.vllm_worker_async import ( VllmAsyncGenerationWorkerImpl, ) -LOGGER = logging.getLogger(__name__) - # vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them # (dynamo/vllm/cache_info.py MAIN_ATTENTION_KV_CACHE_KINDS, which NeMo RL may # not import); test_main_attention_kinds_match_dynamos checks the copy. @@ -89,24 +72,6 @@ def dynamo_kv_events_config( } -def _annotation_value(request: GenerateRequest, prefix: str) -> Optional[str]: - """The value of the first ```` annotation, or None.""" - for annotation in request.get("annotations") or (): - if annotation.startswith(prefix): - return annotation[len(prefix) :] - return None - - -def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: - """The rollout spec if ``request`` is one of NeMo RL's own rollouts. - - Those come from ``VllmGeneration.generate_async`` and are built here - exactly as the native path builds them. - """ - rollout = _annotation_value(request, ROLLOUT_ANNOTATION) - return None if rollout is None else json.loads(rollout) - - class SnapshotStatLogger(StatLoggerBase): """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" @@ -143,18 +108,40 @@ def log_engine_initialized(self) -> None: pass -class NativeVllmEngine(LLMEngine): - """LLMEngine over a running NeMo RL vLLM worker. Does not own its lifecycle.""" +class DynamoVllmAdapter(DynamoEngineAdapter): + """The vLLM backend, over a running NeMo RL vLLM worker it does not own.""" - def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int): + def __init__( + self, host: "VllmAsyncGenerationWorkerImpl", *, leader_worker_idx: int + ): + super().__init__(leader_worker_idx=leader_worker_idx) self._host = host - self._leader_worker_idx = leader_worker_idx self._llm = host.llm self._max_model_len = self._llm.vllm_config.model_config.max_model_len - @classmethod - async def from_args(cls, argv=None): - raise NotImplementedError("constructed in-process by the vLLM worker") + async def serve_rollout( + self, + prompt_token_ids: list[int], + *, + greedy: bool, + stop_strings: Optional[list[str]], + request_id: str, + ) -> ServedRollout: + result, final_output = await self._host._serve_rollout( + TokensPrompt(prompt_token_ids=prompt_token_ids), + prompt_token_ids=prompt_token_ids, + greedy=greedy, + stop_strings=stop_strings, + request_id=request_id, + device=torch.device("cpu"), + ) + if final_output is None: + return ServedRollout(result, finish_reason="length", usage=None) + return ServedRollout( + result, + finish_reason=final_output.outputs[0].finish_reason, + usage=_usage(final_output, len(result.token_ids)), + ) async def start(self, worker_id: int) -> EngineConfig: del worker_id @@ -184,57 +171,6 @@ async def start(self, worker_id: int) -> EngineConfig: ), ) - async def generate( - self, request: GenerateRequest, context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - rollout = _rollout_request(request) - if rollout is None: - raise InvalidArgument( - "NeMo RL's vLLM engines serve only NeMo RL rollouts through Dynamo " - f"(requests annotated {ROLLOUT_ANNOTATION!r})." - ) - async for chunk in self._generate_rollout(request, rollout, context): - yield chunk - - async def _generate_rollout( - self, request: GenerateRequest, rollout: dict[str, Any], context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - """One NeMo RL rollout, served by the worker's own ``_serve_rollout``. - - One chunk carries the result, routed experts included, in ``engine_data``; - the driver assembles the same output ``generate_async`` returns. - """ - prompt_token_ids = list(request["token_ids"]) - try: - result, final_output = await self._host._serve_rollout( - TokensPrompt(prompt_token_ids=prompt_token_ids), - prompt_token_ids=prompt_token_ids, - greedy=rollout["greedy"], - stop_strings=rollout["stop_strings"], - request_id=context.id(), - device=torch.device("cpu"), - ) - except RuntimeError as error: - # A 5xx reaches the driver as "Internal server error" and is re-sent; - # a 4xx keeps the cause, and the rollout retry policy still applies. - raise InvalidArgument(f"{type(error).__name__}: {error}") from error - chunk: GenerateChunk = { - "token_ids": result.token_ids, - "index": 0, - "finish_reason": ( - "length" - if final_output is None - else final_output.outputs[0].finish_reason - ), - "engine_data": { - "rollout": rollout_result_to_wire(result), - "gen_leader_worker_idx": self._leader_worker_idx, - }, - } - if final_output is not None: - chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) - yield chunk - async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) @@ -265,43 +201,3 @@ def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: "total_tokens": prompt_tokens + num_output_tokens, "prompt_tokens_details": {"cached_tokens": output.num_cached_tokens or 0}, } - - -def start_dynamo_endpoint( - host: "VllmAsyncGenerationWorkerImpl", - *, - model_path: str, - namespace: str, - engine_env: dict[str, str], - leader_worker_idx: int, -) -> asyncio.Task: - """Register ``host``'s engine with Dynamo and serve it on the running loop. - - ``model_path`` is a local directory with the model's config and tokenizer - files; the engine is served under ``host.model_name``. - - The Rust Worker serves until the process gets SIGTERM. It installs a chained - handler, so Ray's actor handler still runs and the actor exits as usual; - discovery drops the engine with its process. - """ - # The Rust runtime reads these when the first Worker is constructed. - os.environ.update(engine_env) - kv_events = host.llm.vllm_config.kv_events_config - publishes = kv_events is not None and kv_events.enable_kv_cache_events - # The Dynamo functional overlays check the router applied these. - print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) - config = WorkerConfig( - namespace=namespace, - component=DYNAMO_COMPONENT, - endpoint=DYNAMO_ENDPOINT, - model_name=model_path, - served_model_name=host.model_name, - # NeMo RL's rollouts go to /v1/completions only. - endpoint_types="completions", - discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], - request_plane=engine_env["DYN_REQUEST_PLANE"], - event_plane=engine_env["DYN_EVENT_PLANE"], - ) - return asyncio.get_running_loop().create_task( - Worker(NativeVllmEngine(host, leader_worker_idx), config).run() - ) diff --git a/nemo_rl/models/generation/rollout_result.py b/nemo_rl/models/generation/rollout_result.py new file mode 100644 index 00000000000..fe965fed8d0 --- /dev/null +++ b/nemo_rl/models/generation/rollout_result.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One rollout's result and its JSON wire form, for every generation backend. + +An engine served through Dynamo sends the result to the driver in +``engine_data``; the vLLM worker builds it with ``request_pipeline.rollout_result``. +""" + +import math +from dataclasses import dataclass +from typing import Any, Optional + +import torch + +from nemo_rl.utils.routed_experts_codec import ( + decode_routed_experts, + encode_routed_experts, +) + + +@dataclass +class RolloutResult: + """One rollout's outcome, independent of where its output is assembled.""" + + token_ids: list[int] + # The sampled token's log probability at each position; 0.0 where vLLM + # reported none. + logprobs: list[float] + # The engine stopped at max_tokens. + truncated: bool + # [prompt + generated, num_moe_layers, topk], aligned for router replay. + routed_experts: Optional[torch.Tensor] + # missing/expected/actual route counts; set when router replay is on. + r3_stats: Optional[dict[str, int]] + + @classmethod + def nothing_generated(cls) -> "RolloutResult": + """The result when no new token fits: the prompt alone, untruncated.""" + return cls([], [], False, None, None) + + +def rollout_result_to_wire(result: RolloutResult) -> dict[str, Any]: + """JSON-safe form of ``result``, for engines that answer over HTTP.""" + return { + "token_ids": result.token_ids, + # Strict JSON has no NaN or infinity, and Dynamo's frontend turns them + # into null; send them as the strings float() reads back. + "logprobs": [lp if math.isfinite(lp) else str(lp) for lp in result.logprobs], + "truncated": result.truncated, + "routed_experts": ( + None + if result.routed_experts is None + else encode_routed_experts(result.routed_experts) + ), + "r3_stats": result.r3_stats, + } + + +def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: + """Inverse of :func:`rollout_result_to_wire`; routes keep their own dtype.""" + routed_experts = wire["routed_experts"] + if routed_experts is not None: + routed_experts = decode_routed_experts(routed_experts, dtype=None) + return RolloutResult( + token_ids=list(wire["token_ids"]), + logprobs=[float(lp) for lp in wire["logprobs"]], + truncated=bool(wire["truncated"]), + routed_experts=routed_experts, + r3_stats=wire["r3_stats"], + ) diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index 107a2529a49..62ddf51fa6f 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -19,44 +19,18 @@ """ import logging -import math -from dataclasses import dataclass from typing import Any, Optional, cast import torch from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.models.generation.interfaces import GenerationOutputSpec +from nemo_rl.models.generation.rollout_result import RolloutResult from nemo_rl.models.generation.vllm.utils import pad_and_align_routed_expert_indices -from nemo_rl.utils.routed_experts_codec import ( - decode_routed_experts, - encode_routed_experts, -) LOGGER = logging.getLogger(__name__) -@dataclass -class RolloutResult: - """One rollout's outcome, independent of where its output is assembled.""" - - token_ids: list[int] - # The sampled token's log probability at each position; 0.0 where vLLM - # reported none. - logprobs: list[float] - # The engine stopped at max_tokens. - truncated: bool - # [prompt + generated, num_moe_layers, topk], aligned for router replay. - routed_experts: Optional[torch.Tensor] - # missing/expected/actual route counts; set when router replay is on. - r3_stats: Optional[dict[str, int]] - - @classmethod - def nothing_generated(cls) -> "RolloutResult": - """The result when no new token fits: the prompt alone, untruncated.""" - return cls([], [], False, None, None) - - def _selected_logprobs(completion: Any) -> list[float]: """The sampled token's log probability at each position of a completion.""" by_position = completion.logprobs or () @@ -171,34 +145,3 @@ def rollout_output( if result.routed_experts is not None: output["routed_experts"] = result.routed_experts.to(device).unsqueeze(0) return BatchedDataDict[GenerationOutputSpec](output) - - -def rollout_result_to_wire(result: RolloutResult) -> dict[str, Any]: - """JSON-safe form of ``result``, for engines that answer over HTTP.""" - return { - "token_ids": result.token_ids, - # Strict JSON has no NaN or infinity, and Dynamo's frontend turns them - # into null; send them as the strings float() reads back. - "logprobs": [lp if math.isfinite(lp) else str(lp) for lp in result.logprobs], - "truncated": result.truncated, - "routed_experts": ( - None - if result.routed_experts is None - else encode_routed_experts(result.routed_experts) - ), - "r3_stats": result.r3_stats, - } - - -def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: - """Inverse of :func:`rollout_result_to_wire`; routes keep their own dtype.""" - routed_experts = wire["routed_experts"] - if routed_experts is not None: - routed_experts = decode_routed_experts(routed_experts, dtype=None) - return RolloutResult( - token_ids=list(wire["token_ids"]), - logprobs=[float(lp) for lp in wire["logprobs"]], - truncated=bool(wire["truncated"]), - routed_experts=routed_experts, - r3_stats=wire["r3_stats"], - ) diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index cc165461797..79ec319bc04 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -52,6 +52,7 @@ agreed_media_columns, verify_right_padding, ) +from nemo_rl.models.generation.rollout_result import rollout_result_from_wire from nemo_rl.models.generation.vllm.config import ( REFITTABLE_FP8_KV_CACHE_DTYPES, VllmConfig, @@ -63,10 +64,7 @@ draft_module_sharing_disable_required, ) from nemo_rl.models.generation.vllm.metric_names import BATCH_DURATION_KEY -from nemo_rl.models.generation.vllm.request_pipeline import ( - rollout_output, - rollout_result_from_wire, -) +from nemo_rl.models.generation.vllm.request_pipeline import rollout_output from nemo_rl.models.generation.vllm.utils import ( aggregate_spec_decode_counters, assert_reload_refit_config_supported, diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 9562dab8232..38ba30aaebc 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -69,8 +69,8 @@ resolve_collective_rpc_result, ) from nemo_rl.models.generation.vllm.config import parse_nvfp4_pertoken_rollout +from nemo_rl.models.generation.rollout_result import RolloutResult from nemo_rl.models.generation.vllm.request_pipeline import ( - RolloutResult, rollout_output, rollout_result, ) @@ -446,7 +446,7 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: else [] ) if dynamo_enabled(self.cfg): - from nemo_rl.models.generation.dynamo.native_engine import ( + from nemo_rl.models.generation.dynamo.vllm_adapter import ( SnapshotStatLogger, dynamo_kv_events_config, ) @@ -663,9 +663,10 @@ async def start_dynamo_endpoint_async( # ai-dynamo is optional: import it only for engines served through Dynamo. from dynamo._core import fetch_model - from nemo_rl.models.generation.dynamo.native_engine import ( + from nemo_rl.models.generation.dynamo.engine_adapter import ( start_dynamo_endpoint, ) + from nemo_rl.models.generation.dynamo.vllm_adapter import DynamoVllmAdapter # Dynamo fetches a Hub ID's whole checkpoint before it registers; the engine's # weights come from refit, so register the config and tokenizer files only. @@ -674,12 +675,16 @@ async def start_dynamo_endpoint_async( if os.path.exists(self.model_name) else await fetch_model(self.model_name, ignore_weights=True) ) + kv_events = self.llm.vllm_config.kv_events_config + publishes = kv_events is not None and kv_events.enable_kv_cache_events + # The Dynamo functional overlays check the router applied these. + print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) self._dynamo_endpoint_task = start_dynamo_endpoint( - self, + DynamoVllmAdapter(self, leader_worker_idx=leader_worker_idx), model_path=model_path, + served_model_name=self.model_name, namespace=namespace, engine_env=engine_env, - leader_worker_idx=leader_worker_idx, ) async def dynamo_endpoint_error(self) -> Optional[str]: @@ -2116,7 +2121,7 @@ async def _serve_rollout( Returns: The result, and vLLM's final RequestOutput (None when no new token fits). The final output supplies finish and usage metadata to Dynamo - (``dynamo/native_engine.py``). + (``dynamo/vllm_adapter.py``). """ max_new_tokens = self._rollout_max_new_tokens(len(prompt_token_ids)) if max_new_tokens == 0: diff --git a/pyrefly.toml b/pyrefly.toml index 1361d58cad0..52beefd9ee2 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -241,7 +241,8 @@ project-includes = [ "nemo_rl/models/generation/dynamo/__init__.py", "nemo_rl/models/generation/dynamo/config.py", "nemo_rl/models/generation/dynamo/managed_runtime.py", - "nemo_rl/models/generation/dynamo/native_engine.py", + "nemo_rl/models/generation/dynamo/engine_adapter.py", + "nemo_rl/models/generation/dynamo/vllm_adapter.py", "nemo_rl/models/generation/dynamo/rollout_client.py", "nemo_rl/models/generation/engine_supervisor.py", "nemo_rl/models/generation/fleet_health.py", @@ -253,6 +254,7 @@ project-includes = [ "nemo_rl/models/generation/megatron/token_capture.py", "nemo_rl/models/generation/megatron/utils.py", "nemo_rl/models/generation/openai_server_utils.py", + "nemo_rl/models/generation/rollout_result.py", "nemo_rl/models/generation/capture_context.py", "nemo_rl/models/generation/sglang/config.py", "nemo_rl/models/generation/sglang/utils/http_utils.py", diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py similarity index 90% rename from tests/unit/models/generation/test_dynamo_native_engine.py rename to tests/unit/models/generation/test_dynamo_vllm_adapter.py index 0ef89990af2..bb52d519ff9 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""NeMo RL's Dynamo engine adapter over a fake AsyncLLM. +"""The vLLM Dynamo adapter over a fake AsyncLLM. The host is a real ``VllmAsyncGenerationWorkerImpl`` built without its constructor, so rollouts are sampled by the native worker's own helpers. @@ -34,7 +34,10 @@ from dynamo.llm.exceptions import InvalidArgument # noqa: E402 -from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.dynamo import ( # noqa: E402 + engine_adapter, + vllm_adapter, +) from nemo_rl.models.generation.dynamo.config import ( # noqa: E402 DynamoCfg, DynamoFrontendArgs, @@ -42,11 +45,13 @@ from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) +from nemo_rl.models.generation.rollout_result import ( # noqa: E402 + rollout_result_from_wire, + rollout_result_to_wire, +) from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, - rollout_result_from_wire, - rollout_result_to_wire, ) from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, @@ -137,7 +142,7 @@ def test_rollouts_are_served_like_generate_async() -> None: final = _output([1, 2, 3], _completion([10, 11, 12], "stop")) llm = _FakeLLM([final]) host = _host(llm) - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=3) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=3) (chunk,) = _collect( engine, _rollout_request([1, 2, 3], greedy=True, stop_strings=[""]) @@ -171,7 +176,7 @@ async def abort(request_id): aborted.append(request_id) llm.abort = abort - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) _collect(engine, _rollout_request([1, 2, 3])) # Dynamo's worker calls abort() when the frontend cancels a request. @@ -189,7 +194,7 @@ def test_router_replay_routes_reach_rollouts_as_natively() -> None: prompt_routed_experts=routes[:3], ) host = _host(_FakeLLM([final]), vllm_kwargs={"enable_return_routed_experts": True}) - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) (chunk,) = _collect(engine, _rollout_request([1, 2, 3])) @@ -218,7 +223,7 @@ def test_router_replay_routes_reach_rollouts_as_natively() -> None: def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) (chunk,) = _collect(engine, _rollout_request(list(range(MAX_MODEL_LEN)))) @@ -227,7 +232,7 @@ def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: assert rollout["token_ids"] == [] and rollout["truncated"] is False -def _start_worker_config(monkeypatch, host) -> native_engine.WorkerConfig: +def _start_worker_config(monkeypatch, host) -> engine_adapter.WorkerConfig: """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" configs = [] @@ -238,7 +243,7 @@ def __init__(self, engine, config): async def run(self): return None - monkeypatch.setattr(native_engine, "Worker", _Worker) + monkeypatch.setattr(engine_adapter, "Worker", _Worker) env = { "DYN_DISCOVERY_BACKEND": "etcd", "DYN_REQUEST_PLANE": "tcp", @@ -249,12 +254,12 @@ async def run(self): monkeypatch.setenv(key, value) async def start(): - await native_engine.start_dynamo_endpoint( - host, + await engine_adapter.start_dynamo_endpoint( + vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0), model_path="/snapshots/model", + served_model_name=host.model_name, namespace="ns", engine_env=env, - leader_worker_idx=0, ) asyncio.run(start()) @@ -277,7 +282,7 @@ def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: def test_requests_other_than_rollouts_are_refused() -> None: llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) with pytest.raises(InvalidArgument, match="only NeMo RL rollouts"): _collect(engine, {"token_ids": [1, 2, 3, 4]}) @@ -292,7 +297,7 @@ async def serve_rollout(*args, **kwargs): raise RuntimeError("the generation output did not include routed_experts") host._serve_rollout = serve_rollout - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) # A 5xx would reach the driver as "Internal server error". with pytest.raises(InvalidArgument, match="RuntimeError: .*routed_experts"): @@ -336,7 +341,7 @@ async def call_utility_async(method): ] llm.engine_core = SimpleNamespace(call_utility_async=call_utility_async) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) registration = asyncio.run(engine.start(worker_id=0)).llm @@ -356,7 +361,7 @@ def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( ) -> None: cfg = DynamoCfg(frontend_args=DynamoFrontendArgs(router_mode=router_mode)) - config = native_engine.dynamo_kv_events_config(cfg, prefix_caching=prefix_caching) + config = vllm_adapter.dynamo_kv_events_config(cfg, prefix_caching=prefix_caching) # vLLM publishes block events only for blocks its prefix cache stores. assert (config is not None) is publishes @@ -365,7 +370,7 @@ def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( # A per-engine abstract socket, which vLLM's publisher binds and Dynamo's # source connects to: no port for two engines on a node to collide on. assert config["endpoint"].startswith("ipc://@") - other = native_engine.dynamo_kv_events_config(cfg, prefix_caching=True) + other = vllm_adapter.dynamo_kv_events_config(cfg, prefix_caching=True) assert other["endpoint"] != config["endpoint"] @@ -382,4 +387,4 @@ def test_main_attention_kinds_match_dynamos() -> None: for t in node.targets ) ] - assert {kind.value for kind in native_engine._MAIN_ATTENTION_KINDS} == dynamos + assert {kind.value for kind in vllm_adapter._MAIN_ATTENTION_KINDS} == dynamos diff --git a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py index 62ff7e4b7c2..536f9e404f6 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py +++ b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py @@ -15,9 +15,10 @@ The engines run NeMo RL's own vLLM and use Dynamo only through its engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only in -NeMo RL's own adapter (``dynamo/native_engine.py``). These guards fail if +NeMo RL's own adapter (``dynamo/vllm_adapter.py``). These guards fail if NeMo RL starts importing Dynamo's vLLM integration (or any other Dynamo -module), or if the SDK itself starts importing vLLM. +module), if the SDK itself starts importing vLLM, or if the Dynamo side shared +by every backend starts importing one. """ import ast @@ -41,22 +42,36 @@ } +# The Dynamo side every generation backend builds on. +BACKEND_NEUTRAL_MODULES = ( + "models/generation/dynamo/engine_adapter.py", + "models/generation/rollout_result.py", +) +GENERATION_BACKENDS = {"vllm", "sglang", "trtllm", "megatron"} + + +def _imports(path: Path) -> set[str]: + """The absolute modules ``path`` imports, at any depth.""" + modules: set[str] = set() + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if isinstance(node, ast.ImportFrom) and node.level == 0: + modules.add(node.module or "") + elif isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + return modules + + def _dynamo_imports() -> dict[str, set[str]]: """The Dynamo modules each nemo_rl source file imports.""" imports: dict[str, set[str]] = {} for path in NEMO_RL_ROOT.rglob("*.py"): - for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): - if isinstance(node, ast.ImportFrom) and node.level == 0: - modules = [node.module or ""] - elif isinstance(node, ast.Import): - modules = [alias.name for alias in node.names] - else: - continue - for module in modules: - if module == "dynamo" or module.startswith("dynamo."): - imports.setdefault(str(path.relative_to(NEMO_RL_ROOT)), set()).add( - module - ) + dynamo = { + module + for module in _imports(path) + if module == "dynamo" or module.startswith("dynamo.") + } + if dynamo: + imports[str(path.relative_to(NEMO_RL_ROOT))] = dynamo return imports @@ -74,6 +89,19 @@ def test_nemo_rl_imports_only_dynamos_backend_sdk() -> None: ) +def test_the_shared_dynamo_side_imports_no_generation_backend() -> None: + backends = { + path: sorted( + module + for module in _imports(NEMO_RL_ROOT / path) + if GENERATION_BACKENDS & set(module.split(".")) + ) + for path in BACKEND_NEUTRAL_MODULES + } + + assert backends == {path: [] for path in BACKEND_NEUTRAL_MODULES} + + @pytest.mark.vllm def test_dynamos_backend_sdk_imports_no_vllm() -> None: pytest.importorskip("dynamo._core") diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 1e8c4440a53..5c9a1a0b971 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2867,11 +2867,11 @@ def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error def test_vllm_dynamo_rollouts_go_through_the_frontend(monkeypatch): from nemo_rl.experience.failures import GenerationUnavailable from nemo_rl.models.generation.dynamo import rollout_client - from nemo_rl.models.generation.vllm.request_pipeline import ( + from nemo_rl.models.generation.rollout_result import ( RolloutResult, - rollout_output, rollout_result_to_wire, ) + from nemo_rl.models.generation.vllm.request_pipeline import rollout_output result = RolloutResult([7, 8], [-0.5, -0.25], False, None, None) responses = [ diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py index cc9b7cf0725..11244bb1af0 100644 --- a/tests/unit/models/generation/test_vllm_request_pipeline.py +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -21,13 +21,15 @@ import pytest import torch -from nemo_rl.models.generation.vllm.request_pipeline import ( +from nemo_rl.models.generation.rollout_result import ( RolloutResult, - rollout_output, - rollout_result, rollout_result_from_wire, rollout_result_to_wire, ) +from nemo_rl.models.generation.vllm.request_pipeline import ( + rollout_output, + rollout_result, +) from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL LAYERS, TOPK = 2, 2 From 096e8728b8f4c64c5581c12683848a01aa78364e Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 6 Nov 2026 18:10:33 -0800 Subject: [PATCH 31/82] docs(dynamo): scope the Dynamo guide to the SingleController C58. Every example already uses the SingleController, which is the path the Dynamo overlays test. The design doc names the adapter split. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 18 ++++++++++++++---- docs/guides/dynamo-generation.md | 9 +++++---- docs/index.md | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 4b7e95277e7..245e9629127 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -7,8 +7,8 @@ Dynamo's frontend and KV-aware router. Two requirements shape the design: - Dynamo CI coverage must stay consistent with native vLLM's. Both hold only if the engine process contains no Dynamo-owned vLLM code. A vLLM -bump can then break Dynamo serving only in NeMo RL's own adapter -(`native_engine.py`), which calls a few vLLM APIs native NeMo RL does not: the +bump can then break Dynamo serving only in NeMo RL's own vLLM adapter +(`vllm_adapter.py`), which calls a few vLLM APIs native NeMo RL does not: the `get_kv_cache_group_metadata` engine utility and `KVCacheSpecKind` at registration, and `StatLoggerBase` for the router's load signal. A renamed import fails the vLLM-lane unit tests at collection, and the Dynamo functional overlays @@ -22,8 +22,18 @@ also runs `dynamo.common.backend.Worker` as a task on its own actor event loop, which is the loop that owns the `AsyncLLM`. `dynamo.common.backend` is Dynamo's engine-agnostic backend SDK and imports no -vLLM. The adapter, `nemo_rl/models/generation/dynamo/native_engine.py`, -implements its `LLMEngine` over the worker's `AsyncLLM`: +vLLM. NeMo RL implements its `LLMEngine` in two parts: + +- `dynamo/engine_adapter.py`: `DynamoEngineAdapter`, the Dynamo side shared by + every generation backend. It serves request kinds as handlers keyed by their + annotation, maps engine errors to a 400 that keeps the cause, and registers the + endpoint for `/v1/completions` only. It imports no backend, and the engine only + ever receives token IDs. +- `dynamo/vllm_adapter.py`: `DynamoVllmAdapter`, the vLLM backend over the + worker's `AsyncLLM`. A backend implements `serve_rollout`, `start`, `abort` and + `kv_event_sources`. + +The two together: - `generate` serves only the rollouts NeMo RL itself sends, which carry an annotation, and refuses other requests. The worker's own `_serve_rollout` runs diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 52d65c6cfa3..1f5652e1e5d 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -1,6 +1,7 @@ # Dynamo serving for vLLM generation -Any non-colocated vLLM generation config can serve its rollouts through +SingleController runs (`examples/run_grpo_single_controller.py`) with non-colocated +vLLM generation can serve their rollouts through [Dynamo](https://github.com/ai-dynamo/dynamo)'s OpenAI frontend and KV-aware router. Add one line: @@ -80,8 +81,7 @@ Startup rejects these with `dynamo: true`: - NeMo-Gym (`env.should_use_nemo_gym`): the frontend does not serve Gym's calls yet. Run Gym recipes with `dynamo: false`. - `colocated.enabled: true`: a colocated engine sleeps while the policy trains, - and the frontend would keep routing to it. The SingleController never runs - colocated vLLM; the other entrypoints need non-colocated generation. + and the frontend would keep routing to it. - `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, and a restarted shard does not register with it yet. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data @@ -120,7 +120,8 @@ bash docker/dynamo/install.sh ## Run -Use any non-colocated vLLM recipe that does not use NeMo-Gym, and add the switch: +Use any non-colocated SingleController vLLM recipe that does not use NeMo-Gym, +and add the switch: ```bash uv run examples/run_grpo_single_controller.py \ diff --git a/docs/index.md b/docs/index.md index 9f03e656b34..f99244492a7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -83,7 +83,7 @@ Configure and launch NeMo RL on multi-node Slurm or Kubernetes clusters for dist :link: guides/dynamo-generation :link-type: doc -Serve non-colocated vLLM generation through Dynamo's frontend and KV-aware router with `dynamo: true`. +Serve the SingleController's non-colocated vLLM generation through Dynamo's frontend and KV-aware router with `dynamo: true`. ::: :::: From 3370812df0cb29b23db4a834d02c1ee94a4a3b2a Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 12:29:21 -0800 Subject: [PATCH 32/82] refactor(dynamo): drop the endpoint_types setting; engines always serve chat and completions NeMo RL's rollouts always go to /v1/completions and NeMo-Gym's dialect always calls chat, so every other value of worker_args.endpoint_types breaks one of them after placement and model load (C20). start_dynamo_endpoint registers both; an old config now fails at startup with the removed-key message. The guide's list of what generic OpenAI clients get goes with the generic path (C21), and test_dynamo_cfg_rejects_unknown_keys goes with the setting: its other check only tested pydantic's extra="forbid" (C24). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 11 ----------- ...grpo_math_1B_dynamo_single_controller.yaml | 1 - nemo_rl/models/generation/dynamo/config.py | 11 +---------- nemo_rl/utils/outdated_config_checks.py | 19 ++++++++++++++----- .../models/generation/test_dynamo_config.py | 8 -------- .../generation/test_dynamo_vllm_adapter.py | 7 +------ .../unit/utils/test_outdated_config_checks.py | 4 ++++ 7 files changed, 20 insertions(+), 41 deletions(-) diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index cdaf7fcfccb..01beed5ca46 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -75,17 +75,6 @@ native path: token IDs, log probabilities, truncation, routed experts with their `r3_routed_experts_*` counts, and `gen_leader_worker_idx`. Both paths build it with `nemo_rl/models/generation/vllm/request_pipeline.py`. -OpenAI clients of the frontend get what vLLM's own OpenAI server applies: -- the model's generation-config sampling defaults; -- `n > 1`; -- guided decoding (`guided_*`, `response_format`); -- `bad_words_token_ids` and `allowed_token_ids`; -- prompt log probabilities (`nvext.extra_fields: ["prompt_logprobs"]`); -- vLLM's queue limits, rejected with HTTP 503; -- a 400 naming the "maximum context length" for a prompt that leaves no room - for output. The engine, not the frontend, checks the context, so it checks - the prompt it actually runs. - ## NeMo-Gym NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 2c27bf7f89c..9c6eb20a2c7 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -31,7 +31,6 @@ policy: # A chat template that replaces the model's own. null follows # vllm_cfg.http_server_serving_chat_kwargs.chat_template. custom_jinja_template: null - endpoint_types: [chat, completions] frontend_args: # kv: route each request to the engine holding most of its prefix. router_mode: kv diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 0bb39711d86..621afa087de 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -21,7 +21,7 @@ from typing import Annotated, Any, Literal, Mapping -from pydantic import BaseModel, Field, PositiveFloat, model_validator +from pydantic import BaseModel, Field, PositiveFloat class DynamoWorkerArgs(BaseModel, extra="forbid"): @@ -39,15 +39,6 @@ class DynamoWorkerArgs(BaseModel, extra="forbid"): # A Jinja chat template file that replaces the model's own. None follows # vllm_cfg.http_server_serving_chat_kwargs.chat_template. custom_jinja_template: str | None = None - endpoint_types: list[Literal["chat", "completions"]] = ["chat", "completions"] - - @model_validator(mode="after") - def _validate_endpoint_types(self) -> "DynamoWorkerArgs": - if not self.endpoint_types: - raise ValueError("endpoint_types must contain at least one endpoint") - if len(self.endpoint_types) != len(set(self.endpoint_types)): - raise ValueError("endpoint_types must not contain duplicates") - return self class DynamoFrontendArgs(BaseModel, extra="forbid"): diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 6d75c8bc19e..39084f01c7f 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -209,6 +209,13 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: "metrics_include_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", "metrics_exclude_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", } +_REMOVED_DYNAMO_WORKER_KEYS = { + "extra_cli_args": "engines are configured through vllm_cfg and vllm_kwargs", + "endpoint_types": ( + "the engines always serve /v1/completions (NeMo RL's rollouts) and chat " + "(NeMo-Gym)" + ), +} _REMOVED_DYNAMO_FRONTEND_KEYS = ( "tokenizer", "tokenizer_cache", @@ -257,11 +264,13 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: "frontend never tokenizes." ) worker_args = dynamo_cfg.get("worker_args") - if isinstance(worker_args, dict) and "extra_cli_args" in worker_args: - problems.append( - "policy.generation.dynamo_cfg.worker_args.extra_cli_args was " - "removed: engines are configured through vllm_cfg and vllm_kwargs." - ) + if isinstance(worker_args, dict): + for key, replacement in _REMOVED_DYNAMO_WORKER_KEYS.items(): + if key in worker_args: + problems.append( + f"policy.generation.dynamo_cfg.worker_args.{key} was " + f"removed: {replacement}." + ) if problems: raise ValueError(" ".join(problems)) diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 524032dcd63..379efd5c66d 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import pytest -from pydantic import ValidationError from nemo_rl.models.generation import configure_generation_config from nemo_rl.models.generation.dynamo.config import ( @@ -118,13 +117,6 @@ def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( configure_generation_config(config, _Tokenizer()) -def test_dynamo_cfg_rejects_unknown_keys() -> None: - with pytest.raises(ValidationError, match="extra_cli_args"): - dynamo_cfg({"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}) - with pytest.raises(ValidationError, match="duplicates"): - dynamo_cfg({"dynamo_cfg": {"worker_args": {"endpoint_types": ["chat"] * 2}}}) - - def test_frontend_argv_is_runtime_owned() -> None: cfg = DynamoCfg( frontend_args=DynamoFrontendArgs( diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index 1a6920e4fcc..57efd0fbee1 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -345,7 +345,6 @@ def test_every_worker_arg_reaches_the_dynamo_worker_config( "structural_tag_scope": "always", "structural_tag_schema": "strict", "custom_jinja_template": str(template), - "endpoint_types": ["chat"], } assert set(worker_args) == set(DynamoWorkerArgs.model_fields) host = _host(_FakeLLM(), dynamo_cfg={"worker_args": worker_args}) @@ -363,11 +362,7 @@ def test_every_worker_arg_reaches_the_dynamo_worker_config( assert config.structural_tag_mode == "on" with open(config.custom_jinja_template) as template_file: assert template_file.read() == "{{ messages }}" - for key in worker_args.keys() - { - "endpoint_types", - "enable_structural_tag", - "custom_jinja_template", - }: + for key in worker_args.keys() - {"enable_structural_tag", "custom_jinja_template"}: assert getattr(config, key) == worker_args[key], key diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 0bdcdc14f1b..95b82b13161 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -434,6 +434,10 @@ def _generation_config(**generation) -> dict: {"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}, r"worker_args\.extra_cli_args", ), + ( + {"dynamo_cfg": {"worker_args": {"endpoint_types": ["chat"]}}}, + r"worker_args\.endpoint_types was removed", + ), ( {"dynamo_cfg": {"frontend_args": {"tokenizer_cache": False}}}, r"frontend_args\.tokenizer_cache was", From 790feb77dd79b683318537157c42bd0597ba189b Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 14:21:55 -0800 Subject: [PATCH 33/82] refactor(vllm): move the chat server out of the async worker The classes and routes _setup_vllm_openai_api_server defined inside itself move unchanged to nemo_rl/models/generation/vllm/chat_server.py, so a second host of the same chat path (the Dynamo frontend's chat engine, C27 on #4979) can build the same renderer and chat serving. The worker passes itself in as ``host`` where the closures captured ``worker_self``; behavior is unchanged. Tests that build the server against fake vLLM modules now re-import the module against their fakes and patch names on it rather than on the worker. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/models/generation/vllm/chat_server.py | 847 ++++++++++++++++++ .../generation/vllm/vllm_worker_async.py | 827 +---------------- .../test_vllm_chat_template_wiring.py | 14 +- .../models/generation/test_vllm_generation.py | 11 +- 4 files changed, 875 insertions(+), 824 deletions(-) create mode 100644 nemo_rl/models/generation/vllm/chat_server.py diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py new file mode 100644 index 00000000000..d2f2f2199a6 --- /dev/null +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -0,0 +1,847 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NeMo RL's OpenAI chat and tokenize server over a vLLM engine. + +The vLLM async worker builds it for its HTTP server. The classes live here, not +inside the worker, so another host of the same chat path (Dynamo's frontend) +can build the same renderer and chat serving. ``host`` is the worker: it owns +token capture. This module imports vLLM's OpenAI server stack; import it only +where vLLM is installed. +""" + +import asyncio +import copy +import logging +from copy import deepcopy +from dataclasses import dataclass +from logging import Filter as LoggingFilter +from logging import LogRecord +from typing import Any, List, Literal, Optional, Union + +import torch +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, StreamingResponse +from vllm.entrypoints.chat_utils import load_chat_template +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, +) +from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat +from vllm.entrypoints.openai.models.protocol import BaseModelPath +from vllm.entrypoints.openai.models.serving import OpenAIServingModels + +# vLLM 0.29 moved this out of the openai package (vllm-project/vllm#54492). +from vllm.entrypoints.serve.engine.protocol import ErrorResponse +from vllm.entrypoints.serve.tokenize.protocol import ( + TokenizeChatRequest, + TokenizeCompletionRequest, + TokenizeResponse, +) +from vllm.entrypoints.serve.tokenize.serving import ServingTokenization +from vllm.entrypoints.serve.utils.api_utils import with_cancellation +from vllm.exceptions import VLLMValidationError +from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager +from vllm.renderers.hf import HfRenderer +from vllm.renderers.online_renderer import OnlineRenderer +from vllm.tool_parsers.abstract_tool_parser import ToolParserManager +from vllm.v1.engine.async_llm import logger as vllm_async_llm_logger + +from nemo_rl.data.captured_media import MediaCaptureRejected +from nemo_rl.models.generation.capture_context import decide_capture_input +from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens +from nemo_rl.models.generation.vllm.captured_chat_logprobs import ( + CapturedChatLogprobsMixin, +) +from nemo_rl.models.generation.vllm.utils import ( + attach_routed_experts_to_chat_response_choices, + attach_sampler_topk_logprobs_to_chat_response_choices, + attach_token_information_to_chat_response_choices, + check_sampling_param, + model_dump_chat_response_with_dynamic_message_fields, + remap_multimodal_placeholders, +) + +LOGGER = logging.getLogger(__name__) + + +class NeMoRLOpenAIChatRequestMixin: + def model_post_init(self, context): + # NeMo-Gym specific processing. This is just how NeMo-Gym returns the extra token information. + if self.required_prefix_token_ids is None: + for message in reversed(self.messages): + if "prompt_token_ids" in message: + self.required_prefix_token_ids = ( + message["prompt_token_ids"] + message["generation_token_ids"] + ) + break + + return super().model_post_init(context) + + +# This MRO is necessary i.e. NeMoRLOpenAIChatRequestMixin > ChatCompletionRequest +class NeMoRLChatCompletionRequest(NeMoRLOpenAIChatRequestMixin, ChatCompletionRequest): + required_prefix_token_ids: Optional[List[int]] = None + # Ledger-authoritative token capture: the call identity the ledger + # attaches (capture_key, call_id, parent_call_id, prev_len, mode). + ng_capture: Optional[dict[str, Any]] = None + + +# This MRO is necessary i.e. NeMoRLOpenAIChatRequestMixin > TokenizeRequest +class NeMoRLTokenizeChatRequest(NeMoRLOpenAIChatRequestMixin, TokenizeChatRequest): + required_prefix_token_ids: Optional[List[int]] = None + ng_capture: Optional[dict[str, Any]] = None + ng_measure_only: Literal[True] = True + + +NeMoRLTokenizeRequest = Union[TokenizeCompletionRequest, NeMoRLTokenizeChatRequest] + + +class NeMoRLOpenAIServingMixin: + """The chat preprocessing NeMo RL adds: prefix splice and token capture.""" + + @staticmethod + def _set_max_tokens(request, max_tokens: int) -> None: + """Set the request's max output tokens. + + Mutates the request in place. Handles both max_completion_tokens (newer OpenAI API) + and max_tokens (deprecated but still supported by vLLM). + """ + if request.max_completion_tokens is not None: + request.max_completion_tokens = max_tokens + elif request.max_tokens is not None: + request.max_tokens = max_tokens + + def _clamp_max_tokens( + self, request, request_max_tokens: int, prompt_token_ids: list[int] + ) -> None: + """Clamp the request's max output tokens so that input + output <= max_model_len.""" + remaining = self.model_config.max_model_len - len(prompt_token_ids) + if remaining <= 0: + # preserve the literal "context length" in this message to match Gym's overflow handling + message = ( + f"Prompt length ({len(prompt_token_ids)}) fills or exceeds " + f"this model's maximum context length ({self.model_config.max_model_len}). " + f"No room for output tokens." + ) + LOGGER.warning("Prompt exceeds max_model_len: %s", message) + raise VLLMValidationError( + message, + parameter="input_tokens", + value=len(prompt_token_ids), + ) + max_tokens = min(request_max_tokens, remaining) + self._set_max_tokens(request, max_tokens) + + # vLLM 0.25 moved chat preprocessing to + # OnlineRenderer.preprocess_chat (tool_parser/reasoning_parser were + # folded into a single `parser`), so this override now applies via + # the renderer subclass. + async def preprocess_chat( + self, + request, + messages, + default_template, + default_template_content_format, + default_template_kwargs, + tool_dicts=None, + parser=None, + *, + skip_mm_cache: bool = False, + ): + host = self._host + for message in messages: + if message.get("tool_calls"): + message["tool_calls"] = list(message["tool_calls"]) + + messages_for_replace_prefix_tokens = deepcopy(messages) + # #4124: processor-only cache reads retain concrete pixels even + # when the engine's sender cache would return references. + if host._capture_media and host._capture_admission(request) is not None: + skip_mm_cache = True + + # Temporarily set to 1 so vLLM's pre-tokenization length check passes; + # the actual value will be set through _clamp_max_tokens later. + actual_request_max_tokens = None + if isinstance(request, NeMoRLChatCompletionRequest): + actual_request_max_tokens = ( + request.max_completion_tokens + if request.max_completion_tokens is not None + else request.max_tokens + ) + # If max_completion_tokens or max_tokens is not set, we don't need to do _clamp_max_tokens. + # So we don't need to set the request's max output tokens to 1 here. + if actual_request_max_tokens is not None: + self._set_max_tokens(request, 1) + + try: + res = await super().preprocess_chat( + request=request, + messages=messages, + default_template=default_template, + default_template_content_format=default_template_content_format, + default_template_kwargs=default_template_kwargs, + tool_dicts=tool_dicts, + parser=parser, + skip_mm_cache=skip_mm_cache, + ) + except (ValueError, VLLMValidationError) as e: + if "maximum context length" in str(e): + LOGGER.warning("Prompt exceeds max_model_len: %s", e) + raise + + # Token capture: build the admission once, before branching, + # and resolve its prefix from it (staging_chain -> cached TQ + # read, inline ids, or nothing for a text root). The + # ``ng_capture`` dict is never mutated. Off-loop: the chain + # fetch is a blocking TQ read, and Gym's staging protocol + # requires the serving host to move blocking staging I/O off + # its event loop explicitly. The adapter then attaches the + # prefix to the request, so the inline-prefix branch below is + # the single splice path for staged and inline prefixes. + admission = host._capture_admission(request) + serving_admission = admission + verify_retained_media = False + if admission is not None and ( + admission.mode == "candidate" or admission.request_replay is not None + ): + # Inline assistant token fields cannot override RL's decision. + request.required_prefix_token_ids = None + decision = decide_capture_input( + admission, + messages=messages_for_replace_prefix_tokens, + # HF/string parsing maps null and empty tool content to + # identical template inputs. Structured content and + # other renderers cannot authorize that equivalence. + allow_empty_tool_content=( + type(self.renderer) is HfRenderer + and bool(res[0]) + and all(isinstance(item.get("content"), str) for item in res[0]) + ), + ) + admission = decision.storage + serving_admission = decision.serving + verify_retained_media = decision.verify_retained_media + capture_prefix_token_ids: list[int] | None = None + if serving_admission is not None and serving_admission.mode == "token_in": + capture_prefix_token_ids = await asyncio.to_thread( + host._resolve_admission_prefix, serving_admission + ) + host._enter_request_prefix(request, capture_prefix_token_ids) + + if ( + not hasattr(request, "required_prefix_token_ids") + or request.required_prefix_token_ids is None + ): + # Clamp the request's max output tokens so that input + output <= max_model_len. + if actual_request_max_tokens is not None: + self._clamp_max_tokens( + request, + actual_request_max_tokens, + res[1][0]["prompt_token_ids"], + ) + # Token capture, text mode: the full render is the exact + # engine prompt. + media = await asyncio.to_thread( + host._capture_request_media, + res[1][0], + admission=admission, + verify_retained_media=verify_retained_media, + ) + host._begin_request_capture( + request, + res[1][0]["prompt_token_ids"], + admission=admission, + media=media, + ) + return res + + model_prefix_token_ids = list(request.required_prefix_token_ids) + + # Token-in splice path — shared by staging_chain and direct prefix. + last_assistant_message_idx = None + for i in reversed(range(len(messages_for_replace_prefix_tokens))): + if messages_for_replace_prefix_tokens[i]["role"] == "assistant": + last_assistant_message_idx = i + break + + if last_assistant_message_idx is None: + messages_to_last_assistant_message = messages_for_replace_prefix_tokens + else: + messages_to_last_assistant_message = messages_for_replace_prefix_tokens[ + : last_assistant_message_idx + 1 + ] + + modified_request = request.model_copy(update={"add_generation_prompt": False}) + + corresponding_res = await super().preprocess_chat( + request=modified_request, + messages=messages_to_last_assistant_message, + default_template=default_template, + default_template_content_format=default_template_content_format, + default_template_kwargs=default_template_kwargs, + tool_dicts=tool_dicts, + parser=parser, + skip_mm_cache=skip_mm_cache, + ) + actual_corresponding_token_ids = corresponding_res[1][0]["prompt_token_ids"] + + engine_prompt = res[1][0] + + splice = splice_prefix_tokens( + tokenizer=self.renderer.tokenizer, + model_prefix_token_ids=model_prefix_token_ids, + template_prefix_token_ids=actual_corresponding_token_ids, + template_token_ids=engine_prompt["prompt_token_ids"], + # The template's own EOS must stay countable even when the + # configured stops don't include it. + eos_token_id=self._model_stop_token_ids + | {self.renderer.tokenizer.eos_token_id}, + ) + final_prompt_token_ids = splice.token_ids + # Reject overflows before media validation can mask the length error. + if actual_request_max_tokens is not None: + self._clamp_max_tokens( + request, + actual_request_max_tokens, + final_prompt_token_ids, + ) + + media = await asyncio.to_thread( + host._capture_request_media, + engine_prompt, + admission=serving_admission, + splice=splice, + verify_retained_media=verify_retained_media, + ) + # Must precede the reassignment below: the remap reads the + # current value of prompt_token_ids as the template-coordinate + # sequence the placeholders were computed against. Reordering + # these two statements silently degrades to a no-op. + # Successful media capture already remapped these ranges. + if media is None and (mm_placeholders := engine_prompt.get("mm_placeholders")): + engine_prompt["mm_placeholders"] = remap_multimodal_placeholders( + template_token_ids=engine_prompt["prompt_token_ids"], + final_token_ids=final_prompt_token_ids, + mm_placeholders=mm_placeholders, + splice=splice, + ) + + engine_prompt["prompt_token_ids"] = final_prompt_token_ids + + # Token capture, token-in mode: the spliced prompt is the + # exact engine prompt; begin_call re-checks the prefix it + # was spliced from against the admission. + host._begin_request_capture( + request, + final_prompt_token_ids, + admission=admission, + prefix_token_ids=( + capture_prefix_token_ids + if admission is not None and admission.mode == "token_in" + else None + ), + media=media, + ) + + return res + + +class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): + def __init__(self, *args, host: Any, model_stop_token_ids: set[int], **kwargs): + super().__init__(*args, **kwargs) + self._host = host + # vLLM adds the generation config's EOS to every request's stop set + # regardless of the configured stops, so a turn can end on either. + self._model_stop_token_ids = model_stop_token_ids + + +class NeMoRLOpenAIServingChatMixin: + async def chat_completion_full_generator( + self, + request, + result_generator, + *args, + **kwargs, + ): + host = self._host + return_as_token_id = ( + request.return_tokens_as_token_ids + if request.return_tokens_as_token_ids is not None + else self.return_tokens_as_token_ids + ) + if request.logprobs and return_as_token_id and request.top_logprobs is None: + raise VLLMValidationError( + "`top_logprobs` must be set when requesting token " + "information from the NeMo-RL chat endpoint.", + parameter="top_logprobs", + ) + + final_res = None + capture_routes = id(request) in host._capture_calls + cancelled = False + + async def capture_result_generator(): + nonlocal final_res, cancelled + try: + async for res in result_generator: + final_res = res + if capture_routes: + # vLLM encodes choice-level routes in its response. + # Keep engine arrays for our native capture, and + # hide them only in the superclass's shallow view. + res = copy.copy(res) + res.outputs = [copy.copy(output) for output in res.outputs] + for output in res.outputs: + output.routed_experts = None + yield res + except asyncio.CancelledError: + cancelled = True + raise + + # Capture stages the raw arrays below and strips OpenAI logprobs + # after the terminal PUT. Skip only their object construction; + # changing request.logprobs would also change engine sampling. + bypass_logprobs = bool( + not request.stream + and request.logprobs + and return_as_token_id + and id(request) in host._capture_calls + ) + with self._bypass_chat_logprobs(enabled=bypass_logprobs): + response = await super().chat_completion_full_generator( + request, + capture_result_generator(), + *args, + **kwargs, + ) + if cancelled: + # vLLM's chat_completion_full_generator catches the + # CancelledError that with_cancellation throws into the + # handler task on a client disconnect and returns + # ErrorResponse("Client disconnected") instead of + # re-raising. Nobody reads that response (the wrapper has + # already returned), and letting it through would make the + # route label the capture "error_response". Re-raise so the + # task finishes cancelled and the route sees the disconnect. + raise asyncio.CancelledError() + if not isinstance(response, ChatCompletionResponse) or final_res is None: + return response + + if request.logprobs and return_as_token_id: + response = attach_token_information_to_chat_response_choices( + response, + final_res, + ) + if request.ng_capture and host._capture_top_logprobs: + # Extraction scales with completion length and K. + response = await asyncio.to_thread( + attach_sampler_topk_logprobs_to_chat_response_choices, + response, + final_res, + top_k=host._capture_top_logprobs, + ) + + if host._return_routed_experts_enabled(): + response = attach_routed_experts_to_chat_response_choices( + response, + final_res, + device=torch.device("cpu"), + logger=LOGGER, + routed_experts_dtype=host.routed_experts_dtype, + encode_for_wire=not capture_routes, + ) + + return response + + +class NeMoRLOpenAIServingChat( + NeMoRLOpenAIServingChatMixin, + CapturedChatLogprobsMixin, + OpenAIServingChat, +): + def __init__(self, *args, host: Any, **kwargs): + super().__init__(*args, **kwargs) + self._host = host + + +# Tokenize path delegates to OnlineRenderer.preprocess_chat, +# where the prefix-token override lives. +class NeMoRLServingTokenization(ServingTokenization): + pass + + +@dataclass(frozen=True) +class ChatServer: + """The serving objects behind /v1/chat/completions and /tokenize.""" + + online_renderer: NeMoRLOnlineRenderer + serving_chat: NeMoRLOpenAIServingChat + serving_tokenization: NeMoRLServingTokenization + + +def model_stop_token_ids(model_config: Any, cfg: dict[str, Any]) -> set[int]: + """The configured stops plus the generation config's EOS.""" + stop_token_ids = set(cfg.get("stop_token_ids") or ()) + generation_eos_token_ids = model_config.try_get_generation_config().get( + "eos_token_id" + ) + if isinstance(generation_eos_token_ids, int): + generation_eos_token_ids = [generation_eos_token_ids] + return stop_token_ids | set(generation_eos_token_ids or ()) + + +def build_chat_server( + host: Any, *, engine_client: Any, model_config: Any, cfg: dict[str, Any] +) -> ChatServer: + """Build the chat serving the worker's HTTP server answers with. + + Args: + host: The vLLM worker; it owns token capture. + engine_client: The engine the chat serving generates with. + model_config: The engine's vLLM ``ModelConfig``. + cfg: ``policy.generation``. + """ + maybe_tool_parser_plugin = cfg["vllm_cfg"].get("tool_parser_plugin") + if maybe_tool_parser_plugin: + ToolParserManager.import_tool_parser(maybe_tool_parser_plugin) + + maybe_reasoning_parser_plugin = cfg["vllm_cfg"].get("reasoning_parser_plugin") + if maybe_reasoning_parser_plugin: + ReasoningParserManager.import_reasoning_parser(maybe_reasoning_parser_plugin) + + base_model_paths = [ + BaseModelPath( + name=model_config.served_model_name, model_path=model_config.model + ), + BaseModelPath(name=model_config.model, model_path=model_config.model), + ] + + openai_serving_models_kwargs = dict( + engine_client=engine_client, + base_model_paths=base_model_paths, + lora_modules=None, + ) + openai_serving_models = OpenAIServingModels(**openai_serving_models_kwargs) + + serving_chat_default_kwargs = dict( + response_role="assistant", + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + enable_auto_tools=True, + ) + serving_chat_kwargs = serving_chat_default_kwargs | cfg["vllm_cfg"].get( + "http_server_serving_chat_kwargs", dict() + ) + # The embedded server is constructed directly instead of through + # vLLM's CLI, where chat-template file paths are normally loaded. + # OnlineRenderer expects literal Jinja content; passing a path makes + # Transformers render the path itself and drops multimodal + # placeholders such as . + configured_chat_template = serving_chat_kwargs.get("chat_template") + if configured_chat_template is not None: + serving_chat_kwargs["chat_template"] = load_chat_template( + configured_chat_template + ) + # Recipes may name the parameter either way: ``default_chat_template_kwargs`` + # is vLLM's own spelling, ``chat_template_kwargs`` is accepted for recipes + # written against the older name. Normalize onto the native key rather + # than popping it: OnlineRenderer, OpenAIServingChat and ServingTokenization + # each keep their *own* copy and read it independently -- the chat serving + # builds its reasoning parser from it, and the tokenize path passes its own + # into preprocess_chat -- so the renderer's copy does not reach either. + # vLLM's api_server hands the same value to all three for that reason. + # + # Popped separately, not `A or B`: short-circuiting on a truthy A would + # leave B in the bag and OpenAIServingChat(**kwargs) would reject it. + _legacy_chat_template_kwargs = serving_chat_kwargs.pop("chat_template_kwargs", None) + if serving_chat_kwargs.get("default_chat_template_kwargs") is None: + serving_chat_kwargs["default_chat_template_kwargs"] = ( + _legacy_chat_template_kwargs + ) + default_chat_template_kwargs: dict[str, Any] = ( + serving_chat_kwargs["default_chat_template_kwargs"] or {} + ) + online_renderer = NeMoRLOnlineRenderer( + model_config=engine_client.model_config, + renderer=engine_client.renderer, + request_logger=serving_chat_kwargs["request_logger"], + chat_template=serving_chat_kwargs["chat_template"], + chat_template_content_format=serving_chat_kwargs[ + "chat_template_content_format" + ], + enable_auto_tools=serving_chat_kwargs["enable_auto_tools"], + # Keep the renderer's parser consistent with any parser overrides + # passed to OpenAIServingChat via http_server_serving_chat_kwargs. + tool_parser=serving_chat_kwargs.get("tool_parser"), + reasoning_parser=serving_chat_kwargs.get("reasoning_parser"), + # vLLM merges these into every render, with request-supplied keys + # winning (preprocess_chat's default_template_kwargs). The renderer + # is shared by /v1/chat/completions and /tokenize, so setting it + # here keeps the two endpoints rendering identical prompts. + default_chat_template_kwargs=default_chat_template_kwargs, + host=host, + model_stop_token_ids=model_stop_token_ids(model_config, cfg), + ) + serving_chat_kwargs.update( + dict( + engine_client=engine_client, + models=openai_serving_models, + online_renderer=online_renderer, + return_tokens_as_token_ids=True, + ) + ) + openai_serving_chat = NeMoRLOpenAIServingChat(**serving_chat_kwargs, host=host) + + serving_tokenization_kwargs = dict( + request_logger=serving_chat_kwargs["request_logger"], + chat_template=serving_chat_kwargs["chat_template"], + chat_template_content_format=serving_chat_kwargs[ + "chat_template_content_format" + ], + models=serving_chat_kwargs["models"], + online_renderer=online_renderer, + # ServingTokenization reads its own copy in preprocess_chat rather + # than the renderer's, so /tokenize would otherwise render with {} + # and diverge from /v1/chat/completions under multi-turn. + default_chat_template_kwargs=default_chat_template_kwargs, + ) + openai_serving_tokenization = NeMoRLServingTokenization( + **serving_tokenization_kwargs + ) + return ChatServer(online_renderer, openai_serving_chat, openai_serving_tokenization) + + +def register_routes( + app: FastAPI, server: ChatServer, *, host: Any, generation_config: dict[str, Any] +) -> None: + """Add /v1/chat/completions and /tokenize to ``app``.""" + openai_serving_chat = server.serving_chat + openai_serving_tokenization = server.serving_tokenization + + # The create_chat_completion and tokenize methods are taken from vllm/entrypoints/openai/api_server.py + @app.post("/v1/chat/completions") + # Like vLLM's own routes: cancel the handler (and with it the engine + # request) when the client disconnects, instead of generating for nobody. + @with_cancellation + async def create_chat_completion( + request: NeMoRLChatCompletionRequest, raw_request: Request + ): + # Accept train and validation profiles; reject off-policy requests. + # vLLM fills an unset top_k/min_p/repetition_penalty from + # generation_config.json when vllm_kwargs sets generation_config="auto", + # so pin top_k to the matched profile's (as + # vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params does) + # and disable min_p and the repetition penalty. vLLM applies these and + # the presence/frequency penalties and logit bias before sampling; the + # trainer replays none of them, so a request may not set them. + top_k = check_sampling_param( + temperature=request.temperature, + top_p=request.top_p, + top_k=request.top_k, + generation_config=generation_config, + ) + assert request.min_p in (None, 0.0), ( + f"request min_p={request.min_p} is not supported; leave it unset" + ) + assert request.repetition_penalty in (None, 1.0), ( + f"request repetition_penalty={request.repetition_penalty} is not " + "supported; leave it unset" + ) + assert request.presence_penalty in (None, 0.0), ( + f"request presence_penalty={request.presence_penalty} is not " + "supported; leave it unset" + ) + assert request.frequency_penalty in (None, 0.0), ( + f"request frequency_penalty={request.frequency_penalty} is not " + "supported; leave it unset" + ) + assert not request.logit_bias, "request logit_bias is not supported" + request.top_k = top_k + request.min_p = 0.0 + request.repetition_penalty = 1.0 + + try: + generator = await openai_serving_chat.create_chat_completion( + request, raw_request + ) + except VLLMValidationError as e: + # vLLM raises VLLMValidationError for prompts exceeding + # max_model_len during tokenization, instead of returning an + # ErrorResponse. Convert to HTTP 400 so the Gym proxy can + # detect context-length overflow and handle it gracefully. + host._abort_request_capture(request, reason="context_length") + return JSONResponse( + content={ + "error": { + "message": str(e), + "type": "invalid_request_error", + "param": e.parameter, + "code": 400, + } + }, + status_code=400, + headers=host._context_refusal_headers(request), + ) + except MediaCaptureRejected as e: + # Raised inside preprocess_chat before begin_call, so no capture + # state exists yet and the abort below is a no-op kept for + # symmetry. Return a 400 carrying a stable code so Gym and the + # worker log can distinguish a retained-media re-tile from a + # real engine error (which stays a 500 below). + host._abort_request_capture(request, reason=e.code) + LOGGER.warning( + "Rejected captured call before inference (%s): %s", e.code, e + ) + return JSONResponse( + content={ + "error": { + "message": str(e), + "type": "invalid_request_error", + "param": "messages", + "code": e.code, + } + }, + status_code=400, + ) + except ValueError as e: + # vLLM's get_max_tokens raises a plain ValueError ("Input length + # (N) exceeds model's maximum context length (M).") after + # preprocess_chat, e.g. when a token-in splice grows the prompt + # past max_model_len on a request with no max_tokens to clamp. + # Convert only that overflow to the same 400 as above; any other + # ValueError is a server bug and stays a 500. Must follow + # MediaCaptureRejected, which is also a ValueError. + if "maximum context length" not in str(e): + # Re-raising here bypasses the BaseException handler below. + host._abort_request_capture(request, reason="engine_error") + raise + LOGGER.warning("Prompt exceeds max_model_len: %s", e) + host._abort_request_capture(request, reason="context_length") + return JSONResponse( + content={ + "error": { + "message": str(e), + "type": "invalid_request_error", + "param": "input_tokens", + "code": 400, + } + }, + status_code=400, + # Only vLLM's known pre-generation get_max_tokens refusal + # can settle capture; an arbitrary ValueError cannot. + headers=( + host._context_refusal_headers(request) + if str(e).startswith("Input length (") + and "exceeds model's maximum context length" in str(e) + else None + ), + ) + except asyncio.CancelledError: + # Client disconnected (see with_cancellation above). A cancel + # before generation starts propagates here directly; one that + # lands mid-generation is swallowed by vLLM and re-raised by + # NeMoRLOpenAIServingChatMixin.chat_completion_full_generator. + host._abort_request_capture(request, reason="client_disconnect") + raise + except BaseException: + host._abort_request_capture(request, reason="engine_error") + raise + + if isinstance(generator, ErrorResponse): + host._abort_request_capture(request, reason="error_response") + return JSONResponse( + content=generator.model_dump(), status_code=generator.error.code + ) + + elif isinstance(generator, ChatCompletionResponse): + content = model_dump_chat_response_with_dynamic_message_fields(generator) + # Token capture: stage the delta and ride the coords on the + # response; strips logprobs/ids (no-op when capture is off). + # Off-loop: the sink write inside complete_call is a blocking + # TQ round trip (see the staging protocol's serving-host rule). + try: + content = await asyncio.to_thread( + host._finish_request_capture, request, content + ) + except asyncio.CancelledError: + # Queued executor work may never start after cancellation. + # If staging already claimed the capture, abort is a no-op. + host._abort_request_capture(request, reason="client_disconnect") + raise + return JSONResponse(content=content) + + host._abort_request_capture(request, reason="streaming_response") + return StreamingResponse(content=generator, media_type="text/event-stream") + + @app.post("/tokenize") + async def tokenize(request: NeMoRLTokenizeRequest, raw_request: Request): + generator = await openai_serving_tokenization.create_tokenize( + request, raw_request + ) + + if isinstance(generator, ErrorResponse): + return JSONResponse( + content=generator.model_dump(), status_code=generator.error.code + ) + elif isinstance(generator, TokenizeResponse): + if ( + isinstance(request, NeMoRLTokenizeChatRequest) + and request.ng_capture is not None + ): + return JSONResponse( + content={ + "prompt_token_count": generator.count, + "ng_context_measured": True, + } + ) + return JSONResponse(content=generator.model_dump()) + + +class _CleanLoggingFilter(LoggingFilter): + def filter(self, record: LogRecord) -> bool: + msg = record.getMessage() + + # vLLM does not accept `strict` tool definitions and reporting it to the user is not useful either. + return ( + "Added request" not in msg + and "The following fields were present in the request but ignored: {'strict'}" + not in msg + ) + + +# Suppress the noisy vLLM traceback when a prompt exceeds max_model_len. +# This is expected during multi-turn rollouts; we log a clean one-line +# warning from preprocess_chat instead. +class _MaxContextLengthFilter(LoggingFilter): + def filter(self, record: LogRecord) -> bool: + if record.exc_info and record.exc_info[1]: + if "maximum context length" in str(record.exc_info[1]): + return False + return True + + +def install_log_filters() -> None: + """Keep vLLM's per-request and context-overflow noise out of the logs.""" + print( + "Adding a vLLM logging filter so that the logs aren't spammed with not useful messages like `Added request ...`. This is to help errors pop up better and filter out noise." + ) + vllm_async_llm_logger.addFilter(_CleanLoggingFilter()) + logging.getLogger("vllm.entrypoints.openai.engine.protocol").addFilter( + _CleanLoggingFilter() + ) + logging.getLogger("vllm.entrypoints.openai.chat_completion.serving").addFilter( + _MaxContextLengthFilter() + ) diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 98b56fab9e2..6d86e61b6eb 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -1161,831 +1161,20 @@ def _context_refusal_headers(self, request: Any) -> dict[str, str]: # ruff: noqa def _setup_vllm_openai_api_server(self, app: FastAPI) -> FastAPI: - worker_self = self - from copy import deepcopy - from logging import Filter as LoggingFilter - from logging import LogRecord - from typing import List, Optional, Union - - from fastapi import Request - from fastapi.responses import JSONResponse, StreamingResponse - from vllm.entrypoints.chat_utils import load_chat_template - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - ) - from vllm.entrypoints.openai.chat_completion.serving import ( - OpenAIServingChat, - ) - - # vLLM 0.29 moved this out of the openai package (vllm-project/vllm#54492). - from vllm.entrypoints.serve.engine.protocol import ErrorResponse - from vllm.entrypoints.openai.models.protocol import BaseModelPath - from vllm.entrypoints.openai.models.serving import OpenAIServingModels - from vllm.entrypoints.serve.tokenize.protocol import ( - TokenizeChatRequest, - TokenizeCompletionRequest, - TokenizeResponse, - ) - from vllm.entrypoints.serve.tokenize.serving import ( - ServingTokenization, - ) - from vllm.entrypoints.serve.utils.api_utils import with_cancellation - from vllm.renderers.online_renderer import OnlineRenderer - from vllm.renderers.hf import HfRenderer - from vllm.exceptions import VLLMValidationError - from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager - from vllm.tool_parsers.abstract_tool_parser import ToolParserManager - from vllm.v1.engine.async_llm import logger as vllm_async_llm_logger - - maybe_tool_parser_plugin = self.cfg["vllm_cfg"].get("tool_parser_plugin") - if maybe_tool_parser_plugin: - ToolParserManager.import_tool_parser(maybe_tool_parser_plugin) - - maybe_reasoning_parser_plugin = self.cfg["vllm_cfg"].get( - "reasoning_parser_plugin" - ) - if maybe_reasoning_parser_plugin: - ReasoningParserManager.import_reasoning_parser( - maybe_reasoning_parser_plugin - ) + # Deferred: the chat server imports vLLM's OpenAI server stack. + from nemo_rl.models.generation.vllm import chat_server engine_client = self._http_engine_client if engine_client is None: raise RuntimeError("The HTTP engine client is not initialized.") - model_config = self.llm_async_engine_args.create_model_config() - # vLLM adds the generation config's EOS to every request's stop set - # regardless of the configured stops, so a turn can end on either. - model_stop_token_ids = set(self.cfg.get("stop_token_ids") or ()) - generation_eos_token_ids = model_config.try_get_generation_config().get( - "eos_token_id" - ) - if isinstance(generation_eos_token_ids, int): - generation_eos_token_ids = [generation_eos_token_ids] - model_stop_token_ids |= set(generation_eos_token_ids or ()) - base_model_paths = [ - BaseModelPath( - name=model_config.served_model_name, model_path=model_config.model - ), - BaseModelPath(name=model_config.model, model_path=model_config.model), - ] - - openai_serving_models_kwargs = dict( + server = chat_server.build_chat_server( + self, engine_client=engine_client, - base_model_paths=base_model_paths, - lora_modules=None, - ) - openai_serving_models = OpenAIServingModels(**openai_serving_models_kwargs) - - class NeMoRLOpenAIChatRequestMixin: - def model_post_init(self, context): - # NeMo-Gym specific processing. This is just how NeMo-Gym returns the extra token information. - if self.required_prefix_token_ids is None: - for message in reversed(self.messages): - if "prompt_token_ids" in message: - self.required_prefix_token_ids = ( - message["prompt_token_ids"] - + message["generation_token_ids"] - ) - break - - return super().model_post_init(context) - - class NeMoRLOpenAIServingMixin: - @staticmethod - def _set_max_tokens(request, max_tokens: int) -> None: - """Set the request's max output tokens. - - Mutates the request in place. Handles both max_completion_tokens (newer OpenAI API) - and max_tokens (deprecated but still supported by vLLM). - """ - if request.max_completion_tokens is not None: - request.max_completion_tokens = max_tokens - elif request.max_tokens is not None: - request.max_tokens = max_tokens - - def _clamp_max_tokens( - self, request, request_max_tokens: int, prompt_token_ids: list[int] - ) -> None: - """Clamp the request's max output tokens so that input + output <= max_model_len.""" - remaining = self.model_config.max_model_len - len(prompt_token_ids) - if remaining <= 0: - # preserve the literal "context length" in this message to match Gym's overflow handling - message = ( - f"Prompt length ({len(prompt_token_ids)}) fills or exceeds " - f"this model's maximum context length ({self.model_config.max_model_len}). " - f"No room for output tokens." - ) - LOGGER.warning("Prompt exceeds max_model_len: %s", message) - raise VLLMValidationError( - message, - parameter="input_tokens", - value=len(prompt_token_ids), - ) - max_tokens = min(request_max_tokens, remaining) - self._set_max_tokens(request, max_tokens) - - # vLLM 0.25 moved chat preprocessing to - # OnlineRenderer.preprocess_chat (tool_parser/reasoning_parser were - # folded into a single `parser`), so this override now applies via - # the renderer subclass. - async def preprocess_chat( - self, - request, - messages, - default_template, - default_template_content_format, - default_template_kwargs, - tool_dicts=None, - parser=None, - *, - skip_mm_cache: bool = False, - ): - for message in messages: - if message.get("tool_calls"): - message["tool_calls"] = list(message["tool_calls"]) - - messages_for_replace_prefix_tokens = deepcopy(messages) - # #4124: processor-only cache reads retain concrete pixels even - # when the engine's sender cache would return references. - if ( - worker_self._capture_media - and worker_self._capture_admission(request) is not None - ): - skip_mm_cache = True - - # Temporarily set to 1 so vLLM's pre-tokenization length check passes; - # the actual value will be set through _clamp_max_tokens later. - actual_request_max_tokens = None - if isinstance(request, NeMoRLChatCompletionRequest): - actual_request_max_tokens = ( - request.max_completion_tokens - if request.max_completion_tokens is not None - else request.max_tokens - ) - # If max_completion_tokens or max_tokens is not set, we don't need to do _clamp_max_tokens. - # So we don't need to set the request's max output tokens to 1 here. - if actual_request_max_tokens is not None: - self._set_max_tokens(request, 1) - - try: - res = await super().preprocess_chat( - request=request, - messages=messages, - default_template=default_template, - default_template_content_format=default_template_content_format, - default_template_kwargs=default_template_kwargs, - tool_dicts=tool_dicts, - parser=parser, - skip_mm_cache=skip_mm_cache, - ) - except (ValueError, VLLMValidationError) as e: - if "maximum context length" in str(e): - import logging - - logging.getLogger(__name__).warning( - "Prompt exceeds max_model_len: %s", e - ) - raise - - # Token capture: build the admission once, before branching, - # and resolve its prefix from it (staging_chain -> cached TQ - # read, inline ids, or nothing for a text root). The - # ``ng_capture`` dict is never mutated. Off-loop: the chain - # fetch is a blocking TQ read, and Gym's staging protocol - # requires the serving host to move blocking staging I/O off - # its event loop explicitly. The adapter then attaches the - # prefix to the request, so the inline-prefix branch below is - # the single splice path for staged and inline prefixes. - admission = worker_self._capture_admission(request) - serving_admission = admission - verify_retained_media = False - if admission is not None and ( - admission.mode == "candidate" - or admission.request_replay is not None - ): - # Inline assistant token fields cannot override RL's decision. - request.required_prefix_token_ids = None - decision = decide_capture_input( - admission, - messages=messages_for_replace_prefix_tokens, - # HF/string parsing maps null and empty tool content to - # identical template inputs. Structured content and - # other renderers cannot authorize that equivalence. - allow_empty_tool_content=( - type(self.renderer) is HfRenderer - and bool(res[0]) - and all( - isinstance(item.get("content"), str) for item in res[0] - ) - ), - ) - admission = decision.storage - serving_admission = decision.serving - verify_retained_media = decision.verify_retained_media - capture_prefix_token_ids: list[int] | None = None - if ( - serving_admission is not None - and serving_admission.mode == "token_in" - ): - capture_prefix_token_ids = await asyncio.to_thread( - worker_self._resolve_admission_prefix, serving_admission - ) - worker_self._enter_request_prefix(request, capture_prefix_token_ids) - - if ( - not hasattr(request, "required_prefix_token_ids") - or request.required_prefix_token_ids is None - ): - # Clamp the request's max output tokens so that input + output <= max_model_len. - if actual_request_max_tokens is not None: - self._clamp_max_tokens( - request, - actual_request_max_tokens, - res[1][0]["prompt_token_ids"], - ) - # Token capture, text mode: the full render is the exact - # engine prompt. - media = await asyncio.to_thread( - worker_self._capture_request_media, - res[1][0], - admission=admission, - verify_retained_media=verify_retained_media, - ) - worker_self._begin_request_capture( - request, - res[1][0]["prompt_token_ids"], - admission=admission, - media=media, - ) - return res - - model_prefix_token_ids = list(request.required_prefix_token_ids) - - # Token-in splice path — shared by staging_chain and direct prefix. - last_assistant_message_idx = None - for i in reversed(range(len(messages_for_replace_prefix_tokens))): - if messages_for_replace_prefix_tokens[i]["role"] == "assistant": - last_assistant_message_idx = i - break - - if last_assistant_message_idx is None: - messages_to_last_assistant_message = ( - messages_for_replace_prefix_tokens - ) - else: - messages_to_last_assistant_message = ( - messages_for_replace_prefix_tokens[ - : last_assistant_message_idx + 1 - ] - ) - - modified_request = request.model_copy( - update={"add_generation_prompt": False} - ) - - corresponding_res = await super().preprocess_chat( - request=modified_request, - messages=messages_to_last_assistant_message, - default_template=default_template, - default_template_content_format=default_template_content_format, - default_template_kwargs=default_template_kwargs, - tool_dicts=tool_dicts, - parser=parser, - skip_mm_cache=skip_mm_cache, - ) - actual_corresponding_token_ids = corresponding_res[1][0][ - "prompt_token_ids" - ] - - engine_prompt = res[1][0] - - splice = splice_prefix_tokens( - tokenizer=self.renderer.tokenizer, - model_prefix_token_ids=model_prefix_token_ids, - template_prefix_token_ids=actual_corresponding_token_ids, - template_token_ids=engine_prompt["prompt_token_ids"], - # The template's own EOS must stay countable even when the - # configured stops don't include it. - eos_token_id=model_stop_token_ids - | {self.renderer.tokenizer.eos_token_id}, - ) - final_prompt_token_ids = splice.token_ids - # Reject overflows before media validation can mask the length error. - if actual_request_max_tokens is not None: - self._clamp_max_tokens( - request, - actual_request_max_tokens, - final_prompt_token_ids, - ) - - media = await asyncio.to_thread( - worker_self._capture_request_media, - engine_prompt, - admission=serving_admission, - splice=splice, - verify_retained_media=verify_retained_media, - ) - # Must precede the reassignment below: the remap reads the - # current value of prompt_token_ids as the template-coordinate - # sequence the placeholders were computed against. Reordering - # these two statements silently degrades to a no-op. - # Successful media capture already remapped these ranges. - if media is None and ( - mm_placeholders := engine_prompt.get("mm_placeholders") - ): - engine_prompt["mm_placeholders"] = remap_multimodal_placeholders( - template_token_ids=engine_prompt["prompt_token_ids"], - final_token_ids=final_prompt_token_ids, - mm_placeholders=mm_placeholders, - splice=splice, - ) - - engine_prompt["prompt_token_ids"] = final_prompt_token_ids - - # Token capture, token-in mode: the spliced prompt is the - # exact engine prompt; begin_call re-checks the prefix it - # was spliced from against the admission. - worker_self._begin_request_capture( - request, - final_prompt_token_ids, - admission=admission, - prefix_token_ids=( - capture_prefix_token_ids - if admission is not None and admission.mode == "token_in" - else None - ), - media=media, - ) - - return res - - ######################################## - # /v1/chat/completions endpoint - ######################################## - - # This MRO is necessary i.e. NeMoRLOpenAIChatRequestMixin > ChatCompletionRequest - class NeMoRLChatCompletionRequest( - NeMoRLOpenAIChatRequestMixin, ChatCompletionRequest - ): - required_prefix_token_ids: Optional[List[int]] = None - # Ledger-authoritative token capture: the call identity the ledger - # attaches (capture_key, call_id, parent_call_id, prev_len, mode). - ng_capture: Optional[dict[str, Any]] = None - - # vLLM 0.25 routes both /v1/chat/completions and /tokenize through - # OnlineRenderer.preprocess_chat, so the prefix-token override - # belongs on the renderer subclass. - worker_self = self - - class NeMoRLOpenAIServingChatMixin: - async def chat_completion_full_generator( - self, - request, - result_generator, - *args, - **kwargs, - ): - return_as_token_id = ( - request.return_tokens_as_token_ids - if request.return_tokens_as_token_ids is not None - else self.return_tokens_as_token_ids - ) - if ( - request.logprobs - and return_as_token_id - and request.top_logprobs is None - ): - raise VLLMValidationError( - "`top_logprobs` must be set when requesting token " - "information from the NeMo-RL chat endpoint.", - parameter="top_logprobs", - ) - - final_res = None - capture_routes = id(request) in worker_self._capture_calls - cancelled = False - - async def capture_result_generator(): - nonlocal final_res, cancelled - try: - async for res in result_generator: - final_res = res - if capture_routes: - # vLLM encodes choice-level routes in its response. - # Keep engine arrays for our native capture, and - # hide them only in the superclass's shallow view. - res = copy.copy(res) - res.outputs = [ - copy.copy(output) for output in res.outputs - ] - for output in res.outputs: - output.routed_experts = None - yield res - except asyncio.CancelledError: - cancelled = True - raise - - # Capture stages the raw arrays below and strips OpenAI logprobs - # after the terminal PUT. Skip only their object construction; - # changing request.logprobs would also change engine sampling. - bypass_logprobs = bool( - not request.stream - and request.logprobs - and return_as_token_id - and id(request) in worker_self._capture_calls - ) - with self._bypass_chat_logprobs(enabled=bypass_logprobs): - response = await super().chat_completion_full_generator( - request, - capture_result_generator(), - *args, - **kwargs, - ) - if cancelled: - # vLLM's chat_completion_full_generator catches the - # CancelledError that with_cancellation throws into the - # handler task on a client disconnect and returns - # ErrorResponse("Client disconnected") instead of - # re-raising. Nobody reads that response (the wrapper has - # already returned), and letting it through would make the - # route label the capture "error_response". Re-raise so the - # task finishes cancelled and the route sees the disconnect. - raise asyncio.CancelledError() - if ( - not isinstance(response, ChatCompletionResponse) - or final_res is None - ): - return response - - if request.logprobs and return_as_token_id: - response = attach_token_information_to_chat_response_choices( - response, - final_res, - ) - if request.ng_capture and worker_self._capture_top_logprobs: - # Extraction scales with completion length and K. - response = await asyncio.to_thread( - attach_sampler_topk_logprobs_to_chat_response_choices, - response, - final_res, - top_k=worker_self._capture_top_logprobs, - ) - - if worker_self._return_routed_experts_enabled(): - response = attach_routed_experts_to_chat_response_choices( - response, - final_res, - device=torch.device("cpu"), - logger=LOGGER, - routed_experts_dtype=worker_self.routed_experts_dtype, - encode_for_wire=not capture_routes, - ) - - return response - - class NeMoRLOpenAIServingChat( - NeMoRLOpenAIServingChatMixin, - CapturedChatLogprobsMixin, - OpenAIServingChat, - ): - pass - - class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): - pass - - serving_chat_default_kwargs = dict( - response_role="assistant", - request_logger=None, - chat_template=None, - chat_template_content_format="auto", - enable_auto_tools=True, - ) - serving_chat_kwargs = serving_chat_default_kwargs | self.cfg["vllm_cfg"].get( - "http_server_serving_chat_kwargs", dict() - ) - # The embedded server is constructed directly instead of through - # vLLM's CLI, where chat-template file paths are normally loaded. - # OnlineRenderer expects literal Jinja content; passing a path makes - # Transformers render the path itself and drops multimodal - # placeholders such as . - configured_chat_template = serving_chat_kwargs.get("chat_template") - if configured_chat_template is not None: - serving_chat_kwargs["chat_template"] = load_chat_template( - configured_chat_template - ) - # Recipes may name the parameter either way: ``default_chat_template_kwargs`` - # is vLLM's own spelling, ``chat_template_kwargs`` is accepted for recipes - # written against the older name. Normalize onto the native key rather - # than popping it: OnlineRenderer, OpenAIServingChat and ServingTokenization - # each keep their *own* copy and read it independently -- the chat serving - # builds its reasoning parser from it, and the tokenize path passes its own - # into preprocess_chat -- so the renderer's copy does not reach either. - # vLLM's api_server hands the same value to all three for that reason. - # - # Popped separately, not `A or B`: short-circuiting on a truthy A would - # leave B in the bag and OpenAIServingChat(**kwargs) would reject it. - _legacy_chat_template_kwargs = serving_chat_kwargs.pop( - "chat_template_kwargs", None - ) - if serving_chat_kwargs.get("default_chat_template_kwargs") is None: - serving_chat_kwargs["default_chat_template_kwargs"] = ( - _legacy_chat_template_kwargs - ) - default_chat_template_kwargs: dict[str, Any] = ( - serving_chat_kwargs["default_chat_template_kwargs"] or {} - ) - online_renderer = NeMoRLOnlineRenderer( - model_config=engine_client.model_config, - renderer=engine_client.renderer, - request_logger=serving_chat_kwargs["request_logger"], - chat_template=serving_chat_kwargs["chat_template"], - chat_template_content_format=serving_chat_kwargs[ - "chat_template_content_format" - ], - enable_auto_tools=serving_chat_kwargs["enable_auto_tools"], - # Keep the renderer's parser consistent with any parser overrides - # passed to OpenAIServingChat via http_server_serving_chat_kwargs. - tool_parser=serving_chat_kwargs.get("tool_parser"), - reasoning_parser=serving_chat_kwargs.get("reasoning_parser"), - # vLLM merges these into every render, with request-supplied keys - # winning (preprocess_chat's default_template_kwargs). The renderer - # is shared by /v1/chat/completions and /tokenize, so setting it - # here keeps the two endpoints rendering identical prompts. - default_chat_template_kwargs=default_chat_template_kwargs, - ) - serving_chat_kwargs.update( - dict( - engine_client=engine_client, - models=openai_serving_models, - online_renderer=online_renderer, - return_tokens_as_token_ids=True, - ) + model_config=self.llm_async_engine_args.create_model_config(), + cfg=self.cfg, ) - openai_serving_chat = NeMoRLOpenAIServingChat(**serving_chat_kwargs) - - generation_config = self.cfg - - # The create_chat_completion and tokenize methods are taken from vllm/entrypoints/openai/api_server.py - @app.post("/v1/chat/completions") - # Like vLLM's own routes: cancel the handler (and with it the engine - # request) when the client disconnects, instead of generating for nobody. - @with_cancellation - async def create_chat_completion( - request: NeMoRLChatCompletionRequest, raw_request: Request - ): - # Accept train and validation profiles; reject off-policy requests. - # vLLM fills an unset top_k/min_p/repetition_penalty from - # generation_config.json when vllm_kwargs sets generation_config="auto", - # so pin top_k to the matched profile's (as - # vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params does) - # and disable min_p and the repetition penalty. vLLM applies these and - # the presence/frequency penalties and logit bias before sampling; the - # trainer replays none of them, so a request may not set them. - top_k = check_sampling_param( - temperature=request.temperature, - top_p=request.top_p, - top_k=request.top_k, - generation_config=generation_config, - ) - assert request.min_p in (None, 0.0), ( - f"request min_p={request.min_p} is not supported; leave it unset" - ) - assert request.repetition_penalty in (None, 1.0), ( - f"request repetition_penalty={request.repetition_penalty} is not " - "supported; leave it unset" - ) - assert request.presence_penalty in (None, 0.0), ( - f"request presence_penalty={request.presence_penalty} is not " - "supported; leave it unset" - ) - assert request.frequency_penalty in (None, 0.0), ( - f"request frequency_penalty={request.frequency_penalty} is not " - "supported; leave it unset" - ) - assert not request.logit_bias, "request logit_bias is not supported" - request.top_k = top_k - request.min_p = 0.0 - request.repetition_penalty = 1.0 - - try: - generator = await openai_serving_chat.create_chat_completion( - request, raw_request - ) - except VLLMValidationError as e: - # vLLM raises VLLMValidationError for prompts exceeding - # max_model_len during tokenization, instead of returning an - # ErrorResponse. Convert to HTTP 400 so the Gym proxy can - # detect context-length overflow and handle it gracefully. - worker_self._abort_request_capture(request, reason="context_length") - return JSONResponse( - content={ - "error": { - "message": str(e), - "type": "invalid_request_error", - "param": e.parameter, - "code": 400, - } - }, - status_code=400, - headers=worker_self._context_refusal_headers(request), - ) - except MediaCaptureRejected as e: - # Raised inside preprocess_chat before begin_call, so no capture - # state exists yet and the abort below is a no-op kept for - # symmetry. Return a 400 carrying a stable code so Gym and the - # worker log can distinguish a retained-media re-tile from a - # real engine error (which stays a 500 below). - worker_self._abort_request_capture(request, reason=e.code) - LOGGER.warning( - "Rejected captured call before inference (%s): %s", e.code, e - ) - return JSONResponse( - content={ - "error": { - "message": str(e), - "type": "invalid_request_error", - "param": "messages", - "code": e.code, - } - }, - status_code=400, - ) - except ValueError as e: - # vLLM's get_max_tokens raises a plain ValueError ("Input length - # (N) exceeds model's maximum context length (M).") after - # preprocess_chat, e.g. when a token-in splice grows the prompt - # past max_model_len on a request with no max_tokens to clamp. - # Convert only that overflow to the same 400 as above; any other - # ValueError is a server bug and stays a 500. Must follow - # MediaCaptureRejected, which is also a ValueError. - if "maximum context length" not in str(e): - # Re-raising here bypasses the BaseException handler below. - worker_self._abort_request_capture(request, reason="engine_error") - raise - LOGGER.warning("Prompt exceeds max_model_len: %s", e) - worker_self._abort_request_capture(request, reason="context_length") - return JSONResponse( - content={ - "error": { - "message": str(e), - "type": "invalid_request_error", - "param": "input_tokens", - "code": 400, - } - }, - status_code=400, - # Only vLLM's known pre-generation get_max_tokens refusal - # can settle capture; an arbitrary ValueError cannot. - headers=( - worker_self._context_refusal_headers(request) - if str(e).startswith("Input length (") - and "exceeds model's maximum context length" in str(e) - else None - ), - ) - except asyncio.CancelledError: - # Client disconnected (see with_cancellation above). A cancel - # before generation starts propagates here directly; one that - # lands mid-generation is swallowed by vLLM and re-raised by - # NeMoRLOpenAIServingChatMixin.chat_completion_full_generator. - worker_self._abort_request_capture(request, reason="client_disconnect") - raise - except BaseException: - worker_self._abort_request_capture(request, reason="engine_error") - raise - - if isinstance(generator, ErrorResponse): - worker_self._abort_request_capture(request, reason="error_response") - return JSONResponse( - content=generator.model_dump(), status_code=generator.error.code - ) - - elif isinstance(generator, ChatCompletionResponse): - content = model_dump_chat_response_with_dynamic_message_fields( - generator - ) - # Token capture: stage the delta and ride the coords on the - # response; strips logprobs/ids (no-op when capture is off). - # Off-loop: the sink write inside complete_call is a blocking - # TQ round trip (see the staging protocol's serving-host rule). - try: - content = await asyncio.to_thread( - worker_self._finish_request_capture, request, content - ) - except asyncio.CancelledError: - # Queued executor work may never start after cancellation. - # If staging already claimed the capture, abort is a no-op. - worker_self._abort_request_capture( - request, reason="client_disconnect" - ) - raise - return JSONResponse(content=content) - - worker_self._abort_request_capture(request, reason="streaming_response") - return StreamingResponse(content=generator, media_type="text/event-stream") - - ######################################## - # /tokenize endpoint - ######################################## - - # This MRO is necessary i.e. NeMoRLOpenAIChatRequestMixin > TokenizeRequest - class NeMoRLTokenizeChatRequest( - NeMoRLOpenAIChatRequestMixin, TokenizeChatRequest - ): - required_prefix_token_ids: Optional[List[int]] = None - ng_capture: Optional[dict[str, Any]] = None - ng_measure_only: Literal[True] = True - - NeMoRLTokenizeRequest = Union[ - TokenizeCompletionRequest, NeMoRLTokenizeChatRequest - ] - - # Tokenize path delegates to OnlineRenderer.preprocess_chat, - # where the prefix-token override lives. - class NeMoRLServingTokenization(ServingTokenization): - pass - - serving_tokenization_kwargs = dict( - request_logger=serving_chat_kwargs["request_logger"], - chat_template=serving_chat_kwargs["chat_template"], - chat_template_content_format=serving_chat_kwargs[ - "chat_template_content_format" - ], - models=serving_chat_kwargs["models"], - online_renderer=online_renderer, - # ServingTokenization reads its own copy in preprocess_chat rather - # than the renderer's, so /tokenize would otherwise render with {} - # and diverge from /v1/chat/completions under multi-turn. - default_chat_template_kwargs=default_chat_template_kwargs, - ) - openai_serving_tokenization = NeMoRLServingTokenization( - **serving_tokenization_kwargs - ) - - @app.post("/tokenize") - async def tokenize(request: NeMoRLTokenizeRequest, raw_request: Request): - generator = await openai_serving_tokenization.create_tokenize( - request, raw_request - ) - - if isinstance(generator, ErrorResponse): - return JSONResponse( - content=generator.model_dump(), status_code=generator.error.code - ) - elif isinstance(generator, TokenizeResponse): - if ( - isinstance(request, NeMoRLTokenizeChatRequest) - and request.ng_capture is not None - ): - return JSONResponse( - content={ - "prompt_token_count": generator.count, - "ng_context_measured": True, - } - ) - return JSONResponse(content=generator.model_dump()) - - ######################################## - # Logging - ######################################## - print( - "Adding a vLLM logging filter so that the logs aren't spammed with not useful messages like `Added request ...`. This is to help errors pop up better and filter out noise." - ) - - class CleanLoggingFilter(LoggingFilter): - def filter(self, record: LogRecord) -> bool: - msg = record.getMessage() - - # vLLM does not accept `strict` tool definitions and reporting it to the user is not useful either. - return ( - "Added request" not in msg - and "The following fields were present in the request but ignored: {'strict'}" - not in msg - ) - - vllm_async_llm_logger.addFilter(CleanLoggingFilter()) - - from logging import getLogger as _getLogger - - _getLogger("vllm.entrypoints.openai.engine.protocol").addFilter( - CleanLoggingFilter() - ) - - # Suppress the noisy vLLM traceback when a prompt exceeds max_model_len. - # This is expected during multi-turn rollouts; we log a clean one-line - # warning from _preprocess_chat instead. - class MaxContextLengthFilter(LoggingFilter): - def filter(self, record: LogRecord) -> bool: - if record.exc_info and record.exc_info[1]: - if "maximum context length" in str(record.exc_info[1]): - return False - return True - - _getLogger("vllm.entrypoints.openai.chat_completion.serving").addFilter( - MaxContextLengthFilter() - ) - + chat_server.register_routes(app, server, host=self, generation_config=self.cfg) + chat_server.install_log_filters() return app def _setup_vllm_server(self) -> "tuple[threading.Thread, str, uvicorn.Server]": diff --git a/tests/unit/models/generation/test_vllm_chat_template_wiring.py b/tests/unit/models/generation/test_vllm_chat_template_wiring.py index a449d165292..4f5298524a3 100644 --- a/tests/unit/models/generation/test_vllm_chat_template_wiring.py +++ b/tests/unit/models/generation/test_vllm_chat_template_wiring.py @@ -39,6 +39,7 @@ import torch from nemo_rl.data.captured_media import capture_processed_media +from nemo_rl.models.generation import vllm as vllm_generation_pkg from nemo_rl.models.generation.vllm import utils as vllm_utils from nemo_rl.models.generation.vllm.vllm_worker_async import ( VllmAsyncGenerationWorkerImpl, @@ -49,6 +50,7 @@ # A class attribute would be shadowed by the subclass and the construction would # be recorded somewhere the assertions never look. _BUILT: dict[str, list] = {"renderer": [], "chat": [], "tokenize": []} +_CHAT_SERVER = "nemo_rl.models.generation.vllm.chat_server" def _recorder(slot: str): @@ -87,6 +89,12 @@ def get(self, path, **_kwargs): def _install_fake_vllm(monkeypatch): """Stub exactly the vLLM surface _setup_vllm_openai_api_server imports.""" + # chat_server imports vLLM at module level: drop the cached copy (and the + # package attribute an earlier import set) so the server is built against + # these fakes; monkeypatch restores both afterwards. + monkeypatch.setitem(sys.modules, _CHAT_SERVER, None) + del sys.modules[_CHAT_SERVER] + monkeypatch.delattr(vllm_generation_pkg, "chat_server", raising=False) for name in ( "vllm", "vllm.entrypoints", @@ -677,11 +685,9 @@ async def preprocess(self, *, messages, **kwargs): monkeypatch.setattr( VllmAsyncGenerationWorkerImpl, "_capture_request_media", capture ) - monkeypatch.setattr( - "nemo_rl.models.generation.vllm.vllm_worker_async.remap_multimodal_placeholders", - remap, - ) renderers, _, _ = _build_server(monkeypatch, {}, app=app) + # The server module was just imported against the fakes; patch that copy. + monkeypatch.setattr(sys.modules[_CHAT_SERVER], "remap_multimodal_placeholders", remap) renderer = renderers[0] renderer.renderer = SimpleNamespace(tokenizer=SimpleNamespace(eos_token_id=2)) renderer.model_config = SimpleNamespace(max_model_len=8 if overflows else 16) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index b5428ec0bd9..9c6cc9179bb 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -983,6 +983,14 @@ def skip_fp8_known_failures() -> None: def _install_fake_vllm_openai_modules(monkeypatch): + # chat_server imports vLLM at module level: drop the cached copy so the + # server is built against these fakes, and restore it afterwards. + chat_server = "nemo_rl.models.generation.vllm.chat_server" + monkeypatch.setitem(sys.modules, chat_server, None) + del sys.modules[chat_server] + monkeypatch.delattr( + sys.modules["nemo_rl.models.generation.vllm"], "chat_server", raising=False + ) for module_name in ( "vllm", "vllm.entrypoints", @@ -1634,8 +1642,9 @@ def test_chat_disconnect_releases_capture_queued_for_finalization( "vllm.entrypoints.openai.chat_completion.protocol" ].ChatCompletionResponse serving_chat.create_chat_completion = AsyncMock(return_value=response_type()) + # The server module was just imported against the fakes; patch that copy. monkeypatch.setattr( - "nemo_rl.models.generation.vllm.vllm_worker_async." + sys.modules["nemo_rl.models.generation.vllm.chat_server"], "model_dump_chat_response_with_dynamic_message_fields", lambda _response: {"choices": []}, ) From 0b7894a4aa12b02c0f981158687ee80cfaf7f1de Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 14:32:56 -0800 Subject: [PATCH 34/82] refactor(vllm): share the chat path's render, capture decision, turn ends and sampling The steps every host of NeMo-Gym's chat path runs, so the Dynamo frontend's chat engine (C27 on #4979) can run the same code as the native server: - NeMoRLOpenAIServingMixin._render_chat_call: the render, the capture decision for a Gym candidate, and the history render the splice cuts against; preprocess_chat keeps the TQ read, splice, clamp, media and begin_call around it, in the same order (C26). - capture_context.resolve_capture_input: decide_capture_input with the renderer fact (HF renderer, string tool content) the host proves (C26). - openai_server_utils.model_stop_token_ids / turn_end_token_ids: every id a turn can end on; splice_prefix_tokens and replace_prefix_tokens now require eos_token_id, so no caller falls back to the tokenizer's single EOS (gpt-oss <|call|>); TRT-LLM passes its set or the tokenizer's (C16). - vllm.utils.on_policy_sampling: the profile match at f32, the matched profile's exact temperature, top_p and top_k, min_p and the repetition penalty pinned, penalties and logit_bias refused with OffPolicySamplingError instead of asserts; the native route uses it (C41). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/models/generation/capture_context.py | 31 ++ .../models/generation/dynamo/vllm_adapter.py | 10 +- .../models/generation/openai_server_utils.py | 47 ++- .../generation/trtllm/trtllm_http_server.py | 2 +- nemo_rl/models/generation/vllm/chat_server.py | 274 +++++++++--------- nemo_rl/models/generation/vllm/utils.py | 97 ++++++- tests/unit/data/test_captured_media.py | 5 + .../generation/test_dynamo_vllm_adapter.py | 1 + .../generation/test_openai_server_utils.py | 22 +- .../models/generation/test_vllm_generation.py | 6 + 10 files changed, 334 insertions(+), 161 deletions(-) diff --git a/nemo_rl/models/generation/capture_context.py b/nemo_rl/models/generation/capture_context.py index 032038bfd94..e8d49009bbe 100644 --- a/nemo_rl/models/generation/capture_context.py +++ b/nemo_rl/models/generation/capture_context.py @@ -89,3 +89,34 @@ def decide_capture_input( required_prefix_token_ids=admission.required_prefix_token_ids, ) return CaptureInputDecision(serving, serving, True) + + +def resolve_capture_input( + admission: "CaptureAdmission", + *, + messages: list[dict[str, Any]], + conversation: list[dict[str, Any]], + renderer_is_hf: bool, +) -> CaptureInputDecision: + """:func:`decide_capture_input` with the rendering fact the serving host proves. + + Shared by every host of the Gym chat path, which calls it after rendering + the conversation. HF/string parsing maps null and empty tool content to + identical template inputs; structured content and other renderers cannot + authorize that equivalence. + + Args: + admission: The call's capture admission. + messages: The request's messages, before rendering. + conversation: The rendered conversation the template saw. + renderer_is_hf: The host renders with vLLM's ``HfRenderer``. + """ + return decide_capture_input( + admission, + messages=messages, + allow_empty_tool_content=( + renderer_is_hf + and bool(conversation) + and all(isinstance(item.get("content"), str) for item in conversation) + ), + ) diff --git a/nemo_rl/models/generation/dynamo/vllm_adapter.py b/nemo_rl/models/generation/dynamo/vllm_adapter.py index 1c0758abdb2..f3a82cd500a 100644 --- a/nemo_rl/models/generation/dynamo/vllm_adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm_adapter.py @@ -75,7 +75,11 @@ ServedRollout, _annotation_value, ) -from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens +from nemo_rl.models.generation.openai_server_utils import ( + model_stop_token_ids, + splice_prefix_tokens, + turn_end_token_ids, +) from nemo_rl.models.generation.vllm.utils import ( chat_generation_tokens, chat_routed_experts, @@ -475,6 +479,10 @@ async def _serve_gym_call( model_prefix_token_ids=prefix_token_ids, template_prefix_token_ids=None, template_token_ids=template_token_ids, + eos_token_id=turn_end_token_ids( + model_stop_token_ids(self._llm.model_config, host.cfg), + self._llm.tokenizer, + ), ) token_ids = splice.token_ids self._check_prompt_fits(len(token_ids)) diff --git a/nemo_rl/models/generation/openai_server_utils.py b/nemo_rl/models/generation/openai_server_utils.py index cd6b49ad2e1..f1e26bd1a8b 100644 --- a/nemo_rl/models/generation/openai_server_utils.py +++ b/nemo_rl/models/generation/openai_server_utils.py @@ -27,6 +27,30 @@ from typing import Any +def model_stop_token_ids(model_config: Any, cfg: dict[str, Any]) -> frozenset[int]: + """The configured stops plus the generation config's EOS, read once per server. + + ``model_config`` is vLLM's ``ModelConfig``; ``cfg`` is ``policy.generation``. + """ + stop_token_ids = set(cfg.get("stop_token_ids") or ()) + generation_eos = model_config.try_get_generation_config().get("eos_token_id") + if isinstance(generation_eos, int): + generation_eos = [generation_eos] + return frozenset(stop_token_ids | set(generation_eos or ())) + + +def turn_end_token_ids( + stop_token_ids: Collection[int], tokenizer: Any +) -> frozenset[int]: + """Every id a turn can end on, for :func:`splice_prefix_tokens`. + + vLLM stops on the generation config's EOS as well as the configured stops, + and the chat template closes turns with the tokenizer's EOS, so a turn can + end on any of them. ``stop_token_ids`` is :func:`model_stop_token_ids`. + """ + return frozenset(stop_token_ids) | {tokenizer.eos_token_id} + + @dataclass(frozen=True) class PrefixSplice: """Exact token splice and its source coordinates for multimodal ranges.""" @@ -42,7 +66,7 @@ def replace_prefix_tokens( template_prefix_token_ids: list[int], template_token_ids: list[int], *, - eos_token_id: int | Collection[int] | None = None, + eos_token_id: int | Collection[int], ) -> list[int]: """Replace a rendered history with the exact previously generated tokens. @@ -64,7 +88,7 @@ def splice_prefix_tokens( model_prefix_token_ids: list[int], template_prefix_token_ids: list[int] | None, template_token_ids: list[int], - eos_token_id: int | Collection[int] | None = None, + eos_token_id: int | Collection[int], ) -> PrefixSplice: """This is a subroutine used inside the OpenAI-compatible Chat Completion server. @@ -124,14 +148,14 @@ def splice_prefix_tokens( resumes from the template after that EOS: output => [11,12,13,40,41,220,17,2,21,22,40,41] - ``eos_token_id`` overrides ``tokenizer.eos_token_id``; with it, ``tokenizer`` - may be ``None`` (callers that only hold token ids, e.g. the Megatron prompt - preparer) and the failure message skips the detokenized reprs. It may also - be a collection of ids for models that declare several EOS tokens (the - Megatron chat endpoint ships the model's full EOS set): any of them counts - toward the boundary, and when the model's prefix ends on one of them that - exact id is kept in place of the template's, so the spliced prompt still - starts with the model's tokens verbatim. + ``eos_token_id`` is every id a turn can end on (:func:`turn_end_token_ids` + for vLLM); the tokenizer's own EOS alone misses a turn that ended on another + stop (gpt-oss ``<|call|>``). ``tokenizer`` may be ``None`` (callers that only + hold token ids, e.g. the Megatron prompt preparer); the failure message then + skips the detokenized reprs. Any id in the set counts toward the boundary, + and when the model's prefix ends on one of them that exact id is kept in + place of the template's, so the spliced prompt still starts with the + model's tokens verbatim. A host that sees only the rendered prompt, not the messages (the Dynamo engine), passes ``template_prefix_token_ids=None``. The prefix render is only @@ -144,9 +168,6 @@ def splice_prefix_tokens( if not model_prefix_token_ids: return PrefixSplice(template_token_ids, 0, 0) - if eos_token_id is None: - eos_token_id = tokenizer.eos_token_id - assert eos_token_id is not None, "Tokenizer must have an EOS token ID" eos_token_ids = ( frozenset([eos_token_id]) if isinstance(eos_token_id, int) diff --git a/nemo_rl/models/generation/trtllm/trtllm_http_server.py b/nemo_rl/models/generation/trtllm/trtllm_http_server.py index 035481c1c3a..bc2e6de3c04 100644 --- a/nemo_rl/models/generation/trtllm/trtllm_http_server.py +++ b/nemo_rl/models/generation/trtllm/trtllm_http_server.py @@ -168,7 +168,7 @@ def _sync() -> list[int]: model_prefix_token_ids=required_prefix_ids, template_prefix_token_ids=template_prefix_ids, template_token_ids=prompt_token_ids, - eos_token_id=eos_token_ids or None, + eos_token_id=eos_token_ids or tokenizer.eos_token_id, ) return await asyncio.to_thread(_sync) diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index d2f2f2199a6..98d6169edc6 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -59,8 +59,12 @@ from vllm.v1.engine.async_llm import logger as vllm_async_llm_logger from nemo_rl.data.captured_media import MediaCaptureRejected -from nemo_rl.models.generation.capture_context import decide_capture_input -from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens +from nemo_rl.models.generation.capture_context import resolve_capture_input +from nemo_rl.models.generation.openai_server_utils import ( + model_stop_token_ids, + splice_prefix_tokens, + turn_end_token_ids, +) from nemo_rl.models.generation.vllm.captured_chat_logprobs import ( CapturedChatLogprobsMixin, ) @@ -68,8 +72,8 @@ attach_routed_experts_to_chat_response_choices, attach_sampler_topk_logprobs_to_chat_response_choices, attach_token_information_to_chat_response_choices, - check_sampling_param, model_dump_chat_response_with_dynamic_message_fields, + on_policy_sampling, remap_multimodal_placeholders, ) @@ -144,6 +148,85 @@ def _clamp_max_tokens( max_tokens = min(request_max_tokens, remaining) self._set_max_tokens(request, max_tokens) + async def _render_chat_call( + self, request, messages, render_kwargs: dict[str, Any], *, admission + ) -> "RenderedChat": + """Render a chat call and decide how its capture prefix is served. + + The steps every host of the chat path shares, before any TQ read: the + render with the checkpoint's template and processor, the capture + decision for a Gym candidate (C26 on #4979), and, when a prefix will be + spliced, the render of the history up to the last assistant turn that + the splice cuts against. + """ + for message in messages: + if message.get("tool_calls"): + message["tool_calls"] = list(message["tool_calls"]) + messages_for_replace_prefix_tokens = deepcopy(messages) + + try: + conversation, (engine_prompt,) = await super().preprocess_chat( + request=request, messages=messages, **render_kwargs + ) + except (ValueError, VLLMValidationError) as e: + if "maximum context length" in str(e): + LOGGER.warning("Prompt exceeds max_model_len: %s", e) + raise + + # Token capture: the admission is built once by the host and resolved + # here, before branching; the ``ng_capture`` dict is never mutated. + serving_admission = admission + verify_retained_media = False + if admission is not None and ( + admission.mode == "candidate" or admission.request_replay is not None + ): + # Inline assistant token fields cannot override RL's decision. + request.required_prefix_token_ids = None + decision = resolve_capture_input( + admission, + messages=messages_for_replace_prefix_tokens, + conversation=conversation, + renderer_is_hf=type(self.renderer) is HfRenderer, + ) + admission = decision.storage + serving_admission = decision.serving + verify_retained_media = decision.verify_retained_media + + history_token_ids = None + if getattr(request, "required_prefix_token_ids", None) is not None or ( + serving_admission is not None and serving_admission.mode == "token_in" + ): + # The splice cuts the full render at the turn this history ends. + last_assistant_message_idx = None + for i in reversed(range(len(messages_for_replace_prefix_tokens))): + if messages_for_replace_prefix_tokens[i]["role"] == "assistant": + last_assistant_message_idx = i + break + if last_assistant_message_idx is None: + messages_to_last_assistant_message = messages_for_replace_prefix_tokens + else: + messages_to_last_assistant_message = messages_for_replace_prefix_tokens[ + : last_assistant_message_idx + 1 + ] + modified_request = request.model_copy( + update={"add_generation_prompt": False} + ) + _, (history_prompt,) = await super().preprocess_chat( + request=modified_request, + messages=messages_to_last_assistant_message, + **render_kwargs, + ) + history_token_ids = history_prompt["prompt_token_ids"] + + return RenderedChat( + conversation=conversation, + engine_prompt=engine_prompt, + admission=admission, + serving_admission=serving_admission, + verify_retained_media=verify_retained_media, + history_token_ids=history_token_ids, + ) + # vLLM 0.25 moved chat preprocessing to # OnlineRenderer.preprocess_chat (tool_parser/reasoning_parser were # folded into a single `parser`), so this override now applies via @@ -161,14 +244,10 @@ async def preprocess_chat( skip_mm_cache: bool = False, ): host = self._host - for message in messages: - if message.get("tool_calls"): - message["tool_calls"] = list(message["tool_calls"]) - - messages_for_replace_prefix_tokens = deepcopy(messages) + admission = host._capture_admission(request) # #4124: processor-only cache reads retain concrete pixels even # when the engine's sender cache would return references. - if host._capture_media and host._capture_admission(request) is not None: + if host._capture_media and admission is not None: skip_mm_cache = True # Temporarily set to 1 so vLLM's pre-tokenization length check passes; @@ -185,54 +264,31 @@ async def preprocess_chat( if actual_request_max_tokens is not None: self._set_max_tokens(request, 1) - try: - res = await super().preprocess_chat( - request=request, - messages=messages, + rendered = await self._render_chat_call( + request, + messages, + dict( default_template=default_template, default_template_content_format=default_template_content_format, default_template_kwargs=default_template_kwargs, tool_dicts=tool_dicts, parser=parser, skip_mm_cache=skip_mm_cache, - ) - except (ValueError, VLLMValidationError) as e: - if "maximum context length" in str(e): - LOGGER.warning("Prompt exceeds max_model_len: %s", e) - raise - - # Token capture: build the admission once, before branching, - # and resolve its prefix from it (staging_chain -> cached TQ - # read, inline ids, or nothing for a text root). The - # ``ng_capture`` dict is never mutated. Off-loop: the chain - # fetch is a blocking TQ read, and Gym's staging protocol - # requires the serving host to move blocking staging I/O off - # its event loop explicitly. The adapter then attaches the - # prefix to the request, so the inline-prefix branch below is - # the single splice path for staged and inline prefixes. - admission = host._capture_admission(request) - serving_admission = admission - verify_retained_media = False - if admission is not None and ( - admission.mode == "candidate" or admission.request_replay is not None - ): - # Inline assistant token fields cannot override RL's decision. - request.required_prefix_token_ids = None - decision = decide_capture_input( - admission, - messages=messages_for_replace_prefix_tokens, - # HF/string parsing maps null and empty tool content to - # identical template inputs. Structured content and - # other renderers cannot authorize that equivalence. - allow_empty_tool_content=( - type(self.renderer) is HfRenderer - and bool(res[0]) - and all(isinstance(item.get("content"), str) for item in res[0]) - ), - ) - admission = decision.storage - serving_admission = decision.serving - verify_retained_media = decision.verify_retained_media + ), + admission=admission, + ) + res = (rendered.conversation, [rendered.engine_prompt]) + engine_prompt = rendered.engine_prompt + admission = rendered.admission + serving_admission = rendered.serving_admission + verify_retained_media = rendered.verify_retained_media + + # Resolve the admission's prefix (staging_chain -> cached TQ read). + # Off-loop: the chain fetch is a blocking TQ read, and Gym's staging + # protocol requires the serving host to move blocking staging I/O off + # its event loop explicitly. The adapter then attaches the prefix to + # the request, so the inline-prefix branch below is the single splice + # path for staged and inline prefixes. capture_prefix_token_ids: list[int] | None = None if serving_admission is not None and serving_admission.mode == "token_in": capture_prefix_token_ids = await asyncio.to_thread( @@ -249,65 +305,32 @@ async def preprocess_chat( self._clamp_max_tokens( request, actual_request_max_tokens, - res[1][0]["prompt_token_ids"], + engine_prompt["prompt_token_ids"], ) # Token capture, text mode: the full render is the exact # engine prompt. media = await asyncio.to_thread( host._capture_request_media, - res[1][0], + engine_prompt, admission=admission, verify_retained_media=verify_retained_media, ) host._begin_request_capture( request, - res[1][0]["prompt_token_ids"], + engine_prompt["prompt_token_ids"], admission=admission, media=media, ) return res - model_prefix_token_ids = list(request.required_prefix_token_ids) - - # Token-in splice path — shared by staging_chain and direct prefix. - last_assistant_message_idx = None - for i in reversed(range(len(messages_for_replace_prefix_tokens))): - if messages_for_replace_prefix_tokens[i]["role"] == "assistant": - last_assistant_message_idx = i - break - - if last_assistant_message_idx is None: - messages_to_last_assistant_message = messages_for_replace_prefix_tokens - else: - messages_to_last_assistant_message = messages_for_replace_prefix_tokens[ - : last_assistant_message_idx + 1 - ] - - modified_request = request.model_copy(update={"add_generation_prompt": False}) - - corresponding_res = await super().preprocess_chat( - request=modified_request, - messages=messages_to_last_assistant_message, - default_template=default_template, - default_template_content_format=default_template_content_format, - default_template_kwargs=default_template_kwargs, - tool_dicts=tool_dicts, - parser=parser, - skip_mm_cache=skip_mm_cache, - ) - actual_corresponding_token_ids = corresponding_res[1][0]["prompt_token_ids"] - - engine_prompt = res[1][0] - splice = splice_prefix_tokens( tokenizer=self.renderer.tokenizer, - model_prefix_token_ids=model_prefix_token_ids, - template_prefix_token_ids=actual_corresponding_token_ids, + model_prefix_token_ids=list(request.required_prefix_token_ids), + template_prefix_token_ids=rendered.history_token_ids, template_token_ids=engine_prompt["prompt_token_ids"], - # The template's own EOS must stay countable even when the - # configured stops don't include it. - eos_token_id=self._model_stop_token_ids - | {self.renderer.tokenizer.eos_token_id}, + eos_token_id=turn_end_token_ids( + self._model_stop_token_ids, self.renderer.tokenizer + ), ) final_prompt_token_ids = splice.token_ids # Reject overflows before media validation can mask the length error. @@ -358,8 +381,26 @@ async def preprocess_chat( return res +@dataclass(frozen=True) +class RenderedChat: + """One chat call rendered for the engine, before its prefix is spliced in.""" + + conversation: list[Any] + # The full render, multimodal inputs included; its ids are the template's. + engine_prompt: dict[str, Any] + # What capture stores, after the candidate decision; None without capture. + admission: Any + # How the prompt is served: "token_in" reads the prefix from TQ. + serving_admission: Any + verify_retained_media: bool + # The render up to the last assistant turn, when a prefix will be spliced. + history_token_ids: Optional[list[int]] + + class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): - def __init__(self, *args, host: Any, model_stop_token_ids: set[int], **kwargs): + def __init__( + self, *args, host: Any, model_stop_token_ids: frozenset[int], **kwargs + ): super().__init__(*args, **kwargs) self._host = host # vLLM adds the generation config's EOS to every request's stop set @@ -491,17 +532,6 @@ class ChatServer: serving_tokenization: NeMoRLServingTokenization -def model_stop_token_ids(model_config: Any, cfg: dict[str, Any]) -> set[int]: - """The configured stops plus the generation config's EOS.""" - stop_token_ids = set(cfg.get("stop_token_ids") or ()) - generation_eos_token_ids = model_config.try_get_generation_config().get( - "eos_token_id" - ) - if isinstance(generation_eos_token_ids, int): - generation_eos_token_ids = [generation_eos_token_ids] - return stop_token_ids | set(generation_eos_token_ids or ()) - - def build_chat_server( host: Any, *, engine_client: Any, model_config: Any, cfg: dict[str, Any] ) -> ChatServer: @@ -642,36 +672,22 @@ async def create_chat_completion( # Accept train and validation profiles; reject off-policy requests. # vLLM fills an unset top_k/min_p/repetition_penalty from # generation_config.json when vllm_kwargs sets generation_config="auto", - # so pin top_k to the matched profile's (as + # so serve the matched profile's top_k (as # vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params does) - # and disable min_p and the repetition penalty. vLLM applies these and - # the presence/frequency penalties and logit bias before sampling; the - # trainer replays none of them, so a request may not set them. - top_k = check_sampling_param( + # with min_p and the repetition penalty off; the trainer replays none of + # them, nor the penalties or logit bias a request may therefore not set. + for field, value in on_policy_sampling( temperature=request.temperature, top_p=request.top_p, top_k=request.top_k, + min_p=request.min_p, + repetition_penalty=request.repetition_penalty, + presence_penalty=request.presence_penalty, + frequency_penalty=request.frequency_penalty, + logit_bias=request.logit_bias, generation_config=generation_config, - ) - assert request.min_p in (None, 0.0), ( - f"request min_p={request.min_p} is not supported; leave it unset" - ) - assert request.repetition_penalty in (None, 1.0), ( - f"request repetition_penalty={request.repetition_penalty} is not " - "supported; leave it unset" - ) - assert request.presence_penalty in (None, 0.0), ( - f"request presence_penalty={request.presence_penalty} is not " - "supported; leave it unset" - ) - assert request.frequency_penalty in (None, 0.0), ( - f"request frequency_penalty={request.frequency_penalty} is not " - "supported; leave it unset" - ) - assert not request.logit_bias, "request logit_bias is not supported" - request.top_k = top_k - request.min_p = 0.0 - request.repetition_penalty = 1.0 + ).items(): + setattr(request, field, value) try: generator = await openai_serving_chat.create_chat_completion( diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index b92b285078f..83134b92fcf 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict +from collections import ChainMap, defaultdict from collections.abc import Mapping from dataclasses import is_dataclass, replace from typing import Any, Optional, cast +import numpy as np import torch from nemo_rl.data.multimodal_utils import ( @@ -388,6 +389,100 @@ def check_sampling_param( return profile_top_k +class OffPolicySamplingError(ValueError): + """A NeMo-Gym call that would sample off-policy; the trainer replays none of it.""" + + +# Sampling a NeMo-Gym call may not change, and the value it is served with: vLLM +# would otherwise fill min_p and the repetition penalty from the model's +# generation_config.json, and the trainer replays none of these. +_FIXED_SAMPLING = { + "min_p": 0.0, + "repetition_penalty": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, +} + + +def _f32(value: Optional[float]) -> Optional[float]: + return None if value is None else float(np.float32(value)) + + +def on_policy_sampling( + *, + temperature: Optional[float], + top_p: Optional[float], + top_k: Optional[int], + min_p: Optional[float], + repetition_penalty: Optional[float], + presence_penalty: Optional[float], + frequency_penalty: Optional[float], + logit_bias: Optional[Mapping[Any, float]], + generation_config: Any, +) -> dict[str, Any]: + """The sampling a NeMo-Gym chat call is served with, on every serving path. + + The request must match the train or validation profile + (:func:`check_sampling_param`) and set no sampling the trainer does not + replay. Floats are compared at f32: Dynamo's frontend carries them as f32, + so 0.7 arrives as 0.69999998. + + Args: + temperature, top_p, top_k, min_p, repetition_penalty, presence_penalty, + frequency_penalty, logit_bias: The request's values. + generation_config: ``policy.generation``. + + Returns: + The ``SamplingParams`` fields to serve with: the matched profile's exact + temperature, top_p and top_k, and the fixed min_p and penalties. + + Raises: + OffPolicySamplingError: The request would sample off-policy. + """ + # The profile's floats at f32; top_k and val_top_k as configured. + profile = ChainMap( + { + key: _f32(generation_config[key]) + for key in ("temperature", "top_p", "val_temperature", "val_top_p") + }, + generation_config, + ) + try: + served_top_k = check_sampling_param( + temperature=_f32(temperature), + top_p=_f32(top_p), + top_k=top_k, + generation_config=profile, + ) + except ValueError as error: + raise OffPolicySamplingError(str(error)) from error + requested = { + "min_p": min_p, + "repetition_penalty": repetition_penalty, + "presence_penalty": presence_penalty, + "frequency_penalty": frequency_penalty, + } + for key, served in _FIXED_SAMPLING.items(): + if requested[key] not in (None, served): + raise OffPolicySamplingError( + f"request {key}={requested[key]} is not supported; leave it unset" + ) + if logit_bias: + raise OffPolicySamplingError("request logit_bias is not supported") + is_train = (_f32(temperature), _f32(top_p)) == ( + profile["temperature"], + profile["top_p"], + ) + prefix = "" if is_train else "val_" + return { + "temperature": generation_config[f"{prefix}temperature"], + "top_p": generation_config[f"{prefix}top_p"], + "top_k": served_top_k, + "min_p": _FIXED_SAMPLING["min_p"], + "repetition_penalty": _FIXED_SAMPLING["repetition_penalty"], + } + + def validate_rollout_prompt(expected: list[int], actual: list[int] | None) -> None: """Reject rollouts generated from a different prompt than the learner's.""" if actual != expected: diff --git a/tests/unit/data/test_captured_media.py b/tests/unit/data/test_captured_media.py index 6a79307d142..a64db936950 100644 --- a/tests/unit/data/test_captured_media.py +++ b/tests/unit/data/test_captured_media.py @@ -447,6 +447,7 @@ def test_splice_coordinates_handle_reasoning_shift_and_repeated_pad_runs(): model_prefix_token_ids=original, template_prefix_token_ids=template[:6], template_token_ids=template, + eos_token_id=2, ) prompt = engine_prompt(template, [(Span(1, 2), a), (Span(7, 2), b)]) captured = capture_processed_media( @@ -471,6 +472,7 @@ def test_continuation_captures_only_new_media_and_rejects_changed_pixels(): model_prefix_token_ids=original, template_prefix_token_ids=template[:6], template_token_ids=template, + eos_token_id=2, ) retained = (StagedMediaTensors(**first.tensors, num_frames=None),) prompt = engine_prompt(template, [(Span(1, 2), a), (Span(7, 2), b)]) @@ -540,6 +542,7 @@ def test_changed_retained_pad_run_cannot_steal_the_next_image(): model_prefix_token_ids=original, template_prefix_token_ids=template_prefix, template_token_ids=template, + eos_token_id=2, ) with pytest.raises(ValueError, match="Retained media"): capture_processed_media( @@ -568,6 +571,7 @@ def test_span_mask_is_preserved_on_remap(): model_prefix_token_ids=[10, 77, 31, 2], template_prefix_token_ids=[10, 31, 2], template_token_ids=[10, 31, 2, 11, 18, 18, 12], + eos_token_id=2, ) prompt = engine_prompt( [10, 31, 2, 11, 18, 18, 12], [(Span(3, 4, mask), torch.ones(3, 2, 3))] @@ -959,6 +963,7 @@ def test_video_placeholder_remap_preserves_all_timestamp_tokens(): model_prefix_token_ids=original, template_prefix_token_ids=template_prefix, template_token_ids=template, + eos_token_id=2, ) prompt = video_prompt(template, [(Span(1, 4), frames), (Span(7, 4), frames)]) added = capture_processed_media( diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index 57efd0fbee1..ce63c1c9cd7 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -145,6 +145,7 @@ def __init__(self, outputs=(), *, defaults=None, admission_error=None): self.renderer = _FakeRenderer() self.model_config = SimpleNamespace( get_diff_sampling_param=lambda: dict(defaults or {}), + try_get_generation_config=lambda: {}, multimodal_config=None, allowed_local_media_path="", allowed_media_domains=None, diff --git a/tests/unit/models/generation/test_openai_server_utils.py b/tests/unit/models/generation/test_openai_server_utils.py index 7fafb1ea04d..71c470fd3a2 100644 --- a/tests/unit/models/generation/test_openai_server_utils.py +++ b/tests/unit/models/generation/test_openai_server_utils.py @@ -37,6 +37,7 @@ class _T: model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == template_token_ids @@ -60,22 +61,7 @@ def decode(self, *args, **kwargs): model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, - ) - - -def test_replace_prefix_tokens_tokenizer_without_eos_raises(): - """A tokenizer that has no EOS token cannot locate the splice boundary.""" - - class _T: - eos_token_id = None - - tokenizer = _T() - with pytest.raises(AssertionError): - replace_prefix_tokens( - tokenizer=tokenizer, - model_prefix_token_ids=[1], - template_prefix_token_ids=[1, 2], - template_token_ids=[1, 2], + eos_token_id=tokenizer.eos_token_id, ) @@ -236,12 +222,14 @@ def generate(text): model_prefix_token_ids=chain, template_prefix_token_ids=template_prefix, template_token_ids=template, + eos_token_id=tokenizer.eos_token_id, ) assert spliced == splice_prefix_tokens( tokenizer=tokenizer, model_prefix_token_ids=chain, template_prefix_token_ids=None, template_token_ids=template, + eos_token_id=tokenizer.eos_token_id, ) chain = spliced.token_ids + generate(next_reply) @@ -273,6 +261,7 @@ class _T: model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == [100, 2, 77, 88] @@ -300,6 +289,7 @@ def decode(self, ids, **kwargs): model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == [11, 12, 99, 99, 99, 55, 2, 70, 71, 2, 40, 41] diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 9c6cc9179bb..29fc86d4a3a 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -4018,6 +4018,7 @@ def test_VllmAsyncGenerationWorker_replace_prefix_tokens(tokenizer): model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == og_model_token_ids @@ -4031,6 +4032,7 @@ def test_VllmAsyncGenerationWorker_replace_prefix_tokens(tokenizer): model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == og_model_token_ids @@ -4045,6 +4047,7 @@ def test_VllmAsyncGenerationWorker_replace_prefix_tokens(tokenizer): model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == og_model_token_ids @@ -4060,6 +4063,7 @@ def test_VllmAsyncGenerationWorker_replace_prefix_tokens(tokenizer): model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == og_model_token_ids @@ -4072,6 +4076,7 @@ def test_VllmAsyncGenerationWorker_replace_prefix_tokens(tokenizer): model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == model_token_ids @@ -4085,6 +4090,7 @@ def test_VllmAsyncGenerationWorker_replace_prefix_tokens(tokenizer): model_prefix_token_ids=model_prefix_token_ids, template_prefix_token_ids=template_prefix_token_ids, template_token_ids=template_token_ids, + eos_token_id=tokenizer.eos_token_id, ) assert result == model_token_ids From 43efaba3bd6aabe86e88a98d9b0bd2e619e2d323 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 15:38:56 -0800 Subject: [PATCH 35/82] feat(dynamo)!: serve NeMo-Gym chat with NeMo RL's chat engine in Dynamo's frontend NeMo-Gym's chat calls through Dynamo no longer go through Dynamo's Rust renderer and parsers, and the engine no longer re-runs a copy of the native server's steps. Dynamo's frontend now runs NeMo RL's own chat engine (chat_engine_factory), and each call takes the native server's steps with the native server's code: - dynamo/vllm/frontend.py starts dynamo.frontend's main with NeMo RL's chat_engine_factory added; /v1/completions (NeMo RL's rollouts) is unchanged. - dynamo/vllm/render_pool.py: spawned processes that build the vLLM renderer and chat serving the native server builds, from the engine args each engine publishes in its registration's runtime_data. They run the shared render step (_render_chat_call, with the C26 capture decision) and build replies with vLLM's response builder and parsers (C39). - dynamo/vllm/chat_engine.py: on-policy sampling (C41), the TQ prefix read and splice on the model's turn-end tokens (C16), the context clamp, ng_measure (C36), and routing with a capture plan in extra_args. - dynamo/vllm/adapter.py (was dynamo/vllm_adapter.py): the Gym handler runs the steps after the render (media snapshot, begin_call, generation with the SamplingParams vLLM's chat serving builds, one TQ write, sampler top-k for score centering (C29), routes on every call (C23, C34)); a candidate admission reaching the engine is refused (C26). - gym_call.py: GymCallExtras / GymCallResult define the fields both paths exchange with Gym (C37); Gym packs them into one nemo_rl:gym: annotation and reads nvext.nemo_rl. - The native route answers off-policy sampling with a 400 instead of a 500. Config: dynamo_cfg.worker_args is removed (rejected at startup); the native vllm_cfg chat settings apply unchanged. New frontend_args.render_workers. --dyn-chat-processor and --custom-jinja-template are rejected. The Dynamo top_logprobs rejection is lifted. Tests: test_gym_chat_path.py sends the same calls through the native routes and through Dynamo's path and checks they run the same prompt and SamplingParams, stage the same capture record and return the same reply (C27 AI-1); new chat-engine, render-pool and Gym field-name tests; the decoupling test covers every file outside dynamo// (C42). Re-pins Gym to a9e61ef (one extras annotation, nvext.nemo_rl results). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/design-docs/dynamo-integration.md | 90 +- docs/guides/dynamo-generation.md | 81 +- ...grpo_math_1B_dynamo_single_controller.yaml | 14 +- nemo_rl/environments/nemo_gym.py | 35 +- nemo_rl/models/generation/__init__.py | 21 +- nemo_rl/models/generation/dynamo/config.py | 170 +--- .../generation/dynamo/engine_adapter.py | 13 +- .../generation/dynamo/managed_runtime.py | 71 +- .../models/generation/dynamo/vllm/__init__.py | 13 + .../models/generation/dynamo/vllm/adapter.py | 567 ++++++++++++ .../generation/dynamo/vllm/chat_engine.py | 406 +++++++++ .../models/generation/dynamo/vllm/frontend.py | 54 ++ .../generation/dynamo/vllm/render_pool.py | 381 ++++++++ .../models/generation/dynamo/vllm_adapter.py | 817 ------------------ nemo_rl/models/generation/gym_call.py | 79 ++ nemo_rl/models/generation/vllm/chat_server.py | 198 +++-- .../models/generation/vllm/vllm_generation.py | 8 + .../generation/vllm/vllm_worker_async.py | 8 +- nemo_rl/utils/outdated_config_checks.py | 24 +- pyrefly.toml | 8 +- .../test_nemo_gym_token_capture.py | 16 +- .../generation/test_dynamo_chat_engine.py | 393 +++++++++ .../models/generation/test_dynamo_config.py | 94 +- .../generation/test_dynamo_managed_runtime.py | 34 +- .../generation/test_dynamo_render_pool.py | 152 ++++ .../generation/test_dynamo_vllm_adapter.py | 471 +++++----- .../test_dynamo_vllm_adapter_capture.py | 270 +++--- .../generation/test_dynamo_vllm_decoupling.py | 45 +- tests/unit/models/generation/test_gym_call.py | 89 ++ .../models/generation/test_gym_chat_path.py | 563 ++++++++++++ .../models/generation/test_vllm_generation.py | 14 +- .../unit/utils/test_outdated_config_checks.py | 8 +- 33 files changed, 3549 insertions(+), 1660 deletions(-) create mode 100644 nemo_rl/models/generation/dynamo/vllm/__init__.py create mode 100644 nemo_rl/models/generation/dynamo/vllm/adapter.py create mode 100644 nemo_rl/models/generation/dynamo/vllm/chat_engine.py create mode 100644 nemo_rl/models/generation/dynamo/vllm/frontend.py create mode 100644 nemo_rl/models/generation/dynamo/vllm/render_pool.py delete mode 100644 nemo_rl/models/generation/dynamo/vllm_adapter.py create mode 100644 nemo_rl/models/generation/gym_call.py create mode 100644 tests/unit/models/generation/test_dynamo_chat_engine.py create mode 100644 tests/unit/models/generation/test_dynamo_render_pool.py create mode 100644 tests/unit/models/generation/test_gym_call.py create mode 100644 tests/unit/models/generation/test_gym_chat_path.py diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index a7be5c5f3a4..a9e61ef75b6 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit a7be5c5f3a4d0957c9e84a8f7face90e30b7171e +Subproject commit a9e61ef75b69c964f843f847987b021b1b2d6ddf diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index dab903361be..5bf185fac66 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -6,13 +6,18 @@ Dynamo's frontend and KV-aware router. Two requirements shape the design: - A vLLM bump must never be blocked by Dynamo. - Dynamo CI coverage must stay consistent with native vLLM's. -Both hold only if the engine process contains no Dynamo-owned vLLM code. A vLLM -bump can then break Dynamo serving only in NeMo RL's own vLLM adapter -(`vllm_adapter.py`), which calls a few vLLM APIs native NeMo RL does not: the -`get_kv_cache_group_metadata` engine utility and `KVCacheSpecKind` at -registration, and `StatLoggerBase` for the router's load signal. A renamed import -fails the vLLM-lane unit tests at collection, and the Dynamo functional overlays -run registration. +Both hold only if no process runs Dynamo-owned vLLM code. A vLLM bump can then +break Dynamo serving only in NeMo RL's own `dynamo/vllm/` package: + +- the engine adapter (`adapter.py`) calls a few vLLM APIs native NeMo RL does + not: the `get_kv_cache_group_metadata` engine utility and `KVCacheSpecKind` + at registration, and `StatLoggerBase` for the router's load signal; +- the frontend's chat engine (`chat_engine.py`, `render_pool.py`) builds the + same vLLM renderer and chat serving the native server builds + (`vllm/chat_server.py`), from the engine's own engine args. + +A renamed import fails the vLLM-lane unit tests at collection, and the Dynamo +functional overlays run registration and NeMo-Gym's chat calls. ## Engine hosting @@ -29,9 +34,9 @@ vLLM. NeMo RL implements its `LLMEngine` in two parts: annotation, maps engine errors to a 400 that keeps the cause, and registers the endpoint for `/v1/completions` (NeMo RL's rollouts) and chat (NeMo-Gym). It imports no backend, and the engine only ever receives token IDs. -- `dynamo/vllm_adapter.py`: `DynamoVllmAdapter`, the vLLM backend over the +- `dynamo/vllm/adapter.py`: `DynamoVllmAdapter`, the vLLM backend over the worker's `AsyncLLM`. A backend implements `serve_rollout`, `start`, `abort` and - `kv_event_sources`. + `kv_event_sources`, and may add request handlers. The two together: @@ -47,19 +52,14 @@ The two together: The result travels in `engine_data`, and the driver assembles the same output with `request_pipeline.rollout_output`. - NeMo-Gym's calls also carry an annotation (Gym's Dynamo dialect). The engine - applies the native chat endpoint's rules to them (on-policy sampling, the - prefix splice, the context clamp), stages a token-capture call through the - worker's own capture lifecycle, and takes the token fields and routed experts - from the native endpoint's helpers. - - A request with media carries the frontend's tokens, with one unexpanded - placeholder per item, and the media URLs. The adapter loads the media with - vLLM's media connector. It then builds the processed prompt with the engine's - renderer (`process_for_engine_async`), the same step that follows rendering - on the native endpoint. The splice, the placeholder remap - (`remap_multimodal_placeholders`) and media capture (`_capture_request_media`) - all run on that processed prompt. + NeMo-Gym's calls arrive rendered and spliced by NeMo RL's chat engine (see + [NeMo-Gym chat](#nemo-gym-chat)), annotated for the adapter's Gym handler, + with the capture plan in `extra_args`. The handler runs the native server's + steps after the render, with the worker's own helpers: the media snapshot + (`_capture_request_media`), `begin_call` (which stamps the weight version), + the generation with the `SamplingParams` vLLM's chat serving builds from the + request, and the one TQ write. Its `engine_data` carries the output the + frontend builds the reply from, and the call's `GymCallResult`. - `abort` cancels the vLLM request on client disconnect. - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It @@ -88,6 +88,37 @@ No HTTP control routes are involved. The Rust Worker installs a SIGTERM handler that chains to Ray's, so actor shutdown is unchanged, and discovery drops an engine when its process exits. +## NeMo-Gym chat + +Dynamo's frontend hands chat calls to a Python `chat_engine_factory` when one is +given. `dynamo/vllm/frontend.py` starts `dynamo.frontend`'s own `main` with +NeMo RL's factory added to the `EntrypointArgs` it builds, so the frontend +keeps its HTTP server, router and `/v1/completions` path, and only chat goes to +NeMo RL's chat engine (`dynamo/vllm/chat_engine.py`). + +Each engine publishes, in its registration's `runtime_data`, its own engine +args and the `policy.generation` chat settings. The chat engine builds a +render pool (`dynamo/vllm/render_pool.py`) from them: `render_workers` spawned +processes, each holding the vLLM renderer and chat serving the native server +builds. Rendering is CPU-bound Python, and the frontend serves every call on +one event loop, so it runs in the pool, first in first out. Per call: + +1. the pool renders it with the native server's render step + (`_render_chat_call`, shared with `vllm/chat_server.py`), including the + token-capture decision for a Gym candidate and the history render the + splice cuts against; +2. the chat engine checks on-policy sampling (`on_policy_sampling`, shared + with the native route), reads a token-in prefix from TQ through a read-only + client, splices it in with the model's turn-end tokens, and clamps the + output budget; +3. the final token IDs go through the KV router to one engine (see above); +4. the pool builds the reply with vLLM's own response builder and parsers. + +`GymCallExtras` and `GymCallResult` (`models/generation/gym_call.py`) define +the fields both paths exchange with Gym. `tests/unit/models/generation/test_gym_chat_path.py` +sends the same calls through the native routes and through this path and +checks they match step for step. + ## Driver `VllmGeneration` starts `ManagedDynamoRuntime`, which runs etcd, NATS and @@ -105,15 +136,20 @@ lists what goes where. ## Dependency coupling The vLLM workers' venv (the `vllm` extra) carries the released `ai-dynamo` and -`ai-dynamo-runtime` wheels, never `ai-dynamo[vllm]`. The engine process only -imports `dynamo.common.backend`, `dynamo.llm.exceptions` and `dynamo._core`. +`ai-dynamo-runtime` wheels, never `ai-dynamo[vllm]`. NeMo RL imports only the +Dynamo modules `tests/unit/models/generation/test_dynamo_vllm_decoupling.py` +lists: the backend SDK, `dynamo.llm` and the frontend's `main`, none of which +import vLLM. What a Dynamo upgrade has to re-check: - the `dynamo.common.backend` contract (`LLMEngine`, `Worker`, `WorkerConfig`, publisher and logprob helpers); -- the request fields of Dynamo's preprocessed request; -- the frontend's `nvext` extensions and annotation forwarding; -- the parser names the native settings map to (`dynamo/config.py`). +- the request fields of Dynamo's preprocessed request, and `extra_args` + reaching the engine; +- the frontend's `chat_engine_factory` hook and the `EntrypointArgs` name + `dynamo/vllm/frontend.py` wraps, which are internal API; +- the Python chat request the hook receives, its `nvext` and annotation + fields, and `EngineConfig.runtime_data` reaching the frontend. The one vLLM↔Dynamo wire coupling is vLLM's KV-event msgpack, which Dynamo's router decodes. A change there degrades routing, not serving. The managed diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 01beed5ca46..532313ee740 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -52,11 +52,10 @@ shows them all: |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | | `request_timeout_s` | null | Deadline for one NeMo RL rollout through the frontend. null waits as long as generation takes, as native does; on the SingleController, `async_rl.rollout_failure.native.generation_timeout_s` bounds both paths | -| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | -| `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | | `frontend_args.session_affinity_ttl_secs` | 300 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it. Dynamo keeps at most 65,536 sessions and answers a new one past that with HTTP 529 | -| `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | +| `frontend_args.render_workers` | 8 | Processes in the frontend that render NeMo-Gym's chat calls and build their replies. Each serves about 640 single-turn calls/s, 45 for a 21K-token agentic history, or 200 with 1-4 images | +| `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags. `--dyn-chat-processor` and `--custom-jinja-template` are rejected: NeMo RL renders chat itself | The etcd and nats-server binaries are found in `NEMO_RL_DYNAMO_BIN_DIR` when it is set (the image sets it to `/opt/dynamo/bin`), and on `PATH` otherwise. @@ -68,7 +67,7 @@ is set (the image sets it to `/opt/dynamo/bin`), and on `PATH` otherwise. | `generate_async` on text token prompts, router replay (R3) included | Frontend (KV-aware); the engine samples and assembles it exactly as native `generate_async` does | | NeMo RL's own multimodal rollouts | Native shard dispatch. Those rollouts send media to the engine directly rather than as OpenAI content | | `generate_text_async`, which evaluation uses | Native shard dispatch | -| NeMo-Gym's OpenAI requests | Frontend, in Gym's Dynamo dialect; the engine serves them as the native chat endpoint does. See [NeMo-Gym](#nemo-gym) | +| NeMo-Gym's chat calls | Frontend, rendered by NeMo RL's chat engine with the native server's code. See [NeMo-Gym](#nemo-gym) | A rollout through the frontend has the same `GenerationOutputSpec` as the native path: token IDs, log probabilities, truncation, routed experts with @@ -79,48 +78,40 @@ build it with `nemo_rl/models/generation/vllm/request_pipeline.py`. NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect (`vllm_model.dynamo_frontend`, from -[NVIDIA-NeMo/Gym#4817](https://github.com/NVIDIA-NeMo/Gym/pull/4817)). Gym marks -every call with an annotation, and the engine serves a marked call as the native -chat endpoint does: - -- Sampling must match `temperature`/`top_p` or `val_temperature`/`val_top_p`, - with `top_k` unset and `n` 1. vLLM samples with the configured values. -- The token prefix Gym echoes replaces the history the frontend rendered. -- Stop strings stop vLLM itself, so the token IDs match the native endpoint's. -- The response carries `prompt_token_ids`, `completion_token_ids` and - `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. - With router replay on, it also carries the call's `routed_experts`, as the - native endpoint attaches them. -- Images, video and audio in the messages work as on the native endpoint, - token capture included. - -With `token_capture.enabled`, Gym uses its `dynamo_worker` capture backend. The -same vLLM workers stage each call's tokens, log probabilities, routed experts and -processed media from their own process, exactly as the native endpoint does, and -only the commit coordinates travel back through the frontend. - -The native server's chat settings carry over: - -| Native setting (`vllm_cfg`) | Under Dynamo | -|---|---| -| `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | -| `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | -| `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | -| `http_server_serving_chat_kwargs.default_chat_template_kwargs` | Sent by Gym under its own `chat_template_kwargs` | -| `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | +[NVIDIA-NeMo/Gym#4817](https://github.com/NVIDIA-NeMo/Gym/pull/4817)). The +frontend takes only OpenAI fields plus `nvext`, so Gym sends the native +request's extra fields (`ng_capture`, `required_prefix_token_ids`, `ng_measure`) +in one `nemo_rl:gym:` annotation, and reads the call's token fields +(`prompt_token_ids`, `generation_token_ids`, `generation_log_probs`, +`routed_experts`, `ng_commit_coords`) from `nvext.nemo_rl`. + +The frontend serves chat with NeMo RL's chat engine instead of Dynamo's own +renderer and parsers (Dynamo's `chat_engine_factory` hook). Each call takes the +native server's steps, with the native server's code: + +1. In the frontend, a render-pool process renders the call with the engine's + own tokenizer, template, processor and `vllm_cfg` chat settings (parser + plugins included), which each engine publishes when it registers. It also + decides how a token-capture call's prefix is served. +2. The chat engine checks on-policy sampling, reads a captured prefix from TQ, + splices the prefix into the render, clamps the output budget to the + context, and routes the final token IDs through the KV router. +3. On the engine, the adapter snapshots the call's media, begins its capture, + generates with the `SamplingParams` the native server builds, and stages the + call in one TQ write. Only the commit coordinates travel back. +4. A render-pool process builds the reply with vLLM's reasoning and tool parsers. + +`tests/unit/models/generation/test_gym_chat_path.py` sends the same calls down +both paths and checks they run the same prompt and `SamplingParams`, stage the +same record and return the same reply. Where the behavior differs from the native endpoint: -- The engine sees the rendered prompt, not the messages. It finds where the - echoed prefix ends by counting the prefix's EOS tokens, not by rendering the - history. -- When a request sets `chat_template_kwargs.force_nonempty_content`, Dynamo - reports a reasoning-only turn as content. -- `default_chat_template_kwargs` sit under the template kwargs set in - `env.nemo_gym` only. A key that a Gym YAML also sets loses to the default, and - template kwargs sent through `extra_body` from a Gym YAML or a row's - `metadata.extra_body` are dropped. - The frontend refuses more than 4 `stop` strings or `stop_token_ids` (HTTP 500). +- A token-capture call refused for its context length is answered without the + native endpoint's `x-ng-capture-refused` header, which Dynamo's errors cannot + carry. +- A call's processed media travels to the engine inside the request. ## Not supported @@ -141,12 +132,8 @@ Startup rejects these with `dynamo: true`: rank 0, vLLM rejects each engine's `ipc://` KV-event endpoint. - `vllm_cfg.http_server_api_key_required`: the frontend serves the same engines on 0.0.0.0 and checks no bearer token. -- `top_logprobs > 0` (`loss_fn.score_centering`): the engine's token capture - does not stage sampler top-k log-probs yet. - `frontend_args.router_mode: direct`: it needs a worker ID on every request, and NeMo RL's rollouts name none. -- Parser plugins with no Dynamo parser, and the chat settings above that the - frontend has no counterpart for. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. @@ -193,5 +180,5 @@ is the same run with every `dynamo_cfg` default spelled out. `backend: dynamo` launched Dynamo's own vLLM engine and was removed. Set `backend: vllm` and `dynamo: true`; `vllm_cfg` and `vllm_kwargs` keep their meaning. Startup rejects the old backend and each removed `dynamo_cfg` key -(`engine`, `control_timeout_s`, `metrics_*_prefixes`, `worker_args.extra_cli_args`, +(`engine`, `control_timeout_s`, `metrics_*_prefixes`, `worker_args`, `frontend_args.tokenizer*`) with what replaces it. diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 9c6eb20a2c7..2c1ff204b1d 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -19,24 +19,14 @@ policy: # async_rl.rollout_failure.native.generation_timeout_s bounds both paths. # NeMo-Gym's own calls keep Gym's timeouts. request_timeout_s: null - worker_args: - # Dynamo's built-in parser names. null follows the native server's - # vllm_cfg.http_server_serving_chat_kwargs (tool_parser, reasoning_parser). - tool_call_parser: null - reasoning_parser: null - exclude_tools_when_tool_choice_none: false - enable_structural_tag: false - structural_tag_scope: auto - structural_tag_schema: auto - # A chat template that replaces the model's own. null follows - # vllm_cfg.http_server_serving_chat_kwargs.chat_template. - custom_jinja_template: null frontend_args: # kv: route each request to the engine holding most of its prefix. router_mode: kv # Keep a NeMo-Gym token-capture rollout on one engine while it idles up # to this many seconds (null disables it). session_affinity_ttl_secs: 300 + # Processes that render NeMo-Gym's chat calls and build their replies. + render_workers: 8 extra_cli_args: [] logger: diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 2c37c56bd9b..435f41e5235 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -128,7 +128,6 @@ ) from nemo_rl.models.generation.dynamo.config import ( dynamo_enabled, - native_default_chat_template_kwargs, ) from nemo_rl.models.generation.interfaces import ( resolve_routed_experts_dtype_name_for_model, @@ -1353,7 +1352,8 @@ def _spinup(self) -> None: ) policy_overrides["return_token_id_information"] = False if ( - _external_staging_backend(token_capture) == "vllm_worker" + _external_staging_backend(token_capture) + in ("vllm_worker", "dynamo_worker") and token_capture.get("top_logprobs", 0) > 0 ): policy_overrides.setdefault("sampling_overrides", {}).update( @@ -2942,13 +2942,10 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_token_ids"] = None if dynamo_enabled(generation_config): - # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect, - # with the native server's default template kwargs. Each actor applies it - # to its own composed config (_apply_dynamo_dialect), so it never becomes - # a policy_model overlay that a shard must claim. - config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = ( - native_default_chat_template_kwargs(generation_config) - ) + # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect. + # Each actor applies it to its own composed config (_apply_dynamo_dialect), + # so it never becomes a policy_model overlay that a shard must claim. + config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = True # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for @@ -3026,9 +3023,7 @@ def build_nemo_gym_config( ) -def _apply_dynamo_dialect( - nemo_gym_dict: dict[str, Any], default_chat_template_kwargs: Mapping[str, Any] -) -> None: +def _apply_dynamo_dialect(nemo_gym_dict: dict[str, Any]) -> None: """Point one actor's policy model at the Dynamo frontend, in Gym's Dynamo dialect. The frontend takes token data in nvext instead of vLLM's request extensions. @@ -3050,17 +3045,6 @@ def _apply_dynamo_dialect( # The frontend reads `reasoning` as another name for `reasoning_content` and # refuses an echoed assistant turn that carries both, Gym's default. vllm_model.setdefault("reasoning_field", "reasoning_content") - # The native server merges these under every request's own template - # kwargs; the frontend has no such default, so Gym sends them. They sit - # under the kwargs Gym sends: vllm_model's own, else extra_body's. - if default_chat_template_kwargs: - gym_kwargs = vllm_model.get("chat_template_kwargs") or ( - (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} - ) - vllm_model["chat_template_kwargs"] = { - **default_chat_template_kwargs, - **gym_kwargs, - } models["vllm_model"] = vllm_model policy_model["responses_api_models"] = models nemo_gym_dict["policy_model"] = policy_model @@ -3096,9 +3080,8 @@ def _build_gym_actor_config( invalid_tool_call_patterns = nemo_gym_dict.pop("invalid_tool_call_patterns", None) thinking_tags = nemo_gym_dict.pop("thinking_tags", None) tokenizer_config = nemo_gym_dict.pop("tokenizer_config", None) - dynamo_dialect = nemo_gym_dict.pop("dynamo_dialect", None) - if dynamo_dialect is not None: - _apply_dynamo_dialect(nemo_gym_dict, dynamo_dialect) + if nemo_gym_dict.pop("dynamo_dialect", False): + _apply_dynamo_dialect(nemo_gym_dict) external_service_readiness = extract_external_service_readiness(nemo_gym_dict) port_range = { key: value diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 5f04f9a0811..a3b80bb19b5 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -22,12 +22,7 @@ check_top_p_has_top_k, need_top_k_or_top_p_filtering, ) -from nemo_rl.models.generation.dynamo.config import ( - check_dynamo_serving_chat_kwargs, - dynamo_cfg, - dynamo_enabled, - dynamo_parsers, -) +from nemo_rl.models.generation.dynamo.config import dynamo_cfg, dynamo_enabled from nemo_rl.models.generation.interfaces import ( GenerationConfig, GenerationInterface, @@ -219,11 +214,8 @@ def configure_generation_config( # vision parameters; freezing alone does not exclude them from refit. config["vllm_cfg"].setdefault("load_format", "auto" if is_eval else "dummy") if dynamo_enabled(config): - # Validate before any Ray allocation; dynamo_parsers also rejects - # parser plugins that Dynamo's Rust frontend cannot run. + # Validate before any Ray allocation. dynamo_cfg(config) - dynamo_parsers(config) - check_dynamo_serving_chat_kwargs(config) if config["vllm_cfg"].get("http_server_api_key_required"): raise ValueError( "policy.generation.vllm_cfg.http_server_api_key_required is not " @@ -231,15 +223,6 @@ def configure_generation_config( "frontend serves the same engines on 0.0.0.0 and checks no " "bearer token." ) - if config.get("top_logprobs"): - raise ValueError( - "policy.generation.top_logprobs > 0 (loss_fn.score_centering, " - "policy.generation.sampler_replay) is " - "not supported with policy.generation.dynamo=true yet: the " - "Dynamo engine's token capture does not stage sampler top-k " - "log-probs. Set top_logprobs to 0 and turn off " - "loss_fn.score_centering, or remove policy.generation.dynamo." - ) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 621afa087de..9e9181fc308 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -15,30 +15,14 @@ """Configuration for serving NeMo RL's vLLM workers behind Dynamo. ``policy.generation.dynamo: true`` on any vLLM config is the whole switch; -``policy.generation.dynamo_cfg`` only tunes the frontend and registration, and -every field below carries its default. +``policy.generation.dynamo_cfg`` only tunes the frontend, and every field below +carries its default. NeMo-Gym's chat calls are rendered and parsed with the +native server's own settings (``vllm_cfg.http_server_serving_chat_kwargs``). """ from typing import Annotated, Any, Literal, Mapping -from pydantic import BaseModel, Field, PositiveFloat - - -class DynamoWorkerArgs(BaseModel, extra="forbid"): - """How each vLLM engine registers with Dynamo (``WorkerConfig`` fields).""" - - # Dynamo's built-in parser names (dynamo-parsers). None follows the native - # server's vllm_cfg.http_server_serving_chat_kwargs (see dynamo_parsers). - tool_call_parser: str | None = None - reasoning_parser: str | None = None - # False, as vLLM's OpenAIServingChat (the native server) defaults it. - exclude_tools_when_tool_choice_none: bool = False - enable_structural_tag: bool = False - structural_tag_scope: Literal["auto", "always"] = "auto" - structural_tag_schema: Literal["auto", "strict"] = "auto" - # A Jinja chat template file that replaces the model's own. None follows - # vllm_cfg.http_server_serving_chat_kwargs.chat_template. - custom_jinja_template: str | None = None +from pydantic import BaseModel, Field, PositiveFloat, PositiveInt, model_validator class DynamoFrontendArgs(BaseModel, extra="forbid"): @@ -63,11 +47,30 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): # and answers a new one past that with HTTP 529, so keep (captured rollouts # started per second) x TTL below 65,536. The bound is the frontend's own. session_affinity_ttl_secs: Annotated[int, Field(gt=0, le=31_536_000)] | None = 300 + # Processes that render NeMo-Gym's chat calls and build their replies (the + # checkpoint's template, tokenizer, HF processor and vLLM's parsers). Each + # serves on one core: about 640 single-turn calls/s, 45 for a 21K-token + # agentic history (22 with its prefix splice), or 200 with 1-4 images. + render_workers: PositiveInt = 8 extra_cli_args: list[str] = Field(default_factory=list) + @model_validator(mode="after") + def _chat_is_nemo_rls(self) -> "DynamoFrontendArgs": + # NeMo-Gym's chat calls are rendered and parsed by NeMo RL's chat + # engine, never by Dynamo's own renderer or chat processor. + for flag in ("--dyn-chat-processor", "--custom-jinja-template"): + if any(arg.split("=", 1)[0] == flag for arg in self.extra_cli_args): + raise ValueError( + f"dynamo_cfg.frontend_args.extra_cli_args may not set {flag}: " + "NeMo RL's chat engine renders and parses chat calls with " + "vllm_cfg.http_server_serving_chat_kwargs, as the native server " + "does." + ) + return self + class DynamoCfg(BaseModel, extra="forbid"): - """``policy.generation.dynamo_cfg``: the frontend and engine registration.""" + """``policy.generation.dynamo_cfg``: the managed frontend.""" # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: PositiveFloat = 600.0 @@ -76,134 +79,9 @@ class DynamoCfg(BaseModel, extra="forbid"): # async_rl.rollout_failure.native.generation_timeout_s bounds both paths. # NeMo-Gym's own calls keep Gym's timeouts. request_timeout_s: PositiveFloat | None = None - worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) -# vLLM parser names whose Dynamo built-in parser goes by another name. nano_v3 is -# NeMo RL's vLLM plugin: DeepSeek-R1 reasoning parsing, with the whole output as -# content when chat_template_kwargs.enable_thinking is false, as Dynamo's -# nemotron_v3 parser does. -_DYNAMO_REASONING_PARSER_NAMES = {"nano_v3": "nemotron_v3"} -_DYNAMO_TOOL_PARSER_NAMES = {"openai": "harmony"} - - -def _dynamo_parser( - configured: str | None, - native: str | None, - plugin: str | None, - renames: Mapping[str, str], - *, - kind: str, - worker_arg: str, -) -> str | None: - if configured is not None or native is None: - return configured - if native in renames: - return renames[native] - if plugin: - raise ValueError( - f"policy.generation.vllm_cfg.{kind}_parser_plugin loads a Python parser " - "into vLLM's OpenAI server, which Dynamo's Rust frontend cannot run. Set " - f"policy.generation.dynamo_cfg.worker_args.{worker_arg} to the Dynamo " - f"built-in parser for {native!r}." - ) - return native - - -def dynamo_parsers( - generation_config: Mapping[str, Any], -) -> tuple[str | None, str | None]: - """The ``(tool_call_parser, reasoning_parser)`` Dynamo's frontend applies. - - ``dynamo_cfg.worker_args`` names them explicitly. Otherwise they follow the - native server's ``vllm_cfg.http_server_serving_chat_kwargs``, renamed where - Dynamo calls the same parser differently. A parser plugin without such a - rename has no Dynamo equivalent and is rejected. - """ - worker_args = dynamo_cfg(generation_config).worker_args - vllm_cfg = generation_config["vllm_cfg"] - serving = vllm_cfg.get("http_server_serving_chat_kwargs") or {} - tool_call_parser = _dynamo_parser( - worker_args.tool_call_parser, - serving.get("tool_parser"), - vllm_cfg.get("tool_parser_plugin"), - _DYNAMO_TOOL_PARSER_NAMES, - kind="tool", - worker_arg="tool_call_parser", - ) - reasoning_parser = _dynamo_parser( - worker_args.reasoning_parser, - serving.get("reasoning_parser"), - vllm_cfg.get("reasoning_parser_plugin"), - _DYNAMO_REASONING_PARSER_NAMES, - kind="reasoning", - worker_arg="reasoning_parser", - ) - return tool_call_parser, reasoning_parser - - -def check_dynamo_serving_chat_kwargs(generation_config: Mapping[str, Any]) -> None: - """Reject native chat-server settings Dynamo's frontend has no counterpart for. - - The frontend renders templates and parses tool calls itself: it always - parses ``tool_choice="auto"`` calls when a tool parser is set, and has no - content-format setting. The native defaults (``enable_auto_tools: true``, - ``chat_template_content_format: auto``) match it; other values would - silently change what the policy sees. - """ - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - content_format = serving.get("chat_template_content_format") - if content_format not in (None, "auto"): - raise ValueError( - "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." - f"chat_template_content_format={content_format!r} is not supported " - "with policy.generation.dynamo=true: Dynamo's frontend renders chat " - "templates itself and has no content-format setting. Remove it or " - "set it to 'auto'." - ) - # Without a tool parser the frontend parses no tool calls, so turning auto - # tools off changes nothing there (single-turn recipes set exactly this). - if serving.get("enable_auto_tools") is False and ( - dynamo_parsers(generation_config)[0] is not None - ): - raise ValueError( - "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." - "enable_auto_tools=false is not supported with " - "policy.generation.dynamo=true: Dynamo's frontend always parses " - "tool_choice='auto' calls when a tool parser is set. Remove it, or " - "remove the tool parser." - ) - - -def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: - """The chat template the frontend renders with: a file path or literal Jinja. - - ``dynamo_cfg.worker_args.custom_jinja_template``, else the native server's - ``http_server_serving_chat_kwargs.chat_template``; None uses the model's own. - """ - configured = dynamo_cfg(generation_config).worker_args.custom_jinja_template - if configured is not None: - return configured - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - return serving.get("chat_template") - - -def native_default_chat_template_kwargs( - generation_config: Mapping[str, Any], -) -> dict[str, Any]: - """The template kwargs the native server merges under every request's own. - - ``default_chat_template_kwargs``, or the older ``chat_template_kwargs`` - spelling, of ``vllm_cfg.http_server_serving_chat_kwargs``. - """ - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - kwargs = serving.get("default_chat_template_kwargs") - if kwargs is None: - kwargs = serving.get("chat_template_kwargs") - return dict(kwargs or {}) - - def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" return bool(generation_config.get("dynamo")) diff --git a/nemo_rl/models/generation/dynamo/engine_adapter.py b/nemo_rl/models/generation/dynamo/engine_adapter.py index 95f53d8279c..33be6cb4e81 100644 --- a/nemo_rl/models/generation/dynamo/engine_adapter.py +++ b/nemo_rl/models/generation/dynamo/engine_adapter.py @@ -18,9 +18,9 @@ request kinds served, how errors reach the driver, and the endpoint's registration. A backend implements four methods over its worker's engine (``serve_rollout``, ``start``, ``abort`` and ``kv_event_sources``); -``vllm_adapter.DynamoVllmAdapter`` is the vLLM one. The engine only ever receives +``vllm.adapter.DynamoVllmAdapter`` is the vLLM one. The engine only ever receives token IDs: NeMo RL's rollouts on ``/v1/completions``, and NeMo-Gym's chat calls -once the frontend has rendered them. This module imports no generation backend. +once NeMo RL's chat engine in the frontend has rendered them. This module imports no generation backend. """ from __future__ import annotations @@ -106,7 +106,7 @@ async def generate( break else: raise InvalidArgument( - "NeMo RL's engines serve only NeMo RL rollouts through Dynamo " + "NeMo RL's engines serve only NeMo RL's own requests through Dynamo " f"(requests annotated {', '.join(map(repr, handlers))})." ) try: @@ -179,13 +179,13 @@ def start_dynamo_endpoint( served_model_name: str, namespace: str, engine_env: dict[str, str], - chat_config: Optional[dict[str, Any]] = None, ) -> asyncio.Task: """Register ``adapter``'s engine with Dynamo and serve it on the running loop. ``model_path`` is a local directory with the model's config and tokenizer - files; the engine is served under ``served_model_name``. ``chat_config`` holds - the ``WorkerConfig`` fields that set up the frontend's chat path for NeMo-Gym. + files; the engine is served under ``served_model_name``. Chat calls are + rendered by NeMo RL's chat engine in the frontend, not Dynamo's renderer, + so no template or parser is registered. The Rust Worker serves until the process gets SIGTERM. It installs a chained handler, so Ray's actor handler still runs and the actor exits as usual; @@ -204,6 +204,5 @@ def start_dynamo_endpoint( discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], request_plane=engine_env["DYN_REQUEST_PLANE"], event_plane=engine_env["DYN_EVENT_PLANE"], - **(chat_config or {}), ) return asyncio.get_running_loop().create_task(Worker(adapter, config).run()) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index 9e895747beb..f966acd8ca0 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -22,6 +22,7 @@ import json import logging import os +import pickle import re import shutil import signal @@ -31,7 +32,7 @@ import time import urllib.error import urllib.request -from typing import Callable +from typing import Any, Callable from nemo_rl.distributed.virtual_cluster import ( DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, @@ -118,11 +119,24 @@ def _managed_namespace() -> str: return re.sub(r"[^a-zA-Z0-9_-]+", "-", raw).strip("-_").lower() +# The frontend process's environment for NeMo RL's chat engine +# (dynamo/vllm/chat_engine.py): its render workers, where setup_token_capture +# writes the capture config, and the Ray cluster its TQ reads go through. +RENDER_WORKERS_ENV = "NEMO_RL_DYNAMO_RENDER_WORKERS" +CAPTURE_FILE_ENV = "NEMO_RL_DYNAMO_CAPTURE_FILE" +RAY_ADDRESS_ENV = "NEMO_RL_DYNAMO_RAY_ADDRESS" + + class ManagedDynamoRuntime: """Own etcd, NATS and the frontend that NeMo RL's vLLM engines register with.""" def __init__( - self, *, dynamo_cfg: DynamoCfg, model_name: str, frontend_python: str + self, + *, + dynamo_cfg: DynamoCfg, + model_name: str, + frontend_python: str, + ray_address: str, ) -> None: """Describe the services; nothing starts until :meth:`start`. @@ -131,10 +145,14 @@ def __init__( model_name: The served model; the frontend is ready once it lists it. frontend_python: An interpreter whose environment has ``ai-dynamo``: the vLLM workers' own. + ray_address: The Ray cluster's GCS address; the frontend's chat + engine reads token-capture prefixes from TQ through it. """ self._dynamo_cfg = dynamo_cfg self._model_name = model_name self._frontend_python = frontend_python + self._ray_address = ray_address + self._chat_dir: str | None = None self._namespace = _managed_namespace() self._host = "" self._etcd_port = 0 @@ -198,10 +216,9 @@ def start(self) -> None: "NATS_SERVER": f"nats://{self._host}:{self._nats_port}", "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", - # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. - # Its v2 parser path preserves engine_data, which NeMo-Gym needs for - # exact token IDs and log probabilities. - "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", + # A field the frontend doesn't know fails the call with a 400 rather + # than vanishing; an inherited setting can't turn that off. + "DYN_IGNORE_OPENAI_FE_UNSUPPORTED_FIELDS": "0", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", # NeMo-Gym sends media inline: raise the frontend's 45 MB HTTP body @@ -335,14 +352,21 @@ def _start_frontend(self) -> None: namespace=self._namespace, dynamo_cfg=self._dynamo_cfg, ) + # Dynamo's frontend, with NeMo RL's chat engine serving chat calls. command = [ self._frontend_python, "-m", - "dynamo.frontend", + "nemo_rl.models.generation.dynamo.vllm.frontend", *argv, ] print(f" [Dynamo] launching frontend argv={command!r}", flush=True) + self._chat_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_chat_") frontend_env = self._service_env() + frontend_env[RENDER_WORKERS_ENV] = str( + self._dynamo_cfg.frontend_args.render_workers + ) + frontend_env[CAPTURE_FILE_ENV] = os.path.join(self._chat_dir, "capture.pkl") + frontend_env[RAY_ADDRESS_ENV] = self._ray_address dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) frontend_env["VIRTUAL_ENV"] = dynamo_venv frontend_env["UV_PROJECT_ENVIRONMENT"] = dynamo_venv @@ -350,6 +374,36 @@ def _start_frontend(self) -> None: command, env=frontend_env, start_new_session=True ) + def publish_chat_capture( + self, + *, + dp_cfg: dict[str, Any], + staging_partition: str, + capture_media: bool, + ) -> None: + """Tell the frontend's chat engine where token-capture prefixes are read. + + Capture is set up after the engines register, so the chat engine reads + this on its first captured call. + """ + if self._chat_dir is None: + raise RuntimeError("Managed Dynamo runtime has not been started") + path = os.path.join(self._chat_dir, "capture.pkl") + # Pickled: the data-plane config holds pydantic storage configs. + with tempfile.NamedTemporaryFile( + "wb", dir=self._chat_dir, suffix=".pkl", delete=False + ) as capture_file: + pickle.dump( + { + "data_plane": dp_cfg, + "staging_partition": staging_partition, + "capture_media": capture_media, + }, + capture_file, + ) + # Renamed into place, so the chat engine never reads a partial file. + os.replace(capture_file.name, path) + def wait_for_engines( self, expected_engines: int, check_engines: Callable[[], None] ) -> None: @@ -471,6 +525,9 @@ def shutdown(self) -> None: if self._etcd_data_dir is not None: shutil.rmtree(self._etcd_data_dir, ignore_errors=True) self._etcd_data_dir = None + if self._chat_dir is not None: + shutil.rmtree(self._chat_dir, ignore_errors=True) + self._chat_dir = None if self._nats_data_dir is not None: shutil.rmtree(self._nats_data_dir, ignore_errors=True) self._nats_data_dir = None diff --git a/nemo_rl/models/generation/dynamo/vllm/__init__.py b/nemo_rl/models/generation/dynamo/vllm/__init__.py new file mode 100644 index 00000000000..4fc25d0d3c9 --- /dev/null +++ b/nemo_rl/models/generation/dynamo/vllm/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py new file mode 100644 index 00000000000..4f7eebc36bd --- /dev/null +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -0,0 +1,567 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serve a NeMo RL vLLM worker's own AsyncLLM through Dynamo's backend SDK. + +The engine process holds no Dynamo-owned vLLM code. ``DynamoVllmAdapter`` is the +vLLM backend of ``engine_adapter.DynamoEngineAdapter``, over the AsyncLLM that +``VllmAsyncGenerationWorkerImpl`` owns, so refit, pause, sleep, quantization and +speculative decoding are that worker's own. + +It serves two request kinds, both token ids in: NeMo RL's own rollouts, and +NeMo-Gym's chat calls, which NeMo RL's chat engine in the frontend rendered, +spliced and routed (``chat_engine``). For those the adapter runs the native +server's steps after the render: the media snapshot, ``begin_call``, the +generation with the SamplingParams the native server would build, and the one +TQ write. +""" + +from __future__ import annotations + +import asyncio +import base64 +import dataclasses +import json +import logging +import os +import pickle +from collections.abc import AsyncGenerator +from typing import TYPE_CHECKING, Any, Optional + +import torch +from dynamo._core import Context +from dynamo.common.backend import ( + EngineConfig, + GenerateChunk, + GenerateRequest, + LlmRegistration, +) +from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource +from dynamo.llm.exceptions import InvalidArgument +from vllm.entrypoints.serve.utils.api_utils import get_max_tokens +from vllm.exceptions import VLLMClientError +from vllm.inputs import TokensPrompt +from vllm.v1.kv_cache_interface import KVCacheSpecKind +from vllm.v1.metrics.loggers import StatLoggerBase + +from nemo_rl.data.captured_media import MediaCaptureRejected +from nemo_rl.models.generation.dynamo.config import DynamoCfg +from nemo_rl.models.generation.dynamo.engine_adapter import ( + DynamoEngineAdapter, + RequestHandler, + ServedRollout, +) +from nemo_rl.models.generation.dynamo.vllm.chat_engine import ( + GYM_CALL_ANNOTATION, + PLAN_KEY, + RUNTIME_DATA_KEY, +) +from nemo_rl.models.generation.gym_call import GymCallResult +from nemo_rl.models.generation.openai_server_utils import PrefixSplice +from nemo_rl.models.generation.score_centering import ( + extract_score_centering_samples, +) +from nemo_rl.models.generation.vllm.utils import ( + chat_generation_tokens, + chat_routed_experts, + remap_multimodal_placeholders, +) + +LOGGER = logging.getLogger(__name__) + +if TYPE_CHECKING: + from nemo_rl.models.generation.vllm.vllm_worker_async import ( + VllmAsyncGenerationWorkerImpl, + ) + +# vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them +# (dynamo/vllm/cache_info.py MAIN_ATTENTION_KV_CACHE_KINDS, which NeMo RL may +# not import); test_main_attention_kinds_match_dynamos checks the copy. +_MAIN_ATTENTION_KINDS = ( + KVCacheSpecKind.FULL_ATTENTION, + KVCacheSpecKind.MLA_ATTENTION, + KVCacheSpecKind.SINK_FULL_ATTENTION, +) +# The policy.generation keys the frontend's chat engine renders and checks +# sampling with. +_CHAT_GENERATION_KEYS = ( + "temperature", + "top_p", + "top_k", + "val_temperature", + "val_top_p", + "val_top_k", + "stop_token_ids", +) +_CHAT_VLLM_CFG_KEYS = ( + "http_server_serving_chat_kwargs", + "tool_parser_plugin", + "reasoning_parser_plugin", +) + + +def dynamo_kv_events_config( + cfg: DynamoCfg, *, prefix_caching: bool +) -> Optional[dict[str, Any]]: + """The ``kv_events_config`` that feeds Dynamo's KV router. + + None unless the frontend routes by KV and the engine caches prefixes: vLLM + publishes block events only for the blocks its prefix cache stores. + """ + if cfg.frontend_args.router_mode != "kv" or not prefix_caching: + return None + return { + "enable_kv_cache_events": True, + "publisher": "zmq", + # A random abstract-namespace socket per engine (dynamo-integration.md). + "endpoint": f"ipc://@nemo-rl-kv-events-{os.urandom(8).hex()}", + } + + +def chat_runtime_data(host: "VllmAsyncGenerationWorkerImpl") -> dict[str, Any]: + """What the frontend's chat engine renders with, published at registration. + + The engine's own args (those a JSON payload can carry, which include every + one ``ModelConfig`` reads), so the frontend builds the same tokenizer, + template and processor; and the chat settings and sampling profiles. + """ + engine_args: dict[str, Any] = {} + for field in dataclasses.fields(host.llm_async_engine_args): + value = getattr(host.llm_async_engine_args, field.name) + if field.default is not dataclasses.MISSING: + default = field.default + elif field.default_factory is not dataclasses.MISSING: + default = field.default_factory() + else: + default = dataclasses.MISSING + try: + if value == default: + continue + json.dumps(value) + except (TypeError, ValueError): + continue + engine_args[field.name] = value + # The frontend renders chat calls, so it always loads the tokenizer. + engine_args["skip_tokenizer_init"] = False + cfg = host.cfg + return { + RUNTIME_DATA_KEY: { + "engine_args": engine_args, + "generation": { + **{key: cfg[key] for key in _CHAT_GENERATION_KEYS}, + "vllm_cfg": { + key: cfg["vllm_cfg"][key] + for key in _CHAT_VLLM_CFG_KEYS + if key in cfg["vllm_cfg"] + }, + }, + } + } + + +class SnapshotStatLogger(StatLoggerBase): + """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" + + # Set by Dynamo after vLLM built the logger, so shared through the class. + publisher: Optional[Any] = None + + def __init__(self, vllm_config: Any, engine_index: int = 0) -> None: + self.dp_rank = engine_index + # vLLM sizes the KV cache before it builds its stat loggers. + self.num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0 + + def record(self, scheduler_stats, iteration_stats, *args, **kwargs) -> None: + cls = type(self) + if scheduler_stats is None or cls.publisher is None: + return + total = max(1, self.num_gpu_blocks) + usage = scheduler_stats.kv_cache_usage + hit_rate = None + prefix = scheduler_stats.prefix_cache_stats + if prefix.queries > 0: + hit_rate = prefix.hits / prefix.queries + cls.publisher.publish( + self.dp_rank, + ComponentSnapshot( + kv_used_blocks=int(total * usage), + kv_total_blocks=total, + gpu_cache_usage=usage, + kv_cache_hit_rate=hit_rate, + dp_rank=self.dp_rank, + ), + ) + + def log_engine_initialized(self) -> None: + pass + + +class DynamoVllmAdapter(DynamoEngineAdapter): + """The vLLM backend, over a running NeMo RL vLLM worker it does not own.""" + + def __init__( + self, host: "VllmAsyncGenerationWorkerImpl", *, leader_worker_idx: int + ): + super().__init__(leader_worker_idx=leader_worker_idx) + self._host = host + self._llm = host.llm + self._max_model_len = self._llm.vllm_config.model_config.max_model_len + + def _handlers(self) -> dict[str, RequestHandler]: + return {**super()._handlers(), GYM_CALL_ANNOTATION: self._generate_gym} + + async def serve_rollout( + self, + prompt_token_ids: list[int], + *, + greedy: bool, + stop_strings: Optional[list[str]], + request_id: str, + ) -> ServedRollout: + result, final_output = await self._host._serve_rollout( + TokensPrompt(prompt_token_ids=prompt_token_ids), + prompt_token_ids=prompt_token_ids, + greedy=greedy, + stop_strings=stop_strings, + request_id=request_id, + device=torch.device("cpu"), + ) + if final_output is None: + return ServedRollout(result, finish_reason="length", usage=None) + return ServedRollout( + result, + finish_reason=final_output.outputs[0].finish_reason, + usage=_usage(final_output, len(result.token_ids)), + ) + + async def start(self, worker_id: int) -> EngineConfig: + del worker_id + vllm_config = self._llm.vllm_config + cache = vllm_config.cache_config + scheduler = vllm_config.scheduler_config + # The KV-event block size is the main-attention group's, which can + # differ from cache_config.block_size on hybrid models. + block_size = cache.block_size + for group in await self._llm.engine_core.call_utility_async( + "get_kv_cache_group_metadata" + ): + if group.get("kind") in _MAIN_ATTENTION_KINDS: + block_size = group["block_size"] + break + model = self._host.model_name + return EngineConfig( + model=model, + served_model_name=model, + runtime_data=chat_runtime_data(self._host), + # No token budget: the engine owns overflow (dynamo-integration.md). + llm=LlmRegistration( + context_length=self._max_model_len, + kv_cache_block_size=block_size, + total_kv_blocks=cache.num_gpu_blocks, + max_num_seqs=scheduler.max_num_seqs, + max_num_batched_tokens=scheduler.max_num_batched_tokens, + ), + ) + + async def _generate_gym( + self, request: GenerateRequest, annotation: str, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + del annotation + plan = (request.get("extra_args") or {}).get(PLAN_KEY) + if plan is None: + raise InvalidArgument( + "NeMo-Gym calls reach NeMo RL's engines through NeMo RL's chat " + f"engine in the Dynamo frontend (extra_args.{PLAN_KEY})." + ) + try: + chunk = await self._serve_gym_call( + plan, list(request["token_ids"]), context + ) + except VLLMClientError as error: + # vLLM's request errors are not RuntimeErrors; the native server + # answers them with a 4xx that carries the message. + raise InvalidArgument(str(error)) from error + except MediaCaptureRejected as error: + raise InvalidArgument(f"{error} ({error.code})") from error + if chunk is not None: + yield chunk + + async def _serve_gym_call( + self, plan: dict[str, Any], prompt_token_ids: list[int], context: Context + ) -> Optional[GenerateChunk]: + """One NeMo-Gym chat call, after the frontend rendered and spliced it. + + The native server's steps after the render, in its order: the media + snapshot, ``begin_call`` (which stamps the weight version), the + generation, and the one TQ write. Returns one chunk whose + ``engine_data`` carries the output the frontend builds the reply from + and the call's :class:`GymCallResult`; None for a captured call whose + client went away. + """ + # Deferred: chat_server imports vLLM's OpenAI server stack. + from nemo_rl.models.generation.vllm.chat_server import ( + NeMoRLChatCompletionRequest, + ) + + host = self._host + # The reply never echoes messages; the frontend sends the request + # without them. + chat = NeMoRLChatCompletionRequest.model_validate( + {**plan["request"], "messages": [{"role": "user", "content": ""}]} + ) + admission = host._parse_capture_admission(plan["admission"]) + serving_admission = host._parse_capture_admission(plan["serving_admission"]) + if admission is not None and admission.mode == "candidate": + # C26: the frontend's chat engine decides a candidate; a request + # still marked one skipped that decision. + raise InvalidArgument( + "a candidate admission reached the engine undecided; NeMo RL's " + "chat engine must resolve it before capture" + ) + splice = None + if plan["splice"] is not None: + splice = PrefixSplice(prompt_token_ids, *plan["splice"]) + engine_prompt: dict[str, Any] + if plan["multimodal"] is not None: + # The processed media, in the template's coordinates. + engine_prompt = { + **pickle.loads(base64.b64decode(plan["multimodal"])), + "prompt_token_ids": list(plan["template_token_ids"]), + } + else: + engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) + media = None + if admission is not None: + # Off-loop: a retained-media check reads staged calls. + media = await asyncio.to_thread( + host._capture_request_media, + engine_prompt, + admission=serving_admission if splice is not None else admission, + splice=splice, + verify_retained_media=plan["verify_retained_media"], + ) + if splice is not None and plan["multimodal"] is not None: + # Successful media capture already remapped these ranges. + if media is None and (placeholders := engine_prompt.get("mm_placeholders")): + engine_prompt["mm_placeholders"] = remap_multimodal_placeholders( + template_token_ids=engine_prompt["prompt_token_ids"], + final_token_ids=prompt_token_ids, + mm_placeholders=placeholders, + splice=splice, + ) + engine_prompt["prompt_token_ids"] = prompt_token_ids + prefix_token_ids = None + if admission is not None and admission.mode == "token_in": + prefix_token_ids = prompt_token_ids[: plan["prefix_len"]] + host._begin_request_capture( + chat, + prompt_token_ids, + admission=admission, + prefix_token_ids=prefix_token_ids, + media=media, + ) + captured = id(chat) in host._capture_calls + try: + final_output = await self._generate(chat, engine_prompt, plan, context) + if admission is not None and context.is_stopped(): + # Dynamo aborts the vLLM request on a disconnect but never + # cancels this generator: fail the capture (below), as the + # native endpoint's cancellation does, instead of staging it. + return None + completion = final_output.outputs[0] + if completion.finish_reason == "error": + # As vLLM's chat server does (_raise_if_error): never stage a failed call. + raise RuntimeError(f"Generation failed for request {context.id()}") + output: dict[str, Any] = { + "text": completion.text, + "token_ids": list(completion.token_ids), + "finish_reason": completion.finish_reason, + "stop_reason": completion.stop_reason, + } + if captured: + result = await self._stage(chat, final_output) + else: + result = self._call_result(final_output) + if chat.logprobs: + output["logprobs"] = _wire_logprobs(completion.logprobs) + return { + "token_ids": output["token_ids"], + "index": 0, + "finish_reason": completion.finish_reason, + "completion_usage": _usage(final_output, len(completion.token_ids)), + "engine_data": { + "output": output, + "result": result.model_dump(exclude_none=True), + }, + } + finally: + # A no-op once the call was staged: staging pops its capture state. + if admission is not None: + host._abort_request_capture( + chat, + reason=( + "client_disconnect" if context.is_stopped() else "engine_error" + ), + ) + + async def _generate( + self, chat: Any, engine_prompt: Any, plan: dict[str, Any], context: Context + ) -> Any: + """Generate as the native server does, with the SamplingParams it builds.""" + model_config = self._llm.model_config + default_sampling_params = model_config.get_diff_sampling_param() + override_max_tokens = ( + default_sampling_params.get("max_tokens") + if model_config.generation_config not in ("auto", "vllm") + else getattr(model_config, "override_generation_config", {}).get( + "max_new_tokens" + ) + ) + max_tokens = get_max_tokens( + self._max_model_len, + chat.max_completion_tokens + if chat.max_completion_tokens is not None + else chat.max_tokens, + len(engine_prompt["prompt_token_ids"]), + default_sampling_params, + override_max_tokens, + truncate_prompt_tokens=chat.truncate_prompt_tokens, + ) + params = chat.to_sampling_params(max_tokens, default_sampling_params) + # The worker's own engine client, as the native server uses it, with + # the queue limits it checks before generating. + engine_client = self._host._http_engine_client or self._llm + engine_client.check_admission(1, context.id()) + final_output = None + async for output in engine_client.generate( + engine_prompt, + params, + context.id(), + reasoning_ended=plan["reasoning_ended"], + reasoning_parser_kwargs=plan["reasoning_parser_kwargs"], + ): + final_output = output + if final_output is None: + raise RuntimeError(f"No output received for request {context.id()}") + return final_output + + def _routes(self, final_output: Any, *, encode_for_wire: bool) -> Any: + """The call's routes, as the native endpoint attaches them; None if off. + + With router replay on, a call vLLM returned no routes for fails + (``chat_routed_experts`` raises), captured or not. + """ + if not self._host._return_routed_experts_enabled(): + return None + completion = final_output.outputs[0] + return chat_routed_experts( + final_output, + completion, + prompt_token_count=len(final_output.prompt_token_ids), + device=torch.device("cpu"), + logger=LOGGER, + routed_experts_dtype=self._host.routed_experts_dtype, + encode_for_wire=encode_for_wire, + ) + + def _call_result(self, final_output: Any) -> GymCallResult: + """An uncaptured call's tokens and routes, as the native message carries them.""" + completion = final_output.outputs[0] + generation_log_probs = None + if completion.logprobs is not None: + _, generation_log_probs = chat_generation_tokens(completion) + return GymCallResult( + prompt_token_ids=list(final_output.prompt_token_ids), + generation_token_ids=list(completion.token_ids), + generation_log_probs=generation_log_probs, + routed_experts=self._routes(final_output, encode_for_wire=True), + ) + + async def _stage(self, chat: Any, final_output: Any) -> GymCallResult: + """Stage a captured call from this process, in one TQ write.""" + # Deferred: only captured calls reach here, and capture already imports nemo_gym. + from nemo_gym.token_id_capture import NG_COMMIT_COORDS_FIELD + + completion = final_output.outputs[0] + message: dict[str, Any] = {"generation_token_ids": list(completion.token_ids)} + if completion.logprobs is not None: + _, message["generation_log_probs"] = chat_generation_tokens(completion) + routes = self._routes(final_output, encode_for_wire=False) + if routes is not None: + message["routed_experts"] = routes + if self._host._capture_top_logprobs: + # C29: score centering stages the sampler's top-k with the call. + ( + message["sampler_topk_token_ids"], + message["sampler_topk_logprobs"], + ) = extract_score_centering_samples( + completion, num_candidates=self._host._capture_top_logprobs + ) + content = { + "choices": [ + { + "index": 0, + "finish_reason": completion.finish_reason, + "message": message, + } + ] + } + # The sink write is a blocking data-plane round trip. + content = await asyncio.to_thread( + self._host._finish_request_capture, chat, content + ) + return GymCallResult(ng_commit_coords=content[NG_COMMIT_COORDS_FIELD]) + + async def abort(self, context: Context) -> None: + await self._llm.abort(context.id()) + + async def kv_event_sources(self) -> list[KvEventSource]: + kv_events = self._llm.vllm_config.kv_events_config + if kv_events is None or not kv_events.enable_kv_cache_events: + return [] + # A user's own kv_events_config can keep vLLM's wildcard default + # (tcp://*:5557): vLLM binds it, but a subscriber cannot connect to "*". + return [ZmqSource(endpoint=kv_events.endpoint.replace("*", "127.0.0.1"))] + + def component_metrics_dp_ranks(self) -> list[int]: + return [0] + + def attach_snapshot_publisher(self, publisher: Any) -> None: + SnapshotStatLogger.publisher = publisher + + async def cleanup(self) -> None: + # The worker owns the AsyncLLM; its shutdown() tears the engine down. + SnapshotStatLogger.publisher = None + + +def _wire_logprobs(logprobs: Any) -> Optional[list[dict[str, list[Any]]]]: + """Per-position logprobs as JSON, for the frontend's OpenAI ``logprobs``.""" + if logprobs is None: + return None + return [ + { + str(token_id): [entry.logprob, entry.rank, entry.decoded_token] + for token_id, entry in position.items() + } + for position in logprobs + ] + + +def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: + prompt_tokens = len(output.prompt_token_ids or ()) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": num_output_tokens, + "total_tokens": prompt_tokens + num_output_tokens, + "prompt_tokens_details": {"cached_tokens": output.num_cached_tokens or 0}, + } diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py new file mode 100644 index 00000000000..f0a59a2d259 --- /dev/null +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -0,0 +1,406 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NeMo RL's chat engine inside Dynamo's frontend, for NeMo-Gym's calls. + +Dynamo's frontend hands every chat call to this engine (``chat_engine_factory``) +instead of its Rust preprocessor, so NeMo-Gym's calls are rendered, spliced and +parsed by the same code as NeMo RL's native vLLM server: + +1. the call's :class:`~nemo_rl.models.generation.gym_call.GymCallExtras` and + on-policy sampling are checked; +2. a render-pool worker renders it (:mod:`render_pool`); +3. here, the token-capture prefix is read from TQ and spliced in, and the + output budget is clamped to the context; +4. the final token ids go through Dynamo's KV router to one engine, whose + adapter runs the capture and generation (``adapter.DynamoVllmAdapter``); +5. a render-pool worker builds the reply with vLLM's own parsers. + +The engine publishes its engine args and chat settings at registration +(``EngineConfig.runtime_data``), so this engine renders exactly as it would. +""" + +from __future__ import annotations + +import asyncio +import base64 +import os +import pickle +import time +import uuid +from typing import Any, AsyncGenerator, Optional + +from dynamo.llm import PythonAsyncEngine +from dynamo.llm.exceptions import HttpError + +from nemo_rl.models.generation.dynamo.managed_runtime import ( + CAPTURE_FILE_ENV, + RAY_ADDRESS_ENV, + RENDER_WORKERS_ENV, +) +from nemo_rl.models.generation.dynamo.vllm.render_pool import ( + RenderedCall, + RenderPool, +) +from nemo_rl.models.generation.gym_call import ( + GYM_RESULT_NVEXT_KEY, + GymCallExtras, + GymCallResult, +) +from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens + +# The key the engine publishes its chat settings under in runtime_data. +RUNTIME_DATA_KEY = "nemo_rl_chat" +# The annotation the adapter serves NeMo-Gym calls by; the capture plan rides +# in extra_args under PLAN_KEY. +GYM_CALL_ANNOTATION = "nemo_gym:rollout" +PLAN_KEY = "nemo_rl" + +# Fields of Dynamo's chat request that are not vLLM request fields +# (chat_template_args is renamed to vLLM's chat_template_kwargs). +_DYNAMO_ONLY_FIELDS = {"nvext", "chat_template_args", "unsupported_fields"} + + +class _ContextLengthError(ValueError): + """The prompt leaves no room for output; Gym reads "context length" in a 400.""" + + +def _drop_none(value: Any) -> Any: + """Dynamo hands Python unset fields as None, which vLLM's models reject.""" + if isinstance(value, dict): + return {k: _drop_none(v) for k, v in value.items() if v is not None} + if isinstance(value, list): + return [_drop_none(v) for v in value] + return value + + +def _clamp( + request_max_tokens: Optional[int], prompt_token_ids: list[int], max_model_len: int +) -> Optional[int]: + """The output budget, as the native server's _clamp_max_tokens sets it.""" + remaining = max_model_len - len(prompt_token_ids) + if remaining <= 0: + # Gym reads "context length" in the 400 as an overflow. + raise _ContextLengthError( + f"Prompt length ({len(prompt_token_ids)}) fills or exceeds this " + f"model's maximum context length ({max_model_len}). No room " + "for output tokens." + ) + if request_max_tokens is None: + return None + return min(request_max_tokens, remaining) + + +class _CaptureSource: + """The read-only TQ client the token-capture prefix is read with. + + Built on the first captured call, from what ``setup_token_capture`` wrote + for the frontend: capture starts after the engines register. + """ + + def __init__(self) -> None: + self._prefix_cache: Any = None + + def resolve_prefix(self, serving_admission: dict[str, Any]) -> list[int]: + """The admission's prefix, read from TQ (blocking; run off the loop).""" + from nemo_gym.token_id_capture.staging.records import CaptureAdmission + + from nemo_rl.data_plane.tq_token_sink import resolve_admission_prefix + + if self._prefix_cache is None: + self._prefix_cache = self._connect() + return resolve_admission_prefix( + CaptureAdmission.model_validate(serving_admission), self._prefix_cache + ) + + @staticmethod + def _connect() -> Any: + import ray + + from nemo_rl.data_plane import build_data_plane_client + from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache, TQTokenSource + + path = os.environ[CAPTURE_FILE_ENV] + if not os.path.exists(path): + raise RuntimeError( + "a token-capture call reached the Dynamo frontend before token " + "capture was set up for it" + ) + with open(path, "rb") as capture_file: + capture = pickle.load(capture_file) + if not ray.is_initialized(): + ray.init(address=os.environ[RAY_ADDRESS_ENV], log_to_driver=False) + client = build_data_plane_client( + capture["data_plane"], bootstrap=False, segment_size=0 + ) + cache = ChainPrefixCache() + cache.install( + TQTokenSource( + client, + staging_partition=capture["staging_partition"], + capture_media=capture["capture_media"], + ) + ) + return cache + + +class NemoRLChatEngine: + """Serves NeMo-Gym's chat calls for one model's engines.""" + + def __init__( + self, + *, + pool: RenderPool, + routed_engine: Any, + spec: dict[str, Any], + capture: _CaptureSource, + ): + self._pool = pool + self._routed_engine = routed_engine + self._generation = spec["generation"] + self._capture = capture + + async def generate( + self, request: dict[str, Any], context: Any = None + ) -> AsyncGenerator[dict[str, Any], None]: + from vllm.exceptions import VLLMValidationError + + from nemo_rl.data.captured_media import MediaCaptureRejected + from nemo_rl.models.generation.vllm.utils import OffPolicySamplingError + + try: + yield await self._serve(request, context) + except ( + OffPolicySamplingError, + _ContextLengthError, + MediaCaptureRejected, + VLLMValidationError, + ) as error: + raise HttpError(400, str(error)) from error + + async def _serve(self, request: dict[str, Any], context: Any) -> dict[str, Any]: + from nemo_rl.models.generation.vllm.utils import on_policy_sampling + + extras = GymCallExtras.from_annotations( + (request.get("nvext") or {}).get("annotations") or () + ) + if extras is None: + raise HttpError( + 400, + "NeMo RL's Dynamo frontend serves NeMo-Gym's calls (the " + "nemo_rl:gym: annotation) on chat; NeMo RL's own rollouts use " + "/v1/completions.", + ) + body = _drop_none( + {k: v for k, v in request.items() if k not in _DYNAMO_ONLY_FIELDS} + ) + if request.get("chat_template_args"): + body["chat_template_kwargs"] = request["chat_template_args"] + if (body.get("n") or 1) != 1: + raise HttpError(400, "NeMo-Gym calls take n=1.") + body.update( + on_policy_sampling( + temperature=body.get("temperature"), + top_p=body.get("top_p"), + top_k=body.get("top_k"), + min_p=body.get("min_p"), + repetition_penalty=body.get("repetition_penalty"), + presence_penalty=body.get("presence_penalty"), + frequency_penalty=body.get("frequency_penalty"), + logit_bias=body.get("logit_bias"), + generation_config=self._generation, + ) + ) + body["ng_capture"] = extras.ng_capture + body["required_prefix_token_ids"] = extras.required_prefix_token_ids + info = await self._pool.info() + rendered = await self._pool.render(body) + prompt_token_ids, splice, prefix = await self._splice( + rendered, info["turn_end_token_ids"] + ) + if extras.ng_measure: + return self._chunk( + request, + GymCallResult( + prompt_token_count=len(prompt_token_ids), ng_context_measured=True + ), + message={"role": "assistant", "content": ""}, + finish_reason="stop", + ) + max_tokens = _clamp( + rendered.request_max_tokens, prompt_token_ids, info["max_model_len"] + ) + engine_request = dict(rendered.request) + if max_tokens is not None: + engine_request["max_tokens"] = max_tokens + engine_request["max_completion_tokens"] = None + plan = { + "request": engine_request, + "admission": rendered.admission, + "serving_admission": rendered.serving_admission, + "verify_retained_media": rendered.verify_retained_media, + # The model's own prefix is the spliced prompt's first ids. + "prefix_len": None if prefix is None else len(prefix), + "template_token_ids": rendered.template_token_ids, + "splice": None + if splice is None + else [splice.model_cut_end, splice.template_cut_start], + "multimodal": None + if rendered.multimodal is None + else base64.b64encode(rendered.multimodal).decode(), + "reasoning_ended": rendered.reasoning_ended, + "reasoning_parser_kwargs": rendered.reasoning_parser_kwargs, + } + preprocessed = { + "model": request["model"], + "token_ids": prompt_token_ids, + "stop_conditions": {"max_tokens": max_tokens}, + "sampling_options": { + "temperature": body["temperature"], + "top_p": body["top_p"], + }, + "output_options": {}, + "eos_token_ids": [], + "annotations": [GYM_CALL_ANNOTATION], + "extra_args": {PLAN_KEY: plan}, + } + engine_data = await self._call_engine(preprocessed, context) + result = GymCallResult.model_validate(engine_data["result"]) + reply_request = engine_request + if result.ng_commit_coords is not None: + # A captured call's logprobs were staged with it; as natively, its + # reply carries none. + reply_request = {**engine_request, "logprobs": False, "top_logprobs": None} + response = await self._pool.respond( + reply_request, prompt_token_ids, engine_data["output"] + ) + (choice,) = response["choices"] + return self._chunk( + request, + result, + message=choice["message"], + finish_reason=choice["finish_reason"], + logprobs=choice.get("logprobs"), + usage=response.get("usage"), + ) + + async def _splice( + self, rendered: RenderedCall, turn_end_token_ids: list[int] + ) -> tuple[list[int], Any, Any]: + """The prompt the engine runs: the model's own prefix, then the template's.""" + prefix: Optional[list[int]] = None + serving = rendered.serving_admission + if serving is not None and serving["mode"] == "token_in": + # A blocking data-plane read; keep it off the frontend's loop. + prefix = await asyncio.to_thread(self._capture.resolve_prefix, serving) + elif rendered.required_prefix_token_ids is not None: + prefix = rendered.required_prefix_token_ids + if prefix is None: + return rendered.template_token_ids, None, None + splice = splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=prefix, + template_prefix_token_ids=rendered.history_token_ids, + template_token_ids=rendered.template_token_ids, + eos_token_id=turn_end_token_ids, + ) + return splice.token_ids, splice, prefix + + async def _call_engine( + self, preprocessed: dict[str, Any], context: Any + ) -> dict[str, Any]: + engine_data = None + stream = await self._routed_engine.generate(preprocessed, context=context) + async for item in stream: + if item.is_error(): + comments = item.comments() or [] + raise HttpError(500, "; ".join(comments) or "engine error") + data = item.data() + if data and data.get("engine_data"): + engine_data = data["engine_data"] + if engine_data is None: + raise HttpError(500, "the engine returned no result for this call") + return engine_data + + @staticmethod + def _chunk( + request: dict[str, Any], + result: GymCallResult, + *, + message: dict[str, Any], + finish_reason: str, + logprobs: Optional[dict[str, Any]] = None, + usage: Optional[dict[str, Any]] = None, + ) -> dict[str, Any]: + """The OpenAI chunk Dynamo's frontend aggregates into the reply.""" + delta: dict[str, Any] = { + "role": "assistant", + "content": message.get("content"), + } + # vLLM names it "reasoning"; Dynamo's OpenAI types "reasoning_content". + reasoning = message.get("reasoning", message.get("reasoning_content")) + if reasoning is not None: + delta["reasoning_content"] = reasoning + if message.get("tool_calls"): + delta["tool_calls"] = [ + {**tool_call, "index": index} + for index, tool_call in enumerate(message["tool_calls"]) + ] + chunk: dict[str, Any] = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": request["model"], + "choices": [ + { + "index": 0, + "delta": delta, + "finish_reason": finish_reason, + "logprobs": logprobs, + } + ], + "nvext": {GYM_RESULT_NVEXT_KEY: result.model_dump(exclude_none=True)}, + } + if usage is not None: + chunk["usage"] = usage + return chunk + + +class ChatEngineFactory: + """Dynamo's ``chat_engine_factory``: one chat engine per model's engines.""" + + def __init__(self) -> None: + self._capture = _CaptureSource() + self._pools: list[RenderPool] = [] + + async def chat_engine_factory( + self, instance_id: Any, mdc: Any, routed_engine: Any + ) -> PythonAsyncEngine: + del instance_id + runtime_data = (mdc.runtime_config() or {}).get("runtime_data") or {} + spec = runtime_data.get(RUNTIME_DATA_KEY) + if spec is None: + raise RuntimeError( + f"model {mdc.name()!r} registered without NeMo RL's chat settings " + f"(runtime_data[{RUNTIME_DATA_KEY!r}]); only NeMo RL's engines are " + "served by this frontend" + ) + pool = RenderPool( + spec, mdc.local_dir(), workers=int(os.environ[RENDER_WORKERS_ENV]) + ) + self._pools.append(pool) + engine = NemoRLChatEngine( + pool=pool, routed_engine=routed_engine, spec=spec, capture=self._capture + ) + return PythonAsyncEngine(engine.generate, asyncio.get_running_loop()) diff --git a/nemo_rl/models/generation/dynamo/vllm/frontend.py b/nemo_rl/models/generation/dynamo/vllm/frontend.py new file mode 100644 index 00000000000..4f9ba822ba9 --- /dev/null +++ b/nemo_rl/models/generation/dynamo/vllm/frontend.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run Dynamo's frontend with NeMo RL's chat engine serving chat calls. + +``python -m nemo_rl.models.generation.dynamo.vllm.frontend `` + +The arguments are ``dynamo.frontend``'s own. Its ``main`` builds one +``EntrypointArgs``; this wraps that name to add NeMo RL's +``chat_engine_factory``, so every chat call reaches +:mod:`~nemo_rl.models.generation.dynamo.vllm.chat_engine` and never Dynamo's +Rust preprocessor. ``/v1/completions`` (NeMo RL's own rollouts, token ids in) +is unchanged. +""" + +import sys + + +def main() -> None: + import dynamo.frontend.main as frontend_main + + from nemo_rl.models.generation.dynamo.vllm.chat_engine import ChatEngineFactory + + factory = ChatEngineFactory() + entrypoint_args = frontend_main.EntrypointArgs + + def with_chat_engine(engine_type, **kwargs): + if "chat_engine_factory" in kwargs: + raise RuntimeError( + "NeMo RL's frontend serves chat with its own chat engine; remove " + "--dyn-chat-processor from dynamo_cfg.frontend_args.extra_cli_args" + ) + return entrypoint_args( + engine_type, chat_engine_factory=factory.chat_engine_factory, **kwargs + ) + + frontend_main.EntrypointArgs = with_chat_engine + sys.argv[0] = "dynamo.frontend" + frontend_main.main() + + +if __name__ == "__main__": + main() diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py new file mode 100644 index 00000000000..ab08bde6808 --- /dev/null +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -0,0 +1,381 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render and answer NeMo-Gym chat calls in worker processes of Dynamo's frontend. + +Rendering (template, tokenizer, the checkpoint's HF processor) and building the +reply (detokenized text through vLLM's reasoning and tool parsers) are +CPU-bound Python; on the frontend's one event loop they would serialize every +call in flight. Each worker process builds the same vLLM objects NeMo RL's +native chat server uses, from the engine's own engine args, and runs: + +- ``render``: the native server's render step (``_render_chat_call``: the + render, the capture decision, the history render the splice cuts against); +- ``respond``: vLLM's own non-streaming response builder over the engine's + output, so the reply is the one the native server builds. + +Jobs queue FIFO in the pool; nothing renders on the event loop. +""" + +from __future__ import annotations + +import asyncio +import multiprocessing +import pickle +import uuid +from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass +from typing import Any, Optional + +# Set in each worker process by _init_worker. +_WORKER: Optional["_Worker"] = None + + +@dataclass(frozen=True) +class RenderedCall: + """What the frontend needs from a render to splice, route and serve a call.""" + + template_token_ids: list[int] + # The render up to the last assistant turn, when a prefix will be spliced. + history_token_ids: Optional[list[int]] + # The request's own prefix (Gym's echo) after the capture decision. + required_prefix_token_ids: Optional[list[int]] + # Capture admissions as JSON: what is stored, and how the prompt is served. + admission: Optional[dict[str, Any]] + serving_admission: Optional[dict[str, Any]] + verify_retained_media: bool + # The rendered engine input's multimodal part (pickled), or None for text. + multimodal: Optional[bytes] + # The chat request as rendered (parsers may adjust it), without messages: + # the engine builds its SamplingParams from it, as the native server does. + request: dict[str, Any] + # The output-token budget the request asked for, before the clamp. + request_max_tokens: Optional[int] + # What vLLM's chat serving hands the engine besides the prompt. + reasoning_ended: Optional[bool] + reasoning_parser_kwargs: Optional[dict[str, Any]] + + +class _Worker: + """The vLLM objects one worker process renders and responds with.""" + + def __init__(self, spec: dict[str, Any], model_dir: str): + import vllm.platforms + + # A frontend host may have no GPU; vLLM only parses here, as in + # Dynamo's own frontend (dynamo.frontend.main.parse_args). + if vllm.platforms.current_platform.device_type == "": + from vllm.platforms.cpu import CpuPlatform + + vllm.platforms.current_platform = CpuPlatform() + + from types import SimpleNamespace + + from vllm.config import CacheConfig, LoadConfig, VllmConfig + from vllm.engine.arg_utils import AsyncEngineArgs + from vllm.entrypoints.openai.chat_completion.serving import ( + OpenAIServingChat, + ) + from vllm.entrypoints.openai.models.protocol import BaseModelPath + from vllm.entrypoints.openai.models.serving import OpenAIServingModels + from vllm.renderers.registry import renderer_from_config + from vllm.v1.engine.input_processor import InputProcessor + + from nemo_rl.models.generation.openai_server_utils import ( + model_stop_token_ids, + turn_end_token_ids, + ) + from nemo_rl.models.generation.vllm import chat_server + + cfg = spec["generation"] + chat_server.register_parser_plugins(cfg) + # The engine's own args, with the weight-free snapshot Dynamo fetched. + args = AsyncEngineArgs( + **{**spec["engine_args"], "model": model_dir, "tokenizer": model_dir} + ) + self.model_config = args.create_model_config() + vllm_config = VllmConfig( + model_config=self.model_config, + load_config=LoadConfig(load_format="dummy"), + cache_config=CacheConfig(), + ) + renderer = renderer_from_config(vllm_config) + self.tokenizer = renderer.tokenizer + serving_chat_kwargs, default_chat_template_kwargs = ( + chat_server.serving_chat_settings(cfg) + ) + self.online_renderer = chat_server.FrontendRenderer( + model_config=self.model_config, + renderer=renderer, + request_logger=None, + chat_template=serving_chat_kwargs["chat_template"], + chat_template_content_format=serving_chat_kwargs[ + "chat_template_content_format" + ], + enable_auto_tools=serving_chat_kwargs["enable_auto_tools"], + tool_parser=serving_chat_kwargs.get("tool_parser"), + reasoning_parser=serving_chat_kwargs.get("reasoning_parser"), + default_chat_template_kwargs=default_chat_template_kwargs, + ) + # vLLM's chat serving builds replies only here; it never generates. + engine_client = SimpleNamespace( + model_config=self.model_config, + renderer=renderer, + vllm_config=vllm_config, + input_processor=InputProcessor(vllm_config), + io_processor=None, + errored=False, + ) + models = OpenAIServingModels( + engine_client=engine_client, + base_model_paths=[ + BaseModelPath( + name=self.model_config.served_model_name, model_path=model_dir + ) + ], + lora_modules=None, + ) + self.serving_chat = OpenAIServingChat( + **{ + **serving_chat_kwargs, + "engine_client": engine_client, + "models": models, + "online_renderer": self.online_renderer, + "return_tokens_as_token_ids": True, + } + ) + self.loop = asyncio.new_event_loop() + # Every id a turn can end on: the splice in the frontend cuts by them. + self.turn_end_token_ids = sorted( + turn_end_token_ids( + model_stop_token_ids(self.model_config, cfg), self.tokenizer + ) + ) + + def _parser(self, chat: Any) -> Any: + parser_cls = self.serving_chat.parser_cls + if parser_cls is None: + return None + return parser_cls( + self.tokenizer, + chat.tools, + chat_template_kwargs=self.serving_chat._effective_chat_template_kwargs( + chat + ), + model_config=self.model_config, + ) + + def render(self, body: dict[str, Any]) -> RenderedCall: + from nemo_gym.token_id_capture.staging.records import CaptureAdmission + + from nemo_rl.models.generation.vllm.chat_server import ( + NeMoRLChatCompletionRequest, + set_max_tokens, + ) + + chat = NeMoRLChatCompletionRequest.model_validate(body) + request_max_tokens = ( + chat.max_completion_tokens + if chat.max_completion_tokens is not None + else chat.max_tokens + ) + admission = ( + None + if chat.ng_capture is None + else CaptureAdmission.model_validate(chat.ng_capture) + ) + rendered = self.loop.run_until_complete( + self.online_renderer.render_for_frontend(chat, admission=admission) + ) + # The serving reads these back from the request; restore the budget + # the render lowered for vLLM's pre-tokenization length check. + if request_max_tokens is not None: + set_max_tokens(chat, request_max_tokens) + parser = self._parser(chat) + reasoning_ended = None + reasoning_parser_kwargs = None + if not chat.include_reasoning or chat._grammar_from_parser: + reasoning_ended = True + elif parser is not None and parser.reasoning_parser is not None: + reasoning_ended = parser.is_reasoning_end( + rendered.engine_prompt["prompt_token_ids"] + ) + if parser is not None and parser.reasoning_parser is not None: + reasoning_parser_kwargs = { + "chat_template_kwargs": self.serving_chat._engine_chat_template_kwargs( + self.serving_chat._effective_chat_template_kwargs(chat) + ) + } + multimodal = None + if rendered.engine_prompt.get("mm_kwargs"): + multimodal = pickle.dumps( + { + key: value + for key, value in rendered.engine_prompt.items() + if key != "prompt_token_ids" + }, + protocol=pickle.HIGHEST_PROTOCOL, + ) + return RenderedCall( + template_token_ids=list(rendered.engine_prompt["prompt_token_ids"]), + history_token_ids=rendered.history_token_ids, + required_prefix_token_ids=chat.required_prefix_token_ids, + admission=_admission_json(rendered.admission), + serving_admission=_admission_json(rendered.serving_admission), + verify_retained_media=rendered.verify_retained_media, + multimodal=multimodal, + request=chat.model_dump(mode="json", exclude={"messages"}), + request_max_tokens=request_max_tokens, + reasoning_ended=reasoning_ended, + reasoning_parser_kwargs=reasoning_parser_kwargs, + ) + + def respond( + self, + request: dict[str, Any], + prompt_token_ids: list[int], + output: dict[str, Any], + ) -> dict[str, Any]: + from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionResponse, + ) + from vllm.logprobs import Logprob + from vllm.outputs import CompletionOutput, RequestOutput + + from nemo_rl.models.generation.vllm.chat_server import ( + NeMoRLChatCompletionRequest, + ) + + # The reply never echoes messages, so the request's are not sent back. + chat = NeMoRLChatCompletionRequest.model_validate( + {**request, "messages": [{"role": "user", "content": ""}]} + ) + logprobs = None + if output.get("logprobs") is not None: + logprobs = [ + { + int(token_id): Logprob( + logprob=entry[0], rank=entry[1], decoded_token=entry[2] + ) + for token_id, entry in position.items() + } + for position in output["logprobs"] + ] + request_output = RequestOutput( + request_id=f"chatcmpl-{uuid.uuid4().hex}", + prompt=None, + prompt_token_ids=prompt_token_ids, + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text=output["text"], + token_ids=output["token_ids"], + cumulative_logprob=None, + logprobs=logprobs, + finish_reason=output["finish_reason"], + stop_reason=output.get("stop_reason"), + ) + ], + finished=True, + ) + + async def results(): + yield request_output + + response = self.loop.run_until_complete( + self.serving_chat.chat_completion_full_generator( + chat, + results(), + request_output.request_id, + request["model"], + [], + self.tokenizer, + RequestResponseMetadata(request_id=request_output.request_id), + parser=self._parser(chat), + ) + ) + if not isinstance(response, ChatCompletionResponse): + raise ValueError(f"vLLM could not build the reply: {response}") + return response.model_dump(mode="json", exclude_none=True) + + +def _admission_json(admission: Any) -> Optional[dict[str, Any]]: + return None if admission is None else admission.model_dump(mode="json") + + +def _init_worker(spec: dict[str, Any], model_dir: str) -> None: + global _WORKER + _WORKER = _Worker(spec, model_dir) + + +def _render(body: dict[str, Any]) -> RenderedCall: + assert _WORKER is not None + return _WORKER.render(body) + + +def _respond( + request: dict[str, Any], prompt_token_ids: list[int], output: dict[str, Any] +) -> dict[str, Any]: + assert _WORKER is not None + return _WORKER.respond(request, prompt_token_ids, output) + + +def _info() -> dict[str, Any]: + assert _WORKER is not None + return { + "turn_end_token_ids": _WORKER.turn_end_token_ids, + "max_model_len": _WORKER.model_config.max_model_len, + } + + +class RenderPool: + """``workers`` processes that render and answer chat calls, FIFO.""" + + def __init__(self, spec: dict[str, Any], model_dir: str, *, workers: int): + self._executor = ProcessPoolExecutor( + max_workers=workers, + # Fork would copy the frontend's Rust runtime and event loop. + mp_context=multiprocessing.get_context("spawn"), + initializer=_init_worker, + initargs=(spec, model_dir), + ) + self._info: Optional[dict[str, Any]] = None + + async def info(self) -> dict[str, Any]: + """The model's turn-end ids and context length, from a started worker. + + Workers start on first use, so a run that sends no chat call never + builds them. + """ + if self._info is None: + self._info = await asyncio.wrap_future(self._executor.submit(_info)) + return self._info + + async def render(self, body: dict[str, Any]) -> RenderedCall: + return await asyncio.wrap_future(self._executor.submit(_render, body)) + + async def respond( + self, + request: dict[str, Any], + prompt_token_ids: list[int], + output: dict[str, Any], + ) -> dict[str, Any]: + return await asyncio.wrap_future( + self._executor.submit(_respond, request, prompt_token_ids, output) + ) + + def shutdown(self) -> None: + self._executor.shutdown(wait=False, cancel_futures=True) diff --git a/nemo_rl/models/generation/dynamo/vllm_adapter.py b/nemo_rl/models/generation/dynamo/vllm_adapter.py deleted file mode 100644 index f3a82cd500a..00000000000 --- a/nemo_rl/models/generation/dynamo/vllm_adapter.py +++ /dev/null @@ -1,817 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Serve a NeMo RL vLLM worker's own AsyncLLM through Dynamo's backend SDK. - -The engine process holds no Dynamo-owned vLLM code. ``DynamoVllmAdapter`` is the -vLLM backend of ``engine_adapter.DynamoEngineAdapter``, over the AsyncLLM that -``VllmAsyncGenerationWorkerImpl`` owns, so refit, pause, sleep, quantization and -speculative decoding are that worker's own. -""" - -from __future__ import annotations - -import asyncio -import functools -import hashlib -import json -import logging -import os -import tempfile -import time -from collections.abc import AsyncGenerator -from typing import TYPE_CHECKING, Any, Optional - -import numpy as np -import torch -from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names -from dynamo.common.backend import ( - EngineConfig, - GenerateChunk, - GenerateRequest, - LlmRegistration, -) -from dynamo.common.backend.logprobs import ( - extract_from_completion_output, - parse_logprob_options, -) -from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource -from dynamo.llm.exceptions import HttpError, InvalidArgument -from vllm import envs as vllm_envs -from vllm.entrypoints.chat_utils import load_chat_template -from vllm.exceptions import GracefulHTTPError, VLLMClientError -from vllm.inputs import TokensPrompt -from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY -from vllm.sampling_params import ( - RequestOutputKind, - SamplingParams, - StructuredOutputsParams, -) -from vllm.transformers_utils.processor import get_video_processor_cls_name -from vllm.v1.kv_cache_interface import KVCacheSpecKind -from vllm.v1.metrics.loggers import StatLoggerBase - -from nemo_rl.data.captured_media import MediaCaptureRejected -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - dynamo_cfg, - dynamo_chat_template, - dynamo_parsers, -) -from nemo_rl.models.generation.dynamo.engine_adapter import ( - DynamoEngineAdapter, - RequestHandler, - ServedRollout, - _annotation_value, -) -from nemo_rl.models.generation.openai_server_utils import ( - model_stop_token_ids, - splice_prefix_tokens, - turn_end_token_ids, -) -from nemo_rl.models.generation.vllm.utils import ( - chat_generation_tokens, - chat_routed_experts, - check_sampling_param, - remap_multimodal_placeholders, -) - -LOGGER = logging.getLogger(__name__) - -if TYPE_CHECKING: - from nemo_rl.models.generation.vllm.vllm_worker_async import ( - VllmAsyncGenerationWorkerImpl, - ) - -# vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them -# (dynamo/vllm/cache_info.py MAIN_ATTENTION_KV_CACHE_KINDS, which NeMo RL may -# not import); test_main_attention_kinds_match_dynamos checks the copy. -_MAIN_ATTENTION_KINDS = ( - KVCacheSpecKind.FULL_ATTENTION, - KVCacheSpecKind.MLA_ATTENTION, - KVCacheSpecKind.SINK_FULL_ATTENTION, -) - -# Annotations NeMo-Gym's Dynamo dialect (vllm_model with dynamo_frontend, Gym#4817) -# sets on its requests; Dynamo forwards nvext.annotations to the engine verbatim. -GYM_ROLLOUT_ANNOTATION = "nemo_gym:rollout" -# A token-capture call's admission JSON. -GYM_CAPTURE_ANNOTATION_PREFIX = "ng_capture:" -# Without capture: the last echoed assistant turn's prompt + generation token -# IDs, which the native server reads from the messages instead. -GYM_PREFIX_ANNOTATION_PREFIX = "ng_prefix:" -_GYM_SAMPLING_KEYS = ("temperature", "top_p", "val_temperature", "val_top_p") -_GYM_TOP_K_KEYS = ("top_k", "val_top_k") -# Sampling a Gym call may not change, with the value it is served at: vLLM -# would otherwise fill min_p and the repetition penalty from the model's -# generation_config.json, and the trainer replays none of them. -_GYM_FIXED_SAMPLING = { - "min_p": 0.0, - "repetition_penalty": 1.0, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, -} -# The engine_data name Gym's dialect reads for each native message field. -_GYM_ENGINE_DATA_KEYS = { - "generation_token_ids": "completion_token_ids", - "generation_log_probs": "completion_logprobs", - "routed_experts": "routed_experts", -} - -# The vLLM modality each key of Dynamo's multi_modal_data holds. -_DYNAMO_MEDIA_MODALITIES = { - "image_url": "image", - "video_url": "video", - "audio_url": "audio", -} - -# Filled from the model's generation config when a request leaves them unset, -# as vLLM's OpenAI server fills them. -_GENERATION_CONFIG_KEYS = ( - "repetition_penalty", - "temperature", - "top_k", - "top_p", - "min_p", -) -_SAMPLING_KEYS = ( - *_GENERATION_CONFIG_KEYS, - "seed", - "presence_penalty", - "frequency_penalty", -) - - -def dynamo_kv_events_config( - cfg: DynamoCfg, *, prefix_caching: bool -) -> Optional[dict[str, Any]]: - """The ``kv_events_config`` that feeds Dynamo's KV router. - - None unless the frontend routes by KV and the engine caches prefixes: vLLM - publishes block events only for the blocks its prefix cache stores. - """ - if cfg.frontend_args.router_mode != "kv" or not prefix_caching: - return None - return { - "enable_kv_cache_events": True, - "publisher": "zmq", - # A random abstract-namespace socket per engine (dynamo-integration.md). - "endpoint": f"ipc://@nemo-rl-kv-events-{os.urandom(8).hex()}", - } - - -def _as_f32(value: Optional[float]) -> Optional[float]: - """Dynamo's frontend carries sampling floats as f32 (0.7 arrives as 0.69999998).""" - return None if value is None else float(np.float32(value)) - - -def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: - """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. - - Dynamo sets at most one constraint. The decoding backend is the engine's - own (``structured_outputs_config``), as for the native server. - """ - structural_tag = guided_decoding.get("structural_tag") - if isinstance(structural_tag, dict): - structural_tag = json.dumps(structural_tag) - return StructuredOutputsParams( - json=guided_decoding.get("json"), - regex=guided_decoding.get("regex"), - choice=guided_decoding.get("choice"), - grammar=guided_decoding.get("grammar"), - whitespace_pattern=guided_decoding.get("whitespace_pattern"), - structural_tag=structural_tag, - ) - - -def build_sampling_params( - request: GenerateRequest, - defaults: dict[str, Any], - *, - output_room: int, - overrides: Optional[dict[str, Any]] = None, - output_kind: RequestOutputKind = RequestOutputKind.DELTA, -) -> SamplingParams: - """Map an OpenAI-shaped Dynamo request onto vLLM SamplingParams. - - As vLLM's OpenAI server builds them: the model's generation-config - ``defaults`` for what the request leaves unset, and ``max_tokens`` capped - by the context. Stop strings are applied by Dynamo's Rust detokenizer, so - only token-level stops reach vLLM unless ``overrides`` sets them. - - Args: - request: Dynamo's preprocessed request. - defaults: ``ModelConfig.get_diff_sampling_param()``. - output_room: The context left after the prompt the engine runs, which - media expansion or a prefix splice can make longer than the - request's ``token_ids``. - overrides: SamplingParams fields that replace the request's. - output_kind: How vLLM reports outputs. - """ - sampling = request.get("sampling_options") or {} - passthrough = (request.get("extra_args") or {}).get("sampling_options") or {} - stop = request.get("stop_conditions") or {} - n = sampling.get("n") or 1 - if sampling.get("best_of") not in (None, n): - raise InvalidArgument("best_of is not supported by vLLM's V1 engine") - - kwargs: dict[str, Any] = { - key: defaults[key] for key in _GENERATION_CONFIG_KEYS if key in defaults - } - kwargs.update( - {key: sampling[key] for key in _SAMPLING_KEYS if sampling.get(key) is not None} - ) - kwargs["n"] = n - # As vLLM's get_max_tokens: the generation config's max_tokens applies only - # when the request leaves max_tokens unset. - requested_max_tokens = stop.get("max_tokens") - if requested_max_tokens is None: - requested_max_tokens = defaults.get("max_tokens") - kwargs["max_tokens"] = min( - limit - for limit in (max(1, output_room), requested_max_tokens) - if limit is not None - ) - if stop.get("min_tokens") is not None: - kwargs["min_tokens"] = stop["min_tokens"] - if stop.get("ignore_eos") is not None: - kwargs["ignore_eos"] = stop["ignore_eos"] - stop_ids = set(stop.get("stop_token_ids") or ()) | set( - stop.get("stop_token_ids_hidden") or () - ) - if stop_ids: - kwargs["stop_token_ids"] = sorted(stop_ids) - if sampling.get("guided_decoding"): - kwargs["structured_outputs"] = structured_outputs(sampling["guided_decoding"]) - for key in ("allowed_token_ids", "logprob_token_ids"): - if passthrough.get(key) is not None: - kwargs[key] = passthrough[key] - - logprobs, prompt_logprobs = parse_logprob_options( - request.get("output_options") or {} - ) - # Explicit logprob_token_ids replace top-k selection; vLLM rejects both. - if logprobs is not None and "logprob_token_ids" not in kwargs: - kwargs["logprobs"] = logprobs - if prompt_logprobs is not None: - kwargs["prompt_logprobs"] = prompt_logprobs - kwargs["detokenize"] = False - kwargs.update(overrides or {}) - params = SamplingParams(**kwargs, output_kind=output_kind) - if passthrough.get("bad_words_token_ids") is not None: - # vLLM only tokenizes bad_words strings into this field; a token-in - # client sends the token IDs themselves. - if not hasattr(params, "_bad_words_token_ids"): - raise RuntimeError( - "vLLM SamplingParams has no _bad_words_token_ids; update the " - "bad_words_token_ids passthrough for this vLLM" - ) - params._bad_words_token_ids = passthrough["bad_words_token_ids"] - return params - - -class SnapshotStatLogger(StatLoggerBase): - """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" - - # Set by Dynamo after vLLM built the logger, so shared through the class. - publisher: Optional[Any] = None - - def __init__(self, vllm_config: Any, engine_index: int = 0) -> None: - self.dp_rank = engine_index - # vLLM sizes the KV cache before it builds its stat loggers. - self.num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0 - - def record(self, scheduler_stats, iteration_stats, *args, **kwargs) -> None: - cls = type(self) - if scheduler_stats is None or cls.publisher is None: - return - total = max(1, self.num_gpu_blocks) - usage = scheduler_stats.kv_cache_usage - hit_rate = None - prefix = scheduler_stats.prefix_cache_stats - if prefix.queries > 0: - hit_rate = prefix.hits / prefix.queries - cls.publisher.publish( - self.dp_rank, - ComponentSnapshot( - kv_used_blocks=int(total * usage), - kv_total_blocks=total, - gpu_cache_usage=usage, - kv_cache_hit_rate=hit_rate, - dp_rank=self.dp_rank, - ), - ) - - def log_engine_initialized(self) -> None: - pass - - -class DynamoVllmAdapter(DynamoEngineAdapter): - """The vLLM backend, over a running NeMo RL vLLM worker it does not own.""" - - def __init__( - self, host: "VllmAsyncGenerationWorkerImpl", *, leader_worker_idx: int - ): - super().__init__(leader_worker_idx=leader_worker_idx) - self._host = host - self._llm = host.llm - self._max_model_len = self._llm.vllm_config.model_config.max_model_len - self._defaults = self._llm.model_config.get_diff_sampling_param() - # The train and validation sampling NeMo-Gym rollouts must match, at the - # f32 precision the frontend delivers. - self._gym_sampling_profile = { - **{key: _as_f32(host.cfg[key]) for key in _GYM_SAMPLING_KEYS}, - **{key: host.cfg[key] for key in _GYM_TOP_K_KEYS}, - } - - def _handlers(self) -> dict[str, RequestHandler]: - return {**super()._handlers(), GYM_ROLLOUT_ANNOTATION: self._generate_gym} - - async def serve_rollout( - self, - prompt_token_ids: list[int], - *, - greedy: bool, - stop_strings: Optional[list[str]], - request_id: str, - ) -> ServedRollout: - result, final_output = await self._host._serve_rollout( - TokensPrompt(prompt_token_ids=prompt_token_ids), - prompt_token_ids=prompt_token_ids, - greedy=greedy, - stop_strings=stop_strings, - request_id=request_id, - device=torch.device("cpu"), - ) - if final_output is None: - return ServedRollout(result, finish_reason="length", usage=None) - return ServedRollout( - result, - finish_reason=final_output.outputs[0].finish_reason, - usage=_usage(final_output, len(result.token_ids)), - ) - - async def start(self, worker_id: int) -> EngineConfig: - del worker_id - vllm_config = self._llm.vllm_config - cache = vllm_config.cache_config - scheduler = vllm_config.scheduler_config - # The KV-event block size is the main-attention group's, which can - # differ from cache_config.block_size on hybrid models. - block_size = cache.block_size - for group in await self._llm.engine_core.call_utility_async( - "get_kv_cache_group_metadata" - ): - if group.get("kind") in _MAIN_ATTENTION_KINDS: - block_size = group["block_size"] - break - model = self._host.model_name - return EngineConfig( - model=model, - served_model_name=model, - # No token budget: the engine owns overflow (dynamo-integration.md). - llm=LlmRegistration( - context_length=self._max_model_len, - kv_cache_block_size=block_size, - total_kv_blocks=cache.num_gpu_blocks, - max_num_seqs=scheduler.max_num_seqs, - max_num_batched_tokens=scheduler.max_num_batched_tokens, - ), - ) - - async def _generate_gym( - self, request: GenerateRequest, annotation: str, context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - del annotation - try: - async for chunk in self._serve_gym_call(request, context): - yield chunk - except VLLMClientError as error: - # vLLM's request errors are not RuntimeErrors; the native server - # answers them with a 4xx that carries the message. - raise InvalidArgument(str(error)) from error - - async def _serve_gym_call( - self, request: GenerateRequest, context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - """One NeMo-Gym model call, served as the native chat endpoint serves it. - - The request was rendered by the frontend. As on the native server, it must - sample on-policy, its echoed or staged prefix replaces the rendered - history, output is clamped to the context, and a token-capture call stages - its delta from this process. Tokens, log probabilities and routed experts - come from vLLM's final output through the native endpoint's helpers. One - chunk carries the whole completion and, in ``engine_data``, the token - arrays Gym's dialect reads, or a captured call's commit coordinates. - """ - host = self._host - sampling: dict[str, Any] = request.get("sampling_options") or {} - temperature = _as_f32(sampling.get("temperature")) - top_p = _as_f32(sampling.get("top_p")) - try: - top_k = check_sampling_param( - temperature=temperature, - top_p=top_p, - top_k=sampling.get("top_k"), - generation_config=self._gym_sampling_profile, - ) - except ValueError as error: - raise InvalidArgument(str(error)) from error - for key, served in _GYM_FIXED_SAMPLING.items(): - if sampling.get(key) not in (None, served): - raise InvalidArgument( - f"request {key}={sampling[key]} is not supported; leave it unset" - ) - if (sampling.get("n") or 1) != 1: - raise InvalidArgument( - "NeMo-Gym calls through Dynamo take n=1: engine_data carries one " - "choice's tokens." - ) - # The train or validation profile's own values, so vLLM samples exactly - # as natively. - profile = self._gym_sampling_profile - if (temperature, top_p) == (profile["temperature"], profile["top_p"]): - temperature, top_p = host.cfg["temperature"], host.cfg["top_p"] - else: - temperature, top_p = host.cfg["val_temperature"], host.cfg["val_top_p"] - - capture_context = _annotation_value(request, GYM_CAPTURE_ANNOTATION_PREFIX) - admission = host._parse_capture_admission( - None if capture_context is None else json.loads(capture_context) - ) - prefix_token_ids = None - if admission is not None: - if admission.mode == "token_in": - # A blocking data-plane read; keep it off the engine loop. - prefix_token_ids = await asyncio.to_thread( - host._resolve_admission_prefix, admission - ) - elif ( - echoed := _annotation_value(request, GYM_PREFIX_ANNOTATION_PREFIX) - ) is not None: - prefix_token_ids = json.loads(echoed) - # As the native endpoint does: a captured call reads the processor's own - # pixels, not cache references (#4124). - engine_input = await self._engine_input( - request, skip_mm_cache=admission is not None and host._capture_media - ) - template_token_ids = engine_input["prompt_token_ids"] - self._check_prompt_fits(len(template_token_ids)) - token_ids, splice = template_token_ids, None - if prefix_token_ids is not None: - # The frontend rendered the whole conversation; replace its history - # with the exact tokens the model generated for it. The engine sees no - # messages, so the prefix stands in for the native endpoint's render - # of the history. - splice = splice_prefix_tokens( - tokenizer=self._llm.tokenizer, - model_prefix_token_ids=prefix_token_ids, - template_prefix_token_ids=None, - template_token_ids=template_token_ids, - eos_token_id=turn_end_token_ids( - model_stop_token_ids(self._llm.model_config, host.cfg), - self._llm.tokenizer, - ), - ) - token_ids = splice.token_ids - self._check_prompt_fits(len(token_ids)) - try: - # Off the loop: a retained-media check reads staged calls. - media = await asyncio.to_thread( - host._capture_request_media, - engine_input, - admission=admission, - splice=splice, - ) - except MediaCaptureRejected as error: - raise InvalidArgument(f"{error} ({error.code})") from error - if ( - media is None - and splice is not None - and (placeholders := engine_input.get("mm_placeholders")) - ): - # Media capture already moved them to spliced coordinates. - engine_input["mm_placeholders"] = remap_multimodal_placeholders( - template_token_ids=template_token_ids, - final_token_ids=token_ids, - mm_placeholders=placeholders, - splice=splice, - ) - engine_input["prompt_token_ids"] = token_ids - - stop = (request.get("stop_conditions") or {}).get("stop") - overrides: dict[str, Any] = { - "temperature": temperature, - "top_p": top_p, - # As the native endpoint does: the matched profile's top_k. - "top_k": top_k, - **_GYM_FIXED_SAMPLING, - } - if stop: - # vLLM stops on them as natively, so the token arrays match; the - # frontend then cuts the text at the same place, so both must decode - # special tokens alike (the frontend defaults this off for some parsers). - skip = (request.get("output_options") or {}).get("skip_special_tokens") - overrides.update( - stop=stop, - detokenize=True, - skip_special_tokens=True if skip is None else skip, - ) - if admission is not None: - # A captured call stages the sampled tokens' log probabilities. - overrides["logprobs"] = 0 - sampling_params = build_sampling_params( - request, - self._defaults, - output_room=self._max_model_len - len(token_ids), - overrides=overrides, - output_kind=RequestOutputKind.FINAL_ONLY, - ) - try: - self._llm.check_admission(1, context.id()) - except GracefulHTTPError as error: - raise HttpError(int(error.http_status), error.message) from error - - if admission is not None: - host._begin_request_capture( - request, - token_ids, - admission=admission, - prefix_token_ids=prefix_token_ids, - media=media, - ) - try: - final_output = None - async for output in self._llm.generate( - engine_input, sampling_params, context.id() - ): - final_output = output - if final_output is None: - raise RuntimeError(f"No output received for request {context.id()}") - if admission is not None and context.is_stopped(): - # Dynamo aborts the vLLM request on a disconnect but never - # cancels this generator: fail the capture (below), as the - # native endpoint's cancellation does, instead of staging it. - return - completion = final_output.outputs[0] - if completion.finish_reason == "error": - # As vLLM's chat server does (_raise_if_error): never stage a failed call. - raise RuntimeError(f"Generation failed for request {context.id()}") - chunk: GenerateChunk = { - "token_ids": list(completion.token_ids), - "index": 0, - "finish_reason": completion.finish_reason, - "completion_usage": _usage(final_output, len(completion.token_ids)), - } - if admission is not None: - chunk["engine_data"] = await self._stage_gym_call(request, final_output) - else: - engine_data = {"prompt_token_ids": list(final_output.prompt_token_ids)} - for field, value in self._gym_message( - final_output, encode_for_wire=True - ).items(): - engine_data[_GYM_ENGINE_DATA_KEYS[field]] = value - chunk["engine_data"] = engine_data - log_probs, top_logprobs = extract_from_completion_output( - completion, 0, tokenizer=self._llm.tokenizer, include_bytes=True - ) - if log_probs is not None: - chunk["log_probs"] = log_probs - chunk["top_logprobs"] = top_logprobs - yield chunk - finally: - # A no-op once the call was staged: staging pops its capture state. - if admission is not None: - host._abort_request_capture( - request, - reason=( - "client_disconnect" if context.is_stopped() else "engine_error" - ), - ) - - async def _load_media( - self, request: GenerateRequest - ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: - """The request's media and their UUIDs by modality, as vLLM's chat parser loads them. - - Dynamo passes each media URL through (data: URIs included); the engine - fetches and decodes it with vLLM's own media connector and settings. - """ - media = request.get("multi_modal_data") - if not media: - return None - model_config = self._llm.model_config - mm_config = model_config.multimodal_config - connector = MEDIA_CONNECTOR_REGISTRY.load( - vllm_envs.VLLM_MEDIA_CONNECTOR, - media_io_kwargs=request.get("media_io_kwargs") - or (mm_config.media_io_kwargs if mm_config else None), - allowed_local_media_path=model_config.allowed_local_media_path, - allowed_media_domains=model_config.allowed_media_domains, - ) - fetch = { - "image": connector.fetch_image_async, - "audio": connector.fetch_audio_async, - } - if "video_url" in media: - fetch["video"] = functools.partial( - connector.fetch_video_async, - video_processor=get_video_processor_cls_name(model_config), - ) - uuids = request.get("multi_modal_uuids") or {} - mm_data: dict[str, list[Any]] = {} - mm_uuids: dict[str, list[Any]] = {} - for key, items in media.items(): - modality = _DYNAMO_MEDIA_MODALITIES.get(key) - if modality is None: - raise InvalidArgument(f"Unsupported media type {key!r}") - if any("Url" not in item for item in items): - raise InvalidArgument( - f"{key} items must carry their URL: NeMo RL's vLLM engines load " - "media themselves" - ) - try: - mm_data[modality] = list( - await asyncio.gather( - *(fetch[modality](item["Url"]) for item in items) - ) - ) - except ValueError as error: - raise InvalidArgument(f"Cannot load {key}: {error}") from error - mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) - return mm_data, mm_uuids - - async def _engine_input( - self, request: GenerateRequest, *, skip_mm_cache: bool = False - ) -> dict[str, Any]: - """The processed vLLM prompt for the request, as the native chat endpoint builds it. - - The frontend's tokens carry one unexpanded placeholder per media item; - vLLM's processor expands them and processes the media, exactly as after - the native endpoint renders and tokenizes the messages. - """ - prompt = TokensPrompt(prompt_token_ids=list(request["token_ids"])) - media = await self._load_media(request) - if media is not None: - prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media - if request.get("mm_processor_kwargs"): - prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] - return await self._llm.renderer.process_for_engine_async( - prompt, time.time(), skip_mm_cache=skip_mm_cache - ) - - def _check_prompt_fits(self, prompt_len: int) -> None: - """Refuse a prompt that leaves no room for output, as the native endpoint does. - - NeMo-Gym reads "context length" in a 400 as a length refusal rather than an - engine failure. - """ - if prompt_len >= self._max_model_len: - raise InvalidArgument( - f"Prompt length ({prompt_len}) fills or exceeds this model's " - f"maximum context length ({self._max_model_len}). No room for " - "output tokens." - ) - - def _gym_message( - self, final_output: Any, *, encode_for_wire: bool - ) -> dict[str, Any]: - """The token fields the native endpoint puts on the call's message.""" - completion = final_output.outputs[0] - message: dict[str, Any] = {"generation_token_ids": list(completion.token_ids)} - if completion.logprobs is not None: - _, message["generation_log_probs"] = chat_generation_tokens(completion) - if self._host._return_routed_experts_enabled(): - # Full prompt + generation rows, as the native endpoint attaches them. - message["routed_experts"] = chat_routed_experts( - final_output, - completion, - prompt_token_count=len(final_output.prompt_token_ids), - device=torch.device("cpu"), - logger=LOGGER, - routed_experts_dtype=self._host.routed_experts_dtype, - encode_for_wire=encode_for_wire, - ) - return message - - async def _stage_gym_call( - self, request: GenerateRequest, final_output: Any - ) -> dict[str, Any]: - """Stage a captured call from this process; return the engine_data carrying its commit coordinates.""" - # Deferred: only captured calls reach here, and capture already imports nemo_gym. - from nemo_gym.token_id_capture import NG_COMMIT_COORDS_FIELD - - # The native endpoint's response shape; the prompt is the one recorded at - # begin_call. - content = { - "choices": [ - { - "index": 0, - "finish_reason": final_output.outputs[0].finish_reason, - "message": self._gym_message(final_output, encode_for_wire=False), - } - ] - } - # The sink write is a blocking data-plane round trip. - content = await asyncio.to_thread( - self._host._finish_request_capture, request, content - ) - return {NG_COMMIT_COORDS_FIELD: content[NG_COMMIT_COORDS_FIELD]} - - async def abort(self, context: Context) -> None: - await self._llm.abort(context.id()) - - async def kv_event_sources(self) -> list[KvEventSource]: - kv_events = self._llm.vllm_config.kv_events_config - if kv_events is None or not kv_events.enable_kv_cache_events: - return [] - # A user's own kv_events_config can keep vLLM's wildcard default - # (tcp://*:5557): vLLM binds it, but a subscriber cannot connect to "*". - return [ZmqSource(endpoint=kv_events.endpoint.replace("*", "127.0.0.1"))] - - def component_metrics_dp_ranks(self) -> list[int]: - return [0] - - def attach_snapshot_publisher(self, publisher: Any) -> None: - SnapshotStatLogger.publisher = publisher - - async def cleanup(self) -> None: - # The worker owns the AsyncLLM; its shutdown() tears the engine down. - SnapshotStatLogger.publisher = None - - -def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: - prompt_tokens = len(output.prompt_token_ids or ()) - return { - "prompt_tokens": prompt_tokens, - "completion_tokens": num_output_tokens, - "total_tokens": prompt_tokens + num_output_tokens, - "prompt_tokens_details": {"cached_tokens": output.num_cached_tokens or 0}, - } - - -def _check_parser_name(name: Optional[str], known: list[str], field: str) -> None: - if name is not None and name not in known: - raise ValueError( - f"Dynamo has no {field.replace('_', ' ')} named {name!r}; set " - f"policy.generation.dynamo_cfg.worker_args.{field} to one of " - f"{sorted(known)}." - ) - - -def _chat_template_file(chat_template: Optional[str]) -> Optional[str]: - """A file holding ``chat_template``, which Dynamo reads by path. - - Resolved as the native server resolves it (vLLM's ``load_chat_template``): - a file, a vLLM built-in template name, or literal Jinja. Named by content, - so every engine on a node, and every restart, shares one file. - """ - if chat_template is None: - return None - content = load_chat_template(chat_template) - digest = hashlib.sha256(content.encode()).hexdigest()[:16] - path = os.path.join(tempfile.gettempdir(), f"nemo_rl_chat_template_{digest}.jinja") - if not os.path.exists(path): - # Written aside and renamed, so a concurrent reader never sees a part. - with tempfile.NamedTemporaryFile( - "w", dir=os.path.dirname(path), suffix=".jinja", delete=False - ) as template_file: - template_file.write(content) - os.replace(template_file.name, path) - return path - - -def chat_worker_config(host: "VllmAsyncGenerationWorkerImpl") -> dict[str, Any]: - """The ``WorkerConfig`` fields that set up the frontend's chat path for Gym calls. - - The frontend renders and parses NeMo-Gym's chat calls with the native server's - template and parsers, under Dynamo's names. - """ - worker_args = dynamo_cfg(host.cfg).worker_args - tool_call_parser, reasoning_parser = dynamo_parsers(host.cfg) - _check_parser_name(tool_call_parser, get_tool_parser_names(), "tool_call_parser") - _check_parser_name( - reasoning_parser, get_reasoning_parser_names(), "reasoning_parser" - ) - return { - "custom_jinja_template": _chat_template_file(dynamo_chat_template(host.cfg)), - "tool_call_parser": tool_call_parser, - "reasoning_parser": reasoning_parser, - "exclude_tools_when_tool_choice_none": ( - worker_args.exclude_tools_when_tool_choice_none - ), - "structural_tag_mode": "on" if worker_args.enable_structural_tag else "off", - "structural_tag_scope": worker_args.structural_tag_scope, - "structural_tag_schema": worker_args.structural_tag_schema, - } diff --git a/nemo_rl/models/generation/gym_call.py b/nemo_rl/models/generation/gym_call.py new file mode 100644 index 00000000000..5a4346f6b86 --- /dev/null +++ b/nemo_rl/models/generation/gym_call.py @@ -0,0 +1,79 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The extra fields NeMo RL and NeMo-Gym exchange on a chat call, defined once. + +The native vLLM server reads :class:`GymCallExtras` as request fields and +answers with :class:`GymCallResult`'s fields on the message. Dynamo's frontend +carries only OpenAI fields plus ``nvext``, so through it NeMo-Gym's Dynamo +dialect packs the extras into one ``nemo_rl:gym:`` annotation and NeMo +RL's chat engine returns the result as ``nvext["nemo_rl"]``. A field added here +reaches both paths. +""" + +import json +from collections.abc import Iterable +from typing import Any, Optional + +from pydantic import BaseModel, ConfigDict + +# The annotation NeMo-Gym's Dynamo dialect carries the extras in. +GYM_EXTRAS_ANNOTATION_PREFIX = "nemo_rl:gym:" +# The response nvext key the call result rides in through Dynamo. +GYM_RESULT_NVEXT_KEY = "nemo_rl" + + +class GymCallExtras(BaseModel): + """What a NeMo-Gym chat call adds to an OpenAI request.""" + + # A field Gym sends that NeMo RL does not know is refused, not ignored. + model_config = ConfigDict(extra="forbid") + + # The token-capture admission (capture_key, call ids, prev_len, mode, ...). + ng_capture: Optional[dict[str, Any]] = None + # Without capture: the last echoed assistant turn's prompt + generation ids. + required_prefix_token_ids: Optional[list[int]] = None + # Return only the prompt's length (Gym's /context/measure); no generation. + ng_measure: bool = False + + @classmethod + def from_annotations(cls, annotations: Iterable[str]) -> "GymCallExtras | None": + """The extras in ``nvext.annotations``; None when the call carries none.""" + for annotation in annotations or (): + if annotation.startswith(GYM_EXTRAS_ANNOTATION_PREFIX): + return cls.model_validate( + json.loads(annotation[len(GYM_EXTRAS_ANNOTATION_PREFIX) :]) + ) + return None + + +class GymCallResult(BaseModel): + """What a NeMo-Gym chat call returns beyond the OpenAI response. + + The native server puts these on ``choices[0].message`` (token fields, + ``routed_experts``) or the response (``ng_commit_coords``); field names are + the native ones, so Gym lifts them without renaming. + """ + + # A generation call without token capture. + prompt_token_ids: Optional[list[int]] = None + generation_token_ids: Optional[list[int]] = None + generation_log_probs: Optional[list[float]] = None + # Router replay: the wire-encoded routes, as the native message carries them. + routed_experts: Optional[Any] = None + # A captured call: where its record was staged. + ng_commit_coords: Optional[dict[str, Any]] = None + # A measure-only call. + prompt_token_count: Optional[int] = None + ng_context_measured: Optional[bool] = None diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index 98d6169edc6..d35a2b995a0 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -69,6 +69,7 @@ CapturedChatLogprobsMixin, ) from nemo_rl.models.generation.vllm.utils import ( + OffPolicySamplingError, attach_routed_experts_to_chat_response_choices, attach_sampler_topk_logprobs_to_chat_response_choices, attach_token_information_to_chat_response_choices, @@ -81,6 +82,10 @@ class NeMoRLOpenAIChatRequestMixin: + # Declared by the request models this mixes into. + required_prefix_token_ids: Optional[list[int]] + messages: Any + def model_post_init(self, context): # NeMo-Gym specific processing. This is just how NeMo-Gym returns the extra token information. if self.required_prefix_token_ids is None: @@ -112,20 +117,20 @@ class NeMoRLTokenizeChatRequest(NeMoRLOpenAIChatRequestMixin, TokenizeChatReques NeMoRLTokenizeRequest = Union[TokenizeCompletionRequest, NeMoRLTokenizeChatRequest] -class NeMoRLOpenAIServingMixin: - """The chat preprocessing NeMo RL adds: prefix splice and token capture.""" +def set_max_tokens(request: Any, max_tokens: int) -> None: + """Set the request's max output tokens. - @staticmethod - def _set_max_tokens(request, max_tokens: int) -> None: - """Set the request's max output tokens. + Mutates the request in place. Handles both max_completion_tokens (newer OpenAI API) + and max_tokens (deprecated but still supported by vLLM). + """ + if request.max_completion_tokens is not None: + request.max_completion_tokens = max_tokens + elif request.max_tokens is not None: + request.max_tokens = max_tokens - Mutates the request in place. Handles both max_completion_tokens (newer OpenAI API) - and max_tokens (deprecated but still supported by vLLM). - """ - if request.max_completion_tokens is not None: - request.max_completion_tokens = max_tokens - elif request.max_tokens is not None: - request.max_tokens = max_tokens + +class NeMoRLOpenAIServingMixin: + """The chat preprocessing NeMo RL adds: prefix splice and token capture.""" def _clamp_max_tokens( self, request, request_max_tokens: int, prompt_token_ids: list[int] @@ -146,7 +151,7 @@ def _clamp_max_tokens( value=len(prompt_token_ids), ) max_tokens = min(request_max_tokens, remaining) - self._set_max_tokens(request, max_tokens) + set_max_tokens(request, max_tokens) async def _render_chat_call( self, request, messages, render_kwargs: dict[str, Any], *, admission @@ -262,7 +267,7 @@ async def preprocess_chat( # If max_completion_tokens or max_tokens is not set, we don't need to do _clamp_max_tokens. # So we don't need to set the request's max output tokens to 1 here. if actual_request_max_tokens is not None: - self._set_max_tokens(request, 1) + set_max_tokens(request, 1) rendered = await self._render_chat_call( request, @@ -408,6 +413,58 @@ def __init__( self._model_stop_token_ids = model_stop_token_ids +class FrontendRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): + """The render step for NeMo RL's chat engine in Dynamo's frontend. + + Runs only the shared step (:meth:`_render_chat_call`): the frontend reads + the TQ prefix and splices, and the engine captures. Media is always + processed here, never served from a processor cache, since the engine runs + on the pixels the frontend sends. + """ + + # The render the overridden preprocess_chat is asked for, and its result. + _admission: Any = None + _rendered: Any = None + + async def render_for_frontend(self, request, *, admission) -> "RenderedChat": + """Render ``request``; the caller restores its lowered output budget.""" + self._admission = admission + # As preprocess_chat does: vLLM's pre-tokenization length check must + # pass; the frontend clamps the budget after the splice. + set_max_tokens(request, 1) + result = await self.render_chat(request, skip_mm_cache=True) + if not isinstance(result, tuple): + raise ValueError(result.error.message) + return self._rendered + + async def preprocess_chat( + self, + request, + messages, + default_template, + default_template_content_format, + default_template_kwargs, + tool_dicts=None, + parser=None, + *, + skip_mm_cache: bool = False, + ): + self._rendered = await self._render_chat_call( + request, + messages, + dict( + default_template=default_template, + default_template_content_format=default_template_content_format, + default_template_kwargs=default_template_kwargs, + tool_dicts=tool_dicts, + parser=parser, + skip_mm_cache=True, + ), + admission=self._admission, + ) + return self._rendered.conversation, [self._rendered.engine_prompt] + + class NeMoRLOpenAIServingChatMixin: async def chat_completion_full_generator( self, @@ -532,17 +589,8 @@ class ChatServer: serving_tokenization: NeMoRLServingTokenization -def build_chat_server( - host: Any, *, engine_client: Any, model_config: Any, cfg: dict[str, Any] -) -> ChatServer: - """Build the chat serving the worker's HTTP server answers with. - - Args: - host: The vLLM worker; it owns token capture. - engine_client: The engine the chat serving generates with. - model_config: The engine's vLLM ``ModelConfig``. - cfg: ``policy.generation``. - """ +def register_parser_plugins(cfg: dict[str, Any]) -> None: + """Import the tool and reasoning parser plugins ``policy.generation`` names.""" maybe_tool_parser_plugin = cfg["vllm_cfg"].get("tool_parser_plugin") if maybe_tool_parser_plugin: ToolParserManager.import_tool_parser(maybe_tool_parser_plugin) @@ -551,27 +599,21 @@ def build_chat_server( if maybe_reasoning_parser_plugin: ReasoningParserManager.import_reasoning_parser(maybe_reasoning_parser_plugin) - base_model_paths = [ - BaseModelPath( - name=model_config.served_model_name, model_path=model_config.model - ), - BaseModelPath(name=model_config.model, model_path=model_config.model), - ] - openai_serving_models_kwargs = dict( - engine_client=engine_client, - base_model_paths=base_model_paths, - lora_modules=None, - ) - openai_serving_models = OpenAIServingModels(**openai_serving_models_kwargs) +def serving_chat_settings(cfg: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + """The chat serving's settings and default chat-template kwargs. - serving_chat_default_kwargs = dict( - response_role="assistant", - request_logger=None, - chat_template=None, - chat_template_content_format="auto", - enable_auto_tools=True, - ) + One copy for every host of the chat path, so each renders and parses + alike. Returns the ``OpenAIServingChat`` kwargs (template loaded, + ``default_chat_template_kwargs`` normalized) and those template kwargs. + """ + serving_chat_default_kwargs: dict[str, Any] = { + "response_role": "assistant", + "request_logger": None, + "chat_template": None, + "chat_template_content_format": "auto", + "enable_auto_tools": True, + } serving_chat_kwargs = serving_chat_default_kwargs | cfg["vllm_cfg"].get( "http_server_serving_chat_kwargs", dict() ) @@ -604,6 +646,37 @@ def build_chat_server( default_chat_template_kwargs: dict[str, Any] = ( serving_chat_kwargs["default_chat_template_kwargs"] or {} ) + return serving_chat_kwargs, default_chat_template_kwargs + + +def build_chat_server( + host: Any, *, engine_client: Any, model_config: Any, cfg: dict[str, Any] +) -> ChatServer: + """Build the chat serving the worker's HTTP server answers with. + + Args: + host: The vLLM worker; it owns token capture. + engine_client: The engine the chat serving generates with. + model_config: The engine's vLLM ``ModelConfig``. + cfg: ``policy.generation``. + """ + register_parser_plugins(cfg) + + base_model_paths = [ + BaseModelPath( + name=model_config.served_model_name, model_path=model_config.model + ), + BaseModelPath(name=model_config.model, model_path=model_config.model), + ] + + openai_serving_models_kwargs = dict( + engine_client=engine_client, + base_model_paths=base_model_paths, + lora_modules=None, + ) + openai_serving_models = OpenAIServingModels(**openai_serving_models_kwargs) + + serving_chat_kwargs, default_chat_template_kwargs = serving_chat_settings(cfg) online_renderer = NeMoRLOnlineRenderer( model_config=engine_client.model_config, renderer=engine_client.renderer, @@ -676,17 +749,32 @@ async def create_chat_completion( # vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params does) # with min_p and the repetition penalty off; the trainer replays none of # them, nor the penalties or logit bias a request may therefore not set. - for field, value in on_policy_sampling( - temperature=request.temperature, - top_p=request.top_p, - top_k=request.top_k, - min_p=request.min_p, - repetition_penalty=request.repetition_penalty, - presence_penalty=request.presence_penalty, - frequency_penalty=request.frequency_penalty, - logit_bias=request.logit_bias, - generation_config=generation_config, - ).items(): + try: + sampling = on_policy_sampling( + temperature=request.temperature, + top_p=request.top_p, + top_k=request.top_k, + min_p=request.min_p, + repetition_penalty=request.repetition_penalty, + presence_penalty=request.presence_penalty, + frequency_penalty=request.frequency_penalty, + logit_bias=request.logit_bias, + generation_config=generation_config, + ) + except OffPolicySamplingError as e: + # Before capture begins, so there is no capture state to settle. + return JSONResponse( + content={ + "error": { + "message": str(e), + "type": "invalid_request_error", + "param": "sampling", + "code": 400, + } + }, + status_code=400, + ) + for field, value in sampling.items(): setattr(request, field, value) try: diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index d4c72206365..0e4f8547c1f 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -785,6 +785,7 @@ def _start_dynamo_services(self) -> None: # The workers' venv carries ai-dynamo, so the frontend runs the same # release as every engine. frontend_python=self.worker_group.py_executable, + ray_address=ray.get_runtime_context().gcs_address, ) runtime.start() self._dynamo_runtime = runtime @@ -878,6 +879,13 @@ def setup_token_capture( ) self._token_capture_media = agreed_media_columns(ray.get(futures)) self._token_capture_setup = setup + if self._dynamo_runtime is not None: + # NeMo-Gym's calls through Dynamo read their prefix in the frontend. + self._dynamo_runtime.publish_chat_capture( + dp_cfg=dp_cfg, + staging_partition=staging_partition, + capture_media=capture_media, + ) return self._token_capture_media def get_data_plane_snapshots(self) -> list[dict[str, Any]]: diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 6d86e61b6eb..2bebcd08b8b 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -447,7 +447,7 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: else [] ) if dynamo_enabled(self.cfg): - from nemo_rl.models.generation.dynamo.vllm_adapter import ( + from nemo_rl.models.generation.dynamo.vllm.adapter import ( SnapshotStatLogger, dynamo_kv_events_config, ) @@ -667,10 +667,7 @@ async def start_dynamo_endpoint_async( from nemo_rl.models.generation.dynamo.engine_adapter import ( start_dynamo_endpoint, ) - from nemo_rl.models.generation.dynamo.vllm_adapter import ( - DynamoVllmAdapter, - chat_worker_config, - ) + from nemo_rl.models.generation.dynamo.vllm.adapter import DynamoVllmAdapter # Dynamo fetches a Hub ID's whole checkpoint before it registers; the engine's # weights come from refit, so register the config and tokenizer files only. @@ -689,7 +686,6 @@ async def start_dynamo_endpoint_async( served_model_name=self.model_name, namespace=namespace, engine_env=engine_env, - chat_config=chat_worker_config(self), ) async def dynamo_endpoint_error(self) -> Optional[str]: diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 39084f01c7f..eb90910df14 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -208,12 +208,11 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: "control_timeout_s": "refit and pause use the vLLM backend's own controls", "metrics_include_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", "metrics_exclude_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", -} -_REMOVED_DYNAMO_WORKER_KEYS = { - "extra_cli_args": "engines are configured through vllm_cfg and vllm_kwargs", - "endpoint_types": ( - "the engines always serve /v1/completions (NeMo RL's rollouts) and chat " - "(NeMo-Gym)" + "worker_args": ( + "NeMo RL's chat engine in the frontend renders and parses NeMo-Gym's calls " + "with the native server's vllm_cfg.http_server_serving_chat_kwargs and " + "parser plugins, and engines are configured through vllm_cfg and " + "vllm_kwargs" ), } _REMOVED_DYNAMO_FRONTEND_KEYS = ( @@ -260,16 +259,9 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: if key in frontend_args: problems.append( f"policy.generation.dynamo_cfg.frontend_args.{key} was " - "removed: NeMo RL's rollouts send token IDs, which the " - "frontend never tokenizes." - ) - worker_args = dynamo_cfg.get("worker_args") - if isinstance(worker_args, dict): - for key, replacement in _REMOVED_DYNAMO_WORKER_KEYS.items(): - if key in worker_args: - problems.append( - f"policy.generation.dynamo_cfg.worker_args.{key} was " - f"removed: {replacement}." + "removed: NeMo RL never uses the frontend's tokenizer: " + "rollouts send token IDs, and NeMo-Gym's chat calls are " + "rendered by NeMo RL's own chat engine." ) if problems: raise ValueError(" ".join(problems)) diff --git a/pyrefly.toml b/pyrefly.toml index b1b864bf4d4..9f7cff41e1e 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -263,7 +263,12 @@ project-includes = [ "nemo_rl/models/generation/dynamo/config.py", "nemo_rl/models/generation/dynamo/managed_runtime.py", "nemo_rl/models/generation/dynamo/engine_adapter.py", - "nemo_rl/models/generation/dynamo/vllm_adapter.py", + "nemo_rl/models/generation/dynamo/vllm/__init__.py", + "nemo_rl/models/generation/dynamo/vllm/adapter.py", + "nemo_rl/models/generation/dynamo/vllm/chat_engine.py", + "nemo_rl/models/generation/dynamo/vllm/frontend.py", + "nemo_rl/models/generation/dynamo/vllm/render_pool.py", + "nemo_rl/models/generation/gym_call.py", "nemo_rl/models/generation/dynamo/rollout_client.py", "nemo_rl/models/generation/engine_supervisor.py", "nemo_rl/models/generation/fleet_health.py", @@ -275,6 +280,7 @@ project-includes = [ "nemo_rl/models/generation/megatron/token_capture.py", "nemo_rl/models/generation/megatron/utils.py", "nemo_rl/models/generation/openai_server_utils.py", + "nemo_rl/models/generation/vllm/chat_server.py", "nemo_rl/models/generation/rollout_result.py", "nemo_rl/models/generation/capture_context.py", "nemo_rl/models/generation/sglang/config.py", diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index f4e31f04241..bd056990e7e 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -922,27 +922,19 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> @pytest.mark.parametrize( ("vllm_model", "expected"), [ - ({}, {"enable_thinking": True, "truncate_history_thinking": False}), + # NeMo RL's chat engine renders with the native default template kwargs + # beneath these, as the native server does; Gym sends them as written. ( {"chat_template_kwargs": {"enable_thinking": False}}, - {"enable_thinking": False, "truncate_history_thinking": False}, - ), - ( - {"extra_body": {"chat_template_kwargs": {"force_nonempty_content": True}}}, - { - "enable_thinking": True, - "truncate_history_thinking": False, - "force_nonempty_content": True, - }, + {"enable_thinking": False}, ), # The frontend answers every request carrying bad_words with a 400. ({"extra_body": {"bad_words": ["x"]}}, None), ], ) -def test_gym_actor_config_dynamo_sends_the_native_default_template_kwargs( +def test_gym_actor_config_dynamo_turns_on_gyms_dynamo_dialect( vllm_model: dict, expected: dict | None ) -> None: - """The native server's default template kwargs sit under the ones Gym sends.""" from responses_api_models.vllm_model.app import VLLMModelConfig from nemo_rl.algorithms.grpo import MasterConfig diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py new file mode 100644 index 00000000000..f1a2824f008 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -0,0 +1,393 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NeMo RL's chat engine in Dynamo's frontend, over a fake render pool and router. + +The render pool's own workers are covered by ``test_dynamo_render_pool.py``; +here they are replaced by what they return, so each test checks what the +engine does between the render and the reply. +""" + +import asyncio +import json +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.vllm +pytest.importorskip("dynamo._core") +pytest.importorskip("vllm") + +from dynamo.llm.exceptions import HttpError # noqa: E402 + +from nemo_rl.models.generation.dynamo.vllm import chat_engine # noqa: E402 +from nemo_rl.models.generation.dynamo.vllm.render_pool import ( # noqa: E402 + RenderedCall, +) +from nemo_rl.models.generation.gym_call import ( # noqa: E402 + GYM_EXTRAS_ANNOTATION_PREFIX, + GymCallResult, +) + +EOS = 2 +MAX_MODEL_LEN = 16 +GENERATION = { + "temperature": 0.7, + "top_p": 0.9, + "top_k": None, + "val_temperature": 0.0, + "val_top_p": 1.0, + "val_top_k": None, + "stop_token_ids": [EOS], + "vllm_cfg": {}, +} +# Turn 2: the model's own turn 1 replaces the template's re-render of it. +PREFIX = [1, 5, 6, EOS] +HISTORY = [1, 7, EOS] +TEMPLATE = [*HISTORY, 3, 4] +SPLICED = [*PREFIX, 3, 4] +TOKEN_IN = {"mode": "token_in", "capture_key": "rollout-1", "model_call_id": "c2"} + + +def _rendered(**fields) -> RenderedCall: + """What a render-pool worker returns for a call.""" + return RenderedCall( + **{ + "template_token_ids": [1, 3, 4], + "history_token_ids": None, + "required_prefix_token_ids": None, + "admission": None, + "serving_admission": None, + "verify_retained_media": False, + "multimodal": None, + "request": {"model": "model", "max_tokens": None}, + "request_max_tokens": None, + "reasoning_ended": None, + "reasoning_parser_kwargs": None, + **fields, + } + ) + + +class _Pool: + def __init__(self, rendered: RenderedCall, reply: dict | None = None) -> None: + self.rendered = rendered + self.reply = reply or { + "choices": [ + { + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ] + } + self.bodies: list[dict] = [] + self.responded: list[tuple] = [] + + async def info(self) -> dict: + return {"turn_end_token_ids": [EOS], "max_model_len": MAX_MODEL_LEN} + + async def render(self, body: dict) -> RenderedCall: + self.bodies.append(body) + return self.rendered + + async def respond(self, request, prompt_token_ids, output) -> dict: + self.responded.append((request, prompt_token_ids, output)) + return self.reply + + +class _Item: + """One annotated response from Dynamo's router.""" + + def __init__(self, data=None, comments=None) -> None: + self._data, self._comments = data, comments + + def is_error(self) -> bool: + return self._comments is not None + + def data(self): + return self._data + + def comments(self): + return self._comments + + +class _Router: + def __init__(self, engine_data=None, error=None) -> None: + self.engine_data = engine_data or { + "output": {"text": "ok", "token_ids": [10], "finish_reason": "stop"}, + "result": {"prompt_token_ids": [1, 3, 4], "generation_token_ids": [10]}, + } + self.error = error + self.requests: list[dict] = [] + + async def generate(self, request, context=None): + self.requests.append(request) + + async def stream(): + if self.error is not None: + yield _Item(comments=[self.error]) + return + yield _Item({"token_ids": [10], "engine_data": self.engine_data}) + + return stream() + + +class _Capture: + """The TQ prefix source: returns the staged prefix for a token_in admission.""" + + def __init__(self) -> None: + self.resolved: list[dict] = [] + + def resolve_prefix(self, serving_admission: dict) -> list[int]: + self.resolved.append(serving_admission) + return PREFIX + + +def _call(extras: dict | None = None, **fields) -> dict: + """A chat call as Dynamo's frontend hands it over: unset fields are None.""" + nvext = None + if extras is not None: + nvext = {"annotations": [GYM_EXTRAS_ANNOTATION_PREFIX + json.dumps(extras)]} + return { + "model": "model", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": {"url": "u", "detail": None}}, + ], + } + ], + "temperature": 0.7, + "top_p": 0.9, + "top_k": None, + "n": None, + "min_p": None, + "presence_penalty": None, + "logit_bias": None, + "max_tokens": 20, + "chat_template_args": None, + "unsupported_fields": None, + "nvext": nvext, + **fields, + } + + +def _serve(pool, router, request, capture=None) -> list[dict]: + engine = chat_engine.NemoRLChatEngine( + pool=pool, + routed_engine=router, + spec={"generation": GENERATION}, + capture=capture or _Capture(), + ) + + async def run(): + return [chunk async for chunk in engine.generate(request, None)] + + return asyncio.run(run()) + + +def test_a_gym_call_is_rendered_spliced_and_routed_with_its_capture_plan() -> None: + pool = _Pool( + _rendered( + template_token_ids=TEMPLATE, + history_token_ids=HISTORY, + required_prefix_token_ids=PREFIX, + request={"model": "model", "max_tokens": 20, "max_completion_tokens": None}, + request_max_tokens=20, + reasoning_ended=False, + ), + reply={ + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "reasoning": "thinking", + "tool_calls": [ + { + "id": "call-0", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": {"prompt_tokens": 6, "completion_tokens": 1, "total_tokens": 7}, + }, + ) + router = _Router() + + (chunk,) = _serve( + pool, + router, + _call( + {"required_prefix_token_ids": PREFIX}, + chat_template_args={"enable_thinking": True}, + ), + ) + + # The worker renders the call without Dynamo's None fields, with the + # profile's exact sampling and Gym's extras as the native request fields. + (body,) = pool.bodies + assert body["messages"][0]["content"][1] == { + "type": "image_url", + "image_url": {"url": "u"}, + } + assert {"n", "logit_bias", "nvext", "chat_template_args"}.isdisjoint(body) + assert body["chat_template_kwargs"] == {"enable_thinking": True} + assert (body["temperature"], body["top_p"], body["top_k"]) == (0.7, 0.9, -1) + assert (body["min_p"], body["repetition_penalty"]) == (0.0, 1.0) + assert (body["ng_capture"], body["required_prefix_token_ids"]) == (None, PREFIX) + # The router gets the spliced prompt, the budget clamped to the context, + # and the plan the engine's adapter serves the call by. + (routed,) = router.requests + assert routed["token_ids"] == SPLICED + assert routed["stop_conditions"] == {"max_tokens": MAX_MODEL_LEN - len(SPLICED)} + assert routed["annotations"] == [chat_engine.GYM_CALL_ANNOTATION] + plan = routed["extra_args"][chat_engine.PLAN_KEY] + assert plan["request"] == { + "model": "model", + "max_tokens": MAX_MODEL_LEN - len(SPLICED), + "max_completion_tokens": None, + } + assert (plan["template_token_ids"], plan["splice"], plan["prefix_len"]) == ( + TEMPLATE, + # Each side is cut at its turn end: PREFIX[:3] + TEMPLATE[2:]. + [3, 2], + len(PREFIX), + ) + assert (plan["admission"], plan["multimodal"], plan["reasoning_ended"]) == ( + None, + None, + False, + ) + # The reply is vLLM's, built from the engine's output, with the result in nvext. + ((_, prompt_token_ids, output),) = pool.responded + assert (prompt_token_ids, output) == (SPLICED, router.engine_data["output"]) + (choice,) = chunk["choices"] + assert choice["delta"]["reasoning_content"] == "thinking" + assert choice["delta"]["tool_calls"][0]["index"] == 0 + assert choice["finish_reason"] == "tool_calls" + assert chunk["nvext"] == {"nemo_rl": router.engine_data["result"]} + assert chunk["usage"]["prompt_tokens"] == 6 + + +def test_a_token_in_call_splices_the_prefix_staged_in_tq() -> None: + pool = _Pool( + _rendered( + template_token_ids=TEMPLATE, + history_token_ids=HISTORY, + admission=TOKEN_IN, + serving_admission=TOKEN_IN, + ) + ) + router = _Router() + capture = _Capture() + + _serve(pool, router, _call({"ng_capture": TOKEN_IN}), capture) + + assert pool.bodies[0]["ng_capture"] == TOKEN_IN + assert capture.resolved == [TOKEN_IN] + (routed,) = router.requests + assert routed["token_ids"] == SPLICED + plan = routed["extra_args"][chat_engine.PLAN_KEY] + assert (plan["admission"], plan["serving_admission"], plan["prefix_len"]) == ( + TOKEN_IN, + TOKEN_IN, + len(PREFIX), + ) + + +def test_a_measure_call_returns_the_prompt_length_without_generating() -> None: + pool = _Pool( + _rendered( + template_token_ids=TEMPLATE, + history_token_ids=HISTORY, + required_prefix_token_ids=PREFIX, + ) + ) + router = _Router() + + (chunk,) = _serve( + pool, + router, + _call({"required_prefix_token_ids": PREFIX, "ng_measure": True}), + ) + + # The length of the prompt the call would run, as spliced. + assert chunk["nvext"] == { + "nemo_rl": GymCallResult( + prompt_token_count=len(SPLICED), ng_context_measured=True + ).model_dump(exclude_none=True) + } + assert (router.requests, pool.responded) == ([], []) + + +@pytest.mark.parametrize( + ("request_fields", "rendered", "error"), + [ + ({"extras": None}, {}, "nemo_rl:gym: annotation"), + ({"temperature": 0.5}, {}, "matches neither"), + ({"presence_penalty": 0.5}, {}, "presence_penalty=0.5"), + ({"n": 2}, {}, "n=1"), + # The echoed prefix, spliced in, leaves no room for output. + ( + {}, + { + "template_token_ids": TEMPLATE, + "history_token_ids": HISTORY, + "required_prefix_token_ids": [1, *range(20, 20 + MAX_MODEL_LEN), EOS], + }, + "maximum context length", + ), + ], +) +def test_gym_calls_nemo_rl_cannot_serve_are_refused_with_a_400( + request_fields, rendered, error +) -> None: + pool = _Pool(_rendered(**rendered)) + router = _Router() + request_fields = {"extras": {}, **request_fields} + + with pytest.raises(HttpError, match=error) as refused: + _serve(pool, router, _call(**request_fields)) + + assert refused.value.code == 400 + assert router.requests == [] + + +def test_an_engine_error_fails_the_call() -> None: + with pytest.raises(HttpError, match="engine died") as failed: + _serve(_Pool(_rendered()), _Router(error="engine died"), _call({})) + + assert failed.value.code == 500 + + +def test_only_nemo_rls_engines_are_served() -> None: + model = SimpleNamespace( + runtime_config=lambda: {"runtime_data": {}}, + name=lambda: "other-model", + local_dir=lambda: "/snapshots/other", + ) + + async def build(): + return await chat_engine.ChatEngineFactory().chat_engine_factory( + 0, model, _Router() + ) + + with pytest.raises(RuntimeError, match="without NeMo RL's chat settings"): + asyncio.run(build()) diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 379efd5c66d..f04662d1dfc 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -18,9 +18,6 @@ DynamoCfg, DynamoFrontendArgs, dynamo_cfg, - dynamo_chat_template, - dynamo_parsers, - native_default_chat_template_kwargs, ) from nemo_rl.models.generation.dynamo.managed_runtime import frontend_argv @@ -99,8 +96,6 @@ def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: ), "checks no bearer token", ), - # Dynamo's token capture stages no sampler top-k for score centering. - (_vllm_config(dynamo=True, top_logprobs=4), "sampler top-k"), # Direct routing needs a worker ID that rollouts never carry. ( _vllm_config( @@ -108,6 +103,16 @@ def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: ), "router_mode", ), + # NeMo-Gym's chat calls are NeMo RL's chat engine's, never Dynamo's. + ( + _vllm_config( + dynamo=True, + dynamo_cfg={ + "frontend_args": {"extra_cli_args": ["--dyn-chat-processor=vllm"]} + }, + ), + "may not set --dyn-chat-processor", + ), ], ) def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( @@ -142,82 +147,3 @@ def test_frontend_argv_is_runtime_owned() -> None: frontend_argv( host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=clashing ) - - -def _serving(**serving_chat_kwargs) -> dict: - return _vllm_config( - dynamo=True, - vllm_cfg={ - "async_engine": True, - "max_model_len": 512, - "tensor_parallel_size": 1, - "expert_parallel_size": 1, - "http_server_serving_chat_kwargs": serving_chat_kwargs, - }, - ) - - -def test_parsers_follow_the_native_server_under_dynamos_names() -> None: - assert dynamo_parsers(_serving(tool_parser="hermes", reasoning_parser="qwen3")) == ( - "hermes", - "qwen3", - ) - # NeMo RL's nano_v3 vLLM plugin is Dynamo's nemotron_v3 parser. - nano_v3 = _serving(reasoning_parser="nano_v3") - nano_v3["vllm_cfg"]["reasoning_parser_plugin"] = "nano_v3_reasoning_parser.py" - assert dynamo_parsers(nano_v3) == (None, "nemotron_v3") - assert dynamo_parsers(_serving(tool_parser="openai")) == ("harmony", None) - # dynamo_cfg names them explicitly. - explicit = _serving(tool_parser="hermes") - explicit["dynamo_cfg"] = {"worker_args": {"tool_call_parser": "qwen3_coder"}} - assert dynamo_parsers(explicit) == ("qwen3_coder", None) - - -def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> None: - config = _serving(tool_parser="my_parser") - config["vllm_cfg"]["tool_parser_plugin"] = "my_parser.py" - - with pytest.raises(ValueError, match="worker_args.tool_call_parser"): - configure_generation_config(config, _Tokenizer()) - - -def test_chat_template_and_default_kwargs_follow_the_native_server() -> None: - config = _serving( - chat_template="templates/chat.jinja", - default_chat_template_kwargs={"enable_thinking": False}, - ) - assert dynamo_chat_template(config) == "templates/chat.jinja" - assert native_default_chat_template_kwargs(config) == {"enable_thinking": False} - # The older spelling the native server also accepts. - legacy = _serving(chat_template_kwargs={"enable_thinking": True}) - assert native_default_chat_template_kwargs(legacy) == {"enable_thinking": True} - assert dynamo_chat_template(_serving()) is None - - -@pytest.mark.parametrize( - ("serving_chat_kwargs", "error"), - [ - ({"chat_template_content_format": "string"}, "no content-format setting"), - ({"enable_auto_tools": False, "tool_parser": "hermes"}, "always parses"), - ], -) -def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( - serving_chat_kwargs, error -) -> None: - with pytest.raises(ValueError, match=error): - configure_generation_config(_serving(**serving_chat_kwargs), _Tokenizer()) - # The native defaults are what the frontend does anyway, and with no tool - # parser there are no tool calls for auto tools to change. - for accepted in ( - _serving(chat_template_content_format="auto", enable_auto_tools=True), - _serving(enable_auto_tools=False, tool_parser=None), - ): - configure_generation_config(accepted, _Tokenizer()) - - -def test_a_bearer_token_is_rejected_because_the_frontend_checks_none() -> None: - config = _vllm_config(dynamo=True) - config["vllm_cfg"]["http_server_api_key_required"] = True - - with pytest.raises(ValueError, match="checks no bearer token"): - configure_generation_config(config, _Tokenizer()) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index c6dde7fcf51..639f25fc404 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +import pickle import signal import subprocess import sys @@ -19,6 +20,7 @@ import pytest +from nemo_rl.models.generation.dynamo import managed_runtime from nemo_rl.models.generation.dynamo.config import DynamoCfg, DynamoFrontendArgs from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, @@ -34,6 +36,7 @@ def _runtime(**frontend_args) -> ManagedDynamoRuntime: ), model_name="model", frontend_python="/venvs/vllm/bin/python", + ray_address="10.0.0.1:6379", ) @@ -285,19 +288,42 @@ def check_engines() -> None: runtime.wait_for_engines(1, check_engines=check_engines) -def test_frontend_launches_from_the_given_interpreter(monkeypatch) -> None: - runtime = _runtime() +def test_frontend_launches_nemo_rls_chat_engine_from_the_given_interpreter( + monkeypatch, +) -> None: + runtime = _runtime(render_workers=3) runtime._frontend_port = 3000 runtime._namespace = "nemo-rl-test" launched = [] monkeypatch.setattr( "nemo_rl.models.generation.dynamo.managed_runtime.subprocess.Popen", - lambda command, **kwargs: launched.append(command) or _FakeProcess(), + lambda command, **kwargs: launched.append((command, kwargs["env"])) + or _FakeProcess(), ) runtime._start_frontend() + runtime.publish_chat_capture( + dp_cfg={"backend": "simple"}, staging_partition="staging", capture_media=False + ) - assert launched[0][:3] == ["/venvs/vllm/bin/python", "-m", "dynamo.frontend"] + ((command, env),) = launched + # Dynamo's frontend, with NeMo RL's chat engine serving chat calls. + assert command[:3] == [ + "/venvs/vllm/bin/python", + "-m", + "nemo_rl.models.generation.dynamo.vllm.frontend", + ] + assert env[managed_runtime.RENDER_WORKERS_ENV] == "3" + assert env[managed_runtime.RAY_ADDRESS_ENV] == "10.0.0.1:6379" + # Capture is set up after the engines register; the chat engine reads it + # from the file the frontend was told about. + with open(env[managed_runtime.CAPTURE_FILE_ENV], "rb") as capture_file: + assert pickle.load(capture_file) == { + "data_plane": {"backend": "simple"}, + "staging_partition": "staging", + "capture_media": False, + } + runtime.shutdown() def test_shutdown_logs_the_kv_events_the_router_applied(monkeypatch, capsys) -> None: diff --git a/tests/unit/models/generation/test_dynamo_render_pool.py b/tests/unit/models/generation/test_dynamo_render_pool.py new file mode 100644 index 00000000000..e4ecd4ccb82 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_render_pool.py @@ -0,0 +1,152 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The Dynamo frontend's render pool, over a tiny Qwen3 checkpoint. + +The pool is spawned as in the frontend, so the spawn context, pickling and +parser registration are exercised; renders are checked against the +checkpoint's own chat template, replies against vLLM's Qwen3 parsers. +""" + +import asyncio +import json + +import pytest + +pytestmark = pytest.mark.vllm +pytest.importorskip("vllm") + +from transformers import AutoTokenizer # noqa: E402 + +from nemo_rl.models.generation.dynamo.vllm.render_pool import RenderPool # noqa: E402 + +MAX_MODEL_LEN = 512 +SPEC = { + # The engine's own args, as it publishes them at registration. + "engine_args": {"max_model_len": MAX_MODEL_LEN}, + "generation": { + "temperature": 0.7, + "top_p": 0.9, + "top_k": None, + "val_temperature": 0.0, + "val_top_p": 1.0, + "val_top_k": None, + "stop_token_ids": None, + "vllm_cfg": { + "http_server_serving_chat_kwargs": { + "tool_parser": "hermes", + "reasoning_parser": "qwen3", + "enable_auto_tools": True, + } + }, + }, +} +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "The weather in a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] +MESSAGES = [ + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call-0", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call-0", "content": "sunny"}, +] + + +def _template_ids(tokenizer, messages, *, add_generation_prompt: bool) -> list[int]: + text = tokenizer.apply_chat_template( + messages, + tools=TOOLS, + add_generation_prompt=add_generation_prompt, + tokenize=False, + ) + return tokenizer.encode(text, add_special_tokens=False) + + +def test_the_pool_renders_and_answers_with_the_checkpoints_template_and_parsers( + tiny_qwen3_model_path, +) -> None: + tokenizer = AutoTokenizer.from_pretrained(tiny_qwen3_model_path) + reply_text = ( + "\nchecking\n\n\n\n" + '{"name": "get_weather", "arguments": {"city": "Lyon"}}\n' + ) + reply_ids = tokenizer.encode(reply_text, add_special_tokens=False) + + async def serve(): + pool = RenderPool(SPEC, tiny_qwen3_model_path, workers=1) + try: + info = await pool.info() + rendered = await pool.render( + { + "model": "model", + "messages": MESSAGES, + "tools": TOOLS, + "max_tokens": 64, + "temperature": 0.7, + "top_p": 0.9, + # Gym's echo of the last assistant turn, without capture. + "required_prefix_token_ids": [1, 2, 3], + } + ) + reply = await pool.respond( + rendered.request, + rendered.template_token_ids, + {"text": reply_text, "token_ids": reply_ids, "finish_reason": "stop"}, + ) + finally: + pool.shutdown() + return info, rendered, reply + + info, rendered, reply = asyncio.run(serve()) + + # Turns end on <|im_end|>, which the splice in the frontend cuts at. + assert tokenizer.convert_tokens_to_ids("<|im_end|>") in info["turn_end_token_ids"] + assert info["max_model_len"] == MAX_MODEL_LEN + # The call, and the history up to its last assistant turn the splice cuts against. + assert rendered.template_token_ids == _template_ids( + tokenizer, MESSAGES, add_generation_prompt=True + ) + assert rendered.history_token_ids == _template_ids( + tokenizer, MESSAGES[:2], add_generation_prompt=False + ) + assert rendered.required_prefix_token_ids == [1, 2, 3] + assert rendered.request_max_tokens == 64 + assert "messages" not in rendered.request + assert (rendered.admission, rendered.multimodal) == (None, None) + # The reply through vLLM's own reasoning and tool parsers. + (choice,) = reply["choices"] + assert choice["finish_reason"] == "tool_calls" + assert choice["message"]["reasoning"].strip() == "checking" + (tool_call,) = choice["message"]["tool_calls"] + assert tool_call["function"]["name"] == "get_weather" + assert json.loads(tool_call["function"]["arguments"]) == {"city": "Lyon"} diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index ce63c1c9cd7..7305346757e 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -14,22 +14,21 @@ """The vLLM Dynamo adapter over a fake AsyncLLM. The host is a real ``VllmAsyncGenerationWorkerImpl`` built without its -constructor, so rollouts are sampled by the native worker's own helpers. +constructor, so rollouts and NeMo-Gym calls run the native worker's own helpers. """ import ast import asyncio import base64 import importlib.util -import io import json +import pickle from pathlib import Path from types import SimpleNamespace import numpy as np import pytest import torch -from PIL import Image pytestmark = pytest.mark.vllm pytest.importorskip("dynamo._core") @@ -38,36 +37,43 @@ from dynamo.llm.exceptions import InvalidArgument # noqa: E402 from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 -from nemo_rl.models.generation.dynamo import ( # noqa: E402 - engine_adapter, - vllm_adapter, -) +from nemo_rl.models.generation.dynamo import engine_adapter # noqa: E402 from nemo_rl.models.generation.dynamo.config import ( # noqa: E402 DynamoCfg, DynamoFrontendArgs, - DynamoWorkerArgs, ) from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) +from nemo_rl.models.generation.dynamo.vllm import ( # noqa: E402 + adapter as vllm_adapter, +) +from nemo_rl.models.generation.dynamo.vllm.chat_engine import ( # noqa: E402 + GYM_CALL_ANNOTATION, + PLAN_KEY, + RUNTIME_DATA_KEY, +) from nemo_rl.models.generation.rollout_result import ( # noqa: E402 rollout_result_from_wire, rollout_result_to_wire, ) +from nemo_rl.models.generation.vllm.chat_server import ( # noqa: E402 + NeMoRLChatCompletionRequest, +) from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, ) +from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) +from nemo_rl.utils.routed_experts_codec import decode_routed_experts # noqa: E402 MAX_MODEL_LEN = 16 - - EOS = 2 -# The fake processor expands each image placeholder to IMAGE_TOKENS image tokens. -IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS = 90, 91, 3 +# A processed image's placeholder tokens. +IMAGE_TOKEN, IMAGE_TOKENS = 91, 3 class _Context: @@ -78,23 +84,18 @@ def is_stopped(self) -> bool: return False -class _Tokenizer: - eos_token_id = EOS - - def decode(self, token_ids) -> str: - return "".join(f"<{token_id}>" for token_id in token_ids) - - -def _logprob(value: float): - return SimpleNamespace(logprob=value, rank=1, decoded_token=None) +def _logprob(value: float, rank: int = 1): + return SimpleNamespace(logprob=value, rank=rank, decoded_token=None) def _completion(token_ids, finish_reason=None, index=0, routed_experts=None): return SimpleNamespace( index=index, + text="".join(f"<{token_id}>" for token_id in token_ids), token_ids=token_ids, logprobs=[{t: _logprob(-0.5)} for t in token_ids], finish_reason=finish_reason, + stop_reason=None, routed_experts=routed_experts, ) @@ -109,31 +110,6 @@ def _output(prompt, *completions, prompt_routed_experts=None, prompt_logprobs=No ) -class _FakeRenderer: - """vLLM's processing step: expands each image placeholder, as a VLM processor does.""" - - def __init__(self) -> None: - self.prompts = [] - - async def process_for_engine_async(self, prompt, arrival_time, *, skip_mm_cache): - self.prompts.append((prompt, skip_mm_cache)) - if "multi_modal_data" not in prompt: - return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} - tokens, placeholders = [], [] - for token in prompt["prompt_token_ids"]: - if token == IMAGE_PLACEHOLDER: - placeholders.append(PlaceholderRange(len(tokens), IMAGE_TOKENS)) - tokens.extend([IMAGE_TOKEN] * IMAGE_TOKENS) - else: - tokens.append(token) - return { - "type": "multimodal", - "prompt_token_ids": tokens, - "mm_kwargs": {"image": [object() for _ in placeholders]}, - "mm_placeholders": {"image": placeholders}, - } - - class _FakeLLM: """Yields scripted RequestOutputs and records what it was asked to generate.""" @@ -142,13 +118,11 @@ def __init__(self, outputs=(), *, defaults=None, admission_error=None): self.calls = [] self.admission_error = admission_error self.tokenizer = None - self.renderer = _FakeRenderer() self.model_config = SimpleNamespace( get_diff_sampling_param=lambda: dict(defaults or {}), try_get_generation_config=lambda: {}, - multimodal_config=None, - allowed_local_media_path="", - allowed_media_domains=None, + generation_config="auto", + override_generation_config={}, ) self.vllm_config = SimpleNamespace( model_config=SimpleNamespace(max_model_len=MAX_MODEL_LEN), @@ -159,8 +133,8 @@ def check_admission(self, n, request_id): if self.admission_error is not None: raise self.admission_error - async def generate(self, prompt, sampling_params, request_id): - self.calls.append((prompt, sampling_params, request_id)) + async def generate(self, prompt, sampling_params, request_id, **kwargs): + self.calls.append((prompt, sampling_params, request_id, kwargs)) for output in self.outputs: yield output @@ -175,7 +149,7 @@ def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: "val_top_p": 1.0, "top_k": None, "val_top_k": None, - "stop_token_ids": [2], + "stop_token_ids": [EOS], "stop_strings": [""], "vllm_cfg": {"max_model_len": MAX_MODEL_LEN}, **cfg, @@ -184,7 +158,10 @@ def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: host.model_name = "model" host.routed_experts_dtype = torch.int16 host.llm = llm + host._http_engine_client = None host._capture_media = False + host._capture_calls = {} + host._capture_top_logprobs = 0 host.token_capture = None return host @@ -213,7 +190,7 @@ def test_rollouts_are_served_like_generate_async() -> None: engine, _rollout_request([1, 2, 3], greedy=True, stop_strings=[""]) ) - ((_, sampling_params, _),) = llm.calls + ((_, sampling_params, _, _),) = llm.calls # The worker's _serve_rollout samples it, as it does for generate_async... assert sampling_params.max_tokens == 8 assert sampling_params.temperature == 0.0 @@ -247,7 +224,7 @@ async def abort(request_id): # Dynamo's worker calls abort() when the frontend cancels a request. asyncio.run(engine.abort(_Context())) - ((_, _, request_id),) = llm.calls + ((_, _, request_id, _),) = llm.calls assert aborted == [request_id] @@ -325,7 +302,6 @@ async def start(): served_model_name=host.model_name, namespace="ns", engine_env=env, - chat_config=vllm_adapter.chat_worker_config(host), ) asyncio.run(start()) @@ -333,24 +309,8 @@ async def start(): return config -def test_every_worker_arg_reaches_the_dynamo_worker_config( - monkeypatch, tmp_path -) -> None: - template = tmp_path / "chat.jinja" - template.write_text("{{ messages }}") - worker_args = { - "tool_call_parser": "qwen3_coder", - "reasoning_parser": "nemotron_nano", - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": True, - "structural_tag_scope": "always", - "structural_tag_schema": "strict", - "custom_jinja_template": str(template), - } - assert set(worker_args) == set(DynamoWorkerArgs.model_fields) - host = _host(_FakeLLM(), dynamo_cfg={"worker_args": worker_args}) - - config = _start_worker_config(monkeypatch, host) +def test_engines_register_for_rollouts_and_nemo_gym_chat(monkeypatch) -> None: + config = _start_worker_config(monkeypatch, _host(_FakeLLM())) assert config.namespace == "ns" # Registered from the weight-free snapshot, served under the model's name. @@ -360,55 +320,18 @@ def test_every_worker_arg_reaches_the_dynamo_worker_config( ) # NeMo RL's rollouts go to /v1/completions, NeMo-Gym's calls to chat. assert config.endpoint_types == "chat,completions" - assert config.structural_tag_mode == "on" - with open(config.custom_jinja_template) as template_file: - assert template_file.read() == "{{ messages }}" - for key in worker_args.keys() - {"enable_structural_tag", "custom_jinja_template"}: - assert getattr(config, key) == worker_args[key], key -def test_requests_other_than_rollouts_are_refused() -> None: +def test_requests_other_than_nemo_rls_are_refused() -> None: llm = _FakeLLM() engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) - with pytest.raises(InvalidArgument, match="only NeMo RL rollouts"): + with pytest.raises(InvalidArgument, match="only NeMo RL's own requests"): _collect(engine, {"token_ids": [1, 2, 3, 4]}) assert llm.calls == [] -def test_the_native_servers_chat_settings_reach_the_frontend(monkeypatch) -> None: - host = _host(_FakeLLM()) - host.cfg["vllm_cfg"] = { - "max_model_len": MAX_MODEL_LEN, - "reasoning_parser_plugin": "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", - "http_server_serving_chat_kwargs": { - "tool_parser": "qwen3_coder", - "reasoning_parser": "nano_v3", - # Literal Jinja, which the native server also accepts. - "chat_template": "{{ messages }}\n", - }, - } - - config = _start_worker_config(monkeypatch, host) - - assert (config.tool_call_parser, config.reasoning_parser) == ( - "qwen3_coder", - "nemotron_v3", - ) - with open(config.custom_jinja_template) as template_file: - assert template_file.read() == "{{ messages }}\n" - - -def test_a_parser_dynamo_lacks_fails_registration(monkeypatch) -> None: - host = _host( - _FakeLLM(), dynamo_cfg={"worker_args": {"tool_call_parser": "no_such_parser"}} - ) - - with pytest.raises(ValueError, match="no tool call parser named 'no_such_parser'"): - _start_worker_config(monkeypatch, host) - - def test_an_engine_error_reaches_the_driver_with_its_cause() -> None: host = _host(_FakeLLM()) @@ -460,199 +383,265 @@ async def call_utility_async(method): ] llm.engine_core = SimpleNamespace(call_utility_async=call_utility_async) - engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) + host = _host(llm) + host.llm_async_engine_args = vllm.AsyncEngineArgs(model="/models/m") + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) - registration = asyncio.run(engine.start(worker_id=0)).llm + config = asyncio.run(engine.start(worker_id=0)) + registration = config.llm # The KV router hashes events in blocks of the main-attention group; a # wrong size would quietly turn KV routing into load-only routing. assert registration.kv_cache_block_size == 528 assert registration.total_kv_blocks == 64 assert registration.context_length == MAX_MODEL_LEN + # The frontend's chat engine renders with what the engine publishes. + assert config.runtime_data == vllm_adapter.chat_runtime_data(host) -def _image_url() -> str: - buffer = io.BytesIO() - Image.new("RGB", (8, 8), color="red").save(buffer, format="PNG") - return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() +def test_engines_publish_what_the_frontends_chat_engine_renders_with() -> None: + host = _host(_FakeLLM()) + host.llm_async_engine_args = vllm.AsyncEngineArgs( + model="/models/m", + max_model_len=MAX_MODEL_LEN, + enable_prefix_caching=True, + skip_tokenizer_init=True, + ) + serving_chat_kwargs = {"tool_parser": "hermes", "enable_auto_tools": True} + host.cfg["vllm_cfg"] = { + "max_model_len": MAX_MODEL_LEN, + "http_server_serving_chat_kwargs": serving_chat_kwargs, + "reasoning_parser_plugin": "/plugins/reasoning.py", + } + # Dynamo carries runtime_data as JSON. + spec = json.loads(json.dumps(vllm_adapter.chat_runtime_data(host)))[ + RUNTIME_DATA_KEY + ] -def test_media_is_loaded_and_processed_as_vllms_chat_parser_does() -> None: - final = _output([1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3], _completion([5], "stop")) - llm = _FakeLLM([final]) - engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) + # The engine's own args rebuild the same model config, tokenizer included. + published = vllm.AsyncEngineArgs(**spec["engine_args"]) + for field in ("model", "max_model_len", "enable_prefix_caching"): + assert getattr(published, field) == getattr( + host.llm_async_engine_args, field + ), field + assert published.skip_tokenizer_init is False + # The native server's chat settings and the sampling profiles it checks. + assert spec["generation"] == { + "temperature": 0.7, + "top_p": 0.9, + "top_k": None, + "val_temperature": 0.0, + "val_top_p": 1.0, + "val_top_k": None, + "stop_token_ids": [EOS], + "vllm_cfg": { + "http_server_serving_chat_kwargs": serving_chat_kwargs, + "reasoning_parser_plugin": "/plugins/reasoning.py", + }, + } - _collect( - engine, - _gym_request( - token_ids=[1, IMAGE_PLACEHOLDER, 3], - multi_modal_data={"image_url": [{"Url": _image_url()}]}, - mm_processor_kwargs={"max_num_tiles": 2}, - ), - ) - ((prompt, skip_mm_cache),) = llm.renderer.prompts - # The frontend's unexpanded tokens, the decoded image and the processor kwargs. - assert prompt["prompt_token_ids"] == [1, IMAGE_PLACEHOLDER, 3] - (image,) = prompt["multi_modal_data"]["image"] - assert image.size == (8, 8) and image.mode == "RGB" - assert prompt["multi_modal_uuids"] == {"image": [None]} - assert prompt["mm_processor_kwargs"] == {"max_num_tiles": 2} - assert skip_mm_cache is False - # vLLM generates from the processed prompt, and the output cap counts its tokens. - ((engine_input, params, _),) = llm.calls - assert engine_input["prompt_token_ids"] == [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] - assert params.max_tokens == MAX_MODEL_LEN - 2 - IMAGE_TOKENS - - -# --- NeMo-Gym's Dynamo dialect ------------------------------------------------- - - -def _gym_request( - *, temperature=0.7, top_p=0.9, prefix=None, sampling=None, **fields -) -> dict: - annotations = [vllm_adapter.GYM_ROLLOUT_ANNOTATION] - if prefix is not None: - annotations.insert( - 0, vllm_adapter.GYM_PREFIX_ANNOTATION_PREFIX + json.dumps(prefix) - ) - sampling_options = {"temperature": temperature, "top_p": top_p, **(sampling or {})} - return { - "token_ids": [1, 3, 4], - # The frontend carries sampling floats as f32. - "sampling_options": { - key: vllm_adapter._as_f32(value) if isinstance(value, float) else value - for key, value in sampling_options.items() - }, - # Gym's token information: logprobs=true, top_logprobs=0. - "output_options": {"logprobs": 0}, - "annotations": annotations, +# --- NeMo-Gym calls, after the frontend's chat engine rendered them ----------- + + +def _chat(**fields) -> dict: + """A rendered chat request, as the chat engine sends it (without messages).""" + return NeMoRLChatCompletionRequest( + model="model", + messages=[{"role": "user", "content": "hi"}], + temperature=0.7, + top_p=0.9, **fields, + ).model_dump(mode="json", exclude={"messages"}) + + +def _gym_call(token_ids: list[int], **plan) -> dict: + """A Gym call as the chat engine routes it: final token ids and the plan.""" + plan = { + "request": _chat(), + "admission": None, + "serving_admission": None, + "verify_retained_media": False, + "prefix_len": None, + "template_token_ids": token_ids, + "splice": None, + "multimodal": None, + "reasoning_ended": None, + "reasoning_parser_kwargs": None, + **plan, + } + return { + "token_ids": token_ids, + "annotations": [GYM_CALL_ANNOTATION], + "extra_args": {PLAN_KEY: plan}, } @pytest.mark.parametrize( - ("temperature", "top_p", "skip_special_tokens"), - [ - # The train profile; the frontend left skip_special_tokens unset. - (0.7, 0.9, None), - # The validation profile; the frontend decodes special tokens. - (0.0, 1.0, False), - ], + ("max_tokens", "served_max_tokens"), [(5, 5), (None, MAX_MODEL_LEN - 3)] ) -def test_gym_calls_sample_with_nemo_rls_exact_profile( - temperature, top_p, skip_special_tokens +def test_gym_calls_generate_with_the_native_servers_sampling_params( + max_tokens, served_max_tokens ) -> None: final = _output([1, 3, 4], _completion([10, 11], "stop")) llm = _FakeLLM([final]) engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) - output_options = {"logprobs": 0} - if skip_special_tokens is not None: - output_options["skip_special_tokens"] = skip_special_tokens + reasoning = {"chat_template_kwargs": {"enable_thinking": True}} (chunk,) = _collect( engine, - _gym_request( - temperature=temperature, - top_p=top_p, - stop_conditions={"max_tokens": 100, "stop": [""]}, - output_options=output_options, + _gym_call( + [1, 3, 4], + # Gym's token information: logprobs=true, top_logprobs=0. + request=_chat( + max_tokens=max_tokens, stop=[""], logprobs=True, top_logprobs=0 + ), + reasoning_ended=False, + reasoning_parser_kwargs=reasoning, ), ) - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == [1, 3, 4] - # The configured values, not the f32 values the frontend delivered. - assert (params.temperature, params.top_p) == (temperature, top_p) - # The profile's top_k (unset here), as vLLM normalizes it for this temperature, - # and none of the generation config's min_p or penalties. - assert params.top_k == vllm.SamplingParams(temperature=temperature, top_k=-1).top_k - assert (params.min_p, params.repetition_penalty) == (0.0, 1.0) + ((prompt, params, _, kwargs),) = llm.calls + assert prompt == {"prompt_token_ids": [1, 3, 4]} + # The request's own sampling, as vLLM's chat server builds it. + assert (params.temperature, params.top_p) == (0.7, 0.9) + assert (params.max_tokens, params.stop, params.logprobs) == ( + served_max_tokens, + [""], + 0, + ) assert params.output_kind == vllm.sampling_params.RequestOutputKind.FINAL_ONLY - # Output fits the context, and vLLM stops on the stop strings itself, - # decoding special tokens as the frontend does. - assert params.max_tokens == MAX_MODEL_LEN - 3 - assert (params.stop, params.detokenize) == ([""], True) - assert params.skip_special_tokens is (skip_special_tokens is not False) + # What vLLM's chat serving hands the engine besides the prompt. + assert kwargs == {"reasoning_ended": False, "reasoning_parser_kwargs": reasoning} + # The output the frontend builds the reply from, and the call's result. assert chunk["engine_data"] == { - "prompt_token_ids": [1, 3, 4], - "completion_token_ids": [10, 11], - "completion_logprobs": [-0.5, -0.5], + "output": { + "text": "<10><11>", + "token_ids": [10, 11], + "finish_reason": "stop", + "stop_reason": None, + "logprobs": [{"10": [-0.5, 1, None]}, {"11": [-0.5, 1, None]}], + }, + "result": { + "prompt_token_ids": [1, 3, 4], + "generation_token_ids": [10, 11], + "generation_log_probs": [-0.5, -0.5], + }, } - assert (chunk["token_ids"], chunk["finish_reason"]) == ([10, 11], "stop") - assert chunk["log_probs"] == [-0.5, -0.5] + assert chunk["completion_usage"]["prompt_tokens"] == 3 + + +def test_an_uncaptured_gym_call_carries_its_routes() -> None: + # vLLM routes every token but the last sampled one. + routes = np.arange(4 * 2 * 2, dtype=np.int16).reshape(4, 2, 2) + final = _output( + [1, 3, 4], + _completion([10, 11], "stop", routed_experts=routes[3:]), + prompt_routed_experts=routes[:3], + ) + host = _host(_FakeLLM([final]), vllm_kwargs={"enable_return_routed_experts": True}) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) + + (chunk,) = _collect(engine, _gym_call([1, 3, 4])) + + # As the native endpoint attaches them: one row per token of the sequence. + expected = torch.full((5, 2, 2), R3_MISSING_ROUTE_SENTINEL, dtype=torch.int16) + expected[:4] = torch.from_numpy(routes) + shipped = chunk["engine_data"]["result"]["routed_experts"] + assert torch.equal(decode_routed_experts(shipped, dtype=None), expected) + + +def test_a_gym_call_without_routes_fails_under_router_replay() -> None: + final = _output([1, 3, 4], _completion([10, 11], "stop")) + host = _host(_FakeLLM([final]), vllm_kwargs={"enable_return_routed_experts": True}) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) + + # Training would replay the router on routes the call never returned. + with pytest.raises(InvalidArgument, match="did not include routed_experts"): + _collect(engine, _gym_call([1, 3, 4])) @pytest.mark.parametrize( - ("request_fields", "error"), + ("call", "error"), [ - ({"temperature": 0.5}, "matches neither"), - ({"top_p": None}, "top_p must be set explicitly"), - ({"sampling": {"top_k": 5}}, "must be unset or match the configured top_k"), - ({"sampling": {"presence_penalty": 0.5}}, "presence_penalty=0.5 is not"), - ({"sampling": {"n": 2}}, "take n=1"), - ({"multi_modal_data": {"image_url": [{"UuidOnly": "u"}]}}, "carry their URL"), ( - { - "multi_modal_data": { - "image_url": [{"Url": "data:image/png;base64,eA=="}] - } - }, - "Cannot load image_url", + {"token_ids": [1, 3, 4], "annotations": [GYM_CALL_ANNOTATION]}, + r"extra_args\.nemo_rl", ), - # vLLM's own request errors (not ValueErrors) keep their message, as the - # native server's 4xx does, instead of becoming a bare 500. - ({"stop_conditions": {"min_tokens": 10, "max_tokens": 5}}, "min_tokens"), - # The echoed prefix, spliced in, leaves no room for output. + # vLLM's own request errors (not RuntimeErrors) keep their message, as + # the native server's 4xx does, instead of becoming a bare 500. ( - { - "prefix": [1, *range(20, 20 + MAX_MODEL_LEN), EOS], - "token_ids": [1, 7, EOS, 3], - }, - "maximum context length", + _gym_call([1, 3, 4], request=_chat(min_tokens=10, max_tokens=5)), + "min_tokens", ), ], ) -def test_gym_calls_the_engine_cannot_serve_are_refused(request_fields, error) -> None: +def test_gym_calls_the_engine_cannot_serve_are_refused(call, error) -> None: llm = _FakeLLM() - llm.tokenizer = _Tokenizer() engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) with pytest.raises(InvalidArgument, match=error): - _collect(engine, _gym_request(**request_fields)) + _collect(engine, call) assert llm.calls == [] -def test_gym_echoed_prefix_keeps_media_placeholders_on_their_tokens() -> None: - # Turn 1 ran [1, , 3] and generated [5, 6, EOS]; the frontend +def test_a_gym_call_over_the_engines_queue_limits_is_refused_before_it_runs() -> None: + full = vllm.exceptions.QueueOverflowError() + llm = _FakeLLM( + [_output([1, 3, 4], _completion([10], "stop"))], admission_error=full + ) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) + + # As the native server checks max_num_queued_reqs / max_num_queued_tokens. + with pytest.raises(vllm.exceptions.QueueOverflowError): + _collect(engine, _gym_call([1, 3, 4])) + + assert llm.calls == [] + + +def test_a_spliced_media_call_keeps_placeholders_on_their_tokens() -> None: + # Turn 1 ran [1, , 3] and generated [5, 6, EOS]; the template # re-rendered that turn as [7, EOS] and appended a turn with a second image. - turn1 = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] - prefix = [*turn1, 5, 6, EOS] - rendered = [1, IMAGE_PLACEHOLDER, 3, 7, EOS, IMAGE_PLACEHOLDER, 4] - spliced = [*prefix, *[IMAGE_TOKEN] * IMAGE_TOKENS, 4] + image = [IMAGE_TOKEN] * IMAGE_TOKENS + template = [1, *image, 3, 7, EOS, *image, 4] + prefix = [1, *image, 3, 5, 6, EOS] + spliced = [*prefix, *image, 4] + multimodal = { + "type": "multimodal", + "mm_kwargs": {"image": ["pixels-0", "pixels-1"]}, + "mm_hashes": {"image": ["h0", "h1"]}, + "mm_placeholders": { + "image": [ + PlaceholderRange(1, IMAGE_TOKENS), + PlaceholderRange(7, IMAGE_TOKENS), + ] + }, + } llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) - llm.tokenizer = _Tokenizer() engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) - (chunk,) = _collect( + _collect( engine, - _gym_request( - prefix=prefix, - token_ids=rendered, - multi_modal_data={"image_url": [{"Url": _image_url()}] * 2}, + _gym_call( + spliced, + template_token_ids=template, + # Each side cut at its turn end: prefix[:7] + template[6:]. + splice=[7, 6], + multimodal=base64.b64encode(pickle.dumps(multimodal)).decode(), ), ) - ((engine_input, params, _),) = llm.calls + ((engine_input, params, _, _),) = llm.calls assert engine_input["prompt_token_ids"] == spliced + assert engine_input["mm_kwargs"] == multimodal["mm_kwargs"] # The first image sits in the model's own prefix; the second moved with the # longer generated turn, as the native endpoint remaps them. assert [ (span.offset, span.length) for span in engine_input["mm_placeholders"]["image"] ] == [(1, IMAGE_TOKENS), (len(prefix), IMAGE_TOKENS)] assert params.max_tokens == MAX_MODEL_LEN - len(spliced) - assert chunk["engine_data"]["prompt_token_ids"] == spliced @pytest.mark.parametrize( diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py index 4c488637d69..3beffbd720c 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py @@ -13,11 +13,14 @@ # limitations under the License. """NeMo-Gym token capture through the Dynamo engine adapter. -The worker's own capture lifecycle stages each call, as on the native chat -endpoint; only Gym's capture core, and in the media tests the worker's media -snapshot, are faked. +The frontend's chat engine rendered and spliced each call; the adapter runs the +worker's own capture lifecycle on it, as the native chat endpoint does. Only +Gym's capture core, and in the media tests the worker's media snapshot, are +faked. """ +import base64 +import pickle from types import SimpleNamespace import numpy as np @@ -30,43 +33,58 @@ # The vLLM lanes import this module without the nemo_gym extra. pytest.importorskip("nemo_gym.token_id_capture.staging") +from dynamo.llm.exceptions import InvalidArgument # noqa: E402 from nemo_gym.token_id_capture import NG_COMMIT_COORDS_FIELD # noqa: E402 from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 -from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 - DYNAMO_CAPTURE_ANNOTATION_PREFIX, - DYNAMO_GYM_REQUEST_ANNOTATION, - DYNAMO_PREFIX_ANNOTATION_PREFIX, +from nemo_gym.token_id_capture.replay import ( # noqa: E402 + replay_context, + summarize_replay, ) from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 -from responses_api_models.vllm_model.app import ( # noqa: E402 - _DYNAMO_ENGINE_DATA_FIELDS, -) from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 -from nemo_rl.models.generation.dynamo import vllm_adapter # noqa: E402 +from nemo_rl.models.generation.dynamo.vllm import ( # noqa: E402 + adapter as vllm_adapter, +) from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 from nemo_rl.utils.routed_experts_codec import ( # noqa: E402 routed_experts_tensor_metadata, ) from tests.unit.models.generation.test_dynamo_vllm_adapter import ( # noqa: E402 EOS, - IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS, + _chat, _collect, _completion, _Context, _FakeLLM, - _gym_request, + _gym_call, _host, - _image_url, + _logprob, _output, - _Tokenizer, ) COORDS = {"staging_key": "rollout-1/c2"} +# Turn 2 of a rollout: the staged prefix [1, 5, 6, EOS] replaces the template's +# re-render of turn 1, [1, 7, EOS], before the new turn [3, 4]. +PREFIX = [1, 5, 6, EOS] +TEMPLATE = [1, 7, EOS, 3, 4] +# Each side cut at its turn end: PREFIX[:3] + TEMPLATE[2:]. +SPLICE = [3, 2] +SPLICED = [*PREFIX, 3, 4] +TOKEN_IN = CaptureAdmission( + capture_key="rollout-1", + model_call_id="c2", + parent_call_id="c1", + prev_len=len(PREFIX), + mode="token_in", + required_prefix_token_ids=PREFIX, + parent_chain_hash="0" * 64, +) +TEXT = CaptureAdmission(capture_key="rollout-1", model_call_id="c1", mode="text") class _Capture: @@ -96,74 +114,57 @@ def fail_call(self, call, *, reason) -> None: def _capturing_host(llm, **cfg): host = _host(llm, **cfg) host.token_capture = _Capture() - host._capture_calls = {} - host._capture_top_logprobs = 0 host._chain_prefix = ChainPrefixCache() return host -def _captured_request(admission: CaptureAdmission, **fields) -> dict: - request = _gym_request(**fields) - # Gym's dynamo_worker handler: no logprobs, the admission in an annotation. - request["output_options"] = {} - request["annotations"] = [ - DYNAMO_CAPTURE_ANNOTATION_PREFIX + admission.model_dump_json(), - *request["annotations"], - ] - return request - - -def test_annotations_and_engine_data_are_the_ones_gyms_dynamo_dialect_uses() -> None: - assert vllm_adapter.GYM_ROLLOUT_ANNOTATION == DYNAMO_GYM_REQUEST_ANNOTATION - assert vllm_adapter.GYM_CAPTURE_ANNOTATION_PREFIX == ( - DYNAMO_CAPTURE_ANNOTATION_PREFIX - ) - assert vllm_adapter.GYM_PREFIX_ANNOTATION_PREFIX == ( - DYNAMO_PREFIX_ANNOTATION_PREFIX +def _captured_call( + token_ids: list[int], admission: CaptureAdmission, top_logprobs: int = 0, **plan +) -> dict: + """A captured Gym call, as the chat engine routes it after its render.""" + admission_json = admission.model_dump(mode="json") + return _gym_call( + token_ids, + # Gym's dynamo_worker handler asks for the sampled tokens' logprobs. + request=_chat( + logprobs=True, top_logprobs=top_logprobs, ng_capture=admission_json + ), + admission=admission_json, + serving_admission=admission_json, + **plan, ) - # The engine_data keys Gym requires on a call without capture. - llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) - engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) - (chunk,) = _collect(engine, _gym_request()) - assert chunk["engine_data"].keys() == _DYNAMO_ENGINE_DATA_FIELDS.keys() def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: - # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. - prefix = [1, 5, 6, EOS] - admission = CaptureAdmission( - capture_key="rollout-1", - model_call_id="c2", - parent_call_id="c1", - prev_len=len(prefix), - mode="token_in", - required_prefix_token_ids=prefix, - parent_chain_hash="0" * 64, - ) - spliced = [*prefix, 3, 4] - routes = np.arange(8 * 2 * 2, dtype=np.int16).reshape(8, 2, 2) + # vLLM routes every token but the last sampled one. + routes = np.arange(7 * 2 * 2, dtype=np.int16).reshape(7, 2, 2) final = _output( - spliced, + SPLICED, _completion([10, 11], "stop", routed_experts=routes[6:]), prompt_routed_experts=routes[:6], ) llm = _FakeLLM([final]) - llm.tokenizer = _Tokenizer() host = _capturing_host(llm, vllm_kwargs={"enable_return_routed_experts": True}) engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) (chunk,) = _collect( - engine, _captured_request(admission, token_ids=[1, 7, EOS, 3, 4]) + engine, + _captured_call( + SPLICED, + TOKEN_IN, + prefix_len=len(PREFIX), + template_token_ids=TEMPLATE, + splice=SPLICE, + ), ) - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == spliced - # The sampled tokens' log probabilities are staged; Gym asked for none. + ((prompt, params, _, _),) = llm.calls + assert prompt["prompt_token_ids"] == SPLICED assert params.logprobs == 0 - ((begun_admission, begun_prefix, stream),) = host.token_capture.begun - assert (begun_admission, begun_prefix, stream) == (admission, prefix, False) + # The model's own prefix, as the frontend spliced it, begins the call. + assert host.token_capture.begun == [(TOKEN_IN, PREFIX, False)] (payload,) = host.token_capture.staged - assert payload["prompt_token_ids"] == spliced + assert payload["prompt_token_ids"] == SPLICED message = payload["choices"][0]["message"] assert message["generation_token_ids"] == [10, 11] assert message["generation_log_probs"] == [-0.5, -0.5] @@ -175,14 +176,62 @@ def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: expected[:4] = torch.from_numpy(routes[3:7]) assert torch.equal(staged_routes, expected) assert message["routed_experts"] == routed_experts_tensor_metadata(staged_routes) - # Only the commit coordinates go back through the frontend. - assert chunk["engine_data"] == {NG_COMMIT_COORDS_FIELD: COORDS} - assert "log_probs" not in chunk + # Only the commit coordinates go back through the frontend, beside the text. + assert chunk["engine_data"]["result"] == {NG_COMMIT_COORDS_FIELD: COORDS} + assert "logprobs" not in chunk["engine_data"]["output"] assert host._capture_calls == {} +def test_a_captured_call_stages_the_samplers_top_k_for_score_centering() -> None: + completion = _completion([10, 11], "stop") + completion.logprobs = [ + {10: _logprob(-0.5), 20: _logprob(-2.0, rank=2)}, + {11: _logprob(-0.7), 21: _logprob(-3.0, rank=2)}, + ] + llm = _FakeLLM([_output([1, 3, 4], completion)]) + host = _capturing_host(llm) + host._capture_top_logprobs = 2 + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) + + _collect(engine, _captured_call([1, 3, 4], TEXT, top_logprobs=2)) + + ((_, params, _, _),) = llm.calls + assert params.logprobs == 2 + (attachments,) = host.token_capture.attachments + assert attachments["sampler_topk_token_ids"].tolist() == [[10, 20], [11, 21]] + assert torch.equal( + attachments["sampler_topk_logprobs"], + torch.tensor([[-0.5, -2.0], [-0.7, -3.0]], dtype=torch.float32), + ) + assert host.token_capture.failed == [] + + +def test_an_undecided_candidate_admission_is_refused() -> None: + history = [{"role": "user", "content": "q"}, {"role": "assistant", "content": "a"}] + candidate = CaptureAdmission( + **{ + **TOKEN_IN.model_dump(), + "mode": "candidate", + "requires_empty_tool_content_equivalence": False, + "candidate_replay": summarize_replay( + replay_context(history, render_digest="b" * 64) + ), + "request_replay": replay_context(history, render_digest="b" * 64), + } + ) + llm = _FakeLLM() + host = _capturing_host(llm) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) + + # The frontend's render decides a candidate; one reaching the engine skipped it. + with pytest.raises(InvalidArgument, match="candidate"): + _collect(engine, _captured_call(SPLICED, candidate)) + + assert (llm.calls, host.token_capture.begun) == ([], []) + + class _FailingLLM(_FakeLLM): - async def generate(self, prompt, sampling_params, request_id): + async def generate(self, prompt, sampling_params, request_id, **kwargs): raise RuntimeError("engine died") yield # pragma: no cover @@ -216,44 +265,46 @@ def is_stopped(self) -> bool: def test_a_call_that_does_not_finish_cleanly_fails_its_capture( llm, context, error, reason ) -> None: - admission = CaptureAdmission( - capture_key="rollout-1", model_call_id="c1", mode="text" - ) host = _capturing_host(llm) engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) if error is None: - assert _collect(engine, _captured_request(admission), context) == [] + assert _collect(engine, _captured_call([1, 3, 4], TEXT), context) == [] else: - with pytest.raises(RuntimeError, match=error): - _collect(engine, _captured_request(admission), context) + with pytest.raises(InvalidArgument, match=error): + _collect(engine, _captured_call([1, 3, 4], TEXT), context) assert host.token_capture.staged == [] assert host.token_capture.failed == [reason] assert host._capture_calls == {} +def _multimodal(placeholders: list[PlaceholderRange]) -> str: + """The rendered engine input's media, as the chat engine ships it.""" + multimodal = { + "type": "multimodal", + "mm_kwargs": {"image": [f"pixels-{i}" for i in range(len(placeholders))]}, + "mm_hashes": {"image": [f"hash-{i}" for i in range(len(placeholders))]}, + "mm_placeholders": {"image": placeholders}, + } + return base64.b64encode(pickle.dumps(multimodal)).decode() + + def test_a_captured_media_call_keeps_the_placeholders_capture_moved() -> None: - # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. - prefix = [1, 5, 6, EOS] - admission = CaptureAdmission( - capture_key="rollout-1", - model_call_id="c2", - parent_call_id="c1", - prev_len=len(prefix), - mode="token_in", - required_prefix_token_ids=prefix, - parent_chain_hash="0" * 64, - ) - spliced = [*prefix, *[IMAGE_TOKEN] * IMAGE_TOKENS, 4] + image = [IMAGE_TOKEN] * IMAGE_TOKENS + template = [1, 7, EOS, *image, 4] + spliced = [*PREFIX, *image, 4] llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) - llm.tokenizer = _Tokenizer() host = _capturing_host(llm) host._capture_media = True - moved = {"image": [PlaceholderRange(len(prefix), IMAGE_TOKENS)]} + moved = {"image": [PlaceholderRange(len(PREFIX), IMAGE_TOKENS)]} + snapshots = [] - def capture_request_media(engine_input, *, admission, splice): + def capture_request_media( + engine_input, *, admission, splice, verify_retained_media + ): # As capture_processed_media, which moves the placeholders itself. + snapshots.append((admission, splice, verify_retained_media)) engine_input["mm_placeholders"] = moved return CapturedMedia((), None) @@ -262,51 +313,56 @@ def capture_request_media(engine_input, *, admission, splice): _collect( engine, - _captured_request( - admission, - token_ids=[1, 7, EOS, IMAGE_PLACEHOLDER, 4], - multi_modal_data={"image_url": [{"Url": _image_url()}]}, + _captured_call( + spliced, + TOKEN_IN, + prefix_len=len(PREFIX), + template_token_ids=template, + splice=SPLICE, + multimodal=_multimodal([PlaceholderRange(3, IMAGE_TOKENS)]), + verify_retained_media=True, ), ) - ((engine_input, _, _),) = llm.calls + ((admission, splice, verify),) = snapshots + assert (admission, verify) == (TOKEN_IN, True) + assert (splice.token_ids, splice.model_cut_end, splice.template_cut_start) == ( + spliced, + *SPLICE, + ) + ((engine_input, _, _, _),) = llm.calls assert engine_input["prompt_token_ids"] == spliced # Not remapped a second time. assert engine_input["mm_placeholders"] == moved def test_a_captured_media_call_stages_the_media_the_engine_ran_on() -> None: - admission = CaptureAdmission( - capture_key="rollout-1", model_call_id="c1", mode="text" - ) processed = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] llm = _FakeLLM([_output(processed, _completion([10], "stop"))]) host = _capturing_host(llm) host._capture_media = True - captured = CapturedMedia((), None) snapshots = [] - def capture_request_media(engine_input, *, admission, splice): + def capture_request_media( + engine_input, *, admission, splice, verify_retained_media + ): snapshots.append((list(engine_input["prompt_token_ids"]), admission, splice)) - return captured + return CapturedMedia((), None) host._capture_request_media = capture_request_media engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) (chunk,) = _collect( engine, - _captured_request( - admission, - token_ids=[1, IMAGE_PLACEHOLDER, 3], - multi_modal_data={"image_url": [{"Url": _image_url()}]}, + _captured_call( + processed, + TEXT, + multimodal=_multimodal([PlaceholderRange(1, IMAGE_TOKENS)]), ), ) - # The processor's own pixels, not cache references, as natively (#4124). - ((_, skip_mm_cache),) = llm.renderer.prompts - assert skip_mm_cache is True # Media is snapshotted from the processed prompt the engine runs. - assert snapshots == [(processed, admission, None)] + assert snapshots == [(processed, TEXT, None)] (payload,) = host.token_capture.staged assert payload["prompt_token_ids"] == processed - assert chunk["engine_data"] == {NG_COMMIT_COORDS_FIELD: COORDS} + assert chunk["engine_data"]["result"] == {NG_COMMIT_COORDS_FIELD: COORDS} diff --git a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py index 536f9e404f6..fff9809fc72 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py +++ b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py @@ -14,11 +14,12 @@ """Dynamo stays out of NeMo RL's vLLM engines. The engines run NeMo RL's own vLLM and use Dynamo only through its -engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only in -NeMo RL's own adapter (``dynamo/vllm_adapter.py``). These guards fail if -NeMo RL starts importing Dynamo's vLLM integration (or any other Dynamo -module), if the SDK itself starts importing vLLM, or if the Dynamo side shared -by every backend starts importing one. +engine-agnostic backend SDK, and the frontend serves chat with NeMo RL's own +chat engine, so a vLLM bump can break Dynamo serving only in NeMo RL's own +``dynamo/vllm/`` package. These guards fail if NeMo RL starts importing +Dynamo's vLLM integration (or any other Dynamo module), if the Dynamo modules +it imports start importing vLLM, or if the Dynamo side shared by every backend +starts importing one. """ import ast @@ -38,16 +39,30 @@ "dynamo.common.backend", "dynamo.common.backend.logprobs", "dynamo.common.backend.publisher", + # The frontend's entrypoint and chat-engine hook; the frontend loads Dynamo's + # own vLLM chat processor only for --dyn-chat-processor, which NeMo RL rejects. + "dynamo.frontend.main", + "dynamo.llm", "dynamo.llm.exceptions", } - -# The Dynamo side every generation backend builds on. -BACKEND_NEUTRAL_MODULES = ( - "models/generation/dynamo/engine_adapter.py", - "models/generation/rollout_result.py", -) GENERATION_BACKENDS = {"vllm", "sglang", "trtllm", "megatron"} +DYNAMO_ROOT = NEMO_RL_ROOT / "models/generation/dynamo" + + +def _backend_neutral_modules() -> list[str]: + """The Dynamo side every generation backend builds on: all of ``dynamo/`` + outside its per-backend packages, and the wire types it shares.""" + paths = [ + path + for path in DYNAMO_ROOT.rglob("*.py") + if path.relative_to(DYNAMO_ROOT).parts[0] not in GENERATION_BACKENDS + ] + paths += [ + NEMO_RL_ROOT / "models/generation/rollout_result.py", + NEMO_RL_ROOT / "models/generation/gym_call.py", + ] + return sorted(str(path.relative_to(NEMO_RL_ROOT)) for path in paths) def _imports(path: Path) -> set[str]: @@ -90,20 +105,22 @@ def test_nemo_rl_imports_only_dynamos_backend_sdk() -> None: def test_the_shared_dynamo_side_imports_no_generation_backend() -> None: + modules = _backend_neutral_modules() + assert "models/generation/dynamo/engine_adapter.py" in modules backends = { path: sorted( module for module in _imports(NEMO_RL_ROOT / path) if GENERATION_BACKENDS & set(module.split(".")) ) - for path in BACKEND_NEUTRAL_MODULES + for path in modules } - assert backends == {path: [] for path in BACKEND_NEUTRAL_MODULES} + assert backends == {path: [] for path in modules} @pytest.mark.vllm -def test_dynamos_backend_sdk_imports_no_vllm() -> None: +def test_the_dynamo_modules_nemo_rl_imports_import_no_vllm() -> None: pytest.importorskip("dynamo._core") # A fresh interpreter, so modules other tests imported do not count. code = ( diff --git a/tests/unit/models/generation/test_gym_call.py b/tests/unit/models/generation/test_gym_call.py new file mode 100644 index 00000000000..331a191ffb4 --- /dev/null +++ b/tests/unit/models/generation/test_gym_call.py @@ -0,0 +1,89 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NeMo-Gym's Dynamo dialect packs and lifts the fields NeMo RL's call models define.""" + +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.nemo_gym +pytest.importorskip("nemo_gym.token_id_capture.external_capture") + +from nemo_gym.openai_utils import TOKEN_METADATA_FIELDS # noqa: E402 +from nemo_gym.token_id_capture import ( # noqa: E402 + NG_CAPTURE_FIELD, + NG_COMMIT_COORDS_FIELD, +) +from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 + DYNAMO_GYM_EXTRAS_ANNOTATION_PREFIX, + add_dynamo_gym_extras, + pop_dynamo_call_result, +) +from responses_api_models.vllm_model.app import VLLMModel # noqa: E402 + +from nemo_rl.models.generation.gym_call import ( # noqa: E402 + GYM_EXTRAS_ANNOTATION_PREFIX, + GYM_RESULT_NVEXT_KEY, + GymCallExtras, + GymCallResult, +) + + +def test_gyms_extras_are_the_ones_nemo_rl_reads() -> None: + assert DYNAMO_GYM_EXTRAS_ANNOTATION_PREFIX == GYM_EXTRAS_ANNOTATION_PREFIX + # Gym's own request preparation: the last echoed turn becomes the prefix. + body = { + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": "a", + "prompt_token_ids": [1, 2], + "generation_token_ids": [3], + "generation_log_probs": [-0.5], + }, + ], + "nvext": {"annotations": ["other"]}, + } + VLLMModel._prepare_dynamo_request( + SimpleNamespace(_external_capture_handler=None), body + ) + # The capture handler's admission and the measure flag ride in the same object. + add_dynamo_gym_extras( + body, {NG_CAPTURE_FIELD: {"capture_key": "r"}, "ng_measure": True} + ) + + # Unknown names are refused, so a field Gym renames fails here. + extras = GymCallExtras.from_annotations(body["nvext"]["annotations"]) + assert extras == GymCallExtras( + ng_capture={"capture_key": "r"}, + required_prefix_token_ids=[1, 2, 3], + ng_measure=True, + ) + assert body["nvext"]["annotations"][0] == "other" + assert "prompt_token_ids" not in body["messages"][1] + + +def test_gym_lifts_the_call_result_nemo_rl_returns() -> None: + result = GymCallResult( + prompt_token_ids=[1, 2], + generation_token_ids=[3], + generation_log_probs=[-0.5], + routed_experts="nrlre1:int16:3x1x1:AAABAAIA", + ng_commit_coords={"staging_key": "r/c1"}, + ).model_dump(exclude_none=True) + + assert pop_dynamo_call_result({"nvext": {GYM_RESULT_NVEXT_KEY: result}}) == result + # The fields Gym copies onto the message, and its capture acknowledgement. + assert TOKEN_METADATA_FIELDS | {NG_COMMIT_COORDS_FIELD} <= result.keys() diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py new file mode 100644 index 00000000000..6d2dff024ae --- /dev/null +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -0,0 +1,563 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NeMo-Gym's chat calls take the same steps on both of NeMo RL's serving paths. + +Each case sends one call to the native vLLM server (its real routes, over ASGI) +and through Dynamo (NeMo RL's chat engine, a spawned render pool, and the +engine adapter), over the same tiny Qwen3 checkpoint, the same scripted engine +and Gym's real capture core. Both must run the engine on the same prompt with +the same SamplingParams, stage the same capture record, and answer Gym with the +same message and token fields. + +Not covered here: media (the checkpoint is text-only; the VLM functional +overlay runs it) and a model whose EOS is not its turn end. +""" + +import asyncio +import json +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Any + +import pytest +import torch + +pytestmark = [pytest.mark.vllm, pytest.mark.nemo_gym] +pytest.importorskip("dynamo._core") +pytest.importorskip("vllm") +pytest.importorskip("nemo_gym.token_id_capture.staging") + +import httpx # noqa: E402 +from dynamo.llm.exceptions import HttpError # noqa: E402 +from fastapi import FastAPI # noqa: E402 +from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 +from nemo_gym.token_id_capture.replay import ( # noqa: E402 + replay_context, + summarize_replay, +) +from nemo_gym.token_id_capture.staging.capture import ( # noqa: E402 + RolloutTokenCapture, +) +from nemo_gym.token_id_capture.staging.records import ( # noqa: E402 + CaptureAdmission, + StageResult, +) +from transformers import AutoTokenizer # noqa: E402 +from vllm.config import CacheConfig, LoadConfig, VllmConfig # noqa: E402 +from vllm.engine.arg_utils import AsyncEngineArgs # noqa: E402 +from vllm.logprobs import Logprob # noqa: E402 +from vllm.outputs import CompletionOutput, RequestOutput # noqa: E402 +from vllm.renderers.registry import renderer_from_config # noqa: E402 +from vllm.v1.engine.input_processor import InputProcessor # noqa: E402 + +from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache, StagedPrefix # noqa: E402 +from nemo_rl.models.generation.dynamo.vllm import chat_engine # noqa: E402 +from nemo_rl.models.generation.dynamo.vllm.adapter import ( # noqa: E402 + DynamoVllmAdapter, + chat_runtime_data, +) +from nemo_rl.models.generation.dynamo.vllm.render_pool import RenderPool # noqa: E402 +from nemo_rl.models.generation.gym_call import GYM_EXTRAS_ANNOTATION_PREFIX # noqa: E402 +from nemo_rl.models.generation.vllm import chat_server # noqa: E402 +from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 + VllmAsyncGenerationWorkerImpl, +) + +MAX_MODEL_LEN = 512 +GENERATION = { + "temperature": 0.7, + "top_p": 0.9, + "top_k": None, + "val_temperature": 0.0, + "val_top_p": 1.0, + "val_top_k": None, + "stop_token_ids": None, + "max_new_tokens": 64, + "vllm_cfg": { + "max_model_len": MAX_MODEL_LEN, + "http_server_serving_chat_kwargs": { + "tool_parser": "hermes", + "reasoning_parser": "qwen3", + "enable_auto_tools": True, + # Rendered beneath the request's own on both paths. + "default_chat_template_kwargs": {"enable_thinking": False}, + }, + }, +} +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "The weather in a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] +# Turn 1 as Gym sends it back: the template drops its reasoning when it +# re-renders the history, so the model's own tokens differ from the render. +HISTORY = [ + {"role": "user", "content": "Weather in Paris?"}, + {"role": "assistant", "content": "It is sunny.", "reasoning_content": "easy"}, +] +OBSERVATION = {"role": "user", "content": "And in Lyon?"} +REPLY = ( + "\nchecking\n\n\n\n" + '{"name": "get_weather", "arguments": {"city": "Lyon"}}\n' +) +TURN_1_REPLY = "\neasy\n\n\nIt is sunny." + + +@dataclass +class _Kit: + """The checkpoint, and the vLLM objects both paths share.""" + + path: str + tokenizer: Any + engine_args: Any + model_config: Any + vllm_config: Any + renderer: Any + pool: RenderPool + + def ids(self, text: str) -> list[int]: + return self.tokenizer.encode(text, add_special_tokens=False) + + def turn_1_tokens(self) -> list[int]: + """Turn 1's prompt and the model's own reply, as capture staged them.""" + prompt = self.tokenizer.apply_chat_template( + HISTORY[:1], tools=TOOLS, add_generation_prompt=True, tokenize=False + ) + return self.ids(prompt + TURN_1_REPLY + "<|im_end|>") + + +class _Engine: + """vLLM's engine client: answers every call with REPLY and records its runs.""" + + def __init__(self, kit: _Kit) -> None: + self.model_config = kit.model_config + self.renderer = kit.renderer + self.vllm_config = kit.vllm_config + self.input_processor = InputProcessor(kit.vllm_config) + self.io_processor = None + self.errored = False + self._kit = kit + self.runs: list[tuple[list[int], Any]] = [] + + async def generate(self, prompt, sampling_params, request_id, **kwargs): + prompt_token_ids = list(prompt["prompt_token_ids"]) + self.runs.append((prompt_token_ids, sampling_params)) + reply_ids = self._kit.ids(REPLY) + logprobs = None + if sampling_params.logprobs is not None: + decode = self._kit.tokenizer.decode + # The sampled token, and the runner-up a top_logprobs > 0 call sees. + logprobs = [ + { + token_id: Logprob(-0.5, 1, decode([token_id])), + token_id + 1: Logprob(-2.0, 2, decode([token_id + 1])), + } + for token_id in reply_ids + ] + yield RequestOutput( + request_id=request_id, + prompt=None, + prompt_token_ids=prompt_token_ids, + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text=REPLY, + token_ids=reply_ids, + cumulative_logprob=None, + logprobs=logprobs, + finish_reason="stop", + ) + ], + finished=True, + num_cached_tokens=0, + ) + + def check_admission(self, n, request_id=None) -> None: + pass + + async def is_tracing_enabled(self) -> bool: + return False + + async def abort(self, request_id) -> None: + pass + + +class _Sink: + def __init__(self) -> None: + self.staged: list[tuple[Any, Any]] = [] + + def stage(self, record, *, attachments=None) -> StageResult: + self.staged.append((record, attachments)) + return StageResult(ok=True, staging_key=record.staging_key) + + +class _Staged: + """TQ's staged calls: each call's token delta, by staging key.""" + + def __init__(self, deltas: dict[str, list[int]]) -> None: + self.deltas = deltas + + def fetch_prefix_chains(self, staging_keys: list[str]) -> StagedPrefix: + return StagedPrefix( + expanded=[token for key in staging_keys for token in self.deltas[key]] + ) + + +@dataclass +class _Outcome: + """What one path did with a call.""" + + status: int + reply: dict[str, Any] + runs: list[tuple[list[int], Any]] + staged: list[tuple[Any, Any]] + failed: list[str] = field(default_factory=list) + + +def _host(kit: _Kit, deltas: dict[str, list[int]], top_logprobs: int): + host = object.__new__(VllmAsyncGenerationWorkerImpl) + host.cfg = GENERATION + host.llm = host._http_engine_client = _Engine(kit) + host.llm_async_engine_args = kit.engine_args + host.model_name = "model" + host.routed_experts_dtype = torch.int16 + host._capture_media = False + host._capture_calls = {} + host._capture_top_logprobs = top_logprobs + host._staging_source = None + host._rollout_weight_version = 0 + host._chain_prefix = ChainPrefixCache() + host._chain_prefix.install(_Staged(deltas)) + sink = _Sink() + capture = RolloutTokenCapture( + sink=sink, weight_version_fn=lambda: 0, adapter=VLLMCaptureAdapter() + ) + failed: list[str] = [] + fail_call = capture.fail_call + + def record_failure(call, *, reason): + failed.append(reason) + return fail_call(call, reason=reason) + + capture.fail_call = record_failure + host.token_capture = capture + return host, sink, failed + + +async def _native(kit, host, sink, failed, body, *, measure: bool) -> _Outcome: + server = chat_server.build_chat_server( + host, + engine_client=host._http_engine_client, + model_config=kit.model_config, + cfg=GENERATION, + ) + app = FastAPI() + chat_server.register_routes(app, server, host=host, generation_config=GENERATION) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://native" + ) as client: + response = await client.post( + "/tokenize" if measure else "/v1/chat/completions", json=body + ) + return _Outcome( + response.status_code, response.json(), host.llm.runs, sink.staged, failed + ) + + +class _Router: + """Dynamo's router to one engine: the request crosses the wire as JSON.""" + + def __init__(self, adapter: DynamoVllmAdapter) -> None: + self._adapter = adapter + + async def generate(self, request, context=None): + wire = json.loads(json.dumps(request)) + chunks = [chunk async for chunk in self._adapter.generate(wire, _Context())] + + async def stream(): + for chunk in chunks: + yield SimpleItem(chunk) + + return stream() + + +class _Context: + def id(self) -> str: + return "chatcmpl-0" + + def is_stopped(self) -> bool: + return False + + +class SimpleItem: + def __init__(self, data) -> None: + self._data = data + + def is_error(self) -> bool: + return False + + def data(self): + return self._data + + +async def _dynamo(kit, host, sink, failed, body, extras) -> _Outcome: + request = {k: v for k, v in body.items() if k not in extras} + request["nvext"] = { + "annotations": [GYM_EXTRAS_ANNOTATION_PREFIX + json.dumps(extras)] + } + capture = chat_engine._CaptureSource() + capture._prefix_cache = host._chain_prefix + engine = chat_engine.NemoRLChatEngine( + pool=kit.pool, + routed_engine=_Router(DynamoVllmAdapter(host, leader_worker_idx=0)), + spec=json.loads(json.dumps(chat_runtime_data(host)))[ + chat_engine.RUNTIME_DATA_KEY + ], + capture=capture, + ) + try: + (chunk,) = [chunk async for chunk in engine.generate(request, None)] + except HttpError as error: + return _Outcome( + error.code, {"error": error.message}, host.llm.runs, sink.staged, failed + ) + return _Outcome(200, chunk, host.llm.runs, sink.staged, failed) + + +@pytest.fixture(scope="module") +def kit(tiny_qwen3_model_path): + engine_args = AsyncEngineArgs( + model=tiny_qwen3_model_path, max_model_len=MAX_MODEL_LEN + ) + model_config = engine_args.create_model_config() + vllm_config = VllmConfig( + model_config=model_config, + load_config=LoadConfig(load_format="dummy"), + cache_config=CacheConfig(), + ) + # The pool renders with what an engine publishes at registration. + spec = json.loads( + json.dumps( + chat_runtime_data( + SimpleNamespace(llm_async_engine_args=engine_args, cfg=GENERATION) + ) + ) + )[chat_engine.RUNTIME_DATA_KEY] + pool = RenderPool(spec, tiny_qwen3_model_path, workers=1) + yield _Kit( + path=tiny_qwen3_model_path, + tokenizer=AutoTokenizer.from_pretrained(tiny_qwen3_model_path), + engine_args=engine_args, + model_config=model_config, + vllm_config=vllm_config, + renderer=renderer_from_config(vllm_config), + pool=pool, + ) + pool.shutdown() + + +def _admission(**fields) -> dict[str, Any]: + return CaptureAdmission( + capture_key="rollout-1", model_call_id="c2", **fields + ).model_dump(mode="json") + + +def _staged_turn_1(kit: _Kit, **fields) -> dict[str, Any]: + """Turn 2's admission, chained to turn 1 staged under rollout-1/c1.""" + return _admission( + parent_call_id="c1", + prev_len=len(kit.turn_1_tokens()), + parent_chain_hash="1" * 64, + staging_chain=["rollout-1/c1"], + **fields, + ) + + +def _candidate(kit: _Kit, messages: list[dict]) -> dict[str, Any]: + """Gym's candidate: turn 1's source, and this request's, for RL to check the cut.""" + return _staged_turn_1( + kit, + mode="candidate", + requires_empty_tool_content_equivalence=False, + candidate_replay=summarize_replay( + replay_context(HISTORY, render_digest="b" * 64) + ), + request_replay=replay_context(messages, render_digest="b" * 64), + ) + + +# id, messages, extras(kit), request fields, capture top-k +CASES = [ + # A plain call: the render and the reply's reasoning and tool parsers (C39). + ("plain", [*HISTORY, OBSERVATION], lambda kit: {}, {}, 0), + # Gym's echo of turn 1 without capture: the model's tokens replace the + # template's re-render of that turn (C16's turn-end cut). + ( + "inline_prefix", + [*HISTORY, OBSERVATION], + lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, + {}, + 0, + ), + # Token capture: turn 1's staged tokens, read from TQ. + ( + "token_in", + [*HISTORY, OBSERVATION], + lambda kit: {"ng_capture": _staged_turn_1(kit, mode="token_in")}, + {}, + 0, + ), + # Gym's candidate whose cut holds is served token-in (C26)... + ( + "candidate_cut_holds", + [*HISTORY, OBSERVATION], + lambda kit: {"ng_capture": _candidate(kit, [*HISTORY, OBSERVATION])}, + {}, + 0, + ), + # ...and one whose conversion moved the cut roots a new text segment. + ( + "candidate_cut_fails", + [*HISTORY, OBSERVATION, HISTORY[-1]], + lambda kit: {"ng_capture": _candidate(kit, [*HISTORY, OBSERVATION])}, + {}, + 0, + ), + # Score centering stages the sampler's top-k with the call (C29). + ( + "score_centering", + [*HISTORY, OBSERVATION], + lambda kit: {"ng_capture": _admission(mode="text")}, + {"top_logprobs": 2}, + 2, + ), + # Off-policy sampling is refused before anything runs (C41). + ("off_policy", [*HISTORY, OBSERVATION], lambda kit: {}, {"temperature": 0.5}, 0), +] + + +@pytest.mark.parametrize( + ("messages", "extras", "fields", "top_logprobs"), + [case[1:] for case in CASES], + ids=[case[0] for case in CASES], +) +def test_a_gym_call_takes_the_same_steps_through_dynamo_as_natively( + kit, messages, extras, fields, top_logprobs +) -> None: + extras = extras(kit) + body = { + "model": kit.path, + "messages": messages, + "tools": TOOLS, + "temperature": 0.7, + "top_p": 0.9, + "max_tokens": 32, + # Gym's token information. + "logprobs": True, + "top_logprobs": 0, + **fields, + **extras, + } + deltas = {"rollout-1/c1": kit.turn_1_tokens()} + + async def both(): + native = await _native( + kit, *_host(kit, deltas, top_logprobs), body, measure=False + ) + dynamo = await _dynamo(kit, *_host(kit, deltas, top_logprobs), body, extras) + return native, dynamo + + native, dynamo = asyncio.run(both()) + + assert (native.status, dynamo.status) == ( + native.status, + native.status, + ), (native.reply, dynamo.reply) + if native.status != 200: + assert native.runs == dynamo.runs == [] + return + # The engine ran the same prompt with the same SamplingParams. + ((native_prompt, native_params),) = native.runs + ((dynamo_prompt, dynamo_params),) = dynamo.runs + assert dynamo_prompt == native_prompt + assert dynamo_params == native_params + # Capture staged the same record and tensors. + assert len(dynamo.staged) == len(native.staged) + for (native_record, native_tensors), (dynamo_record, dynamo_tensors) in zip( + native.staged, dynamo.staged + ): + assert dynamo_record == native_record + assert (dynamo_tensors or {}).keys() == (native_tensors or {}).keys() + for name, tensor in (native_tensors or {}).items(): + assert torch.equal(dynamo_tensors[name], tensor), name + assert dynamo.failed == native.failed == [] + # Gym reads the same reply and token fields. + message = native.reply["choices"][0]["message"] + (choice,) = dynamo.reply["choices"] + assert choice["finish_reason"] == native.reply["choices"][0]["finish_reason"] + assert choice["delta"]["content"] == message["content"] + assert choice["delta"].get("reasoning_content") == message.get("reasoning") + assert [call["function"] for call in choice["delta"]["tool_calls"]] == [ + call["function"] for call in message["tool_calls"] + ] + result = dynamo.reply["nvext"]["nemo_rl"] + if native.staged: + assert result == {"ng_commit_coords": native.reply["ng_commit_coords"]} + else: + assert result == { + key: message[key] + for key in ( + "prompt_token_ids", + "generation_token_ids", + "generation_log_probs", + ) + } + + +def test_a_measure_call_counts_the_prompt_the_call_would_run(kit) -> None: + extras = {"required_prefix_token_ids": kit.turn_1_tokens()} + body = { + "model": kit.path, + "messages": [*HISTORY, OBSERVATION], + "tools": TOOLS, + **extras, + } + + async def both(): + native = await _native(kit, *_host(kit, {}, 0), body, measure=True) + dynamo = await _dynamo( + kit, + *_host(kit, {}, 0), + {**body, "temperature": 0.7, "top_p": 0.9}, + {**extras, "ng_measure": True}, + ) + return native, dynamo + + native, dynamo = asyncio.run(both()) + + # Native Gym measures with /tokenize; through Dynamo, the chat call (C36). + assert ( + dynamo.reply["nvext"]["nemo_rl"]["prompt_token_count"] == native.reply["count"] + ) + assert dynamo.runs == [] diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 29fc86d4a3a..4edc0476861 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1365,9 +1365,12 @@ def test_chat_request_rejects_unreplayed_sampling_params( request.top_k = None setattr(request, field, value) - # check_sampling_param raises ValueError for top_k; the rest are asserts. - with pytest.raises((AssertionError, ValueError), match=field): - asyncio.run(chat_handler(request, MagicMock())) + response = asyncio.run(chat_handler(request, MagicMock())) + + # A client error Gym sees, not a server failure. + assert response.status_code == 400 + assert field in json.loads(response.body)["error"]["message"] + serving_chat.create_chat_completion.assert_not_called() def _track_request_capture(worker, request): @@ -1580,8 +1583,9 @@ async def test_chat_handler_rejects_off_policy_sampling_and_pins_top_k(monkeypat off_policy = _fake_chat_request() off_policy.temperature = 0.5 - with pytest.raises(ValueError, match="matches neither"): - await chat_handler(off_policy, MagicMock()) + response = await chat_handler(off_policy, MagicMock()) + assert response.status_code == 400 + assert "matches neither" in json.loads(response.body)["error"]["message"] serving_chat.create_chat_completion.assert_not_awaited() request = _fake_chat_request() diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 95b82b13161..67358a15467 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -431,12 +431,8 @@ def _generation_config(**generation) -> dict: r"dynamo_cfg\.metrics_exclude_prefixes", ), ( - {"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}, - r"worker_args\.extra_cli_args", - ), - ( - {"dynamo_cfg": {"worker_args": {"endpoint_types": ["chat"]}}}, - r"worker_args\.endpoint_types was removed", + {"dynamo_cfg": {"worker_args": {"tool_call_parser": "hermes"}}}, + r"dynamo_cfg\.worker_args was removed", ), ( {"dynamo_cfg": {"frontend_args": {"tokenizer_cache": False}}}, From 817107f3a07699c34c63c93ad71f2d15759746d6 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 16:51:29 -0800 Subject: [PATCH 36/82] build(gym): re-pin Gym to 4c84f173 Gym's Dynamo capture handler now asks for the sampled tokens' log probabilities, as its vLLM and Megatron handlers do. NeMo RL's engine behind Dynamo generates what the request asks for, so without them a captured call staged no log probabilities and failed its capture. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index a9e61ef75b6..4c84f1732ad 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit a9e61ef75b69c964f843f847987b021b1b2d6ddf +Subproject commit 4c84f1732ad5503f3bb695c360b28d24065de2d7 From a20b76de7158c6dfeb6b6cc6f98c54910bbb5e67 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 19:10:29 -0800 Subject: [PATCH 37/82] test(dynamo): run every Gym overlay in fast CI; drop the swe1 recipe test - Gym's default mode and router replay with deferred routes now run with FAST=1, which CI sets whenever it reuses an image (C17). - Add the context-compaction overlay, which turns on forest training, with dynamo=true (C26). - Delete test_swe1_dynamo_config.py, which only pinned recipe values, and its L0 entries (C18). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../functional/L1_Functional_Tests_Dynamo.sh | 8 +- tests/unit/L0_Unit_Tests_Vllm_1.sh | 2 +- tests/unit/L0_Unit_Tests_Vllm_2.sh | 2 +- tests/unit/L0_Unit_Tests_Vllm_3.sh | 2 +- .../generation/test_swe1_dynamo_config.py | 125 ------------------ 5 files changed, 8 insertions(+), 131 deletions(-) delete mode 100644 tests/unit/models/generation/test_swe1_dynamo_config.py diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index 23d08060f17..f5349069023 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -37,11 +37,13 @@ run_test() { # Each keeps the native test's checks and adds check_dynamo_overlay.sh's. run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true ++policy.generation.dynamo=true -# Gym's default mode: token arrays come back through the frontend in engine_data. -run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++policy.generation.dynamo=true +# Gym's default mode: token arrays come back in the reply's nvext.nemo_rl. +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true +# Forest training: token-capture calls that may continue an earlier call. +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller_context_compaction.sh ++policy.generation.dynamo=true # Router replay with deferred routes: the engine stages each captured call's routed experts. -run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/unit/L0_Unit_Tests_Vllm_1.sh b/tests/unit/L0_Unit_Tests_Vllm_1.sh index 26a82b73e57..b88f62208ed 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_1.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_1.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_score_centering.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "unit/models/generation/test_chat_template_parity_common.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_score_centering.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "unit/models/generation/test_chat_template_parity_common.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/L0_Unit_Tests_Vllm_2.sh b/tests/unit/L0_Unit_Tests_Vllm_2.sh index ecdfd22e8bc..90d23fcfd7b 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_2.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_2.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_score_centering.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "unit/models/generation/test_chat_template_parity_common.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_score_centering.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "unit/models/generation/test_chat_template_parity_common.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/L0_Unit_Tests_Vllm_3.sh b/tests/unit/L0_Unit_Tests_Vllm_3.sh index cf0f712ef82..1209d60d48d 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_3.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_3.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --extra modelopt bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_score_centering.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "unit/models/generation/test_chat_template_parity_common.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --extra modelopt bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_score_centering.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "unit/models/generation/test_chat_template_parity_common.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/models/generation/test_swe1_dynamo_config.py b/tests/unit/models/generation/test_swe1_dynamo_config.py deleted file mode 100644 index 9bd87f972b1..00000000000 --- a/tests/unit/models/generation/test_swe1_dynamo_config.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. - -from pathlib import Path -from types import SimpleNamespace - -import pytest -from omegaconf import OmegaConf - -from nemo_rl.environments.nemo_gym import setup_nemo_gym_config -from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.config import dynamo_cfg -from nemo_rl.utils.config import load_config, register_omegaconf_resolvers - -REPO_ROOT = Path(__file__).resolve().parents[4] -RECIPE = ( - REPO_ROOT / "examples/configs/recipes/llm/" - "grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml" -) -DRIVER = ( - REPO_ROOT / "tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh" -) - - -def _load_recipe() -> dict: - register_omegaconf_resolvers() - return OmegaConf.to_container(load_config(RECIPE), resolve=True) - - -def test_public_swe_recipe_has_supported_topology_and_telemetry( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("HF_HOME", "/hf-home") - config = _load_recipe() - generation = config["policy"]["generation"] - - class Tokenizer: - pad_token_id = 0 - eos_token_id = 1 - - tokenizer = Tokenizer() - configured_generation = configure_generation_config(generation, tokenizer) - config["policy"]["generation"] = configured_generation - setup_nemo_gym_config( - SimpleNamespace( - policy=config["policy"], env=config["env"], data=config["data"] - ), - tokenizer, - ) - vllm_cfg = configured_generation["vllm_cfg"] - assert configured_generation["backend"] == "vllm" - assert configured_generation["dynamo"] is True - assert vllm_cfg["async_engine"] is True - assert vllm_cfg["expose_http_server"] is True - - assert config["policy"]["model_name"] == ( - "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" - ) - assert config["cluster"]["gpus_per_node"] == 8 - assert config["cluster"]["num_nodes"] == 3 - assert config["cluster"]["segment_size"] == 1 - assert config["grpo"]["max_num_steps"] == 4 - assert config["grpo"]["async_grpo"]["enabled"] is True - assert config["grpo"]["async_grpo"]["in_flight_weight_updates"] is False - assert ( - config["grpo"]["async_grpo"]["recompute_kv_cache_after_weight_updates"] is True - ) - assert generation["colocated"]["resources"] == { - "gpus_per_node": 8, - "num_nodes": 1, - } - assert vllm_cfg["tensor_parallel_size"] * vllm_cfg["pipeline_parallel_size"] == 4 - assert vllm_cfg["expert_parallel_size"] == 4 - assert dynamo_cfg(configured_generation).frontend_args.router_mode == "kv" - assert vllm_cfg["enable_vllm_metrics_logger"] is True - assert vllm_cfg["load_format"] == "dummy" - assert config["env"]["nemo_gym"]["config_paths"][-1].endswith( - "swe_pivot_single_step_tool_use_with_argument_comparison.yaml" - ) - assert ( - config["env"]["nemo_gym"]["policy_model"]["responses_api_models"]["vllm_model"][ - "chat_template_kwargs" - ]["force_nonempty_content"] - is True - ) - assert ( - config["env"]["nemo_gym"]["single_step_tool_use_with_argument_comparison_swe"][ - "responses_api_agents" - ]["tool_simulation_agent"]["resources_server"]["name"] - == "swe_pivot_single_step_tool_use_with_argument_comparison_resources_server" - ) - assert config["logger"]["wandb_enabled"] is True - assert config["logger"]["tensorboard_enabled"] is True - assert config["logger"]["wandb"]["project"] == "nemo-rl" - assert config["data"]["train"]["data_path"].endswith( - "/superv3_data/swe1/train-split.jsonl" - ) - assert config["data"]["validation"]["data_path"].endswith( - "/superv3_data/swe1/val-split.jsonl" - ) - - -def test_recipe_and_driver_have_no_removed_swe_modes() -> None: - recipe_text = RECIPE.read_text(encoding="utf-8") - text = recipe_text + DRIVER.read_text(encoding="utf-8") - for forbidden in ( - "/lustre/", - "/path/to", - "/home/", - "jthomson", - "openhands", - "container_formatter", - "SIF_", - "effort_levels", - "hsg_r2", - "USES_SANDBOX", - ): - assert forbidden.lower() not in text.lower() - assert "load_format:" not in recipe_text - assert "--require-tag-prefix generation_metrics/" in text - assert "project: nemo-rl" in text - assert "Expected step ${MAX_STEPS}" in text - assert 'median(data["train/token_mult_prob_error"]) < 1.1' in text - assert "data['train/token_mult_prob_error']['${MAX_STEPS}'] < 1.1" in text - assert 'mean(data["train/gen_kl_error"]) < 0.02' in text - assert "Expected one cache invalidation per refit" in text From 97e2a4b1d39c9d8838110eedfb9c54419138ae45 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 19:10:29 -0800 Subject: [PATCH 38/82] docs(dynamo): review nits - The session-affinity comment is four lines, and drops the claim that the engine splices after routing: the frontend now splices before (C25). - _apply_dynamo_dialect names Gym's policy_model server (C38). - The guide links Gym's vllm_model config instead of Gym#4817 (C40). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 6 +++--- nemo_rl/environments/nemo_gym.py | 2 +- nemo_rl/models/generation/dynamo/config.py | 11 ++++------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 532313ee740..aee4d7e1eda 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -77,9 +77,9 @@ build it with `nemo_rl/models/generation/vllm/request_pipeline.py`. ## NeMo-Gym NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect -(`vllm_model.dynamo_frontend`, from -[NVIDIA-NeMo/Gym#4817](https://github.com/NVIDIA-NeMo/Gym/pull/4817)). The -frontend takes only OpenAI fields plus `nvext`, so Gym sends the native +(`dynamo_frontend` in Gym's +[`vllm_model`](https://github.com/NVIDIA-NeMo/Gym/tree/main/responses_api_models/vllm_model) +config). The frontend takes only OpenAI fields plus `nvext`, so Gym sends the native request's extra fields (`ng_capture`, `required_prefix_token_ids`, `ng_measure`) in one `nemo_rl:gym:` annotation, and reads the call's token fields (`prompt_token_ids`, `generation_token_ids`, `generation_log_probs`, diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 435f41e5235..177bc68042d 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -3024,7 +3024,7 @@ def build_nemo_gym_config( def _apply_dynamo_dialect(nemo_gym_dict: dict[str, Any]) -> None: - """Point one actor's policy model at the Dynamo frontend, in Gym's Dynamo dialect. + """Point Gym's `policy_model` server in one NemoGym actor's config at the Dynamo frontend. The frontend takes token data in nvext instead of vLLM's request extensions. Applied to the actor's composed config (a shard's overrides included), so diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 9e9181fc308..f31758011fa 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -39,13 +39,10 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "least-loaded", "device-aware-weighted", ] = "kv" - # Idle TTL in seconds for keeping one session on one engine, or None to - # disable it. NeMo-Gym names the session on token-capture calls only (its - # capture key), as the native backend pins each Gym session to one server; the - # engine splices a call's staged prefix after routing, so the KV router alone - # cannot see where that prefix is cached. Dynamo keeps at most 65,536 sessions - # and answers a new one past that with HTTP 529, so keep (captured rollouts - # started per second) x TTL below 65,536. The bound is the frontend's own. + # Idle seconds a NeMo-Gym token-capture session stays on one engine, as native + # pins each Gym session to one server; None disables it. Dynamo keeps at most + # 65,536 sessions (HTTP 529 past that), so keep (captured rollouts started per + # second) x TTL below that. session_affinity_ttl_secs: Annotated[int, Field(gt=0, le=31_536_000)] | None = 300 # Processes that render NeMo-Gym's chat calls and build their replies (the # checkpoint's template, tokenizer, HF processor and vLLM's parsers). Each From 95849865360bcd84bc185d0b0e685292ce7d4729 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 19:10:29 -0800 Subject: [PATCH 39/82] fix(dynamo): acknowledge a refused capture call to Gym through the frontend Natively, a token-capture call refused for its context length before it runs gets a 400 with the x-ng-capture-refused header, and Gym settles the call as refused only on that header. Dynamo's frontend forwards only an error's status and message, so through Dynamo the refused call stayed pending and its rollout untrainable. The chat engine now ends the 400's message with Gym's acknowledgement for the call (dynamo_capture_refusal_suffix), for the same refusals the native route acknowledges: the render's and the clamp's context overflows. Gym's dialect moves it back onto the header. Re-pins Gym to 9aac6094, which reads it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/guides/dynamo-generation.md | 7 ++-- .../generation/dynamo/vllm/chat_engine.py | 42 +++++++++++++------ .../generation/test_dynamo_chat_engine.py | 29 +++++++++++++ 4 files changed, 63 insertions(+), 17 deletions(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 4c84f1732ad..9aac6094048 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 4c84f1732ad5503f3bb695c360b28d24065de2d7 +Subproject commit 9aac609404809fc00cec07b5bf2c29df7197fa9d diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index aee4d7e1eda..229f9d2a498 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -108,9 +108,10 @@ same record and return the same reply. Where the behavior differs from the native endpoint: - The frontend refuses more than 4 `stop` strings or `stop_token_ids` (HTTP 500). -- A token-capture call refused for its context length is answered without the - native endpoint's `x-ng-capture-refused` header, which Dynamo's errors cannot - carry. +- A token-capture call refused for its context length acknowledges the refusal + at the end of the error message instead of in the native endpoint's + `x-ng-capture-refused` header, which Dynamo's errors cannot carry. Gym's + dialect moves it back to the header, so the harness sees the native error. - A call's processed media travels to the engine inside the request. ## Not supported diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index f0a59a2d259..52981f4144c 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -76,6 +76,22 @@ class _ContextLengthError(ValueError): """The prompt leaves no room for output; Gym reads "context length" in a 400.""" +def _capture_refusal_ack(extras: GymCallExtras) -> str: + """Gym's acknowledgement that a capture call was refused before it ran. + + The native route sends it as a header; Dynamo's frontend forwards only an + error's status and message, so it ends the message instead. + """ + if extras.ng_capture is None: + return "" + # Deferred: Gym is installed wherever a call carries ng_capture. + from nemo_gym.token_id_capture.external_capture import ( + dynamo_capture_refusal_suffix, + ) + + return dynamo_capture_refusal_suffix(extras.ng_capture["model_call_id"]) + + def _drop_none(value: Any) -> Any: """Dynamo hands Python unset fields as None, which vLLM's models reject.""" if isinstance(value, dict): @@ -179,19 +195,6 @@ async def generate( from nemo_rl.data.captured_media import MediaCaptureRejected from nemo_rl.models.generation.vllm.utils import OffPolicySamplingError - try: - yield await self._serve(request, context) - except ( - OffPolicySamplingError, - _ContextLengthError, - MediaCaptureRejected, - VLLMValidationError, - ) as error: - raise HttpError(400, str(error)) from error - - async def _serve(self, request: dict[str, Any], context: Any) -> dict[str, Any]: - from nemo_rl.models.generation.vllm.utils import on_policy_sampling - extras = GymCallExtras.from_annotations( (request.get("nvext") or {}).get("annotations") or () ) @@ -202,6 +205,19 @@ async def _serve(self, request: dict[str, Any], context: Any) -> dict[str, Any]: "nemo_rl:gym: annotation) on chat; NeMo RL's own rollouts use " "/v1/completions.", ) + try: + yield await self._serve(request, extras, context) + except (OffPolicySamplingError, MediaCaptureRejected) as error: + raise HttpError(400, str(error)) from error + except (_ContextLengthError, VLLMValidationError) as error: + # Refused before the engine ran, so nothing was staged: as natively. + raise HttpError(400, str(error) + _capture_refusal_ack(extras)) from error + + async def _serve( + self, request: dict[str, Any], extras: GymCallExtras, context: Any + ) -> dict[str, Any]: + from nemo_rl.models.generation.vllm.utils import on_policy_sampling + body = _drop_none( {k: v for k, v in request.items() if k not in _DYNAMO_ONLY_FIELDS} ) diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index f1a2824f008..e6d51b01085 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -92,12 +92,15 @@ def __init__(self, rendered: RenderedCall, reply: dict | None = None) -> None: } self.bodies: list[dict] = [] self.responded: list[tuple] = [] + self.error: Exception | None = None async def info(self) -> dict: return {"turn_end_token_ids": [EOS], "max_model_len": MAX_MODEL_LEN} async def render(self, body: dict) -> RenderedCall: self.bodies.append(body) + if self.error is not None: + raise self.error return self.rendered async def respond(self, request, prompt_token_ids, output) -> dict: @@ -370,6 +373,32 @@ def test_gym_calls_nemo_rl_cannot_serve_are_refused_with_a_400( assert router.requests == [] +@pytest.mark.nemo_gym +@pytest.mark.parametrize("captured", [True, False]) +@pytest.mark.parametrize("refused_by", ["render", "clamp"]) +def test_a_call_refused_before_it_runs_acknowledges_its_capture_to_gym( + captured, refused_by +) -> None: + from nemo_gym.token_id_capture.external_capture import ( + split_dynamo_capture_refusal, + ) + from vllm.exceptions import VLLMValidationError + + # The prompt alone fills the context. + pool = _Pool(_rendered(template_token_ids=list(range(MAX_MODEL_LEN)))) + if refused_by == "render": + pool.error = VLLMValidationError("maximum context length is 16 tokens") + extras = {"ng_capture": TOKEN_IN} if captured else {} + + with pytest.raises(HttpError) as refused: + _serve(pool, _Router(), _call(extras)) + + # Gym reads the acknowledgement off the message, as natively off the header. + message, refused_call = split_dynamo_capture_refusal(refused.value.message) + assert (refused.value.code, "maximum context length" in message) == (400, True) + assert refused_call == (TOKEN_IN["model_call_id"] if captured else None) + + def test_an_engine_error_fails_the_call() -> None: with pytest.raises(HttpError, match="engine died") as failed: _serve(_Pool(_rendered()), _Router(error="engine died"), _call({})) From 9331ee7077356669fe8f52c1aaadf81f85096110 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 19:49:35 -0800 Subject: [PATCH 40/82] test(dynamo): add the step-order cases the review asked for Each case now runs on a kit (a checkpoint and the chat settings both paths serve), and states the status both paths must answer and the mode capture stages it in, so a failure both paths share no longer passes as parity. New cases: - turn_end_not_eos: a checkpoint whose turns end on a generation-config EOS that is not the tokenizer's, as gpt-oss's <|call|> (C16). - empty_tool_content / _structured: a candidate whose tool call's empty content became none holds its cut with string content, and roots a new segment with structured content (C26). - image_after_echoed_turn: a Qwen2.5-VL call after an echoed turn; both engines run the same pixels and image-token ranges (C27 case 4, C30). - nano_v3_thinking_off: Nemotron-3-Nano's reasoning parser plugin (C39). - on_policy_sampling and one off_policy case per sampling field, logit_bias included (C41, C46). Each new case fails when the code it covers is reverted. Staging an image's pixels is not covered: only the Omni processor supports it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../models/generation/test_gym_chat_path.py | 421 ++++++++++++++---- 1 file changed, 339 insertions(+), 82 deletions(-) diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 6d2dff024ae..2ebcd888dae 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -15,20 +15,24 @@ Each case sends one call to the native vLLM server (its real routes, over ASGI) and through Dynamo (NeMo RL's chat engine, a spawned render pool, and the -engine adapter), over the same tiny Qwen3 checkpoint, the same scripted engine -and Gym's real capture core. Both must run the engine on the same prompt with -the same SamplingParams, stage the same capture record, and answer Gym with the -same message and token fields. +engine adapter), with the same kit (a checkpoint and the chat settings both +paths serve), the same scripted engine and Gym's real capture core. Both must +run the engine on the same prompt, media and SamplingParams, stage the same +capture record, and answer Gym with the same message and token fields. -Not covered here: media (the checkpoint is text-only; the VLM functional -overlay runs it) and a model whose EOS is not its turn end. +Not covered here: staging an image's pixels, which needs the Omni processor +and its gated checkpoint. """ import asyncio +import base64 +import io import json +import shutil from dataclasses import dataclass, field +from pathlib import Path from types import SimpleNamespace -from typing import Any +from typing import Any, Callable, Optional import pytest import torch @@ -41,6 +45,7 @@ import httpx # noqa: E402 from dynamo.llm.exceptions import HttpError # noqa: E402 from fastapi import FastAPI # noqa: E402 +from PIL import Image # noqa: E402 from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 from nemo_gym.token_id_capture.replay import ( # noqa: E402 replay_context, @@ -53,7 +58,7 @@ CaptureAdmission, StageResult, ) -from transformers import AutoTokenizer # noqa: E402 +from transformers import AutoConfig, AutoProcessor, AutoTokenizer # noqa: E402 from vllm.config import CacheConfig, LoadConfig, VllmConfig # noqa: E402 from vllm.engine.arg_utils import AsyncEngineArgs # noqa: E402 from vllm.logprobs import Logprob # noqa: E402 @@ -121,6 +126,42 @@ '{"name": "get_weather", "arguments": {"city": "Lyon"}}\n' ) TURN_1_REPLY = "\neasy\n\n\nIt is sunny." +# Turn 1 as a tool call, which Gym's source history recorded with empty +# content and the harness now sends with none. +TOOL_TURN = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-0", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], +} +TOOL_HISTORY = [HISTORY[0], {**TOOL_TURN, "content": ""}] +TOOL_RESULT = {"role": "tool", "tool_call_id": "call-0", "content": "sunny"} + + +def _image_url() -> str: + image = Image.new("RGB", (56, 56)) + image.putdata([(4 * x, 4 * y, 128) for y in range(56) for x in range(56)]) + png = io.BytesIO() + image.save(png, format="PNG") + return "data:image/png;base64," + base64.b64encode(png.getvalue()).decode() + + +IMAGE_TURN = { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": _image_url()}}, + {"type": "text", "text": "And here?"}, + ], +} +NANO_V3_PLUGIN = str( + Path(__file__).parents[4] + / "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py" +) @dataclass @@ -128,6 +169,8 @@ class _Kit: """The checkpoint, and the vLLM objects both paths share.""" path: str + generation: dict[str, Any] + capture_media: bool tokenizer: Any engine_args: Any model_config: Any @@ -157,11 +200,11 @@ def __init__(self, kit: _Kit) -> None: self.io_processor = None self.errored = False self._kit = kit - self.runs: list[tuple[list[int], Any]] = [] + self.runs: list[tuple[list[int], Any, Any]] = [] async def generate(self, prompt, sampling_params, request_id, **kwargs): prompt_token_ids = list(prompt["prompt_token_ids"]) - self.runs.append((prompt_token_ids, sampling_params)) + self.runs.append((prompt_token_ids, sampling_params, _media(prompt))) reply_ids = self._kit.ids(REPLY) logprobs = None if sampling_params.logprobs is not None: @@ -203,6 +246,36 @@ async def abort(self, request_id) -> None: pass +def _media(prompt) -> Any: + """The processed media an engine prompt runs: its token ranges and tensors.""" + if "mm_kwargs" not in prompt: + return None + return ( + { + modality: [(item.offset, item.length) for item in ranges] + for modality, ranges in prompt["mm_placeholders"].items() + }, + { + modality: [{key: elem.data for key, elem in item.items()} for item in items] + for modality, items in prompt["mm_kwargs"].items() + }, + ) + + +def _assert_same_media(dynamo, native) -> None: + if native is None: + assert dynamo is None + return + assert dynamo[0] == native[0] + assert dynamo[1].keys() == native[1].keys() + for modality, items in native[1].items(): + assert len(dynamo[1][modality]) == len(items) + for dynamo_item, item in zip(dynamo[1][modality], items): + assert dynamo_item.keys() == item.keys() + for key, tensor in item.items(): + assert torch.equal(dynamo_item[key], tensor), (modality, key) + + class _Sink: def __init__(self) -> None: self.staged: list[tuple[Any, Any]] = [] @@ -230,19 +303,19 @@ class _Outcome: status: int reply: dict[str, Any] - runs: list[tuple[list[int], Any]] + runs: list[tuple[list[int], Any, Any]] staged: list[tuple[Any, Any]] failed: list[str] = field(default_factory=list) def _host(kit: _Kit, deltas: dict[str, list[int]], top_logprobs: int): host = object.__new__(VllmAsyncGenerationWorkerImpl) - host.cfg = GENERATION + host.cfg = kit.generation host.llm = host._http_engine_client = _Engine(kit) host.llm_async_engine_args = kit.engine_args host.model_name = "model" host.routed_experts_dtype = torch.int16 - host._capture_media = False + host._capture_media = kit.capture_media host._capture_calls = {} host._capture_top_logprobs = top_logprobs host._staging_source = None @@ -270,10 +343,12 @@ async def _native(kit, host, sink, failed, body, *, measure: bool) -> _Outcome: host, engine_client=host._http_engine_client, model_config=kit.model_config, - cfg=GENERATION, + cfg=kit.generation, ) app = FastAPI() - chat_server.register_routes(app, server, host=host, generation_config=GENERATION) + chat_server.register_routes( + app, server, host=host, generation_config=kit.generation + ) async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://native" ) as client: @@ -323,6 +398,9 @@ def data(self): async def _dynamo(kit, host, sink, failed, body, extras) -> _Outcome: request = {k: v for k, v in body.items() if k not in extras} + # Dynamo's request names the template kwargs chat_template_args. + if "chat_template_kwargs" in request: + request["chat_template_args"] = request.pop("chat_template_kwargs") request["nvext"] = { "annotations": [GYM_EXTRAS_ANNOTATION_PREFIX + json.dumps(extras)] } @@ -345,10 +423,80 @@ async def _dynamo(kit, host, sink, failed, body, extras) -> _Outcome: return _Outcome(200, chunk, host.llm.runs, sink.staged, failed) -@pytest.fixture(scope="module") -def kit(tiny_qwen3_model_path): +def _generation(*, vllm_cfg=None, **chat_settings) -> dict[str, Any]: + """GENERATION with the native chat settings a recipe changes.""" + base = GENERATION["vllm_cfg"] + return { + **GENERATION, + "vllm_cfg": { + **base, + **(vllm_cfg or {}), + "http_server_serving_chat_kwargs": { + **base["http_server_serving_chat_kwargs"], + **chat_settings, + }, + }, + } + + +def _turn_end_not_eos(path: str, tmp_path_factory) -> str: + """The checkpoint, shaped like gpt-oss: its turns end on a generation-config + EOS (<|im_end|>) that is not the tokenizer's (<|endoftext|>).""" + copy = tmp_path_factory.mktemp("turn_end_not_eos") + shutil.copytree(path, copy, dirs_exist_ok=True) + tokenizer_config = json.loads((copy / "tokenizer_config.json").read_text()) + tokenizer = AutoTokenizer.from_pretrained(path) + eos = tokenizer.convert_tokens_to_ids(["<|im_end|>", "<|endoftext|>"]) + tokenizer_config["eos_token"] = "<|endoftext|>" + (copy / "tokenizer_config.json").write_text(json.dumps(tokenizer_config)) + (copy / "generation_config.json").write_text(json.dumps({"eos_token_id": eos})) + return str(copy) + + +def _vlm(path: str, tmp_path_factory) -> str: + """Qwen2.5-VL's config and processor: rendering never loads weights.""" + del path + checkpoint = tmp_path_factory.mktemp("qwen2_5_vl") + AutoConfig.from_pretrained("Qwen/Qwen2.5-VL-3B-Instruct").save_pretrained( + checkpoint + ) + AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-3B-Instruct").save_pretrained( + checkpoint + ) + return str(checkpoint) + + +@dataclass(frozen=True) +class _KitSpec: + generation: dict[str, Any] + # The checkpoint, from the tiny Qwen3's path. + checkpoint: Callable[..., str] = lambda path, _: path + capture_media: bool = False + + +KITS = { + "qwen3": _KitSpec(GENERATION), + "turn_end_not_eos": _KitSpec(GENERATION, _turn_end_not_eos), + # The template gets structured content parts instead of strings. + "structured_content": _KitSpec(_generation(chat_template_content_format="openai")), + # Nemotron-3-Nano's parser, a plugin file the native recipe loads. + "nano_v3": _KitSpec( + _generation( + vllm_cfg={"reasoning_parser_plugin": NANO_V3_PLUGIN}, + reasoning_parser="nano_v3", + ) + ), + # Qwen2.5's tokenizer has no thinking tokens for Qwen3's reasoning parser. + "vlm": _KitSpec(_generation(reasoning_parser=""), _vlm), +} + + +def _build_kit(kit_spec: _KitSpec, path: str) -> _Kit: + generation = kit_spec.generation + # No processor cache: each case's engine gets the media itself, not a handle + # to media an earlier case processed. engine_args = AsyncEngineArgs( - model=tiny_qwen3_model_path, max_model_len=MAX_MODEL_LEN + model=path, max_model_len=MAX_MODEL_LEN, mm_processor_cache_gb=0 ) model_config = engine_args.create_model_config() vllm_config = VllmConfig( @@ -360,21 +508,39 @@ def kit(tiny_qwen3_model_path): spec = json.loads( json.dumps( chat_runtime_data( - SimpleNamespace(llm_async_engine_args=engine_args, cfg=GENERATION) + SimpleNamespace(llm_async_engine_args=engine_args, cfg=generation) ) ) )[chat_engine.RUNTIME_DATA_KEY] - pool = RenderPool(spec, tiny_qwen3_model_path, workers=1) - yield _Kit( - path=tiny_qwen3_model_path, - tokenizer=AutoTokenizer.from_pretrained(tiny_qwen3_model_path), + return _Kit( + path=path, + generation=generation, + capture_media=kit_spec.capture_media, + tokenizer=AutoTokenizer.from_pretrained(path), engine_args=engine_args, model_config=model_config, vllm_config=vllm_config, renderer=renderer_from_config(vllm_config), - pool=pool, + pool=RenderPool(spec, path, workers=1), ) - pool.shutdown() + + +@pytest.fixture(scope="module") +def kits(tiny_qwen3_model_path, tmp_path_factory): + """Each kit in KITS, built on first use.""" + built: dict[str, _Kit] = {} + + def kit(name: str) -> _Kit: + if name not in built: + spec = KITS[name] + built[name] = _build_kit( + spec, spec.checkpoint(tiny_qwen3_model_path, tmp_path_factory) + ) + return built[name] + + yield kit + for built_kit in built.values(): + built_kit.pool.shutdown() def _admission(**fields) -> dict[str, Any]: @@ -394,81 +560,164 @@ def _staged_turn_1(kit: _Kit, **fields) -> dict[str, Any]: ) -def _candidate(kit: _Kit, messages: list[dict]) -> dict[str, Any]: +def _candidate( + kit: _Kit, + messages: list[dict], + source: list[dict] = HISTORY, + *, + empty_tool_content: bool = False, +) -> dict[str, Any]: """Gym's candidate: turn 1's source, and this request's, for RL to check the cut.""" return _staged_turn_1( kit, mode="candidate", - requires_empty_tool_content_equivalence=False, + requires_empty_tool_content_equivalence=empty_tool_content, candidate_replay=summarize_replay( - replay_context(HISTORY, render_digest="b" * 64) + replay_context(source, render_digest="b" * 64) ), request_replay=replay_context(messages, render_digest="b" * 64), ) -# id, messages, extras(kit), request fields, capture top-k +@dataclass(frozen=True) +class _Case: + """One Gym call, and what both paths must do with it.""" + + id: str + messages: list[dict] = field(default_factory=lambda: [*HISTORY, OBSERVATION]) + extras: Callable[[_Kit], dict[str, Any]] = lambda kit: {} + fields: dict[str, Any] = field(default_factory=dict) + # The sampler top-k capture stages, for score centering. + top_logprobs: int = 0 + # 200, or the 400 both paths refuse the call with before it runs. + status: int = 200 + # The mode capture stages the call in; None when the call is not captured. + mode: Optional[str] = None + kit: str = "qwen3" + + +def _off_policy(name: str, value: Any) -> _Case: + return _Case(f"off_policy_{name}", fields={name: value}, status=400) + + CASES = [ # A plain call: the render and the reply's reasoning and tool parsers (C39). - ("plain", [*HISTORY, OBSERVATION], lambda kit: {}, {}, 0), + _Case("plain"), # Gym's echo of turn 1 without capture: the model's tokens replace the # template's re-render of that turn (C16's turn-end cut). - ( + _Case( "inline_prefix", - [*HISTORY, OBSERVATION], - lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, - {}, - 0, + extras=lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, + ), + # The same, for a model whose turns end on a token that is not its + # tokenizer's EOS, as gpt-oss's <|call|> (C16). + _Case( + "turn_end_not_eos", + extras=lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, + kit="turn_end_not_eos", ), # Token capture: turn 1's staged tokens, read from TQ. - ( + _Case( "token_in", - [*HISTORY, OBSERVATION], - lambda kit: {"ng_capture": _staged_turn_1(kit, mode="token_in")}, - {}, - 0, + extras=lambda kit: {"ng_capture": _staged_turn_1(kit, mode="token_in")}, + mode="token_in", ), # Gym's candidate whose cut holds is served token-in (C26)... - ( + _Case( "candidate_cut_holds", - [*HISTORY, OBSERVATION], - lambda kit: {"ng_capture": _candidate(kit, [*HISTORY, OBSERVATION])}, - {}, - 0, + extras=lambda kit: {"ng_capture": _candidate(kit, [*HISTORY, OBSERVATION])}, + mode="token_in", ), # ...and one whose conversion moved the cut roots a new text segment. - ( + _Case( "candidate_cut_fails", - [*HISTORY, OBSERVATION, HISTORY[-1]], - lambda kit: {"ng_capture": _candidate(kit, [*HISTORY, OBSERVATION])}, - {}, - 0, + messages=[*HISTORY, OBSERVATION, HISTORY[-1]], + extras=lambda kit: {"ng_capture": _candidate(kit, [*HISTORY, OBSERVATION])}, + mode="text", + ), + # A candidate whose tool call had empty content and now has none: string + # content renders both alike, so the cut holds (C26)... + _Case( + "empty_tool_content", + messages=[HISTORY[0], TOOL_TURN, TOOL_RESULT], + extras=lambda kit: { + "ng_capture": _candidate( + kit, + [HISTORY[0], TOOL_TURN, TOOL_RESULT], + TOOL_HISTORY, + empty_tool_content=True, + ) + }, + mode="token_in", + ), + # ...and structured content cannot prove that, so it roots a new segment. + _Case( + "empty_tool_content_structured", + messages=[HISTORY[0], TOOL_TURN, TOOL_RESULT], + extras=lambda kit: { + "ng_capture": _candidate( + kit, + [HISTORY[0], TOOL_TURN, TOOL_RESULT], + TOOL_HISTORY, + empty_tool_content=True, + ) + }, + mode="text", + kit="structured_content", ), # Score centering stages the sampler's top-k with the call (C29). - ( + _Case( "score_centering", - [*HISTORY, OBSERVATION], - lambda kit: {"ng_capture": _admission(mode="text")}, - {"top_logprobs": 2}, - 2, + extras=lambda kit: {"ng_capture": _admission(mode="text")}, + fields={"top_logprobs": 2}, + top_logprobs=2, + mode="text", + ), + # An image after the echoed turn 1: processed once in the frontend, both + # engines run the same pixels, and the splice moves its tokens (C30). + _Case( + "image_after_echoed_turn", + messages=[*HISTORY, IMAGE_TURN], + extras=lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, + kit="vlm", + ), + # Nemotron-3-Nano's reasoning parser plugin, with thinking off (C39). + _Case( + "nano_v3_thinking_off", + fields={"chat_template_kwargs": {"enable_thinking": False}}, + kit="nano_v3", + ), + # The sampling the trainer replays, set explicitly (C41)... + _Case( + "on_policy_sampling", + fields={ + "top_k": -1, + "min_p": 0.0, + "repetition_penalty": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "logit_bias": {}, + }, ), - # Off-policy sampling is refused before anything runs (C41). - ("off_policy", [*HISTORY, OBSERVATION], lambda kit: {}, {"temperature": 0.5}, 0), + # ...and any it does not is refused before anything runs. + _off_policy("temperature", 0.5), + _off_policy("top_p", 0.5), + _off_policy("top_k", 50), + _off_policy("min_p", 0.1), + _off_policy("repetition_penalty", 1.05), + _off_policy("presence_penalty", 0.5), + _off_policy("frequency_penalty", 0.5), + _off_policy("logit_bias", {"10": 5.0}), ] -@pytest.mark.parametrize( - ("messages", "extras", "fields", "top_logprobs"), - [case[1:] for case in CASES], - ids=[case[0] for case in CASES], -) -def test_a_gym_call_takes_the_same_steps_through_dynamo_as_natively( - kit, messages, extras, fields, top_logprobs -) -> None: - extras = extras(kit) +@pytest.mark.parametrize("case", CASES, ids=[case.id for case in CASES]) +def test_a_gym_call_takes_the_same_steps_through_dynamo_as_natively(kits, case) -> None: + kit = kits(case.kit) + extras = case.extras(kit) body = { "model": kit.path, - "messages": messages, + "messages": case.messages, "tools": TOOLS, "temperature": 0.7, "top_p": 0.9, @@ -476,33 +725,40 @@ def test_a_gym_call_takes_the_same_steps_through_dynamo_as_natively( # Gym's token information. "logprobs": True, "top_logprobs": 0, - **fields, + **case.fields, **extras, } deltas = {"rollout-1/c1": kit.turn_1_tokens()} async def both(): native = await _native( - kit, *_host(kit, deltas, top_logprobs), body, measure=False + kit, *_host(kit, deltas, case.top_logprobs), body, measure=False + ) + dynamo = await _dynamo( + kit, *_host(kit, deltas, case.top_logprobs), body, extras ) - dynamo = await _dynamo(kit, *_host(kit, deltas, top_logprobs), body, extras) return native, dynamo native, dynamo = asyncio.run(both()) - assert (native.status, dynamo.status) == ( - native.status, - native.status, - ), (native.reply, dynamo.reply) - if native.status != 200: + assert (native.status, dynamo.status) == (case.status, case.status), ( + native.reply, + dynamo.reply, + ) + if case.status != 200: assert native.runs == dynamo.runs == [] return - # The engine ran the same prompt with the same SamplingParams. - ((native_prompt, native_params),) = native.runs - ((dynamo_prompt, dynamo_params),) = dynamo.runs + # The engine ran the same prompt, media and SamplingParams. + ((native_prompt, native_params, native_media),) = native.runs + ((dynamo_prompt, dynamo_params, dynamo_media),) = dynamo.runs assert dynamo_prompt == native_prompt + _assert_same_media(dynamo_media, native_media) + assert (native_media is not None) == (case.kit == "vlm") assert dynamo_params == native_params - # Capture staged the same record and tensors. + # Capture staged the same record and tensors, in the expected mode. + assert [record.mode for record, _ in native.staged] == ( + [case.mode] if case.mode else [] + ) assert len(dynamo.staged) == len(native.staged) for (native_record, native_tensors), (dynamo_record, dynamo_tensors) in zip( native.staged, dynamo.staged @@ -518,8 +774,8 @@ async def both(): assert choice["finish_reason"] == native.reply["choices"][0]["finish_reason"] assert choice["delta"]["content"] == message["content"] assert choice["delta"].get("reasoning_content") == message.get("reasoning") - assert [call["function"] for call in choice["delta"]["tool_calls"]] == [ - call["function"] for call in message["tool_calls"] + assert [call["function"] for call in choice["delta"].get("tool_calls") or []] == [ + call["function"] for call in message.get("tool_calls") or [] ] result = dynamo.reply["nvext"]["nemo_rl"] if native.staged: @@ -535,7 +791,8 @@ async def both(): } -def test_a_measure_call_counts_the_prompt_the_call_would_run(kit) -> None: +def test_a_measure_call_counts_the_prompt_the_call_would_run(kits) -> None: + kit = kits("qwen3") extras = {"required_prefix_token_ids": kit.turn_1_tokens()} body = { "model": kit.path, From cca9af2cfd827e6cbdf56e707bae45f66ccd6041 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 19:49:35 -0800 Subject: [PATCH 41/82] test(dynamo): drop the adapter test helper's unused prompt_logprobs Nothing on the Dynamo path reads a request's prompt log probabilities (C44). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/models/generation/test_dynamo_vllm_adapter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index 7305346757e..89b2dcad3c1 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -100,11 +100,10 @@ def _completion(token_ids, finish_reason=None, index=0, routed_experts=None): ) -def _output(prompt, *completions, prompt_routed_experts=None, prompt_logprobs=None): +def _output(prompt, *completions, prompt_routed_experts=None): return SimpleNamespace( prompt_token_ids=prompt, num_cached_tokens=0, - prompt_logprobs=prompt_logprobs, prompt_routed_experts=prompt_routed_experts, outputs=list(completions), ) From 53a5298a5ec5d5f03f73d2539ddfbf67dc75df4d Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 20:38:58 -0800 Subject: [PATCH 42/82] fix(dynamo): answer a Gym call the engine refuses with the native status Dynamo hands an error the engine raises to the frontend's chat engine as an error item without its status, so a Gym call the engine refused before it ran came back as a 500 with the message hidden. Natively it is a 4xx (a vLLM request error, a refused media capture) or vLLM's 503 for a full queue, which clients and rollout retries treat differently. The adapter now answers such a call in engine_data, with the status vLLM's create_error_response gives the error (400 for a refused media capture), and the chat engine raises it. A validation error also acknowledges a capture call's refusal to Gym, as the native server's header does. The step-order test's image_capture_unsupported case covers it on both paths; reverting the adapter to raise fails it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 6 ++- .../models/generation/dynamo/vllm/adapter.py | 42 +++++++++++++--- .../generation/dynamo/vllm/chat_engine.py | 4 ++ .../generation/test_dynamo_chat_engine.py | 48 +++++++++++++++---- .../generation/test_dynamo_vllm_adapter.py | 48 +++++++++---------- .../models/generation/test_gym_chat_path.py | 9 ++++ 6 files changed, 118 insertions(+), 39 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 5bf185fac66..0b48b7c536e 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -59,7 +59,11 @@ The two together: (`_capture_request_media`), `begin_call` (which stamps the weight version), the generation with the `SamplingParams` vLLM's chat serving builds from the request, and the one TQ write. Its `engine_data` carries the output the - frontend builds the reply from, and the call's `GymCallResult`. + frontend builds the reply from, and the call's `GymCallResult`. A call the + engine refuses before it runs (a vLLM request error, a full queue, media + capture refused) comes back in `engine_data` too, with the status the native + server answers it with: Dynamo hands a raised error to the chat engine + without its status. - `abort` cancels the vLLM request on client disconnect. - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index 4f7eebc36bd..044f7e22b12 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -37,6 +37,7 @@ import os import pickle from collections.abc import AsyncGenerator +from http import HTTPStatus from typing import TYPE_CHECKING, Any, Optional import torch @@ -49,8 +50,11 @@ ) from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource from dynamo.llm.exceptions import InvalidArgument +from vllm.entrypoints.serve.exception_handling.error_response import ( + create_error_response, +) from vllm.entrypoints.serve.utils.api_utils import get_max_tokens -from vllm.exceptions import VLLMClientError +from vllm.exceptions import GracefulHTTPError, VLLMClientError, VLLMValidationError from vllm.inputs import TokensPrompt from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase @@ -287,12 +291,17 @@ async def _generate_gym( chunk = await self._serve_gym_call( plan, list(request["token_ids"]), context ) - except VLLMClientError as error: - # vLLM's request errors are not RuntimeErrors; the native server - # answers them with a 4xx that carries the message. - raise InvalidArgument(str(error)) from error + except (VLLMClientError, GracefulHTTPError) as error: + # The status vLLM's server answers it with; a native 400 for a + # validation error also acknowledges a capture call's refusal. + response = create_error_response(error).error + chunk = _refusal( + response.code, + response.message, + acknowledge=isinstance(error, VLLMValidationError), + ) except MediaCaptureRejected as error: - raise InvalidArgument(f"{error} ({error.code})") from error + chunk = _refusal(HTTPStatus.BAD_REQUEST, f"{error} ({error.code})") if chunk is not None: yield chunk @@ -557,6 +566,27 @@ def _wire_logprobs(logprobs: Any) -> Optional[list[dict[str, list[Any]]]]: ] +def _refusal(status: int, message: str, *, acknowledge: bool = False) -> GenerateChunk: + """A Gym call the engine refused before it ran, answered in the chunk. + + Dynamo hands an error the engine raises to the frontend's chat engine + without its status, so it could only answer 500; the chat engine answers + this with ``status``, as the native server does. + """ + return { + "token_ids": [], + "index": 0, + "finish_reason": "stop", + "engine_data": { + "refusal": { + "status": int(status), + "message": message, + "acknowledge": acknowledge, + } + }, + } + + def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: prompt_tokens = len(output.prompt_token_ids or ()) return { diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 52981f4144c..25aa8197cbb 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -292,6 +292,10 @@ async def _serve( "extra_args": {PLAN_KEY: plan}, } engine_data = await self._call_engine(preprocessed, context) + if (refusal := engine_data.get("refusal")) is not None: + # The engine refused the call before it ran, with the native status. + ack = _capture_refusal_ack(extras) if refusal["acknowledge"] else "" + raise HttpError(refusal["status"], refusal["message"] + ack) result = GymCallResult.model_validate(engine_data["result"]) reply_request = engine_request if result.ng_commit_coords is not None: diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index e6d51b01085..fa2b950a39a 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -375,7 +375,7 @@ def test_gym_calls_nemo_rl_cannot_serve_are_refused_with_a_400( @pytest.mark.nemo_gym @pytest.mark.parametrize("captured", [True, False]) -@pytest.mark.parametrize("refused_by", ["render", "clamp"]) +@pytest.mark.parametrize("refused_by", ["render", "clamp", "engine"]) def test_a_call_refused_before_it_runs_acknowledges_its_capture_to_gym( captured, refused_by ) -> None: @@ -384,14 +384,25 @@ def test_a_call_refused_before_it_runs_acknowledges_its_capture_to_gym( ) from vllm.exceptions import VLLMValidationError - # The prompt alone fills the context. - pool = _Pool(_rendered(template_token_ids=list(range(MAX_MODEL_LEN)))) + pool = _Pool(_rendered()) + router = _Router() if refused_by == "render": pool.error = VLLMValidationError("maximum context length is 16 tokens") + elif refused_by == "clamp": + # The prompt alone fills the context. + pool.rendered = _rendered(template_token_ids=list(range(MAX_MODEL_LEN))) + else: + router.engine_data = { + "refusal": { + "status": 400, + "message": "maximum context length is 16 tokens", + "acknowledge": True, + } + } extras = {"ng_capture": TOKEN_IN} if captured else {} with pytest.raises(HttpError) as refused: - _serve(pool, _Router(), _call(extras)) + _serve(pool, router, _call(extras)) # Gym reads the acknowledgement off the message, as natively off the header. message, refused_call = split_dynamo_capture_refusal(refused.value.message) @@ -399,11 +410,32 @@ def test_a_call_refused_before_it_runs_acknowledges_its_capture_to_gym( assert refused_call == (TOKEN_IN["model_call_id"] if captured else None) -def test_an_engine_error_fails_the_call() -> None: - with pytest.raises(HttpError, match="engine died") as failed: - _serve(_Pool(_rendered()), _Router(error="engine died"), _call({})) +@pytest.mark.parametrize( + ("router", "status", "message"), + [ + (_Router(error="engine died"), 500, "engine died"), + # Refused before it ran, with the status the native server answers. + ( + _Router( + engine_data={ + "refusal": { + "status": 503, + "message": "queue full", + "acknowledge": False, + } + } + ), + 503, + "queue full", + ), + ], + ids=["error", "refusal"], +) +def test_an_engine_that_does_not_serve_the_call_fails_it(router, status, message): + with pytest.raises(HttpError, match=message) as failed: + _serve(_Pool(_rendered()), router, _call({})) - assert failed.value.code == 500 + assert failed.value.code == status def test_only_nemo_rls_engines_are_served() -> None: diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index 89b2dcad3c1..e7a95926cdc 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -561,42 +561,42 @@ def test_a_gym_call_without_routes_fails_under_router_replay() -> None: _collect(engine, _gym_call([1, 3, 4])) -@pytest.mark.parametrize( - ("call", "error"), - [ - ( - {"token_ids": [1, 3, 4], "annotations": [GYM_CALL_ANNOTATION]}, - r"extra_args\.nemo_rl", - ), - # vLLM's own request errors (not RuntimeErrors) keep their message, as - # the native server's 4xx does, instead of becoming a bare 500. - ( - _gym_call([1, 3, 4], request=_chat(min_tokens=10, max_tokens=5)), - "min_tokens", - ), - ], -) -def test_gym_calls_the_engine_cannot_serve_are_refused(call, error) -> None: +def test_a_gym_call_without_the_chat_engines_plan_is_refused() -> None: llm = _FakeLLM() engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) - with pytest.raises(InvalidArgument, match=error): - _collect(engine, call) + with pytest.raises(InvalidArgument, match=r"extra_args\.nemo_rl"): + _collect(engine, {"token_ids": [1, 3, 4], "annotations": [GYM_CALL_ANNOTATION]}) assert llm.calls == [] -def test_a_gym_call_over_the_engines_queue_limits_is_refused_before_it_runs() -> None: - full = vllm.exceptions.QueueOverflowError() +@pytest.mark.parametrize( + ("chat", "admission_error", "status", "acknowledge", "message"), + [ + # vLLM's own validation errors keep their 400 and message, as natively, + # and acknowledge a capture call's refusal. + (_chat(min_tokens=10, max_tokens=5), None, 400, True, "min_tokens"), + # The queue limits the native server checks: a 503 to retry elsewhere. + (_chat(), vllm.exceptions.QueueOverflowError(), 503, False, "busy"), + ], + ids=["request_error", "queue_full"], +) +def test_a_gym_call_the_engine_refuses_answers_with_the_native_status( + chat, admission_error, status, acknowledge, message +) -> None: llm = _FakeLLM( - [_output([1, 3, 4], _completion([10], "stop"))], admission_error=full + [_output([1, 3, 4], _completion([10], "stop"))], + admission_error=admission_error, ) engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) - # As the native server checks max_num_queued_reqs / max_num_queued_tokens. - with pytest.raises(vllm.exceptions.QueueOverflowError): - _collect(engine, _gym_call([1, 3, 4])) + # In the chunk: Dynamo hands a raised error to the chat engine without it. + (chunk,) = _collect(engine, _gym_call([1, 3, 4], request=chat)) + refusal = chunk["engine_data"]["refusal"] + assert (refusal["status"], refusal["acknowledge"]) == (status, acknowledge) + assert message in refusal["message"] assert llm.calls == [] diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 2ebcd888dae..6bfa54b2719 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -673,6 +673,15 @@ def _off_policy(name: str, value: Any) -> _Case: top_logprobs=2, mode="text", ), + # Capture with a processor that has no media capture is refused before it + # runs, with the native server's 400. + _Case( + "image_capture_unsupported", + messages=[IMAGE_TURN], + extras=lambda kit: {"ng_capture": _admission(mode="text")}, + status=400, + kit="vlm", + ), # An image after the echoed turn 1: processed once in the frontend, both # engines run the same pixels, and the splice moves its tokens (C30). _Case( From 6c566031ac08ae5083dc7506c239dd09cc6d03af Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 7 Nov 2026 20:59:06 -0800 Subject: [PATCH 43/82] docs(dynamo): say what token_capture.generation_backend picks The comment now says the value picks how NeMo-Gym reaches the capture host, and why Dynamo needs its own handler (C34). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/algorithms/single_controller_utils/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py index 6b75c510a0c..adee5a9dfab 100644 --- a/nemo_rl/algorithms/single_controller_utils/config.py +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -689,10 +689,11 @@ class TokenCaptureConfig(BaseModel, extra="allow"): # derived at setup # under the run's log dir. capture_dir: Optional[str] = None - # Generation backend hosting token capture. This is derived from - # policy.generation.backend during setup; users should not set it separately. - # "dynamo" is the vLLM backend served through Dynamo (policy.generation.dynamo): - # the same workers host capture, and NeMo-Gym reaches them through the frontend. + # How NeMo-Gym reaches the capture host; derived during setup, users should not + # set it. "dynamo" = vLLM served through the Dynamo frontend + # (policy.generation.dynamo): same workers and capture code, but Gym's + # dynamo_worker handler sends capture fields in nvext, because the frontend + # carries only OpenAI fields plus nvext. generation_backend: Optional[Literal["vllm", "megatron", "dynamo"]] = None # Keep routed_experts out of canonical rows and assemble them on policy # workers from strict staged-fragment plans. From 62b375d559825129cf1c99bdf8af971d203335ce Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 18:57:39 -0800 Subject: [PATCH 44/82] test(dynamo): fix the lint and Gym-dialect failures CI hit Lint failed on import order in test_gym_chat_path.py and on a long line in test_vllm_chat_template_wiring.py, so no L0/L1 job ran (C47). The Dynamo dialect test still expected the native default template kwargs merged into Gym's config, which 43efaba3bd rightly stopped: the frontend renders with them itself. It now checks that only a shard's own kwargs reach Gym (C48). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/environments/test_nemo_gym_utils.py | 13 +++++++++---- tests/unit/models/generation/test_gym_chat_path.py | 11 ++++++++--- .../generation/test_vllm_chat_template_wiring.py | 4 +++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index 4f0a315ffc8..d60ceeeddb5 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -1214,7 +1214,8 @@ def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( """With Dynamo, each actor's composed config speaks Gym's Dynamo dialect. The dialect is not a policy_model overlay, which a sharded plan would reject - as unowned. A shard's own template kwargs still win over the native defaults. + as unowned. The frontend applies the native default template kwargs, so only + a shard's own kwargs reach Gym. """ from nemo_rl.algorithms.grpo import MasterConfig @@ -1255,14 +1256,18 @@ def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( turn_checkpointing_enabled=False, ) - thinking = [ + template_kwargs = [ cfg["initial_global_config_dict"]["policy_model"]["responses_api_models"][ "vllm_model" - ]["chat_template_kwargs"]["enable_thinking"] + ].get("chat_template_kwargs") for cfg in cluster.actor_configs ] # judged, then the tools shard's two replicas with their own override. - assert thinking == [True, False, False] + assert template_kwargs == [ + None, + {"enable_thinking": False}, + {"enable_thinking": False}, + ] for cfg in cluster.actor_configs: gym_config = cfg["initial_global_config_dict"] vllm_model = gym_config["policy_model"]["responses_api_models"]["vllm_model"] diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 6bfa54b2719..411bae5d5a5 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -45,7 +45,6 @@ import httpx # noqa: E402 from dynamo.llm.exceptions import HttpError # noqa: E402 from fastapi import FastAPI # noqa: E402 -from PIL import Image # noqa: E402 from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 from nemo_gym.token_id_capture.replay import ( # noqa: E402 replay_context, @@ -58,6 +57,7 @@ CaptureAdmission, StageResult, ) +from PIL import Image # noqa: E402 from transformers import AutoConfig, AutoProcessor, AutoTokenizer # noqa: E402 from vllm.config import CacheConfig, LoadConfig, VllmConfig # noqa: E402 from vllm.engine.arg_utils import AsyncEngineArgs # noqa: E402 @@ -66,14 +66,19 @@ from vllm.renderers.registry import renderer_from_config # noqa: E402 from vllm.v1.engine.input_processor import InputProcessor # noqa: E402 -from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache, StagedPrefix # noqa: E402 +from nemo_rl.data_plane.tq_token_sink import ( # noqa: E402 + ChainPrefixCache, + StagedPrefix, +) from nemo_rl.models.generation.dynamo.vllm import chat_engine # noqa: E402 from nemo_rl.models.generation.dynamo.vllm.adapter import ( # noqa: E402 DynamoVllmAdapter, chat_runtime_data, ) from nemo_rl.models.generation.dynamo.vllm.render_pool import RenderPool # noqa: E402 -from nemo_rl.models.generation.gym_call import GYM_EXTRAS_ANNOTATION_PREFIX # noqa: E402 +from nemo_rl.models.generation.gym_call import ( # noqa: E402 + GYM_EXTRAS_ANNOTATION_PREFIX, +) from nemo_rl.models.generation.vllm import chat_server # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, diff --git a/tests/unit/models/generation/test_vllm_chat_template_wiring.py b/tests/unit/models/generation/test_vllm_chat_template_wiring.py index 4f5298524a3..31b738877d3 100644 --- a/tests/unit/models/generation/test_vllm_chat_template_wiring.py +++ b/tests/unit/models/generation/test_vllm_chat_template_wiring.py @@ -687,7 +687,9 @@ async def preprocess(self, *, messages, **kwargs): ) renderers, _, _ = _build_server(monkeypatch, {}, app=app) # The server module was just imported against the fakes; patch that copy. - monkeypatch.setattr(sys.modules[_CHAT_SERVER], "remap_multimodal_placeholders", remap) + monkeypatch.setattr( + sys.modules[_CHAT_SERVER], "remap_multimodal_placeholders", remap + ) renderer = renderers[0] renderer.renderer = SimpleNamespace(tokenizer=SimpleNamespace(eos_token_id=2)) renderer.model_config = SimpleNamespace(max_model_len=8 if overflows else 16) From fd91d7641dc5898ed9ecf952bc3825bb20c802d0 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 19:00:24 -0800 Subject: [PATCH 45/82] fix(dynamo): fail engine errors and dead render workers as native does - Dynamo's binding raises an engine's error frame as a ValueError, which the frontend answers with 400, so Gym never retried a failed generation. The chat engine now answers it with 500, as natively; every status the engine means to send already comes back as a refusal (C49). - A dead render-pool worker broke the executor for the rest of the run. The pool now restarts once it has served a call, and names the cause when its workers die before serving any (C50). - The first concurrent token-in calls each ran ray.init; the TQ client is now connected once, under a lock (C57). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/dynamo/vllm/chat_engine.py | 24 ++++++--- .../generation/dynamo/vllm/render_pool.py | 49 +++++++++++++++---- .../generation/test_dynamo_chat_engine.py | 20 ++++---- .../generation/test_dynamo_render_pool.py | 46 +++++++++++++++++ .../models/generation/test_gym_chat_path.py | 3 -- 5 files changed, 110 insertions(+), 32 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 25aa8197cbb..46c8a246d29 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -37,6 +37,7 @@ import base64 import os import pickle +import threading import time import uuid from typing import Any, AsyncGenerator, Optional @@ -127,6 +128,8 @@ class _CaptureSource: def __init__(self) -> None: self._prefix_cache: Any = None + # Calls run resolve_prefix on several threads at once; connect once. + self._connect_lock = threading.Lock() def resolve_prefix(self, serving_admission: dict[str, Any]) -> list[int]: """The admission's prefix, read from TQ (blocking; run off the loop).""" @@ -135,7 +138,9 @@ def resolve_prefix(self, serving_admission: dict[str, Any]) -> list[int]: from nemo_rl.data_plane.tq_token_sink import resolve_admission_prefix if self._prefix_cache is None: - self._prefix_cache = self._connect() + with self._connect_lock: + if self._prefix_cache is None: + self._prefix_cache = self._connect() return resolve_admission_prefix( CaptureAdmission.model_validate(serving_admission), self._prefix_cache ) @@ -342,13 +347,16 @@ async def _call_engine( ) -> dict[str, Any]: engine_data = None stream = await self._routed_engine.generate(preprocessed, context=context) - async for item in stream: - if item.is_error(): - comments = item.comments() or [] - raise HttpError(500, "; ".join(comments) or "engine error") - data = item.data() - if data and data.get("engine_data"): - engine_data = data["engine_data"] + try: + async for item in stream: + data = item.data() + if data and data.get("engine_data"): + engine_data = data["engine_data"] + except ValueError as error: + # Dynamo raises an engine's error as a ValueError, which it would + # answer with 400. The engine sends every status it means as a + # refusal, so anything raised is an engine failure: 500, as natively. + raise HttpError(500, str(error)) from error if engine_data is None: raise HttpError(500, "the engine returned no result for this call") return engine_data diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index ab08bde6808..73102cf6bd1 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -31,12 +31,16 @@ from __future__ import annotations import asyncio +import logging import multiprocessing import pickle import uuid from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool from dataclasses import dataclass -from typing import Any, Optional +from typing import Any, Callable, Optional + +LOGGER = logging.getLogger(__name__) # Set in each worker process by _init_worker. _WORKER: Optional["_Worker"] = None @@ -345,14 +349,41 @@ class RenderPool: """``workers`` processes that render and answer chat calls, FIFO.""" def __init__(self, spec: dict[str, Any], model_dir: str, *, workers: int): - self._executor = ProcessPoolExecutor( - max_workers=workers, + self._spec = spec + self._model_dir = model_dir + self._workers = workers + self._executor = self._start() + self._info: Optional[dict[str, Any]] = None + + def _start(self) -> ProcessPoolExecutor: + return ProcessPoolExecutor( + max_workers=self._workers, # Fork would copy the frontend's Rust runtime and event loop. mp_context=multiprocessing.get_context("spawn"), initializer=_init_worker, - initargs=(spec, model_dir), + initargs=(self._spec, self._model_dir), ) - self._info: Optional[dict[str, Any]] = None + + async def _submit(self, fn: Callable[..., Any], *args: Any) -> Any: + executor = self._executor + try: + return await asyncio.wrap_future(executor.submit(fn, *args)) + except BrokenProcessPool as error: + if self._info is None: + # No worker ever served a job: most often _init_worker failed (a + # config error, which a restart cannot fix), so do not restart. + raise RuntimeError( + "the render pool's workers died before serving a call; see " + "this log for an 'Exception in initializer' traceback or a " + "kill (e.g. OOM)" + ) from error + if self._executor is executor: + # A worker died (e.g. OOM-killed). The calls in flight fail; + # later calls get a fresh pool instead of this broken one. + LOGGER.error("A render-pool worker died; restarting the pool") + executor.shutdown(wait=False, cancel_futures=True) + self._executor = self._start() + raise async def info(self) -> dict[str, Any]: """The model's turn-end ids and context length, from a started worker. @@ -361,11 +392,11 @@ async def info(self) -> dict[str, Any]: builds them. """ if self._info is None: - self._info = await asyncio.wrap_future(self._executor.submit(_info)) + self._info = await self._submit(_info) return self._info async def render(self, body: dict[str, Any]) -> RenderedCall: - return await asyncio.wrap_future(self._executor.submit(_render, body)) + return await self._submit(_render, body) async def respond( self, @@ -373,9 +404,7 @@ async def respond( prompt_token_ids: list[int], output: dict[str, Any], ) -> dict[str, Any]: - return await asyncio.wrap_future( - self._executor.submit(_respond, request, prompt_token_ids, output) - ) + return await self._submit(_respond, request, prompt_token_ids, output) def shutdown(self) -> None: self._executor.shutdown(wait=False, cancel_futures=True) diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index fa2b950a39a..470448e0017 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -111,18 +111,12 @@ async def respond(self, request, prompt_token_ids, output) -> dict: class _Item: """One annotated response from Dynamo's router.""" - def __init__(self, data=None, comments=None) -> None: - self._data, self._comments = data, comments - - def is_error(self) -> bool: - return self._comments is not None + def __init__(self, data) -> None: + self._data = data def data(self): return self._data - def comments(self): - return self._comments - class _Router: def __init__(self, engine_data=None, error=None) -> None: @@ -138,8 +132,8 @@ async def generate(self, request, context=None): async def stream(): if self.error is not None: - yield _Item(comments=[self.error]) - return + # Dynamo's binding raises an engine's error frame as a ValueError. + raise ValueError(self.error) yield _Item({"token_ids": [10], "engine_data": self.engine_data}) return stream() @@ -413,7 +407,11 @@ def test_a_call_refused_before_it_runs_acknowledges_its_capture_to_gym( @pytest.mark.parametrize( ("router", "status", "message"), [ - (_Router(error="engine died"), 500, "engine died"), + ( + _Router(error="BackendInvalidArgument: RuntimeError: engine died"), + 500, + "engine died", + ), # Refused before it ran, with the status the native server answers. ( _Router( diff --git a/tests/unit/models/generation/test_dynamo_render_pool.py b/tests/unit/models/generation/test_dynamo_render_pool.py index e4ecd4ccb82..4ccff42aec4 100644 --- a/tests/unit/models/generation/test_dynamo_render_pool.py +++ b/tests/unit/models/generation/test_dynamo_render_pool.py @@ -20,6 +20,9 @@ import asyncio import json +import os +import signal +from concurrent.futures.process import BrokenProcessPool import pytest @@ -150,3 +153,46 @@ async def serve(): (tool_call,) = choice["message"]["tool_calls"] assert tool_call["function"]["name"] == "get_weather" assert json.loads(tool_call["function"]["arguments"]) == {"city": "Lyon"} + + +def test_a_dead_worker_fails_only_its_calls(tiny_qwen3_model_path) -> None: + body = { + "model": "model", + "messages": MESSAGES[:1], + "max_tokens": 8, + "temperature": 0.7, + "top_p": 0.9, + } + + async def serve(): + pool = RenderPool(SPEC, tiny_qwen3_model_path, workers=1) + try: + await pool.info() + (pid,) = pool._executor._processes + # As the OOM killer would. + os.kill(pid, signal.SIGKILL) + with pytest.raises(BrokenProcessPool): + await pool.render(body) + # The next call gets a fresh pool, not the broken one. + return await pool.render(body) + finally: + pool.shutdown() + + rendered = asyncio.run(serve()) + + assert rendered.template_token_ids + + +def test_workers_that_cannot_start_fail_the_call_by_name(tiny_qwen3_model_path): + vllm_cfg = {"http_server_serving_chat_kwargs": {"reasoning_parser": "no_such"}} + spec = {**SPEC, "generation": {**SPEC["generation"], "vllm_cfg": vllm_cfg}} + + async def serve(): + pool = RenderPool(spec, tiny_qwen3_model_path, workers=1) + try: + await pool.info() + finally: + pool.shutdown() + + with pytest.raises(RuntimeError, match="before serving a call"): + asyncio.run(serve()) diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 411bae5d5a5..50ae46ad770 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -394,9 +394,6 @@ class SimpleItem: def __init__(self, data) -> None: self._data = data - def is_error(self) -> bool: - return False - def data(self): return self._data From 3e81bcadfa84502c2c0c5435089561806b696397 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 19:18:37 -0800 Subject: [PATCH 46/82] fix(dynamo): render Gym calls as native does - Publish all of policy.generation to the frontend instead of a hand-kept key list, whose misses made the frontend fall back silently (C59). - Register NeMo RL's TorchCodec video loader in the render workers, which decode Gym's videos through Dynamo; one helper serves both hosts (C51). - Count a measure call as /tokenize does, without the context cap, and never acknowledge a capture refusal on it (C52). - Keep the client's nulls inside tool schemas and other free-form JSON, which the template renders into the prompt (C56). - Rebuild the rendered request without the structured outputs a tool parser set for a forced tool, which the request's validators reject beside a named tool_choice (C63). - Refuse gpt-oss (Harmony) calls by name: vLLM renders them without the step NeMo RL's capture and splice run in (C64). - One output-budget clamp for both paths, with the frontend's rule: a prompt that leaves no room for output is refused on every chat call. Natively this changes only the message for a prompt of exactly max_model_len without max_tokens, to the "context length" one Gym marks context_length_exceeded (C66). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../models/generation/dynamo/vllm/adapter.py | 37 +------ .../generation/dynamo/vllm/chat_engine.py | 53 +++++----- .../generation/dynamo/vllm/render_pool.py | 31 +++--- nemo_rl/models/generation/vllm/chat_server.py | 97 +++++++++++++++---- nemo_rl/models/generation/vllm/video_utils.py | 16 +++ nemo_rl/models/generation/vllm/vllm_worker.py | 12 +-- .../generation/test_dynamo_chat_engine.py | 2 +- .../generation/test_dynamo_render_pool.py | 29 ++++++ .../generation/test_dynamo_vllm_adapter.py | 16 +-- .../models/generation/test_gym_chat_path.py | 48 ++++++++- .../test_vllm_chat_template_wiring.py | 4 + .../models/generation/test_vllm_generation.py | 1 + 12 files changed, 226 insertions(+), 120 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index 044f7e22b12..887838ce752 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -97,22 +97,6 @@ KVCacheSpecKind.MLA_ATTENTION, KVCacheSpecKind.SINK_FULL_ATTENTION, ) -# The policy.generation keys the frontend's chat engine renders and checks -# sampling with. -_CHAT_GENERATION_KEYS = ( - "temperature", - "top_p", - "top_k", - "val_temperature", - "val_top_p", - "val_top_k", - "stop_token_ids", -) -_CHAT_VLLM_CFG_KEYS = ( - "http_server_serving_chat_kwargs", - "tool_parser_plugin", - "reasoning_parser_plugin", -) def dynamo_kv_events_config( @@ -138,7 +122,8 @@ def chat_runtime_data(host: "VllmAsyncGenerationWorkerImpl") -> dict[str, Any]: The engine's own args (those a JSON payload can carry, which include every one ``ModelConfig`` reads), so the frontend builds the same tokenizer, - template and processor; and the chat settings and sampling profiles. + template and processor; and all of ``policy.generation``, which the native + server's chat helpers read. """ engine_args: dict[str, Any] = {} for field in dataclasses.fields(host.llm_async_engine_args): @@ -158,18 +143,10 @@ def chat_runtime_data(host: "VllmAsyncGenerationWorkerImpl") -> dict[str, Any]: engine_args[field.name] = value # The frontend renders chat calls, so it always loads the tokenizer. engine_args["skip_tokenizer_init"] = False - cfg = host.cfg return { RUNTIME_DATA_KEY: { "engine_args": engine_args, - "generation": { - **{key: cfg[key] for key in _CHAT_GENERATION_KEYS}, - "vllm_cfg": { - key: cfg["vllm_cfg"][key] - for key in _CHAT_VLLM_CFG_KEYS - if key in cfg["vllm_cfg"] - }, - }, + "generation": json.loads(json.dumps(host.cfg)), } } @@ -319,15 +296,11 @@ async def _serve_gym_call( """ # Deferred: chat_server imports vLLM's OpenAI server stack. from nemo_rl.models.generation.vllm.chat_server import ( - NeMoRLChatCompletionRequest, + chat_request_from_render, ) host = self._host - # The reply never echoes messages; the frontend sends the request - # without them. - chat = NeMoRLChatCompletionRequest.model_validate( - {**plan["request"], "messages": [{"role": "user", "content": ""}]} - ) + chat = chat_request_from_render(plan["request"]) admission = host._parse_capture_admission(plan["admission"]) serving_admission = host._parse_capture_admission(plan["serving_admission"]) if admission is not None and admission.mode == "candidate": diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 46c8a246d29..333dec8e8aa 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -73,17 +73,14 @@ _DYNAMO_ONLY_FIELDS = {"nvext", "chat_template_args", "unsupported_fields"} -class _ContextLengthError(ValueError): - """The prompt leaves no room for output; Gym reads "context length" in a 400.""" - - def _capture_refusal_ack(extras: GymCallExtras) -> str: """Gym's acknowledgement that a capture call was refused before it ran. The native route sends it as a header; Dynamo's frontend forwards only an error's status and message, so it ends the message instead. """ - if extras.ng_capture is None: + if extras.ng_capture is None or extras.ng_measure: + # Native's /tokenize never acknowledges a measure call. return "" # Deferred: Gym is installed wherever a call carries ng_capture. from nemo_gym.token_id_capture.external_capture import ( @@ -93,32 +90,30 @@ def _capture_refusal_ack(extras: GymCallExtras) -> str: return dynamo_capture_refusal_suffix(extras.ng_capture["model_call_id"]) +# Free-form JSON the client wrote (a tool's parameters, a response_format +# schema, processor and media kwargs) keeps its nulls, as natively: a schema's +# `"default": null` is rendered into the prompt. +_CLIENT_JSON_KEYS = frozenset( + {"parameters", "schema", "mm_processor_kwargs", "media_io_kwargs"} +) + + def _drop_none(value: Any) -> Any: - """Dynamo hands Python unset fields as None, which vLLM's models reject.""" + """Drop the Nones Dynamo adds for unset struct fields, which vLLM rejects. + + An image part's unset ``detail`` arrives as None, for example. + """ if isinstance(value, dict): - return {k: _drop_none(v) for k, v in value.items() if v is not None} + return { + k: v if k in _CLIENT_JSON_KEYS else _drop_none(v) + for k, v in value.items() + if v is not None + } if isinstance(value, list): return [_drop_none(v) for v in value] return value -def _clamp( - request_max_tokens: Optional[int], prompt_token_ids: list[int], max_model_len: int -) -> Optional[int]: - """The output budget, as the native server's _clamp_max_tokens sets it.""" - remaining = max_model_len - len(prompt_token_ids) - if remaining <= 0: - # Gym reads "context length" in the 400 as an overflow. - raise _ContextLengthError( - f"Prompt length ({len(prompt_token_ids)}) fills or exceeds this " - f"model's maximum context length ({max_model_len}). No room " - "for output tokens." - ) - if request_max_tokens is None: - return None - return min(request_max_tokens, remaining) - - class _CaptureSource: """The read-only TQ client the token-capture prefix is read with. @@ -214,13 +209,15 @@ async def generate( yield await self._serve(request, extras, context) except (OffPolicySamplingError, MediaCaptureRejected) as error: raise HttpError(400, str(error)) from error - except (_ContextLengthError, VLLMValidationError) as error: + except VLLMValidationError as error: # Refused before the engine ran, so nothing was staged: as natively. raise HttpError(400, str(error) + _capture_refusal_ack(extras)) from error async def _serve( self, request: dict[str, Any], extras: GymCallExtras, context: Any ) -> dict[str, Any]: + # Deferred: chat_server imports vLLM's OpenAI server stack. + from nemo_rl.models.generation.vllm.chat_server import clamp_max_tokens from nemo_rl.models.generation.vllm.utils import on_policy_sampling body = _drop_none( @@ -246,7 +243,7 @@ async def _serve( body["ng_capture"] = extras.ng_capture body["required_prefix_token_ids"] = extras.required_prefix_token_ids info = await self._pool.info() - rendered = await self._pool.render(body) + rendered = await self._pool.render(body, measure=extras.ng_measure) prompt_token_ids, splice, prefix = await self._splice( rendered, info["turn_end_token_ids"] ) @@ -259,8 +256,8 @@ async def _serve( message={"role": "assistant", "content": ""}, finish_reason="stop", ) - max_tokens = _clamp( - rendered.request_max_tokens, prompt_token_ids, info["max_model_len"] + max_tokens = clamp_max_tokens( + rendered.request_max_tokens, len(prompt_token_ids), info["max_model_len"] ) engine_request = dict(rendered.request) if max_tokens is not None: diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index 73102cf6bd1..4bcd4a1a632 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -101,9 +101,14 @@ def __init__(self, spec: dict[str, Any], model_dir: str): turn_end_token_ids, ) from nemo_rl.models.generation.vllm import chat_server + from nemo_rl.models.generation.vllm.video_utils import ( + register_vllm_video_loader, + ) cfg = spec["generation"] chat_server.register_parser_plugins(cfg) + # This worker decodes the call's videos; the engine runs its pixels. + register_vllm_video_loader(cfg) # The engine's own args, with the weight-free snapshot Dynamo fetched. args = AsyncEngineArgs( **{**spec["engine_args"], "model": model_dir, "tokenizer": model_dir} @@ -180,15 +185,19 @@ def _parser(self, chat: Any) -> Any: model_config=self.model_config, ) - def render(self, body: dict[str, Any]) -> RenderedCall: + def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall: from nemo_gym.token_id_capture.staging.records import CaptureAdmission from nemo_rl.models.generation.vllm.chat_server import ( NeMoRLChatCompletionRequest, + NeMoRLMeasureChatRequest, set_max_tokens, ) - chat = NeMoRLChatCompletionRequest.model_validate(body) + request_cls = ( + NeMoRLMeasureChatRequest if measure else NeMoRLChatCompletionRequest + ) + chat = request_cls.model_validate(body) request_max_tokens = ( chat.max_completion_tokens if chat.max_completion_tokens is not None @@ -259,13 +268,10 @@ def respond( from vllm.outputs import CompletionOutput, RequestOutput from nemo_rl.models.generation.vllm.chat_server import ( - NeMoRLChatCompletionRequest, + chat_request_from_render, ) - # The reply never echoes messages, so the request's are not sent back. - chat = NeMoRLChatCompletionRequest.model_validate( - {**request, "messages": [{"role": "user", "content": ""}]} - ) + chat = chat_request_from_render(request) logprobs = None if output.get("logprobs") is not None: logprobs = [ @@ -325,9 +331,9 @@ def _init_worker(spec: dict[str, Any], model_dir: str) -> None: _WORKER = _Worker(spec, model_dir) -def _render(body: dict[str, Any]) -> RenderedCall: +def _render(body: dict[str, Any], measure: bool) -> RenderedCall: assert _WORKER is not None - return _WORKER.render(body) + return _WORKER.render(body, measure=measure) def _respond( @@ -395,8 +401,11 @@ async def info(self) -> dict[str, Any]: self._info = await self._submit(_info) return self._info - async def render(self, body: dict[str, Any]) -> RenderedCall: - return await self._submit(_render, body) + async def render( + self, body: dict[str, Any], *, measure: bool = False + ) -> RenderedCall: + """Render a call; a measure call is counted past the context, as natively.""" + return await self._submit(_render, body, measure) async def respond( self, diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index d35a2b995a0..184f5a6f3eb 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -25,7 +25,7 @@ import copy import logging from copy import deepcopy -from dataclasses import dataclass +from dataclasses import dataclass, replace from logging import Filter as LoggingFilter from logging import LogRecord from typing import Any, List, Literal, Optional, Union @@ -55,6 +55,7 @@ from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager from vllm.renderers.hf import HfRenderer from vllm.renderers.online_renderer import OnlineRenderer +from vllm.sampling_params import StructuredOutputsParams from vllm.tool_parsers.abstract_tool_parser import ToolParserManager from vllm.v1.engine.async_llm import logger as vllm_async_llm_logger @@ -107,6 +108,39 @@ class NeMoRLChatCompletionRequest(NeMoRLOpenAIChatRequestMixin, ChatCompletionRe ng_capture: Optional[dict[str, Any]] = None +class NeMoRLMeasureChatRequest(NeMoRLChatCompletionRequest): + """A chat call Gym measures, counted as ``/tokenize`` counts it. + + vLLM's tokenize request sets no context limit, so a prompt longer than the + context is counted instead of refused. + """ + + def build_tok_params(self, model_config: Any) -> Any: + return replace(super().build_tok_params(model_config), max_total_tokens=None) + + +def chat_request_from_render(request: dict[str, Any]) -> NeMoRLChatCompletionRequest: + """The chat request a render produced, rebuilt from its JSON without messages. + + vLLM's tool parser sets ``structured_outputs`` during the render (a forced + tool's schema), which the request's own validators reject beside a named + ``tool_choice``; it is set back after validation, as the render left it. + """ + structured_outputs = request.get("structured_outputs") + chat = NeMoRLChatCompletionRequest.model_validate( + { + **request, + "structured_outputs": None, + "messages": [{"role": "user", "content": ""}], + } + ) + if structured_outputs is not None: + chat.structured_outputs = StructuredOutputsParams( + **{k: v for k, v in structured_outputs.items() if not k.startswith("_")} + ) + return chat + + # This MRO is necessary i.e. NeMoRLOpenAIChatRequestMixin > TokenizeRequest class NeMoRLTokenizeChatRequest(NeMoRLOpenAIChatRequestMixin, TokenizeChatRequest): required_prefix_token_ids: Optional[List[int]] = None @@ -129,29 +163,41 @@ def set_max_tokens(request: Any, max_tokens: int) -> None: request.max_tokens = max_tokens +def clamp_max_tokens( + request_max_tokens: Optional[int], prompt_len: int, max_model_len: int +) -> Optional[int]: + """The output budget that keeps input + output within ``max_model_len``. + + Refuses a prompt that leaves no room for output, whether or not the call set + a budget, and lowers the budget only when it did. Both chat paths call it. + """ + remaining = max_model_len - prompt_len + if remaining <= 0: + # preserve the literal "context length" in this message to match Gym's overflow handling + message = ( + f"Prompt length ({prompt_len}) fills or exceeds " + f"this model's maximum context length ({max_model_len}). " + f"No room for output tokens." + ) + LOGGER.warning("Prompt exceeds max_model_len: %s", message) + raise VLLMValidationError(message, parameter="input_tokens", value=prompt_len) + if request_max_tokens is None: + return None + return min(request_max_tokens, remaining) + + class NeMoRLOpenAIServingMixin: """The chat preprocessing NeMo RL adds: prefix splice and token capture.""" def _clamp_max_tokens( - self, request, request_max_tokens: int, prompt_token_ids: list[int] + self, request, request_max_tokens: Optional[int], prompt_token_ids: list[int] ) -> None: """Clamp the request's max output tokens so that input + output <= max_model_len.""" - remaining = self.model_config.max_model_len - len(prompt_token_ids) - if remaining <= 0: - # preserve the literal "context length" in this message to match Gym's overflow handling - message = ( - f"Prompt length ({len(prompt_token_ids)}) fills or exceeds " - f"this model's maximum context length ({self.model_config.max_model_len}). " - f"No room for output tokens." - ) - LOGGER.warning("Prompt exceeds max_model_len: %s", message) - raise VLLMValidationError( - message, - parameter="input_tokens", - value=len(prompt_token_ids), - ) - max_tokens = min(request_max_tokens, remaining) - set_max_tokens(request, max_tokens) + max_tokens = clamp_max_tokens( + request_max_tokens, len(prompt_token_ids), self.model_config.max_model_len + ) + if max_tokens is not None: + set_max_tokens(request, max_tokens) async def _render_chat_call( self, request, messages, render_kwargs: dict[str, Any], *, admission @@ -257,8 +303,10 @@ async def preprocess_chat( # Temporarily set to 1 so vLLM's pre-tokenization length check passes; # the actual value will be set through _clamp_max_tokens later. + # /tokenize counts a prompt of any length, so only chat calls clamp. + clamps_output = isinstance(request, NeMoRLChatCompletionRequest) actual_request_max_tokens = None - if isinstance(request, NeMoRLChatCompletionRequest): + if clamps_output: actual_request_max_tokens = ( request.max_completion_tokens if request.max_completion_tokens is not None @@ -306,7 +354,7 @@ async def preprocess_chat( or request.required_prefix_token_ids is None ): # Clamp the request's max output tokens so that input + output <= max_model_len. - if actual_request_max_tokens is not None: + if clamps_output: self._clamp_max_tokens( request, actual_request_max_tokens, @@ -339,7 +387,7 @@ async def preprocess_chat( ) final_prompt_token_ids = splice.token_ids # Reject overflows before media validation can mask the length error. - if actual_request_max_tokens is not None: + if clamps_output: self._clamp_max_tokens( request, actual_request_max_tokens, @@ -428,6 +476,13 @@ class FrontendRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): async def render_for_frontend(self, request, *, admission) -> "RenderedChat": """Render ``request``; the caller restores its lowered output budget.""" + if self.use_harmony: + # vLLM renders gpt-oss's Harmony format without preprocess_chat, so + # NeMo RL's render, capture decision and splice would never run. + raise ValueError( + "gpt-oss (Harmony) chat calls are not served through NeMo RL's " + "Dynamo chat engine; use policy.generation.dynamo=false" + ) self._admission = admission # As preprocess_chat does: vLLM's pre-tokenization length check must # pass; the frontend clamps the budget after the splice. diff --git a/nemo_rl/models/generation/vllm/video_utils.py b/nemo_rl/models/generation/vllm/video_utils.py index 4068042d29e..0380912fab3 100644 --- a/nemo_rl/models/generation/vllm/video_utils.py +++ b/nemo_rl/models/generation/vllm/video_utils.py @@ -29,6 +29,7 @@ CACHED_VIDEO_FRAME_MANIFEST_MAGIC, CACHED_VIDEO_FRAME_MANIFEST_MIME, ) +from nemo_rl.models.generation.vllm.config import VllmConfig, resolve_vllm_video_config VideoSamplingStyle = Literal["nemotron_vl"] @@ -518,6 +519,21 @@ def _load_video_frames_torchcodec_with_metadata( return frames, metadata +def register_vllm_video_loader(generation_config: VllmConfig) -> None: + """Register NeMo RL's video loader when ``vllm_cfg.video`` is set. + + Called by every process that decodes the videos of chat calls: the vLLM + engine, and with Dynamo the frontend's render workers, whose pixels the + engine then runs. + """ + video_config = resolve_vllm_video_config(generation_config) + if video_config is not None: + register_torchcodec_vllm_video_loader( + sampling_style=video_config.sampling_style, + temporal_patch_size=video_config.temporal_patch_size, + ) + + def register_torchcodec_vllm_video_loader( *, sampling_style: VideoSamplingStyle, diff --git a/nemo_rl/models/generation/vllm/vllm_worker.py b/nemo_rl/models/generation/vllm/vllm_worker.py index 08dda3dd79c..15340ff911c 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker.py +++ b/nemo_rl/models/generation/vllm/vllm_worker.py @@ -46,7 +46,6 @@ VLLM_SPARSE_REFIT_TRANSPORTS, VllmConfig, parse_nvfp4_pertoken_rollout, - resolve_vllm_video_config, validate_nvfp4_pertoken_model, vllm_fp32_lm_head_enabled, ) @@ -61,9 +60,7 @@ pad_and_align_routed_expert_indices, validate_rollout_prompt, ) -from nemo_rl.models.generation.vllm.video_utils import ( - register_torchcodec_vllm_video_loader, -) +from nemo_rl.models.generation.vllm.video_utils import register_vllm_video_loader from nemo_rl.models.generation.vllm.worker_utils import ( find_tokenizer_required_architectures, resolve_distributed_executor_backend, @@ -810,12 +807,7 @@ def _load_model(self, bundle_indices, seed): if logprobs_mode is not None: llm_kwargs["logprobs_mode"] = logprobs_mode - video_config = resolve_vllm_video_config(self.cfg) - if video_config is not None: - register_torchcodec_vllm_video_loader( - sampling_style=video_config.sampling_style, - temporal_patch_size=video_config.temporal_patch_size, - ) + register_vllm_video_loader(self.cfg) _configure_nvfp4_pertoken_engine_kwargs(self.cfg, llm_kwargs) _use_vllm_sampling_defaults(llm_kwargs) llm_kwargs.setdefault("worker_extension_cls", default_worker_extension_cls) diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index 470448e0017..49ee7af2d1c 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -97,7 +97,7 @@ def __init__(self, rendered: RenderedCall, reply: dict | None = None) -> None: async def info(self) -> dict: return {"turn_end_token_ids": [EOS], "max_model_len": MAX_MODEL_LEN} - async def render(self, body: dict) -> RenderedCall: + async def render(self, body: dict, *, measure: bool = False) -> RenderedCall: self.bodies.append(body) if self.error is not None: raise self.error diff --git a/tests/unit/models/generation/test_dynamo_render_pool.py b/tests/unit/models/generation/test_dynamo_render_pool.py index 4ccff42aec4..36e4c4087b1 100644 --- a/tests/unit/models/generation/test_dynamo_render_pool.py +++ b/tests/unit/models/generation/test_dynamo_render_pool.py @@ -196,3 +196,32 @@ async def serve(): with pytest.raises(RuntimeError, match="before serving a call"): asyncio.run(serve()) + + +def _nemotron_vl_loader() -> str: + """Run in a render worker: the class vLLM decodes nemotron_vl videos with.""" + from vllm.multimodal.video import VIDEO_LOADER_REGISTRY + + return VIDEO_LOADER_REGISTRY.name2class["nemotron_vl"].__name__ + + +def test_workers_decode_videos_with_nemo_rls_loader(tiny_qwen3_model_path) -> None: + # The worker decodes a Gym call's videos, so it needs the engine's loader. + vllm_cfg = { + **SPEC["generation"]["vllm_cfg"], + "video": { + "sampling_style": "nemotron_vl", + "num_frames": 8, + "temporal_patch_size": 2, + }, + } + spec = {**SPEC, "generation": {**SPEC["generation"], "vllm_cfg": vllm_cfg}} + + async def serve(): + pool = RenderPool(spec, tiny_qwen3_model_path, workers=1) + try: + return await pool._submit(_nemotron_vl_loader) + finally: + pool.shutdown() + + assert asyncio.run(serve()) == "TorchCodecNemotronVLVideoBackend" diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index e7a95926cdc..a9a37d83a24 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -425,20 +425,8 @@ def test_engines_publish_what_the_frontends_chat_engine_renders_with() -> None: host.llm_async_engine_args, field ), field assert published.skip_tokenizer_init is False - # The native server's chat settings and the sampling profiles it checks. - assert spec["generation"] == { - "temperature": 0.7, - "top_p": 0.9, - "top_k": None, - "val_temperature": 0.0, - "val_top_p": 1.0, - "val_top_k": None, - "stop_token_ids": [EOS], - "vllm_cfg": { - "http_server_serving_chat_kwargs": serving_chat_kwargs, - "reasoning_parser_plugin": "/plugins/reasoning.py", - }, - } + # All of policy.generation, which the native server's chat helpers read. + assert spec["generation"] == host.cfg # --- NeMo-Gym calls, after the frontend's chat engine rendered them ----------- diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 50ae46ad770..741ba0b2229 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -119,6 +119,26 @@ }, } ] +# A tool whose schema has a null default, as pydantic writes an Optional field. +NULL_DEFAULT_TOOLS = [ + { + "type": "function", + "function": { + **TOOLS[0]["function"], + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "unit": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, + }, + }, + "required": ["city"], + }, + }, + } +] # Turn 1 as Gym sends it back: the template drops its reasoning when it # re-renders the history, so the model's own tokens differ from the render. HISTORY = [ @@ -490,6 +510,8 @@ class _KitSpec: ), # Qwen2.5's tokenizer has no thinking tokens for Qwen3's reasoning parser. "vlm": _KitSpec(_generation(reasoning_parser=""), _vlm), + # A tool parser with no structural tag: a forced tool sets structured outputs. + "pythonic": _KitSpec(_generation(tool_parser="pythonic")), } @@ -692,6 +714,16 @@ def _off_policy(name: str, value: Any) -> _Case: extras=lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, kit="vlm", ), + # The client's nulls in a tool schema are rendered into the prompt. + _Case("tool_schema_null_default", fields={"tools": NULL_DEFAULT_TOOLS}), + # A named tool_choice, whose schema the tool parser sets as structured outputs. + _Case( + "named_tool_choice", + fields={ + "tool_choice": {"type": "function", "function": {"name": "get_weather"}} + }, + kit="pythonic", + ), # Nemotron-3-Nano's reasoning parser plugin, with thinking off (C39). _Case( "nano_v3_thinking_off", @@ -802,12 +834,21 @@ async def both(): } -def test_a_measure_call_counts_the_prompt_the_call_would_run(kits) -> None: +@pytest.mark.parametrize( + "observation", + [ + OBSERVATION, + # Past the context window: Gym measures to decide whether to compact. + {"role": "user", "content": "And in Lyon?" + " Lyon" * MAX_MODEL_LEN}, + ], + ids=["fits", "past_the_context"], +) +def test_a_measure_call_counts_the_prompt_the_call_would_run(kits, observation) -> None: kit = kits("qwen3") extras = {"required_prefix_token_ids": kit.turn_1_tokens()} body = { "model": kit.path, - "messages": [*HISTORY, OBSERVATION], + "messages": [*HISTORY, observation], "tools": TOOLS, **extras, } @@ -824,7 +865,8 @@ async def both(): native, dynamo = asyncio.run(both()) - # Native Gym measures with /tokenize; through Dynamo, the chat call (C36). + # Native Gym measures with /tokenize; through Dynamo, the chat call. + assert (native.status, dynamo.status) == (200, 200), dynamo.reply assert ( dynamo.reply["nvext"]["nemo_rl"]["prompt_token_count"] == native.reply["count"] ) diff --git a/tests/unit/models/generation/test_vllm_chat_template_wiring.py b/tests/unit/models/generation/test_vllm_chat_template_wiring.py index 31b738877d3..89f96793c60 100644 --- a/tests/unit/models/generation/test_vllm_chat_template_wiring.py +++ b/tests/unit/models/generation/test_vllm_chat_template_wiring.py @@ -176,6 +176,10 @@ def __init__(self, message, **kwargs): super().__init__(message) module("vllm.exceptions", VLLMValidationError=ValidationError) + module( + "vllm.sampling_params", + StructuredOutputsParams=placeholder("StructuredOutputsParams"), + ) module( "vllm.reasoning.abs_reasoning_parsers", ReasoningParserManager=type( diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 4edc0476861..357bb8b8b25 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1158,6 +1158,7 @@ def with_cancellation(handler): ServingTokenization=ServingTokenization, ) make_module("vllm.exceptions", VLLMValidationError=VLLMValidationError) + make_module("vllm.sampling_params", StructuredOutputsParams=MagicMock()) make_module( "vllm.reasoning.abs_reasoning_parsers", ReasoningParserManager=ReasoningParserManager, From f9172b443a6c21bfd0f5f8aa6c697b7be690e40b Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 19:20:30 -0800 Subject: [PATCH 47/82] fix(dynamo): reject at startup what the frontend would drop - Dynamo's frontend accepts six vLLM fields (its PASSTHROUGH_EXTRA_FIELDS) but never hands them to a Python chat engine, so a Gym extra_body stop_token_ids got a 200 and was ignored. The Dynamo dialect now refuses them at startup, next to bad_words (C54). - An explicit vllm_cfg.skip_tokenizer_init=true with NeMo-Gym gave every Gym call an empty reply through Dynamo; it is refused at startup (C65). - Restore #4905's tokenizer default: NeMo-Gym runs already load the engine's tokenizer through expose_http_server, which Gym requires, so non-Gym Dynamo runs skip detokenizing their rollouts again. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/environments/nemo_gym.py | 22 ++++++++++++++++++- nemo_rl/models/generation/__init__.py | 2 -- .../models/generation/vllm/vllm_generation.py | 8 +++++++ .../test_nemo_gym_token_capture.py | 10 +++++---- .../models/generation/test_dynamo_config.py | 7 +++--- .../models/generation/test_vllm_generation.py | 9 +++++++- 6 files changed, 47 insertions(+), 11 deletions(-) diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 177bc68042d..dfda6204776 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -3035,12 +3035,32 @@ def _apply_dynamo_dialect(nemo_gym_dict: dict[str, Any]) -> None: policy_model = dict(nemo_gym_dict.get("policy_model") or {}) models = dict(policy_model.get("responses_api_models") or {}) vllm_model = dict(models.get("vllm_model") or {}) - if "bad_words" in (vllm_model.get("extra_body") or {}): + extra_body = vllm_model.get("extra_body") or {} + if "bad_words" in extra_body: raise ValueError( "vllm_model.extra_body.bad_words is not supported with " "policy.generation.dynamo=true: the Dynamo frontend answers every " "request carrying it with HTTP 400." ) + # Dynamo accepts these vLLM fields (its PASSTHROUGH_EXTRA_FIELDS) but never + # serializes them to a Python chat engine, so ours would never see them. + dropped = sorted( + { + "allowed_token_ids", + "bad_words_token_ids", + "cache_salt", + "detokenize", + "logprob_token_ids", + "stop_token_ids", + } + & extra_body.keys() + ) + if dropped: + raise ValueError( + f"vllm_model.extra_body {dropped} is not supported with " + "policy.generation.dynamo=true: the Dynamo frontend accepts these " + "fields but drops them before NeMo RL's chat engine sees them." + ) vllm_model["dynamo_frontend"] = True # The frontend reads `reasoning` as another name for `reasoning_content` and # refuses an echoed assistant turn that carries both, Gym's default. diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index a3b80bb19b5..b5fb07139f2 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -321,8 +321,6 @@ def configure_generation_config( is_eval or config["stop_strings"] is not None or config["vllm_cfg"].get("expose_http_server", None) - # Dynamo serves OpenAI clients as the exposed server does. - or dynamo_enabled(config) ): config["vllm_cfg"]["skip_tokenizer_init"] = False else: diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 0e4f8547c1f..4d9cc99f617 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -234,6 +234,14 @@ def validate_settings(cls, master_config: "MasterConfig") -> None: "picks engines itself, and a restarted shard does not register " "with it. Disable fleet health." ) + if master_config.env.get("should_use_nemo_gym") and generation_config[ + "vllm_cfg" + ].get("skip_tokenizer_init"): + raise ValueError( + "vllm_cfg.skip_tokenizer_init=true does not work with NeMo-Gym " + "and policy.generation.dynamo=true: the engine detokenizes the " + "replies Gym's chat calls get through the frontend." + ) @staticmethod def init_cluster_placement_groups( diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index bd056990e7e..95aa4b33de4 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -929,11 +929,13 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> {"enable_thinking": False}, ), # The frontend answers every request carrying bad_words with a 400. - ({"extra_body": {"bad_words": ["x"]}}, None), + ({"extra_body": {"bad_words": ["x"]}}, "bad_words"), + # The frontend accepts stop_token_ids but drops it before our chat engine. + ({"extra_body": {"stop_token_ids": [7]}}, "stop_token_ids"), ], ) def test_gym_actor_config_dynamo_turns_on_gyms_dynamo_dialect( - vllm_model: dict, expected: dict | None + vllm_model: dict, expected: dict | str ) -> None: from responses_api_models.vllm_model.app import VLLMModelConfig @@ -984,8 +986,8 @@ def build() -> dict: turn_checkpointing_enabled=False, )["initial_global_config_dict"] - if expected is None: - with pytest.raises(ValueError, match="bad_words"): + if isinstance(expected, str): + with pytest.raises(ValueError, match=expected): build() return gym_config = build() diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index f04662d1dfc..7ab5fba52a3 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -50,10 +50,11 @@ def _vllm_config(**overrides) -> dict: def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: config = configure_generation_config(_vllm_config(dynamo=dynamo), _Tokenizer()) - # Dynamo serves the AsyncLLM, so the switch selects the async engine, and it - # serves OpenAI clients, which need the engine's tokenizer. + # Dynamo serves the AsyncLLM, so the switch selects the async engine. + # Rollouts read token IDs only, so the engine skips its tokenizer as natively; + # NeMo-Gym runs load it through expose_http_server, which Gym requires. assert config["vllm_cfg"]["async_engine"] is dynamo - assert config["vllm_cfg"]["skip_tokenizer_init"] is (not dynamo) + assert config["vllm_cfg"]["skip_tokenizer_init"] is True assert "dynamo_cfg" not in config assert dynamo_cfg(config) == DynamoCfg() diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 357bb8b8b25..1fdeff0c8fb 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2849,12 +2849,16 @@ def test_vllm_validate_settings_accepts_missing_vllm_cfg_as_refit_disabled(): VllmGeneration.validate_settings(master_config) -def _dynamo_master_config(*, router=False, fleet_health=False): +def _dynamo_master_config( + *, router=False, fleet_health=False, gym=False, skip_tokenizer_init=False +): vllm_config = deepcopy(basic_vllm_test_config) vllm_config["dynamo"] = True vllm_config["vllm_cfg"]["async_engine"] = True vllm_config["vllm_cfg"]["expose_http_server"] = True + vllm_config["vllm_cfg"]["skip_tokenizer_init"] = skip_tokenizer_init return types.SimpleNamespace( + env={"should_use_nemo_gym": gym}, policy={"generation": vllm_config}, async_rl=types.SimpleNamespace( generation_router=types.SimpleNamespace(enabled=router), @@ -2869,6 +2873,9 @@ def _dynamo_master_config(*, router=False, fleet_health=False): ({}, None), ({"router": True}, "routes the whole fleet behind one URL"), ({"fleet_health": True}, "Disable fleet health"), + # Gym's replies need the engine's tokenizer; NeMo RL's rollouts do not. + ({"gym": True, "skip_tokenizer_init": True}, "skip_tokenizer_init"), + ({"skip_tokenizer_init": True}, None), ], ) def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error): From 53d7c9df85885e3ba817985f7bc0aa46479a564d Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 19:40:49 -0800 Subject: [PATCH 48/82] refactor(dynamo): one request contract, a typed plan, no dead code - GymRequestFields holds the request fields both paths read; the native request classes inherit it, GymCallExtras extends it with the Dynamo-only measure flag, and the chat engine forwards exactly those fields to the render. Native's measure reply is a GymCallResult (C53). - The capture plan and the engine's reply are TypedDicts, so pyrefly checks their keys at the producer and the readers (C67). - splice_prefix_tokens requires the template's prefix again: no caller passes None since the engine stopped splicing (C69). - Render workers select vLLM's CPU platform with VLLM_TARGET_DEVICE instead of replacing vllm.platforms.current_platform by hand (C70). - model_stop_token_ids reads vLLM's ModelConfig, so it moves to vllm/utils.py (C71). - vllm_worker_async.py's file-wide `ruff: noqa`, left from the classes that moved to chat_server.py, hid 10 unused imports (C72). - Drop the dict-key entry for the now-boolean dynamo_dialect (C76) and the chat-engine factory's unread pool list (C77). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/environments/nemo_gym_shards.py | 1 - .../models/generation/dynamo/vllm/adapter.py | 30 ++++--- .../generation/dynamo/vllm/chat_engine.py | 57 ++++++++++-- .../generation/dynamo/vllm/render_pool.py | 18 ++-- nemo_rl/models/generation/gym_call.py | 30 ++++--- .../models/generation/openai_server_utils.py | 31 +------ nemo_rl/models/generation/vllm/chat_server.py | 29 ++++--- nemo_rl/models/generation/vllm/utils.py | 12 +++ .../generation/vllm/vllm_worker_async.py | 24 ++---- .../generation/test_dynamo_vllm_adapter.py | 5 +- .../generation/test_openai_server_utils.py | 86 ------------------- 11 files changed, 130 insertions(+), 193 deletions(-) diff --git a/nemo_rl/environments/nemo_gym_shards.py b/nemo_rl/environments/nemo_gym_shards.py index 910680269d1..84771acef99 100644 --- a/nemo_rl/environments/nemo_gym_shards.py +++ b/nemo_rl/environments/nemo_gym_shards.py @@ -39,7 +39,6 @@ # The actor factory or rollout code consumes these separately. NEMO_RL_DICT_CONFIG_KEYS = frozenset( { - "dynamo_dialect", "effort_levels", "external_service_readiness", "tokenizer_config", diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index 887838ce752..d1806696571 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -70,6 +70,8 @@ GYM_CALL_ANNOTATION, PLAN_KEY, RUNTIME_DATA_KEY, + GymCallPlan, + GymEngineReply, ) from nemo_rl.models.generation.gym_call import GymCallResult from nemo_rl.models.generation.openai_server_utils import PrefixSplice @@ -283,7 +285,7 @@ async def _generate_gym( yield chunk async def _serve_gym_call( - self, plan: dict[str, Any], prompt_token_ids: list[int], context: Context + self, plan: GymCallPlan, prompt_token_ids: list[int], context: Context ) -> Optional[GenerateChunk]: """One NeMo-Gym chat call, after the frontend rendered and spliced it. @@ -376,15 +378,16 @@ async def _serve_gym_call( result = self._call_result(final_output) if chat.logprobs: output["logprobs"] = _wire_logprobs(completion.logprobs) + reply: GymEngineReply = { + "output": output, + "result": result.model_dump(exclude_none=True), + } return { "token_ids": output["token_ids"], "index": 0, "finish_reason": completion.finish_reason, "completion_usage": _usage(final_output, len(completion.token_ids)), - "engine_data": { - "output": output, - "result": result.model_dump(exclude_none=True), - }, + "engine_data": reply, } finally: # A no-op once the call was staged: staging pops its capture state. @@ -397,7 +400,7 @@ async def _serve_gym_call( ) async def _generate( - self, chat: Any, engine_prompt: Any, plan: dict[str, Any], context: Context + self, chat: Any, engine_prompt: Any, plan: GymCallPlan, context: Context ) -> Any: """Generate as the native server does, with the SamplingParams it builds.""" model_config = self._llm.model_config @@ -546,17 +549,18 @@ def _refusal(status: int, message: str, *, acknowledge: bool = False) -> Generat without its status, so it could only answer 500; the chat engine answers this with ``status``, as the native server does. """ + reply: GymEngineReply = { + "refusal": { + "status": int(status), + "message": message, + "acknowledge": acknowledge, + } + } return { "token_ids": [], "index": 0, "finish_reason": "stop", - "engine_data": { - "refusal": { - "status": int(status), - "message": message, - "acknowledge": acknowledge, - } - }, + "engine_data": reply, } diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 333dec8e8aa..5a4fe7fe9d1 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -40,7 +40,7 @@ import threading import time import uuid -from typing import Any, AsyncGenerator, Optional +from typing import Any, AsyncGenerator, Optional, TypedDict from dynamo.llm import PythonAsyncEngine from dynamo.llm.exceptions import HttpError @@ -58,6 +58,7 @@ GYM_RESULT_NVEXT_KEY, GymCallExtras, GymCallResult, + GymRequestFields, ) from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens @@ -68,6 +69,46 @@ GYM_CALL_ANNOTATION = "nemo_gym:rollout" PLAN_KEY = "nemo_rl" + +class GymCallPlan(TypedDict): + """What the chat engine sends the adapter in ``extra_args[PLAN_KEY]``.""" + + # The rendered chat request, without messages. + request: dict[str, Any] + # Capture admissions as JSON: what is stored, and how the prompt is served. + admission: Optional[dict[str, Any]] + serving_admission: Optional[dict[str, Any]] + verify_retained_media: bool + # The model's own prefix length at the start of the spliced prompt. + prefix_len: Optional[int] + template_token_ids: list[int] + # PrefixSplice's (model_cut_end, template_cut_start), when spliced. + splice: Optional[list[int]] + # The render's multimodal input, pickled then base64-encoded. + multimodal: Optional[str] + reasoning_ended: Optional[bool] + reasoning_parser_kwargs: Optional[dict[str, Any]] + + +class GymCallRefusal(TypedDict): + """A call the engine refused before it ran, with the native status.""" + + status: int + message: str + # Whether Gym is told the capture call was refused (a native 400). + acknowledge: bool + + +class GymEngineReply(TypedDict, total=False): + """The adapter's ``engine_data`` for a Gym call: an output, or a refusal.""" + + # A served call: the completion the reply is built from, and its + # GymCallResult as JSON. + output: dict[str, Any] + result: dict[str, Any] + refusal: GymCallRefusal + + # Fields of Dynamo's chat request that are not vLLM request fields # (chat_template_args is renamed to vLLM's chat_template_kwargs). _DYNAMO_ONLY_FIELDS = {"nvext", "chat_template_args", "unsupported_fields"} @@ -240,8 +281,8 @@ async def _serve( generation_config=self._generation, ) ) - body["ng_capture"] = extras.ng_capture - body["required_prefix_token_ids"] = extras.required_prefix_token_ids + request_fields: set[str] = set(GymRequestFields.model_fields) + body.update(extras.model_dump(include=request_fields)) info = await self._pool.info() rendered = await self._pool.render(body, measure=extras.ng_measure) prompt_token_ids, splice, prefix = await self._splice( @@ -263,7 +304,7 @@ async def _serve( if max_tokens is not None: engine_request["max_tokens"] = max_tokens engine_request["max_completion_tokens"] = None - plan = { + plan: GymCallPlan = { "request": engine_request, "admission": rendered.admission, "serving_admission": rendered.serving_admission, @@ -330,6 +371,8 @@ async def _splice( prefix = rendered.required_prefix_token_ids if prefix is None: return rendered.template_token_ids, None, None + # The render includes the history whenever a prefix will be spliced. + assert rendered.history_token_ids is not None splice = splice_prefix_tokens( tokenizer=None, model_prefix_token_ids=prefix, @@ -341,8 +384,8 @@ async def _splice( async def _call_engine( self, preprocessed: dict[str, Any], context: Any - ) -> dict[str, Any]: - engine_data = None + ) -> GymEngineReply: + engine_data: Optional[GymEngineReply] = None stream = await self._routed_engine.generate(preprocessed, context=context) try: async for item in stream: @@ -407,7 +450,6 @@ class ChatEngineFactory: def __init__(self) -> None: self._capture = _CaptureSource() - self._pools: list[RenderPool] = [] async def chat_engine_factory( self, instance_id: Any, mdc: Any, routed_engine: Any @@ -424,7 +466,6 @@ async def chat_engine_factory( pool = RenderPool( spec, mdc.local_dir(), workers=int(os.environ[RENDER_WORKERS_ENV]) ) - self._pools.append(pool) engine = NemoRLChatEngine( pool=pool, routed_engine=routed_engine, spec=spec, capture=self._capture ) diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index 4bcd4a1a632..3a6db0359cc 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -33,6 +33,7 @@ import asyncio import logging import multiprocessing +import os import pickle import uuid from concurrent.futures import ProcessPoolExecutor @@ -75,14 +76,9 @@ class _Worker: """The vLLM objects one worker process renders and responds with.""" def __init__(self, spec: dict[str, Any], model_dir: str): - import vllm.platforms - - # A frontend host may have no GPU; vLLM only parses here, as in - # Dynamo's own frontend (dynamo.frontend.main.parse_args). - if vllm.platforms.current_platform.device_type == "": - from vllm.platforms.cpu import CpuPlatform - - vllm.platforms.current_platform = CpuPlatform() + # Render workers only parse and render; pin vLLM to its CPU platform so + # a frontend host needs no accelerator (an explicit target is final). + os.environ["VLLM_TARGET_DEVICE"] = "cpu" from types import SimpleNamespace @@ -96,11 +92,9 @@ def __init__(self, spec: dict[str, Any], model_dir: str): from vllm.renderers.registry import renderer_from_config from vllm.v1.engine.input_processor import InputProcessor - from nemo_rl.models.generation.openai_server_utils import ( - model_stop_token_ids, - turn_end_token_ids, - ) + from nemo_rl.models.generation.openai_server_utils import turn_end_token_ids from nemo_rl.models.generation.vllm import chat_server + from nemo_rl.models.generation.vllm.utils import model_stop_token_ids from nemo_rl.models.generation.vllm.video_utils import ( register_vllm_video_loader, ) diff --git a/nemo_rl/models/generation/gym_call.py b/nemo_rl/models/generation/gym_call.py index 5a4346f6b86..87d933d4c9f 100644 --- a/nemo_rl/models/generation/gym_call.py +++ b/nemo_rl/models/generation/gym_call.py @@ -14,12 +14,14 @@ """The extra fields NeMo RL and NeMo-Gym exchange on a chat call, defined once. -The native vLLM server reads :class:`GymCallExtras` as request fields and -answers with :class:`GymCallResult`'s fields on the message. Dynamo's frontend -carries only OpenAI fields plus ``nvext``, so through it NeMo-Gym's Dynamo -dialect packs the extras into one ``nemo_rl:gym:`` annotation and NeMo -RL's chat engine returns the result as ``nvext["nemo_rl"]``. A field added here -reaches both paths. +The native vLLM server's request classes inherit :class:`GymRequestFields`, and +it answers with :class:`GymCallResult`'s fields on the message. Dynamo's +frontend carries only OpenAI fields plus ``nvext``, so through it NeMo-Gym's +Dynamo dialect packs :class:`GymCallExtras` (those request fields, plus the +measure flag) into one ``nemo_rl:gym:`` annotation. NeMo RL's chat engine +hands the request fields to the same render and returns the result as +``nvext["nemo_rl"]``. A field added to :class:`GymRequestFields` reaches both +paths. """ import json @@ -34,17 +36,23 @@ GYM_RESULT_NVEXT_KEY = "nemo_rl" -class GymCallExtras(BaseModel): - """What a NeMo-Gym chat call adds to an OpenAI request.""" - - # A field Gym sends that NeMo RL does not know is refused, not ignored. - model_config = ConfigDict(extra="forbid") +class GymRequestFields(BaseModel): + """The request fields both paths read; the native request classes inherit them.""" # The token-capture admission (capture_key, call ids, prev_len, mode, ...). ng_capture: Optional[dict[str, Any]] = None # Without capture: the last echoed assistant turn's prompt + generation ids. required_prefix_token_ids: Optional[list[int]] = None + + +class GymCallExtras(GymRequestFields): + """What a NeMo-Gym chat call adds to an OpenAI request through Dynamo.""" + + # A field Gym sends that NeMo RL does not know is refused, not ignored. + model_config = ConfigDict(extra="forbid") + # Return only the prompt's length (Gym's /context/measure); no generation. + # Natively Gym calls /tokenize for this instead. ng_measure: bool = False @classmethod diff --git a/nemo_rl/models/generation/openai_server_utils.py b/nemo_rl/models/generation/openai_server_utils.py index f1e26bd1a8b..0f520f75eb4 100644 --- a/nemo_rl/models/generation/openai_server_utils.py +++ b/nemo_rl/models/generation/openai_server_utils.py @@ -27,18 +27,6 @@ from typing import Any -def model_stop_token_ids(model_config: Any, cfg: dict[str, Any]) -> frozenset[int]: - """The configured stops plus the generation config's EOS, read once per server. - - ``model_config`` is vLLM's ``ModelConfig``; ``cfg`` is ``policy.generation``. - """ - stop_token_ids = set(cfg.get("stop_token_ids") or ()) - generation_eos = model_config.try_get_generation_config().get("eos_token_id") - if isinstance(generation_eos, int): - generation_eos = [generation_eos] - return frozenset(stop_token_ids | set(generation_eos or ())) - - def turn_end_token_ids( stop_token_ids: Collection[int], tokenizer: Any ) -> frozenset[int]: @@ -46,7 +34,8 @@ def turn_end_token_ids( vLLM stops on the generation config's EOS as well as the configured stops, and the chat template closes turns with the tokenizer's EOS, so a turn can - end on any of them. ``stop_token_ids`` is :func:`model_stop_token_ids`. + end on any of them. ``stop_token_ids`` is vLLM's + :func:`nemo_rl.models.generation.vllm.utils.model_stop_token_ids`. """ return frozenset(stop_token_ids) | {tokenizer.eos_token_id} @@ -86,7 +75,7 @@ def splice_prefix_tokens( *, tokenizer: Any, model_prefix_token_ids: list[int], - template_prefix_token_ids: list[int] | None, + template_prefix_token_ids: list[int], template_token_ids: list[int], eos_token_id: int | Collection[int], ) -> PrefixSplice: @@ -156,14 +145,6 @@ def splice_prefix_tokens( and when the model's prefix ends on one of them that exact id is kept in place of the template's, so the spliced prompt still starts with the model's tokens verbatim. - - A host that sees only the rendered prompt, not the messages (the Dynamo - engine), passes ``template_prefix_token_ids=None``. The prefix render is only - used for its EOS count, and every EOS in the model's prefix closed one - message, plus the one the template adds to a final assistant turn the model - did not close. The counts agree when the history up to that turn is the one - the prefix was generated from and the model emitted no EOS text as ordinary - tokens. """ if not model_prefix_token_ids: return PrefixSplice(template_token_ids, 0, 0) @@ -181,12 +162,6 @@ def splice_prefix_tokens( model_ended_on_eos = model_prefix_token_ids[-1] in eos_token_ids if model_ended_on_eos: model_cut_end -= 1 - if template_prefix_token_ids is None: - template_prefix_token_ids = ( - model_prefix_token_ids - if model_ended_on_eos - else [*model_prefix_token_ids, next(iter(eos_token_ids))] - ) # Locate the turn boundary by EOS count rather than token position. Qwen3 # templates may strip prior reasoning blocks when re-rendering history; diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index 184f5a6f3eb..cf04f249a7d 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -28,7 +28,7 @@ from dataclasses import dataclass, replace from logging import Filter as LoggingFilter from logging import LogRecord -from typing import Any, List, Literal, Optional, Union +from typing import Any, Literal, Optional, Union import torch from fastapi import FastAPI, Request @@ -61,8 +61,8 @@ from nemo_rl.data.captured_media import MediaCaptureRejected from nemo_rl.models.generation.capture_context import resolve_capture_input +from nemo_rl.models.generation.gym_call import GymCallResult, GymRequestFields from nemo_rl.models.generation.openai_server_utils import ( - model_stop_token_ids, splice_prefix_tokens, turn_end_token_ids, ) @@ -75,6 +75,7 @@ attach_sampler_topk_logprobs_to_chat_response_choices, attach_token_information_to_chat_response_choices, model_dump_chat_response_with_dynamic_message_fields, + model_stop_token_ids, on_policy_sampling, remap_multimodal_placeholders, ) @@ -101,11 +102,10 @@ def model_post_init(self, context): # This MRO is necessary i.e. NeMoRLOpenAIChatRequestMixin > ChatCompletionRequest -class NeMoRLChatCompletionRequest(NeMoRLOpenAIChatRequestMixin, ChatCompletionRequest): - required_prefix_token_ids: Optional[List[int]] = None - # Ledger-authoritative token capture: the call identity the ledger - # attaches (capture_key, call_id, parent_call_id, prev_len, mode). - ng_capture: Optional[dict[str, Any]] = None +class NeMoRLChatCompletionRequest( + NeMoRLOpenAIChatRequestMixin, GymRequestFields, ChatCompletionRequest +): + pass class NeMoRLMeasureChatRequest(NeMoRLChatCompletionRequest): @@ -142,9 +142,9 @@ def chat_request_from_render(request: dict[str, Any]) -> NeMoRLChatCompletionReq # This MRO is necessary i.e. NeMoRLOpenAIChatRequestMixin > TokenizeRequest -class NeMoRLTokenizeChatRequest(NeMoRLOpenAIChatRequestMixin, TokenizeChatRequest): - required_prefix_token_ids: Optional[List[int]] = None - ng_capture: Optional[dict[str, Any]] = None +class NeMoRLTokenizeChatRequest( + NeMoRLOpenAIChatRequestMixin, GymRequestFields, TokenizeChatRequest +): ng_measure_only: Literal[True] = True @@ -376,6 +376,8 @@ async def preprocess_chat( ) return res + # The render includes the history whenever a prefix will be spliced. + assert rendered.history_token_ids is not None splice = splice_prefix_tokens( tokenizer=self.renderer.tokenizer, model_prefix_token_ids=list(request.required_prefix_token_ids), @@ -961,10 +963,9 @@ async def tokenize(request: NeMoRLTokenizeRequest, raw_request: Request): and request.ng_capture is not None ): return JSONResponse( - content={ - "prompt_token_count": generator.count, - "ng_context_measured": True, - } + content=GymCallResult( + prompt_token_count=generator.count, ng_context_measured=True + ).model_dump(exclude_none=True) ) return JSONResponse(content=generator.model_dump()) diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index 83134b92fcf..80c19d975c3 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -72,6 +72,18 @@ def _format_refit_key_error(label: str, keys: set[str]) -> str: ) +def model_stop_token_ids(model_config: Any, cfg: dict[str, Any]) -> frozenset[int]: + """The configured stops plus the generation config's EOS, read once per server. + + ``model_config`` is vLLM's ``ModelConfig``; ``cfg`` is ``policy.generation``. + """ + stop_token_ids = set(cfg.get("stop_token_ids") or ()) + generation_eos = model_config.try_get_generation_config().get("eos_token_id") + if isinstance(generation_eos, int): + generation_eos = [generation_eos] + return frozenset(stop_token_ids | set(generation_eos or ())) + + def assert_reload_refit_config_supported(config: VllmConfig) -> None: """Reject pure-config combinations unsupported by vLLM reload refit.""" vllm_cfg = config.get("vllm_cfg") diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 2bebcd08b8b..5accdbb13b0 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -24,7 +24,7 @@ import warnings from collections.abc import Awaitable, Callable from dataclasses import dataclass -from typing import Literal, TYPE_CHECKING, Any, AsyncGenerator, Optional, cast +from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, cast import ray import torch @@ -58,10 +58,11 @@ GenerationOutputSpec, verify_right_padding, ) -from nemo_rl.models.generation.score_centering import validate_score_centering_request -from nemo_rl.models.generation.vllm.captured_chat_logprobs import ( - CapturedChatLogprobsMixin, +from nemo_rl.models.generation.openai_server_utils import ( + PrefixSplice, ) +from nemo_rl.models.generation.rollout_result import RolloutResult +from nemo_rl.models.generation.score_centering import validate_score_centering_request from nemo_rl.models.generation.vllm.checkpoint_engine import ( VllmAsyncCheckpointEngineRpcMixin, ) @@ -69,27 +70,15 @@ resolve_collective_rpc_result, ) from nemo_rl.models.generation.vllm.config import parse_nvfp4_pertoken_rollout -from nemo_rl.models.generation.rollout_result import RolloutResult from nemo_rl.models.generation.vllm.request_pipeline import ( rollout_output, rollout_result, ) from nemo_rl.models.generation.vllm.utils import ( - attach_routed_experts_to_chat_response_choices, - attach_token_information_to_chat_response_choices, - attach_sampler_topk_logprobs_to_chat_response_choices, format_prompt_for_vllm_generation, - check_sampling_param, validate_rollout_prompt, - model_dump_chat_response_with_dynamic_message_fields, - remap_multimodal_placeholders, ) from nemo_rl.models.generation.vllm.vllm_worker import BaseVllmGenerationWorker -from nemo_rl.models.generation.capture_context import decide_capture_input -from nemo_rl.models.generation.openai_server_utils import ( - PrefixSplice, - splice_prefix_tokens, -) from nemo_rl.telemetry.setup import shutdown_telemetry from nemo_rl.utils.routed_experts_codec import routed_experts_tensor_metadata @@ -500,7 +489,7 @@ def _start_vllm_metrics_logger(self) -> None: Controlled by the required vllm_metrics_logger_interval in vllm_cfg. Runs only on the model-owner actor. """ - from vllm.v1.metrics.reader import Gauge, Counter, get_metrics_snapshot + from vllm.v1.metrics.reader import Counter, Gauge, get_metrics_snapshot assert self.cfg["vllm_cfg"].get("async_engine", False), ( "vLLM metrics logger is only supported with async engine enabled" @@ -1155,7 +1144,6 @@ def _context_refusal_headers(self, request: Any) -> dict[str, str]: return {CAPTURE_REFUSAL_HEADER: admission.model_call_id} - # ruff: noqa def _setup_vllm_openai_api_server(self, app: FastAPI) -> FastAPI: # Deferred: the chat server imports vLLM's OpenAI server stack. from nemo_rl.models.generation.vllm import chat_server diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index a9a37d83a24..d0284818adc 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -52,6 +52,7 @@ GYM_CALL_ANNOTATION, PLAN_KEY, RUNTIME_DATA_KEY, + GymCallPlan, ) from nemo_rl.models.generation.rollout_result import ( # noqa: E402 rollout_result_from_wire, @@ -445,7 +446,7 @@ def _chat(**fields) -> dict: def _gym_call(token_ids: list[int], **plan) -> dict: """A Gym call as the chat engine routes it: final token ids and the plan.""" - plan = { + full_plan: GymCallPlan = { "request": _chat(), "admission": None, "serving_admission": None, @@ -461,7 +462,7 @@ def _gym_call(token_ids: list[int], **plan) -> dict: return { "token_ids": token_ids, "annotations": [GYM_CALL_ANNOTATION], - "extra_args": {PLAN_KEY: plan}, + "extra_args": {PLAN_KEY: full_plan}, } diff --git a/tests/unit/models/generation/test_openai_server_utils.py b/tests/unit/models/generation/test_openai_server_utils.py index 71c470fd3a2..36e7fcc910d 100644 --- a/tests/unit/models/generation/test_openai_server_utils.py +++ b/tests/unit/models/generation/test_openai_server_utils.py @@ -146,92 +146,6 @@ def test_splice_prefix_tokens_without_tokenizer_honors_explicit_eos_ids( ) assert result == expected assert result.token_ids[: len(model_prefix)] == model_prefix - # A host without the messages (the Dynamo engine) passes no template prefix. - assert ( - splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=model_prefix, - template_prefix_token_ids=None, - template_token_ids=template, - eos_token_id=eos_token_id, - ) - == expected - ) - - -@pytest.mark.parametrize( - "model_name", ["Qwen/Qwen3-0.6B", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"] -) -def test_splicing_without_the_template_prefix_matches_real_template_renders( - model_name, -): - """Across tool-calling turns with reasoning, splicing without the template's - render of the history (as the Dynamo engine does) matches splicing with it.""" - transformers = pytest.importorskip("transformers") - try: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - except OSError as error: - pytest.skip(f"{model_name} tokenizer unavailable: {error}") - eos = tokenizer.eos_token_id - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Weather for a city.", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - }, - } - ] - - def render(messages, *, add_generation_prompt): - rendered = tokenizer.apply_chat_template( - messages, - tools=tools, - add_generation_prompt=add_generation_prompt, - tokenize=True, - ) - return list(getattr(rendered, "input_ids", rendered)) - - def generate(text): - return tokenizer.encode(text, add_special_tokens=False) + [eos] - - messages = [ - {"role": "system", "content": "You are a weather assistant."}, - {"role": "user", "content": "Weather in Paris?"}, - ] - replies = [ - "\nI should look it up.\n\n\nChecking the forecast.", - "\nIt is sunny.\n\n\nIt is sunny in Paris.", - ] - follow_ups = [ - {"role": "tool", "content": "sunny, 22C"}, - {"role": "user", "content": "And London?"}, - ] - chain = render(messages, add_generation_prompt=True) + generate(replies[0]) - for reply, follow_up, next_reply in zip(replies, follow_ups, replies[1:] + ["ok"]): - messages = [*messages, {"role": "assistant", "content": reply}, follow_up] - template_prefix = render(messages[:-1], add_generation_prompt=False) - template = render(messages, add_generation_prompt=True) - spliced = splice_prefix_tokens( - tokenizer=tokenizer, - model_prefix_token_ids=chain, - template_prefix_token_ids=template_prefix, - template_token_ids=template, - eos_token_id=tokenizer.eos_token_id, - ) - assert spliced == splice_prefix_tokens( - tokenizer=tokenizer, - model_prefix_token_ids=chain, - template_prefix_token_ids=None, - template_token_ids=template, - eos_token_id=tokenizer.eos_token_id, - ) - chain = spliced.token_ids + generate(next_reply) def test_replace_prefix_tokens_without_tokenizer_reports_missing_eos_without_decoding(): From 28fd0e810709bead51d62238a9037e81c9b99ac2 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 19:45:28 -0800 Subject: [PATCH 49/82] test(dynamo): tie the measure call to Gym; cover the budget fallback and the frontend hook - Run Gym's real /context/measure in its Dynamo dialect against NeMo RL's models, so a renamed measure flag or count fails here (C61). - A step-order case without max_tokens: the adapter's copy of vLLM's budget fallback was never compared with native (C62). - Check that Dynamo's frontend still builds its entrypoint through the module global NeMo RL wraps (C68). - The overlay's leak check matches the frontend module NeMo RL now launches (C58). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/functional/check_dynamo_overlay.sh | 2 +- .../generation/test_dynamo_chat_engine.py | 9 ++++ tests/unit/models/generation/test_gym_call.py | 50 ++++++++++++++++++- .../models/generation/test_gym_chat_path.py | 2 + 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/tests/functional/check_dynamo_overlay.sh b/tests/functional/check_dynamo_overlay.sh index 42a2b381e41..e77a178f2e2 100755 --- a/tests/functional/check_dynamo_overlay.sh +++ b/tests/functional/check_dynamo_overlay.sh @@ -49,6 +49,6 @@ echo "KV events the router applied: ${applied:-0}" || fail "The KV router applied none of the engines' KV-cache events" dynamo_bin_dir=${NEMO_RL_DYNAMO_BIN_DIR:-/opt/dynamo/bin} -if pgrep -f "[d]ynamo.frontend|${dynamo_bin_dir}/[e]tcd|${dynamo_bin_dir}/[n]ats-server"; then +if pgrep -f "[d]ynamo\.vllm\.frontend|${dynamo_bin_dir}/[e]tcd|${dynamo_bin_dir}/[n]ats-server"; then fail "Managed Dynamo services remain after shutdown" fi diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index 49ee7af2d1c..53e17a17cba 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -436,6 +436,15 @@ def test_an_engine_that_does_not_serve_the_call_fails_it(router, status, message assert failed.value.code == status +def test_dynamos_frontend_still_builds_its_entrypoint_by_the_name_nemo_rl_wraps(): + import dynamo.frontend.main as frontend_main + + # frontend.main() swaps this module global for one that adds NeMo RL's + # chat_engine_factory; if Dynamo stops reading it there, Dynamo's own + # preprocessor serves chat and every NeMo-Gym call is refused. + assert "EntrypointArgs" in frontend_main.async_main.__code__.co_names + + def test_only_nemo_rls_engines_are_served() -> None: model = SimpleNamespace( runtime_config=lambda: {"runtime_data": {}}, diff --git a/tests/unit/models/generation/test_gym_call.py b/tests/unit/models/generation/test_gym_call.py index 331a191ffb4..ec31a63b82d 100644 --- a/tests/unit/models/generation/test_gym_call.py +++ b/tests/unit/models/generation/test_gym_call.py @@ -20,7 +20,10 @@ pytestmark = pytest.mark.nemo_gym pytest.importorskip("nemo_gym.token_id_capture.external_capture") -from nemo_gym.openai_utils import TOKEN_METADATA_FIELDS # noqa: E402 +from nemo_gym.openai_utils import ( # noqa: E402 + TOKEN_METADATA_FIELDS, + NeMoGymAsyncOpenAI, +) from nemo_gym.token_id_capture import ( # noqa: E402 NG_CAPTURE_FIELD, NG_COMMIT_COORDS_FIELD, @@ -31,6 +34,9 @@ pop_dynamo_call_result, ) from responses_api_models.vllm_model.app import VLLMModel # noqa: E402 +from responses_api_models.vllm_model.tests import ( # noqa: E402 + test_framework_context as gym_harness, +) from nemo_rl.models.generation.gym_call import ( # noqa: E402 GYM_EXTRAS_ANNOTATION_PREFIX, @@ -87,3 +93,45 @@ def test_gym_lifts_the_call_result_nemo_rl_returns() -> None: assert pop_dynamo_call_result({"nvext": {GYM_RESULT_NVEXT_KEY: result}}) == result # The fields Gym copies onto the message, and its capture acknowledgement. assert TOKEN_METADATA_FIELDS | {NG_COMMIT_COORDS_FIELD} <= result.keys() + + +def test_gyms_measure_call_is_the_one_nemo_rl_answers(monkeypatch, tmp_path) -> None: + # Gym's own /context/measure, in its Dynamo dialect. + harness = gym_harness.make_capture_harness(monkeypatch, tmp_path, dynamo=True) + first = gym_harness.assert_clean(harness.post("attempt", gym_harness.HISTORY)) + seen = [] + + async def frontend(client, *, extra_headers=None, **body): + # Unknown names are refused, so a measure flag Gym renames fails here. + seen.append(GymCallExtras.from_annotations(body["nvext"]["annotations"])) + result = GymCallResult(prompt_token_count=19, ng_context_measured=True) + return { + "id": "chatcmpl-measure", + "object": "chat.completion", + "created": 0, + "model": "m", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": ""}, + } + ], + "nvext": {GYM_RESULT_NVEXT_KEY: result.model_dump(exclude_none=True)}, + } + + monkeypatch.setattr(NeMoGymAsyncOpenAI, "create_chat_completion", frontend) + response = harness.client.post( + "/ng-rollout/attempt/training-token-capture/context/measure", + json={ + "input": gym_harness.HISTORY + + first["output"] + + [{"role": "user", "content": "next"}] + }, + ) + + # Gym read the result back by NeMo RL's names; a renamed one answers 502. + assert response.status_code == 200, response.text + assert response.json() == {"prompt_token_count": 19} + (extras,) = seen + assert extras.ng_measure is True diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 741ba0b2229..01549bc93b1 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -627,6 +627,8 @@ def _off_policy(name: str, value: Any) -> _Case: CASES = [ # A plain call: the render and the reply's reasoning and tool parsers (C39). _Case("plain"), + # No max_tokens: each path fills the budget the way vLLM's server does. + _Case("no_max_tokens", fields={"max_tokens": None}), # Gym's echo of turn 1 without capture: the model's tokens replace the # template's re-render of that turn (C16's turn-end cut). _Case( From 4fd57e8f01f0b49e164cab3da5b8d5b7720e6369 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 19:46:15 -0800 Subject: [PATCH 50/82] docs(dynamo): correct the guide and comments - The 4-stop limit lives in Dynamo's Rust preprocessor, which chat calls skip once a chat_engine_factory is set: the guide now lists what the frontend does refuse (C55). - Startup's new rejections (the six dropped extra_body fields and skip_tokenizer_init with Gym) and the gpt-oss refusal are listed. - session_affinity_ttl_secs takes effect from Dynamo 1.7: 1.5.1 drops a chat call's session id before the router reads it (C60). - render_workers' rates move from the code comment to the guide, with the setup they were measured on and each worker's memory (C73). - ng_measure is the Dynamo dialect's flag, not a native request field (C53); any non-colocated SingleController recipe can switch to Dynamo, Gym ones included (C74); review-thread tags are gone from comments (C75). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 22 +++++++++++++------ nemo_rl/models/generation/dynamo/config.py | 11 +++++----- .../models/generation/dynamo/vllm/adapter.py | 4 ++-- nemo_rl/models/generation/vllm/chat_server.py | 2 +- .../models/generation/test_gym_chat_path.py | 18 +++++++-------- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 229f9d2a498..944e5919e30 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -53,8 +53,8 @@ shows them all: | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | | `request_timeout_s` | null | Deadline for one NeMo RL rollout through the frontend. null waits as long as generation takes, as native does; on the SingleController, `async_rl.rollout_failure.native.generation_timeout_s` bounds both paths | | `frontend_args.router_mode` | `kv` | Frontend routing policy | -| `frontend_args.session_affinity_ttl_secs` | 300 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it. Dynamo keeps at most 65,536 sessions and answers a new one past that with HTTP 529 | -| `frontend_args.render_workers` | 8 | Processes in the frontend that render NeMo-Gym's chat calls and build their replies. Each serves about 640 single-turn calls/s, 45 for a 21K-token agentic history, or 200 with 1-4 images | +| `frontend_args.session_affinity_ttl_secs` | 300 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it. Takes effect from Dynamo 1.7: Dynamo 1.5.1 drops the session id of a chat call served by NeMo RL's chat engine. Dynamo keeps at most 65,536 sessions and answers a new one past that with HTTP 529 | +| `frontend_args.render_workers` | 8 | Processes in the frontend that render NeMo-Gym's chat calls and build their replies; every Gym call in the fleet shares them. Each holds its own tokenizer and processor, about 1.7–2.3 GB. On one Grace core of a GB200, one serves about 640 single-turn calls/s, 45 for a 21K-token agentic history (22 with its prefix splice), or 200 with 1–4 images | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags. `--dyn-chat-processor` and `--custom-jinja-template` are rejected: NeMo RL renders chat itself | The etcd and nats-server binaries are found in `NEMO_RL_DYNAMO_BIN_DIR` when it @@ -80,8 +80,9 @@ NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect (`dynamo_frontend` in Gym's [`vllm_model`](https://github.com/NVIDIA-NeMo/Gym/tree/main/responses_api_models/vllm_model) config). The frontend takes only OpenAI fields plus `nvext`, so Gym sends the native -request's extra fields (`ng_capture`, `required_prefix_token_ids`, `ng_measure`) -in one `nemo_rl:gym:` annotation, and reads the call's token fields +request's extra fields (`ng_capture`, `required_prefix_token_ids`), and +`ng_measure` for the measure call it makes in place of `/tokenize`, in one +`nemo_rl:gym:` annotation, and reads the call's token fields (`prompt_token_ids`, `generation_token_ids`, `generation_log_probs`, `routed_experts`, `ng_commit_coords`) from `nvext.nemo_rl`. @@ -107,7 +108,10 @@ same record and return the same reply. Where the behavior differs from the native endpoint: -- The frontend refuses more than 4 `stop` strings or `stop_token_ids` (HTTP 500). +- The frontend refuses more than 32 `stop` strings, and vLLM request fields its + own request type lacks (for example `structured_outputs`, `return_token_ids`, + `truncate_prompt_tokens`, `priority`, `chat_template`), with HTTP 400. +- gpt-oss (Harmony) chat calls are refused with HTTP 400. - A token-capture call refused for its context length acknowledges the refusal at the end of the error message instead of in the native endpoint's `x-ng-capture-refused` header, which Dynamo's errors cannot carry. Gym's @@ -128,6 +132,11 @@ Startup rejects these with `dynamo: true`: (`env.nemo_gym.policy_model.responses_api_models.vllm_model`): the frontend answers every request carrying it with HTTP 400. Startup checks only this overlay, not Gym's own YAMLs. +- `extra_body` `stop_token_ids`, `allowed_token_ids`, `bad_words_token_ids`, + `cache_salt`, `logprob_token_ids` or `detokenize` on the policy model: the + frontend accepts them but drops them before NeMo RL's chat engine sees them. +- `vllm_cfg.skip_tokenizer_init: true` with NeMo-Gym: the engine detokenizes + the replies Gym's calls get through the frontend. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data parallelism inside vLLM, which Dynamo does not serve yet. Above data-parallel rank 0, vLLM rejects each engine's `ipc://` KV-event endpoint. @@ -164,8 +173,7 @@ bash docker/dynamo/install.sh ## Run -Use any non-colocated SingleController vLLM recipe that does not use NeMo-Gym, -and add the switch: +Use any non-colocated SingleController vLLM recipe and add the switch: ```bash uv run examples/run_grpo_single_controller.py \ diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index f31758011fa..f8cc16dfed5 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -40,14 +40,13 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "device-aware-weighted", ] = "kv" # Idle seconds a NeMo-Gym token-capture session stays on one engine, as native - # pins each Gym session to one server; None disables it. Dynamo keeps at most - # 65,536 sessions (HTTP 529 past that), so keep (captured rollouts started per - # second) x TTL below that. + # pins each Gym session to one server; None disables it. Needs Dynamo 1.7 + # (ai-dynamo/dynamo#15333): 1.5.1 drops a chat call's session id. Dynamo keeps + # at most 65,536 sessions (HTTP 529 past that). session_affinity_ttl_secs: Annotated[int, Field(gt=0, le=31_536_000)] | None = 300 # Processes that render NeMo-Gym's chat calls and build their replies (the - # checkpoint's template, tokenizer, HF processor and vLLM's parsers). Each - # serves on one core: about 640 single-turn calls/s, 45 for a 21K-token - # agentic history (22 with its prefix splice), or 200 with 1-4 images. + # checkpoint's template, tokenizer, HF processor and vLLM's parsers); each + # uses one core. See docs/guides/dynamo-generation.md for sizing. render_workers: PositiveInt = 8 extra_cli_args: list[str] = Field(default_factory=list) diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index d1806696571..3c381210a40 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -306,7 +306,7 @@ async def _serve_gym_call( admission = host._parse_capture_admission(plan["admission"]) serving_admission = host._parse_capture_admission(plan["serving_admission"]) if admission is not None and admission.mode == "candidate": - # C26: the frontend's chat engine decides a candidate; a request + # The frontend's chat engine decides a candidate; a request # still marked one skipped that decision. raise InvalidArgument( "a candidate admission reached the engine undecided; NeMo RL's " @@ -485,7 +485,7 @@ async def _stage(self, chat: Any, final_output: Any) -> GymCallResult: if routes is not None: message["routed_experts"] = routes if self._host._capture_top_logprobs: - # C29: score centering stages the sampler's top-k with the call. + # Score centering stages the sampler's top-k with the call. ( message["sampler_topk_token_ids"], message["sampler_topk_logprobs"], diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index cf04f249a7d..e5d48a46a22 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -206,7 +206,7 @@ async def _render_chat_call( The steps every host of the chat path shares, before any TQ read: the render with the checkpoint's template and processor, the capture - decision for a Gym candidate (C26 on #4979), and, when a prefix will be + decision for a Gym candidate, and, when a prefix will be spliced, the render of the history up to the last assistant turn that the splice cuts against. """ diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 01549bc93b1..03917dde1d6 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -625,18 +625,18 @@ def _off_policy(name: str, value: Any) -> _Case: CASES = [ - # A plain call: the render and the reply's reasoning and tool parsers (C39). + # A plain call: the render and the reply's reasoning and tool parsers. _Case("plain"), # No max_tokens: each path fills the budget the way vLLM's server does. _Case("no_max_tokens", fields={"max_tokens": None}), # Gym's echo of turn 1 without capture: the model's tokens replace the - # template's re-render of that turn (C16's turn-end cut). + # template's re-render of that turn (the turn-end cut). _Case( "inline_prefix", extras=lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, ), # The same, for a model whose turns end on a token that is not its - # tokenizer's EOS, as gpt-oss's <|call|> (C16). + # tokenizer's EOS, as gpt-oss's <|call|>. _Case( "turn_end_not_eos", extras=lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, @@ -648,7 +648,7 @@ def _off_policy(name: str, value: Any) -> _Case: extras=lambda kit: {"ng_capture": _staged_turn_1(kit, mode="token_in")}, mode="token_in", ), - # Gym's candidate whose cut holds is served token-in (C26)... + # Gym's candidate whose cut holds is served token-in... _Case( "candidate_cut_holds", extras=lambda kit: {"ng_capture": _candidate(kit, [*HISTORY, OBSERVATION])}, @@ -662,7 +662,7 @@ def _off_policy(name: str, value: Any) -> _Case: mode="text", ), # A candidate whose tool call had empty content and now has none: string - # content renders both alike, so the cut holds (C26)... + # content renders both alike, so the cut holds... _Case( "empty_tool_content", messages=[HISTORY[0], TOOL_TURN, TOOL_RESULT], @@ -691,7 +691,7 @@ def _off_policy(name: str, value: Any) -> _Case: mode="text", kit="structured_content", ), - # Score centering stages the sampler's top-k with the call (C29). + # Score centering stages the sampler's top-k with the call. _Case( "score_centering", extras=lambda kit: {"ng_capture": _admission(mode="text")}, @@ -709,7 +709,7 @@ def _off_policy(name: str, value: Any) -> _Case: kit="vlm", ), # An image after the echoed turn 1: processed once in the frontend, both - # engines run the same pixels, and the splice moves its tokens (C30). + # engines run the same pixels, and the splice moves its tokens. _Case( "image_after_echoed_turn", messages=[*HISTORY, IMAGE_TURN], @@ -726,13 +726,13 @@ def _off_policy(name: str, value: Any) -> _Case: }, kit="pythonic", ), - # Nemotron-3-Nano's reasoning parser plugin, with thinking off (C39). + # Nemotron-3-Nano's reasoning parser plugin, with thinking off. _Case( "nano_v3_thinking_off", fields={"chat_template_kwargs": {"enable_thinking": False}}, kit="nano_v3", ), - # The sampling the trainer replays, set explicitly (C41)... + # The sampling the trainer replays, set explicitly... _Case( "on_policy_sampling", fields={ From f7be6c1205e28df346eef0604f89a29ea642e448 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 20:06:08 -0800 Subject: [PATCH 51/82] test(dynamo): fake vLLM's chat request as a pydantic model in the wiring test NeMoRLChatCompletionRequest now inherits GymRequestFields, a pydantic model, so the wiring test's plain placeholder base made it require its messages. The fake keeps unknown fields, as vLLM's request models do, and the overflow test builds its request with model_construct. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../test_vllm_chat_template_wiring.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/unit/models/generation/test_vllm_chat_template_wiring.py b/tests/unit/models/generation/test_vllm_chat_template_wiring.py index 89f96793c60..a81d4357595 100644 --- a/tests/unit/models/generation/test_vllm_chat_template_wiring.py +++ b/tests/unit/models/generation/test_vllm_chat_template_wiring.py @@ -37,6 +37,7 @@ import pytest import torch +from pydantic import BaseModel, ConfigDict from nemo_rl.data.captured_media import capture_processed_media from nemo_rl.models.generation import vllm as vllm_generation_pkg @@ -128,7 +129,12 @@ def placeholder(name): ) module( "vllm.entrypoints.openai.chat_completion.protocol", - ChatCompletionRequest=placeholder("ChatCompletionRequest"), + # A pydantic model that keeps unknown fields, as vLLM's request models are. + ChatCompletionRequest=type( + "ChatCompletionRequest", + (BaseModel,), + {"model_config": ConfigDict(extra="allow")}, + ), ChatCompletionResponse=placeholder("ChatCompletionResponse"), ) module( @@ -698,10 +704,12 @@ async def preprocess(self, *, messages, **kwargs): renderer.renderer = SimpleNamespace(tokenizer=SimpleNamespace(eos_token_id=2)) renderer.model_config = SimpleNamespace(max_model_len=8 if overflows else 16) endpoint = next(fn for path, fn in app.routes if path == "/v1/chat/completions") - request = endpoint.__annotations__["request"]() - request.required_prefix_token_ids = [10, 77, 31, 2] - request.max_tokens, request.max_completion_tokens = 4, None - request.model_copy = lambda **kwargs: request + request = endpoint.__annotations__["request"].model_construct( + messages=[], + required_prefix_token_ids=[10, 77, 31, 2], + max_tokens=4, + max_completion_tokens=None, + ) message = "maximum context length" if overflows else "retiled media" with pytest.raises(ValueError, match=message): From 45add8b0a14ba8eff3b4b8047f45a7cc60941e18 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sun, 8 Nov 2026 20:15:14 -0800 Subject: [PATCH 52/82] fix(dynamo): publish pydantic config blocks; keep the frontend host's platform The Dynamo overlays caught two faults in this round's changes: - policy.generation carries pydantic models once MasterConfig validates it (sampler_replay), which json.dumps refused, so no engine could register. pydantic's to_jsonable_python serializes them and still fails on anything that is not JSON. The adapter test publishes one. - Forcing vLLM's CPU platform in every render worker made ModelConfig refuse the engine's sleep mode, and it would refuse FP8 quantization too. A worker now keeps an accelerator host's platform and selects the CPU one, through VLLM_TARGET_DEVICE, only where none resolves. The worker's own engine args turn sleep mode off, which vLLM's CPU platform refuses. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../models/generation/dynamo/vllm/adapter.py | 5 ++++- .../generation/dynamo/vllm/render_pool.py | 22 +++++++++++++++---- .../generation/test_dynamo_vllm_adapter.py | 5 ++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index 3c381210a40..55c7d5ee3ad 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -50,6 +50,7 @@ ) from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource from dynamo.llm.exceptions import InvalidArgument +from pydantic_core import to_jsonable_python from vllm.entrypoints.serve.exception_handling.error_response import ( create_error_response, ) @@ -148,7 +149,9 @@ def chat_runtime_data(host: "VllmAsyncGenerationWorkerImpl") -> dict[str, Any]: return { RUNTIME_DATA_KEY: { "engine_args": engine_args, - "generation": json.loads(json.dumps(host.cfg)), + # MasterConfig validates some blocks into pydantic models + # (sampler_replay); anything else that is not JSON fails here. + "generation": to_jsonable_python(host.cfg), } } diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index 3a6db0359cc..31c89b8ec3e 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -76,9 +76,16 @@ class _Worker: """The vLLM objects one worker process renders and responds with.""" def __init__(self, spec: dict[str, Any], model_dir: str): - # Render workers only parse and render; pin vLLM to its CPU platform so - # a frontend host needs no accelerator (an explicit target is final). - os.environ["VLLM_TARGET_DEVICE"] = "cpu" + import vllm.platforms + + # A frontend host may have no accelerator; vLLM only renders here, as in + # Dynamo's own frontend. vLLM's explicit CPU target selects its CPU + # platform. An accelerator host keeps its own, which validates the + # engine's args (quantization, for one) as the engine does. + if vllm.platforms.resolve_current_platform_cls_qualname().endswith( + ".UnspecifiedPlatform" + ): + os.environ["VLLM_TARGET_DEVICE"] = "cpu" from types import SimpleNamespace @@ -104,8 +111,15 @@ def __init__(self, spec: dict[str, Any], model_dir: str): # This worker decodes the call's videos; the engine runs its pixels. register_vllm_video_loader(cfg) # The engine's own args, with the weight-free snapshot Dynamo fetched. + # Sleep mode is the engine's memory management, and vLLM's CPU platform + # refuses it. args = AsyncEngineArgs( - **{**spec["engine_args"], "model": model_dir, "tokenizer": model_dir} + **{ + **spec["engine_args"], + "model": model_dir, + "tokenizer": model_dir, + "enable_sleep_mode": False, + } ) self.model_config = args.create_model_config() vllm_config = VllmConfig( diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index d0284818adc..a34e80b8369 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -54,6 +54,7 @@ RUNTIME_DATA_KEY, GymCallPlan, ) +from nemo_rl.models.generation.interfaces import SamplerReplayConfig # noqa: E402 from nemo_rl.models.generation.rollout_result import ( # noqa: E402 rollout_result_from_wire, rollout_result_to_wire, @@ -413,6 +414,8 @@ def test_engines_publish_what_the_frontends_chat_engine_renders_with() -> None: "http_server_serving_chat_kwargs": serving_chat_kwargs, "reasoning_parser_plugin": "/plugins/reasoning.py", } + # MasterConfig validates this block into a pydantic model. + host.cfg["sampler_replay"] = SamplerReplayConfig(enabled=True) # Dynamo carries runtime_data as JSON. spec = json.loads(json.dumps(vllm_adapter.chat_runtime_data(host)))[ @@ -427,7 +430,7 @@ def test_engines_publish_what_the_frontends_chat_engine_renders_with() -> None: ), field assert published.skip_tokenizer_init is False # All of policy.generation, which the native server's chat helpers read. - assert spec["generation"] == host.cfg + assert spec["generation"] == {**host.cfg, "sampler_replay": {"enabled": True}} # --- NeMo-Gym calls, after the frontend's chat engine rendered them ----------- From c6a4c66bceb4ed9994d92d4ad9557a5d8d982d9e Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 9 Nov 2026 10:49:36 -0800 Subject: [PATCH 53/82] fix(dynamo): keep the generation worker's logging under Dynamo's Worker C59. Dynamo's Worker.run() starts with configure_dynamo_logging(), which replaces the whole actor's Python logging: it removes NeMo RL's root handler, sets the root level from DYN_LOG (so NRL_LOG_LEVEL=DEBUG records are dropped), and swaps vLLM's handlers for an always-ANSI formatter. start_dynamo_endpoint now replaces that module global with a no-op before the Worker starts. A test checks the replacement keeps the root handlers, and another fails if Worker.run() stops calling the setup by that name, so a Dynamo bump cannot silently undo it. ai-dynamo/dynamo#18447 asks for a WorkerConfig opt-out that replaces this. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../models/generation/dynamo/engine_adapter.py | 11 +++++++++++ .../generation/test_dynamo_vllm_adapter.py | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/nemo_rl/models/generation/dynamo/engine_adapter.py b/nemo_rl/models/generation/dynamo/engine_adapter.py index 9d21a0383fc..40ae920be17 100644 --- a/nemo_rl/models/generation/dynamo/engine_adapter.py +++ b/nemo_rl/models/generation/dynamo/engine_adapter.py @@ -41,6 +41,7 @@ Worker, WorkerConfig, ) +from dynamo.common.backend import worker as dynamo_worker from dynamo.common.backend.publisher import KvEventSource from dynamo.llm.exceptions import InvalidArgument @@ -71,6 +72,14 @@ class ServedRollout: usage: Optional[dict[str, Any]] +def _keep_host_logging(*args: object, **kwargs: object) -> None: + """Replaces Dynamo's ``configure_dynamo_logging`` (ai-dynamo/dynamo#18447). + + ``Worker.run()`` calls it first, and it would replace this process's Python + logging: NeMo RL's root handler and level, and vLLM's handlers. + """ + + def _annotation_value(request: GenerateRequest, prefix: str) -> Optional[str]: """The value of the first ```` annotation, or None.""" for annotation in request.get("annotations") or (): @@ -190,6 +199,8 @@ def start_dynamo_endpoint( """ # The Rust runtime reads these when the first Worker is constructed. os.environ.update(engine_env) + # The generation worker keeps its own logging until WorkerConfig can opt out. + dynamo_worker.configure_dynamo_logging = _keep_host_logging config = WorkerConfig( namespace=namespace, component=DYNAMO_COMPONENT, diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index bb52d519ff9..04ba37a7c77 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -21,6 +21,7 @@ import asyncio import importlib.util import json +import logging from pathlib import Path from types import SimpleNamespace @@ -244,6 +245,12 @@ async def run(self): return None monkeypatch.setattr(engine_adapter, "Worker", _Worker) + # Registered so the engine's replacement of Dynamo's logging setup is undone. + monkeypatch.setattr( + engine_adapter.dynamo_worker, + "configure_dynamo_logging", + engine_adapter.dynamo_worker.configure_dynamo_logging, + ) env = { "DYN_DISCOVERY_BACKEND": "etcd", "DYN_REQUEST_PLANE": "tcp", @@ -278,6 +285,17 @@ def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: ) # NeMo RL's rollouts go to /v1/completions; no chat surface is registered. assert config.endpoint_types == "completions" + # Worker.run() starts with Dynamo's logging setup, which keeps the worker's. + handlers = logging.getLogger().handlers[:] + engine_adapter.dynamo_worker.configure_dynamo_logging() + assert logging.getLogger().handlers == handlers + + +def test_worker_run_still_calls_the_logging_setup_the_engine_replaces() -> None: + # Otherwise a Dynamo bump would silently take over the worker's logging again. + run = engine_adapter.Worker.run + assert run.__globals__ is vars(engine_adapter.dynamo_worker) + assert "configure_dynamo_logging" in run.__code__.co_names def test_requests_other_than_rollouts_are_refused() -> None: From aa4493ec186bdcbb89041429c27c1817cc04dfef Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 9 Nov 2026 13:48:35 -0800 Subject: [PATCH 54/82] test(dynamo): run Gym turn recovery behind Dynamo Adds the turn-recovery test to the Dynamo L1 overlays. Turn recovery never calls the generation worker during a save, so only the test needed changes: - Its policy proxy read ng_capture from the top of the request body, and Gym's Dynamo dialect carries it in the nemo_rl:gym: annotation. Without the fallback, the held call was logged with no rollout ID and the snapshot selector never matched. - Phase one is killed with SIGKILL. Behind Dynamo, NeMo RL runs etcd, NATS and the frontend in their own sessions and stops them only on a clean exit, so they outlived phase one and failed the overlay's leak check. stop_phase1 now stops the processes carrying phase one's DYN_NAMESPACE. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/functional/L1_Functional_Tests_Dynamo.sh | 2 ++ tests/functional/_gym_policy_proxy.py | 7 +++++++ .../grpo_async_gym_single_controller_turn_recovery.sh | 10 ++++++++++ 3 files changed, 19 insertions(+) diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index f5349069023..49b43d2f6c2 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -44,6 +44,8 @@ run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller_context_compaction.sh ++policy.generation.dynamo=true # Router replay with deferred routes: the engine stages each captured call's routed experts. run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true +# Turn recovery: Gym checkpoint + TQ snapshot, kill, restore, re-sent calls. +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller_turn_recovery.sh ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/_gym_policy_proxy.py b/tests/functional/_gym_policy_proxy.py index f8c1958bcc3..518580f55c2 100644 --- a/tests/functional/_gym_policy_proxy.py +++ b/tests/functional/_gym_policy_proxy.py @@ -46,6 +46,7 @@ from typing import Any, Optional from nemo_rl.distributed.virtual_cluster import _get_node_ip_local +from nemo_rl.models.generation.gym_call import GymCallExtras from tests.functional._single_controller_hook_utils import append_event _SKIPPED_REQUEST_HEADERS = frozenset({"host", "content-length", "connection"}) @@ -84,6 +85,12 @@ def _tool_results(body: dict[str, Any]) -> dict[str, Any]: def _rollout_id(body: dict[str, Any]) -> Optional[str]: capture = body.get("ng_capture") + if capture is None: + # Gym's Dynamo dialect carries it in the nemo_rl:gym: annotation. + extras = GymCallExtras.from_annotations( + (body.get("nvext") or {}).get("annotations") + ) + capture = extras.ng_capture if extras is not None else None if isinstance(capture, dict): # Gym names it capture_key; Gyms before NVIDIA-NeMo/Gym#4128 sent rollout_id. rollout_id = capture.get("capture_key", capture.get("rollout_id")) diff --git a/tests/functional/grpo_async_gym_single_controller_turn_recovery.sh b/tests/functional/grpo_async_gym_single_controller_turn_recovery.sh index 4d3060287f7..9d22b3c73d5 100755 --- a/tests/functional/grpo_async_gym_single_controller_turn_recovery.sh +++ b/tests/functional/grpo_async_gym_single_controller_turn_recovery.sh @@ -98,6 +98,16 @@ stop_phase1() { fi sleep 1 done + # Behind Dynamo, NeMo RL runs etcd, NATS and the frontend in their own + # sessions and stops them only on a clean exit, so the kill orphans them. + # Each carries phase one's DYN_NAMESPACE in its environment. + local namespace pid + namespace=$(grep -aoE "'DYN_NAMESPACE': '[^']+'" "$BASE_RUN_LOG" | head -1 | cut -d"'" -f4 || true) + if [[ -n "$namespace" ]]; then + for pid in $(grep -lsxz "DYN_NAMESPACE=$namespace" /proc/[0-9]*/environ | cut -d/ -f3 || true); do + kill -KILL "$pid" 2>/dev/null || true + done + fi } cleanup() { From 59179cd14ee4be3d08b959a50d6e9a548cf1af16 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 9 Nov 2026 14:57:23 -0800 Subject: [PATCH 55/82] fix(dynamo): render calls without capture where NeMo-Gym is not installed RenderPool.render imported Gym's CaptureAdmission on every call. Only a captured call needs it, and CI's vLLM lane has no nemo_gym, so the two render-pool tests that send plain calls failed there with ModuleNotFoundError. The import now happens only when the call carries ng_capture, as the adapter's does. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../models/generation/dynamo/vllm/render_pool.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index 31c89b8ec3e..a30ec235a91 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -194,8 +194,6 @@ def _parser(self, chat: Any) -> Any: ) def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall: - from nemo_gym.token_id_capture.staging.records import CaptureAdmission - from nemo_rl.models.generation.vllm.chat_server import ( NeMoRLChatCompletionRequest, NeMoRLMeasureChatRequest, @@ -211,11 +209,12 @@ def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall if chat.max_completion_tokens is not None else chat.max_tokens ) - admission = ( - None - if chat.ng_capture is None - else CaptureAdmission.model_validate(chat.ng_capture) - ) + admission = None + if chat.ng_capture is not None: + # Deferred: only captured calls need it, and capture already imports nemo_gym. + from nemo_gym.token_id_capture.staging.records import CaptureAdmission + + admission = CaptureAdmission.model_validate(chat.ng_capture) rendered = self.loop.run_until_complete( self.online_renderer.render_for_frontend(chat, admission=admission) ) From a2c74e630bd9faa1ac5c5b9ba79c14528b80609c Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 10 Nov 2026 09:35:39 -0800 Subject: [PATCH 56/82] fix(dynamo): fail Gym calls with non-finite log-probs with 500, as natively An engine NaN cascade left one sequence's generation log-probs NaN to the token cap (HSG job 8814903, 22 calls). The worker sent them in engine_data, Dynamo's strict JSON turned each into null, and the frontend's GymCallResult.model_validate raised a ValueError that Dynamo answered with 400. Gym retries neither a 400 nor a model 400 in the agent, so the rollout's /run failed before /verify and its GenRM cohort waited out its 1800 s timeout. The native server cannot send NaN either: Starlette's JSONResponse uses allow_nan=False, so the same call fails there with 500, which Gym retries with a fresh generation. The worker now refuses an uncaptured call whose sampled-token log-probs are not all finite, naming the first bad position; the chat engine already answers an engine's error with 500. Captured calls stage in-process without crossing JSON and are unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/models/generation/dynamo/vllm/adapter.py | 9 +++++++++ .../models/generation/test_dynamo_vllm_adapter.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index 55c7d5ee3ad..b552e708e64 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -34,6 +34,7 @@ import dataclasses import json import logging +import math import os import pickle from collections.abc import AsyncGenerator @@ -468,6 +469,14 @@ def _call_result(self, final_output: Any) -> GymCallResult: generation_log_probs = None if completion.logprobs is not None: _, generation_log_probs = chat_generation_tokens(completion) + for index, log_prob in enumerate(generation_log_probs): + if not math.isfinite(log_prob): + # Strict JSON has no NaN or inf (Dynamo sends null). The native + # server's JSONResponse fails this call with 500, which Gym retries. + raise RuntimeError( + f"Generation produced a non-finite log-prob ({log_prob}) at " + f"token {index} of {len(generation_log_probs)}" + ) return GymCallResult( prompt_token_ids=list(final_output.prompt_token_ids), generation_token_ids=list(completion.token_ids), diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index 2071c166133..8958b63567c 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -571,6 +571,20 @@ def test_a_gym_call_without_routes_fails_under_router_replay() -> None: _collect(engine, _gym_call([1, 3, 4])) +def test_an_uncaptured_gym_call_with_non_finite_logprobs_fails_as_natively() -> None: + completion = _completion([10, 11], "length") + completion.logprobs[1] = {11: _logprob(float("nan"))} + llm = _FakeLLM([_output([1, 3, 4], completion)]) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) + + # Dynamo would hand the frontend a null log-prob. Fail the call instead: the + # chat engine answers an engine's error with 500, as natively, and Gym retries. + with pytest.raises( + InvalidArgument, match=r"non-finite log-prob \(nan\) at token 1" + ): + _collect(engine, _gym_call([1, 3, 4])) + + def test_a_gym_call_without_the_chat_engines_plan_is_refused() -> None: llm = _FakeLLM() engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) From a9ff06bf922e6f207f01c61ebab44ffd0bcfc22f Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 10 Nov 2026 15:24:23 -0800 Subject: [PATCH 57/82] fix(dynamo): answer an engine's InvalidArgument with 500 on the Gym chat path a2c74e630b fails a Gym call with non-finite log-probs by raising a RuntimeError in the engine, which engine_adapter.generate wraps as InvalidArgument. Dynamo 1.5.1's response stream raises any engine error as a ValueError, which _call_engine answers with 500. From 1.7, a classified InvalidRequest is raised as InvalidArgument instead, which is a DynamoException and not a ValueError. _call_engine let it through, and Dynamo answered 400, which Gym does not retry (HSG job 8836297). Catch both types. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../models/generation/dynamo/vllm/chat_engine.py | 11 ++++++----- .../models/generation/test_dynamo_chat_engine.py | 13 +++++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 5a4fe7fe9d1..cea86e0d551 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -43,7 +43,7 @@ from typing import Any, AsyncGenerator, Optional, TypedDict from dynamo.llm import PythonAsyncEngine -from dynamo.llm.exceptions import HttpError +from dynamo.llm.exceptions import HttpError, InvalidArgument from nemo_rl.models.generation.dynamo.managed_runtime import ( CAPTURE_FILE_ENV, @@ -392,10 +392,11 @@ async def _call_engine( data = item.data() if data and data.get("engine_data"): engine_data = data["engine_data"] - except ValueError as error: - # Dynamo raises an engine's error as a ValueError, which it would - # answer with 400. The engine sends every status it means as a - # refusal, so anything raised is an engine failure: 500, as natively. + except (ValueError, InvalidArgument) as error: + # Dynamo raises an engine's error as a ValueError, or from 1.7 the + # engine adapter's InvalidArgument as itself (not a ValueError), and + # would answer either with 400. The engine sends every status it means + # as a refusal, so anything raised is an engine failure: 500, as natively. raise HttpError(500, str(error)) from error if engine_data is None: raise HttpError(500, "the engine returned no result for this call") diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index 53e17a17cba..3b4ce87ae61 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -28,7 +28,7 @@ pytest.importorskip("dynamo._core") pytest.importorskip("vllm") -from dynamo.llm.exceptions import HttpError # noqa: E402 +from dynamo.llm.exceptions import HttpError, InvalidArgument # noqa: E402 from nemo_rl.models.generation.dynamo.vllm import chat_engine # noqa: E402 from nemo_rl.models.generation.dynamo.vllm.render_pool import ( # noqa: E402 @@ -131,6 +131,8 @@ async def generate(self, request, context=None): self.requests.append(request) async def stream(): + if isinstance(self.error, Exception): + raise self.error if self.error is not None: # Dynamo's binding raises an engine's error frame as a ValueError. raise ValueError(self.error) @@ -412,6 +414,13 @@ def test_a_call_refused_before_it_runs_acknowledges_its_capture_to_gym( 500, "engine died", ), + # From Dynamo 1.7, the engine adapter's InvalidArgument (a RuntimeError it + # wrapped, e.g. non-finite log-probs) is raised as itself, not a ValueError. + ( + _Router(error=InvalidArgument("RuntimeError: non-finite log-prob (nan)")), + 500, + "non-finite log-prob", + ), # Refused before it ran, with the status the native server answers. ( _Router( @@ -427,7 +436,7 @@ def test_a_call_refused_before_it_runs_acknowledges_its_capture_to_gym( "queue full", ), ], - ids=["error", "refusal"], + ids=["error", "engine-invalid-argument", "refusal"], ) def test_an_engine_that_does_not_serve_the_call_fails_it(router, status, message): with pytest.raises(HttpError, match=message) as failed: From e47ce35d6055fe2806d13bb49e11e765960247e5 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 10 Nov 2026 18:34:44 -0800 Subject: [PATCH 58/82] feat(dynamo): cut running NeMo-Gym calls at a checkpoint With prefix recovery (#4506), NeMo-Gym posts the calls it has in flight to /ng-control/v1/generation-cut beside the base URL they went to. Behind Dynamo that URL is the frontend, which has no such route and does not hold the calls. Every cut got a 404, and its calls regenerated after a restore. The frontend process now serves the route on a port of its own and sends the cut to every engine over Dynamo's request plane: - frontend.py keeps the frontend's DistributedRuntime, the way it already wraps EntrypointArgs, and starts the route (dynamo/vllm/generation_cut.py). The route is the native one (register_generation_cut_route), so the path, the bearer check and the validation are shared. - Each engine runs its own cut and answers with the calls it cut (a nemo_rl:generation_cut handler in the adapter). A call no engine cut gets Gym's failure ack and regenerates, as before. - setup_token_capture publishes the cut's bearer token and timeout to the frontend with the capture settings. Until then the route answers 404, as the native route does. - Gym is told where to send cuts (generation_cut_control_url) only when the base URLs do not take them, which is under Dynamo. The port is needed because a Dynamo frontend route can only be a synchronous GET (ai-dynamo/dynamo#18486). Checked against Dynamo 1.5.1 with the real frontend launcher and two stub engines: 404 before capture is set up, 401 for a wrong bearer, and a receipt Gym validates in which each call is acked by its engine. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- .../single_controller_utils/setup.py | 10 + nemo_rl/environments/nemo_gym.py | 23 +++ .../generation/dynamo/managed_runtime.py | 47 ++++- .../models/generation/dynamo/vllm/adapter.py | 41 +++- .../generation/dynamo/vllm/chat_engine.py | 9 +- .../models/generation/dynamo/vllm/frontend.py | 15 +- .../generation/dynamo/vllm/generation_cut.py | 192 ++++++++++++++++++ .../generation/vllm/generation_cut_route.py | 25 ++- .../models/generation/vllm/vllm_generation.py | 22 +- pyrefly.toml | 1 + tests/unit/environments/test_nemo_gym.py | 2 + .../unit/environments/test_nemo_gym_utils.py | 8 +- .../generation/test_dynamo_chat_engine.py | 3 + .../generation/test_dynamo_generation_cut.py | 169 +++++++++++++++ .../generation/test_dynamo_managed_runtime.py | 11 +- .../test_dynamo_vllm_adapter_capture.py | 59 ++++++ tests/unit/single_controller/test_setup.py | 7 + 17 files changed, 620 insertions(+), 24 deletions(-) create mode 100644 nemo_rl/models/generation/dynamo/vllm/generation_cut.py create mode 100644 tests/unit/models/generation/test_dynamo_generation_cut.py diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index 3119c29d070..023e70bd473 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -800,6 +800,7 @@ def _spinup_gym( base_urls: list[str], tokenizer: PreTrainedTokenizerBase, policy_api_key: Optional[str], + generation_cut_control_urls: Optional[list[str]], ) -> tuple[NemoGymShardSet, float]: """Spin up the NeMo-Gym shard set against the reserved vLLM URLs. @@ -810,6 +811,9 @@ def _spinup_gym( call. See NemoGym.set_tokenizer. policy_api_key: The bearer token the vLLM server requires on its OpenAI routes (VllmGeneration.http_server_api_key), or None when it checks none. + generation_cut_control_urls: Where Gym sends checkpoint generation cuts, + one per base URL (VllmGeneration.generation_cut_control_urls), or + None when each base URL takes its own. Returns: A tuple of (NeMo-Gym shard set, wall time spent in this call). @@ -832,6 +836,7 @@ def _spinup_gym( generation_prefix_cuts_enabled=( master_config.rollout_recovery.generation_prefix_cuts_enabled ), + generation_cut_control_urls=generation_cut_control_urls, # Capture transport config includes the generation request's candidate count. token_capture=( { @@ -1988,6 +1993,11 @@ def _build_trainer_then_megatron_generation() -> tuple[ policy_api_key=( generation.http_server_api_key if generation is not None else None ), + generation_cut_control_urls=( + generation.generation_cut_control_urls + if generation is not None + else None + ), ) if megatron_backend and colocated: diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 569458bd912..e7e0d7d4b2d 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -2964,6 +2964,7 @@ def build_nemo_gym_config( use_fastokens: bool, turn_checkpointing_enabled: bool, generation_prefix_cuts_enabled: bool = False, + generation_cut_control_urls: Optional[list[str]] = None, token_capture: Optional[dict[str, Any]] = None, policy_api_key: Optional[str] = None, ) -> NemoGymConfig: @@ -2988,6 +2989,10 @@ def build_nemo_gym_config( generation_prefix_cuts_enabled: Let Gym cut active model calls at a checkpoint so a restore resumes them from their decoded prefix. Requires ``turn_checkpointing_enabled``. + generation_cut_control_urls: Where Gym sends those cuts, one per entry + of ``base_urls``; None when each base URL takes its own, as a + vLLM server does. A Dynamo frontend takes them on another port + (VllmGeneration.generation_cut_control_urls). policy_api_key: The bearer token the generation server requires on its OpenAI routes (VllmGeneration.http_server_api_key), or None when it checks none. @@ -3023,6 +3028,7 @@ def build_nemo_gym_config( ), turn_checkpointing_enabled=turn_checkpointing_enabled, generation_prefix_cuts_enabled=generation_prefix_cuts_enabled, + generation_cut_control_urls=generation_cut_control_urls, token_capture=token_capture, policy_api_key=policy_api_key, ) @@ -3085,6 +3091,7 @@ def _build_gym_actor_config( checkpoint_instance: GymCheckpointInstance, turn_checkpointing_enabled: bool, generation_prefix_cuts_enabled: bool, + generation_cut_control_urls: Optional[list[str]], token_capture: Optional[dict[str, Any]] = None, policy_api_key: Optional[str] = None, ) -> NemoGymConfig: @@ -3180,6 +3187,11 @@ def _build_gym_actor_config( policy_overrides["checkpoint"] = {"policy": True} if generation_prefix_cuts_enabled: policy_overrides["checkpoint"]["generation_cuts"] = True + if generation_cut_control_urls is not None: + # The base URLs do not take cuts themselves (a Dynamo frontend). + policy_overrides["generation_cut_control_url"] = list( + generation_cut_control_urls + ) return NemoGymConfig( model_name=model_name, @@ -3484,6 +3496,7 @@ def build_nemo_gym_actors( use_fastokens: bool, turn_checkpointing_enabled: bool, generation_prefix_cuts_enabled: bool = False, + generation_cut_control_urls: Optional[list[str]] = None, token_capture: Optional[dict[str, Any]] = None, policy_api_key: Optional[str] = None, pg_ready_timeout: float = DEFAULT_SHARD_PG_READY_TIMEOUT_SECONDS, @@ -3505,6 +3518,10 @@ def build_nemo_gym_actors( generation_prefix_cuts_enabled: Let Gym cut active model calls at a checkpoint so a restore resumes them from their decoded prefix. Requires ``turn_checkpointing_enabled``. + generation_cut_control_urls: Where Gym sends those cuts, one per entry + of ``base_urls``; None when each base URL takes its own, as a + vLLM server does. A Dynamo frontend takes them on another port + (VllmGeneration.generation_cut_control_urls). Returns: A :class:`NemoGymShardSet` whose actors are all running and validated. @@ -3527,6 +3544,7 @@ def build_nemo_gym_actors( use_fastokens=use_fastokens, turn_checkpointing_enabled=turn_checkpointing_enabled, generation_prefix_cuts_enabled=generation_prefix_cuts_enabled, + generation_cut_control_urls=generation_cut_control_urls, token_capture=token_capture, policy_api_key=policy_api_key, ) @@ -3541,6 +3559,7 @@ def build_nemo_gym_actors( use_fastokens=use_fastokens, turn_checkpointing_enabled=turn_checkpointing_enabled, generation_prefix_cuts_enabled=generation_prefix_cuts_enabled, + generation_cut_control_urls=generation_cut_control_urls, token_capture=token_capture, policy_api_key=policy_api_key, pg_ready_timeout=pg_ready_timeout, @@ -3558,6 +3577,7 @@ def _build_single_gym_actor( use_fastokens: bool, turn_checkpointing_enabled: bool, generation_prefix_cuts_enabled: bool, + generation_cut_control_urls: Optional[list[str]], token_capture: Optional[dict[str, Any]], policy_api_key: Optional[str] = None, ) -> NemoGymShardSet: @@ -3574,6 +3594,7 @@ def _build_single_gym_actor( ), turn_checkpointing_enabled=turn_checkpointing_enabled, generation_prefix_cuts_enabled=generation_prefix_cuts_enabled, + generation_cut_control_urls=generation_cut_control_urls, token_capture=token_capture, policy_api_key=policy_api_key, ) @@ -3624,6 +3645,7 @@ def _build_sharded_gym_actors( use_fastokens: bool, turn_checkpointing_enabled: bool, generation_prefix_cuts_enabled: bool, + generation_cut_control_urls: Optional[list[str]], token_capture: Optional[dict[str, Any]], policy_api_key: Optional[str] = None, pg_ready_timeout: float, @@ -3714,6 +3736,7 @@ def _build_sharded_gym_actors( ), turn_checkpointing_enabled=turn_checkpointing_enabled, generation_prefix_cuts_enabled=generation_prefix_cuts_enabled, + generation_cut_control_urls=generation_cut_control_urls, token_capture=token_capture, policy_api_key=policy_api_key, ) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index f966acd8ca0..ce712165232 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -121,10 +121,21 @@ def _managed_namespace() -> str: # The frontend process's environment for NeMo RL's chat engine # (dynamo/vllm/chat_engine.py): its render workers, where setup_token_capture -# writes the capture config, and the Ray cluster its TQ reads go through. +# writes the capture config, the Ray cluster its TQ reads go through, and the +# port it takes NeMo-Gym's generation cuts on (dynamo/vllm/generation_cut.py). RENDER_WORKERS_ENV = "NEMO_RL_DYNAMO_RENDER_WORKERS" CAPTURE_FILE_ENV = "NEMO_RL_DYNAMO_CAPTURE_FILE" RAY_ADDRESS_ENV = "NEMO_RL_DYNAMO_RAY_ADDRESS" +GENERATION_CUT_PORT_ENV = "NEMO_RL_DYNAMO_GENERATION_CUT_PORT" + + +def read_chat_capture() -> dict[str, Any] | None: + """What ``publish_chat_capture`` wrote for this frontend; None before it has.""" + path = os.environ[CAPTURE_FILE_ENV] + if not os.path.exists(path): + return None + with open(path, "rb") as capture_file: + return pickle.load(capture_file) class ManagedDynamoRuntime: @@ -159,6 +170,7 @@ def __init__( self._etcd_peer_port = 0 self._nats_port = 0 self._frontend_port = 0 + self._generation_cut_port = 0 self._manager_env: dict[str, str] = {} self._started = False self._etcd_process: subprocess.Popen | None = None @@ -178,6 +190,14 @@ def frontend_url(self) -> str: host = f"[{self._host}]" if ":" in self._host else self._host return f"http://{host}:{self._frontend_port}/v1" + @property + def generation_cut_control_url(self) -> str: + """Where the frontend process takes NeMo-Gym's checkpoint generation cuts.""" + if not self._started: + raise RuntimeError("Managed Dynamo runtime has not been started") + host = f"[{self._host}]" if ":" in self._host else self._host + return f"http://{host}:{self._generation_cut_port}" + @property def engine_env(self) -> dict[str, str]: """The environment each engine's Dynamo worker needs to register.""" @@ -202,15 +222,19 @@ def start(self) -> None: ) http = (DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH) ports: list[int] = [] - for low, high in (control, control, control, http): + for low, high in (control, control, control, http, control): ports.append( _get_free_port_local( low, high, max_retries=None, excluded_ports=set(ports) ) ) - self._etcd_port, self._etcd_peer_port, self._nats_port, self._frontend_port = ( - ports - ) + ( + self._etcd_port, + self._etcd_peer_port, + self._nats_port, + self._frontend_port, + self._generation_cut_port, + ) = ports self._manager_env = { "ETCD_ENDPOINTS": f"http://{self._host}:{self._etcd_port}", "NATS_SERVER": f"nats://{self._host}:{self._nats_port}", @@ -367,6 +391,7 @@ def _start_frontend(self) -> None: ) frontend_env[CAPTURE_FILE_ENV] = os.path.join(self._chat_dir, "capture.pkl") frontend_env[RAY_ADDRESS_ENV] = self._ray_address + frontend_env[GENERATION_CUT_PORT_ENV] = str(self._generation_cut_port) dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) frontend_env["VIRTUAL_ENV"] = dynamo_venv frontend_env["UV_PROJECT_ENVIRONMENT"] = dynamo_venv @@ -380,11 +405,15 @@ def publish_chat_capture( dp_cfg: dict[str, Any], staging_partition: str, capture_media: bool, + generation_cut_control_token: str | None, + generation_cut_control_timeout_s: float | None, ) -> None: - """Tell the frontend's chat engine where token-capture prefixes are read. + """Tell the frontend where token-capture prefixes are read, and who may cut. Capture is set up after the engines register, so the chat engine reads - this on its first captured call. + this on its first captured call. ``generation_cut_control_token`` is the + bearer token a generation cut must carry, None to refuse every cut; + ``generation_cut_control_timeout_s`` bounds one cut across the engines. """ if self._chat_dir is None: raise RuntimeError("Managed Dynamo runtime has not been started") @@ -398,6 +427,10 @@ def publish_chat_capture( "data_plane": dp_cfg, "staging_partition": staging_partition, "capture_media": capture_media, + "generation_cut_control_token": generation_cut_control_token, + "generation_cut_control_timeout_s": ( + generation_cut_control_timeout_s + ), }, capture_file, ) diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index 8cfa818d415..7d2ba41b15f 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -75,6 +75,11 @@ GymCallPlan, GymEngineReply, ) +from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( + CUT_PREFIXES_KEY, + GENERATION_CUT_ANNOTATION, + INVENTORY_KEY, +) from nemo_rl.models.generation.gym_call import GymCallResult from nemo_rl.models.generation.openai_server_utils import PrefixSplice from nemo_rl.models.generation.score_centering import ( @@ -205,7 +210,41 @@ def __init__( self._max_model_len = self._llm.vllm_config.model_config.max_model_len def _handlers(self) -> dict[str, RequestHandler]: - return {**super()._handlers(), GYM_CALL_ANNOTATION: self._generate_gym} + return { + **super()._handlers(), + GYM_CALL_ANNOTATION: self._generate_gym, + GENERATION_CUT_ANNOTATION: self._generate_cut, + } + + async def _generate_cut( + self, request: GenerateRequest, annotation: str, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + """Cut the calls of a checkpoint's inventory that this engine holds. + + The frontend sends every engine the whole inventory + (:mod:`~nemo_rl.models.generation.dynamo.vllm.generation_cut`), so the + answer names only the calls cut here. + """ + del annotation, context + # Deferred: Gym is optional outside framework-owned capture. + from nemo_gym._checkpoint.generation_cut import GenerationCutInventory + + inventory = GenerationCutInventory.model_validate( + (request.get("extra_args") or {})[INVENTORY_KEY] + ) + receipt = await self._host._prefix_recovery.run_cut(inventory) + yield { + "token_ids": [], + "index": 0, + "finish_reason": "stop", + "engine_data": { + CUT_PREFIXES_KEY: [ + ack.model_dump(mode="json") + for ack in receipt.prefixes + if ack.disposition == "durable_prefix" + ] + }, + } async def serve_rollout( self, diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index cea86e0d551..6faefc8881a 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -36,7 +36,6 @@ import asyncio import base64 import os -import pickle import threading import time import uuid @@ -46,9 +45,9 @@ from dynamo.llm.exceptions import HttpError, InvalidArgument from nemo_rl.models.generation.dynamo.managed_runtime import ( - CAPTURE_FILE_ENV, RAY_ADDRESS_ENV, RENDER_WORKERS_ENV, + read_chat_capture, ) from nemo_rl.models.generation.dynamo.vllm.render_pool import ( RenderedCall, @@ -188,14 +187,12 @@ def _connect() -> Any: from nemo_rl.data_plane import build_data_plane_client from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache, TQTokenSource - path = os.environ[CAPTURE_FILE_ENV] - if not os.path.exists(path): + capture = read_chat_capture() + if capture is None: raise RuntimeError( "a token-capture call reached the Dynamo frontend before token " "capture was set up for it" ) - with open(path, "rb") as capture_file: - capture = pickle.load(capture_file) if not ray.is_initialized(): ray.init(address=os.environ[RAY_ADDRESS_ENV], log_to_driver=False) client = build_data_plane_client( diff --git a/nemo_rl/models/generation/dynamo/vllm/frontend.py b/nemo_rl/models/generation/dynamo/vllm/frontend.py index 4f9ba822ba9..86e32335583 100644 --- a/nemo_rl/models/generation/dynamo/vllm/frontend.py +++ b/nemo_rl/models/generation/dynamo/vllm/frontend.py @@ -21,7 +21,9 @@ ``chat_engine_factory``, so every chat call reaches :mod:`~nemo_rl.models.generation.dynamo.vllm.chat_engine` and never Dynamo's Rust preprocessor. ``/v1/completions`` (NeMo RL's own rollouts, token ids in) -is unchanged. +is unchanged. It wraps ``DistributedRuntime`` the same way, to serve NeMo-Gym's +checkpoint generation cuts through the frontend's own runtime +(:mod:`~nemo_rl.models.generation.dynamo.vllm.generation_cut`). """ import sys @@ -31,6 +33,9 @@ def main() -> None: import dynamo.frontend.main as frontend_main from nemo_rl.models.generation.dynamo.vllm.chat_engine import ChatEngineFactory + from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( + start_generation_cut_server, + ) factory = ChatEngineFactory() entrypoint_args = frontend_main.EntrypointArgs @@ -45,7 +50,15 @@ def with_chat_engine(engine_type, **kwargs): engine_type, chat_engine_factory=factory.chat_engine_factory, **kwargs ) + distributed_runtime = frontend_main.DistributedRuntime + + def with_generation_cuts(loop, *args, **kwargs): + runtime = distributed_runtime(loop, *args, **kwargs) + start_generation_cut_server(runtime, loop) + return runtime + frontend_main.EntrypointArgs = with_chat_engine + frontend_main.DistributedRuntime = with_generation_cuts sys.argv[0] = "dynamo.frontend" frontend_main.main() diff --git a/nemo_rl/models/generation/dynamo/vllm/generation_cut.py b/nemo_rl/models/generation/dynamo/vllm/generation_cut.py new file mode 100644 index 00000000000..3c337d13e33 --- /dev/null +++ b/nemo_rl/models/generation/dynamo/vllm/generation_cut.py @@ -0,0 +1,192 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checkpoint generation cuts for NeMo-Gym calls served through Dynamo. + +At a checkpoint NeMo-Gym posts the calls it has in flight to +``/ng-control/v1/generation-cut`` beside the base URL they went to. Behind +Dynamo that URL is the frontend, and the calls run in the engines. So the +frontend process serves the route on a control port of its own and sends the +cut to every engine over Dynamo's request plane. Each engine answers for the +calls it cut, and the answers are merged into the one receipt Gym expects. + +The control port is there because a Dynamo frontend route can only be a +synchronous GET (ai-dynamo/dynamo#18486). Once routes take POST and async +handlers, this route can move onto the frontend's own port. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import threading +from collections.abc import Iterable, Sequence +from typing import Any + +from nemo_rl.models.generation.dynamo.managed_runtime import ( + DYNAMO_COMPONENT, + DYNAMO_ENDPOINT, + GENERATION_CUT_PORT_ENV, + read_chat_capture, +) + +LOGGER = logging.getLogger(__name__) + +# Marks an engine request as a generation cut; its inventory rides in extra_args. +GENERATION_CUT_ANNOTATION = "nemo_rl:generation_cut" +INVENTORY_KEY = "nemo_rl_generation_cut" +# The calls an engine cut, in its answer's engine_data. +CUT_PREFIXES_KEY = "prefixes" + + +def merge_cut_receipts(inventory: Any, engine_prefixes: Iterable[Sequence[Any]]) -> Any: + """Build Gym's receipt for ``inventory`` from what each engine cut. + + An engine acknowledges only the calls it cut. A call no engine + acknowledged gets Gym's failure ack, so it regenerates after a restore. + """ + from nemo_gym._checkpoint.generation_cut import ( + GenerationCutPrefixAck, + GenerationCutReceipt, + ) + + cut: dict[str, Any] = {} + for prefixes in engine_prefixes: + for prefix in prefixes: + ack = GenerationCutPrefixAck.model_validate(prefix) + cut.setdefault(ack.ticket_id, ack) + return GenerationCutReceipt( + checkpoint_id=inventory.checkpoint_id, + cut_id=f"frontend-{inventory.inventory_digest}", + inventory_digest=inventory.inventory_digest, + inventory=inventory, + backend_snapshot_id=f"tq-{inventory.inventory_digest}", + prefixes=tuple( + cut.get(prefix.ticket_id) or GenerationCutPrefixAck.failure(prefix) + for prefix in inventory.active_prefixes + ), + ) + + +class EngineCutFanOut: + """Cuts a checkpoint's calls on every engine: what the cut route runs.""" + + def __init__(self, runtime: Any, loop: asyncio.AbstractEventLoop) -> None: + """Open the request-plane client. Call it on ``loop``, the frontend's. + + The client is opened now, at startup, so that by the first checkpoint + it has discovered every engine. + """ + # The frontend's loop, which the runtime's request-plane client runs on. + self._loop = loop + path = f"{os.environ['DYN_NAMESPACE']}.{DYNAMO_COMPONENT}.{DYNAMO_ENDPOINT}" + # Dynamo returns a future here, not a coroutine. + self._client = asyncio.ensure_future(runtime.endpoint(path).client()) + + @property + def control_token(self) -> str | None: + """The bearer token a cut needs; None until capture is set up with cuts on.""" + capture = read_chat_capture() + return None if capture is None else capture["generation_cut_control_token"] + + async def run_cut(self, inventory: Any) -> Any: + """Cut ``inventory``'s calls. Safe to await from the control server's loop.""" + return await asyncio.wrap_future( + asyncio.run_coroutine_threadsafe( + self._cut_on_every_engine(inventory), self._loop + ) + ) + + async def _cut_on_every_engine(self, inventory: Any) -> Any: + client = await self._client + request = { + # Dynamo's request type requires the field: without it the request + # never reaches the engine and the caller is left waiting. + "token_ids": [], + "annotations": [GENERATION_CUT_ANNOTATION], + "extra_args": {INVENTORY_KEY: inventory.model_dump(mode="json")}, + } + capture = read_chat_capture() or {} + cuts = { + asyncio.ensure_future( + self._cut_on_engine(client, request, instance_id) + ): instance_id + for instance_id in client.instance_ids() + } + engine_prefixes = [] + if cuts: + done, late = await asyncio.wait( + cuts, timeout=capture.get("generation_cut_control_timeout_s") + ) + for cut in late: + cut.cancel() + if late: + LOGGER.warning( + "generation cut %s: engines %s did not answer in time; calls " + "they hold regenerate after a restore", + inventory.checkpoint_id, + sorted(cuts[cut] for cut in late), + ) + engine_prefixes = [cut.result() for cut in done] + return merge_cut_receipts(inventory, engine_prefixes) + + @staticmethod + async def _cut_on_engine( + client: Any, request: dict[str, Any], instance_id: int + ) -> Sequence[Any]: + prefixes: Sequence[Any] = () + try: + stream = await client.direct(request, instance_id) + async for item in stream: + data = item.data() + if data and data.get("engine_data"): + prefixes = data["engine_data"][CUT_PREFIXES_KEY] + # One engine's failure must cost only the calls it holds, whatever it is. + except Exception: # noqa: BLE001 + LOGGER.warning( + "generation cut failed on engine %s; calls it holds regenerate " + "after a restore", + instance_id, + exc_info=True, + ) + return prefixes + + +def start_generation_cut_server(runtime: Any, loop: asyncio.AbstractEventLoop) -> None: + """Serve Gym's generation-cut route on this frontend process's control port.""" + # Deferred: only the frontend process serves the route. + import uvicorn + from fastapi import FastAPI + + from nemo_rl.models.generation.vllm.generation_cut_route import ( + register_generation_cut_route, + ) + + fan_out = EngineCutFanOut(runtime, loop) + app = FastAPI() + register_generation_cut_route(app, lambda: fan_out) + server = uvicorn.Server( + uvicorn.Config( + app, + host="0.0.0.0", + port=int(os.environ[GENERATION_CUT_PORT_ENV]), + log_level="warning", + ) + ) + # A thread of its own, as each engine's HTTP server has: on the frontend's + # loop uvicorn would take over the frontend's signal handlers. + threading.Thread( + target=server.run, name="nrl-generation-cut-server", daemon=True + ).start() diff --git a/nemo_rl/models/generation/vllm/generation_cut_route.py b/nemo_rl/models/generation/vllm/generation_cut_route.py index f1b0f43da40..d2d936713c2 100644 --- a/nemo_rl/models/generation/vllm/generation_cut_route.py +++ b/nemo_rl/models/generation/vllm/generation_cut_route.py @@ -18,25 +18,38 @@ import secrets from collections.abc import Callable +from typing import Any, Protocol from fastapi import FastAPI, Header, HTTPException -from nemo_rl.models.generation.prefix_recovery import NoPrefixRecovery, PrefixRecovery - # Gym owns this route (nemo_gym._checkpoint.generation_cut.GENERATION_CUT_ROUTE), # but nemo_gym is an optional extra. A drifted copy would 404 every cut, which Gym # treats as "regenerate", so test_generation_cut_route_matches_gym pins it. GENERATION_CUT_ROUTE = "/ng-control/v1/generation-cut" +class GenerationCutBackend(Protocol): + """What runs a cut: a worker's prefix recovery, or the Dynamo frontend's fan-out.""" + + @property + def control_token(self) -> str | None: + """The bearer token a cut must carry; None refuses every cut.""" + ... + + async def run_cut(self, inventory: Any) -> Any: + """Cut the calls Gym's inventory names and return Gym's receipt.""" + ... + + def register_generation_cut_route( - app: FastAPI, recovery: Callable[[], PrefixRecovery | NoPrefixRecovery] + app: FastAPI, recovery: Callable[[], GenerationCutBackend] ) -> None: - """Serve Gym's generation-cut call from the worker's prefix recovery. + """Serve Gym's generation-cut call from the host's cut backend. ``recovery`` returns the worker's current ``PrefixRecovery`` or - ``NoPrefixRecovery``. It is a callable because the server starts before - ``setup_token_capture`` builds the recovery object. + ``NoPrefixRecovery``, or the Dynamo frontend's fan-out to its engines. It + is a callable because the server starts before ``setup_token_capture`` + builds the recovery object. """ @app.post(GENERATION_CUT_ROUTE) diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 645f6c7daf7..07808f28993 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -216,6 +216,7 @@ class VllmGeneration(GenerationInterface): # defaults, so shutdown() also works when __init__ failed before starting them. _dynamo_runtime: Optional["ManagedDynamoRuntime"] = None dynamo_frontend_url: Optional[str] = None + dynamo_generation_cut_url: Optional[str] = None @classmethod def validate_settings(cls, master_config: "MasterConfig") -> None: @@ -784,6 +785,17 @@ def client_base_urls(self) -> list[Optional[str]]: return [self.dynamo_frontend_url] return self.dp_openai_server_base_urls + @property + def generation_cut_control_urls(self) -> Optional[list[str]]: + """Where NeMo-Gym sends checkpoint generation cuts, one per ``client_base_urls`` entry. + + None when each base URL takes its own cuts, as every shard's server + does. The Dynamo frontend takes them on a port of its own. + """ + if self.dynamo_generation_cut_url is not None: + return [self.dynamo_generation_cut_url] + return None + def _start_dynamo_services(self) -> None: """Start the etcd, NATS and frontend this fleet's engines register with.""" from nemo_rl.models.generation.dynamo.managed_runtime import ( @@ -801,6 +813,7 @@ def _start_dynamo_services(self) -> None: runtime.start() self._dynamo_runtime = runtime self.dynamo_frontend_url = runtime.frontend_url + self.dynamo_generation_cut_url = runtime.generation_cut_control_url def _register_dynamo_engines(self) -> None: """Serve every shard's engine through Dynamo; wait until it routes to all.""" @@ -900,11 +913,18 @@ def setup_token_capture( self._token_capture_media = agreed_media_columns(ray.get(futures)) self._token_capture_setup = setup if self._dynamo_runtime is not None: - # NeMo-Gym's calls through Dynamo read their prefix in the frontend. + # NeMo-Gym's calls through Dynamo read their prefix in the frontend, + # which also takes Gym's generation cuts for the engines. self._dynamo_runtime.publish_chat_capture( dp_cfg=dp_cfg, staging_partition=staging_partition, capture_media=capture_media, + generation_cut_control_token=( + generation_cut_control_token + if generation_prefix_cuts_enabled + else None + ), + generation_cut_control_timeout_s=generation_cut_control_timeout_s, ) return self._token_capture_media diff --git a/pyrefly.toml b/pyrefly.toml index c1ead0ea59c..56f826da22f 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -269,6 +269,7 @@ project-includes = [ "nemo_rl/models/generation/dynamo/vllm/adapter.py", "nemo_rl/models/generation/dynamo/vllm/chat_engine.py", "nemo_rl/models/generation/dynamo/vllm/frontend.py", + "nemo_rl/models/generation/dynamo/vllm/generation_cut.py", "nemo_rl/models/generation/dynamo/vllm/render_pool.py", "nemo_rl/models/generation/gym_call.py", "nemo_rl/models/generation/dynamo/rollout_client.py", diff --git a/tests/unit/environments/test_nemo_gym.py b/tests/unit/environments/test_nemo_gym.py index 37194a262dc..1cea48374ad 100644 --- a/tests/unit/environments/test_nemo_gym.py +++ b/tests/unit/environments/test_nemo_gym.py @@ -2602,6 +2602,7 @@ def spin_up(handle, gym_config=None): checkpoint_instance=GymCheckpointInstance("nemo_gym", 0), turn_checkpointing_enabled=False, generation_prefix_cuts_enabled=False, + generation_cut_control_urls=None, ) NemoGym.__ray_metadata__.modified_class(cfg)._spinup() return started[-1] @@ -2662,6 +2663,7 @@ def spin_up(): checkpoint_instance=GymCheckpointInstance("nemo_gym", 0), turn_checkpointing_enabled=False, generation_prefix_cuts_enabled=False, + generation_cut_control_urls=None, ) NemoGym.__ray_metadata__.modified_class(cfg)._spinup() return parse_otel_resource_attributes(os.environ["OTEL_RESOURCE_ATTRIBUTES"]) diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index 00e6fdfad7d..21d9b23cd7b 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -1399,8 +1399,9 @@ def as_config(config): nemo_gym_mod._reject_typed_environment_servers(as_config(typed)) +@pytest.mark.parametrize("control_urls", [None, ["http://frontend:2400"]]) def test_prefix_checkpointing_enables_generation_cuts_on_policy_model( - detected_uv_dirs, + detected_uv_dirs, control_urls ): config = build_nemo_gym_config( _env_configs(), @@ -1410,6 +1411,7 @@ def test_prefix_checkpointing_enables_generation_cuts_on_policy_model( use_fastokens=False, turn_checkpointing_enabled=True, generation_prefix_cuts_enabled=True, + generation_cut_control_urls=control_urls, token_capture={ "enabled": True, "capture_dir": "/capture", @@ -1421,6 +1423,8 @@ def test_prefix_checkpointing_enables_generation_cuts_on_policy_model( "responses_api_models" ]["vllm_model"] assert model["checkpoint"] == {"policy": True, "generation_cuts": True} + # Only a backend that takes cuts elsewhere (a Dynamo frontend) names where. + assert model.get("generation_cut_control_url") == control_urls @pytest.mark.nemo_gym @@ -1437,6 +1441,7 @@ def test_prefix_checkpointing_policy_overrides_are_gym_fields(detected_uv_dirs): use_fastokens=False, turn_checkpointing_enabled=True, generation_prefix_cuts_enabled=True, + generation_cut_control_urls=["http://frontend:2400"], token_capture={ "enabled": True, "capture_dir": "/capture", @@ -1447,6 +1452,7 @@ def test_prefix_checkpointing_policy_overrides_are_gym_fields(detected_uv_dirs): model = config["initial_global_config_dict"]["policy_model"][ "responses_api_models" ]["vllm_model"] + assert "generation_cut_control_url" in model assert set(model) - set(VLLMModelConfig.model_fields) == set() checkpoint_config = VLLMModelConfig.model_fields["checkpoint"].annotation assert set(model["checkpoint"]) == {"policy", "generation_cuts"} diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index 3b4ce87ae61..fdd346dbe3a 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -452,6 +452,9 @@ def test_dynamos_frontend_still_builds_its_entrypoint_by_the_name_nemo_rl_wraps( # chat_engine_factory; if Dynamo stops reading it there, Dynamo's own # preprocessor serves chat and every NeMo-Gym call is refused. assert "EntrypointArgs" in frontend_main.async_main.__code__.co_names + # It swaps this one to serve NeMo-Gym's generation cuts from the frontend's + # runtime; if Dynamo stops reading it there, every cut gets a 404. + assert "DistributedRuntime" in frontend_main.async_main.__code__.co_names def test_only_nemo_rls_engines_are_served() -> None: diff --git a/tests/unit/models/generation/test_dynamo_generation_cut.py b/tests/unit/models/generation/test_dynamo_generation_cut.py new file mode 100644 index 00000000000..df55e9ea53f --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_generation_cut.py @@ -0,0 +1,169 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""NeMo-Gym's checkpoint generation cut through the Dynamo frontend.""" + +import asyncio + +import pytest + +pytest.importorskip("nemo_gym") + +from nemo_gym._checkpoint.generation_cut import ( # noqa: E402 + GenerationCutInventory, + GenerationCutPrefix, + GenerationCutPrefixAck, +) + +from nemo_rl.models.generation.dynamo import managed_runtime # noqa: E402 +from nemo_rl.models.generation.dynamo.config import DynamoCfg # noqa: E402 +from nemo_rl.models.generation.dynamo.managed_runtime import ( # noqa: E402 + ManagedDynamoRuntime, +) +from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( # noqa: E402 + CUT_PREFIXES_KEY, + GENERATION_CUT_ANNOTATION, + INVENTORY_KEY, + EngineCutFanOut, +) + +pytestmark = pytest.mark.nemo_gym + + +class _Item: + def __init__(self, data) -> None: + self._data = data + + def data(self): + return self._data + + +class _Client: + """Dynamo's request-plane client: each engine answers a direct request.""" + + def __init__(self, engines: dict[int, object]) -> None: + self._engines = engines + self.requests: dict[int, dict] = {} + + def instance_ids(self) -> list[int]: + return list(self._engines) + + async def direct(self, request, instance_id): + self.requests[instance_id] = request + answer = self._engines[instance_id] + if isinstance(answer, Exception): + raise answer + + async def stream(): + if answer is None: + await asyncio.Event().wait() + yield _Item({"token_ids": [], "engine_data": {CUT_PREFIXES_KEY: answer}}) + + return stream() + + +class _Runtime: + def __init__(self, client: _Client) -> None: + self._client = client + self.paths: list[str] = [] + + def endpoint(self, path: str) -> "_Runtime": + self.paths.append(path) + return self + + def client(self) -> "asyncio.Future[_Client]": + # A future, as Dynamo's binding returns: not a coroutine. + future = asyncio.get_running_loop().create_future() + future.set_result(self._client) + return future + + +def _call(name: str) -> GenerationCutPrefix: + return GenerationCutPrefix( + ticket_id=f"ticket-{name}", + rollout_id="r0", + attempt=0, + model_call_id=name, + admitted_at=1.0, + ) + + +def test_a_cut_reaches_every_engine_and_each_call_gets_its_engines_answer( + monkeypatch, tmp_path +) -> None: + held, lost = _call("held"), _call("lost") + inventory = GenerationCutInventory.build( + checkpoint_id="checkpoint-1", + server_name="policy_model", + active_prefixes=[held, lost], + ) + cut = GenerationCutPrefixAck( + **held.model_dump(), + disposition="durable_prefix", + cut_kind="active_prefix", + frozen_buffer_id="active/checkpoint-1/0", + staging_keys=("__generation_cut__/checkpoint-1/r0/held/0",), + prefix_token_count=3, + prefix_digest="0" * 64, + effective_output_limit=8, + ) + client = _Client( + { + 1: [], # holds neither call + 2: [cut.model_dump(mode="json")], + 3: RuntimeError("engine gone"), + 4: None, # never answers + } + ) + runtime = _Runtime(client) + monkeypatch.setenv("DYN_NAMESPACE", "nemo-rl-test") + monkeypatch.setenv(managed_runtime.CAPTURE_FILE_ENV, str(tmp_path / "capture.pkl")) + driver = ManagedDynamoRuntime( + dynamo_cfg=DynamoCfg(), + model_name="model", + frontend_python="python", + ray_address="10.0.0.1:6379", + ) + driver._chat_dir = str(tmp_path) + + async def run(): + fan_out = EngineCutFanOut(runtime, asyncio.get_running_loop()) + # Before capture is set up the route refuses every cut, as natively. + assert fan_out.control_token is None + driver.publish_chat_capture( + dp_cfg={"backend": "simple"}, + staging_partition="staging", + capture_media=False, + generation_cut_control_token="secret", + generation_cut_control_timeout_s=0.5, + ) + assert fan_out.control_token == "secret" + return await fan_out.run_cut(inventory) + + receipt = asyncio.run(run()) + + # Gym accepts it as the answer to its inventory: every call acked once. + receipt.validate_for(inventory) + assert {ack.model_call_id: ack.disposition for ack in receipt.prefixes} == { + "held": "durable_prefix", + "lost": "durable_failure", + } + assert runtime.paths == ["nemo-rl-test.backend.generate"] + # Every engine gets the whole inventory; token_ids is always present. + request = { + "token_ids": [], + "annotations": [GENERATION_CUT_ANNOTATION], + "extra_args": {INVENTORY_KEY: inventory.model_dump(mode="json")}, + } + assert client.requests == {1: request, 2: request, 3: request, 4: request} diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 639f25fc404..9d96422e1d9 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -293,6 +293,7 @@ def test_frontend_launches_nemo_rls_chat_engine_from_the_given_interpreter( ) -> None: runtime = _runtime(render_workers=3) runtime._frontend_port = 3000 + runtime._generation_cut_port = 2400 runtime._namespace = "nemo-rl-test" launched = [] monkeypatch.setattr( @@ -303,7 +304,11 @@ def test_frontend_launches_nemo_rls_chat_engine_from_the_given_interpreter( runtime._start_frontend() runtime.publish_chat_capture( - dp_cfg={"backend": "simple"}, staging_partition="staging", capture_media=False + dp_cfg={"backend": "simple"}, + staging_partition="staging", + capture_media=False, + generation_cut_control_token="secret", + generation_cut_control_timeout_s=60.0, ) ((command, env),) = launched @@ -315,6 +320,8 @@ def test_frontend_launches_nemo_rls_chat_engine_from_the_given_interpreter( ] assert env[managed_runtime.RENDER_WORKERS_ENV] == "3" assert env[managed_runtime.RAY_ADDRESS_ENV] == "10.0.0.1:6379" + # NeMo-Gym's checkpoint generation cuts go to a port of the frontend's own. + assert env[managed_runtime.GENERATION_CUT_PORT_ENV] == "2400" # Capture is set up after the engines register; the chat engine reads it # from the file the frontend was told about. with open(env[managed_runtime.CAPTURE_FILE_ENV], "rb") as capture_file: @@ -322,6 +329,8 @@ def test_frontend_launches_nemo_rls_chat_engine_from_the_given_interpreter( "data_plane": {"backend": "simple"}, "staging_partition": "staging", "capture_media": False, + "generation_cut_control_token": "secret", + "generation_cut_control_timeout_s": 60.0, } runtime.shutdown() diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py index 7f75c2670c4..bf64504b8c2 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py @@ -34,12 +34,19 @@ pytest.importorskip("nemo_gym.token_id_capture.staging") from dynamo.llm.exceptions import InvalidArgument # noqa: E402 +from nemo_gym._checkpoint.generation_cut import ( # noqa: E402 + GenerationCutInventory, + GenerationCutPrefix, +) from nemo_gym.token_id_capture import NG_COMMIT_COORDS_FIELD # noqa: E402 from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 from nemo_gym.token_id_capture.replay import ( # noqa: E402 replay_context, summarize_replay, ) +from nemo_gym.token_id_capture.staging.capture import ( # noqa: E402 + RolloutTokenCapture, +) from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 @@ -48,6 +55,11 @@ from nemo_rl.models.generation.dynamo.vllm import ( # noqa: E402 adapter as vllm_adapter, ) +from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( # noqa: E402 + CUT_PREFIXES_KEY, + GENERATION_CUT_ANNOTATION, + INVENTORY_KEY, +) from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 from nemo_rl.utils.routed_experts_codec import ( # noqa: E402 routed_experts_tensor_metadata, @@ -66,6 +78,11 @@ _logprob, _output, ) +from tests.unit.models.generation.test_vllm_token_capture_hosting import ( # noqa: E402 + _active_call_inventory, + _cuts, + _MemorySink, +) COORDS = {"staging_key": "rollout-1/c2"} # Turn 2 of a rollout: the staged prefix [1, 5, 6, EOS] replaces the template's @@ -380,3 +397,45 @@ def capture_request_media( (payload,) = host.token_capture.staged assert payload["prompt_token_ids"] == processed assert chunk["engine_data"]["result"] == {NG_COMMIT_COORDS_FIELD: COORDS} + + +def test_an_engine_answers_a_cut_with_the_calls_it_cut() -> None: + sink = _MemorySink() + host = _host(_FakeLLM()) + host._capture_sink = sink + host.token_capture = RolloutTokenCapture( + sink=sink, weight_version_fn=lambda: 0, adapter=VLLMCaptureAdapter() + ) + _cuts(host) + inventory, _ = _active_call_inventory(host, ("held", [12, 13])) + # The frontend sends every engine the whole inventory: one call here, + # one that another engine holds. + inventory = GenerationCutInventory.build( + checkpoint_id=inventory.checkpoint_id, + server_name=inventory.server_name, + active_prefixes=[ + *inventory.active_prefixes, + GenerationCutPrefix( + ticket_id="ticket-elsewhere", + rollout_id="r9", + attempt=0, + model_call_id="elsewhere", + admitted_at=1.0, + ), + ], + ) + + (chunk,) = _collect( + vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0), + { + "token_ids": [], + "annotations": [GENERATION_CUT_ANNOTATION], + "extra_args": {INVENTORY_KEY: inventory.model_dump(mode="json")}, + }, + ) + + (ack,) = chunk["engine_data"][CUT_PREFIXES_KEY] + assert (ack["model_call_id"], ack["disposition"]) == ("held", "durable_prefix") + assert ack["prefix_token_count"] == 2 + # The cut's rows are in TQ before the engine answers. + assert ack["staging_keys"] == sink.generation_prefix_keys diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index d147ff9f3be..653b2430cb6 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -1208,6 +1208,7 @@ def test_periodic_checkpointing_does_not_enable_gym_participants(self): ["http://generation"], MagicMock(name="tokenizer"), policy_api_key=None, + generation_cut_control_urls=None, ) assert mock_spinup.call_args.kwargs["turn_checkpointing_enabled"] is False @@ -1234,11 +1235,13 @@ def test_gym_spinup_passes_generation_prefix_cuts_explicitly( ["http://generation"], MagicMock(name="tokenizer"), policy_api_key=None, + generation_cut_control_urls=["http://generation-cuts"], ) kwargs = mock_spinup.call_args.kwargs assert kwargs["turn_checkpointing_enabled"] is True assert kwargs["generation_prefix_cuts_enabled"] is generation_prefix_cuts + assert kwargs["generation_cut_control_urls"] == ["http://generation-cuts"] # The flag is a typed keyword, not an undeclared key in the capture config. assert "generation_prefix_cuts_enabled" not in kwargs["token_capture"] @@ -2613,6 +2616,10 @@ def test_nemo_gym_wires_env_handle(self, patched_factories): use_fastokens=False, turn_checkpointing_enabled=False, generation_prefix_cuts_enabled=False, + # Where the generation backend takes Gym's checkpoint cuts. + generation_cut_control_urls=patched_factories[ + "fake_gen" + ].generation_cut_control_urls, # The vLLM server's bearer token, handed to the Gym model server. policy_api_key=patched_factories["fake_gen"].http_server_api_key, token_capture=None, From 5c7fa69ef7d5715fddf6e2458d31ad6daa383f46 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 10 Nov 2026 18:54:06 -0800 Subject: [PATCH 59/82] feat(dynamo): resume a cut NeMo-Gym call from its saved tokens After a restore, NeMo-Gym re-sends each call a checkpoint cut, with the cut in its capture admission. The native server resumes it from the tokens the cut saved (#4506). Behind Dynamo the call was served as a new one, and with the cut in its admission capture refused it. The engine now resumes it, with the native server's steps (resolve_restored_call and its helpers in vllm/chat_server.py): - A cut the engine can use replaces the prompt the frontend sent with the capture prefix plus the saved rows, and lowers max_tokens and min_tokens to what is left. A prefix that had already finished its call is answered without decoding. - A cut it cannot use (its rows are gone, or they are newer than the engine's weights) is dropped and the frontend's prompt runs, so the call regenerates, as natively. - Capture begins with the cut, so the call's one TQ row joins the saved tokens and the rest. The adapter also reports each call's resolved output budget and its outputs, which a cut needs, and aborts through the native routes' helper, which never blocks the loop behind a cut. The engine's reply names what it resumed from. The frontend's render pool puts the saved text back before vLLM's parsers run, gives the parsers the saved token ids, and counts the saved tokens as output, so Gym reads the reply the uninterrupted call would have given. The resolve is in the engine because it needs the engine's weight version and capture state. The KV router therefore scores a resumed call on the prompt without its saved tokens. Tests: a call is cut with the worker's own cut, then resumed natively and through Dynamo at four cut points and with its rows gone, and both must match the uninterrupted call (test_gym_chat_path.py); #4506's shared restore cases run through the render pool's reply builder. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 32 +++ docs/guides/dynamo-generation.md | 21 ++ .../models/generation/dynamo/vllm/adapter.py | 108 +++++++-- .../generation/dynamo/vllm/chat_engine.py | 8 +- .../generation/dynamo/vllm/render_pool.py | 66 +++++- .../generation/test_dynamo_chat_engine.py | 2 +- .../generation/test_dynamo_render_pool.py | 114 +++++++++- .../test_dynamo_vllm_adapter_capture.py | 24 ++ .../models/generation/test_gym_chat_path.py | 208 +++++++++++++++++- 9 files changed, 559 insertions(+), 24 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 0b48b7c536e..ac7021722f2 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -123,6 +123,38 @@ the fields both paths exchange with Gym. `tests/unit/models/generation/test_gym_ sends the same calls through the native routes and through this path and checks they match step for step. +### Prefix recovery + +With `rollout_recovery.target_level: prefix`, a checkpoint cuts each running +call (its decoded tokens are staged in TQ), and a restore resumes it from +those tokens. The engines hold the calls and the worker-side state +(`models/generation/prefix_recovery`), so both halves run there, with the +native server's code: + +- **The cut.** NeMo-Gym posts it beside the URL its calls go to, the + frontend. A Dynamo frontend route can only be a synchronous GET, so + `dynamo/vllm/frontend.py` also wraps `DistributedRuntime` and serves the + native route (`vllm/generation_cut_route.py`) on a second port, in the + frontend process (`dynamo/vllm/generation_cut.py`). It sends the cut to + every engine with `Client.direct`; the adapter's `nemo_rl:generation_cut` + handler runs the worker's own cut and answers with the calls it cut; and + the answers are merged into the one receipt Gym expects. A call no engine + cut gets Gym's failure ack and regenerates after a restore. +- **The resume.** The frontend renders and routes a re-sent call like any + other. The adapter then runs the steps the native server runs + (`resolve_restored_call` and its helpers in `vllm/chat_server.py`): it + reads and checks the saved rows, replaces the prompt with them, lowers the + output budget, and begins capture with the cut. A cut it cannot use (rows + gone, or newer than its weights) is dropped, and the frontend's prompt + runs. The engine's reply names what it resumed from, and the render pool + puts the saved text back before vLLM's parsers run and counts the saved + tokens as output (`prefix_recovery/restore.py`). + +The resolve is in the engine, not the frontend, because the check needs the +engine's weight version and its capture state, and a cut the engine drops then +needs no second pass through the frontend. The cost is that the KV router +scores a resumed call on the prompt without its saved tokens. + ## Driver `VllmGeneration` starts `ManagedDynamoRuntime`, which runs etcd, NATS and diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 944e5919e30..5ac208ec3f8 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -118,6 +118,27 @@ Where the behavior differs from the native endpoint: dialect moves it back to the header, so the harness sees the native error. - A call's processed media travels to the engine inside the request. +### Rollout recovery + +`rollout_recovery.target_level: turn` and `prefix` work through Dynamo with no +extra setting. With `prefix`, NeMo-Gym asks the generation backend to cut its +running calls at each checkpoint, and re-sends each cut call after a restore. +Two things differ from the native endpoint: + +- **Where the cut goes.** Gym sends the cut beside the URL its calls go to, + which is the frontend, but the calls run in the engines. The frontend process + takes the cut on a port of its own, from the same range as etcd and NATS, and + sends it to every engine over Dynamo's request plane; each engine cuts the + calls it holds. Setup tells Gym the port (`generation_cut_control_url` in + Gym's `vllm_model` config). The port is needed until a Dynamo frontend route + can take a POST + ([ai-dynamo/dynamo#18486](https://github.com/ai-dynamo/dynamo/issues/18486)). +- **What the router sees for a resumed call.** The engine replaces the prompt + the frontend sent with the call's saved tokens and generates the rest, and + the frontend puts the saved text back in front of the reply. So the KV router + scores that one call on a prompt without the saved tokens. A cut the engine + cannot use is dropped and the call regenerates, as natively. + ## Not supported Startup rejects these with `dynamo: true`: diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index 7d2ba41b15f..42209d53992 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -341,13 +341,22 @@ async def _serve_gym_call( """ # Deferred: chat_server imports vLLM's OpenAI server stack. from nemo_rl.models.generation.vllm.chat_server import ( + abort_request_capture, chat_request_from_render, + clamp_max_tokens, + classify_restored_terminal, + resolve_restored_call, + serve_restored_terminal, + set_max_tokens, ) host = self._host chat = chat_request_from_render(plan["request"]) admission = host._parse_capture_admission(plan["admission"]) - serving_admission = host._parse_capture_admission(plan["serving_admission"]) + # One object when storage and serving are one admission, as natively. + serving_admission = admission + if plan["serving_admission"] != plan["admission"]: + serving_admission = host._parse_capture_admission(plan["serving_admission"]) if admission is not None and admission.mode == "candidate": # The frontend's chat engine decides a candidate; a request # still marked one skipped that decision. @@ -355,20 +364,62 @@ async def _serve_gym_call( "a candidate admission reached the engine undecided; NeMo RL's " "chat engine must resolve it before capture" ) + prefix_token_ids = None + if admission is not None and admission.mode == "token_in": + prefix_token_ids = prompt_token_ids[: plan["prefix_len"]] + restored = None + if serving_admission is not None: + # A call a checkpoint cut resumes from the tokens the cut saved. A + # cut this engine cannot use comes back as None: the frontend's + # prompt stands, and the call regenerates. + resolved_admission, restored = await resolve_restored_call( + host, + chat, + serving_admission, + # The model's own prefix is the spliced prompt's first ids. + prompt_token_ids[: plan["prefix_len"]] + if serving_admission.mode == "token_in" + else [], + ) + if serving_admission is admission: + admission = resolved_admission + serving_admission = resolved_admission + splice_cut, multimodal = plan["splice"], plan["multimodal"] + terminal = None + if restored is not None: + # The saved rows are the exact engine prompt at the cut, so the + # frontend's splice is dropped; cuts are text-only, so no media. + splice_cut = multimodal = None + prompt_token_ids = restored.prompt_token_ids + terminal = classify_restored_terminal( + restored, prompt_token_ids, admission, max_model_len=self._max_model_len + ) + if terminal is None: + max_tokens = clamp_max_tokens( + restored.max_tokens + if restored.max_tokens is not None + else chat.max_completion_tokens + if chat.max_completion_tokens is not None + else chat.max_tokens, + len(prompt_token_ids), + self._max_model_len, + ) + if max_tokens is not None: + set_max_tokens(chat, max_tokens) splice = None - if plan["splice"] is not None: - splice = PrefixSplice(prompt_token_ids, *plan["splice"]) + if splice_cut is not None: + splice = PrefixSplice(prompt_token_ids, *splice_cut) engine_prompt: dict[str, Any] - if plan["multimodal"] is not None: + if multimodal is not None: # The processed media, in the template's coordinates. engine_prompt = { - **pickle.loads(base64.b64decode(plan["multimodal"])), + **pickle.loads(base64.b64decode(multimodal)), "prompt_token_ids": list(plan["template_token_ids"]), } else: engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) media = None - if admission is not None: + if admission is not None and restored is None: # Off-loop: a retained-media check reads staged calls. media = await asyncio.to_thread( host._capture_request_media, @@ -377,7 +428,7 @@ async def _serve_gym_call( splice=splice, verify_retained_media=plan["verify_retained_media"], ) - if splice is not None and plan["multimodal"] is not None: + if splice is not None and multimodal is not None: # Successful media capture already remapped these ranges. if media is None and (placeholders := engine_prompt.get("mm_placeholders")): engine_prompt["mm_placeholders"] = remap_multimodal_placeholders( @@ -387,16 +438,26 @@ async def _serve_gym_call( splice=splice, ) engine_prompt["prompt_token_ids"] = prompt_token_ids - prefix_token_ids = None - if admission is not None and admission.mode == "token_in": - prefix_token_ids = prompt_token_ids[: plan["prefix_len"]] host._begin_request_capture( chat, prompt_token_ids, admission=admission, prefix_token_ids=prefix_token_ids, media=media, + generation_cut=None if restored is None else restored.saved, + resumed_generation_token_ids=( + None if restored is None else restored.resumed_generation_token_ids + ), ) + if terminal is not None: + # This task also submits the call: the engine client answers it. + serve_restored_terminal( + chat, + engine_prompt, + terminal, + admission, + max_model_len=self._max_model_len, + ) captured = chat in host._capture_registry try: final_output = await self._generate(chat, engine_prompt, plan, context) @@ -425,6 +486,20 @@ async def _serve_gym_call( "output": output, "result": result.model_dump(exclude_none=True), } + if restored is not None: + # The frontend's reply puts the saved text back in front. + reply["restored"] = { + "prompt_token_ids": prompt_token_ids, + "token_ids": restored.resumed_generation_token_ids, + "terminal": None + if terminal is None + else { + "prefix_token_count": terminal.prefix_token_count, + "reason": terminal.reason, + "finish_reason": terminal.finish_reason, + "stop_reason": terminal.stop_reason, + }, + } return { "token_ids": output["token_ids"], "index": 0, @@ -435,11 +510,10 @@ async def _serve_gym_call( finally: # A no-op once the call was staged: staging pops its capture state. if admission is not None: - host._abort_request_capture( + await abort_request_capture( + host, chat, - reason=( - "client_disconnect" if context.is_stopped() else "engine_error" - ), + "client_disconnect" if context.is_stopped() else "engine_error", ) async def _generate( @@ -466,6 +540,9 @@ async def _generate( truncate_prompt_tokens=chat.truncate_prompt_tokens, ) params = chat.to_sampling_params(max_tokens, default_sampling_params) + if chat.ng_capture is not None: + # As the native request does: a cut needs the call's resolved budget. + self._host._prefix_recovery.on_sampling_params(chat, params) # The worker's own engine client, as the native server uses it, with # the queue limits it checks before generating. engine_client = self._host._http_engine_client or self._llm @@ -478,6 +555,9 @@ async def _generate( reasoning_ended=plan["reasoning_ended"], reasoning_parser_kwargs=plan["reasoning_parser_kwargs"], ): + # Recorded before asking for the next output, which releases + # vLLM's finished state for this request. + self._host._prefix_recovery.observe(chat, output) final_output = output if final_output is None: raise RuntimeError(f"No output received for request {context.id()}") diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 6faefc8881a..3647fead59a 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -52,6 +52,7 @@ from nemo_rl.models.generation.dynamo.vllm.render_pool import ( RenderedCall, RenderPool, + RestoredReply, ) from nemo_rl.models.generation.gym_call import ( GYM_RESULT_NVEXT_KEY, @@ -105,6 +106,8 @@ class GymEngineReply(TypedDict, total=False): # GymCallResult as JSON. output: dict[str, Any] result: dict[str, Any] + # Only for a call resumed from a checkpoint cut. + restored: RestoredReply refusal: GymCallRefusal @@ -343,7 +346,10 @@ async def _serve( # reply carries none. reply_request = {**engine_request, "logprobs": False, "top_logprobs": None} response = await self._pool.respond( - reply_request, prompt_token_ids, engine_data["output"] + reply_request, + prompt_token_ids, + engine_data["output"], + engine_data.get("restored"), ) (choice,) = response["choices"] return self._chunk( diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index a30ec235a91..536461ecf16 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -39,7 +39,7 @@ from concurrent.futures import ProcessPoolExecutor from concurrent.futures.process import BrokenProcessPool from dataclasses import dataclass -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, TypedDict LOGGER = logging.getLogger(__name__) @@ -47,6 +47,22 @@ _WORKER: Optional["_Worker"] = None +class RestoredReply(TypedDict): + """What the engine resumed a call from, after a checkpoint cut it. + + The engine swaps the frontend's prompt for the saved rows and generates + only the rest, so the reply needs these to read as the whole call's. + """ + + # The prompt the engine ran: the capture prefix, then the saved rows. + prompt_token_ids: list[int] + # The saved rows' generated tokens, which the reply's text and usage count. + token_ids: list[int] + # The saved prefix's finish, when it had already ended the call: + # ``_RestoredPrefixTerminal``'s fields without the prompt. + terminal: Optional[dict[str, Any]] + + @dataclass(frozen=True) class RenderedCall: """What the frontend needs from a render to splice, route and serve a call.""" @@ -266,6 +282,7 @@ def respond( request: dict[str, Any], prompt_token_ids: list[int], output: dict[str, Any], + restored: Optional[RestoredReply] = None, ) -> dict[str, Any]: from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -274,11 +291,31 @@ def respond( from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput + from nemo_rl.models.generation.prefix_recovery.restore import ( + _RestoredPrefixTerminal, + restore_reply_text, + restore_reply_usage, + wrap_parser_for_restore, + ) from nemo_rl.models.generation.vllm.chat_server import ( chat_request_from_render, ) chat = chat_request_from_render(request) + resumed_token_ids: list[int] = [] + terminal = None + if restored is not None: + # The engine resumed this call from the tokens a checkpoint cut + # saved: it ran this prompt and generated only the rest. + prompt_token_ids = restored["prompt_token_ids"] + resumed_token_ids = restored["token_ids"] + # The engine lowered its own copy's min_tokens, never this one. + chat._original_min_tokens = chat.min_tokens + if restored["terminal"] is not None: + terminal = _RestoredPrefixTerminal( + prompt_token_ids=tuple(prompt_token_ids), **restored["terminal"] + ) + chat._restored_prefix_terminal = terminal logprobs = None if output.get("logprobs") is not None: logprobs = [ @@ -309,6 +346,17 @@ def respond( finished=True, ) + if restored is not None: + # Before vLLM parses the reply, so tool calls and reasoning are + # parsed from the whole call's text. + restore_reply_text( + chat, + request_output, + prefix_token_ids=resumed_token_ids, + engine_prompt_token_ids=prompt_token_ids, + tokenizer=self.tokenizer, + ) + async def results(): yield request_output @@ -321,11 +369,14 @@ async def results(): [], self.tokenizer, RequestResponseMetadata(request_id=request_output.request_id), - parser=self._parser(chat), + parser=wrap_parser_for_restore(self._parser(chat), resumed_token_ids), ) ) if not isinstance(response, ChatCompletionResponse): raise ValueError(f"vLLM could not build the reply: {response}") + restore_reply_usage( + response, restored_token_count=len(resumed_token_ids), terminal=terminal + ) return response.model_dump(mode="json", exclude_none=True) @@ -344,10 +395,13 @@ def _render(body: dict[str, Any], measure: bool) -> RenderedCall: def _respond( - request: dict[str, Any], prompt_token_ids: list[int], output: dict[str, Any] + request: dict[str, Any], + prompt_token_ids: list[int], + output: dict[str, Any], + restored: Optional[RestoredReply], ) -> dict[str, Any]: assert _WORKER is not None - return _WORKER.respond(request, prompt_token_ids, output) + return _WORKER.respond(request, prompt_token_ids, output, restored) def _info() -> dict[str, Any]: @@ -419,8 +473,10 @@ async def respond( request: dict[str, Any], prompt_token_ids: list[int], output: dict[str, Any], + restored: Optional[RestoredReply] = None, ) -> dict[str, Any]: - return await self._submit(_respond, request, prompt_token_ids, output) + """Build the reply; ``restored`` is the engine's ``RestoredReply``, if any.""" + return await self._submit(_respond, request, prompt_token_ids, output, restored) def shutdown(self) -> None: self._executor.shutdown(wait=False, cancel_futures=True) diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index fdd346dbe3a..dedd015cccc 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -103,7 +103,7 @@ async def render(self, body: dict, *, measure: bool = False) -> RenderedCall: raise self.error return self.rendered - async def respond(self, request, prompt_token_ids, output) -> dict: + async def respond(self, request, prompt_token_ids, output, restored=None) -> dict: self.responded.append((request, prompt_token_ids, output)) return self.reply diff --git a/tests/unit/models/generation/test_dynamo_render_pool.py b/tests/unit/models/generation/test_dynamo_render_pool.py index 36e4c4087b1..f95a0e63b90 100644 --- a/tests/unit/models/generation/test_dynamo_render_pool.py +++ b/tests/unit/models/generation/test_dynamo_render_pool.py @@ -31,7 +31,22 @@ from transformers import AutoTokenizer # noqa: E402 -from nemo_rl.models.generation.dynamo.vllm.render_pool import RenderPool # noqa: E402 +from nemo_rl.models.generation.dynamo.vllm.render_pool import ( # noqa: E402 + RenderPool, + _Worker, +) +from nemo_rl.models.generation.prefix_recovery.restore import ( # noqa: E402 + _classify_restored_prefix_terminal, +) +from tests.unit.models.generation.prefix_restore_cases import ( # noqa: E402 + RESTORE_CASES, + RestoreCase, +) +from tests.unit.models.generation.test_prefix_restore_cases import ( # noqa: E402 + _engine_text, + _ids, + _tokenizer, +) MAX_MODEL_LEN = 512 SPEC = { @@ -225,3 +240,100 @@ async def serve(): pool.shutdown() assert asyncio.run(serve()) == "TorchCodecNemotronVLVideoBackend" + + +class _RecordingParser: + """vLLM's response parser: the content is the text, and the IDs are kept.""" + + def __init__(self) -> None: + self.token_ids: list[int] | None = None + + def parse( + self, model_output, request, *, enable_auto_tools, model_output_token_ids + ): + self.token_ids = list(model_output_token_ids) + return None, model_output, [] + + def count_reasoning_tokens(self, token_ids) -> int: + return 0 + + +@pytest.fixture(scope="module") +def reply_worker(tiny_qwen3_model_path) -> _Worker: + """One render worker, in this process, to build replies with.""" + return _Worker(SPEC, tiny_qwen3_model_path) + + +@pytest.mark.parametrize("style", ["metaspace", "byte_level"]) +@pytest.mark.parametrize("case", RESTORE_CASES, ids=lambda case: case.name) +def test_a_resumed_calls_reply_matches_the_uninterrupted_reply( + reply_worker, monkeypatch, case: RestoreCase, style: str +) -> None: + """The cases the native server passes in test_prefix_restore_cases.py.""" + tokenizer = _tokenizer(style) + parser = _RecordingParser() + monkeypatch.setattr(reply_worker, "tokenizer", tokenizer) + monkeypatch.setattr(reply_worker, "_parser", lambda chat: parser) + prompt = _ids(tokenizer, case.prompt) + output = _ids(tokenizer, case.output) + saved, tail = output[: case.saved], output[case.saved :] + # The engine ran the saved rows as its prompt and decoded only the tail. + terminal = _classify_restored_prefix_terminal( + prompt_token_ids=prompt + saved, + prefix_token_count=len(saved), + requested_output_tokens=len(output) + 8, + model_max_tokens=4096, + terminal_finish_reason=case.finish_reason if not tail else None, + terminal_stop_reason=case.stop_reason if not tail else None, + ) + tail_text = "" + if tail: + tail_text = _engine_text( + tokenizer, + prompt + saved, + tail, + case, + min_tokens=max(0, case.min_tokens - len(saved)), + ) + + reply = reply_worker.respond( + { + "model": "model", + "min_tokens": case.min_tokens, + "stop": list(case.stop) or None, + "include_stop_str_in_output": case.include_stop_str_in_output, + }, + # The prompt the frontend sent, which the engine swapped. + prompt, + { + "text": tail_text, + "token_ids": tail, + "finish_reason": case.finish_reason, + "stop_reason": case.stop_reason, + }, + { + "prompt_token_ids": prompt + saved, + "token_ids": saved, + "terminal": None + if terminal is None + else { + "prefix_token_count": terminal.prefix_token_count, + "reason": terminal.reason, + "finish_reason": terminal.finish_reason, + "stop_reason": terminal.stop_reason, + }, + }, + ) + + (choice,) = reply["choices"] + assert choice["message"]["content"] == _engine_text( + tokenizer, prompt, output, case, min_tokens=case.min_tokens + ) + assert choice["finish_reason"] == case.finish_reason + # Tool-call and reasoning parsers see the whole output, saved tokens first. + assert parser.token_ids == output + usage = reply["usage"] + assert (usage["prompt_tokens"], usage["completion_tokens"]) == ( + len(prompt), + len(output), + ) diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py index bf64504b8c2..288762ebb90 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py @@ -22,6 +22,7 @@ import base64 import pickle from types import SimpleNamespace +from unittest.mock import MagicMock import numpy as np import pytest @@ -60,6 +61,7 @@ GENERATION_CUT_ANNOTATION, INVENTORY_KEY, ) +from nemo_rl.models.generation.prefix_recovery import NoPrefixRecovery # noqa: E402 from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 from nemo_rl.utils.routed_experts_codec import ( # noqa: E402 routed_experts_tensor_metadata, @@ -439,3 +441,25 @@ def test_an_engine_answers_a_cut_with_the_calls_it_cut() -> None: assert ack["prefix_token_count"] == 2 # The cut's rows are in TQ before the engine answers. assert ack["staging_keys"] == sink.generation_prefix_keys + + +def test_a_captured_call_records_what_a_cut_needs() -> None: + """A checkpoint cuts only a call whose output budget and progress it knows.""" + final = _output([1, 3, 4], _completion([10, 11], "stop")) + llm = _FakeLLM([final]) + host = _capturing_host(llm) + host._prefix_recovery = MagicMock(wraps=NoPrefixRecovery()) + + _collect( + vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0), + _captured_call([1, 3, 4], TEXT), + ) + + ((_, params, _, _),) = llm.calls + ((chat, recorded_params),) = [ + call.args for call in host._prefix_recovery.on_sampling_params.call_args_list + ] + assert recorded_params is params + assert [call.args for call in host._prefix_recovery.observe.call_args_list] == [ + (chat, final) + ] diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index c171007ea4b..bfa3f7bc93f 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -45,6 +45,10 @@ import httpx # noqa: E402 from dynamo.llm.exceptions import HttpError # noqa: E402 from fastapi import FastAPI # noqa: E402 +from nemo_gym._checkpoint.generation_cut import ( # noqa: E402 + GenerationCutInventory, + GenerationCutPrefix, +) from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 from nemo_gym.token_id_capture.replay import ( # noqa: E402 replay_context, @@ -79,10 +83,19 @@ from nemo_rl.models.generation.gym_call import ( # noqa: E402 GYM_EXTRAS_ANNOTATION_PREFIX, ) +from nemo_rl.models.generation.prefix_recovery.restore import ( # noqa: E402 + _build_restored_prefix_terminal_output, + restored_terminal, +) from nemo_rl.models.generation.vllm import chat_server # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) +from tests.unit.models.generation.test_vllm_token_capture_hosting import ( # noqa: E402 + _cuts, + _MemoryPrefixSource, + _MemorySink, +) MAX_MODEL_LEN = 512 GENERATION = { @@ -226,11 +239,20 @@ def __init__(self, kit: _Kit) -> None: self.errored = False self._kit = kit self.runs: list[tuple[list[int], Any, Any]] = [] + # How many of REPLY's tokens the prompt already ends with: a call + # resumed from a checkpoint cut generates only the rest. + self.resumed = 0 async def generate(self, prompt, sampling_params, request_id, **kwargs): prompt_token_ids = list(prompt["prompt_token_ids"]) self.runs.append((prompt_token_ids, sampling_params, _media(prompt))) - reply_ids = self._kit.ids(REPLY) + terminal = restored_terminal() + if terminal is not None: + # As the worker's engine client answers a call whose saved prefix + # had already finished it: without decoding. + yield _build_restored_prefix_terminal_output(request_id, terminal) + return + reply_ids = self._kit.ids(REPLY)[self.resumed :] logprobs = None if sampling_params.logprobs is not None: decode = self._kit.tokenizer.decode @@ -250,7 +272,9 @@ async def generate(self, prompt, sampling_params, request_id, **kwargs): outputs=[ CompletionOutput( index=0, - text=REPLY, + text=self._kit.tokenizer.decode(reply_ids) + if self.resumed + else REPLY, token_ids=reply_ids, cumulative_logprob=None, logprobs=logprobs, @@ -873,3 +897,183 @@ async def both(): dynamo.reply["nvext"]["nemo_rl"]["prompt_token_count"] == native.reply["count"] ) assert dynamo.runs == [] + + +def _cut_call( + kit: _Kit, prompt_token_ids: list[int], saved: int, *, output_limit: int +) -> tuple[dict[str, Any], dict[str, Any]]: + """Cut a call that has generated ``saved`` of REPLY's tokens, as a checkpoint does. + + Returns the cut's staged rows by key, and the continuation Gym puts in the + admission of the call it re-sends after a restore. + """ + host, _, _ = _host(kit, {}, 0) + sink = _MemorySink() + host._capture_sink = sink + request = SimpleNamespace(ng_capture=_admission(mode="text"), stream=False) + VllmAsyncGenerationWorkerImpl._begin_request_capture( + host, request, prompt_token_ids + ) + state = host._capture_registry.get(request) + state.set_effective_output_limit(output_limit) + reply_ids = kit.ids(REPLY) + state.record( + reply_ids[:saved], + [-0.5] * saved, + finish_reason="stop" if saved == len(reply_ids) else None, + ) + (ack,) = ( + _cuts(host, tokenizer=kit.tokenizer) + .checkpoint_cut( + GenerationCutInventory.build( + checkpoint_id="checkpoint-1", + server_name="policy_model", + active_prefixes=[ + GenerationCutPrefix( + ticket_id="ticket-1", + rollout_id="rollout-1", + attempt=0, + model_call_id="c2", + admitted_at=1.0, + ) + ], + ) + ) + .prefixes + ) + assert ack.disposition == "durable_prefix" + (cut_key,) = ack.staging_keys + cut_record = sink.generation_prefix_records[-1][1] + return {cut_key: cut_record}, { + "source_capture_key": "rollout-1", + "source_model_call_id": "c2", + "staging_keys": [cut_key], + "prefix_token_count": ack.prefix_token_count, + "prefix_digest": cut_record.digest, + "effective_output_limit": ack.effective_output_limit, + "terminal_finish_reason": ack.terminal_finish_reason, + "terminal_stop_reason": ack.terminal_stop_reason, + } + + +def _restored_host(kit: _Kit, records: dict[str, Any], resumed: int): + """A host after a restore: fresh, with the cut's rows in TQ.""" + host, sink, failed = _host(kit, {}, 0) + host._capture_sink = _MemorySink() + _cuts( + host, + staging_source=_MemoryPrefixSource({}, records=records), + tokenizer=kit.tokenizer, + ) + host.llm.resumed = resumed + return host, sink, failed + + +def _reply_text(saved: int, kit: _Kit) -> str: + return kit.tokenizer.decode(kit.ids(REPLY)[:saved]) + + +@pytest.mark.parametrize( + ("cut_after", "rows_kept"), + [ + # The cut lands inside the reasoning, ... + ("\nchecking", True), + # ... after it ended, so the reasoning parser must see the saved part, ... + ("\n\n", True), + # ... inside the tool call's JSON, ... + ('{"name": "get_weather"', True), + # ... or after the call had finished: nothing is left to decode. + (REPLY, True), + # A cut whose rows are gone is dropped, and the call regenerates. + ("\n\n", False), + ], + ids=["in_reasoning", "after_reasoning", "in_tool_call", "finished", "rows_gone"], +) +def test_a_call_cut_at_a_checkpoint_resumes_to_the_uninterrupted_reply( + kits, cut_after: str, rows_kept: bool +) -> None: + kit = kits("qwen3") + reply_ids = kit.ids(REPLY) + # The cut saved the tokens up to the end of ``cut_after``. + saved = next( + count + for count in range(1, len(reply_ids) + 1) + if cut_after in _reply_text(count, kit) + ) + body = { + "model": kit.path, + "messages": [*HISTORY, OBSERVATION], + "tools": TOOLS, + "temperature": 0.7, + "top_p": 0.9, + "max_tokens": 64, + "logprobs": True, + "top_logprobs": 0, + "ng_capture": _admission(mode="text"), + } + # Each path in a task of its own, as each request has on a server: a + # finished prefix is marked in a context variable. + whole = asyncio.run(_native(kit, *_host(kit, {}, 0), body, measure=False)) + ((prompt, params, _),) = whole.runs + ((whole_record, _),) = whole.staged + records, continuation = _cut_call( + kit, prompt, saved, output_limit=params.max_tokens + ) + resumed = saved if rows_kept else 0 + extras = { + "ng_capture": CaptureAdmission( + capture_key="rollout-1-a1", + model_call_id="c3", + mode="text", + generation_cut=continuation, + ).model_dump(mode="json") + } + rows = records if rows_kept else {} + native = asyncio.run( + _native( + kit, *_restored_host(kit, rows, resumed), {**body, **extras}, measure=False + ) + ) + dynamo = asyncio.run( + _dynamo(kit, *_restored_host(kit, rows, resumed), {**body, **extras}, extras) + ) + + assert (native.status, dynamo.status) == (200, 200), (native.reply, dynamo.reply) + # The engine ran the saved rows as its prompt, with the budget that is left. + ((native_prompt, native_params, _),) = native.runs + ((dynamo_prompt, dynamo_params, _),) = dynamo.runs + assert native_prompt == dynamo_prompt == prompt + reply_ids[:resumed] + assert dynamo_params == native_params + if resumed < len(reply_ids): + assert native_params.max_tokens == params.max_tokens - resumed + # Capture staged one row for the call: the saved tokens, then the rest. + for outcome in (native, dynamo): + ((record, _),) = outcome.staged + assert record.token_ids_delta == whole_record.token_ids_delta + assert record.token_mask_delta == whole_record.token_mask_delta + assert ( + record.generation_log_probs_delta == whole_record.generation_log_probs_delta + ) + assert outcome.failed == [] + # Gym reads the reply the uninterrupted call gave. + whole_choice = whole.reply["choices"][0] + native_choice = native.reply["choices"][0] + (dynamo_choice,) = dynamo.reply["choices"] + for choice, message in ( + (native_choice, native_choice["message"]), + (dynamo_choice, dynamo_choice["delta"]), + ): + assert choice["finish_reason"] == whole_choice["finish_reason"] + assert message["content"] == whole_choice["message"]["content"] + assert ( + message.get("reasoning", message.get("reasoning_content")) + == whole_choice["message"]["reasoning"] + ) + assert [call["function"] for call in message["tool_calls"]] == [ + call["function"] for call in whole_choice["message"]["tool_calls"] + ] + counts = ("prompt_tokens", "completion_tokens", "total_tokens") + for usage in (native.reply["usage"], dynamo.reply["usage"]): + assert [usage[count] for count in counts] == [ + whole.reply["usage"][count] for count in counts + ] From 4002952ca22f733a7b2ee9bda7c257aa60ea9900 Mon Sep 17 00:00:00 2001 From: Terry Kong Date: Wed, 11 Nov 2026 00:01:30 -0800 Subject: [PATCH 60/82] refactor(gym): share the prompt and capture steps of a Gym chat call (sketch for #4979) A sketch for review comment C88 on #4979, not for merge. Built on 5c7fa69e. Both serving paths now share the steps of a Gym chat call after its render, in nemo_rl/models/generation/vllm/prompt_and_capture.py: - prepare_prompt -> PreparedPrompt: TQ prefix read, splice and clamp. Runs where the render runs (the worker natively, the frontend behind Dynamo). PreparedPrompt.to_json / from_json carry it to the engine. - start_capture: on the worker, in one fixed order: resolve a restore from a checkpoint cut, swap in the saved prompt, check for a finished prefix, set the budget, snapshot media, remap placeholders, begin_call, and serve a finished prefix. Native preprocess_chat and the Dynamo adapter both call it; neither keeps its own copy. - save_capture: the one TQ write, on both paths. - error_reply / abort_capture: one error -> status, abort reason and acknowledgement map for the native route, the adapter and the frontend. - ResumedCallReplyMixin: the reply fixes for a resumed call (text, parser, usage), used by the native serving chat and the render pool's. on_policy_sampling also refuses n != 1 and logprobs without top_logprobs on both paths. Dynamo checks the queue before capture begins and builds the staged message off the event loop. Signed-off-by: Terry Kong Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 9 +- .../models/generation/dynamo/vllm/adapter.py | 343 ++++----- .../generation/dynamo/vllm/chat_engine.py | 141 ++-- .../generation/dynamo/vllm/render_pool.py | 69 +- .../generation/prefix_recovery/recovery.py | 21 - .../generation/prefix_recovery/restore.py | 2 +- nemo_rl/models/generation/vllm/chat_server.py | 673 +++++------------- .../generation/vllm/prompt_and_capture.py | 624 ++++++++++++++++ nemo_rl/models/generation/vllm/utils.py | 23 +- .../generation/vllm/vllm_worker_async.py | 16 +- .../generation/test_dynamo_chat_engine.py | 26 +- .../generation/test_dynamo_vllm_adapter.py | 34 +- .../models/generation/test_gym_chat_path.py | 62 +- .../test_vllm_chat_template_wiring.py | 52 +- .../models/generation/test_vllm_generation.py | 53 +- .../test_vllm_token_capture_hosting.py | 11 +- 16 files changed, 1183 insertions(+), 976 deletions(-) create mode 100644 nemo_rl/models/generation/vllm/prompt_and_capture.py diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index ac7021722f2..69997285224 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -114,9 +114,12 @@ one event loop, so it runs in the pool, first in first out. Per call: 2. the chat engine checks on-policy sampling (`on_policy_sampling`, shared with the native route), reads a token-in prefix from TQ through a read-only client, splices it in with the model's turn-end tokens, and clamps the - output budget; -3. the final token IDs go through the KV router to one engine (see above); -4. the pool builds the reply with vLLM's own response builder and parsers. + output budget (`prepare_prompt` in `vllm/prompt_and_capture.py`, shared + with the native server); +3. the final token IDs go through the KV router to one engine, whose adapter + runs the native server's `start_capture` and `save_capture`; +4. the pool builds the reply with vLLM's own response builder and parsers, + through the same reply fixes for restored calls as the native server. `GymCallExtras` and `GymCallResult` (`models/generation/gym_call.py`) define the fields both paths exchange with Gym. `tests/unit/models/generation/test_gym_chat_path.py` diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index 42209d53992..cc9cdbc2802 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -20,11 +20,11 @@ speculative decoding are that worker's own. It serves two request kinds, both token ids in: NeMo RL's own rollouts, and -NeMo-Gym's chat calls, which NeMo RL's chat engine in the frontend rendered, -spliced and routed (``chat_engine``). For those the adapter runs the native -server's steps after the render: the media snapshot, ``begin_call``, the -generation with the SamplingParams the native server would build, and the one -TQ write. +NeMo-Gym's chat calls, whose prompt NeMo RL's chat engine in the frontend +rendered, prepared and routed (``chat_engine``). For those the adapter runs the +native server's steps after that (``prompt_and_capture``): ``start_capture``, the +generation with the SamplingParams the native server would build, and +``save_capture``'s one TQ write. """ from __future__ import annotations @@ -38,7 +38,6 @@ import os import pickle from collections.abc import AsyncGenerator -from http import HTTPStatus from typing import TYPE_CHECKING, Any, Optional import torch @@ -52,16 +51,11 @@ from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource from dynamo.llm.exceptions import InvalidArgument from pydantic_core import to_jsonable_python -from vllm.entrypoints.serve.exception_handling.error_response import ( - create_error_response, -) from vllm.entrypoints.serve.utils.api_utils import get_max_tokens -from vllm.exceptions import GracefulHTTPError, VLLMClientError, VLLMValidationError from vllm.inputs import TokensPrompt from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase -from nemo_rl.data.captured_media import MediaCaptureRejected from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.dynamo.engine_adapter import ( DynamoEngineAdapter, @@ -72,7 +66,7 @@ GYM_CALL_ANNOTATION, PLAN_KEY, RUNTIME_DATA_KEY, - GymCallPlan, + ForwardedCall, GymEngineReply, ) from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( @@ -81,14 +75,19 @@ INVENTORY_KEY, ) from nemo_rl.models.generation.gym_call import GymCallResult -from nemo_rl.models.generation.openai_server_utils import PrefixSplice from nemo_rl.models.generation.score_centering import ( extract_score_centering_samples, ) +from nemo_rl.models.generation.vllm.prompt_and_capture import ( + PreparedPrompt, + abort_request_capture, + start_capture, + abort_capture, + save_capture, +) from nemo_rl.models.generation.vllm.utils import ( chat_generation_tokens, chat_routed_experts, - remap_multimodal_placeholders, ) LOGGER = logging.getLogger(__name__) @@ -302,222 +301,129 @@ async def start(self, worker_id: int) -> EngineConfig: async def _generate_gym( self, request: GenerateRequest, annotation: str, context: Context ) -> AsyncGenerator[GenerateChunk, None]: + # Deferred: chat_server imports vLLM's OpenAI server stack. + from nemo_rl.models.generation.vllm.chat_server import ( + chat_request_from_render, + ) + del annotation - plan = (request.get("extra_args") or {}).get(PLAN_KEY) - if plan is None: + wire: Optional[ForwardedCall] = (request.get("extra_args") or {}).get(PLAN_KEY) + if wire is None: raise InvalidArgument( "NeMo-Gym calls reach NeMo RL's engines through NeMo RL's chat " f"engine in the Dynamo frontend (extra_args.{PLAN_KEY})." ) + host = self._host + plan = PreparedPrompt.from_json( + wire["prepared_prompt"], + prompt_token_ids=list(request["token_ids"]), + parse_admission=host._parse_capture_admission, + ) + chat = chat_request_from_render(wire["request"]) try: - chunk = await self._serve_gym_call( - plan, list(request["token_ids"]), context - ) - except (VLLMClientError, GracefulHTTPError) as error: - # The status vLLM's server answers it with; a native 400 for a - # validation error also acknowledges a capture call's refusal. - response = create_error_response(error).error - chunk = _refusal( - response.code, - response.message, - acknowledge=isinstance(error, VLLMValidationError), - ) - except MediaCaptureRejected as error: - chunk = _refusal(HTTPStatus.BAD_REQUEST, f"{error} ({error.code})") + chunk = await self._serve_gym_call(chat, plan, wire, context) + except BaseException as error: + failure = await abort_capture(host, chat, error) + if failure is None: + raise + message = failure.message + if failure.code != failure.status: + # The frontend forwards only a status and a message. + message = f"{message} ({failure.code})" + chunk = _refusal(failure.status, message, acknowledge=failure.acknowledge) if chunk is not None: yield chunk async def _serve_gym_call( - self, plan: GymCallPlan, prompt_token_ids: list[int], context: Context + self, chat: Any, plan: PreparedPrompt, wire: ForwardedCall, context: Context ) -> Optional[GenerateChunk]: - """One NeMo-Gym chat call, after the frontend rendered and spliced it. - - The native server's steps after the render, in its order: the media - snapshot, ``begin_call`` (which stamps the weight version), the - generation, and the one TQ write. Returns one chunk whose - ``engine_data`` carries the output the frontend builds the reply from - and the call's :class:`GymCallResult`; None for a captured call whose - client went away. + """One NeMo-Gym chat call, after the frontend prepared its prompt. + + The native server's steps after the plan, in its order: the queue + check vLLM's serving runs first, :func:`start_capture` (a restore + from a checkpoint cut, and ``begin_call``, which stamps the weight + version), the generation, and :func:`save_capture`'s one TQ write. + Returns one chunk whose ``engine_data`` carries the output the + frontend builds the reply from and the call's :class:`GymCallResult`; + None for a captured call whose client went away. """ - # Deferred: chat_server imports vLLM's OpenAI server stack. - from nemo_rl.models.generation.vllm.chat_server import ( - abort_request_capture, - chat_request_from_render, - clamp_max_tokens, - classify_restored_terminal, - resolve_restored_call, - serve_restored_terminal, - set_max_tokens, - ) - host = self._host - chat = chat_request_from_render(plan["request"]) - admission = host._parse_capture_admission(plan["admission"]) - # One object when storage and serving are one admission, as natively. - serving_admission = admission - if plan["serving_admission"] != plan["admission"]: - serving_admission = host._parse_capture_admission(plan["serving_admission"]) - if admission is not None and admission.mode == "candidate": - # The frontend's chat engine decides a candidate; a request - # still marked one skipped that decision. - raise InvalidArgument( - "a candidate admission reached the engine undecided; NeMo RL's " - "chat engine must resolve it before capture" - ) - prefix_token_ids = None - if admission is not None and admission.mode == "token_in": - prefix_token_ids = prompt_token_ids[: plan["prefix_len"]] - restored = None - if serving_admission is not None: - # A call a checkpoint cut resumes from the tokens the cut saved. A - # cut this engine cannot use comes back as None: the frontend's - # prompt stands, and the call regenerates. - resolved_admission, restored = await resolve_restored_call( - host, - chat, - serving_admission, - # The model's own prefix is the spliced prompt's first ids. - prompt_token_ids[: plan["prefix_len"]] - if serving_admission.mode == "token_in" - else [], - ) - if serving_admission is admission: - admission = resolved_admission - serving_admission = resolved_admission - splice_cut, multimodal = plan["splice"], plan["multimodal"] - terminal = None - if restored is not None: - # The saved rows are the exact engine prompt at the cut, so the - # frontend's splice is dropped; cuts are text-only, so no media. - splice_cut = multimodal = None - prompt_token_ids = restored.prompt_token_ids - terminal = classify_restored_terminal( - restored, prompt_token_ids, admission, max_model_len=self._max_model_len - ) - if terminal is None: - max_tokens = clamp_max_tokens( - restored.max_tokens - if restored.max_tokens is not None - else chat.max_completion_tokens - if chat.max_completion_tokens is not None - else chat.max_tokens, - len(prompt_token_ids), - self._max_model_len, - ) - if max_tokens is not None: - set_max_tokens(chat, max_tokens) - splice = None - if splice_cut is not None: - splice = PrefixSplice(prompt_token_ids, *splice_cut) + # The worker's own engine client, as the native server uses it, with + # the queue limits vLLM's serving checks before the render. + engine_client = host._http_engine_client or self._llm + engine_client.check_admission(1, context.id()) engine_prompt: dict[str, Any] - if multimodal is not None: + if wire["multimodal"] is not None: # The processed media, in the template's coordinates. engine_prompt = { - **pickle.loads(base64.b64decode(multimodal)), - "prompt_token_ids": list(plan["template_token_ids"]), + **pickle.loads(base64.b64decode(wire["multimodal"])), + "prompt_token_ids": list(plan.template_token_ids), } else: - engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) - media = None - if admission is not None and restored is None: - # Off-loop: a retained-media check reads staged calls. - media = await asyncio.to_thread( - host._capture_request_media, - engine_prompt, - admission=serving_admission if splice is not None else admission, - splice=splice, - verify_retained_media=plan["verify_retained_media"], - ) - if splice is not None and multimodal is not None: - # Successful media capture already remapped these ranges. - if media is None and (placeholders := engine_prompt.get("mm_placeholders")): - engine_prompt["mm_placeholders"] = remap_multimodal_placeholders( - template_token_ids=engine_prompt["prompt_token_ids"], - final_token_ids=prompt_token_ids, - mm_placeholders=placeholders, - splice=splice, - ) - engine_prompt["prompt_token_ids"] = prompt_token_ids - host._begin_request_capture( - chat, - prompt_token_ids, - admission=admission, - prefix_token_ids=prefix_token_ids, - media=media, - generation_cut=None if restored is None else restored.saved, - resumed_generation_token_ids=( - None if restored is None else restored.resumed_generation_token_ids - ), + engine_prompt = TokensPrompt(prompt_token_ids=plan.prompt_token_ids) + await start_capture( + host, chat, engine_prompt, plan, max_model_len=self._max_model_len ) - if terminal is not None: - # This task also submits the call: the engine client answers it. - serve_restored_terminal( - chat, - engine_prompt, - terminal, - admission, - max_model_len=self._max_model_len, - ) captured = chat in host._capture_registry - try: - final_output = await self._generate(chat, engine_prompt, plan, context) - if admission is not None and context.is_stopped(): - # Dynamo aborts the vLLM request on a disconnect but never - # cancels this generator: fail the capture (below), as the - # native endpoint's cancellation does, instead of staging it. - return None - completion = final_output.outputs[0] - if completion.finish_reason == "error": - # As vLLM's chat server does (_raise_if_error): never stage a failed call. - raise RuntimeError(f"Generation failed for request {context.id()}") - output: dict[str, Any] = { - "text": completion.text, - "token_ids": list(completion.token_ids), - "finish_reason": completion.finish_reason, - "stop_reason": completion.stop_reason, - } - if captured: - result = await self._stage(chat, final_output) - else: - result = self._call_result(final_output) - if chat.logprobs: - output["logprobs"] = _wire_logprobs(completion.logprobs) - reply: GymEngineReply = { - "output": output, - "result": result.model_dump(exclude_none=True), - } - if restored is not None: - # The frontend's reply puts the saved text back in front. - reply["restored"] = { - "prompt_token_ids": prompt_token_ids, - "token_ids": restored.resumed_generation_token_ids, - "terminal": None - if terminal is None - else { - "prefix_token_count": terminal.prefix_token_count, - "reason": terminal.reason, - "finish_reason": terminal.finish_reason, - "stop_reason": terminal.stop_reason, - }, - } - return { - "token_ids": output["token_ids"], - "index": 0, - "finish_reason": completion.finish_reason, - "completion_usage": _usage(final_output, len(completion.token_ids)), - "engine_data": reply, + final_output = await self._generate( + chat, engine_prompt, wire, engine_client, context + ) + if plan.admission is not None and context.is_stopped(): + # Dynamo aborts the vLLM request on a disconnect but never + # cancels this generator: fail the capture, as the native + # endpoint's cancellation does, instead of staging it. + await abort_request_capture(host, chat, "client_disconnect") + return None + completion = final_output.outputs[0] + if completion.finish_reason == "error": + # As vLLM's chat server does (_raise_if_error): never stage a failed call. + raise RuntimeError(f"Generation failed for request {context.id()}") + output: dict[str, Any] = { + "text": completion.text, + "token_ids": list(completion.token_ids), + "finish_reason": completion.finish_reason, + "stop_reason": completion.stop_reason, + } + if captured: + result = await self._stage(chat, final_output) + else: + result = self._call_result(final_output) + if chat.logprobs: + output["logprobs"] = _wire_logprobs(completion.logprobs) + reply: GymEngineReply = { + "output": output, + "result": result.model_dump(exclude_none=True), + } + if chat._restored_prompt_token_ids: + # The frontend's reply puts the saved text back in front. + terminal = chat._restored_prefix_terminal + reply["restored"] = { + "prompt_token_ids": list(chat._restored_prompt_token_ids), + "token_ids": list(chat._restored_generation_token_ids), + "terminal": None + if terminal is None + else { + "prefix_token_count": terminal.prefix_token_count, + "reason": terminal.reason, + "finish_reason": terminal.finish_reason, + "stop_reason": terminal.stop_reason, + }, } - finally: - # A no-op once the call was staged: staging pops its capture state. - if admission is not None: - await abort_request_capture( - host, - chat, - "client_disconnect" if context.is_stopped() else "engine_error", - ) + return { + "token_ids": output["token_ids"], + "index": 0, + "finish_reason": completion.finish_reason, + "completion_usage": _usage(final_output, len(completion.token_ids)), + "engine_data": reply, + } async def _generate( - self, chat: Any, engine_prompt: Any, plan: GymCallPlan, context: Context + self, + chat: Any, + engine_prompt: Any, + wire: ForwardedCall, + engine_client: Any, + context: Context, ) -> Any: """Generate as the native server does, with the SamplingParams it builds.""" model_config = self._llm.model_config @@ -543,17 +449,13 @@ async def _generate( if chat.ng_capture is not None: # As the native request does: a cut needs the call's resolved budget. self._host._prefix_recovery.on_sampling_params(chat, params) - # The worker's own engine client, as the native server uses it, with - # the queue limits it checks before generating. - engine_client = self._host._http_engine_client or self._llm - engine_client.check_admission(1, context.id()) final_output = None async for output in engine_client.generate( engine_prompt, params, context.id(), - reasoning_ended=plan["reasoning_ended"], - reasoning_parser_kwargs=plan["reasoning_parser_kwargs"], + reasoning_ended=wire["reasoning_ended"], + reasoning_parser_kwargs=wire["reasoning_parser_kwargs"], ): # Recorded before asking for the next output, which releases # vLLM's finished state for this request. @@ -608,6 +510,14 @@ async def _stage(self, chat: Any, final_output: Any) -> GymCallResult: # Deferred: only captured calls reach here, and capture already imports nemo_gym. from nemo_gym.token_id_capture import NG_COMMIT_COORDS_FIELD + # Off the loop, as natively: the sampler top-k scales with the + # completion's length and K. + content = await asyncio.to_thread(self._response_to_stage, final_output) + content = await save_capture(self._host, chat, content) + return GymCallResult(ng_commit_coords=content[NG_COMMIT_COORDS_FIELD]) + + def _response_to_stage(self, final_output: Any) -> dict[str, Any]: + """A captured call's response, with the fields the native endpoint stages.""" completion = final_output.outputs[0] message: dict[str, Any] = {"generation_token_ids": list(completion.token_ids)} if completion.logprobs is not None: @@ -623,7 +533,7 @@ async def _stage(self, chat: Any, final_output: Any) -> GymCallResult: ) = extract_score_centering_samples( completion, num_candidates=self._host._capture_top_logprobs ) - content = { + return { "choices": [ { "index": 0, @@ -632,11 +542,6 @@ async def _stage(self, chat: Any, final_output: Any) -> GymCallResult: } ] } - # The sink write is a blocking data-plane round trip. - content = await asyncio.to_thread( - self._host._finish_request_capture, chat, content - ) - return GymCallResult(ng_commit_coords=content[NG_COMMIT_COORDS_FIELD]) async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 3647fead59a..bb023442e6f 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -21,10 +21,12 @@ 1. the call's :class:`~nemo_rl.models.generation.gym_call.GymCallExtras` and on-policy sampling are checked; 2. a render-pool worker renders it (:mod:`render_pool`); -3. here, the token-capture prefix is read from TQ and spliced in, and the - output budget is clamped to the context; +3. here, the prompt is prepared as natively + (:func:`~nemo_rl.models.generation.vllm.prompt_and_capture.prepare_prompt`: the + token-capture prefix read from TQ, the splice and the clamp); 4. the final token ids go through Dynamo's KV router to one engine, whose - adapter runs the capture and generation (``adapter.DynamoVllmAdapter``); + adapter runs the capture and generation + (``adapter.DynamoVllmAdapter``); 5. a render-pool worker builds the reply with vLLM's own parsers. The engine publishes its engine args and chat settings at registration @@ -50,7 +52,6 @@ read_chat_capture, ) from nemo_rl.models.generation.dynamo.vllm.render_pool import ( - RenderedCall, RenderPool, RestoredReply, ) @@ -60,30 +61,26 @@ GymCallResult, GymRequestFields, ) -from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens # The key the engine publishes its chat settings under in runtime_data. RUNTIME_DATA_KEY = "nemo_rl_chat" -# The annotation the adapter serves NeMo-Gym calls by; the capture plan rides +# The annotation the adapter serves NeMo-Gym calls by; the ForwardedCall rides # in extra_args under PLAN_KEY. GYM_CALL_ANNOTATION = "nemo_gym:rollout" PLAN_KEY = "nemo_rl" -class GymCallPlan(TypedDict): - """What the chat engine sends the adapter in ``extra_args[PLAN_KEY]``.""" +class ForwardedCall(TypedDict): + """What the chat engine sends the adapter in ``extra_args[PLAN_KEY]``. + The call's ``PreparedPrompt.to_json()`` and what the engine needs that the + native worker has in process: the rendered request, its media, and what + vLLM's chat serving hands the engine besides the prompt. + """ + + prepared_prompt: dict[str, Any] # The rendered chat request, without messages. request: dict[str, Any] - # Capture admissions as JSON: what is stored, and how the prompt is served. - admission: Optional[dict[str, Any]] - serving_admission: Optional[dict[str, Any]] - verify_retained_media: bool - # The model's own prefix length at the start of the spliced prompt. - prefix_len: Optional[int] - template_token_ids: list[int] - # PrefixSplice's (model_cut_end, template_cut_start), when spliced. - splice: Optional[list[int]] # The render's multimodal input, pickled then base64-encoded. multimodal: Optional[str] reasoning_ended: Optional[bool] @@ -169,19 +166,15 @@ def __init__(self) -> None: # Calls run resolve_prefix on several threads at once; connect once. self._connect_lock = threading.Lock() - def resolve_prefix(self, serving_admission: dict[str, Any]) -> list[int]: + def resolve_prefix(self, serving_admission: Any) -> list[int]: """The admission's prefix, read from TQ (blocking; run off the loop).""" - from nemo_gym.token_id_capture.staging.records import CaptureAdmission - from nemo_rl.data_plane.tq_token_sink import resolve_admission_prefix if self._prefix_cache is None: with self._connect_lock: if self._prefix_cache is None: self._prefix_cache = self._connect() - return resolve_admission_prefix( - CaptureAdmission.model_validate(serving_admission), self._prefix_cache - ) + return resolve_admission_prefix(serving_admission, self._prefix_cache) @staticmethod def _connect() -> Any: @@ -231,10 +224,8 @@ def __init__( async def generate( self, request: dict[str, Any], context: Any = None ) -> AsyncGenerator[dict[str, Any], None]: - from vllm.exceptions import VLLMValidationError - - from nemo_rl.data.captured_media import MediaCaptureRejected - from nemo_rl.models.generation.vllm.utils import OffPolicySamplingError + # Deferred: prompt_and_capture imports vLLM. + from nemo_rl.models.generation.vllm.prompt_and_capture import error_reply extras = GymCallExtras.from_annotations( (request.get("nvext") or {}).get("annotations") or () @@ -247,18 +238,24 @@ async def generate( "/v1/completions.", ) try: - yield await self._serve(request, extras, context) - except (OffPolicySamplingError, MediaCaptureRejected) as error: - raise HttpError(400, str(error)) from error - except VLLMValidationError as error: - # Refused before the engine ran, so nothing was staged: as natively. - raise HttpError(400, str(error) + _capture_refusal_ack(extras)) from error + chunk = await self._serve(request, extras, context) + except HttpError: + raise + except Exception as error: + # The steps before the engine refuse a call with these, as + # natively; the frontend holds no capture state to fail. + failure = error_reply(error) + if failure is None: + raise + ack = _capture_refusal_ack(extras) if failure.acknowledge else "" + raise HttpError(failure.status, failure.message + ack) from error + yield chunk async def _serve( self, request: dict[str, Any], extras: GymCallExtras, context: Any ) -> dict[str, Any]: - # Deferred: chat_server imports vLLM's OpenAI server stack. - from nemo_rl.models.generation.vllm.chat_server import clamp_max_tokens + # Deferred: prompt_and_capture imports vLLM. + from nemo_rl.models.generation.vllm.prompt_and_capture import prepare_prompt from nemo_rl.models.generation.vllm.utils import on_policy_sampling body = _drop_none( @@ -266,8 +263,6 @@ async def _serve( ) if request.get("chat_template_args"): body["chat_template_kwargs"] = request["chat_template_args"] - if (body.get("n") or 1) != 1: - raise HttpError(400, "NeMo-Gym calls take n=1.") body.update( on_policy_sampling( temperature=body.get("temperature"), @@ -278,6 +273,10 @@ async def _serve( presence_penalty=body.get("presence_penalty"), frequency_penalty=body.get("frequency_penalty"), logit_bias=body.get("logit_bias"), + n=body.get("n"), + logprobs=body.get("logprobs"), + # From the raw request: _drop_none drops an explicit null. + top_logprobs=request.get("top_logprobs"), generation_config=self._generation, ) ) @@ -285,36 +284,26 @@ async def _serve( body.update(extras.model_dump(include=request_fields)) info = await self._pool.info() rendered = await self._pool.render(body, measure=extras.ng_measure) - prompt_token_ids, splice, prefix = await self._splice( - rendered, info["turn_end_token_ids"] + plan = await prepare_prompt( + rendered, + resolve_prefix=self._capture.resolve_prefix, + turn_end_token_ids=info["turn_end_token_ids"], + max_model_len=info["max_model_len"], + clamp=not extras.ng_measure, ) if extras.ng_measure: return self._chunk( request, GymCallResult( - prompt_token_count=len(prompt_token_ids), ng_context_measured=True + prompt_token_count=len(plan.prompt_token_ids), + ng_context_measured=True, ), message={"role": "assistant", "content": ""}, finish_reason="stop", ) - max_tokens = clamp_max_tokens( - rendered.request_max_tokens, len(prompt_token_ids), info["max_model_len"] - ) - engine_request = dict(rendered.request) - if max_tokens is not None: - engine_request["max_tokens"] = max_tokens - engine_request["max_completion_tokens"] = None - plan: GymCallPlan = { - "request": engine_request, - "admission": rendered.admission, - "serving_admission": rendered.serving_admission, - "verify_retained_media": rendered.verify_retained_media, - # The model's own prefix is the spliced prompt's first ids. - "prefix_len": None if prefix is None else len(prefix), - "template_token_ids": rendered.template_token_ids, - "splice": None - if splice is None - else [splice.model_cut_end, splice.template_cut_start], + wire: ForwardedCall = { + "prepared_prompt": plan.to_json(), + "request": rendered.request, "multimodal": None if rendered.multimodal is None else base64.b64encode(rendered.multimodal).decode(), @@ -323,8 +312,8 @@ async def _serve( } preprocessed = { "model": request["model"], - "token_ids": prompt_token_ids, - "stop_conditions": {"max_tokens": max_tokens}, + "token_ids": plan.prompt_token_ids, + "stop_conditions": {"max_tokens": plan.max_tokens}, "sampling_options": { "temperature": body["temperature"], "top_p": body["top_p"], @@ -332,7 +321,7 @@ async def _serve( "output_options": {}, "eos_token_ids": [], "annotations": [GYM_CALL_ANNOTATION], - "extra_args": {PLAN_KEY: plan}, + "extra_args": {PLAN_KEY: wire}, } engine_data = await self._call_engine(preprocessed, context) if (refusal := engine_data.get("refusal")) is not None: @@ -340,14 +329,14 @@ async def _serve( ack = _capture_refusal_ack(extras) if refusal["acknowledge"] else "" raise HttpError(refusal["status"], refusal["message"] + ack) result = GymCallResult.model_validate(engine_data["result"]) - reply_request = engine_request + reply_request = rendered.request if result.ng_commit_coords is not None: # A captured call's logprobs were staged with it; as natively, its # reply carries none. - reply_request = {**engine_request, "logprobs": False, "top_logprobs": None} + reply_request = {**reply_request, "logprobs": False, "top_logprobs": None} response = await self._pool.respond( reply_request, - prompt_token_ids, + plan.prompt_token_ids, engine_data["output"], engine_data.get("restored"), ) @@ -361,30 +350,6 @@ async def _serve( usage=response.get("usage"), ) - async def _splice( - self, rendered: RenderedCall, turn_end_token_ids: list[int] - ) -> tuple[list[int], Any, Any]: - """The prompt the engine runs: the model's own prefix, then the template's.""" - prefix: Optional[list[int]] = None - serving = rendered.serving_admission - if serving is not None and serving["mode"] == "token_in": - # A blocking data-plane read; keep it off the frontend's loop. - prefix = await asyncio.to_thread(self._capture.resolve_prefix, serving) - elif rendered.required_prefix_token_ids is not None: - prefix = rendered.required_prefix_token_ids - if prefix is None: - return rendered.template_token_ids, None, None - # The render includes the history whenever a prefix will be spliced. - assert rendered.history_token_ids is not None - splice = splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=prefix, - template_prefix_token_ids=rendered.history_token_ids, - template_token_ids=rendered.template_token_ids, - eos_token_id=turn_end_token_ids, - ) - return splice.token_ids, splice, prefix - async def _call_engine( self, preprocessed: dict[str, Any], context: Any ) -> GymEngineReply: diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index 536461ecf16..6e4a2f9e77e 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -65,16 +65,19 @@ class RestoredReply(TypedDict): @dataclass(frozen=True) class RenderedCall: - """What the frontend needs from a render to splice, route and serve a call.""" + """What the frontend needs from a render to plan, route and serve a call. + + A :class:`~nemo_rl.models.generation.vllm.prompt_and_capture.RenderedPrompt`. + """ template_token_ids: list[int] # The render up to the last assistant turn, when a prefix will be spliced. history_token_ids: Optional[list[int]] # The request's own prefix (Gym's echo) after the capture decision. required_prefix_token_ids: Optional[list[int]] - # Capture admissions as JSON: what is stored, and how the prompt is served. - admission: Optional[dict[str, Any]] - serving_admission: Optional[dict[str, Any]] + # Capture admissions: what is stored, and how the prompt is served. + admission: Any + serving_admission: Any verify_retained_media: bool # The rendered engine input's multimodal part (pickled), or None for text. multimodal: Optional[bytes] @@ -107,9 +110,6 @@ def __init__(self, spec: dict[str, Any], model_dir: str): from vllm.config import CacheConfig, LoadConfig, VllmConfig from vllm.engine.arg_utils import AsyncEngineArgs - from vllm.entrypoints.openai.chat_completion.serving import ( - OpenAIServingChat, - ) from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.renderers.registry import renderer_from_config @@ -179,7 +179,7 @@ def __init__(self, spec: dict[str, Any], model_dir: str): ], lora_modules=None, ) - self.serving_chat = OpenAIServingChat( + self.serving_chat = chat_server.FrontendServingChat( **{ **serving_chat_kwargs, "engine_client": engine_client, @@ -210,21 +210,16 @@ def _parser(self, chat: Any) -> Any: ) def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall: + from nemo_rl.models.generation.vllm.prompt_and_capture import set_max_tokens from nemo_rl.models.generation.vllm.chat_server import ( NeMoRLChatCompletionRequest, NeMoRLMeasureChatRequest, - set_max_tokens, ) request_cls = ( NeMoRLMeasureChatRequest if measure else NeMoRLChatCompletionRequest ) chat = request_cls.model_validate(body) - request_max_tokens = ( - chat.max_completion_tokens - if chat.max_completion_tokens is not None - else chat.max_tokens - ) admission = None if chat.ng_capture is not None: # Deferred: only captured calls need it, and capture already imports nemo_gym. @@ -236,8 +231,8 @@ def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall ) # The serving reads these back from the request; restore the budget # the render lowered for vLLM's pre-tokenization length check. - if request_max_tokens is not None: - set_max_tokens(chat, request_max_tokens) + if rendered.request_max_tokens is not None: + set_max_tokens(chat, rendered.request_max_tokens) parser = self._parser(chat) reasoning_ended = None reasoning_parser_kwargs = None @@ -266,13 +261,13 @@ def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall return RenderedCall( template_token_ids=list(rendered.engine_prompt["prompt_token_ids"]), history_token_ids=rendered.history_token_ids, - required_prefix_token_ids=chat.required_prefix_token_ids, - admission=_admission_json(rendered.admission), - serving_admission=_admission_json(rendered.serving_admission), + required_prefix_token_ids=rendered.required_prefix_token_ids, + admission=rendered.admission, + serving_admission=rendered.serving_admission, verify_retained_media=rendered.verify_retained_media, multimodal=multimodal, request=chat.model_dump(mode="json", exclude={"messages"}), - request_max_tokens=request_max_tokens, + request_max_tokens=rendered.request_max_tokens, reasoning_ended=reasoning_ended, reasoning_parser_kwargs=reasoning_parser_kwargs, ) @@ -293,29 +288,25 @@ def respond( from nemo_rl.models.generation.prefix_recovery.restore import ( _RestoredPrefixTerminal, - restore_reply_text, - restore_reply_usage, - wrap_parser_for_restore, ) from nemo_rl.models.generation.vllm.chat_server import ( chat_request_from_render, ) chat = chat_request_from_render(request) - resumed_token_ids: list[int] = [] - terminal = None if restored is not None: # The engine resumed this call from the tokens a checkpoint cut - # saved: it ran this prompt and generated only the rest. + # saved: it ran this prompt and generated only the rest. The + # serving chat's reply fixes read these fields, as natively. prompt_token_ids = restored["prompt_token_ids"] - resumed_token_ids = restored["token_ids"] + chat._restored_prompt_token_ids = tuple(prompt_token_ids) + chat._restored_generation_token_ids = tuple(restored["token_ids"]) # The engine lowered its own copy's min_tokens, never this one. chat._original_min_tokens = chat.min_tokens if restored["terminal"] is not None: - terminal = _RestoredPrefixTerminal( + chat._restored_prefix_terminal = _RestoredPrefixTerminal( prompt_token_ids=tuple(prompt_token_ids), **restored["terminal"] ) - chat._restored_prefix_terminal = terminal logprobs = None if output.get("logprobs") is not None: logprobs = [ @@ -346,17 +337,6 @@ def respond( finished=True, ) - if restored is not None: - # Before vLLM parses the reply, so tool calls and reasoning are - # parsed from the whole call's text. - restore_reply_text( - chat, - request_output, - prefix_token_ids=resumed_token_ids, - engine_prompt_token_ids=prompt_token_ids, - tokenizer=self.tokenizer, - ) - async def results(): yield request_output @@ -369,21 +349,14 @@ async def results(): [], self.tokenizer, RequestResponseMetadata(request_id=request_output.request_id), - parser=wrap_parser_for_restore(self._parser(chat), resumed_token_ids), + parser=self._parser(chat), ) ) if not isinstance(response, ChatCompletionResponse): raise ValueError(f"vLLM could not build the reply: {response}") - restore_reply_usage( - response, restored_token_count=len(resumed_token_ids), terminal=terminal - ) return response.model_dump(mode="json", exclude_none=True) -def _admission_json(admission: Any) -> Optional[dict[str, Any]]: - return None if admission is None else admission.model_dump(mode="json") - - def _init_worker(spec: dict[str, Any], model_dir: str) -> None: global _WORKER _WORKER = _Worker(spec, model_dir) diff --git a/nemo_rl/models/generation/prefix_recovery/recovery.py b/nemo_rl/models/generation/prefix_recovery/recovery.py index a2caee4a862..728351f41a7 100644 --- a/nemo_rl/models/generation/prefix_recovery/recovery.py +++ b/nemo_rl/models/generation/prefix_recovery/recovery.py @@ -37,7 +37,6 @@ from nemo_rl.models.generation.prefix_recovery.parts import PrefixRecoveryParts from nemo_rl.models.generation.prefix_recovery.restore import ( _generation_cut_blockers, - restore_reply_text, ) from nemo_rl.models.generation.prefix_recovery.state import ( _CompletedCaptureState, @@ -201,21 +200,6 @@ def observe(self, request: Any, request_output: Any) -> None: stop_reason=stop_reason, ) - def restore_reply_text( - self, request: Any, request_output: Any, *, tokenizer: Any - ) -> None: - """Prepend a resumed call's restored text to its final output.""" - state = self._registry.get(request) - if state is None or not state.resumed_generation_token_ids: - return - restore_reply_text( - request, - request_output, - prefix_token_ids=state.resumed_generation_token_ids, - engine_prompt_token_ids=state.prompt_token_ids, - tokenizer=tokenizer, - ) - def retire(self, staging_keys: Sequence[str]) -> None: """Queue an ended call's chunk rows for the next snapshot to delete. @@ -869,11 +853,6 @@ def on_sampling_params(self, request: Any, sampling_params: Any) -> None: def observe(self, request: Any, request_output: Any) -> None: pass - def restore_reply_text( - self, request: Any, request_output: Any, *, tokenizer: Any - ) -> None: - pass - def remember_completed( self, model_call_id: str, diff --git a/nemo_rl/models/generation/prefix_recovery/restore.py b/nemo_rl/models/generation/prefix_recovery/restore.py index c8c42635309..805d5723d51 100644 --- a/nemo_rl/models/generation/prefix_recovery/restore.py +++ b/nemo_rl/models/generation/prefix_recovery/restore.py @@ -344,7 +344,7 @@ def _full_generator_signature(method: Callable[..., Any]) -> inspect.Signature: raise RuntimeError( "vLLM's OpenAIServingChat.chat_completion_full_generator has no " f"{missing} parameter, which restored generation prefixes rewrite; " - "update NeMoRLOpenAIServingChatMixin.chat_completion_full_generator" + "update ResumedCallReplyMixin.chat_completion_full_generator" ) return signature diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index d6341484bbc..1076df5c1e4 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -60,37 +60,37 @@ from vllm.tool_parsers.abstract_tool_parser import ToolParserManager from vllm.v1.engine.async_llm import logger as vllm_async_llm_logger -from nemo_rl.data.captured_media import MediaCaptureRejected from nemo_rl.models.generation.capture_context import resolve_capture_input from nemo_rl.models.generation.gym_call import GymCallResult, GymRequestFields -from nemo_rl.models.generation.openai_server_utils import ( - splice_prefix_tokens, - turn_end_token_ids, -) +from nemo_rl.models.generation.openai_server_utils import turn_end_token_ids from nemo_rl.models.generation.prefix_recovery.restore import ( - _classify_restored_prefix_terminal, _full_generator_signature, _RestoredPrefixTerminal, - mark_restored_terminal, - plan_restore, + restore_reply_text, restore_reply_usage, wrap_parser_for_restore, ) from nemo_rl.models.generation.vllm.captured_chat_logprobs import ( CapturedChatLogprobsMixin, ) +from nemo_rl.models.generation.vllm.prompt_and_capture import ( + abort_request_capture, + start_capture, + abort_capture, + save_capture, + prepare_prompt, + set_max_tokens, +) from nemo_rl.models.generation.vllm.generation_cut_route import ( register_generation_cut_route, ) from nemo_rl.models.generation.vllm.utils import ( - OffPolicySamplingError, attach_routed_experts_to_chat_response_choices, attach_sampler_topk_logprobs_to_chat_response_choices, attach_token_information_to_chat_response_choices, model_dump_chat_response_with_dynamic_message_fields, model_stop_token_ids, on_policy_sampling, - remap_multimodal_placeholders, ) LOGGER = logging.getLogger(__name__) @@ -123,6 +123,9 @@ class NeMoRLChatCompletionRequest( default=None ) _restored_generation_token_ids: tuple[int, ...] = PrivateAttr(default=()) + # The engine prompt of a restored call: the capture prefix, then the + # saved rows; empty otherwise. + _restored_prompt_token_ids: tuple[int, ...] = PrivateAttr(default=()) # The client's min_tokens before it is reduced by a restored prefix; # replaying the prefix's stop handling needs the original window. _original_min_tokens: int | None = PrivateAttr(default=None) @@ -171,185 +174,9 @@ class NeMoRLTokenizeChatRequest( NeMoRLTokenizeRequest = Union[TokenizeCompletionRequest, NeMoRLTokenizeChatRequest] -def set_max_tokens(request: Any, max_tokens: int) -> None: - """Set the request's max output tokens. - - Mutates the request in place. Handles both max_completion_tokens (newer OpenAI API) - and max_tokens (deprecated but still supported by vLLM). A request - that set neither gets max_tokens: otherwise vLLM would fall back to - the generation config's budget, e.g. for a restored call whose - remaining budget is smaller. - """ - if request.max_completion_tokens is not None: - request.max_completion_tokens = max_tokens - else: - request.max_tokens = max_tokens - - -def clamp_max_tokens( - request_max_tokens: Optional[int], prompt_len: int, max_model_len: int -) -> Optional[int]: - """The output budget that keeps input + output within ``max_model_len``. - - Refuses a prompt that leaves no room for output, whether or not the call set - a budget, and lowers the budget only when it did. Both chat paths call it. - """ - remaining = max_model_len - prompt_len - if remaining <= 0: - # preserve the literal "context length" in this message to match Gym's overflow handling - message = ( - f"Prompt length ({prompt_len}) fills or exceeds " - f"this model's maximum context length ({max_model_len}). " - f"No room for output tokens." - ) - LOGGER.warning("Prompt exceeds max_model_len: %s", message) - raise VLLMValidationError(message, parameter="input_tokens", value=prompt_len) - if request_max_tokens is None: - return None - return min(request_max_tokens, remaining) - - -@dataclass(frozen=True) -class RestoredCall: - """A call that resumes from the tokens a checkpoint cut saved.""" - - # The verified saved rows, as capture's begin_call takes them. - saved: Any - # The engine prompt: the capture prefix, then the saved rows. - prompt_token_ids: list[int] - # The saved rows' generated tokens alone. - resumed_generation_token_ids: list[int] - # The whole output budget, as the cut call resolved it. - output_limit: int - # What is left of that budget; None keeps the request's own. - max_tokens: Optional[int] - - -async def resolve_restored_call( - host: Any, request: Any, serving_admission: Any, prefix_token_ids: list[int] -) -> tuple[Any, Optional[RestoredCall]]: - """Resolve the prefix a checkpoint cut saved for this call. - - Every host of the chat path calls it once the capture prefix is known. - Returns the admission to serve and the restore. A prefix the worker cannot - use comes back as the admission without its cut and None, so the call - regenerates. For a restore, the request gets its restore fields and its - remaining ``min_tokens``. - """ - # Off-loop: the saved rows are a blocking TQ read. - resolved_admission, saved = await asyncio.to_thread( - host._prefix_recovery.resolve_restore, serving_admission, prefix_token_ids - ) - if saved is None: - return resolved_admission, None - original_min_tokens = getattr(request, "min_tokens", None) - plan = plan_restore( - prefix_token_ids=prefix_token_ids, - saved=saved, - continuation=resolved_admission.generation_cut, - min_tokens=original_min_tokens, - ) - request._restored_generation_token_ids = plan.resumed_generation_token_ids - request._original_min_tokens = original_min_tokens - if plan.min_tokens is not None: - request.min_tokens = plan.min_tokens - return resolved_admission, RestoredCall( - saved=saved, - prompt_token_ids=plan.engine_prefix_token_ids, - resumed_generation_token_ids=list(plan.resumed_generation_token_ids), - output_limit=plan.output_limit, - max_tokens=plan.max_tokens, - ) - - -def classify_restored_terminal( - restored: RestoredCall, - prompt_token_ids: list[int], - admission: Any, - *, - max_model_len: int, -) -> Optional[_RestoredPrefixTerminal]: - """The terminal of a saved prefix that already finished its call, else None.""" - continuation = admission.generation_cut - try: - return _classify_restored_prefix_terminal( - prompt_token_ids=prompt_token_ids, - prefix_token_count=continuation.prefix_token_count, - requested_output_tokens=restored.output_limit, - model_max_tokens=max_model_len, - terminal_finish_reason=continuation.terminal_finish_reason, - terminal_stop_reason=continuation.terminal_stop_reason, - ) - except ValueError as error: - raise VLLMValidationError( - str(error), - parameter="generation_prefix", - value=continuation.prefix_token_count, - ) from error - - -def serve_restored_terminal( - request: Any, - engine_prompt: Any, - terminal: _RestoredPrefixTerminal, - admission: Any, - *, - max_model_len: int, -) -> None: - """Have the engine client answer a finished prefix without decoding. - - Call it in the task that submits the call to the engine: the marker is a - context variable. - """ - # One token is the smallest budget vLLM accepts, so a missed marker costs - # one token, not the whole call. - request.min_tokens = 0 - set_max_tokens(request, 1) - mark_restored_terminal(terminal) - prompt_token_ids = engine_prompt["prompt_token_ids"] - if len(prompt_token_ids) == max_model_len: - engine_prompt["prompt_token_ids"] = prompt_token_ids[:-1] - request._restored_prefix_terminal = terminal - LOGGER.info( - "generation prefix already terminal: " - "rollout_id=%s model_call_id=%s prefix_tokens=%d reason=%s", - admission.capture_key, - admission.model_call_id, - terminal.prefix_token_count, - terminal.reason, - ) - - -async def abort_request_capture(host: Any, request: Any, reason: str) -> None: - """Fail a request's capture without blocking the event loop. - - An uncontended abort finishes here, so it never queues behind - executor work. Only one that must wait out a generation cut or a - terminal write holding the call's lifecycle lock moves off the loop. - """ - if host._abort_request_capture(request, reason=reason, blocking=False): - return - # Shielded: these paths often run after the handler was cancelled, - # and a second cancel must not drop the abort before it starts, - # leaving the call neither completed nor failed. - await asyncio.shield( - asyncio.to_thread(host._abort_request_capture, request, reason=reason) - ) - - class NeMoRLOpenAIServingMixin: """The chat preprocessing NeMo RL adds: prefix splice and token capture.""" - def _clamp_max_tokens( - self, request, request_max_tokens: Optional[int], prompt_token_ids: list[int] - ) -> None: - """Clamp the request's max output tokens so that input + output <= max_model_len.""" - max_tokens = clamp_max_tokens( - request_max_tokens, len(prompt_token_ids), self.model_config.max_model_len - ) - if max_tokens is not None: - set_max_tokens(request, max_tokens) - async def _render_chat_call( self, request, messages, render_kwargs: dict[str, Any], *, admission ) -> "RenderedChat": @@ -359,8 +186,19 @@ async def _render_chat_call( render with the checkpoint's template and processor, the capture decision for a Gym candidate, and, when a prefix will be spliced, the render of the history up to the last assistant turn that - the splice cuts against. + the splice cuts against. A chat request renders with an output budget + of 1, so vLLM's pre-tokenization length check passes; start_capture sets the + budget clamped to the final prompt. """ + request_max_tokens = None + if isinstance(request, ChatCompletionRequest): + request_max_tokens = ( + request.max_completion_tokens + if request.max_completion_tokens is not None + else request.max_tokens + ) + if request_max_tokens is not None: + set_max_tokens(request, 1) for message in messages: if message.get("tool_calls"): message["tool_calls"] = list(message["tool_calls"]) @@ -421,12 +259,17 @@ async def _render_chat_call( history_token_ids = history_prompt["prompt_token_ids"] return RenderedChat( - conversation=conversation, - engine_prompt=engine_prompt, + template_token_ids=list(engine_prompt["prompt_token_ids"]), + history_token_ids=history_token_ids, + required_prefix_token_ids=getattr( + request, "required_prefix_token_ids", None + ), admission=admission, serving_admission=serving_admission, verify_retained_media=verify_retained_media, - history_token_ids=history_token_ids, + request_max_tokens=request_max_tokens, + conversation=conversation, + engine_prompt=engine_prompt, ) # vLLM 0.25 moved chat preprocessing to @@ -452,22 +295,6 @@ async def preprocess_chat( if host._capture_media and admission is not None: skip_mm_cache = True - # Temporarily set to 1 so vLLM's pre-tokenization length check passes; - # the actual value will be set through _clamp_max_tokens later. - # /tokenize counts a prompt of any length, so only chat calls clamp. - clamps_output = isinstance(request, NeMoRLChatCompletionRequest) - actual_request_max_tokens = None - if clamps_output: - actual_request_max_tokens = ( - request.max_completion_tokens - if request.max_completion_tokens is not None - else request.max_tokens - ) - # If max_completion_tokens or max_tokens is not set, we don't need to do _clamp_max_tokens. - # So we don't need to set the request's max output tokens to 1 here. - if actual_request_max_tokens is not None: - set_max_tokens(request, 1) - rendered = await self._render_chat_call( request, messages, @@ -481,184 +308,49 @@ async def preprocess_chat( ), admission=admission, ) - res = (rendered.conversation, [rendered.engine_prompt]) - engine_prompt = rendered.engine_prompt - admission = rendered.admission - serving_admission = rendered.serving_admission - verify_retained_media = rendered.verify_retained_media - - # Resolve the admission's prefix (staging_chain -> cached TQ read). - # Off-loop: the chain fetch is a blocking TQ read, and Gym's staging - # protocol requires the serving host to move blocking staging I/O off - # its event loop explicitly. The adapter then attaches the prefix to - # the request, so the inline-prefix branch below is the single splice - # path for staged and inline prefixes. - capture_prefix_token_ids: list[int] | None = None - restored: RestoredCall | None = None - if serving_admission is not None: - resolved_prefix: list[int] = await asyncio.to_thread( - host._resolve_admission_prefix, serving_admission - ) - capture_prefix_token_ids = resolved_prefix - resolved_admission, restored = await resolve_restored_call( - host, request, serving_admission, resolved_prefix - ) - # An unusable prefix comes back without its cut, so the - # call regenerates; storage and serving share one admission here. - if serving_admission is admission: - admission = resolved_admission - serving_admission = resolved_admission - engine_prefix_token_ids = list(resolved_prefix) - if restored is not None: - engine_prefix_token_ids = restored.prompt_token_ids - if restored.max_tokens is not None: - actual_request_max_tokens = restored.max_tokens - if engine_prefix_token_ids: - host._enter_request_prefix(request, engine_prefix_token_ids) - generation_cut = None if restored is None else restored.saved - resumed_generation_token_ids = ( - [] if restored is None else restored.resumed_generation_token_ids + plan = await prepare_prompt( + rendered, + resolve_prefix=host._resolve_admission_prefix, + turn_end_token_ids=turn_end_token_ids( + self._model_stop_token_ids, self.renderer.tokenizer + ), + max_model_len=self.model_config.max_model_len, + # /tokenize counts a prompt of any length, so only chat calls clamp. + clamp=isinstance(request, NeMoRLChatCompletionRequest), + tokenizer=self.renderer.tokenizer, ) - - if ( - not hasattr(request, "required_prefix_token_ids") - or request.required_prefix_token_ids is None - ): - # Clamp the request's max output tokens so that input + output <= max_model_len. - if clamps_output: - self._clamp_max_tokens( - request, - actual_request_max_tokens, - engine_prompt["prompt_token_ids"], - ) - # Token capture, text mode: the full render is the exact - # engine prompt. - media = await asyncio.to_thread( - host._capture_request_media, - engine_prompt, - admission=admission, - verify_retained_media=verify_retained_media, - ) - host._begin_request_capture( - request, - engine_prompt["prompt_token_ids"], - admission=admission, - prefix_token_ids=capture_prefix_token_ids, - media=media, - generation_cut=generation_cut, - resumed_generation_token_ids=resumed_generation_token_ids, - ) - return res - - splice = None - if restored is not None: - # The durable prefix is the exact engine prompt at the cut; - # rendering/splicing it again could add or drop tokens. - # truncate_prompt_tokens already applied when the original - # call's chat was tokenized, and vLLM does not truncate - # this list again (test_vllm_progress_read_layout.py). - final_prompt_token_ids = list(request.required_prefix_token_ids) - else: - # The render includes the history whenever a prefix will be spliced. - assert rendered.history_token_ids is not None - splice = splice_prefix_tokens( - tokenizer=self.renderer.tokenizer, - model_prefix_token_ids=list(request.required_prefix_token_ids), - template_prefix_token_ids=rendered.history_token_ids, - template_token_ids=engine_prompt["prompt_token_ids"], - eos_token_id=turn_end_token_ids( - self._model_stop_token_ids, self.renderer.tokenizer - ), - ) - final_prompt_token_ids = splice.token_ids - - restored_prefix_terminal = None - if restored is not None: - restored_prefix_terminal = classify_restored_terminal( - restored, - final_prompt_token_ids, - admission, - max_model_len=self.model_config.max_model_len, - ) - - # Reject overflows before media validation can mask the length error. - # A restored terminal prefix generates nothing more. - if clamps_output and restored_prefix_terminal is None: - self._clamp_max_tokens( - request, - actual_request_max_tokens, - final_prompt_token_ids, - ) - - # Generation-prefix cuts are text-only (setup refuses them with media - # capture), so a restored prompt has no media to capture. - media = None - if restored is None: - media = await asyncio.to_thread( - host._capture_request_media, - engine_prompt, - admission=serving_admission, - splice=splice, - verify_retained_media=verify_retained_media, - ) - # Must precede the reassignment below: the remap reads the - # current value of prompt_token_ids as the template-coordinate - # sequence the placeholders were computed against. Reordering - # these two statements silently degrades to a no-op. - # Successful media capture already remapped these ranges. - if media is None and (mm_placeholders := engine_prompt.get("mm_placeholders")): - engine_prompt["mm_placeholders"] = remap_multimodal_placeholders( - template_token_ids=engine_prompt["prompt_token_ids"], - final_token_ids=final_prompt_token_ids, - mm_placeholders=mm_placeholders, - splice=splice, - ) - - engine_prompt["prompt_token_ids"] = final_prompt_token_ids - - # Token capture, token-in mode: the spliced prompt is the - # exact engine prompt; begin_call re-checks the prefix it - # was spliced from against the admission. - host._begin_request_capture( + await start_capture( + host, request, - final_prompt_token_ids, - admission=admission, - prefix_token_ids=( - capture_prefix_token_ids - if admission is not None and admission.mode == "token_in" - else None - ), - media=media, - generation_cut=generation_cut, - resumed_generation_token_ids=resumed_generation_token_ids, + rendered.engine_prompt, + plan, + max_model_len=self.model_config.max_model_len, ) - - if restored_prefix_terminal is not None: - serve_restored_terminal( - request, - engine_prompt, - restored_prefix_terminal, - admission, - max_model_len=self.model_config.max_model_len, - ) - - return res + return rendered.conversation, [rendered.engine_prompt] @dataclass(frozen=True) class RenderedChat: - """One chat call rendered for the engine, before its prefix is spliced in.""" + """One chat call rendered for the engine, before its prefix is spliced in. - conversation: list[Any] - # The full render, multimodal inputs included; its ids are the template's. - engine_prompt: dict[str, Any] + A :class:`~nemo_rl.models.generation.vllm.prompt_and_capture.RenderedPrompt`. + """ + + template_token_ids: list[int] + # The render up to the last assistant turn, when a prefix will be spliced. + history_token_ids: Optional[list[int]] + # The request's own prefix (Gym's echo) after the capture decision. + required_prefix_token_ids: Optional[list[int]] # What capture stores, after the candidate decision; None without capture. admission: Any # How the prompt is served: "token_in" reads the prefix from TQ. serving_admission: Any verify_retained_media: bool - # The render up to the last assistant turn, when a prefix will be spliced. - history_token_ids: Optional[list[int]] + # The output budget the request asked for, before the render lowered it. + request_max_tokens: Optional[int] + conversation: list[Any] + # The full render, multimodal inputs included; its ids are the template's. + engine_prompt: dict[str, Any] class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): @@ -695,10 +387,6 @@ async def render_for_frontend(self, request, *, admission) -> "RenderedChat": "Dynamo chat engine; use policy.generation.dynamo=false" ) self._admission = admission - # As preprocess_chat does: vLLM's pre-tokenization length check must - # pass; the frontend clamps the budget after the splice. - if request.max_completion_tokens is not None or request.max_tokens is not None: - set_max_tokens(request, 1) result = await self.render_chat(request, skip_mm_cache=True) if not isinstance(result, tuple): raise ValueError(result.error.message) @@ -737,6 +425,74 @@ async def preprocess_chat( ) +class ResumedCallReplyMixin: + """vLLM's reply to a call resumed from a checkpoint cut, as the whole call's. + + The engine ran the restored tokens as prompt and generated only the rest. + Before vLLM parses the reply, the restored text goes in front of the final + output and the parser sees the restored tokens first; after, usage counts + them as completion. It reads the request's restore fields, which + ``resolve_restored_call`` sets on the worker and the render pool sets from + the engine's reply. The native serving chat and the render pool's both use + it. + """ + + async def chat_completion_full_generator( + self, + request, + result_generator, + *args, + **kwargs, + ): + restored_token_ids = request._restored_generation_token_ids + + async def restored_results(): + final_res = None + async for res in result_generator: + final_res = res + if final_res is None: + return + # Before vLLM parses the reply, so tool calls and reasoning are + # parsed from the whole call's text. + restore_reply_text( + request, + final_res, + prefix_token_ids=restored_token_ids, + engine_prompt_token_ids=request._restored_prompt_token_ids, + tokenizer=self.renderer.tokenizer, + ) + yield final_res + + if request._restored_prompt_token_ids: + result_generator = restored_results() + # Bound by name: vLLM passes request_metadata by position and + # parser by keyword, and either may move. + bound = _FULL_GENERATOR_SIGNATURE.bind_partial( + self, request, result_generator, *args, **kwargs + ) + if bound.arguments.get("parser") is not None: + bound.arguments["parser"] = wrap_parser_for_restore( + bound.arguments["parser"], restored_token_ids + ) + response = await super().chat_completion_full_generator( + *bound.args[1:], **bound.kwargs + ) + if not isinstance(response, ChatCompletionResponse): + return response + terminal = request._restored_prefix_terminal + if not isinstance(terminal, _RestoredPrefixTerminal): + terminal = None + if restore_reply_usage( + response, + restored_token_count=len(restored_token_ids), + terminal=terminal, + ): + request_metadata = bound.arguments.get("request_metadata") + if request_metadata is not None: + request_metadata.final_usage_info = response.usage + return response + + class NeMoRLOpenAIServingChatMixin: async def chat_completion_full_generator( self, @@ -751,12 +507,6 @@ async def chat_completion_full_generator( if request.return_tokens_as_token_ids is not None else self.return_tokens_as_token_ids ) - if request.logprobs and return_as_token_id and request.top_logprobs is None: - raise VLLMValidationError( - "`top_logprobs` must be set when requesting token " - "information from the NeMo-RL chat endpoint.", - parameter="top_logprobs", - ) final_res = None capture_routes = request in host._capture_registry @@ -782,27 +532,11 @@ async def capture_result_generator(): final_res = res if final_res is None: return - host._prefix_recovery.restore_reply_text( - request, - final_res, - tokenizer=self.renderer.tokenizer, - ) yield (_without_routes(final_res) if capture_routes else final_res) except asyncio.CancelledError: cancelled = True raise - # Bound by name: vLLM passes request_metadata by position and - # parser by keyword, and either may move. - bound = _FULL_GENERATOR_SIGNATURE.bind_partial( - self, request, capture_result_generator(), *args, **kwargs - ) - if bound.arguments.get("parser") is not None: - bound.arguments["parser"] = wrap_parser_for_restore( - bound.arguments["parser"], - request._restored_generation_token_ids, - ) - # Capture stages the raw arrays below and strips OpenAI logprobs # after the terminal PUT. Skip only their object construction; # changing request.logprobs would also change engine sampling. @@ -814,7 +548,7 @@ async def capture_result_generator(): ) with self._bypass_chat_logprobs(enabled=bypass_logprobs): response = await super().chat_completion_full_generator( - *bound.args[1:], **bound.kwargs + request, capture_result_generator(), *args, **kwargs ) if cancelled: # vLLM's chat_completion_full_generator catches the @@ -829,18 +563,6 @@ async def capture_result_generator(): if not isinstance(response, ChatCompletionResponse) or final_res is None: return response - restored_prefix_terminal = request._restored_prefix_terminal - if not isinstance(restored_prefix_terminal, _RestoredPrefixTerminal): - restored_prefix_terminal = None - if restore_reply_usage( - response, - restored_token_count=len(request._restored_generation_token_ids), - terminal=restored_prefix_terminal, - ): - request_metadata = bound.arguments.get("request_metadata") - if request_metadata is not None: - request_metadata.final_usage_info = response.usage - if request.logprobs and return_as_token_id: response = attach_token_information_to_chat_response_choices( response, @@ -871,6 +593,7 @@ async def capture_result_generator(): class NeMoRLOpenAIServingChat( NeMoRLOpenAIServingChatMixin, CapturedChatLogprobsMixin, + ResumedCallReplyMixin, OpenAIServingChat, ): def __init__(self, *args, host: Any, **kwargs): @@ -878,6 +601,13 @@ def __init__(self, *args, host: Any, **kwargs): self._host = host +class FrontendServingChat(ResumedCallReplyMixin, OpenAIServingChat): + """The reply step for NeMo RL's chat engine in Dynamo's frontend. + + Builds the reply from the engine's output and never generates. + """ + + # Tokenize path delegates to OnlineRenderer.preprocess_chat, # where the prefix-token override lives. class NeMoRLServingTokenization(ServingTokenization): @@ -1072,110 +802,45 @@ async def create_chat_completion( presence_penalty=request.presence_penalty, frequency_penalty=request.frequency_penalty, logit_bias=request.logit_bias, + n=request.n, + logprobs=request.logprobs, + # vLLM defaults an unset top_logprobs to 0; Gym must set it. + top_logprobs=( + request.top_logprobs + if "top_logprobs" in request.model_fields_set + else None + ), generation_config=generation_config, ) - except OffPolicySamplingError as e: - # Before capture begins, so there is no capture state to settle. - return JSONResponse( - content={ - "error": { - "message": str(e), - "type": "invalid_request_error", - "param": "sampling", - "code": 400, - } - }, - status_code=400, - ) - for field, value in sampling.items(): - setattr(request, field, value) - - try: + for field, value in sampling.items(): + setattr(request, field, value) generator = await openai_serving_chat.create_chat_completion( request, raw_request ) - except VLLMValidationError as e: - # vLLM raises VLLMValidationError for prompts exceeding - # max_model_len during tokenization, instead of returning an - # ErrorResponse. Convert to HTTP 400 so the Gym proxy can - # detect context-length overflow and handle it gracefully. - await abort_request_capture(host, request, "context_length") - return JSONResponse( - content={ - "error": { - "message": str(e), - "type": "invalid_request_error", - "param": e.parameter, - "code": 400, - } - }, - status_code=400, - headers=host._context_refusal_headers(request), - ) - except MediaCaptureRejected as e: - # Raised inside preprocess_chat before begin_call, so no capture - # state exists yet and the abort below is a no-op kept for - # symmetry. Return a 400 carrying a stable code so Gym and the - # worker log can distinguish a retained-media re-tile from a - # real engine error (which stays a 500 below). - await abort_request_capture(host, request, e.code) - LOGGER.warning( - "Rejected captured call before inference (%s): %s", e.code, e - ) - return JSONResponse( - content={ - "error": { - "message": str(e), - "type": "invalid_request_error", - "param": "messages", - "code": e.code, - } - }, - status_code=400, - ) - except ValueError as e: - # vLLM's get_max_tokens raises a plain ValueError ("Input length - # (N) exceeds model's maximum context length (M).") after - # preprocess_chat, e.g. when a token-in splice grows the prompt - # past max_model_len on a request with no max_tokens to clamp. - # Convert only that overflow to the same 400 as above; any other - # ValueError is a server bug and stays a 500. Must follow - # MediaCaptureRejected, which is also a ValueError. - if "maximum context length" not in str(e): - # Re-raising here bypasses the BaseException handler below. - await abort_request_capture(host, request, "engine_error") + except BaseException as error: + # A cancel before generation starts propagates here directly; one + # that lands mid-generation is swallowed by vLLM and re-raised by + # NeMoRLOpenAIServingChatMixin.chat_completion_full_generator. + failure = await abort_capture(host, request, error) + if failure is None: raise - LOGGER.warning("Prompt exceeds max_model_len: %s", e) - await abort_request_capture(host, request, "context_length") return JSONResponse( content={ "error": { - "message": str(e), - "type": "invalid_request_error", - "param": "input_tokens", - "code": 400, + "message": failure.message, + "type": failure.error_type, + "param": failure.param, + "code": failure.code, } }, - status_code=400, - # Only vLLM's known pre-generation get_max_tokens refusal - # can settle capture; an arbitrary ValueError cannot. + status_code=failure.status, + # Sent after abort_capture drained the capture state. headers=( host._context_refusal_headers(request) - if str(e).startswith("Input length (") - and "exceeds model's maximum context length" in str(e) + if failure.acknowledge else None ), ) - except asyncio.CancelledError: - # Client disconnected (see with_cancellation above). A cancel - # before generation starts propagates here directly; one that - # lands mid-generation is swallowed by vLLM and re-raised by - # NeMoRLOpenAIServingChatMixin.chat_completion_full_generator. - await abort_request_capture(host, request, "client_disconnect") - raise - except BaseException: - await abort_request_capture(host, request, "engine_error") - raise if isinstance(generator, ErrorResponse): await abort_request_capture(host, request, "error_response") @@ -1187,17 +852,7 @@ async def create_chat_completion( content = model_dump_chat_response_with_dynamic_message_fields(generator) # Token capture: stage the delta and ride the coords on the # response; strips logprobs/ids (no-op when capture is off). - # Off-loop: the sink write inside complete_call is a blocking - # TQ round trip (see the staging protocol's serving-host rule). - try: - content = await asyncio.to_thread( - host._finish_request_capture, request, content - ) - except asyncio.CancelledError: - # Queued executor work may never start after cancellation. - # If staging already claimed the capture, abort is a no-op. - await abort_request_capture(host, request, "client_disconnect") - raise + content = await save_capture(host, request, content) return JSONResponse(content=content) await abort_request_capture(host, request, "streaming_response") diff --git a/nemo_rl/models/generation/vllm/prompt_and_capture.py b/nemo_rl/models/generation/vllm/prompt_and_capture.py new file mode 100644 index 00000000000..53bd5286b88 --- /dev/null +++ b/nemo_rl/models/generation/vllm/prompt_and_capture.py @@ -0,0 +1,624 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The steps of one Gym chat call after its render, shared by every serving path. + +1. :func:`prepare_prompt` turns a render into a :class:`PreparedPrompt`: the + token-capture prefix read from TQ, the splice, and the output budget. +2. On the engine's worker, :func:`start_capture` takes the prepared prompt to + ``begin_call``: a restore from a checkpoint cut, the budget, the media + snapshot, the placeholder remap and the final prompt; after generation, + :func:`save_capture` stages the call in one TQ write. +3. :func:`abort_capture` fails the capture of a call that raised, and + :func:`error_reply` says how to answer it. + +The native vLLM server runs all of them in the vLLM worker. Behind Dynamo, +the frontend's chat engine prepares the prompt (its KV router needs the final +token ids before it picks an engine) and the engine's adapter runs the rest, +with the prepared prompt sent across as JSON (:meth:`PreparedPrompt.to_json`). +A restore is resolved on the worker: its weight-version check needs the +engine's own. + +This module imports vLLM; import it only where vLLM is installed. +""" + +import asyncio +import logging +from dataclasses import dataclass +from typing import Any, Callable, Collection, Optional, Protocol + +from vllm.entrypoints.serve.exception_handling.error_response import ( + create_error_response, +) +from vllm.exceptions import GracefulHTTPError, VLLMClientError, VLLMValidationError + +from nemo_rl.data.captured_media import MediaCaptureRejected +from nemo_rl.models.generation.openai_server_utils import ( + PrefixSplice, + splice_prefix_tokens, +) +from nemo_rl.models.generation.prefix_recovery.restore import ( + _classify_restored_prefix_terminal, + _RestoredPrefixTerminal, + mark_restored_terminal, + plan_restore, +) +from nemo_rl.models.generation.vllm.utils import ( + OffPolicySamplingError, + remap_multimodal_placeholders, +) + +LOGGER = logging.getLogger(__name__) + + +def set_max_tokens(request: Any, max_tokens: int) -> None: + """Set the request's max output tokens. + + Mutates the request in place. Handles both max_completion_tokens (newer OpenAI API) + and max_tokens (deprecated but still supported by vLLM). A request + that set neither gets max_tokens: otherwise vLLM would fall back to + the generation config's budget, e.g. for a restored call whose + remaining budget is smaller. + """ + if request.max_completion_tokens is not None: + request.max_completion_tokens = max_tokens + else: + request.max_tokens = max_tokens + + +def clamp_max_tokens( + request_max_tokens: Optional[int], prompt_len: int, max_model_len: int +) -> Optional[int]: + """The output budget that keeps input + output within ``max_model_len``. + + Refuses a prompt that leaves no room for output, whether or not the call set + a budget, and lowers the budget only when it did. + """ + remaining = max_model_len - prompt_len + if remaining <= 0: + # preserve the literal "context length" in this message to match Gym's overflow handling + message = ( + f"Prompt length ({prompt_len}) fills or exceeds " + f"this model's maximum context length ({max_model_len}). " + f"No room for output tokens." + ) + LOGGER.warning("Prompt exceeds max_model_len: %s", message) + raise VLLMValidationError(message, parameter="input_tokens", value=prompt_len) + if request_max_tokens is None: + return None + return min(request_max_tokens, remaining) + + +@dataclass(frozen=True) +class RestoredCall: + """A call that resumes from the tokens a checkpoint cut saved.""" + + # The verified saved rows, as capture's begin_call takes them. + saved: Any + # The engine prompt: the capture prefix, then the saved rows. + prompt_token_ids: list[int] + # The saved rows' generated tokens alone. + resumed_generation_token_ids: list[int] + # The whole output budget, as the cut call resolved it. + output_limit: int + # What is left of that budget; None keeps the request's own. + max_tokens: Optional[int] + + +async def resolve_restored_call( + host: Any, request: Any, serving_admission: Any, prefix_token_ids: list[int] +) -> tuple[Any, Optional[RestoredCall]]: + """Resolve the prefix a checkpoint cut saved for this call. + + Returns the admission to serve and the restore. A prefix the worker cannot + use comes back as the admission without its cut and None, so the call + regenerates. For a restore, the request gets its restore fields (which the + reply reads) and its remaining ``min_tokens``. + """ + # Off-loop: the saved rows are a blocking TQ read. + resolved_admission, saved = await asyncio.to_thread( + host._prefix_recovery.resolve_restore, serving_admission, prefix_token_ids + ) + if saved is None: + return resolved_admission, None + original_min_tokens = getattr(request, "min_tokens", None) + plan = plan_restore( + prefix_token_ids=prefix_token_ids, + saved=saved, + continuation=resolved_admission.generation_cut, + min_tokens=original_min_tokens, + ) + request._restored_generation_token_ids = plan.resumed_generation_token_ids + request._restored_prompt_token_ids = tuple(plan.engine_prefix_token_ids) + request._original_min_tokens = original_min_tokens + if plan.min_tokens is not None: + request.min_tokens = plan.min_tokens + return resolved_admission, RestoredCall( + saved=saved, + prompt_token_ids=plan.engine_prefix_token_ids, + resumed_generation_token_ids=list(plan.resumed_generation_token_ids), + output_limit=plan.output_limit, + max_tokens=plan.max_tokens, + ) + + +def classify_restored_terminal( + restored: RestoredCall, + prompt_token_ids: list[int], + admission: Any, + *, + max_model_len: int, +) -> Optional[_RestoredPrefixTerminal]: + """The terminal of a saved prefix that already finished its call, else None.""" + continuation = admission.generation_cut + try: + return _classify_restored_prefix_terminal( + prompt_token_ids=prompt_token_ids, + prefix_token_count=continuation.prefix_token_count, + requested_output_tokens=restored.output_limit, + model_max_tokens=max_model_len, + terminal_finish_reason=continuation.terminal_finish_reason, + terminal_stop_reason=continuation.terminal_stop_reason, + ) + except ValueError as error: + raise VLLMValidationError( + str(error), + parameter="generation_prefix", + value=continuation.prefix_token_count, + ) from error + + +def serve_restored_terminal( + request: Any, + engine_prompt: Any, + terminal: _RestoredPrefixTerminal, + admission: Any, + *, + max_model_len: int, +) -> None: + """Have the engine client answer a finished prefix without decoding. + + Call it in the task that submits the call to the engine: the marker is a + context variable. + """ + # One token is the smallest budget vLLM accepts, so a missed marker costs + # one token, not the whole call. + request.min_tokens = 0 + set_max_tokens(request, 1) + mark_restored_terminal(terminal) + prompt_token_ids = engine_prompt["prompt_token_ids"] + if len(prompt_token_ids) == max_model_len: + engine_prompt["prompt_token_ids"] = prompt_token_ids[:-1] + request._restored_prefix_terminal = terminal + LOGGER.info( + "generation prefix already terminal: " + "rollout_id=%s model_call_id=%s prefix_tokens=%d reason=%s", + admission.capture_key, + admission.model_call_id, + terminal.prefix_token_count, + terminal.reason, + ) + + +async def abort_request_capture(host: Any, request: Any, reason: str) -> None: + """Fail a request's capture without blocking the event loop. + + An uncontended abort finishes here, so it never queues behind + executor work. Only one that must wait out a generation cut or a + terminal write holding the call's lifecycle lock moves off the loop. + """ + if host._abort_request_capture(request, reason=reason, blocking=False): + return + # Shielded: these paths often run after the handler was cancelled, + # and a second cancel must not drop the abort before it starts, + # leaving the call neither completed nor failed. + await asyncio.shield( + asyncio.to_thread(host._abort_request_capture, request, reason=reason) + ) + + +class RenderedPrompt(Protocol): + """What :func:`prepare_prompt` reads from a call's render; each path's render has it.""" + + # The full render's token ids, in the template's coordinates. + template_token_ids: list[int] + # The render up to the last assistant turn, when a prefix will be spliced. + history_token_ids: Optional[list[int]] + # The request's own prefix (Gym's echo) after the capture decision. + required_prefix_token_ids: Optional[list[int]] + # Capture admissions: what is stored, and how the prompt is served; None + # without capture. + admission: Any + serving_admission: Any + verify_retained_media: bool + # The output budget the request asked for; None when it set none. + request_max_tokens: Optional[int] + + +@dataclass(frozen=True) +class PreparedPrompt: + """A call's prompt as the engine runs it, its output budget, and how capture admits it.""" + + prompt_token_ids: list[int] + template_token_ids: list[int] + # How the model's own prefix replaced the template's history; None if not. + splice: Optional[PrefixSplice] + # The model's own prefix is the prompt's first ``prefix_len`` ids. + prefix_len: Optional[int] + # Whether Gym only measures the prompt; such a call never generates. + measure_only: bool + # The output budget the request asked for, and the one after the clamp; + # None when the request set none. + request_max_tokens: Optional[int] + max_tokens: Optional[int] + admission: Any + serving_admission: Any + verify_retained_media: bool + + @property + def begin_call_prefix_token_ids(self) -> Optional[list[int]]: + """The prefix ``begin_call`` checks a token-in admission against.""" + if self.admission is None or self.admission.mode != "token_in": + return None + return self.prompt_token_ids[: self.prefix_len] + + def to_json(self) -> dict[str, Any]: + """As JSON, without the prompt (it rides as the call's token ids).""" + return { + "template_token_ids": self.template_token_ids, + "splice": None + if self.splice is None + else [self.splice.model_cut_end, self.splice.template_cut_start], + "prefix_len": self.prefix_len, + "measure_only": self.measure_only, + "request_max_tokens": self.request_max_tokens, + "max_tokens": self.max_tokens, + "admission": _admission_json(self.admission), + "serving_admission": _admission_json(self.serving_admission), + "verify_retained_media": self.verify_retained_media, + } + + @classmethod + def from_json( + cls, + wire: dict[str, Any], + *, + prompt_token_ids: list[int], + parse_admission: Callable[[Any], Any], + ) -> "PreparedPrompt": + """What :meth:`to_json` sent, for the prompt it rides beside.""" + splice = None + if wire["splice"] is not None: + splice = PrefixSplice(prompt_token_ids, *wire["splice"]) + admission = parse_admission(wire["admission"]) + # One object when storage and serving are one admission, as natively. + serving_admission = admission + if wire["serving_admission"] != wire["admission"]: + serving_admission = parse_admission(wire["serving_admission"]) + return cls( + prompt_token_ids=prompt_token_ids, + template_token_ids=list(wire["template_token_ids"]), + splice=splice, + prefix_len=wire["prefix_len"], + measure_only=wire["measure_only"], + request_max_tokens=wire["request_max_tokens"], + max_tokens=wire["max_tokens"], + admission=admission, + serving_admission=serving_admission, + verify_retained_media=wire["verify_retained_media"], + ) + + +def _admission_json(admission: Any) -> Optional[dict[str, Any]]: + return None if admission is None else admission.model_dump(mode="json") + + +async def prepare_prompt( + rendered: RenderedPrompt, + *, + resolve_prefix: Callable[[Any], list[int]], + turn_end_token_ids: Collection[int], + max_model_len: int, + clamp: bool, + tokenizer: Any = None, +) -> PreparedPrompt: + """Prepare a rendered call's prompt: the ids the engine runs, and its output budget. + + Args: + rendered: The call's render. + resolve_prefix: Reads a token-in admission's prefix from TQ. It blocks, + so it runs off the event loop. + turn_end_token_ids: Every id a turn can end on; the splice cuts by them. + max_model_len: The engine's context length. + clamp: Whether the call generates, so its budget is clamped to the + context; a measure call counts its prompt at any length. + tokenizer: Only for the splice's error message; may be None. + + Raises: + VLLMValidationError: A generating call's prompt leaves no room for + output. Raised before any media check, so the length error is the + one the call answers with. + """ + prefix = rendered.required_prefix_token_ids + serving = rendered.serving_admission + if serving is not None and serving.mode == "token_in": + prefix = await asyncio.to_thread(resolve_prefix, serving) + splice = None + prompt_token_ids = rendered.template_token_ids + if prefix is not None: + # The render includes the history whenever a prefix will be spliced. + assert rendered.history_token_ids is not None + splice = splice_prefix_tokens( + tokenizer=tokenizer, + model_prefix_token_ids=list(prefix), + template_prefix_token_ids=rendered.history_token_ids, + template_token_ids=rendered.template_token_ids, + eos_token_id=turn_end_token_ids, + ) + prompt_token_ids = splice.token_ids + max_tokens = None + if clamp: + max_tokens = clamp_max_tokens( + rendered.request_max_tokens, len(prompt_token_ids), max_model_len + ) + return PreparedPrompt( + prompt_token_ids=prompt_token_ids, + template_token_ids=rendered.template_token_ids, + splice=splice, + prefix_len=None if prefix is None else len(prefix), + measure_only=not clamp, + request_max_tokens=rendered.request_max_tokens, + max_tokens=max_tokens, + admission=rendered.admission, + serving_admission=rendered.serving_admission, + verify_retained_media=rendered.verify_retained_media, + ) + + +async def start_capture( + host: Any, + request: Any, + engine_prompt: dict[str, Any], + plan: PreparedPrompt, + *, + max_model_len: int, +) -> None: + """Take a prepared prompt to the start of its capture, on the engine's worker. + + In order: resolve a restore from a checkpoint cut (its saved rows replace + the prepared prompt), set the output budget, snapshot the call's media for + capture, make ``engine_prompt`` the final prompt (its media placeholders + moved with the splice), begin the call's capture, and have the engine + answer a restored call that had already finished. + + Args: + host: The vLLM worker; it owns token capture. + request: The chat request the engine serves. + engine_prompt: The render's engine input, in the template's + coordinates; updated in place. + plan: The call's prepared prompt. + max_model_len: The engine's context length. + """ + admission = plan.admission + serving_admission = plan.serving_admission + if admission is not None and admission.mode == "candidate": + raise RuntimeError( + "a candidate admission reached the engine undecided; NeMo RL's " + "render must decide it before capture" + ) + prompt_token_ids = plan.prompt_token_ids + splice = plan.splice + max_tokens = plan.max_tokens + restored = None + terminal = None + if serving_admission is not None: + # A cut this worker cannot use comes back as None: the prepared prompt + # stands, and the call regenerates. + resolved_admission, restored = await resolve_restored_call( + host, + request, + serving_admission, + # The model's own prefix is the prepared prompt's first ids. + prompt_token_ids[: plan.prefix_len] + if serving_admission.mode == "token_in" + else [], + ) + if serving_admission is admission: + admission = resolved_admission + serving_admission = resolved_admission + if restored is not None: + # The saved rows are the exact engine prompt at the cut; splicing them + # again could add or drop tokens (test_vllm_progress_read_layout.py). + splice = None + prompt_token_ids = restored.prompt_token_ids + terminal = classify_restored_terminal( + restored, prompt_token_ids, admission, max_model_len=max_model_len + ) + # A restored terminal prefix generates nothing more. + max_tokens = None + if not plan.measure_only and terminal is None: + max_tokens = clamp_max_tokens( + restored.max_tokens + if restored.max_tokens is not None + else plan.request_max_tokens, + len(prompt_token_ids), + max_model_len, + ) + if max_tokens is not None: + set_max_tokens(request, max_tokens) + media = None + media_admission = admission if splice is None else serving_admission + # Generation-prefix cuts are text-only (setup refuses them with media + # capture), so a restored prompt has no media to capture. + if restored is None and media_admission is not None: + # Off-loop: a retained-media check reads staged calls. + media = await asyncio.to_thread( + host._capture_request_media, + engine_prompt, + admission=media_admission, + splice=splice, + verify_retained_media=plan.verify_retained_media, + ) + if splice is not None or restored is not None: + # Successful media capture already remapped these ranges. The remap + # reads the template's ids, so it runs before they are replaced. + if media is None and (placeholders := engine_prompt.get("mm_placeholders")): + engine_prompt["mm_placeholders"] = remap_multimodal_placeholders( + template_token_ids=engine_prompt["prompt_token_ids"], + final_token_ids=prompt_token_ids, + mm_placeholders=placeholders, + splice=splice, + ) + engine_prompt["prompt_token_ids"] = prompt_token_ids + host._begin_request_capture( + request, + prompt_token_ids, + admission=admission, + prefix_token_ids=plan.begin_call_prefix_token_ids, + media=media, + generation_cut=None if restored is None else restored.saved, + resumed_generation_token_ids=( + None if restored is None else restored.resumed_generation_token_ids + ), + ) + if terminal is not None: + # The caller's task also submits the call: the engine client answers it. + serve_restored_terminal( + request, engine_prompt, terminal, admission, max_model_len=max_model_len + ) + + +async def save_capture(host: Any, request: Any, content: dict) -> dict: + """Stage a finished call and ride its coords on ``content``. + + A no-op without capture. Off the loop: the sink write is a blocking TQ + round trip. + """ + try: + return await asyncio.to_thread(host._finish_request_capture, request, content) + except asyncio.CancelledError: + # Queued executor work may never start after cancellation. + # If staging already claimed the capture, abort is a no-op. + await abort_request_capture(host, request, "client_disconnect") + raise + + +@dataclass(frozen=True) +class ErrorReply: + """How a call that failed before its reply is answered, on every path.""" + + status: int + # The OpenAI error body. + message: str + error_type: str + param: Optional[str] + code: Any + # What the call's capture is failed with. + abort_reason: str + # Whether Gym is told its capture call was refused before it was staged. + acknowledge: bool + + +def error_reply(error: Exception) -> Optional[ErrorReply]: + """The answer to a call that raised ``error``; None for a server error (500).""" + message = str(error) + # Gym detects a context overflow by this text. + overflow = "maximum context length" in message + if isinstance(error, MediaCaptureRejected): + # A stable code, so Gym and the worker log can tell a retained-media + # re-tile from an engine error. + return ErrorReply( + status=400, + message=message, + error_type="invalid_request_error", + param="messages", + code=error.code, + abort_reason=error.code, + acknowledge=False, + ) + if isinstance(error, OffPolicySamplingError): + return ErrorReply( + status=400, + message=message, + error_type="invalid_request_error", + param="sampling", + code=400, + abort_reason="invalid_request", + acknowledge=False, + ) + if isinstance(error, VLLMValidationError): + return ErrorReply( + status=400, + message=message, + error_type="invalid_request_error", + param=error.parameter, + code=400, + abort_reason="context_length" if overflow else "invalid_request", + acknowledge=True, + ) + if isinstance(error, (VLLMClientError, GracefulHTTPError)): + # vLLM's own status: 503 for a full queue, so Gym retries elsewhere. + info = create_error_response(error).error + return ErrorReply( + status=info.code, + message=info.message, + error_type=info.type, + param=info.param, + code=info.code, + abort_reason=( + "invalid_request" + if isinstance(error, VLLMClientError) + else "engine_error" + ), + acknowledge=False, + ) + if isinstance(error, ValueError) and overflow: + # vLLM's get_max_tokens refuses an over-long prompt with a plain + # ValueError ("Input length (N) exceeds model's maximum context length + # (M)."); only that known refusal, before generation, settles capture. + return ErrorReply( + status=400, + message=message, + error_type="invalid_request_error", + param="input_tokens", + code=400, + abort_reason="context_length", + acknowledge=message.startswith("Input length (") + and "exceeds model's maximum context length" in message, + ) + return None + + +async def abort_capture( + host: Any, request: Any, error: BaseException +) -> Optional[ErrorReply]: + """Fail the capture of a call that raised ``error``; say how to answer it. + + Returns None when the call fails as a server error or a disconnect: the + caller re-raises. A no-op on capture once the call was staged. + """ + if isinstance(error, asyncio.CancelledError): + await abort_request_capture(host, request, "client_disconnect") + return None + failure = error_reply(error) if isinstance(error, Exception) else None + await abort_request_capture( + host, request, "engine_error" if failure is None else failure.abort_reason + ) + if failure is not None and failure.status < 500: + LOGGER.warning( + "Refused a chat call before it ran (%s): %s", + failure.abort_reason, + failure.message, + ) + return failure diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index d32ceea1866..9ded53c9cf1 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -531,18 +531,23 @@ def on_policy_sampling( presence_penalty: Optional[float], frequency_penalty: Optional[float], logit_bias: Optional[Mapping[Any, float]], + n: Optional[int], + logprobs: Optional[bool], + top_logprobs: Optional[int], generation_config: Any, ) -> dict[str, Any]: """The sampling a NeMo-Gym chat call is served with, on every serving path. The request must match the train or validation profile - (:func:`check_sampling_param`) and set no sampling the trainer does not - replay. Floats are compared at f32: Dynamo's frontend carries them as f32, - so 0.7 arrives as 0.69999998. + (:func:`check_sampling_param`), set no sampling the trainer does not + replay, sample one choice, and set ``top_logprobs`` when it asks for + token information. Floats are compared at f32: Dynamo's frontend carries + them as f32, so 0.7 arrives as 0.69999998. Args: temperature, top_p, top_k, min_p, repetition_penalty, presence_penalty, - frequency_penalty, logit_bias: The request's values. + frequency_penalty, logit_bias, n, logprobs, top_logprobs: The + request's values; None (or missing) when it did not set them. generation_config: ``policy.generation``. Returns: @@ -582,6 +587,16 @@ def on_policy_sampling( ) if logit_bias: raise OffPolicySamplingError("request logit_bias is not supported") + if (n or 1) != 1: + # Capture stages one choice per call, and the trainer replays one. + raise OffPolicySamplingError( + f"request n={n} is not supported; NeMo-Gym calls take n=1" + ) + if logprobs and top_logprobs is None: + raise OffPolicySamplingError( + "`top_logprobs` must be set when requesting token information from " + "NeMo RL's chat endpoint" + ) is_train = (_f32(temperature), _f32(top_p)) == ( profile["temperature"], profile["top_p"], diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 70d34d55f69..ed10b83ee75 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -964,8 +964,8 @@ def _begin_request_capture( ) -> None: """Admit one ledger-forwarded call into the capture layer. - Called from preprocess_chat once the exact engine prompt is known - (post-splice in token-in mode, full render in text mode). No-op + Called from ``prompt_and_capture.start_capture`` once the exact engine prompt + is known (post-splice in token-in mode, full render in text mode). No-op unless capture is installed and the request carries the ledger's ``ng_capture`` context. @@ -1145,18 +1145,6 @@ def _resolve_admission_prefix(self, admission: Any) -> list[int]: return resolve_admission_prefix(admission, self._chain_prefix) - def _enter_request_prefix(self, request: Any, prefix_token_ids: list[int]) -> None: - """Attach the resolved prefix to the request through the capture adapter. - - ``VLLMCaptureAdapter.enter_prefix`` writes the engine-native field - (``required_prefix_token_ids``) into a payload; the same fields are - applied to the pydantic request so the existing prefix-splice branch - of preprocess_chat handles staged and inline prefixes alike. - """ - adapter = self.token_capture.adapter - for field_name, value in adapter.enter_prefix({}, prefix_token_ids).items(): - setattr(request, field_name, value) - @staticmethod def _delta_align_routed_experts( payload: dict[str, Any], *, prev_len: int, prompt_len: int, generated_len: int diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index dedd015cccc..a214bcf8a0e 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -59,6 +59,11 @@ TOKEN_IN = {"mode": "token_in", "capture_key": "rollout-1", "model_call_id": "c2"} +def _admission(fields: dict) -> SimpleNamespace: + """A parsed capture admission, as the render pool returns it.""" + return SimpleNamespace(**fields, model_dump=lambda mode="python": dict(fields)) + + def _rendered(**fields) -> RenderedCall: """What a render-pool worker returns for a call.""" return RenderedCall( @@ -147,8 +152,8 @@ class _Capture: def __init__(self) -> None: self.resolved: list[dict] = [] - def resolve_prefix(self, serving_admission: dict) -> list[int]: - self.resolved.append(serving_admission) + def resolve_prefix(self, serving_admission) -> list[int]: + self.resolved.append(serving_admission.model_dump(mode="json")) return PREFIX @@ -257,19 +262,22 @@ def test_a_gym_call_is_rendered_spliced_and_routed_with_its_capture_plan() -> No assert routed["token_ids"] == SPLICED assert routed["stop_conditions"] == {"max_tokens": MAX_MODEL_LEN - len(SPLICED)} assert routed["annotations"] == [chat_engine.GYM_CALL_ANNOTATION] - plan = routed["extra_args"][chat_engine.PLAN_KEY] - assert plan["request"] == { + wire = routed["extra_args"][chat_engine.PLAN_KEY] + plan = wire["prepared_prompt"] + # The request as rendered; the engine sets the plan's clamped budget. + assert wire["request"] == { "model": "model", - "max_tokens": MAX_MODEL_LEN - len(SPLICED), + "max_tokens": 20, "max_completion_tokens": None, } + assert plan["max_tokens"] == MAX_MODEL_LEN - len(SPLICED) assert (plan["template_token_ids"], plan["splice"], plan["prefix_len"]) == ( TEMPLATE, # Each side is cut at its turn end: PREFIX[:3] + TEMPLATE[2:]. [3, 2], len(PREFIX), ) - assert (plan["admission"], plan["multimodal"], plan["reasoning_ended"]) == ( + assert (plan["admission"], wire["multimodal"], wire["reasoning_ended"]) == ( None, None, False, @@ -290,8 +298,8 @@ def test_a_token_in_call_splices_the_prefix_staged_in_tq() -> None: _rendered( template_token_ids=TEMPLATE, history_token_ids=HISTORY, - admission=TOKEN_IN, - serving_admission=TOKEN_IN, + admission=_admission(TOKEN_IN), + serving_admission=_admission(TOKEN_IN), ) ) router = _Router() @@ -303,7 +311,7 @@ def test_a_token_in_call_splices_the_prefix_staged_in_tq() -> None: assert capture.resolved == [TOKEN_IN] (routed,) = router.requests assert routed["token_ids"] == SPLICED - plan = routed["extra_args"][chat_engine.PLAN_KEY] + plan = routed["extra_args"][chat_engine.PLAN_KEY]["prepared_prompt"] assert (plan["admission"], plan["serving_admission"], plan["prefix_len"]) == ( TOKEN_IN, TOKEN_IN, diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index 2e0a5383145..98b07cb02e2 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -53,7 +53,7 @@ GYM_CALL_ANNOTATION, PLAN_KEY, RUNTIME_DATA_KEY, - GymCallPlan, + ForwardedCall, ) from nemo_rl.models.generation.interfaces import SamplerReplayConfig # noqa: E402 from nemo_rl.models.generation.rollout_result import ( # noqa: E402 @@ -465,25 +465,39 @@ def _chat(**fields) -> dict: ).model_dump(mode="json", exclude={"messages"}) -def _gym_call(token_ids: list[int], **plan) -> dict: - """A Gym call as the chat engine routes it: final token ids and the plan.""" - full_plan: GymCallPlan = { - "request": _chat(), +def _gym_call(token_ids: list[int], **fields) -> dict: + """A Gym call as the chat engine routes it: final token ids and the plan. + + ``fields`` set the prepared prompt's fields and the call's others by name. + """ + plan = { + "template_token_ids": token_ids, + "splice": None, + "prefix_len": None, + "measure_only": False, + "request_max_tokens": None, + "max_tokens": None, "admission": None, "serving_admission": None, "verify_retained_media": False, - "prefix_len": None, - "template_token_ids": token_ids, - "splice": None, + } + wire: ForwardedCall = { + "prepared_prompt": plan, + "request": _chat(), "multimodal": None, "reasoning_ended": None, "reasoning_parser_kwargs": None, - **plan, } + for name, value in fields.items(): + if name in plan: + plan[name] = value + else: + assert name in wire, name + wire[name] = value return { "token_ids": token_ids, "annotations": [GYM_CALL_ANNOTATION], - "extra_args": {PLAN_KEY: full_plan}, + "extra_args": {PLAN_KEY: wire}, } diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index bfa3f7bc93f..2e07bd2105b 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -65,6 +65,7 @@ from transformers import AutoConfig, AutoProcessor, AutoTokenizer # noqa: E402 from vllm.config import CacheConfig, LoadConfig, VllmConfig # noqa: E402 from vllm.engine.arg_utils import AsyncEngineArgs # noqa: E402 +from vllm.exceptions import QueueOverflowError, VLLMValidationError # noqa: E402 from vllm.logprobs import Logprob # noqa: E402 from vllm.outputs import CompletionOutput, RequestOutput # noqa: E402 from vllm.renderers.registry import renderer_from_config # noqa: E402 @@ -228,9 +229,13 @@ def turn_1_tokens(self) -> list[int]: class _Engine: - """vLLM's engine client: answers every call with REPLY and records its runs.""" + """vLLM's engine client: answers every call with REPLY and records its runs. - def __init__(self, kit: _Kit) -> None: + ``refuse`` is what the engine refuses every call with, and where: its + queue check ("admission") or its input check ("generate"). + """ + + def __init__(self, kit: _Kit, refuse: Optional[tuple[str, Any]] = None) -> None: self.model_config = kit.model_config self.renderer = kit.renderer self.vllm_config = kit.vllm_config @@ -238,12 +243,15 @@ def __init__(self, kit: _Kit) -> None: self.io_processor = None self.errored = False self._kit = kit + self._refuse = refuse self.runs: list[tuple[list[int], Any, Any]] = [] # How many of REPLY's tokens the prompt already ends with: a call # resumed from a checkpoint cut generates only the rest. self.resumed = 0 async def generate(self, prompt, sampling_params, request_id, **kwargs): + if self._refuse is not None and self._refuse[0] == "generate": + raise self._refuse[1] prompt_token_ids = list(prompt["prompt_token_ids"]) self.runs.append((prompt_token_ids, sampling_params, _media(prompt))) terminal = restored_terminal() @@ -286,7 +294,8 @@ async def generate(self, prompt, sampling_params, request_id, **kwargs): ) def check_admission(self, n, request_id=None) -> None: - pass + if self._refuse is not None and self._refuse[0] == "admission": + raise self._refuse[1] async def is_tracing_enabled(self) -> bool: return False @@ -357,11 +366,16 @@ class _Outcome: failed: list[str] = field(default_factory=list) -def _host(kit: _Kit, deltas: dict[str, list[int]], top_logprobs: int): +def _host( + kit: _Kit, + deltas: dict[str, list[int]], + top_logprobs: int, + refuse: Optional[tuple[str, Any]] = None, +): host = object.__new__(VllmAsyncGenerationWorkerImpl) host._init_token_capture_state() host.cfg = kit.generation - host.llm = host._http_engine_client = _Engine(kit) + host.llm = host._http_engine_client = _Engine(kit, refuse) host.llm_async_engine_args = kit.engine_args host.model_name = "model" host.routed_experts_dtype = torch.int16 @@ -637,11 +651,15 @@ class _Case: fields: dict[str, Any] = field(default_factory=dict) # The sampler top-k capture stages, for score centering. top_logprobs: int = 0 - # 200, or the 400 both paths refuse the call with before it runs. + # 200, or the status both paths refuse the call with before it runs. status: int = 200 # The mode capture stages the call in; None when the call is not captured. mode: Optional[str] = None kit: str = "qwen3" + # What the engine refuses the call with, and where (see _Engine). + refuse: Optional[tuple[str, Any]] = None + # What a refused call's capture is failed with, on both paths. + failed: tuple[str, ...] = () def _off_policy(name: str, value: Any) -> _Case: @@ -777,6 +795,28 @@ def _off_policy(name: str, value: Any) -> _Case: _off_policy("presence_penalty", 0.5), _off_policy("frequency_penalty", 0.5), _off_policy("logit_bias", {"10": 5.0}), + # Capture stages one choice, and the trainer replays one. + _Case("n_2", fields={"n": 2}, status=400), + # Token information without top_logprobs, unset or sent as null. + _Case("top_logprobs_unset", fields={"top_logprobs": None}, status=400), + # A full queue: vLLM's 503, so Gym retries elsewhere; checked before capture. + _Case( + "queue_full", + extras=lambda kit: {"ng_capture": _admission(mode="text")}, + status=503, + refuse=("admission", QueueOverflowError()), + ), + # The engine's input check refuses a captured call after it began. + _Case( + "token_out_of_vocabulary", + extras=lambda kit: {"ng_capture": _admission(mode="text")}, + status=400, + refuse=( + "generate", + VLLMValidationError("Token id 999999 is out of vocabulary"), + ), + failed=("invalid_request",), + ), ] @@ -801,10 +841,13 @@ def test_a_gym_call_takes_the_same_steps_through_dynamo_as_natively(kits, case) async def both(): native = await _native( - kit, *_host(kit, deltas, case.top_logprobs), body, measure=False + kit, + *_host(kit, deltas, case.top_logprobs, case.refuse), + body, + measure=False, ) dynamo = await _dynamo( - kit, *_host(kit, deltas, case.top_logprobs), body, extras + kit, *_host(kit, deltas, case.top_logprobs, case.refuse), body, extras ) return native, dynamo @@ -816,6 +859,9 @@ async def both(): ) if case.status != 200: assert native.runs == dynamo.runs == [] + # Capture is failed alike, and neither path staged the call. + assert native.failed == dynamo.failed == list(case.failed) + assert native.staged == dynamo.staged == [] return # The engine ran the same prompt, media and SamplingParams. ((native_prompt, native_params, native_media),) = native.runs diff --git a/tests/unit/models/generation/test_vllm_chat_template_wiring.py b/tests/unit/models/generation/test_vllm_chat_template_wiring.py index 6ef83e26f54..331492e3b9a 100644 --- a/tests/unit/models/generation/test_vllm_chat_template_wiring.py +++ b/tests/unit/models/generation/test_vllm_chat_template_wiring.py @@ -54,6 +54,7 @@ # be recorded somewhere the assertions never look. _BUILT: dict[str, list] = {"renderer": [], "chat": [], "tokenize": []} _CHAT_SERVER = "nemo_rl.models.generation.vllm.chat_server" +_PROMPT_AND_CAPTURE = "nemo_rl.models.generation.vllm.prompt_and_capture" def _recorder(slot: str): @@ -113,9 +114,13 @@ def _install_fake_vllm(monkeypatch): # chat_server imports vLLM at module level: drop the cached copy (and the # package attribute an earlier import set) so the server is built against # these fakes; monkeypatch restores both afterwards. - monkeypatch.setitem(sys.modules, _CHAT_SERVER, None) - del sys.modules[_CHAT_SERVER] - monkeypatch.delattr(vllm_generation_pkg, "chat_server", raising=False) + for module_name, attribute in ( + (_CHAT_SERVER, "chat_server"), + (_PROMPT_AND_CAPTURE, "prompt_and_capture"), + ): + monkeypatch.setitem(sys.modules, module_name, None) + del sys.modules[module_name] + monkeypatch.delattr(vllm_generation_pkg, attribute, raising=False) for name in ( "vllm", "vllm.entrypoints", @@ -125,6 +130,7 @@ def _install_fake_vllm(monkeypatch): "vllm.entrypoints.openai.models", "vllm.entrypoints.serve", "vllm.entrypoints.serve.engine", + "vllm.entrypoints.serve.exception_handling", "vllm.entrypoints.serve.tokenize", "vllm.entrypoints.serve.utils", "vllm.reasoning", @@ -201,7 +207,16 @@ class ValidationError(ValueError): def __init__(self, message, **kwargs): super().__init__(message) - module("vllm.exceptions", VLLMValidationError=ValidationError) + module( + "vllm.exceptions", + VLLMValidationError=ValidationError, + VLLMClientError=placeholder("VLLMClientError"), + GracefulHTTPError=placeholder("GracefulHTTPError"), + ) + module( + "vllm.entrypoints.serve.exception_handling.error_response", + create_error_response=MagicMock(), + ) module( "vllm.sampling_params", StructuredOutputsParams=placeholder("StructuredOutputsParams"), @@ -328,6 +343,7 @@ async def test_captured_chat_logprobs_bypass( top_logprobs=0, # NeMoRLChatCompletionRequest's defaults for a call with no restored prefix. _restored_generation_token_ids=(), + _restored_prompt_token_ids=(), _restored_prefix_terminal=None, return_tokens_as_token_ids=token_ids, ) @@ -433,6 +449,7 @@ async def test_resumed_call_reports_usage_as_the_uninterrupted_call(monkeypatch) top_logprobs=0, # Prompt [1, 2, 3]; the call resumed from generated prefix [4, 5]. _restored_generation_token_ids=(4, 5), + _restored_prompt_token_ids=(), _restored_prefix_terminal=None, return_tokens_as_token_ids=False, ) @@ -637,6 +654,7 @@ def install_with_real_response_type(monkeypatch): for request in requests: # NeMoRLChatCompletionRequest's defaults for a call with no restored prefix. request._restored_generation_token_ids = () + request._restored_prompt_token_ids = () request._restored_prefix_terminal = None _, chats, _ = _build_server(monkeypatch, {}, capture_requests=[requests[0]]) chat = chats[0] @@ -786,13 +804,6 @@ def capture( "_resolve_admission_prefix", lambda self, admission: [10, 77, 31, 2], ) - monkeypatch.setattr( - VllmAsyncGenerationWorkerImpl, - "_enter_request_prefix", - lambda self, request, prefix: setattr( - request, "required_prefix_token_ids", prefix - ), - ) monkeypatch.setattr( VllmAsyncGenerationWorkerImpl, "_capture_request_media", capture ) @@ -802,6 +813,7 @@ def capture( renderers, _, _ = _build_server(monkeypatch, {}) renderer = renderers[0] renderer.renderer = SimpleNamespace(tokenizer=SimpleNamespace(eos_token_id=2)) + renderer.model_config = SimpleNamespace(max_model_len=64) request = SimpleNamespace(required_prefix_token_ids=[10, 77, 31, 2]) request.model_copy = lambda **kwargs: request result = asyncio.run( @@ -843,7 +855,14 @@ async def preprocess(self, *, messages, **kwargs): VllmAsyncGenerationWorkerImpl, "_capture_media", False, raising=False ) monkeypatch.setattr( - VllmAsyncGenerationWorkerImpl, "_capture_admission", lambda self, request: None + VllmAsyncGenerationWorkerImpl, + "_capture_admission", + lambda self, request: SimpleNamespace( + mode="text", request_replay=None, generation_cut=None + ), + ) + monkeypatch.setattr( + VllmAsyncGenerationWorkerImpl, "_begin_request_capture", MagicMock() ) capture = MagicMock( side_effect=ValueError("retiled media") if failure_stage == "capture" else None, @@ -856,7 +875,7 @@ async def preprocess(self, *, messages, **kwargs): renderers, _, _ = _build_server(monkeypatch, {}, app=app) # The server module was just imported against the fakes; patch that copy. monkeypatch.setattr( - sys.modules[_CHAT_SERVER], "remap_multimodal_placeholders", remap + sys.modules[_PROMPT_AND_CAPTURE], "remap_multimodal_placeholders", remap ) renderer = renderers[0] renderer.renderer = SimpleNamespace(tokenizer=SimpleNamespace(eos_token_id=2)) @@ -941,13 +960,6 @@ async def preprocess(self, *, messages, **kwargs): SimpleNamespace(token_ids_delta=delta, token_mask_delta=mask), ), ) - monkeypatch.setattr( - VllmAsyncGenerationWorkerImpl, - "_enter_request_prefix", - lambda self, request, prefix: setattr( - request, "required_prefix_token_ids", prefix - ), - ) begin = MagicMock() monkeypatch.setattr(VllmAsyncGenerationWorkerImpl, "_begin_request_capture", begin) renderers, _, _ = _build_server(monkeypatch, {}, app=app) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 416b33b83e1..1d4c6d1881e 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -988,12 +988,13 @@ def skip_fp8_known_failures() -> None: def _install_fake_vllm_openai_modules(monkeypatch): # chat_server imports vLLM at module level: drop the cached copy so the # server is built against these fakes, and restore it afterwards. - chat_server = "nemo_rl.models.generation.vllm.chat_server" - monkeypatch.setitem(sys.modules, chat_server, None) - del sys.modules[chat_server] - monkeypatch.delattr( - sys.modules["nemo_rl.models.generation.vllm"], "chat_server", raising=False - ) + for name in ("chat_server", "prompt_and_capture"): + module_name = f"nemo_rl.models.generation.vllm.{name}" + monkeypatch.setitem(sys.modules, module_name, None) + del sys.modules[module_name] + monkeypatch.delattr( + sys.modules["nemo_rl.models.generation.vllm"], name, raising=False + ) for module_name in ( "vllm", "vllm.entrypoints", @@ -1003,6 +1004,7 @@ def _install_fake_vllm_openai_modules(monkeypatch): "vllm.entrypoints.openai.models", "vllm.entrypoints.serve", "vllm.entrypoints.serve.engine", + "vllm.entrypoints.serve.exception_handling", "vllm.entrypoints.serve.tokenize", "vllm.entrypoints.serve.utils", "vllm.reasoning", @@ -1174,7 +1176,16 @@ def with_cancellation(handler): "vllm.entrypoints.serve.tokenize.serving", ServingTokenization=ServingTokenization, ) - make_module("vllm.exceptions", VLLMValidationError=VLLMValidationError) + make_module( + "vllm.exceptions", + VLLMValidationError=VLLMValidationError, + VLLMClientError=type("VLLMClientError", (Exception,), {}), + GracefulHTTPError=type("GracefulHTTPError", (Exception,), {}), + ) + make_module( + "vllm.entrypoints.serve.exception_handling.error_response", + create_error_response=MagicMock(), + ) make_module("vllm.sampling_params", StructuredOutputsParams=MagicMock()) make_module( "vllm.reasoning.abs_reasoning_parsers", @@ -1353,14 +1364,18 @@ def _fake_chat_request(): temperature=1.0, max_tokens=1, max_completion_tokens=None, + n=1, # Read by NeMoRLOpenAIServingChatMixin.chat_completion_full_generator. stream=False, return_tokens_as_token_ids=None, logprobs=False, top_logprobs=None, + # The fields the client set, as pydantic records them. + model_fields_set=set(), # NeMoRLChatCompletionRequest's private defaults for a call that # restores no generation prefix. _restored_generation_token_ids=(), + _restored_prompt_token_ids=(), _restored_prefix_terminal=None, ) @@ -1478,16 +1493,18 @@ async def create_chat_completion(request, _raw_request): @pytest.mark.asyncio async def test_context_overflow_returns_http_400_for_nemo_gym(monkeypatch): - """Overflow from _clamp_max_tokens must be HTTP 400 that NeMo Gym recognizes.""" + """Overflow from clamp_max_tokens must be HTTP 400 that NeMo Gym recognizes.""" _, serving_chat, chat_handler = _setup_fake_vllm_chat_handler(monkeypatch) + prompt_and_capture = sys.modules[ + "nemo_rl.models.generation.vllm.prompt_and_capture" + ] max_model_len = 128 - renderer = serving_chat.kwargs["online_renderer"] - renderer.model_config = types.SimpleNamespace(max_model_len=max_model_len) - overflow_prompt = [0] * max_model_len async def create_chat_completion(request, _raw_request): - renderer._clamp_max_tokens(request, request.max_tokens, overflow_prompt) + prompt_and_capture.clamp_max_tokens( + request.max_tokens, max_model_len, max_model_len + ) serving_chat.create_chat_completion = create_chat_completion response = await chat_handler(_fake_chat_request(), MagicMock()) @@ -1520,14 +1537,17 @@ async def create_chat_completion(request, _raw_request): def test_clamp_max_tokens_sets_the_limit_the_request_uses( monkeypatch, max_tokens, max_completion_tokens, expected ): - _, serving_chat, _ = _setup_fake_vllm_chat_handler(monkeypatch) - renderer = serving_chat.kwargs["online_renderer"] - renderer.model_config = types.SimpleNamespace(max_model_len=2048) + _setup_fake_vllm_chat_handler(monkeypatch) + prompt_and_capture = sys.modules[ + "nemo_rl.models.generation.vllm.prompt_and_capture" + ] request = types.SimpleNamespace( max_tokens=max_tokens, max_completion_tokens=max_completion_tokens ) - renderer._clamp_max_tokens(request, 424, [0] * 1024) + prompt_and_capture.set_max_tokens( + request, prompt_and_capture.clamp_max_tokens(424, 1024, 2048) + ) assert vars(request) == expected @@ -1871,6 +1891,7 @@ async def test_chat_completion_cancelled_on_client_disconnect(monkeypatch, logpr request.logprobs = logprobs request.return_tokens_as_token_ids = True request.top_logprobs = 0 + request.model_fields_set = {"top_logprobs"} token_capture, call = _track_request_capture(worker, request) task = asyncio.create_task(chat_handler(request, MagicMock())) try: diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index 094f7506723..3370f247c31 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -2117,9 +2117,12 @@ def replay(tokenizer, **kwargs): ) # vLLM's processed tail: EOS and any matched stop string already stripped. output = SimpleNamespace(token_ids=[14, 151645], text=" tail") - resumed_worker._prefix_recovery.restore_reply_text( + resumed = resumed_worker._capture_registry.get(request) + restore_reply_text( request, SimpleNamespace(outputs=[output]), + prefix_token_ids=resumed.resumed_generation_token_ids, + engine_prompt_token_ids=resumed.prompt_token_ids, tokenizer=_ByteTokenizer(), ) (replay_kwargs,) = replays @@ -2131,7 +2134,6 @@ def replay(tokenizer, **kwargs): # A second checkpoint taken before the resumed call finishes must retain # the original cut key and append the newly generated tail. Gym v2 stores # this complete key list in the replacement attempt's model record. - resumed = resumed_worker._capture_registry.get(request) resumed.record([14], [-0.3]) second_receipt = _cuts(resumed_worker).checkpoint_cut( _single_call_inventory("c2", checkpoint_id="checkpoint-2", attempt=1) @@ -2250,7 +2252,7 @@ def _staging_chain_request(prev_len: int = 3) -> _FakeRequest: def test_staging_chain_prefix_flows_through_adapter_and_begin_call(): - """The resolved prefix reaches both the engine request and capture admission.""" + """The prefix resolved from the staging chain is the one begin_call checks.""" sink = _MemorySink() worker = _worker_with_capture(sink) source = _MemoryPrefixSource({"r0/c1": [10, 11], "r0/c2": [12]}) @@ -2260,7 +2262,6 @@ def test_staging_chain_prefix_flows_through_adapter_and_begin_call(): admission = worker._capture_admission(request) prefix = worker._resolve_admission_prefix(admission) - worker._enter_request_prefix(request, prefix) VllmAsyncGenerationWorkerImpl._begin_request_capture( worker, request, @@ -2274,8 +2275,6 @@ def test_staging_chain_prefix_flows_through_adapter_and_begin_call(): # Never patched back into the wire context. assert request.ng_capture == context_before assert admission.required_prefix_token_ids == [] - # enter_prefix is the production writer of the request field. - assert request.required_prefix_token_ids == prefix state = worker._capture_registry.get(request) assert state.call.prefix_token_ids == prefix assert state.prompt_token_ids == [10, 11, 12, 20] From 7b536f56f77b1b10c9909481cc2626c9c1b3c9ec Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 16:38:35 -0800 Subject: [PATCH 61/82] fix(gym): run the shared chat-call steps under vLLM and Dynamo The sketch had only run against a fake vLLM. Under real vLLM and the Dynamo tests: - ResumedCallReplyMixin iterated its own generator: the wrapper read result_generator after it was rebound to the wrapper, so every resumed reply raised. It reads the engine's results now. - RenderedPrompt's members are read-only, so each path's frozen render dataclass type-checks against it, and prompt_and_capture.py joins pyrefly's list. - Import order. Three comment blocks are trimmed to two lines, and two reasons the move dropped are back: a restored prompt is never truncated again, and MediaCaptureRejected is checked first because it is a ValueError. - test_dynamo_render_pool.py patches the tokenizer the shared reply step reads. Co-authored-by: Terry Kong Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- .../models/generation/dynamo/vllm/adapter.py | 9 ++-- .../generation/dynamo/vllm/render_pool.py | 2 +- nemo_rl/models/generation/vllm/chat_server.py | 20 ++++----- .../generation/vllm/prompt_and_capture.py | 45 ++++++++++++------- pyrefly.toml | 1 + .../generation/test_dynamo_render_pool.py | 2 + 6 files changed, 48 insertions(+), 31 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index cc9cdbc2802..bf1eacf70ce 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -80,10 +80,10 @@ ) from nemo_rl.models.generation.vllm.prompt_and_capture import ( PreparedPrompt, - abort_request_capture, - start_capture, abort_capture, + abort_request_capture, save_capture, + start_capture, ) from nemo_rl.models.generation.vllm.utils import ( chat_generation_tokens, @@ -369,9 +369,8 @@ async def _serve_gym_call( chat, engine_prompt, wire, engine_client, context ) if plan.admission is not None and context.is_stopped(): - # Dynamo aborts the vLLM request on a disconnect but never - # cancels this generator: fail the capture, as the native - # endpoint's cancellation does, instead of staging it. + # Dynamo aborts the vLLM request on a disconnect but never cancels + # this generator: fail the capture, as natively, instead of staging it. await abort_request_capture(host, chat, "client_disconnect") return None completion = final_output.outputs[0] diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index 6e4a2f9e77e..1acfecc1d90 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -210,11 +210,11 @@ def _parser(self, chat: Any) -> Any: ) def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall: - from nemo_rl.models.generation.vllm.prompt_and_capture import set_max_tokens from nemo_rl.models.generation.vllm.chat_server import ( NeMoRLChatCompletionRequest, NeMoRLMeasureChatRequest, ) + from nemo_rl.models.generation.vllm.prompt_and_capture import set_max_tokens request_cls = ( NeMoRLMeasureChatRequest if measure else NeMoRLChatCompletionRequest diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index 1076df5c1e4..807f65492c5 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -73,16 +73,16 @@ from nemo_rl.models.generation.vllm.captured_chat_logprobs import ( CapturedChatLogprobsMixin, ) +from nemo_rl.models.generation.vllm.generation_cut_route import ( + register_generation_cut_route, +) from nemo_rl.models.generation.vllm.prompt_and_capture import ( - abort_request_capture, - start_capture, abort_capture, - save_capture, + abort_request_capture, prepare_prompt, + save_capture, set_max_tokens, -) -from nemo_rl.models.generation.vllm.generation_cut_route import ( - register_generation_cut_route, + start_capture, ) from nemo_rl.models.generation.vllm.utils import ( attach_routed_experts_to_chat_response_choices, @@ -445,10 +445,11 @@ async def chat_completion_full_generator( **kwargs, ): restored_token_ids = request._restored_generation_token_ids + engine_results = result_generator async def restored_results(): final_res = None - async for res in result_generator: + async for res in engine_results: final_res = res if final_res is None: return @@ -818,9 +819,8 @@ async def create_chat_completion( request, raw_request ) except BaseException as error: - # A cancel before generation starts propagates here directly; one - # that lands mid-generation is swallowed by vLLM and re-raised by - # NeMoRLOpenAIServingChatMixin.chat_completion_full_generator. + # A cancel before generation propagates here directly; vLLM swallows a + # later one, and NeMoRLOpenAIServingChatMixin's full generator re-raises it. failure = await abort_capture(host, request, error) if failure is None: raise diff --git a/nemo_rl/models/generation/vllm/prompt_and_capture.py b/nemo_rl/models/generation/vllm/prompt_and_capture.py index 53bd5286b88..8d0d50eb88f 100644 --- a/nemo_rl/models/generation/vllm/prompt_and_capture.py +++ b/nemo_rl/models/generation/vllm/prompt_and_capture.py @@ -229,21 +229,37 @@ async def abort_request_capture(host: Any, request: Any, reason: str) -> None: class RenderedPrompt(Protocol): - """What :func:`prepare_prompt` reads from a call's render; each path's render has it.""" + """What :func:`prepare_prompt` reads from a call's render. + + Read-only members, so each path's frozen render dataclass fits. + """ # The full render's token ids, in the template's coordinates. - template_token_ids: list[int] + @property + def template_token_ids(self) -> list[int]: ... + # The render up to the last assistant turn, when a prefix will be spliced. - history_token_ids: Optional[list[int]] + @property + def history_token_ids(self) -> Optional[list[int]]: ... + # The request's own prefix (Gym's echo) after the capture decision. - required_prefix_token_ids: Optional[list[int]] + @property + def required_prefix_token_ids(self) -> Optional[list[int]]: ... + # Capture admissions: what is stored, and how the prompt is served; None # without capture. - admission: Any - serving_admission: Any - verify_retained_media: bool + @property + def admission(self) -> Any: ... + + @property + def serving_admission(self) -> Any: ... + + @property + def verify_retained_media(self) -> bool: ... + # The output budget the request asked for; None when it set none. - request_max_tokens: Optional[int] + @property + def request_max_tokens(self) -> Optional[int]: ... @dataclass(frozen=True) @@ -438,8 +454,8 @@ async def start_capture( admission = resolved_admission serving_admission = resolved_admission if restored is not None: - # The saved rows are the exact engine prompt at the cut; splicing them - # again could add or drop tokens (test_vllm_progress_read_layout.py). + # The saved rows are the exact engine prompt at the cut: never spliced or + # truncated again (test_vllm_progress_read_layout.py). splice = None prompt_token_ids = restored.prompt_token_ids terminal = classify_restored_terminal( @@ -536,8 +552,8 @@ def error_reply(error: Exception) -> Optional[ErrorReply]: # Gym detects a context overflow by this text. overflow = "maximum context length" in message if isinstance(error, MediaCaptureRejected): - # A stable code, so Gym and the worker log can tell a retained-media - # re-tile from an engine error. + # First, since it is a ValueError too. A stable code lets Gym and the + # worker log tell a retained-media re-tile from an engine error. return ErrorReply( status=400, message=message, @@ -584,9 +600,8 @@ def error_reply(error: Exception) -> Optional[ErrorReply]: acknowledge=False, ) if isinstance(error, ValueError) and overflow: - # vLLM's get_max_tokens refuses an over-long prompt with a plain - # ValueError ("Input length (N) exceeds model's maximum context length - # (M)."); only that known refusal, before generation, settles capture. + # vLLM's get_max_tokens refuses an over-long prompt with a plain ValueError + # ("Input length (N) exceeds ..."); only that refusal settles capture. return ErrorReply( status=400, message=message, diff --git a/pyrefly.toml b/pyrefly.toml index c5e55c48995..332347b63d9 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -316,6 +316,7 @@ project-includes = [ "nemo_rl/models/generation/vllm/metric_names.py", "nemo_rl/models/generation/vllm/patches.py", "nemo_rl/models/generation/vllm/progress_reader.py", + "nemo_rl/models/generation/vllm/prompt_and_capture.py", "nemo_rl/models/generation/vllm/quantization/deepseek_v4_fp8.py", "nemo_rl/models/generation/vllm/quantization/fp8_train_utils.py", "nemo_rl/models/generation/vllm/quantization/mxfp8_utils.py", diff --git a/tests/unit/models/generation/test_dynamo_render_pool.py b/tests/unit/models/generation/test_dynamo_render_pool.py index f95a0e63b90..8728e6c8ddd 100644 --- a/tests/unit/models/generation/test_dynamo_render_pool.py +++ b/tests/unit/models/generation/test_dynamo_render_pool.py @@ -273,6 +273,8 @@ def test_a_resumed_calls_reply_matches_the_uninterrupted_reply( tokenizer = _tokenizer(style) parser = _RecordingParser() monkeypatch.setattr(reply_worker, "tokenizer", tokenizer) + # The serving chat's reply step decodes the saved tokens with its renderer's. + monkeypatch.setattr(reply_worker.serving_chat.renderer, "tokenizer", tokenizer) monkeypatch.setattr(reply_worker, "_parser", lambda chat: parser) prompt = _ids(tokenizer, case.prompt) output = _ids(tokenizer, case.output) From f3d143b8a3aa2539965e46ecb547a9f9f8ff0d7b Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 16:53:36 -0800 Subject: [PATCH 62/82] refactor(gym): trim the shared chat-call steps - A text call sends its prompt once. The template's token ids, which only a media call's placeholder remap reads, ride with the media (ForwardedMedia) instead of in every PreparedPrompt. - resolve_restored_call returns before its thread hop when the admission has no generation_cut: only a resumed call has rows to read. - The adapter builds the staged message in a thread only when score centering's top-k is on, as the native server does. - prepare_prompt takes measure_only (it stored "not clamp"), and measure_only stays off the wire: a measure call is answered where it is prepared. - PLAN_KEY is CALL_KEY: it carries a ForwardedCall, not a plan. Co-authored-by: Terry Kong Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- .../models/generation/dynamo/vllm/adapter.py | 22 ++++++++----- .../generation/dynamo/vllm/chat_engine.py | 33 +++++++++++++------ nemo_rl/models/generation/vllm/chat_server.py | 2 +- .../generation/vllm/prompt_and_capture.py | 21 ++++++------ .../generation/test_dynamo_chat_engine.py | 9 ++--- .../generation/test_dynamo_vllm_adapter.py | 13 +++++--- .../test_dynamo_vllm_adapter_capture.py | 1 - 7 files changed, 61 insertions(+), 40 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index bf1eacf70ce..a66057eab62 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -63,8 +63,8 @@ ServedRollout, ) from nemo_rl.models.generation.dynamo.vllm.chat_engine import ( + CALL_KEY, GYM_CALL_ANNOTATION, - PLAN_KEY, RUNTIME_DATA_KEY, ForwardedCall, GymEngineReply, @@ -307,11 +307,11 @@ async def _generate_gym( ) del annotation - wire: Optional[ForwardedCall] = (request.get("extra_args") or {}).get(PLAN_KEY) + wire: Optional[ForwardedCall] = (request.get("extra_args") or {}).get(CALL_KEY) if wire is None: raise InvalidArgument( "NeMo-Gym calls reach NeMo RL's engines through NeMo RL's chat " - f"engine in the Dynamo frontend (extra_args.{PLAN_KEY})." + f"engine in the Dynamo frontend (extra_args.{CALL_KEY})." ) host = self._host plan = PreparedPrompt.from_json( @@ -353,11 +353,12 @@ async def _serve_gym_call( engine_client = host._http_engine_client or self._llm engine_client.check_admission(1, context.id()) engine_prompt: dict[str, Any] - if wire["multimodal"] is not None: + media = wire["multimodal"] + if media is not None: # The processed media, in the template's coordinates. engine_prompt = { - **pickle.loads(base64.b64decode(wire["multimodal"])), - "prompt_token_ids": list(plan.template_token_ids), + **pickle.loads(base64.b64decode(media["inputs"])), + "prompt_token_ids": list(media["template_token_ids"]), } else: engine_prompt = TokensPrompt(prompt_token_ids=plan.prompt_token_ids) @@ -509,9 +510,12 @@ async def _stage(self, chat: Any, final_output: Any) -> GymCallResult: # Deferred: only captured calls reach here, and capture already imports nemo_gym. from nemo_gym.token_id_capture import NG_COMMIT_COORDS_FIELD - # Off the loop, as natively: the sampler top-k scales with the - # completion's length and K. - content = await asyncio.to_thread(self._response_to_stage, final_output) + if self._host._capture_top_logprobs: + # Off the loop, as natively: the sampler top-k scales with the + # completion's length and K. + content = await asyncio.to_thread(self._response_to_stage, final_output) + else: + content = self._response_to_stage(final_output) content = await save_capture(self._host, chat, content) return GymCallResult(ng_commit_coords=content[NG_COMMIT_COORDS_FIELD]) diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index bb023442e6f..b283470fdbe 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -65,13 +65,22 @@ # The key the engine publishes its chat settings under in runtime_data. RUNTIME_DATA_KEY = "nemo_rl_chat" # The annotation the adapter serves NeMo-Gym calls by; the ForwardedCall rides -# in extra_args under PLAN_KEY. +# in extra_args under CALL_KEY. GYM_CALL_ANNOTATION = "nemo_gym:rollout" -PLAN_KEY = "nemo_rl" +CALL_KEY = "nemo_rl" + + +class ForwardedMedia(TypedDict): + """A media call's render, in the template's coordinates.""" + + # The render's multimodal input, pickled then base64-encoded. + inputs: str + # The template's token ids, which the input's placeholders index. + template_token_ids: list[int] class ForwardedCall(TypedDict): - """What the chat engine sends the adapter in ``extra_args[PLAN_KEY]``. + """What the chat engine sends the adapter in ``extra_args[CALL_KEY]``. The call's ``PreparedPrompt.to_json()`` and what the engine needs that the native worker has in process: the rendered request, its media, and what @@ -81,8 +90,8 @@ class ForwardedCall(TypedDict): prepared_prompt: dict[str, Any] # The rendered chat request, without messages. request: dict[str, Any] - # The render's multimodal input, pickled then base64-encoded. - multimodal: Optional[str] + # None for a text call, which sends its prompt once: as the final token ids. + multimodal: Optional[ForwardedMedia] reasoning_ended: Optional[bool] reasoning_parser_kwargs: Optional[dict[str, Any]] @@ -289,7 +298,7 @@ async def _serve( resolve_prefix=self._capture.resolve_prefix, turn_end_token_ids=info["turn_end_token_ids"], max_model_len=info["max_model_len"], - clamp=not extras.ng_measure, + measure_only=extras.ng_measure, ) if extras.ng_measure: return self._chunk( @@ -301,12 +310,16 @@ async def _serve( message={"role": "assistant", "content": ""}, finish_reason="stop", ) + media: Optional[ForwardedMedia] = None + if rendered.multimodal is not None: + media = ForwardedMedia( + inputs=base64.b64encode(rendered.multimodal).decode(), + template_token_ids=rendered.template_token_ids, + ) wire: ForwardedCall = { "prepared_prompt": plan.to_json(), "request": rendered.request, - "multimodal": None - if rendered.multimodal is None - else base64.b64encode(rendered.multimodal).decode(), + "multimodal": media, "reasoning_ended": rendered.reasoning_ended, "reasoning_parser_kwargs": rendered.reasoning_parser_kwargs, } @@ -321,7 +334,7 @@ async def _serve( "output_options": {}, "eos_token_ids": [], "annotations": [GYM_CALL_ANNOTATION], - "extra_args": {PLAN_KEY: wire}, + "extra_args": {CALL_KEY: wire}, } engine_data = await self._call_engine(preprocessed, context) if (refusal := engine_data.get("refusal")) is not None: diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index 807f65492c5..cd6a68c4415 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -316,7 +316,7 @@ async def preprocess_chat( ), max_model_len=self.model_config.max_model_len, # /tokenize counts a prompt of any length, so only chat calls clamp. - clamp=isinstance(request, NeMoRLChatCompletionRequest), + measure_only=not isinstance(request, NeMoRLChatCompletionRequest), tokenizer=self.renderer.tokenizer, ) await start_capture( diff --git a/nemo_rl/models/generation/vllm/prompt_and_capture.py b/nemo_rl/models/generation/vllm/prompt_and_capture.py index 8d0d50eb88f..20e39eed8fc 100644 --- a/nemo_rl/models/generation/vllm/prompt_and_capture.py +++ b/nemo_rl/models/generation/vllm/prompt_and_capture.py @@ -126,6 +126,8 @@ async def resolve_restored_call( regenerates. For a restore, the request gets its restore fields (which the reply reads) and its remaining ``min_tokens``. """ + if serving_admission.generation_cut is None: + return serving_admission, None # Off-loop: the saved rows are a blocking TQ read. resolved_admission, saved = await asyncio.to_thread( host._prefix_recovery.resolve_restore, serving_admission, prefix_token_ids @@ -267,7 +269,6 @@ class PreparedPrompt: """A call's prompt as the engine runs it, its output budget, and how capture admits it.""" prompt_token_ids: list[int] - template_token_ids: list[int] # How the model's own prefix replaced the template's history; None if not. splice: Optional[PrefixSplice] # The model's own prefix is the prompt's first ``prefix_len`` ids. @@ -292,12 +293,10 @@ def begin_call_prefix_token_ids(self) -> Optional[list[int]]: def to_json(self) -> dict[str, Any]: """As JSON, without the prompt (it rides as the call's token ids).""" return { - "template_token_ids": self.template_token_ids, "splice": None if self.splice is None else [self.splice.model_cut_end, self.splice.template_cut_start], "prefix_len": self.prefix_len, - "measure_only": self.measure_only, "request_max_tokens": self.request_max_tokens, "max_tokens": self.max_tokens, "admission": _admission_json(self.admission), @@ -324,10 +323,10 @@ def from_json( serving_admission = parse_admission(wire["serving_admission"]) return cls( prompt_token_ids=prompt_token_ids, - template_token_ids=list(wire["template_token_ids"]), splice=splice, prefix_len=wire["prefix_len"], - measure_only=wire["measure_only"], + # A measure call is answered where it is prepared. + measure_only=False, request_max_tokens=wire["request_max_tokens"], max_tokens=wire["max_tokens"], admission=admission, @@ -346,7 +345,7 @@ async def prepare_prompt( resolve_prefix: Callable[[Any], list[int]], turn_end_token_ids: Collection[int], max_model_len: int, - clamp: bool, + measure_only: bool, tokenizer: Any = None, ) -> PreparedPrompt: """Prepare a rendered call's prompt: the ids the engine runs, and its output budget. @@ -357,8 +356,9 @@ async def prepare_prompt( so it runs off the event loop. turn_end_token_ids: Every id a turn can end on; the splice cuts by them. max_model_len: The engine's context length. - clamp: Whether the call generates, so its budget is clamped to the - context; a measure call counts its prompt at any length. + measure_only: Whether Gym only measures the prompt. Such a call counts + a prompt of any length; a generating call's budget is clamped to + the context. tokenizer: Only for the splice's error message; may be None. Raises: @@ -384,16 +384,15 @@ async def prepare_prompt( ) prompt_token_ids = splice.token_ids max_tokens = None - if clamp: + if not measure_only: max_tokens = clamp_max_tokens( rendered.request_max_tokens, len(prompt_token_ids), max_model_len ) return PreparedPrompt( prompt_token_ids=prompt_token_ids, - template_token_ids=rendered.template_token_ids, splice=splice, prefix_len=None if prefix is None else len(prefix), - measure_only=not clamp, + measure_only=measure_only, request_max_tokens=rendered.request_max_tokens, max_tokens=max_tokens, admission=rendered.admission, diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index a214bcf8a0e..4b4d38eab25 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -262,7 +262,7 @@ def test_a_gym_call_is_rendered_spliced_and_routed_with_its_capture_plan() -> No assert routed["token_ids"] == SPLICED assert routed["stop_conditions"] == {"max_tokens": MAX_MODEL_LEN - len(SPLICED)} assert routed["annotations"] == [chat_engine.GYM_CALL_ANNOTATION] - wire = routed["extra_args"][chat_engine.PLAN_KEY] + wire = routed["extra_args"][chat_engine.CALL_KEY] plan = wire["prepared_prompt"] # The request as rendered; the engine sets the plan's clamped budget. assert wire["request"] == { @@ -271,12 +271,13 @@ def test_a_gym_call_is_rendered_spliced_and_routed_with_its_capture_plan() -> No "max_completion_tokens": None, } assert plan["max_tokens"] == MAX_MODEL_LEN - len(SPLICED) - assert (plan["template_token_ids"], plan["splice"], plan["prefix_len"]) == ( - TEMPLATE, + assert (plan["splice"], plan["prefix_len"]) == ( # Each side is cut at its turn end: PREFIX[:3] + TEMPLATE[2:]. [3, 2], len(PREFIX), ) + # A text call sends its prompt once: the template's ids ride only with media. + assert "template_token_ids" not in plan assert (plan["admission"], wire["multimodal"], wire["reasoning_ended"]) == ( None, None, @@ -311,7 +312,7 @@ def test_a_token_in_call_splices_the_prefix_staged_in_tq() -> None: assert capture.resolved == [TOKEN_IN] (routed,) = router.requests assert routed["token_ids"] == SPLICED - plan = routed["extra_args"][chat_engine.PLAN_KEY]["prepared_prompt"] + plan = routed["extra_args"][chat_engine.CALL_KEY]["prepared_prompt"] assert (plan["admission"], plan["serving_admission"], plan["prefix_len"]) == ( TOKEN_IN, TOKEN_IN, diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index 98b07cb02e2..9cc05442ebb 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -50,8 +50,8 @@ adapter as vllm_adapter, ) from nemo_rl.models.generation.dynamo.vllm.chat_engine import ( # noqa: E402 + CALL_KEY, GYM_CALL_ANNOTATION, - PLAN_KEY, RUNTIME_DATA_KEY, ForwardedCall, ) @@ -471,10 +471,8 @@ def _gym_call(token_ids: list[int], **fields) -> dict: ``fields`` set the prepared prompt's fields and the call's others by name. """ plan = { - "template_token_ids": token_ids, "splice": None, "prefix_len": None, - "measure_only": False, "request_max_tokens": None, "max_tokens": None, "admission": None, @@ -488,6 +486,13 @@ def _gym_call(token_ids: list[int], **fields) -> dict: "reasoning_ended": None, "reasoning_parser_kwargs": None, } + # A media call's template ids are its own prompt's unless it was spliced. + template_token_ids = fields.pop("template_token_ids", token_ids) + if (inputs := fields.pop("multimodal", None)) is not None: + wire["multimodal"] = { + "inputs": inputs, + "template_token_ids": template_token_ids, + } for name, value in fields.items(): if name in plan: plan[name] = value @@ -497,7 +502,7 @@ def _gym_call(token_ids: list[int], **fields) -> dict: return { "token_ids": token_ids, "annotations": [GYM_CALL_ANNOTATION], - "extra_args": {PLAN_KEY: wire}, + "extra_args": {CALL_KEY: wire}, } diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py index 288762ebb90..9f60a7c5c95 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py @@ -186,7 +186,6 @@ def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: SPLICED, TOKEN_IN, prefix_len=len(PREFIX), - template_token_ids=TEMPLATE, splice=SPLICE, ), ) From c810ee1838f637559d989eed2b12d2d6397e89ed Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 17:04:27 -0800 Subject: [PATCH 63/82] fix(vllm): answer vLLM errors raised while the request body is parsed vLLM raises some request errors from the body's validators (top_logprobs with logprobs off, for one), before any route runs. The native app had no handler for them, so FastAPI answered 500 where the Dynamo path answers 400. The app now answers a VLLMClientError with the shared error map's status and body. No capture began for such a call, so a chat call's refusal is acknowledged to Gym from the raw body's ng_capture, as the chat engine's message suffix does behind Dynamo. The route builds its error body with the same helper. Co-authored-by: Terry Kong Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- nemo_rl/models/generation/vllm/chat_server.py | 62 +++++++++++++------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index cd6a68c4415..dc147d595cf 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -52,7 +52,7 @@ ) from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import with_cancellation -from vllm.exceptions import VLLMValidationError +from vllm.exceptions import VLLMClientError, VLLMValidationError from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager from vllm.renderers.hf import HfRenderer from vllm.renderers.online_renderer import OnlineRenderer @@ -77,8 +77,10 @@ register_generation_cut_route, ) from nemo_rl.models.generation.vllm.prompt_and_capture import ( + ErrorReply, abort_capture, abort_request_capture, + error_reply, prepare_prompt, save_capture, set_max_tokens, @@ -762,6 +764,27 @@ def build_chat_server( return ChatServer(online_renderer, openai_serving_chat, openai_serving_tokenization) +def _error_response( + failure: ErrorReply, headers: Optional[dict[str, str]] +) -> JSONResponse: + """The OpenAI error body of a call refused before its reply.""" + return JSONResponse( + content={ + "error": { + "message": failure.message, + "type": failure.error_type, + "param": failure.param, + "code": failure.code, + } + }, + status_code=failure.status, + headers=headers or None, + ) + + +CHAT_COMPLETIONS_PATH = "/v1/chat/completions" + + def register_routes( app: FastAPI, server: ChatServer, *, host: Any, generation_config: dict[str, Any] ) -> None: @@ -769,6 +792,21 @@ def register_routes( openai_serving_chat = server.serving_chat openai_serving_tokenization = server.serving_tokenization + @app.exception_handler(VLLMClientError) + async def refuse_unparsed_request(raw_request: Request, error: VLLMClientError): + # vLLM raises some request errors from the body's validators, before any + # route runs; unhandled, FastAPI answers them with a 500. + failure = error_reply(error) + if failure is None: + raise error + headers = None + if failure.acknowledge and raw_request.url.path == CHAT_COMPLETIONS_PATH: + # No capture began: the refused call is named by the raw body. + headers = host._context_refusal_headers( + GymRequestFields.model_validate_json(await raw_request.body()) + ) + return _error_response(failure, headers) + class HostedChatCompletionRequest(NeMoRLChatCompletionRequest): def to_sampling_params(self, *args, **kwargs): sampling_params = super().to_sampling_params(*args, **kwargs) @@ -779,7 +817,7 @@ def to_sampling_params(self, *args, **kwargs): register_generation_cut_route(app, lambda: host._prefix_recovery) # The create_chat_completion and tokenize methods are taken from vllm/entrypoints/openai/api_server.py - @app.post("/v1/chat/completions") + @app.post(CHAT_COMPLETIONS_PATH) # Like vLLM's own routes: cancel the handler (and with it the engine # request) when the client disconnects, instead of generating for nobody. @with_cancellation @@ -824,22 +862,10 @@ async def create_chat_completion( failure = await abort_capture(host, request, error) if failure is None: raise - return JSONResponse( - content={ - "error": { - "message": failure.message, - "type": failure.error_type, - "param": failure.param, - "code": failure.code, - } - }, - status_code=failure.status, - # Sent after abort_capture drained the capture state. - headers=( - host._context_refusal_headers(request) - if failure.acknowledge - else None - ), + # The acknowledgement goes out after abort_capture drained capture state. + return _error_response( + failure, + host._context_refusal_headers(request) if failure.acknowledge else None, ) if isinstance(generator, ErrorResponse): From bbb35a20d8a084605f6b84ebd184b97cafa89156 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 17:04:27 -0800 Subject: [PATCH 64/82] test(gym): compare refused and failed calls on both paths The parity test now sends each of these through the native route and through the chat engine plus adapter, and compares status, what reached the engine, what was staged, the failed-capture reason and Gym's acknowledgement (the native header, Dynamo's message suffix): - logprobs with top_logprobs unset, and sent as null (400); - a field vLLM refuses while the body is parsed (400, acknowledged); - a token-in call whose spliced prompt fills the context though its render fits (400, acknowledged); - an engine error (500, capture failed as engine_error); - a client disconnect mid-generation (the call ran, capture failed as client_disconnect, nothing staged). The out-of-vocabulary case also checks its acknowledgement. A router-replay kit makes the fake engine return routed experts, so the staged comparison covers them. The fake router raises an engine's error from its stream, as Dynamo does, and the fake context can be stopped. Co-authored-by: Terry Kong Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- .../models/generation/test_gym_chat_path.py | 188 +++++++++++++++--- 1 file changed, 159 insertions(+), 29 deletions(-) diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 2e07bd2105b..5d51597b94f 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -34,6 +34,7 @@ from types import SimpleNamespace from typing import Any, Callable, Optional +import numpy as np import pytest import torch @@ -50,6 +51,10 @@ GenerationCutPrefix, ) from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 +from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 + split_dynamo_capture_refusal, +) +from nemo_gym.token_id_capture.protocols import CAPTURE_REFUSAL_HEADER # noqa: E402 from nemo_gym.token_id_capture.replay import ( # noqa: E402 replay_context, summarize_replay, @@ -165,6 +170,8 @@ '{"name": "get_weather", "arguments": {"city": "Lyon"}}\n' ) TURN_1_REPLY = "\neasy\n\n\nIt is sunny." +# Reasoning the template drops from history, longer than the context. +LONG_TURN_1_REPLY = "\n" + "hmm " * MAX_MODEL_LEN + "\n\n\nIt is sunny." # Turn 1 as a tool call, which Gym's source history recorded with empty # content and the harness now sends with none. TOOL_TURN = { @@ -220,19 +227,20 @@ class _Kit: def ids(self, text: str) -> list[int]: return self.tokenizer.encode(text, add_special_tokens=False) - def turn_1_tokens(self) -> list[int]: + def turn_1_tokens(self, reply: str = TURN_1_REPLY) -> list[int]: """Turn 1's prompt and the model's own reply, as capture staged them.""" prompt = self.tokenizer.apply_chat_template( HISTORY[:1], tools=TOOLS, add_generation_prompt=True, tokenize=False ) - return self.ids(prompt + TURN_1_REPLY + "<|im_end|>") + return self.ids(prompt + reply + "<|im_end|>") class _Engine: """vLLM's engine client: answers every call with REPLY and records its runs. ``refuse`` is what the engine refuses every call with, and where: its - queue check ("admission") or its input check ("generate"). + queue check ("admission"), its input check ("generate"), or mid-generation + ("running"), where a cancelled native handler gets its CancelledError. """ def __init__(self, kit: _Kit, refuse: Optional[tuple[str, Any]] = None) -> None: @@ -254,6 +262,8 @@ async def generate(self, prompt, sampling_params, request_id, **kwargs): raise self._refuse[1] prompt_token_ids = list(prompt["prompt_token_ids"]) self.runs.append((prompt_token_ids, sampling_params, _media(prompt))) + if self._refuse is not None and self._refuse[0] == "running": + raise self._refuse[1] terminal = restored_terminal() if terminal is not None: # As the worker's engine client answers a call whose saved prefix @@ -272,7 +282,14 @@ async def generate(self, prompt, sampling_params, request_id, **kwargs): } for token_id in reply_ids ] - yield RequestOutput( + routes = None + if self._kit.generation.get("vllm_kwargs", {}).get( + "enable_return_routed_experts" + ): + # vLLM routes every token but the last sampled one: [tokens, layers, k]. + tokens = len(prompt_token_ids) + len(reply_ids) - 1 + routes = np.arange(tokens * 4, dtype=np.int16).reshape(tokens, 2, 2) + output = RequestOutput( request_id=request_id, prompt=None, prompt_token_ids=prompt_token_ids, @@ -287,11 +304,17 @@ async def generate(self, prompt, sampling_params, request_id, **kwargs): cumulative_logprob=None, logprobs=logprobs, finish_reason="stop", + routed_experts=None + if routes is None + else routes[len(prompt_token_ids) :], ) ], finished=True, num_cached_tokens=0, ) + if routes is not None: + output.prompt_routed_experts = routes[: len(prompt_token_ids)] + yield output def check_admission(self, n, request_id=None) -> None: if self._refuse is not None and self._refuse[0] == "admission": @@ -359,11 +382,14 @@ def fetch_prefix_chains(self, staging_keys: list[str]) -> StagedPrefix: class _Outcome: """What one path did with a call.""" - status: int + # None when the client went away: nobody is answered. + status: Optional[int] reply: dict[str, Any] runs: list[tuple[list[int], Any, Any]] staged: list[tuple[Any, Any]] failed: list[str] = field(default_factory=list) + # The capture call Gym is told was refused before it was staged, if any. + acknowledged: Optional[str] = None def _host( @@ -413,39 +439,59 @@ async def _native(kit, host, sink, failed, body, *, measure: bool) -> _Outcome: app, server, host=host, generation_config=kit.generation ) async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="http://native" + # A server error is answered with a 500, as uvicorn does. + transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://native", ) as client: - response = await client.post( - "/tokenize" if measure else "/v1/chat/completions", json=body - ) + try: + response = await client.post( + "/tokenize" if measure else "/v1/chat/completions", json=body + ) + except asyncio.CancelledError: + # The handler was cancelled, as on a client disconnect. + return _Outcome(None, {}, host.llm.runs, sink.staged, failed) + is_json = response.headers["content-type"] == "application/json" return _Outcome( - response.status_code, response.json(), host.llm.runs, sink.staged, failed + response.status_code, + response.json() if is_json else {"error": response.text}, + host.llm.runs, + sink.staged, + failed, + response.headers.get(CAPTURE_REFUSAL_HEADER), ) class _Router: """Dynamo's router to one engine: the request crosses the wire as JSON.""" - def __init__(self, adapter: DynamoVllmAdapter) -> None: + def __init__(self, adapter: DynamoVllmAdapter, *, stopped: bool) -> None: self._adapter = adapter + self._stopped = stopped async def generate(self, request, context=None): wire = json.loads(json.dumps(request)) - chunks = [chunk async for chunk in self._adapter.generate(wire, _Context())] + context = _Context(self._stopped) async def stream(): - for chunk in chunks: - yield SimpleItem(chunk) + try: + async for chunk in self._adapter.generate(wire, context): + yield SimpleItem(chunk) + except Exception as error: + # Dynamo raises an engine's error from the stream, as a ValueError. + raise ValueError(str(error)) from error return stream() class _Context: + def __init__(self, stopped: bool) -> None: + self._stopped = stopped + def id(self) -> str: return "chatcmpl-0" def is_stopped(self) -> bool: - return False + return self._stopped class SimpleItem: @@ -456,7 +502,9 @@ def data(self): return self._data -async def _dynamo(kit, host, sink, failed, body, extras) -> _Outcome: +async def _dynamo( + kit, host, sink, failed, body, extras, *, stopped: bool = False +) -> _Outcome: request = {k: v for k, v in body.items() if k not in extras} # Dynamo's request names the template kwargs chat_template_args. if "chat_template_kwargs" in request: @@ -468,7 +516,9 @@ async def _dynamo(kit, host, sink, failed, body, extras) -> _Outcome: capture._prefix_cache = host._chain_prefix engine = chat_engine.NemoRLChatEngine( pool=kit.pool, - routed_engine=_Router(DynamoVllmAdapter(host, leader_worker_idx=0)), + routed_engine=_Router( + DynamoVllmAdapter(host, leader_worker_idx=0), stopped=stopped + ), spec=json.loads(json.dumps(chat_runtime_data(host)))[ chat_engine.RUNTIME_DATA_KEY ], @@ -477,8 +527,15 @@ async def _dynamo(kit, host, sink, failed, body, extras) -> _Outcome: try: (chunk,) = [chunk async for chunk in engine.generate(request, None)] except HttpError as error: + message, acknowledged = split_dynamo_capture_refusal(error.message) return _Outcome( - error.code, {"error": error.message}, host.llm.runs, sink.staged, failed + # The frontend's answer to a client that went away reaches nobody. + None if stopped else error.code, + {"error": message}, + host.llm.runs, + sink.staged, + failed, + acknowledged, ) return _Outcome(200, chunk, host.llm.runs, sink.staged, failed) @@ -550,6 +607,10 @@ class _KitSpec: "vlm": _KitSpec(_generation(reasoning_parser=""), _vlm), # A tool parser with no structural tag: a forced tool sets structured outputs. "pythonic": _KitSpec(_generation(tool_parser="pythonic")), + # Router replay: the engine returns each token's routed experts. + "router_replay": _KitSpec( + {**GENERATION, "vllm_kwargs": {"enable_return_routed_experts": True}} + ), } @@ -611,11 +672,11 @@ def _admission(**fields) -> dict[str, Any]: ).model_dump(mode="json") -def _staged_turn_1(kit: _Kit, **fields) -> dict[str, Any]: +def _staged_turn_1(kit: _Kit, reply: str = TURN_1_REPLY, **fields) -> dict[str, Any]: """Turn 2's admission, chained to turn 1 staged under rollout-1/c1.""" return _admission( parent_call_id="c1", - prev_len=len(kit.turn_1_tokens()), + prev_len=len(kit.turn_1_tokens(reply)), parent_chain_hash="1" * 64, staging_chain=["rollout-1/c1"], **fields, @@ -651,15 +712,23 @@ class _Case: fields: dict[str, Any] = field(default_factory=dict) # The sampler top-k capture stages, for score centering. top_logprobs: int = 0 - # 200, or the status both paths refuse the call with before it runs. + # Request fields the call leaves out. + omit: tuple[str, ...] = () + # 200, or the status both paths answer a call that did not finish with. status: int = 200 # The mode capture stages the call in; None when the call is not captured. mode: Optional[str] = None kit: str = "qwen3" # What the engine refuses the call with, and where (see _Engine). refuse: Optional[tuple[str, Any]] = None - # What a refused call's capture is failed with, on both paths. + # The client goes away while the engine generates: nobody is answered. + disconnect: bool = False + # What an unfinished call's capture is failed with, on both paths. failed: tuple[str, ...] = () + # Whether Gym is told the capture call was refused before it was staged. + acknowledged: bool = False + # Turn 1's reply as the model wrote it and capture staged it. + turn_1_reply: str = TURN_1_REPLY def _off_policy(name: str, value: Any) -> _Case: @@ -798,7 +867,27 @@ def _off_policy(name: str, value: Any) -> _Case: # Capture stages one choice, and the trainer replays one. _Case("n_2", fields={"n": 2}, status=400), # Token information without top_logprobs, unset or sent as null. - _Case("top_logprobs_unset", fields={"top_logprobs": None}, status=400), + _Case("top_logprobs_unset", omit=("top_logprobs",), status=400), + _Case("top_logprobs_null", fields={"top_logprobs": None}, status=400), + # A field vLLM refuses while the request is parsed, before any step runs. + _Case( + "field_error", + extras=lambda kit: {"ng_capture": _admission(mode="text")}, + fields={"logprobs": False, "top_logprobs": 5}, + status=400, + acknowledged=True, + ), + # The model's own turn 1 is longer than the template's render of it, so the + # spliced prompt fills the context though the render fits. + _Case( + "spliced_prompt_overflow", + extras=lambda kit: { + "ng_capture": _staged_turn_1(kit, LONG_TURN_1_REPLY, mode="token_in") + }, + status=400, + acknowledged=True, + turn_1_reply=LONG_TURN_1_REPLY, + ), # A full queue: vLLM's 503, so Gym retries elsewhere; checked before capture. _Case( "queue_full", @@ -816,6 +905,29 @@ def _off_policy(name: str, value: Any) -> _Case: VLLMValidationError("Token id 999999 is out of vocabulary"), ), failed=("invalid_request",), + acknowledged=True, + ), + # The engine fails a captured call: a server error, and nothing staged. + _Case( + "engine_error", + extras=lambda kit: {"ng_capture": _admission(mode="text")}, + status=500, + refuse=("generate", RuntimeError("engine failed")), + failed=("engine_error",), + ), + # The client goes away mid-generation: the call ran, and is not staged. + _Case( + "client_disconnect", + extras=lambda kit: {"ng_capture": _admission(mode="text")}, + disconnect=True, + failed=("client_disconnect",), + ), + # Router replay stages each token's routed experts with the call. + _Case( + "router_replay", + extras=lambda kit: {"ng_capture": _admission(mode="text")}, + mode="text", + kit="router_replay", ), ] @@ -837,31 +949,46 @@ def test_a_gym_call_takes_the_same_steps_through_dynamo_as_natively(kits, case) **case.fields, **extras, } - deltas = {"rollout-1/c1": kit.turn_1_tokens()} + for name in case.omit: + del body[name] + deltas = {"rollout-1/c1": kit.turn_1_tokens(case.turn_1_reply)} + # A disconnect cancels the native handler while the engine generates; + # Dynamo only marks the call's context stopped. + cancelled = ("running", asyncio.CancelledError()) if case.disconnect else None async def both(): native = await _native( kit, - *_host(kit, deltas, case.top_logprobs, case.refuse), + *_host(kit, deltas, case.top_logprobs, cancelled or case.refuse), body, measure=False, ) dynamo = await _dynamo( - kit, *_host(kit, deltas, case.top_logprobs, case.refuse), body, extras + kit, + *_host(kit, deltas, case.top_logprobs, case.refuse), + body, + extras, + stopped=case.disconnect, ) return native, dynamo native, dynamo = asyncio.run(both()) - assert (native.status, dynamo.status) == (case.status, case.status), ( + status = None if case.disconnect else case.status + assert (native.status, dynamo.status) == (status, status), ( native.reply, dynamo.reply, ) - if case.status != 200: - assert native.runs == dynamo.runs == [] + if status != 200: + # Only a call the client left reached the engine, with the same prompt. + assert [run[0] for run in native.runs] == [run[0] for run in dynamo.runs] + assert len(native.runs) == (1 if case.disconnect else 0) # Capture is failed alike, and neither path staged the call. assert native.failed == dynamo.failed == list(case.failed) assert native.staged == dynamo.staged == [] + # Gym is told alike that its capture call was refused before staging. + acknowledged = "c2" if case.acknowledged else None + assert native.acknowledged == dynamo.acknowledged == acknowledged return # The engine ran the same prompt, media and SamplingParams. ((native_prompt, native_params, native_media),) = native.runs @@ -882,6 +1009,9 @@ async def both(): assert (dynamo_tensors or {}).keys() == (native_tensors or {}).keys() for name, tensor in (native_tensors or {}).items(): assert torch.equal(dynamo_tensors[name], tensor), name + assert ("routed_experts" in (native_tensors or {})) == ( + case.kit == "router_replay" + ) assert dynamo.failed == native.failed == [] # Gym reads the same reply and token fields. message = native.reply["choices"][0]["message"] From 687794476705d59f62d9a682e5e3583c44453264 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 17:12:06 -0800 Subject: [PATCH 65/82] test(vllm): follow the shared chat-call checks in the native route tests Three things the shared steps changed for the native server's own tests: - The app registers an exception handler, so the two fake apps accept one. - logprobs without top_logprobs is refused before anything runs. The three tests that post to a real server now send top_logprobs: 0, as NeMo-Gym does. - The off-policy route test returns a whole chat reply from its fake serving chat instead of a bare iterator. Co-authored-by: Terry Kong Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- .../test_vllm_chat_template_wiring.py | 3 +++ .../models/generation/test_vllm_generation.py | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/unit/models/generation/test_vllm_chat_template_wiring.py b/tests/unit/models/generation/test_vllm_chat_template_wiring.py index 331492e3b9a..250aa573d47 100644 --- a/tests/unit/models/generation/test_vllm_chat_template_wiring.py +++ b/tests/unit/models/generation/test_vllm_chat_template_wiring.py @@ -105,6 +105,9 @@ def decorator(fn): def post(self, path, **_kwargs): return self._register(path) + def exception_handler(self, _error_type): + return lambda fn: fn + def get(self, path, **_kwargs): return self._register(path) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 1d4c6d1881e..f8a49c4f543 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1210,6 +1210,9 @@ def decorator(func): return decorator + def exception_handler(self, error_type): + return lambda func: func + def test_vllm_async_http_server_loads_reasoning_parser_plugin(monkeypatch): ( @@ -1852,7 +1855,16 @@ async def create_chat_completion(_request, _raw_request): async def test_chat_handler_rejects_off_policy_sampling_and_pins_top_k(monkeypatch): """The route rejects off-policy sampling, then hands vLLM top_k=-1.""" _, serving_chat, chat_handler = _setup_fake_vllm_chat_handler(monkeypatch) - serving_chat.create_chat_completion = AsyncMock(return_value=iter(())) + response_type = sys.modules[ + "vllm.entrypoints.openai.chat_completion.protocol" + ].ChatCompletionResponse + serving_chat.create_chat_completion = AsyncMock(return_value=response_type()) + # The server module was just imported against the fakes; patch that copy. + monkeypatch.setattr( + sys.modules["nemo_rl.models.generation.vllm.chat_server"], + "model_dump_chat_response_with_dynamic_message_fields", + lambda _response: {"choices": []}, + ) off_policy = _fake_chat_request() off_policy.temperature = 0.5 @@ -4044,6 +4056,8 @@ def test_vllm_http_server(cluster, tokenizer): top_p=generation_config["top_p"], # We want to test the actual train flow and how this is used. So we need to get logprobs here. logprobs=True, + # As NeMo-Gym sends it: the chosen token's logprob only. + top_logprobs=0, max_tokens=1, ) @@ -4258,6 +4272,7 @@ def test_vllm_deferred_model_load(cluster, tokenizer): temperature=generation_config["temperature"], top_p=generation_config["top_p"], logprobs=True, + top_logprobs=0, return_tokens_as_token_ids=True, max_tokens=1, ) @@ -4418,6 +4433,7 @@ async def test_vllm_http_server_correct_merged_tokens_matches_baseline( top_p=generation_config["top_p"], # We want to test the actual train flow and how this is used. So we need to get logprobs here. logprobs=True, + top_logprobs=0, return_tokens_as_token_ids=True, max_tokens=1, ) From 8415161769ab0b51d17bf3f30dcb991f5093b9be Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 17:12:06 -0800 Subject: [PATCH 66/82] fix(gym): refuse stream: true on NeMo-Gym chat calls Behind Dynamo a streamed call was generated in vLLM's delta mode while the adapter kept only the last output, so the reply held the final piece of the completion. Natively an uncaptured streamed call was served and a captured one failed as a server error in Gym's begin_call. on_policy_sampling, which both paths run before anything else, now refuses stream: true with a 400. NeMo-Gym never streams a chat call. The native route's streaming branch is unreachable and goes. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- .../generation/dynamo/vllm/chat_engine.py | 1 + nemo_rl/models/generation/vllm/chat_server.py | 19 +++++++++---------- nemo_rl/models/generation/vllm/utils.py | 17 ++++++++++++----- .../models/generation/test_gym_chat_path.py | 2 ++ 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index b283470fdbe..a242466ae68 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -286,6 +286,7 @@ async def _serve( logprobs=body.get("logprobs"), # From the raw request: _drop_none drops an explicit null. top_logprobs=request.get("top_logprobs"), + stream=body.get("stream"), generation_config=self._generation, ) ) diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index dc147d595cf..bcbef7b15c9 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -32,7 +32,7 @@ import torch from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, StreamingResponse +from fastapi.responses import JSONResponse from pydantic import PrivateAttr from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -849,6 +849,7 @@ async def create_chat_completion( if "top_logprobs" in request.model_fields_set else None ), + stream=request.stream, generation_config=generation_config, ) for field, value in sampling.items(): @@ -874,15 +875,13 @@ async def create_chat_completion( content=generator.model_dump(), status_code=generator.error.code ) - elif isinstance(generator, ChatCompletionResponse): - content = model_dump_chat_response_with_dynamic_message_fields(generator) - # Token capture: stage the delta and ride the coords on the - # response; strips logprobs/ids (no-op when capture is off). - content = await save_capture(host, request, content) - return JSONResponse(content=content) - - await abort_request_capture(host, request, "streaming_response") - return StreamingResponse(content=generator, media_type="text/event-stream") + # on_policy_sampling refused stream: true, so the reply is whole. + assert isinstance(generator, ChatCompletionResponse) + content = model_dump_chat_response_with_dynamic_message_fields(generator) + # Token capture: stage the delta and ride the coords on the + # response; strips logprobs/ids (no-op when capture is off). + content = await save_capture(host, request, content) + return JSONResponse(content=content) @app.post("/tokenize") async def tokenize(request: NeMoRLTokenizeRequest, raw_request: Request): diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index 9ded53c9cf1..0faafb93f7b 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -534,20 +534,21 @@ def on_policy_sampling( n: Optional[int], logprobs: Optional[bool], top_logprobs: Optional[int], + stream: Optional[bool], generation_config: Any, ) -> dict[str, Any]: """The sampling a NeMo-Gym chat call is served with, on every serving path. The request must match the train or validation profile (:func:`check_sampling_param`), set no sampling the trainer does not - replay, sample one choice, and set ``top_logprobs`` when it asks for - token information. Floats are compared at f32: Dynamo's frontend carries - them as f32, so 0.7 arrives as 0.69999998. + replay, sample one choice, set ``top_logprobs`` when it asks for token + information, and not stream. Floats are compared at f32: Dynamo's frontend + carries them as f32, so 0.7 arrives as 0.69999998. Args: temperature, top_p, top_k, min_p, repetition_penalty, presence_penalty, - frequency_penalty, logit_bias, n, logprobs, top_logprobs: The - request's values; None (or missing) when it did not set them. + frequency_penalty, logit_bias, n, logprobs, top_logprobs, stream: + The request's values; None (or missing) when it did not set them. generation_config: ``policy.generation``. Returns: @@ -597,6 +598,12 @@ def on_policy_sampling( "`top_logprobs` must be set when requesting token information from " "NeMo RL's chat endpoint" ) + if stream: + # Capture refuses a streamed call, and vLLM's delta outputs would leave + # the Dynamo path only the last piece of an uncaptured one. + raise OffPolicySamplingError( + "request stream=true is not supported; NeMo-Gym calls do not stream" + ) is_train = (_f32(temperature), _f32(top_p)) == ( profile["temperature"], profile["top_p"], diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 5d51597b94f..b064e17f553 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -869,6 +869,8 @@ def _off_policy(name: str, value: Any) -> _Case: # Token information without top_logprobs, unset or sent as null. _Case("top_logprobs_unset", omit=("top_logprobs",), status=400), _Case("top_logprobs_null", fields={"top_logprobs": None}, status=400), + # Neither path stages a streamed call, and Gym never sends one. + _Case("stream", fields={"stream": True}, status=400), # A field vLLM refuses while the request is parsed, before any step runs. _Case( "field_error", From 7e29dccaab91301853baf1e47ac8458fe9c9fed0 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 17:21:46 -0800 Subject: [PATCH 67/82] fix(gym): take reasoning_ended from the template render on both paths With a reasoning parser and structured output, vLLM applies the grammar from the first token when reasoning_ended is True. Natively vLLM's serving read it from the spliced prompt, which still holds the model's earlier tokens that the template drops from history, so a multi-turn call could not reason before its structured output. Behind Dynamo the render pool read it from the template render, which is right. Both paths now use one helper, reasoning_ended, called by the shared render step on the template's ids. Dynamo forwards the value as before. Natively the request's SamplingParams carry it in extra_args, and the worker's engine client hands it to the engine in place of the value vLLM's serving computed. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- .../generation/dynamo/vllm/render_pool.py | 9 +-- nemo_rl/models/generation/vllm/chat_server.py | 61 +++++++++++++++++++ nemo_rl/models/generation/vllm/utils.py | 21 +++++++ .../generation/vllm/vllm_worker_async.py | 2 + .../models/generation/test_gym_chat_path.py | 38 +++++++++++- .../test_vllm_captured_call_progress.py | 4 +- .../generation/test_vllm_flat_logprobs.py | 20 +++++- .../models/generation/test_vllm_generation.py | 9 ++- 8 files changed, 151 insertions(+), 13 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index 1acfecc1d90..f64251217ba 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -234,14 +234,7 @@ def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall if rendered.request_max_tokens is not None: set_max_tokens(chat, rendered.request_max_tokens) parser = self._parser(chat) - reasoning_ended = None reasoning_parser_kwargs = None - if not chat.include_reasoning or chat._grammar_from_parser: - reasoning_ended = True - elif parser is not None and parser.reasoning_parser is not None: - reasoning_ended = parser.is_reasoning_end( - rendered.engine_prompt["prompt_token_ids"] - ) if parser is not None and parser.reasoning_parser is not None: reasoning_parser_kwargs = { "chat_template_kwargs": self.serving_chat._engine_chat_template_kwargs( @@ -268,7 +261,7 @@ def render(self, body: dict[str, Any], *, measure: bool = False) -> RenderedCall multimodal=multimodal, request=chat.model_dump(mode="json", exclude={"messages"}), request_max_tokens=rendered.request_max_tokens, - reasoning_ended=reasoning_ended, + reasoning_ended=rendered.reasoning_ended, reasoning_parser_kwargs=reasoning_parser_kwargs, ) diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index bcbef7b15c9..f61d507279b 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -87,6 +87,7 @@ start_capture, ) from nemo_rl.models.generation.vllm.utils import ( + REASONING_ENDED_ARG, attach_routed_experts_to_chat_response_choices, attach_sampler_topk_logprobs_to_chat_response_choices, attach_token_information_to_chat_response_choices, @@ -131,6 +132,8 @@ class NeMoRLChatCompletionRequest( # The client's min_tokens before it is reduced by a restored prefix; # replaying the prefix's stop handling needs the original window. _original_min_tokens: int | None = PrivateAttr(default=None) + # reasoning_ended from the template render; None without a reasoning parser. + _template_reasoning_ended: bool | None = PrivateAttr(default=None) class NeMoRLMeasureChatRequest(NeMoRLChatCompletionRequest): @@ -176,6 +179,24 @@ class NeMoRLTokenizeChatRequest( NeMoRLTokenizeRequest = Union[TokenizeCompletionRequest, NeMoRLTokenizeChatRequest] +def reasoning_ended( + request: Any, parser: Any, template_token_ids: list[int] +) -> Optional[bool]: + """Whether the model's reasoning is over before this call generates. + + vLLM's rule (its chat serving's), read from the call's template render and + never from a spliced prompt: the model's own earlier turns still hold the + end-of-reasoning tokens the template drops from history, and they say + nothing about this turn. None without a reasoning parser. + """ + if not request.include_reasoning or request._grammar_from_parser: + # A Mistral grammar from the parser has its own optional think rule. + return True + if parser is not None and parser.reasoning_parser is not None: + return parser.is_reasoning_end(template_token_ids) + return None + + class NeMoRLOpenAIServingMixin: """The chat preprocessing NeMo RL adds: prefix splice and token capture.""" @@ -215,6 +236,14 @@ async def _render_chat_call( LOGGER.warning("Prompt exceeds max_model_len: %s", e) raise + ended = None + if isinstance(request, ChatCompletionRequest): + ended = reasoning_ended( + request, + self._request_parser(request, render_kwargs), + engine_prompt["prompt_token_ids"], + ) + # Token capture: the admission is built once by the host and resolved # here, before branching; the ``ng_capture`` dict is never mutated. serving_admission = admission @@ -270,10 +299,30 @@ async def _render_chat_call( serving_admission=serving_admission, verify_retained_media=verify_retained_media, request_max_tokens=request_max_tokens, + reasoning_ended=ended, conversation=conversation, engine_prompt=engine_prompt, ) + def _request_parser(self, request, render_kwargs: dict[str, Any]) -> Any: + """The call's parser, built as vLLM's chat serving builds its own. + + None when no reasoning parser is configured: nothing reads it then. + """ + parser_cls = render_kwargs["parser"] + if parser_cls is None or parser_cls.reasoning_parser_cls is None: + return None + chat_params = request.build_chat_params( + render_kwargs["default_template"], + render_kwargs["default_template_content_format"], + ).with_defaults(render_kwargs["default_template_kwargs"]) + return parser_cls( + self.renderer.get_tokenizer(), + request.tools, + chat_template_kwargs=chat_params.chat_template_kwargs, + model_config=self.model_config, + ) + # vLLM 0.25 moved chat preprocessing to # OnlineRenderer.preprocess_chat (tool_parser/reasoning_parser were # folded into a single `parser`), so this override now applies via @@ -310,6 +359,9 @@ async def preprocess_chat( ), admission=admission, ) + if isinstance(request, NeMoRLChatCompletionRequest): + # The request's SamplingParams carry it to the engine client. + request._template_reasoning_ended = rendered.reasoning_ended plan = await prepare_prompt( rendered, resolve_prefix=host._resolve_admission_prefix, @@ -350,6 +402,8 @@ class RenderedChat: verify_retained_media: bool # The output budget the request asked for, before the render lowered it. request_max_tokens: Optional[int] + # Whether reasoning is over before the call generates (reasoning_ended). + reasoning_ended: Optional[bool] conversation: list[Any] # The full render, multimodal inputs included; its ids are the template's. engine_prompt: dict[str, Any] @@ -812,6 +866,13 @@ def to_sampling_params(self, *args, **kwargs): sampling_params = super().to_sampling_params(*args, **kwargs) if self.ng_capture is not None: host._prefix_recovery.on_sampling_params(self, sampling_params) + if self._template_reasoning_ended is not None: + # vLLM's serving reads its own from the spliced prompt; the + # worker's engine client hands the engine this one instead. + sampling_params.extra_args = { + **(sampling_params.extra_args or {}), + REASONING_ENDED_ARG: self._template_reasoning_ended, + } return sampling_params register_generation_cut_route(app, lambda: host._prefix_recovery) diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index 0faafb93f7b..9e3b61328f3 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -521,6 +521,27 @@ def _f32(value: Optional[float]) -> Optional[float]: return None if value is None else float(np.float32(value)) +# SamplingParams.extra_args key: a native chat call's reasoning_ended, as +# NeMo RL read it from the call's template render. +REASONING_ENDED_ARG = "nemo_rl_reasoning_ended" + + +def take_reasoning_ended(sampling_params: Any, engine_kwargs: dict[str, Any]) -> None: + """Hand the engine the ``reasoning_ended`` a call's SamplingParams carry. + + vLLM's chat serving reads its own value from the spliced prompt, whose + earlier turns still hold their own end-of-reasoning tokens; this replaces + it and takes the argument back out of ``extra_args``. A call that carries + none is left as it is. + """ + extra_args = sampling_params.extra_args + if not extra_args or REASONING_ENDED_ARG not in extra_args: + return + engine_kwargs["reasoning_ended"] = extra_args.pop(REASONING_ENDED_ARG) + if not extra_args: + sampling_params.extra_args = None + + def on_policy_sampling( *, temperature: Optional[float], diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index ed10b83ee75..fc89a375fdc 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -88,6 +88,7 @@ ) from nemo_rl.models.generation.vllm.utils import ( format_prompt_for_vllm_generation, + take_reasoning_ended, validate_rollout_prompt, ) from nemo_rl.models.generation.vllm.vllm_worker import BaseVllmGenerationWorker @@ -199,6 +200,7 @@ def generate( and sampling_params.prompt_logprobs is None ): sampling_params.flat_logprobs = True + take_reasoning_ended(sampling_params, kwargs) return self._generate(prompt, sampling_params, request_id, kwargs) async def _generate( diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index b064e17f553..992168e3302 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -94,6 +94,7 @@ restored_terminal, ) from nemo_rl.models.generation.vllm import chat_server # noqa: E402 +from nemo_rl.models.generation.vllm.utils import take_reasoning_ended # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) @@ -253,6 +254,8 @@ def __init__(self, kit: _Kit, refuse: Optional[tuple[str, Any]] = None) -> None: self._kit = kit self._refuse = refuse self.runs: list[tuple[list[int], Any, Any]] = [] + # The reasoning_ended each run was handed. + self.reasoning_ended: list[Optional[bool]] = [] # How many of REPLY's tokens the prompt already ends with: a call # resumed from a checkpoint cut generates only the rest. self.resumed = 0 @@ -260,8 +263,11 @@ def __init__(self, kit: _Kit, refuse: Optional[tuple[str, Any]] = None) -> None: async def generate(self, prompt, sampling_params, request_id, **kwargs): if self._refuse is not None and self._refuse[0] == "generate": raise self._refuse[1] + # As the worker's engine client does before it submits the call. + take_reasoning_ended(sampling_params, kwargs) prompt_token_ids = list(prompt["prompt_token_ids"]) self.runs.append((prompt_token_ids, sampling_params, _media(prompt))) + self.reasoning_ended.append(kwargs.get("reasoning_ended")) if self._refuse is not None and self._refuse[0] == "running": raise self._refuse[1] terminal = restored_terminal() @@ -390,6 +396,8 @@ class _Outcome: failed: list[str] = field(default_factory=list) # The capture call Gym is told was refused before it was staged, if any. acknowledged: Optional[str] = None + # What the engine was told of each run's reasoning. + reasoning_ended: list[Optional[bool]] = field(default_factory=list) def _host( @@ -458,6 +466,7 @@ async def _native(kit, host, sink, failed, body, *, measure: bool) -> _Outcome: sink.staged, failed, response.headers.get(CAPTURE_REFUSAL_HEADER), + host.llm.reasoning_ended, ) @@ -537,7 +546,14 @@ async def _dynamo( failed, acknowledged, ) - return _Outcome(200, chunk, host.llm.runs, sink.staged, failed) + return _Outcome( + 200, + chunk, + host.llm.runs, + sink.staged, + failed, + reasoning_ended=host.llm.reasoning_ended, + ) def _generation(*, vllm_cfg=None, **chat_settings) -> dict[str, Any]: @@ -607,6 +623,8 @@ class _KitSpec: "vlm": _KitSpec(_generation(reasoning_parser=""), _vlm), # A tool parser with no structural tag: a forced tool sets structured outputs. "pythonic": _KitSpec(_generation(tool_parser="pythonic")), + # A reasoning parser that reads the prompt back to its last think token. + "deepseek_r1": _KitSpec(_generation(reasoning_parser="deepseek_r1")), # Router replay: the engine returns each token's routed experts. "router_replay": _KitSpec( {**GENERATION, "vllm_kwargs": {"enable_return_routed_experts": True}} @@ -729,6 +747,8 @@ class _Case: acknowledged: bool = False # Turn 1's reply as the model wrote it and capture staged it. turn_1_reply: str = TURN_1_REPLY + # What the engine is told of the call's reasoning; unchecked when None. + reasoning_ended: Optional[bool] = None def _off_policy(name: str, value: Any) -> _Case: @@ -924,6 +944,18 @@ def _off_policy(name: str, value: Any) -> _Case: disconnect=True, failed=("client_disconnect",), ), + # Structured output after the model's own turn 1, whose reasoning the + # template drops: that turn's does not end this turn's reasoning. + _Case( + "reasoning_not_ended_by_an_earlier_turn", + extras=lambda kit: {"required_prefix_token_ids": kit.turn_1_tokens()}, + fields={ + "chat_template_kwargs": {"enable_thinking": True}, + "response_format": {"type": "json_object"}, + }, + kit="deepseek_r1", + reasoning_ended=False, + ), # Router replay stages each token's routed experts with the call. _Case( "router_replay", @@ -999,6 +1031,10 @@ async def both(): _assert_same_media(dynamo_media, native_media) assert (native_media is not None) == (case.kit == "vlm") assert dynamo_params == native_params + # ...and was told the same of the call's reasoning, read from its template. + assert dynamo.reasoning_ended == native.reasoning_ended + if case.reasoning_ended is not None: + assert native.reasoning_ended == [case.reasoning_ended] # Capture staged the same record and tensors, in the expected mode. assert [record.mode for record, _ in native.staged] == ( [case.mode] if case.mode else [] diff --git a/tests/unit/models/generation/test_vllm_captured_call_progress.py b/tests/unit/models/generation/test_vllm_captured_call_progress.py index 0158b42e03e..5f512d665e2 100644 --- a/tests/unit/models/generation/test_vllm_captured_call_progress.py +++ b/tests/unit/models/generation/test_vllm_captured_call_progress.py @@ -63,7 +63,9 @@ async def exercise() -> None: client = _AsyncLLMHTTPClient( engine, asyncio.get_running_loop(), flat_logprobs=False ) - sampling_params = SimpleNamespace(logprobs=None, prompt_logprobs=None) + sampling_params = SimpleNamespace( + logprobs=None, prompt_logprobs=None, extra_args=None + ) async def serve(model_call_id: Optional[str], request_id: str) -> None: # What _begin_request_capture does in the request's task. diff --git a/tests/unit/models/generation/test_vllm_flat_logprobs.py b/tests/unit/models/generation/test_vllm_flat_logprobs.py index f6f5db9003b..5388a375522 100644 --- a/tests/unit/models/generation/test_vllm_flat_logprobs.py +++ b/tests/unit/models/generation/test_vllm_flat_logprobs.py @@ -16,6 +16,7 @@ import pytest +from nemo_rl.models.generation.vllm.utils import REASONING_ENDED_ARG from nemo_rl.models.generation.vllm.vllm_worker_async import _AsyncLLMHTTPClient pytestmark = pytest.mark.vllm @@ -48,9 +49,26 @@ def test_generate_sets_flat_logprobs( flat_logprobs, logprobs, prompt_logprobs, expected ): sampling_params = SimpleNamespace( - logprobs=logprobs, prompt_logprobs=prompt_logprobs, flat_logprobs=False + logprobs=logprobs, + prompt_logprobs=prompt_logprobs, + flat_logprobs=False, + extra_args=None, ) _client(flat_logprobs=flat_logprobs).generate("prompt", sampling_params, "req-0") assert sampling_params.flat_logprobs is expected + + +def test_generate_hands_the_engine_the_template_reasoning_ended(): + """vLLM's serving passes its own value, read from the spliced prompt.""" + client = _client(flat_logprobs=False) + client._generate = lambda prompt, sampling_params, request_id, kwargs: kwargs + sampling_params = SimpleNamespace( + logprobs=None, extra_args={REASONING_ENDED_ARG: False, "other": 1} + ) + + kwargs = client.generate("prompt", sampling_params, "req-0", reasoning_ended=True) + + assert kwargs == {"reasoning_ended": False} + assert sampling_params.extra_args == {"other": 1} diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index f8a49c4f543..3f6934dc536 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -666,7 +666,10 @@ def use_client_from_http_thread(): async def use_client(): tracing_enabled = await client.is_tracing_enabled() outputs = [ - output async for output in client.generate(None, None, "request") + output + async for output in client.generate( + None, types.SimpleNamespace(extra_args=None), "request" + ) ] errored = client.errored return ( @@ -739,7 +742,9 @@ async def abort(self, request_id): def cancel_from_http_thread(): async def cancel(): async def consume(): - async for _ in client.generate(None, None, "cancelled-request"): + async for _ in client.generate( + None, types.SimpleNamespace(extra_args=None), "cancelled-request" + ): pass task = asyncio.create_task(consume()) From 4589ba2043a0dfc1b611b38fec5238d2df16ccbb Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 17:25:48 -0800 Subject: [PATCH 68/82] fix(dynamo): give every engine the frontend's cut deadline The frontend and each engine timed one generation cut with two clocks. The engine's started later, when the request-plane message arrived, so an engine that ran into its deadline always answered after the frontend had given up, and every call it had cut and staged was reported as failed. The frontend now takes one wall-clock deadline when the cut arrives, waits to it, and sends it with the cut. Each engine converts it to its own monotonic clock, less a second for the answer to travel, and PrefixRecovery.run_cut uses it instead of starting a new control_timeout_s. The native route passes none and is unchanged. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- .../models/generation/dynamo/vllm/adapter.py | 15 ++++- .../generation/dynamo/vllm/generation_cut.py | 23 +++++-- .../generation/prefix_recovery/recovery.py | 18 ++++-- .../generation/test_dynamo_generation_cut.py | 64 +++++++++++++------ .../test_dynamo_vllm_adapter_capture.py | 20 +++++- .../test_vllm_token_capture_hosting.py | 16 +++++ 6 files changed, 120 insertions(+), 36 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/vllm/adapter.py b/nemo_rl/models/generation/dynamo/vllm/adapter.py index a66057eab62..94970160419 100644 --- a/nemo_rl/models/generation/dynamo/vllm/adapter.py +++ b/nemo_rl/models/generation/dynamo/vllm/adapter.py @@ -37,6 +37,7 @@ import math import os import pickle +import time from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional @@ -71,6 +72,8 @@ ) from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( CUT_PREFIXES_KEY, + CUT_REPLY_MARGIN_S, + DEADLINE_KEY, GENERATION_CUT_ANNOTATION, INVENTORY_KEY, ) @@ -228,10 +231,16 @@ async def _generate_cut( # Deferred: Gym is optional outside framework-owned capture. from nemo_gym._checkpoint.generation_cut import GenerationCutInventory - inventory = GenerationCutInventory.model_validate( - (request.get("extra_args") or {})[INVENTORY_KEY] + extra_args = request.get("extra_args") or {} + inventory = GenerationCutInventory.model_validate(extra_args[INVENTORY_KEY]) + deadline = extra_args.get(DEADLINE_KEY) + if deadline is not None: + # The frontend's wall-clock deadline on this host's monotonic clock, + # less the time the answer needs to get back. + deadline = time.monotonic() + (deadline - time.time()) - CUT_REPLY_MARGIN_S + receipt = await self._host._prefix_recovery.run_cut( + inventory, deadline=deadline ) - receipt = await self._host._prefix_recovery.run_cut(inventory) yield { "token_ids": [], "index": 0, diff --git a/nemo_rl/models/generation/dynamo/vllm/generation_cut.py b/nemo_rl/models/generation/dynamo/vllm/generation_cut.py index 3c337d13e33..35a4313be20 100644 --- a/nemo_rl/models/generation/dynamo/vllm/generation_cut.py +++ b/nemo_rl/models/generation/dynamo/vllm/generation_cut.py @@ -32,6 +32,7 @@ import logging import os import threading +import time from collections.abc import Iterable, Sequence from typing import Any @@ -49,6 +50,11 @@ INVENTORY_KEY = "nemo_rl_generation_cut" # The calls an engine cut, in its answer's engine_data. CUT_PREFIXES_KEY = "prefixes" +# The frontend's deadline for the cut, in extra_args: wall-clock seconds, since +# an engine on another host does not share the frontend's monotonic clock. +DEADLINE_KEY = "nemo_rl_generation_cut_deadline" +# What an engine leaves of that deadline for its answer to reach the frontend. +CUT_REPLY_MARGIN_S = 1.0 def merge_cut_receipts(inventory: Any, engine_prefixes: Iterable[Sequence[Any]]) -> Any: @@ -103,22 +109,28 @@ def control_token(self) -> str | None: async def run_cut(self, inventory: Any) -> Any: """Cut ``inventory``'s calls. Safe to await from the control server's loop.""" + # One deadline, from the cut's arrival, for this wait and every engine: + # an engine that started its own later would answer after this gave up. + timeout_s = (read_chat_capture() or {}).get("generation_cut_control_timeout_s") + deadline = None if timeout_s is None else time.time() + timeout_s return await asyncio.wrap_future( asyncio.run_coroutine_threadsafe( - self._cut_on_every_engine(inventory), self._loop + self._cut_on_every_engine(inventory, deadline), self._loop ) ) - async def _cut_on_every_engine(self, inventory: Any) -> Any: + async def _cut_on_every_engine(self, inventory: Any, deadline: float | None) -> Any: client = await self._client request = { # Dynamo's request type requires the field: without it the request # never reaches the engine and the caller is left waiting. "token_ids": [], "annotations": [GENERATION_CUT_ANNOTATION], - "extra_args": {INVENTORY_KEY: inventory.model_dump(mode="json")}, + "extra_args": { + INVENTORY_KEY: inventory.model_dump(mode="json"), + DEADLINE_KEY: deadline, + }, } - capture = read_chat_capture() or {} cuts = { asyncio.ensure_future( self._cut_on_engine(client, request, instance_id) @@ -128,7 +140,8 @@ async def _cut_on_every_engine(self, inventory: Any) -> Any: engine_prefixes = [] if cuts: done, late = await asyncio.wait( - cuts, timeout=capture.get("generation_cut_control_timeout_s") + cuts, + timeout=None if deadline is None else max(0.0, deadline - time.time()), ) for cut in late: cut.cancel() diff --git a/nemo_rl/models/generation/prefix_recovery/recovery.py b/nemo_rl/models/generation/prefix_recovery/recovery.py index 728351f41a7..2355608dbc5 100644 --- a/nemo_rl/models/generation/prefix_recovery/recovery.py +++ b/nemo_rl/models/generation/prefix_recovery/recovery.py @@ -266,11 +266,17 @@ def shutdown(self) -> None: # ---- A checkpoint cut ----------------------------------------------- - async def run_cut(self, inventory: Any) -> Any: - """Cut every call Gym's inventory names, off the serving loop.""" - # Measured from arrival so time queued behind a stale cut counts. - timeout_s = self._parts.control_timeout_s - deadline = None if timeout_s is None else time.monotonic() + timeout_s + async def run_cut(self, inventory: Any, *, deadline: float | None = None) -> Any: + """Cut every call Gym's inventory names, off the serving loop. + + ``deadline`` is a ``time.monotonic()`` bound from a caller that waits + on one of its own (the Dynamo frontend). Without it the cut gets + ``control_timeout_s`` from its arrival here. + """ + if deadline is None: + # Measured from arrival so time queued behind a stale cut counts. + timeout_s = self._parts.control_timeout_s + deadline = None if timeout_s is None else time.monotonic() + timeout_s return await asyncio.get_running_loop().run_in_executor( self._control_executor, functools.partial(self.checkpoint_cut, inventory, deadline=deadline), @@ -878,5 +884,5 @@ def counts(self) -> dict[str, int]: def shutdown(self) -> None: pass - async def run_cut(self, inventory: Any) -> Any: + async def run_cut(self, inventory: Any, *, deadline: float | None = None) -> Any: raise RuntimeError("generation-prefix cuts are disabled on this worker") diff --git a/tests/unit/models/generation/test_dynamo_generation_cut.py b/tests/unit/models/generation/test_dynamo_generation_cut.py index df55e9ea53f..deff97621c3 100644 --- a/tests/unit/models/generation/test_dynamo_generation_cut.py +++ b/tests/unit/models/generation/test_dynamo_generation_cut.py @@ -15,6 +15,7 @@ """NeMo-Gym's checkpoint generation cut through the Dynamo frontend.""" import asyncio +import time import pytest @@ -33,6 +34,7 @@ ) from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( # noqa: E402 CUT_PREFIXES_KEY, + DEADLINE_KEY, GENERATION_CUT_ANNOTATION, INVENTORY_KEY, EngineCutFanOut, @@ -49,6 +51,10 @@ def data(self): return self._data +class _Late(list): + """An engine's answer that reaches the frontend after its deadline.""" + + class _Client: """Dynamo's request-plane client: each engine answers a direct request.""" @@ -66,8 +72,10 @@ async def direct(self, request, instance_id): raise answer async def stream(): - if answer is None: - await asyncio.Event().wait() + if isinstance(answer, _Late): + await asyncio.sleep( + request["extra_args"][DEADLINE_KEY] - time.time() + 5 + ) yield _Item({"token_ids": [], "engine_data": {CUT_PREFIXES_KEY: answer}}) return stream() @@ -102,28 +110,33 @@ def _call(name: str) -> GenerationCutPrefix: def test_a_cut_reaches_every_engine_and_each_call_gets_its_engines_answer( monkeypatch, tmp_path ) -> None: - held, lost = _call("held"), _call("lost") + held, lost, late = _call("held"), _call("lost"), _call("late") inventory = GenerationCutInventory.build( checkpoint_id="checkpoint-1", server_name="policy_model", - active_prefixes=[held, lost], - ) - cut = GenerationCutPrefixAck( - **held.model_dump(), - disposition="durable_prefix", - cut_kind="active_prefix", - frozen_buffer_id="active/checkpoint-1/0", - staging_keys=("__generation_cut__/checkpoint-1/r0/held/0",), - prefix_token_count=3, - prefix_digest="0" * 64, - effective_output_limit=8, + active_prefixes=[held, lost, late], ) + + def cut(call: GenerationCutPrefix) -> dict: + return GenerationCutPrefixAck( + **call.model_dump(), + disposition="durable_prefix", + cut_kind="active_prefix", + frozen_buffer_id="active/checkpoint-1/0", + staging_keys=( + f"__generation_cut__/checkpoint-1/r0/{call.model_call_id}/0", + ), + prefix_token_count=3, + prefix_digest="0" * 64, + effective_output_limit=8, + ).model_dump(mode="json") + client = _Client( { - 1: [], # holds neither call - 2: [cut.model_dump(mode="json")], + 1: [], # holds none of the calls + 2: [cut(held)], 3: RuntimeError("engine gone"), - 4: None, # never answers + 4: _Late([cut(late)]), } ) runtime = _Runtime(client) @@ -149,21 +162,30 @@ async def run(): generation_cut_control_timeout_s=0.5, ) assert fan_out.control_token == "secret" - return await fan_out.run_cut(inventory) + arrived = time.time() + return arrived, await fan_out.run_cut(inventory) - receipt = asyncio.run(run()) + arrived, receipt = asyncio.run(run()) # Gym accepts it as the answer to its inventory: every call acked once. receipt.validate_for(inventory) assert {ack.model_call_id: ack.disposition for ack in receipt.prefixes} == { "held": "durable_prefix", "lost": "durable_failure", + # Its engine cut it, but answered after the frontend stopped waiting. + "late": "durable_failure", } assert runtime.paths == ["nemo-rl-test.backend.generate"] - # Every engine gets the whole inventory; token_ids is always present. + # Every engine gets the whole inventory and the frontend's one deadline, + # counted from the cut's arrival; token_ids is always present. + deadline = client.requests[1]["extra_args"][DEADLINE_KEY] + assert arrived + 0.5 <= deadline <= time.time() request = { "token_ids": [], "annotations": [GENERATION_CUT_ANNOTATION], - "extra_args": {INVENTORY_KEY: inventory.model_dump(mode="json")}, + "extra_args": { + INVENTORY_KEY: inventory.model_dump(mode="json"), + DEADLINE_KEY: deadline, + }, } assert client.requests == {1: request, 2: request, 3: request, 4: request} diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py index 9f60a7c5c95..86e85457db5 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter_capture.py @@ -21,6 +21,7 @@ import base64 import pickle +import time from types import SimpleNamespace from unittest.mock import MagicMock @@ -58,6 +59,8 @@ ) from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( # noqa: E402 CUT_PREFIXES_KEY, + CUT_REPLY_MARGIN_S, + DEADLINE_KEY, GENERATION_CUT_ANNOTATION, INVENTORY_KEY, ) @@ -426,15 +429,30 @@ def test_an_engine_answers_a_cut_with_the_calls_it_cut() -> None: ], ) + run_cut = host._prefix_recovery.run_cut + deadlines = [] + + async def record_deadline(inventory, *, deadline): + deadlines.append(deadline - time.monotonic()) + return await run_cut(inventory, deadline=deadline) + + host._prefix_recovery.run_cut = record_deadline + (chunk,) = _collect( vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0), { "token_ids": [], "annotations": [GENERATION_CUT_ANNOTATION], - "extra_args": {INVENTORY_KEY: inventory.model_dump(mode="json")}, + "extra_args": { + INVENTORY_KEY: inventory.model_dump(mode="json"), + DEADLINE_KEY: time.time() + 30, + }, }, ) + # The cut ran to the frontend's deadline, less the margin for the answer. + (remaining,) = deadlines + assert 30 - CUT_REPLY_MARGIN_S - 5 < remaining <= 30 - CUT_REPLY_MARGIN_S (ack,) = chunk["engine_data"][CUT_PREFIXES_KEY] assert (ack["model_call_id"], ack["disposition"]) == ("held", "durable_prefix") assert ack["prefix_token_count"] == 2 diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index 3370f247c31..c816295ae5d 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -1515,6 +1515,22 @@ def test_generation_cut_past_its_deadline_fails_uncut_calls_without_staging(capl assert _uncut_reasons(caplog) == {"control_deadline": 1} +def test_a_cut_runs_to_the_deadline_its_caller_gives_it(): + """The Dynamo frontend hands every engine the deadline it waits to itself.""" + sink = _MemorySink() + worker = _worker_with_capture(sink) + _live_call(worker, [11, 12], [-0.1, -0.2], prompt_token_ids=(10,)) + inventory = _single_call_inventory() + + # This worker sets no control timeout of its own. + receipt = asyncio.run( + _cuts(worker).run_cut(inventory, deadline=time.monotonic() - 1.0) + ) + + assert [ack.disposition for ack in receipt.prefixes] == ["durable_failure"] + assert sink.generation_prefix_records == [] + + def _active_call_inventory( worker, *calls: tuple[str, list[int]], output_limit: int | None = 16 ) -> tuple[GenerationCutInventory, list]: From a6a13e7efaa09e9d282977d6e6a3e8d67868e247 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 17:26:32 -0800 Subject: [PATCH 69/82] docs(dynamo): the shared error map, refusals and the cut deadline - Design doc: reasoning_ended comes from the template render; one error map answers a refused or failed call on both paths; the frontend sends its deadline with a generation cut; the resume runs start_capture. - Guide: both paths refuse stream: true, n other than 1 and logprobs without top_logprobs; engines cut up to the frontend's deadline. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 17 +++++++++++++---- docs/guides/dynamo-generation.md | 9 +++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 69997285224..c4712fb3e7d 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -109,8 +109,9 @@ one event loop, so it runs in the pool, first in first out. Per call: 1. the pool renders it with the native server's render step (`_render_chat_call`, shared with `vllm/chat_server.py`), including the - token-capture decision for a Gym candidate and the history render the - splice cuts against; + token-capture decision for a Gym candidate, the history render the + splice cuts against, and `reasoning_ended`, read from the template's + render and never from a spliced prompt; 2. the chat engine checks on-policy sampling (`on_policy_sampling`, shared with the native route), reads a token-in prefix from TQ through a read-only client, splices it in with the model's turn-end tokens, and clamps the @@ -121,6 +122,11 @@ one event loop, so it runs in the pool, first in first out. Per call: 4. the pool builds the reply with vLLM's own response builder and parsers, through the same reply fixes for restored calls as the native server. +A call that is refused or fails is answered through one map on both paths +(`error_reply` in `vllm/prompt_and_capture.py`): the status, the error body, +the reason its capture is failed with, and whether Gym is told the capture +call was refused before it was staged. + `GymCallExtras` and `GymCallResult` (`models/generation/gym_call.py`) define the fields both paths exchange with Gym. `tests/unit/models/generation/test_gym_chat_path.py` sends the same calls through the native routes and through this path and @@ -142,10 +148,13 @@ native server's code: every engine with `Client.direct`; the adapter's `nemo_rl:generation_cut` handler runs the worker's own cut and answers with the calls it cut; and the answers are merged into the one receipt Gym expects. A call no engine - cut gets Gym's failure ack and regenerates after a restore. + cut gets Gym's failure ack and regenerates after a restore. The frontend + takes one deadline when the cut arrives and sends it along, so an engine + stops cutting in time for its answer to reach the frontend; an engine + that timed the cut itself would start later and answer too late. - **The resume.** The frontend renders and routes a re-sent call like any other. The adapter then runs the steps the native server runs - (`resolve_restored_call` and its helpers in `vllm/chat_server.py`): it + (`start_capture` in `vllm/prompt_and_capture.py`): it reads and checks the saved rows, replaces the prompt with them, lowers the output budget, and begins capture with the cut. A cut it cannot use (rows gone, or newer than its weights) is dropped, and the frontend's prompt diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 5ac208ec3f8..9486a375ccb 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -106,6 +106,10 @@ native server's steps, with the native server's code: both paths and checks they run the same prompt and `SamplingParams`, stage the same record and return the same reply. +Both paths refuse, with HTTP 400 and before anything runs, a call that streams +(`stream: true`), asks for more than one choice (`n`), or sets `logprobs` +without `top_logprobs`. NeMo-Gym sends none of these. + Where the behavior differs from the native endpoint: - The frontend refuses more than 32 `stop` strings, and vLLM request fields its @@ -129,8 +133,9 @@ Two things differ from the native endpoint: which is the frontend, but the calls run in the engines. The frontend process takes the cut on a port of its own, from the same range as etcd and NATS, and sends it to every engine over Dynamo's request plane; each engine cuts the - calls it holds. Setup tells Gym the port (`generation_cut_control_url` in - Gym's `vllm_model` config). The port is needed until a Dynamo frontend route + calls it holds, up to the frontend's deadline + (`token_capture.control_timeout_s` from the cut's arrival there). Setup tells + Gym the port (`generation_cut_control_url` in Gym's `vllm_model` config). The port is needed until a Dynamo frontend route can take a POST ([ai-dynamo/dynamo#18486](https://github.com/ai-dynamo/dynamo/issues/18486)). - **What the router sees for a resumed call.** The engine replaces the prompt From 26d9d01bbb39cd000293eec45826cb2b34b5d036 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Wed, 11 Nov 2026 18:00:39 -0800 Subject: [PATCH 70/82] test: follow the shared chat-call steps outside the generation tests The full unit lanes found tests elsewhere that the shared steps reach: - test_vllm_http_logprobs_contract pinned that the native server serves logprobs with top_logprobs omitted. The shared sampling check requires the field, since Dynamo's frontend hands NeMo RL an omitted field and an explicit null alike. The test now checks 0 is served and that an omitted field and a null are both refused with a 400. - The fake requests of two capture tests gain the fields the route now reads, and their fake renderers a tokenizer and a model_config. - The fake vLLM request models gain include_reasoning and _grammar_from_parser, which reasoning_ended reads. Co-authored-by: Terry Kong Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- tests/unit/data/test_captured_media.py | 6 +++ tests/unit/environments/test_nemo_gym.py | 43 ++++++++----------- tests/unit/experience/test_capture_context.py | 3 +- .../test_vllm_chat_template_wiring.py | 8 +++- .../models/generation/test_vllm_generation.py | 8 +++- 5 files changed, 40 insertions(+), 28 deletions(-) diff --git a/tests/unit/data/test_captured_media.py b/tests/unit/data/test_captured_media.py index ac68813efbd..ed1ebe6d14e 100644 --- a/tests/unit/data/test_captured_media.py +++ b/tests/unit/data/test_captured_media.py @@ -1430,6 +1430,7 @@ async def test_retained_media_rejection_precedes_inference_and_survives_gym( renderer.renderer = SimpleNamespace( tokenizer=SimpleNamespace(decode=str, eos_token_id=2) ) + renderer.model_config = SimpleNamespace(max_model_len=4096) inference = AsyncMock() async def create_chat_completion(request, raw_request): @@ -1447,6 +1448,11 @@ async def create_chat_completion(request, raw_request): logit_bias=None, top_p=1.0, temperature=1.0, + n=1, + logprobs=False, + top_logprobs=None, + model_fields_set=set(), + stream=False, ng_capture=admission.model_dump(), required_prefix_token_ids=None, model_copy=lambda **kwargs: request, diff --git a/tests/unit/environments/test_nemo_gym.py b/tests/unit/environments/test_nemo_gym.py index 1cea48374ad..b1bfb19428b 100644 --- a/tests/unit/environments/test_nemo_gym.py +++ b/tests/unit/environments/test_nemo_gym.py @@ -2409,14 +2409,17 @@ def _standardize(l: list[dict]): def test_vllm_http_logprobs_contract(nemo_gym_vllm_generation): """Pin the vLLM OpenAI HTTP logprobs contract that NeMo-Gym capture depends on. - NeMo-Gym's vllm_model sets logprobs=True and return_tokens_as_token_ids=True to extract - per-token ids and logprobs for training (Gym omits top_logprobs on the capture path, so - vLLM applies its default; Gym PR #1612 additionally pins top_logprobs=0, which is - equivalent). vLLM computes `logprobs = top_logprobs if logprobs else None`, so omitting - top_logprobs (default 0) or sending 0 returns logprobs, while an explicit null returns - none and silently empties the captured token ids. This exercises the real HTTP path where - that translation lives (the offline LLM API does not), so a vLLM bump that changes the - contract fails here instead of silently freezing training. + NeMo-Gym's vllm_model sets logprobs=True, top_logprobs=0 and + return_tokens_as_token_ids=True to extract per-token ids and logprobs for training. + vLLM computes `logprobs = top_logprobs if logprobs else None`, so sending 0 returns + logprobs, while an explicit null would return none and silently empty the captured + token ids. This exercises the real HTTP path where that translation lives (the offline + LLM API does not), so a vLLM bump that changes the contract fails here instead of + silently freezing training. + + NeMo RL's server requires top_logprobs with logprobs. vLLM alone would serve an + omitted field with its default 0, but Dynamo's frontend hands NeMo RL an omitted + field and an explicit null alike, so both serving paths refuse both. All three cases share the (expensive) vLLM fixture, so they run in a single test rather than as separate parametrized cases. @@ -2461,25 +2464,15 @@ def _assert_has_token_ids(resp, label): token_ids = [int(c["token"].removeprefix("token_id:")) for c in content] assert len(token_ids) == len(content) - # Omitting top_logprobs (what Gym does on the capture path; vLLM default 0) and sending 0 - # (the equivalent explicit pin) must both yield per-token logprobs whose tokens decode to ints. - _assert_has_token_ids(_chat(_OMIT_TOP_LOGPROBS), "omitted top_logprobs") + # Sending 0 (what Gym sends) must yield per-token logprobs whose tokens decode to ints. _assert_has_token_ids(_chat(0), "top_logprobs=0") - # Explicit null is the divergence that motivates the Gym fix: vLLM returns no logprobs - # (200 with logprobs=None) or rejects the request outright. Both mean capture gets - # nothing. If a future vLLM makes null behave like 0, this fails and signals the Gym - # workaround can be relaxed. - null_resp = _chat(None) - if null_resp.status_code == 200: - assert null_resp.json()["choices"][0].get("logprobs") is None - else: - # A rejection must be a client-side validation error, not an unrelated server failure - # that would let this branch pass vacuously. - assert 400 <= null_resp.status_code < 500, ( - f"expected null top_logprobs accepted-with-None or rejected as 4xx, " - f"got {null_resp.status_code}: {null_resp.text}" - ) + # An omitted field and an explicit null are refused before anything runs, so capture + # never gets a reply without token ids. + for field in (_OMIT_TOP_LOGPROBS, None): + refused = _chat(field) + assert refused.status_code == 400, refused.text + assert "top_logprobs" in refused.json()["error"]["message"] @pytest.mark.nemo_gym diff --git a/tests/unit/experience/test_capture_context.py b/tests/unit/experience/test_capture_context.py index e22b9fa7932..81c70407eca 100644 --- a/tests/unit/experience/test_capture_context.py +++ b/tests/unit/experience/test_capture_context.py @@ -322,7 +322,8 @@ async def test_worker_managed_root_clears_untrusted_prefix_and_keeps_media_check ) renderer = _build_server(monkeypatch, {})[0][0] - renderer.renderer = SimpleNamespace() + renderer.renderer = SimpleNamespace(tokenizer=SimpleNamespace(eos_token_id=2)) + renderer.model_config = SimpleNamespace(max_model_len=4096) monkeypatch.setattr( VllmAsyncGenerationWorkerImpl, "token_capture", object(), raising=False ) diff --git a/tests/unit/models/generation/test_vllm_chat_template_wiring.py b/tests/unit/models/generation/test_vllm_chat_template_wiring.py index 250aa573d47..1083e399d7b 100644 --- a/tests/unit/models/generation/test_vllm_chat_template_wiring.py +++ b/tests/unit/models/generation/test_vllm_chat_template_wiring.py @@ -162,7 +162,13 @@ def placeholder(name): ChatCompletionRequest=type( "ChatCompletionRequest", (BaseModel,), - {"model_config": ConfigDict(extra="allow")}, + { + "model_config": ConfigDict(extra="allow"), + # What vLLM's rule for reasoning_ended reads off a request. + "__annotations__": {"include_reasoning": bool}, + "include_reasoning": True, + "_grammar_from_parser": False, + }, ), ChatCompletionResponse=placeholder("ChatCompletionResponse"), ) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 3f6934dc536..d1486b9cf55 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1132,7 +1132,13 @@ class ReasoningParserManager: ChatCompletionRequest=type( "ChatCompletionRequest", (BaseModel,), - {"model_config": ConfigDict(extra="allow")}, + { + "model_config": ConfigDict(extra="allow"), + # What vLLM's rule for reasoning_ended reads off a request. + "__annotations__": {"include_reasoning": bool}, + "include_reasoning": True, + "_grammar_from_parser": False, + }, ), ChatCompletionResponse=ChatCompletionResponse, ) From 56774f269d94db75de0e4da4316349983a5832cb Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 12 Nov 2026 10:08:22 -0800 Subject: [PATCH 71/82] fix(dynamo): start the frontend's chat workers when the engines register Behind Dynamo, the first NeMo-Gym chat call built everything a chat call needs, one piece after another: the frontend imported vLLM on its event loop, the render pool spawned its workers (each imports vLLM and builds the tokenizer, template and processor), and each worker compiled the chat template on its first render. The native worker builds its chat server at startup, so only Dynamo's first step waited: 70-130 s on the cluster parity runs, 10 s on the local Gym capture test. The chat-engine factory now starts all of it in the background when the engines expose chat (vllm_cfg.expose_http_server, which NeMo-Gym requires and the native worker starts its server on), so it runs while the trainer loads: - RenderPool.start() submits one job per worker, which starts every worker; - the steps a call runs are imported in a thread; - each worker runs vLLM's renderer warm-up, as vLLM's API server does. Registration does not wait for it, and a call that arrives first waits for the workers as before. A run that exposes no chat still starts no workers. The first token-capture prefix read still connects to TQ on demand, and now logs how long that took. Local Gym capture test, step 1 exposed generation: 10.6 s before, 0.6 s after, 0.4 s native. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 5 ++- docs/guides/dynamo-generation.md | 3 ++ .../generation/dynamo/vllm/chat_engine.py | 44 +++++++++++++++++++ .../generation/dynamo/vllm/render_pool.py | 29 +++++++++--- .../generation/test_dynamo_chat_engine.py | 34 ++++++++++++++ .../generation/test_dynamo_render_pool.py | 12 +++++ 6 files changed, 120 insertions(+), 7 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index c4712fb3e7d..7ec86482769 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -105,7 +105,10 @@ args and the `policy.generation` chat settings. The chat engine builds a render pool (`dynamo/vllm/render_pool.py`) from them: `render_workers` spawned processes, each holding the vLLM renderer and chat serving the native server builds. Rendering is CPU-bound Python, and the frontend serves every call on -one event loop, so it runs in the pool, first in first out. Per call: +one event loop, so it runs in the pool, first in first out. The workers start +when the first engine registers, if the engines expose chat +(`vllm_cfg.expose_http_server`, which NeMo-Gym requires and the native worker +starts its server on); a run without it starts none. Per call: 1. the pool renders it with the native server's render step (`_render_chat_call`, shared with `vllm/chat_server.py`), including the diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 9486a375ccb..86ab607efcf 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -35,6 +35,9 @@ With `dynamo: true`: - Each engine turns on vLLM's KV-cache events for `router_mode: kv` and registers with the frontend from inside its own worker process. Setup waits until the frontend routes to every engine. +- With NeMo-Gym (`vllm_cfg.expose_http_server: true`), the frontend starts its + render workers when the first engine registers, while the trainer loads, so + Gym's first calls do not wait for them. Other runs start none. - vLLM publishes KV-cache events only for blocks its prefix cache stores. With `vllm_cfg.enable_prefix_caching: false`, the engines publish none, and the KV router routes on load alone. diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index a242466ae68..39c0536028e 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -37,6 +37,8 @@ import asyncio import base64 +import importlib +import logging import os import threading import time @@ -62,6 +64,8 @@ GymRequestFields, ) +LOGGER = logging.getLogger(__name__) + # The key the engine publishes its chat settings under in runtime_data. RUNTIME_DATA_KEY = "nemo_rl_chat" # The annotation the adapter serves NeMo-Gym calls by; the ForwardedCall rides @@ -177,12 +181,18 @@ def __init__(self) -> None: def resolve_prefix(self, serving_admission: Any) -> list[int]: """The admission's prefix, read from TQ (blocking; run off the loop).""" + started = time.monotonic() from nemo_rl.data_plane.tq_token_sink import resolve_admission_prefix if self._prefix_cache is None: with self._connect_lock: if self._prefix_cache is None: self._prefix_cache = self._connect() + LOGGER.info( + "The first token-capture prefix read imported and " + "connected to TQ in %.1fs", + time.monotonic() - started, + ) return resolve_admission_prefix(serving_admission, self._prefix_cache) @staticmethod @@ -214,6 +224,15 @@ def _connect() -> Any: return cache +def _log_failed_start(task: asyncio.Task[None]) -> None: + """Log why the chat engine's start failed; the first call fails the same way.""" + if not task.cancelled() and task.exception() is not None: + LOGGER.error( + "NeMo RL's chat engine could not start its render workers", + exc_info=task.exception(), + ) + + class NemoRLChatEngine: """Serves NeMo-Gym's chat calls for one model's engines.""" @@ -229,6 +248,27 @@ def __init__( self._routed_engine = routed_engine self._generation = spec["generation"] self._capture = capture + # Held here: the event loop keeps only a weak reference to its tasks. + self._start_task: Optional[asyncio.Task[None]] = None + + def start(self) -> None: + """Do in the background what the first chat call would wait for. + + The render workers start, and the steps a call runs are imported (they + import vLLM) in a thread, so the frontend's event loop keeps serving. + A call that arrives first waits for the workers, as without this. + """ + self._start_task = asyncio.get_running_loop().create_task(self._start()) + self._start_task.add_done_callback(_log_failed_start) + + async def _start(self) -> None: + await asyncio.gather( + self._pool.start(), + asyncio.to_thread( + importlib.import_module, + "nemo_rl.models.generation.vllm.prompt_and_capture", + ), + ) async def generate( self, request: dict[str, Any], context: Any = None @@ -452,4 +492,8 @@ async def chat_engine_factory( engine = NemoRLChatEngine( pool=pool, routed_engine=routed_engine, spec=spec, capture=self._capture ) + if spec["generation"]["vllm_cfg"].get("expose_http_server"): + # NeMo-Gym's calls need this flag, and the native worker starts its + # chat server on it. Without it no chat call comes, so nothing starts. + engine.start() return PythonAsyncEngine(engine.generate, asyncio.get_running_loop()) diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index f64251217ba..398fcc418c6 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -35,6 +35,7 @@ import multiprocessing import os import pickle +import time import uuid from concurrent.futures import ProcessPoolExecutor from concurrent.futures.process import BrokenProcessPool @@ -161,6 +162,9 @@ def __init__(self, spec: dict[str, Any], model_dir: str): reasoning_parser=serving_chat_kwargs.get("reasoning_parser"), default_chat_template_kwargs=default_chat_template_kwargs, ) + # As vLLM's API server does at startup: compiles the chat template and + # warms the multimodal processor, so the first render does neither. + self.online_renderer.warmup() # vLLM's chat serving builds replies only here; it never generates. engine_client = SimpleNamespace( model_config=self.model_config, @@ -379,7 +383,11 @@ def _info() -> dict[str, Any]: class RenderPool: - """``workers`` processes that render and answer chat calls, FIFO.""" + """``workers`` processes that render and answer chat calls, FIFO. + + :meth:`start` starts them all. Without it the executor starts one for each + job submitted while none is idle, so the first calls wait for them. + """ def __init__(self, spec: dict[str, Any], model_dir: str, *, workers: int): self._spec = spec @@ -418,12 +426,21 @@ async def _submit(self, fn: Callable[..., Any], *args: Any) -> Any: self._executor = self._start() raise - async def info(self) -> dict[str, Any]: - """The model's turn-end ids and context length, from a started worker. + async def start(self) -> None: + """Start every worker now: one job each, submitted while none is idle.""" + started = time.monotonic() + infos = await asyncio.gather( + *(self._submit(_info) for _ in range(self._workers)) + ) + self._info = infos[0] + LOGGER.info( + "The render pool's first of %d workers is ready after %.1fs", + self._workers, + time.monotonic() - started, + ) - Workers start on first use, so a run that sends no chat call never - builds them. - """ + async def info(self) -> dict[str, Any]: + """The model's turn-end ids and context length, from a started worker.""" if self._info is None: self._info = await self._submit(_info) return self._info diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index 4b4d38eab25..5511ded3005 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -480,3 +480,37 @@ async def build(): with pytest.raises(RuntimeError, match="without NeMo RL's chat settings"): asyncio.run(build()) + + +@pytest.mark.parametrize("exposes_chat", [True, False]) +def test_render_workers_start_at_registration_only_where_chat_is_served( + monkeypatch, exposes_chat: bool +) -> None: + started: list[int] = [] + + class _StartedPool: + def __init__(self, spec: dict, model_dir: str, *, workers: int) -> None: + self._workers = workers + + async def start(self) -> None: + started.append(self._workers) + + monkeypatch.setattr(chat_engine, "RenderPool", _StartedPool) + monkeypatch.setenv(chat_engine.RENDER_WORKERS_ENV, "3") + # NeMo-Gym needs the flag, and the native worker starts its server on it. + generation = {**GENERATION, "vllm_cfg": {"expose_http_server": exposes_chat}} + spec = {"engine_args": {}, "generation": generation} + model = SimpleNamespace( + runtime_config=lambda: {"runtime_data": {chat_engine.RUNTIME_DATA_KEY: spec}}, + name=lambda: "model", + local_dir=lambda: "/snapshots/model", + ) + + async def register() -> None: + await chat_engine.ChatEngineFactory().chat_engine_factory(0, model, _Router()) + # The start runs in the background; registration never waits for it. + await asyncio.gather(*(asyncio.all_tasks() - {asyncio.current_task()})) + + asyncio.run(register()) + + assert started == ([3] if exposes_chat else []) diff --git a/tests/unit/models/generation/test_dynamo_render_pool.py b/tests/unit/models/generation/test_dynamo_render_pool.py index 8728e6c8ddd..640483eae67 100644 --- a/tests/unit/models/generation/test_dynamo_render_pool.py +++ b/tests/unit/models/generation/test_dynamo_render_pool.py @@ -170,6 +170,18 @@ async def serve(): assert json.loads(tool_call["function"]["arguments"]) == {"city": "Lyon"} +def test_start_brings_up_every_worker_before_any_call(tiny_qwen3_model_path) -> None: + async def serve(): + pool = RenderPool(SPEC, tiny_qwen3_model_path, workers=2) + try: + await pool.start() + return len(pool._executor._processes) + finally: + pool.shutdown() + + assert asyncio.run(serve()) == 2 + + def test_a_dead_worker_fails_only_its_calls(tiny_qwen3_model_path) -> None: body = { "model": "model", From 1cd7285fe8e8ed8460156e4537b2e757c1dfae60 Mon Sep 17 00:00:00 2001 From: Zeyu Zhou Date: Thu, 12 Nov 2026 22:16:33 +0000 Subject: [PATCH 72/82] build(gym): bump the Gym submodule Moves the Gym submodule from 5911d628c to 23557b7c2 (Gym main on 2026-11-12). uv.lock is updated with uv 0.11.28, which keeps main's lock format: openai 2.44.0 -> 2.45.0 and nemo-gym's own requirement list. uv 0.13.6, the Dockerfile's version, accepts the file unchanged. Recipes: Gym now ships most of its bundled environment servers as single_agent_turn_legacy. RL recipes declare legacy_agent for many of the same servers on purpose, so each recipe states its path; with both, Gym refused to start the server. Each such block now also deletes single_agent_turn_legacy, so the server keeps the type the recipe declares (102 blocks in 23 files). The single_agent_turn recipe deletes single_agent_turn_legacy instead of legacy_agent. Tests: CaptureContext's rollout_id is now capture_key. The exporter test collects names from the modules that define them, since nemo_gym.openai_utils serves moved names only on request. The checkpoint adapter test's export hooks take checkpoint_id, which Gym now requires. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Zeyu Zhou --- 3rdparty/Gym-workspace/Gym | 2 +- .../mopd-qwen3-1.7b-3n8g-megatron-pack.yaml | 5 +++ ...0n8g-megatron-tp8ep16cp2-async-gym.v1.yaml | 1 + ...-circle-click-2n8g-megatron-tp2ep8.v1.yaml | 1 + ...cle-count-1n4g-megatron_generation.v1.yaml | 1 + ...6n8g-megatron-tp8ep16cp2-async-gym.v1.yaml | 2 ++ .../nemo_gym/distillation_qwen3_0_6b.yaml | 1 + .../gdpo_example_tool_call_multireward.yaml | 1 + examples/nemo_gym/grpo_nanov3.yaml | 5 +++ ...nemotron_nano_v2_9b_single_agent_turn.yaml | 2 +- .../grpo_qwen3_30ba3b_thinking_swe1.yaml | 1 + examples/nemo_gym/grpo_sharded_gym_smoke.yaml | 1 + .../super_rlvr.yaml | 23 ++++++++++++++ .../ultra_rlvr.yaml | 31 +++++++++++++++++++ .../nemotron-3-super/stage2_swe1.yaml | 1 + .../nemotron-3-super/stage3_rlhf.yaml | 1 + .../nemotron-3-ultra/ifbench_teacher.yaml | 4 +++ examples/nemo_gym/nemotron-3-ultra/mopd.yaml | 6 ++++ .../nemotron-3-ultra/reasoning_teacher.yaml | 1 + .../nemotron-3-ultra/student_rlvr2.yaml | 1 + examples/nemo_gym/prefetch_omni_envs.yaml | 6 ++++ .../nemo_gym/prefetch_super_all_envs.yaml | 1 + .../nemo_gym/prefetch_ultra_all_envs.yaml | 6 ++++ ...atron_token_capture_single_controller.yaml | 1 + ..._vllm_token_capture_single_controller.yaml | 1 + .../test_gym_checkpoint_adapter.py | 4 +-- .../test_nemo_gym_token_capture.py | 2 +- .../experience/test_cc_metadata_scaling.py | 2 +- .../test_external_exporter_compatibility.py | 6 ++-- uv.lock | 23 ++++++++------ 30 files changed, 125 insertions(+), 18 deletions(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 5911d628c74..23557b7c2a4 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 5911d628c744d0bf43fe1d9ddf3bda72d59b0f9a +Subproject commit 23557b7c2a48c60975c7719ee25cb6adb42963b7 diff --git a/examples/configs/recipes/llm/mopd-qwen3-1.7b-3n8g-megatron-pack.yaml b/examples/configs/recipes/llm/mopd-qwen3-1.7b-3n8g-megatron-pack.yaml index 5b04ddb0f66..5471145d7d7 100644 --- a/examples/configs/recipes/llm/mopd-qwen3-1.7b-3n8g-megatron-pack.yaml +++ b/examples/configs/recipes/llm/mopd-qwen3-1.7b-3n8g-megatron-pack.yaml @@ -105,6 +105,7 @@ env: # Use legacy agent orchestration. math_with_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -112,6 +113,7 @@ env: name: math_with_judge_simple_agent code_gen_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -126,6 +128,7 @@ env: name: workplace_assistant_simple_agent mcqa_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -133,6 +136,7 @@ env: name: mcqa_simple_agent instruction_following_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -140,6 +144,7 @@ env: name: instruction_following_simple_agent structured_outputs_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/configs/recipes/vlm/mopd-nemotron-super-omni-120ba12b-10n8g-megatron-tp8ep16cp2-async-gym.v1.yaml b/examples/configs/recipes/vlm/mopd-nemotron-super-omni-120ba12b-10n8g-megatron-tp8ep16cp2-async-gym.v1.yaml index 4b33942bf94..c30f3907cfd 100644 --- a/examples/configs/recipes/vlm/mopd-nemotron-super-omni-120ba12b-10n8g-megatron-tp8ep16cp2-async-gym.v1.yaml +++ b/examples/configs/recipes/vlm/mopd-nemotron-super-omni-120ba12b-10n8g-megatron-tp8ep16cp2-async-gym.v1.yaml @@ -55,6 +55,7 @@ env: string_match_environment_server: null circle_count_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-circle-click-2n8g-megatron-tp2ep8.v1.yaml b/examples/configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-circle-click-2n8g-megatron-tp2ep8.v1.yaml index 2dcadfc242f..372e7a5c274 100644 --- a/examples/configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-circle-click-2n8g-megatron-tp2ep8.v1.yaml +++ b/examples/configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-circle-click-2n8g-megatron-tp2ep8.v1.yaml @@ -145,6 +145,7 @@ env: - resources_servers/circle_click/configs/circle_click.yaml circle_click_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-circle-count-1n4g-megatron_generation.v1.yaml b/examples/configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-circle-count-1n4g-megatron_generation.v1.yaml index 3cddda64ee6..080134e12cc 100644 --- a/examples/configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-circle-count-1n4g-megatron_generation.v1.yaml +++ b/examples/configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-circle-count-1n4g-megatron_generation.v1.yaml @@ -89,6 +89,7 @@ env: - resources_servers/circle_count/configs/circle_count.yaml circle_count_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/configs/recipes/vlm/vlm_grpo-nemotron-super-omni-120ba12b-16n8g-megatron-tp8ep16cp2-async-gym.v1.yaml b/examples/configs/recipes/vlm/vlm_grpo-nemotron-super-omni-120ba12b-16n8g-megatron-tp8ep16cp2-async-gym.v1.yaml index 2c55fdcc1da..b3d89573271 100644 --- a/examples/configs/recipes/vlm/vlm_grpo-nemotron-super-omni-120ba12b-16n8g-megatron-tp8ep16cp2-async-gym.v1.yaml +++ b/examples/configs/recipes/vlm/vlm_grpo-nemotron-super-omni-120ba12b-16n8g-megatron-tp8ep16cp2-async-gym.v1.yaml @@ -142,6 +142,7 @@ env: structured_outputs_environment_server: null gui_coordinate_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -149,6 +150,7 @@ env: name: gui_coordinate_simple_agent string_match_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/distillation_qwen3_0_6b.yaml b/examples/nemo_gym/distillation_qwen3_0_6b.yaml index 4dfa593275b..cda3022fc8d 100644 --- a/examples/nemo_gym/distillation_qwen3_0_6b.yaml +++ b/examples/nemo_gym/distillation_qwen3_0_6b.yaml @@ -101,6 +101,7 @@ env: # Use legacy agent orchestration. example_multi_step_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/gdpo_example_tool_call_multireward.yaml b/examples/nemo_gym/gdpo_example_tool_call_multireward.yaml index a7e1826c0dc..9bebd542c8b 100644 --- a/examples/nemo_gym/gdpo_example_tool_call_multireward.yaml +++ b/examples/nemo_gym/gdpo_example_tool_call_multireward.yaml @@ -42,6 +42,7 @@ env: # Use legacy agent orchestration. example_tool_call_multireward_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/grpo_nanov3.yaml b/examples/nemo_gym/grpo_nanov3.yaml index 4a9d39a5062..4c2d3ed529d 100644 --- a/examples/nemo_gym/grpo_nanov3.yaml +++ b/examples/nemo_gym/grpo_nanov3.yaml @@ -331,6 +331,7 @@ env: nemo_gym: math_with_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -338,6 +339,7 @@ env: name: math_with_judge_simple_agent code_gen_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -352,6 +354,7 @@ env: name: workplace_assistant_simple_agent mcqa_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -359,6 +362,7 @@ env: name: mcqa_simple_agent instruction_following_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -366,6 +370,7 @@ env: name: instruction_following_simple_agent structured_outputs_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/grpo_openmathreasoning_nemotron_nano_v2_9b_single_agent_turn.yaml b/examples/nemo_gym/grpo_openmathreasoning_nemotron_nano_v2_9b_single_agent_turn.yaml index fc43a884f14..91c43e70f7c 100644 --- a/examples/nemo_gym/grpo_openmathreasoning_nemotron_nano_v2_9b_single_agent_turn.yaml +++ b/examples/nemo_gym/grpo_openmathreasoning_nemotron_nano_v2_9b_single_agent_turn.yaml @@ -59,7 +59,7 @@ env: openmathreasoning: math_with_judge_environment_server math_with_judge_environment_server: environment_servers: - _delete_key: legacy_agent + _delete_key: single_agent_turn_legacy single_agent_turn: entrypoint: app.py cleanup_timeout_seconds: 180 diff --git a/examples/nemo_gym/grpo_qwen3_30ba3b_thinking_swe1.yaml b/examples/nemo_gym/grpo_qwen3_30ba3b_thinking_swe1.yaml index 836929258dc..478302b0b6a 100644 --- a/examples/nemo_gym/grpo_qwen3_30ba3b_thinking_swe1.yaml +++ b/examples/nemo_gym/grpo_qwen3_30ba3b_thinking_swe1.yaml @@ -228,6 +228,7 @@ env: - resources_servers/single_step_tool_use_with_argument_comparison/configs/swe_pivot_single_step_tool_use_with_argument_comparison.yaml swe_pivot_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/grpo_sharded_gym_smoke.yaml b/examples/nemo_gym/grpo_sharded_gym_smoke.yaml index 0bee4b04c9e..c1f3e9203c1 100644 --- a/examples/nemo_gym/grpo_sharded_gym_smoke.yaml +++ b/examples/nemo_gym/grpo_sharded_gym_smoke.yaml @@ -58,6 +58,7 @@ env: overrides: math_with_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/legacy_environment_servers/super_rlvr.yaml b/examples/nemo_gym/legacy_environment_servers/super_rlvr.yaml index 8032dddb5db..06127d22c22 100644 --- a/examples/nemo_gym/legacy_environment_servers/super_rlvr.yaml +++ b/examples/nemo_gym/legacy_environment_servers/super_rlvr.yaml @@ -3,6 +3,7 @@ env: nemo_gym: math_with_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -10,6 +11,7 @@ env: name: math_with_judge_simple_agent code_gen_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -24,6 +26,7 @@ env: name: workplace_assistant_simple_agent mcqa_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -31,6 +34,7 @@ env: name: mcqa_simple_agent instruction_following_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -38,6 +42,7 @@ env: name: instruction_following_simple_agent structured_outputs_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -45,6 +50,7 @@ env: name: structured_outputs_simple_agent lc_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -52,6 +58,7 @@ env: name: lc_judge_simple_agent calendar_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -73,6 +80,7 @@ env: name: genrm_simple_agent_reasoning_off equivalence_llm_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -80,6 +88,7 @@ env: name: equivalence_llm_judge_simple_agent single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -87,6 +96,7 @@ env: name: single_step_tool_use_with_argument_comparison_agent reasoning_gym_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -94,6 +104,7 @@ env: name: reasoning_gym_simple_agent ns_tools_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -108,6 +119,7 @@ env: name: math_formal_lean_refinement_agent swerl_gen_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -115,6 +127,7 @@ env: name: swerl_gen_simple_agent jailbreak_detection_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -122,6 +135,7 @@ env: name: jailbreak_detection_simple_agent jailbreak_engagement_with_disclaimer_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -129,6 +143,7 @@ env: name: jailbreak_engagement_with_disclaimer jailbreak_refusal_with_explanation_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -136,6 +151,7 @@ env: name: jailbreak_refusal_with_explanation jailbreak_hard_refusal_no_redirection_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -143,6 +159,7 @@ env: name: jailbreak_hard_refusal_no_redirection jailbreak_hard_refusal_with_helplines_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -150,6 +167,7 @@ env: name: jailbreak_hard_refusal_with_helplines over_refusal_detection_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -157,6 +175,7 @@ env: name: over_refusal_detection_simple_agent xstest_eval_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -164,6 +183,7 @@ env: name: xstest_eval_agent multichallenge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -171,6 +191,7 @@ env: name: multichallenge_simple_agent inverse_if_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -178,6 +199,7 @@ env: name: inverse_if_simple_agent search_pivot_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -185,6 +207,7 @@ env: name: search_pivot_single_step_tool_use_with_argument_comparison_agent toolcall_schema_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/legacy_environment_servers/ultra_rlvr.yaml b/examples/nemo_gym/legacy_environment_servers/ultra_rlvr.yaml index 05555543e5f..9a795cbca9c 100644 --- a/examples/nemo_gym/legacy_environment_servers/ultra_rlvr.yaml +++ b/examples/nemo_gym/legacy_environment_servers/ultra_rlvr.yaml @@ -3,6 +3,7 @@ env: nemo_gym: math_with_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -10,6 +11,7 @@ env: name: math_with_judge_simple_agent code_gen_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -24,6 +26,7 @@ env: name: workplace_assistant_simple_agent mcqa_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -31,6 +34,7 @@ env: name: mcqa_simple_agent instruction_following_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -38,6 +42,7 @@ env: name: instruction_following_simple_agent lc_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -45,6 +50,7 @@ env: name: lc_judge_simple_agent calendar_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -66,6 +72,7 @@ env: name: genrm_simple_agent_reasoning_off equivalence_llm_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -73,6 +80,7 @@ env: name: equivalence_llm_judge_simple_agent single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -80,6 +88,7 @@ env: name: single_step_tool_use_with_argument_comparison_agent reasoning_gym_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -87,6 +96,7 @@ env: name: reasoning_gym_simple_agent terminus_judge_string_only_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -94,6 +104,7 @@ env: name: terminus_judge_string_only_simple_agent ns_tools_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -108,6 +119,7 @@ env: name: math_formal_lean_refinement_agent multichallenge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -115,6 +127,7 @@ env: name: multichallenge_simple_agent inverse_if_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -122,6 +135,7 @@ env: name: inverse_if_simple_agent search_pivot_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -129,6 +143,7 @@ env: name: search_pivot_single_step_tool_use_with_argument_comparison_agent toolcall_schema_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -136,6 +151,7 @@ env: name: toolcall_schema_single_step_tool_use_with_argument_comparison_agent swe_pivot_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -143,6 +159,7 @@ env: name: swe_pivot_single_step_tool_use_with_argument_comparison_agent abstention_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -150,6 +167,7 @@ env: name: abstention_simple_agent nvarc_inductive_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -157,6 +175,7 @@ env: name: nvarc_inductive_simple_agent nvarc_transductive_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -164,6 +183,7 @@ env: name: nvarc_transductive_simple_agent droid_pivot_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -171,6 +191,7 @@ env: name: droid_pivot_single_step_tool_use_with_argument_comparison_agent lc_equivalence_rule_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -178,6 +199,7 @@ env: name: lc_equivalence_rule_simple_agent ether0_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -185,6 +207,7 @@ env: name: ether0_simple_agent structured_outputs_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -192,6 +215,7 @@ env: name: structured_outputs_simple_agent structured_outputs_v3_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -199,6 +223,7 @@ env: name: structured_outputs_v3_simple_agent freeform_formatting_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -206,6 +231,7 @@ env: name: freeform_formatting_simple_agent citation_format_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -213,6 +239,7 @@ env: name: citation_format_simple_agent jailbreak_detection_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -220,6 +247,7 @@ env: name: jailbreak_detection_simple_agent jailbreak_engagement_with_disclaimer_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -227,6 +255,7 @@ env: name: jailbreak_engagement_with_disclaimer jailbreak_refusal_with_explanation_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -234,6 +263,7 @@ env: name: jailbreak_refusal_with_explanation jailbreak_hard_refusal_no_redirection_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -241,6 +271,7 @@ env: name: jailbreak_hard_refusal_no_redirection jailbreak_hard_refusal_with_helplines_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/nemotron-3-super/stage2_swe1.yaml b/examples/nemo_gym/nemotron-3-super/stage2_swe1.yaml index 0df1624c77b..384cbcefa14 100644 --- a/examples/nemo_gym/nemotron-3-super/stage2_swe1.yaml +++ b/examples/nemo_gym/nemotron-3-super/stage2_swe1.yaml @@ -313,6 +313,7 @@ env: - resources_servers/single_step_tool_use_with_argument_comparison/configs/swe_pivot_single_step_tool_use_with_argument_comparison.yaml swe_pivot_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/nemotron-3-super/stage3_rlhf.yaml b/examples/nemo_gym/nemotron-3-super/stage3_rlhf.yaml index ff92ba55818..ba6b48b034a 100644 --- a/examples/nemo_gym/nemotron-3-super/stage3_rlhf.yaml +++ b/examples/nemo_gym/nemotron-3-super/stage3_rlhf.yaml @@ -311,6 +311,7 @@ env: name: genrm_simple_agent_reasoning_off single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/nemotron-3-ultra/ifbench_teacher.yaml b/examples/nemo_gym/nemotron-3-ultra/ifbench_teacher.yaml index 87ea3dccce0..e2c5f835496 100644 --- a/examples/nemo_gym/nemotron-3-ultra/ifbench_teacher.yaml +++ b/examples/nemo_gym/nemotron-3-ultra/ifbench_teacher.yaml @@ -391,6 +391,7 @@ env: nemo_gym: abstention_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -412,6 +413,7 @@ env: name: genrm_simple_agent_reasoning_off instruction_following_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -419,6 +421,7 @@ env: name: instruction_following_simple_agent inverse_if_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -426,6 +429,7 @@ env: name: inverse_if_simple_agent multichallenge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/nemotron-3-ultra/mopd.yaml b/examples/nemo_gym/nemotron-3-ultra/mopd.yaml index e5d7ab045a0..9a4ed6fe11f 100644 --- a/examples/nemo_gym/nemotron-3-ultra/mopd.yaml +++ b/examples/nemo_gym/nemotron-3-ultra/mopd.yaml @@ -410,6 +410,7 @@ env: nemo_gym: swe_pivot_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -417,6 +418,7 @@ env: name: swe_pivot_tool_simulation_agent terminal_multi_harness_stirrup_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -424,6 +426,7 @@ env: name: terminal_multi_harness_stirrup_agent terminal_multi_harness_agent006_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -431,6 +434,7 @@ env: name: terminal_multi_harness_agent006_agent terminal_multi_harness_opencode_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -438,6 +442,7 @@ env: name: terminal_multi_harness_opencode_agent terminal_multi_harness_codex_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -445,6 +450,7 @@ env: name: terminal_multi_harness_codex_agent litmus_agent_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/nemotron-3-ultra/reasoning_teacher.yaml b/examples/nemo_gym/nemotron-3-ultra/reasoning_teacher.yaml index 0d132abb2bd..1a82009a817 100644 --- a/examples/nemo_gym/nemotron-3-ultra/reasoning_teacher.yaml +++ b/examples/nemo_gym/nemotron-3-ultra/reasoning_teacher.yaml @@ -392,6 +392,7 @@ env: nemo_gym: equivalence_llm_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/nemotron-3-ultra/student_rlvr2.yaml b/examples/nemo_gym/nemotron-3-ultra/student_rlvr2.yaml index 75180fced4a..015c31dc836 100644 --- a/examples/nemo_gym/nemotron-3-ultra/student_rlvr2.yaml +++ b/examples/nemo_gym/nemotron-3-ultra/student_rlvr2.yaml @@ -394,6 +394,7 @@ env: nemo_gym: litmus_agent_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/prefetch_omni_envs.yaml b/examples/nemo_gym/prefetch_omni_envs.yaml index 0073a236ff1..eb16d7a3701 100644 --- a/examples/nemo_gym/prefetch_omni_envs.yaml +++ b/examples/nemo_gym/prefetch_omni_envs.yaml @@ -33,6 +33,7 @@ env: - resources_servers/circle_count/configs/circle_count.yaml math_with_judge_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -40,6 +41,7 @@ env: name: math_with_judge_simple_agent mcqa_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -47,6 +49,7 @@ env: name: mcqa_simple_agent gui_coordinate_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -54,6 +57,7 @@ env: name: gui_coordinate_simple_agent string_match_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -61,6 +65,7 @@ env: name: string_match_simple_agent circle_click_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -68,6 +73,7 @@ env: name: circle_click_simple_agent circle_count_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/prefetch_super_all_envs.yaml b/examples/nemo_gym/prefetch_super_all_envs.yaml index fb057fcf14b..1549f685673 100644 --- a/examples/nemo_gym/prefetch_super_all_envs.yaml +++ b/examples/nemo_gym/prefetch_super_all_envs.yaml @@ -11,6 +11,7 @@ env: nemo_gym: swe_pivot_single_step_tool_use_with_argument_comparison_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/prefetch_ultra_all_envs.yaml b/examples/nemo_gym/prefetch_ultra_all_envs.yaml index 99d655620c5..08415e5b093 100644 --- a/examples/nemo_gym/prefetch_ultra_all_envs.yaml +++ b/examples/nemo_gym/prefetch_ultra_all_envs.yaml @@ -13,6 +13,7 @@ env: nemo_gym: swe_pivot_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -20,6 +21,7 @@ env: name: swe_pivot_tool_simulation_agent terminal_multi_harness_stirrup_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -27,6 +29,7 @@ env: name: terminal_multi_harness_stirrup_agent terminal_multi_harness_agent006_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -34,6 +37,7 @@ env: name: terminal_multi_harness_agent006_agent terminal_multi_harness_opencode_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -41,6 +45,7 @@ env: name: terminal_multi_harness_opencode_agent terminal_multi_harness_codex_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: @@ -62,6 +67,7 @@ env: name: swe_agents_val litmus_agent_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_megatron_token_capture_single_controller.yaml b/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_megatron_token_capture_single_controller.yaml index 1ed6794435f..b3ec7324a37 100644 --- a/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_megatron_token_capture_single_controller.yaml +++ b/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_megatron_token_capture_single_controller.yaml @@ -47,6 +47,7 @@ env: - resources_servers/string_match/configs/string_match.yaml string_match_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml b/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml index 5353ff36d4c..6a9f2515225 100644 --- a/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml +++ b/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml @@ -42,6 +42,7 @@ env: - resources_servers/string_match/configs/string_match.yaml string_match_environment_server: environment_servers: + _delete_key: single_agent_turn_legacy legacy_agent: entrypoint: app.py agent_server: diff --git a/tests/unit/environments/test_gym_checkpoint_adapter.py b/tests/unit/environments/test_gym_checkpoint_adapter.py index ec5596c82e3..a1c53a5efb9 100644 --- a/tests/unit/environments/test_gym_checkpoint_adapter.py +++ b/tests/unit/environments/test_gym_checkpoint_adapter.py @@ -162,7 +162,7 @@ def test_checkpoint_instance_rejects_unsafe_identity( class _NoResourceSessions: """Resources server hooks with no per-session state to save.""" - async def export_session_states(self, session_ids): + async def export_session_states(self, session_ids, *, checkpoint_id): return {} async def restore_session_states(self, states): @@ -233,7 +233,7 @@ class _AgentHooks: def __init__(self) -> None: self.restored: list[Any] = [] - async def export_agent_sessions(self, session_keys): + async def export_agent_sessions(self, session_keys, *, checkpoint_id): return {key: {"turns": 1} for key in session_keys} async def restore_agent_sessions(self, sessions): diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index aacb80b5269..7c5e9d32103 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -242,7 +242,7 @@ def test_megatron_capture_handler_admits_media_requests(content_part: dict) -> N admission = CaptureAdmission(capture_key="r0", model_call_id="c1", mode="text") context = CaptureContext( - rollout_id="r0", + capture_key="r0", model_call_id="c1", token_sink=None, lineage_store=InMemoryLineageStore(), diff --git a/tests/unit/experience/test_cc_metadata_scaling.py b/tests/unit/experience/test_cc_metadata_scaling.py index bfb5d306c5c..4ee02677055 100644 --- a/tests/unit/experience/test_cc_metadata_scaling.py +++ b/tests/unit/experience/test_cc_metadata_scaling.py @@ -75,7 +75,7 @@ def test_ledger_growth_and_restart_preserve_segment_chains( async def reopen() -> None: ledger = FileLineageStore(tmp_path / "lineage") context = CaptureContext( - rollout_id="attempt", + capture_key="attempt", model_call_id="after-restart", token_sink=None, lineage_store=ledger, diff --git a/tests/unit/experience/test_external_exporter_compatibility.py b/tests/unit/experience/test_external_exporter_compatibility.py index 53df0c93d9c..4d0ad209699 100644 --- a/tests/unit/experience/test_external_exporter_compatibility.py +++ b/tests/unit/experience/test_external_exporter_compatibility.py @@ -20,8 +20,10 @@ pytest.importorskip("nemo_gym", reason="requires the paired Gym checkout") -from nemo_gym import openai_utils, rollout_observability +from nemo_gym import rollout_observability +from nemo_gym.client import openai_client from nemo_gym.config_types import ModelServerRef +from nemo_gym.core import responses_api_types from nemo_gym.openai_utils import ( NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, @@ -72,7 +74,7 @@ def response(index: int, text: str, **updates: Any) -> NeMoGymResponse: def namespace() -> dict: return { name: value - for module in (openai_utils, rollout_observability) + for module in (openai_client, responses_api_types, rollout_observability) for name, value in vars(module).items() if not name.startswith("_") } diff --git a/uv.lock b/uv.lock index f4d027541da..f80b8f8a1d9 100644 --- a/uv.lock +++ b/uv.lock @@ -3388,7 +3388,7 @@ dev = [ { name = "nvidia-modelopt", marker = "extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "nvidia-resiliency-ext" }, { name = "onnxscript" }, - { name = "openai", version = "2.44.0", source = { registry = "https://pypi.org/simple" }, extra = ["aiohttp"], marker = "extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "openai", version = "2.45.0", source = { registry = "https://pypi.org/simple" }, extra = ["aiohttp"], marker = "extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "opentelemetry-api" }, { name = "orjson" }, { name = "pillow" }, @@ -4105,7 +4105,7 @@ dependencies = [ { name = "jsonschema" }, { name = "mcp" }, { name = "omegaconf" }, - { name = "openai", version = "2.44.0", source = { registry = "https://pypi.org/simple" } }, + { name = "openai", version = "2.45.0", source = { registry = "https://pypi.org/simple" } }, { name = "orjson" }, { name = "packaging" }, { name = "psutil" }, @@ -4120,6 +4120,7 @@ dependencies = [ { name = "requests" }, { name = "rich" }, { name = "scipy" }, + { name = "tiktoken" }, { name = "tqdm" }, { name = "urllib3" }, { name = "uvicorn" }, @@ -4156,10 +4157,11 @@ requires-dist = [ { name = "nemo-gym", extras = ["dev", "mlflow", "ray", "sandbox", "wandb"], marker = "extra == 'all'" }, { name = "nemo-lens", extras = ["sdk"], marker = "extra == 'telemetry'", git = "https://github.com/NVIDIA-NeMo/Lens.git?rev=4d045c03186b90afdc815db46b671f6a21bc7e98" }, { name = "omegaconf" }, - { name = "openai", specifier = "==2.44.0" }, + { name = "openai", specifier = "==2.45.0" }, { name = "opensandbox", marker = "extra == 'sandbox'", specifier = ">=0.1.15" }, - { name = "openshell", marker = "extra == 'openshell'", specifier = ">=0.0.92,<0.1" }, + { name = "openshell", marker = "extra == 'openshell'", specifier = ">=0.1.6,<0.2" }, { name = "opentelemetry-instrumentation-fastapi", marker = "extra == 'telemetry'", specifier = ">=0.60b0" }, + { name = "opentelemetry-sdk", marker = "extra == 'dev'" }, { name = "orjson", specifier = ">=3.11.6" }, { name = "packaging", specifier = ">=24.2" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.6.0" }, @@ -4178,12 +4180,13 @@ requires-dist = [ { name = "ray", extras = ["default"], marker = "extra == 'ray'", specifier = ">=2.58.0" }, { name = "requests" }, { name = "requests-mock", marker = "extra == 'dev'" }, - { name = "requests" }, { name = "rich" }, { name = "ruff", marker = "extra == 'dev'" }, { name = "scipy", specifier = ">=1.14.1" }, { name = "sqlparse", marker = "extra == 'mlflow'", specifier = ">=0.6.0" }, { name = "tenacity", marker = "extra == 'sandbox'", specifier = ">=9.1.4" }, + { name = "tiktoken", specifier = ">=0.9.0" }, + { name = "tomlkit", marker = "extra == 'dev'", specifier = ">=0.12" }, { name = "tqdm" }, { name = "urllib3", specifier = ">=2.8.0" }, { name = "uvicorn" }, @@ -5568,7 +5571,7 @@ wheels = [ [[package]] name = "openai" -version = "2.44.0" +version = "2.45.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "platform_machine == 'x86_64' and sys_platform == 'linux'", @@ -5585,9 +5588,9 @@ dependencies = [ { name = "tqdm", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/f5/7c7cb955305cb41f7f3c5fd7e0e38bf6bbf2658468863d4b7b868a5cb8df/openai-2.44.0.tar.gz", hash = "sha256:68a5a5ffad82b8ff7d451c437529fb64f7c3b8123aaf0c021966a882d9e3947d", size = 988753, upload-time = "2026-06-24T20:56:02.293Z" } +sdist = { url = "https://files.pythonhosted.org/packages/78/60/d4219875289b11d2c2f7da93c36283da224a2e55865ed865ab64e0ce9217/openai-2.45.0.tar.gz", hash = "sha256:10d34ca9c5643bce775852fddbfc172505cb1d4de1ccd101696c3ecff358765d", size = 1109653, upload-time = "2026-07-09T18:02:44.091Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/f4/561ed79fd94876160018a5e75254cfcb9b0e62d4dded9dcb20072e86d623/openai-2.44.0-py3-none-any.whl", hash = "sha256:0a2a3ab2e29aeda368700f662ff9ba0f9df17ba4c54577a64e08b8115a3cc0ad", size = 1366216, upload-time = "2026-06-24T20:55:58.882Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b0/2291689e3ec4723fbf5bbf3b54afcd7b160f9ddc98ca7aedfd0132af5677/openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777", size = 1629470, upload-time = "2026-07-09T18:02:42.21Z" }, ] [package.optional-dependencies] @@ -8151,7 +8154,7 @@ dependencies = [ { name = "nvidia-nccl-cu13" }, { name = "nvtx", version = "0.2.16", source = { registry = "https://pypi.org/simple" } }, { name = "omegaconf" }, - { name = "openai", version = "2.44.0", source = { registry = "https://pypi.org/simple" } }, + { name = "openai", version = "2.45.0", source = { registry = "https://pypi.org/simple" } }, { name = "openai-harmony" }, { name = "optimum" }, { name = "ordered-set" }, @@ -9113,7 +9116,7 @@ dependencies = [ { name = "nvidia-cudnn-frontend" }, { name = "nvidia-cutlass-dsl", version = "4.6.2", source = { registry = "https://pypi.org/simple" }, extra = ["cu13"], marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, { name = "nvtx", version = "0.2.15", source = { registry = "https://pypi.org/simple" } }, - { name = "openai", version = "2.44.0", source = { registry = "https://pypi.org/simple" } }, + { name = "openai", version = "2.45.0", source = { registry = "https://pypi.org/simple" } }, { name = "openai-harmony" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp" }, From e1c510f8dc653e28179ece136520693fa66b6fd2 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 12 Nov 2026 16:27:42 -0800 Subject: [PATCH 73/82] fix(dynamo): build the frontend's chat workers while the engines load The render workers started when the engines registered. On a cluster they need 67-86 s to start, and NeMo-Gym's first chat call comes 20-50 s after registration, so the first step still waited for them. The workers are built from the engine's own engine args, which exist only once the engine process has imported vLLM. So for a run that serves chat: - the frontend starts the workers at launch; each imports vLLM and waits for its settings; - an engine sends the driver its chat settings as soon as it has its engine args, before it builds the engine. The driver hands them to the frontend, and the workers build from them; - at registration the chat engine keeps the workers when the engines register the same policy.generation, replaces them when it differs, and builds the waiting workers when no settings came early. The engine args are not compared: vLLM fills in more of them while it builds the engine (scheduler limits, resolved model paths), and no render uses those. Dynamo's discovery is unchanged: the early settings travel over Ray and a file, and the engines register once they have loaded, as before. The first chat call logs how long it waited for the workers, and the first one served logs how long each of its steps took. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 21 +- docs/guides/dynamo-generation.md | 5 +- .../generation/dynamo/engine_adapter.py | 13 +- .../generation/dynamo/managed_runtime.py | 57 ++++-- .../generation/dynamo/vllm/chat_engine.py | 187 +++++++++++++++--- .../models/generation/dynamo/vllm/frontend.py | 14 +- .../generation/dynamo/vllm/render_pool.py | 108 ++++++++-- .../models/generation/vllm/vllm_generation.py | 41 ++++ .../generation/vllm/vllm_worker_async.py | 31 ++- .../generation/test_dynamo_chat_engine.py | 81 +++++++- .../generation/test_dynamo_generation_cut.py | 1 + .../generation/test_dynamo_managed_runtime.py | 21 +- .../generation/test_dynamo_render_pool.py | 49 ++++- .../generation/test_dynamo_vllm_adapter.py | 21 ++ .../models/generation/test_gym_chat_path.py | 4 +- .../models/generation/test_vllm_generation.py | 63 ++++++ 16 files changed, 624 insertions(+), 93 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 7ec86482769..376922b4466 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -105,10 +105,23 @@ args and the `policy.generation` chat settings. The chat engine builds a render pool (`dynamo/vllm/render_pool.py`) from them: `render_workers` spawned processes, each holding the vLLM renderer and chat serving the native server builds. Rendering is CPU-bound Python, and the frontend serves every call on -one event loop, so it runs in the pool, first in first out. The workers start -when the first engine registers, if the engines expose chat -(`vllm_cfg.expose_http_server`, which NeMo-Gym requires and the native worker -starts its server on); a run without it starts none. Per call: +one event loop, so it runs in the pool, first in first out. + +When the engines expose chat (`vllm_cfg.expose_http_server`, which NeMo-Gym +requires and the native worker starts its server on), the workers are built +while the engines load, so NeMo-Gym's first calls do not wait for them: + +- the frontend starts the workers at launch, and each imports vLLM; +- an engine sends the driver its settings as soon as it has its engine args, + before it builds the engine. The driver hands them to the frontend in a + file, as it does the capture config, and the workers build from them; +- at registration the chat engine keeps the workers if the engines register + the same `policy.generation`, and replaces them otherwise. It does not + compare the engine args, which vLLM adds to while it builds the engine + (scheduler limits, resolved model paths; no render uses those). Workers that + got no settings before registration build then. + +A run without chat starts none. Per call: 1. the pool renders it with the native server's render step (`_render_chat_call`, shared with `vllm/chat_server.py`), including the diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 86ab607efcf..e482a9de526 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -36,8 +36,9 @@ With `dynamo: true`: registers with the frontend from inside its own worker process. Setup waits until the frontend routes to every engine. - With NeMo-Gym (`vllm_cfg.expose_http_server: true`), the frontend starts its - render workers when the first engine registers, while the trainer loads, so - Gym's first calls do not wait for them. Other runs start none. + render workers at launch and builds them while the engines load, from the + settings one engine sends before it loads. Gym's first calls then do not + wait for them. Other runs start none. - vLLM publishes KV-cache events only for blocks its prefix cache stores. With `vllm_cfg.enable_prefix_caching: false`, the engines publish none, and the KV router routes on load alone. diff --git a/nemo_rl/models/generation/dynamo/engine_adapter.py b/nemo_rl/models/generation/dynamo/engine_adapter.py index 580080d360e..517f6a0d371 100644 --- a/nemo_rl/models/generation/dynamo/engine_adapter.py +++ b/nemo_rl/models/generation/dynamo/engine_adapter.py @@ -33,7 +33,7 @@ from dataclasses import dataclass from typing import Any, Optional -from dynamo._core import Context +from dynamo._core import Context, fetch_model from dynamo.common.backend import ( EngineConfig, GenerateChunk, @@ -181,6 +181,17 @@ async def cleanup(self) -> None: """Nothing to release: the worker owns its engine and shuts it down.""" +async def weight_free_model_path(model: str) -> str: + """A local directory with ``model``'s config and tokenizer files. + + ``model`` itself when it is a local path; for a Hub ID, Dynamo's snapshot + of it without the weights. + """ + if os.path.exists(model): + return model + return await fetch_model(model, ignore_weights=True) + + def start_dynamo_endpoint( adapter: DynamoEngineAdapter, *, diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index ce712165232..9c6f7d95ac7 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -124,6 +124,10 @@ def _managed_namespace() -> str: # writes the capture config, the Ray cluster its TQ reads go through, and the # port it takes NeMo-Gym's generation cuts on (dynamo/vllm/generation_cut.py). RENDER_WORKERS_ENV = "NEMO_RL_DYNAMO_RENDER_WORKERS" +# Where the driver writes the chat settings an engine sends before it loads. +# Set only for a run that serves chat: the frontend then starts its render +# workers at launch. +CHAT_SETTINGS_FILE_ENV = "NEMO_RL_DYNAMO_CHAT_SETTINGS_FILE" CAPTURE_FILE_ENV = "NEMO_RL_DYNAMO_CAPTURE_FILE" RAY_ADDRESS_ENV = "NEMO_RL_DYNAMO_RAY_ADDRESS" GENERATION_CUT_PORT_ENV = "NEMO_RL_DYNAMO_GENERATION_CUT_PORT" @@ -138,6 +142,15 @@ def read_chat_capture() -> dict[str, Any] | None: return pickle.load(capture_file) +def read_chat_settings() -> dict[str, Any] | None: + """What ``publish_chat_settings`` wrote for this frontend; None before it has.""" + path = os.environ[CHAT_SETTINGS_FILE_ENV] + if not os.path.exists(path): + return None + with open(path) as settings_file: + return json.load(settings_file) + + class ManagedDynamoRuntime: """Own etcd, NATS and the frontend that NeMo RL's vLLM engines register with.""" @@ -148,6 +161,7 @@ def __init__( model_name: str, frontend_python: str, ray_address: str, + serves_chat: bool, ) -> None: """Describe the services; nothing starts until :meth:`start`. @@ -158,11 +172,15 @@ def __init__( the vLLM workers' own. ray_address: The Ray cluster's GCS address; the frontend's chat engine reads token-capture prefixes from TQ through it. + serves_chat: Whether the engines expose chat + (``vllm_cfg.expose_http_server``, which NeMo-Gym requires). The + frontend then starts its render workers at launch. """ self._dynamo_cfg = dynamo_cfg self._model_name = model_name self._frontend_python = frontend_python self._ray_address = ray_address + self.serves_chat = serves_chat self._chat_dir: str | None = None self._namespace = _managed_namespace() self._host = "" @@ -390,6 +408,10 @@ def _start_frontend(self) -> None: self._dynamo_cfg.frontend_args.render_workers ) frontend_env[CAPTURE_FILE_ENV] = os.path.join(self._chat_dir, "capture.pkl") + if self.serves_chat: + frontend_env[CHAT_SETTINGS_FILE_ENV] = os.path.join( + self._chat_dir, "chat_settings.json" + ) frontend_env[RAY_ADDRESS_ENV] = self._ray_address frontend_env[GENERATION_CUT_PORT_ENV] = str(self._generation_cut_port) dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) @@ -415,14 +437,10 @@ def publish_chat_capture( bearer token a generation cut must carry, None to refuse every cut; ``generation_cut_control_timeout_s`` bounds one cut across the engines. """ - if self._chat_dir is None: - raise RuntimeError("Managed Dynamo runtime has not been started") - path = os.path.join(self._chat_dir, "capture.pkl") # Pickled: the data-plane config holds pydantic storage configs. - with tempfile.NamedTemporaryFile( - "wb", dir=self._chat_dir, suffix=".pkl", delete=False - ) as capture_file: - pickle.dump( + self._write_chat_file( + "capture.pkl", + pickle.dumps( { "data_plane": dp_cfg, "staging_partition": staging_partition, @@ -431,11 +449,28 @@ def publish_chat_capture( "generation_cut_control_timeout_s": ( generation_cut_control_timeout_s ), - }, - capture_file, - ) + } + ), + ) + + def publish_chat_settings(self, settings: dict[str, Any]) -> None: + """Hand the frontend the chat settings an engine sent before it loaded. + + ``settings`` is what the engine publishes at registration + (``chat_runtime_data``). The frontend builds its render workers from + them while the engines load, and registration confirms them. + """ + self._write_chat_file("chat_settings.json", json.dumps(settings).encode()) + + def _write_chat_file(self, name: str, content: bytes) -> None: + if self._chat_dir is None: + raise RuntimeError("Managed Dynamo runtime has not been started") + with tempfile.NamedTemporaryFile( + "wb", dir=self._chat_dir, delete=False + ) as chat_file: + chat_file.write(content) # Renamed into place, so the chat engine never reads a partial file. - os.replace(capture_file.name, path) + os.replace(chat_file.name, os.path.join(self._chat_dir, name)) def wait_for_engines( self, expected_engines: int, check_engines: Callable[[], None] diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 39c0536028e..0737e4c2225 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -31,6 +31,8 @@ The engine publishes its engine args and chat settings at registration (``EngineConfig.runtime_data``), so this engine renders exactly as it would. +For a run that serves chat it also sends them before it loads, and the render +workers are built from them meanwhile (:class:`ChatEngineFactory`). """ from __future__ import annotations @@ -43,15 +45,18 @@ import threading import time import uuid +from collections.abc import Coroutine from typing import Any, AsyncGenerator, Optional, TypedDict from dynamo.llm import PythonAsyncEngine from dynamo.llm.exceptions import HttpError, InvalidArgument +from nemo_rl.models.generation.dynamo.engine_adapter import weight_free_model_path from nemo_rl.models.generation.dynamo.managed_runtime import ( RAY_ADDRESS_ENV, RENDER_WORKERS_ENV, read_chat_capture, + read_chat_settings, ) from nemo_rl.models.generation.dynamo.vllm.render_pool import ( RenderPool, @@ -233,6 +238,38 @@ def _log_failed_start(task: asyncio.Task[None]) -> None: ) +async def _start_chat_workers(pool: RenderPool) -> None: + """Do what the first chat call would otherwise wait for. + + The render workers start, and the steps a call runs are imported (they + import vLLM) in a thread, so the frontend's event loop keeps serving. + A call that arrives first waits for the workers, as without this. + """ + await asyncio.gather( + pool.start(), + asyncio.to_thread( + importlib.import_module, + "nemo_rl.models.generation.vllm.prompt_and_capture", + ), + ) + + +class _StepTimes: + """How long each step of a chat call took, for the first call's log line.""" + + def __init__(self) -> None: + self._last = time.monotonic() + self._seconds: dict[str, float] = {} + + def mark(self, step: str) -> None: + now = time.monotonic() + self._seconds[step] = now - self._last + self._last = now + + def __str__(self) -> str: + return ", ".join(f"{step} {took:.1f}s" for step, took in self._seconds.items()) + + class NemoRLChatEngine: """Serves NeMo-Gym's chat calls for one model's engines.""" @@ -248,27 +285,10 @@ def __init__( self._routed_engine = routed_engine self._generation = spec["generation"] self._capture = capture - # Held here: the event loop keeps only a weak reference to its tasks. - self._start_task: Optional[asyncio.Task[None]] = None - - def start(self) -> None: - """Do in the background what the first chat call would wait for. - - The render workers start, and the steps a call runs are imported (they - import vLLM) in a thread, so the frontend's event loop keeps serving. - A call that arrives first waits for the workers, as without this. - """ - self._start_task = asyncio.get_running_loop().create_task(self._start()) - self._start_task.add_done_callback(_log_failed_start) - - async def _start(self) -> None: - await asyncio.gather( - self._pool.start(), - asyncio.to_thread( - importlib.import_module, - "nemo_rl.models.generation.vllm.prompt_and_capture", - ), - ) + # The first call, and the first one served, each log once: what the + # frontend's start-up cost NeMo-Gym's first step. + self._first_call = True + self._first_served_call = True async def generate( self, request: dict[str, Any], context: Any = None @@ -332,8 +352,14 @@ async def _serve( ) request_fields: set[str] = set(GymRequestFields.model_fields) body.update(extras.model_dump(include=request_fields)) + first_call, self._first_call = self._first_call, False + steps = _StepTimes() info = await self._pool.info() + steps.mark("render workers") + if first_call: + LOGGER.info("The first chat call waited for the %s", steps) rendered = await self._pool.render(body, measure=extras.ng_measure) + steps.mark("render") plan = await prepare_prompt( rendered, resolve_prefix=self._capture.resolve_prefix, @@ -341,6 +367,7 @@ async def _serve( max_model_len=info["max_model_len"], measure_only=extras.ng_measure, ) + steps.mark("prefix and splice") if extras.ng_measure: return self._chunk( request, @@ -378,6 +405,7 @@ async def _serve( "extra_args": {CALL_KEY: wire}, } engine_data = await self._call_engine(preprocessed, context) + steps.mark("engine") if (refusal := engine_data.get("refusal")) is not None: # The engine refused the call before it ran, with the native status. ack = _capture_refusal_ack(extras) if refusal["acknowledge"] else "" @@ -394,6 +422,10 @@ async def _serve( engine_data["output"], engine_data.get("restored"), ) + steps.mark("reply") + if self._first_served_call: + self._first_served_call = False + LOGGER.info("The first chat call served took: %s", steps) (choice,) = response["choices"] return self._chunk( request, @@ -469,10 +501,106 @@ def _chunk( class ChatEngineFactory: - """Dynamo's ``chat_engine_factory``: one chat engine per model's engines.""" + """Dynamo's ``chat_engine_factory``: one chat engine per model's engines. + + The render workers are built from an engine's chat settings, which reach + the frontend when the engine registers. For a run that serves chat the + driver also hands over the settings one engine sends before it loads + (``read_chat_settings``): :meth:`start_render_workers` starts the workers + at launch and builds them from those, and registration keeps them when the + engines register the same ``policy.generation``. + """ def __init__(self) -> None: self._capture = _CaptureSource() + # The workers started at launch and the settings they were built from + # (None until those arrive), until a registration takes them. + self._early_pool: Optional[RenderPool] = None + self._early_spec: Optional[dict[str, Any]] = None + # Held here: the event loop keeps only a weak reference to its tasks. + self._tasks: list[asyncio.Task[None]] = [] + + def start_render_workers(self) -> None: + """Start the render workers before any engine registers. + + Call it on the frontend's loop, in a run that serves chat. + """ + self._early_pool = self._new_pool(start=True) + self._run(self._build_early_pool(self._early_pool)) + + def _run(self, work: Coroutine[Any, Any, None]) -> None: + task = asyncio.get_running_loop().create_task(work) + task.add_done_callback(_log_failed_start) + self._tasks.append(task) + + def _new_pool(self, *, start: bool) -> RenderPool: + pool = RenderPool(workers=int(os.environ[RENDER_WORKERS_ENV])) + if start: + self._run(_start_chat_workers(pool)) + return pool + + async def _build_early_pool(self, pool: RenderPool) -> None: + """Build ``pool`` from the settings an engine sends before it loads.""" + launched = time.monotonic() + while (runtime_data := read_chat_settings()) is None: + if self._early_pool is not pool: + return + await asyncio.sleep(0.2) + spec = runtime_data[RUNTIME_DATA_KEY] + # The directory an engine registers from; registration brings Dynamo's + # copy of the same files. + model_dir = await weight_free_model_path(spec["generation"]["model_name"]) + if self._early_pool is not pool: + # The engines registered meanwhile, and their settings built it. + return + self._early_spec = spec + pool.build(spec, model_dir) + LOGGER.info( + "Building the render workers from the chat settings an engine sent " + "%.1fs after the frontend launched, before it loaded", + time.monotonic() - launched, + ) + + def _take_early_pool( + self, spec: dict[str, Any], model_dir: str + ) -> Optional[RenderPool]: + """The workers started at launch, built for ``spec``; None without them. + + Workers built from settings sent early are kept when the engines + register the same ``generation`` (``policy.generation``), which every + engine derives its engine args from. The engine args are not compared: + vLLM fills in more of them while it builds the engine (scheduler + limits, resolved model paths), and no render uses those. + """ + pool, self._early_pool = self._early_pool, None + early_spec, self._early_spec = self._early_spec, None + if pool is None: + return None + if early_spec is None: + # No settings came before registration: the workers have imported + # vLLM and build now. + pool.build(spec, model_dir) + return pool + early, registered = early_spec["generation"], spec["generation"] + changed = sorted( + key + for key in early.keys() | registered.keys() + if early.get(key) != registered.get(key) + ) + if changed: + LOGGER.warning( + "The engines registered policy.generation settings other than " + "the ones one sent before it loaded (%s); restarting the render " + "workers", + ", ".join(changed), + ) + pool.shutdown() + return None + LOGGER.info( + "The engines registered the settings the render workers were built " + "from; keeping the workers" + ) + return pool async def chat_engine_factory( self, instance_id: Any, mdc: Any, routed_engine: Any @@ -486,14 +614,15 @@ async def chat_engine_factory( f"(runtime_data[{RUNTIME_DATA_KEY!r}]); only NeMo RL's engines are " "served by this frontend" ) - pool = RenderPool( - spec, mdc.local_dir(), workers=int(os.environ[RENDER_WORKERS_ENV]) - ) + pool = self._take_early_pool(spec, mdc.local_dir()) + if pool is None: + # NeMo-Gym's calls need this flag, and the native worker starts its + # chat server on it. Without it no chat call comes, so nothing starts. + pool = self._new_pool( + start=bool(spec["generation"]["vllm_cfg"].get("expose_http_server")) + ) + pool.build(spec, mdc.local_dir()) engine = NemoRLChatEngine( pool=pool, routed_engine=routed_engine, spec=spec, capture=self._capture ) - if spec["generation"]["vllm_cfg"].get("expose_http_server"): - # NeMo-Gym's calls need this flag, and the native worker starts its - # chat server on it. Without it no chat call comes, so nothing starts. - engine.start() return PythonAsyncEngine(engine.generate, asyncio.get_running_loop()) diff --git a/nemo_rl/models/generation/dynamo/vllm/frontend.py b/nemo_rl/models/generation/dynamo/vllm/frontend.py index 86e32335583..a6cbfd02d35 100644 --- a/nemo_rl/models/generation/dynamo/vllm/frontend.py +++ b/nemo_rl/models/generation/dynamo/vllm/frontend.py @@ -23,15 +23,20 @@ Rust preprocessor. ``/v1/completions`` (NeMo RL's own rollouts, token ids in) is unchanged. It wraps ``DistributedRuntime`` the same way, to serve NeMo-Gym's checkpoint generation cuts through the frontend's own runtime -(:mod:`~nemo_rl.models.generation.dynamo.vllm.generation_cut`). +(:mod:`~nemo_rl.models.generation.dynamo.vllm.generation_cut`) and, in a run +that serves chat, to start the render workers at launch. """ +import os import sys def main() -> None: import dynamo.frontend.main as frontend_main + from nemo_rl.models.generation.dynamo.managed_runtime import ( + CHAT_SETTINGS_FILE_ENV, + ) from nemo_rl.models.generation.dynamo.vllm.chat_engine import ChatEngineFactory from nemo_rl.models.generation.dynamo.vllm.generation_cut import ( start_generation_cut_server, @@ -52,13 +57,16 @@ def with_chat_engine(engine_type, **kwargs): distributed_runtime = frontend_main.DistributedRuntime - def with_generation_cuts(loop, *args, **kwargs): + def on_the_frontends_loop(loop, *args, **kwargs): runtime = distributed_runtime(loop, *args, **kwargs) start_generation_cut_server(runtime, loop) + if CHAT_SETTINGS_FILE_ENV in os.environ: + # The driver names the file for a run that serves chat. + factory.start_render_workers() return runtime frontend_main.EntrypointArgs = with_chat_engine - frontend_main.DistributedRuntime = with_generation_cuts + frontend_main.DistributedRuntime = on_the_frontends_loop sys.argv[0] = "dynamo.frontend" frontend_main.main() diff --git a/nemo_rl/models/generation/dynamo/vllm/render_pool.py b/nemo_rl/models/generation/dynamo/vllm/render_pool.py index 398fcc418c6..3bd2c775978 100644 --- a/nemo_rl/models/generation/dynamo/vllm/render_pool.py +++ b/nemo_rl/models/generation/dynamo/vllm/render_pool.py @@ -26,15 +26,22 @@ output, so the reply is the one the native server builds. Jobs queue FIFO in the pool; nothing renders on the event loop. + +A worker imports vLLM before it has the settings it is built from, so the +frontend can start the workers at launch, before any engine has registered. """ from __future__ import annotations import asyncio +import importlib +import json import logging import multiprocessing import os import pickle +import shutil +import tempfile import time import uuid from concurrent.futures import ProcessPoolExecutor @@ -92,20 +99,28 @@ class RenderedCall: reasoning_parser_kwargs: Optional[dict[str, Any]] +def _import_vllm() -> None: + """Import everything a worker builds and serves with.""" + import vllm.platforms + + # A frontend host may have no accelerator; vLLM only renders here, as in + # Dynamo's own frontend. vLLM's explicit CPU target selects its CPU + # platform. An accelerator host keeps its own, which validates the + # engine's args (quantization, for one) as the engine does. + if vllm.platforms.resolve_current_platform_cls_qualname().endswith( + ".UnspecifiedPlatform" + ): + os.environ["VLLM_TARGET_DEVICE"] = "cpu" + # The chat server imports the rest of vLLM that a worker uses. + importlib.import_module("nemo_rl.models.generation.vllm.chat_server") + + class _Worker: """The vLLM objects one worker process renders and responds with.""" def __init__(self, spec: dict[str, Any], model_dir: str): - import vllm.platforms - - # A frontend host may have no accelerator; vLLM only renders here, as in - # Dynamo's own frontend. vLLM's explicit CPU target selects its CPU - # platform. An accelerator host keeps its own, which validates the - # engine's args (quantization, for one) as the engine does. - if vllm.platforms.resolve_current_platform_cls_qualname().endswith( - ".UnspecifiedPlatform" - ): - os.environ["VLLM_TARGET_DEVICE"] = "cpu" + started = time.monotonic() + _import_vllm() from types import SimpleNamespace @@ -199,6 +214,7 @@ def __init__(self, spec: dict[str, Any], model_dir: str): model_stop_token_ids(self.model_config, cfg), self.tokenizer ) ) + self.build_seconds = time.monotonic() - started def _parser(self, chat: Any) -> Any: parser_cls = self.serving_chat.parser_cls @@ -354,9 +370,26 @@ async def results(): return response.model_dump(mode="json", exclude_none=True) -def _init_worker(spec: dict[str, Any], model_dir: str) -> None: +def _wait_for_settings(path: str) -> tuple[dict[str, Any], str]: + """The settings :meth:`RenderPool.build` writes, once it has.""" + frontend = multiprocessing.parent_process() + while not os.path.exists(path): + # None will come once the pool is shut down or the frontend is gone. + if not os.path.isdir(os.path.dirname(path)) or ( + frontend is not None and not frontend.is_alive() + ): + raise SystemExit("the render pool went away before it had settings") + time.sleep(0.1) + with open(path) as settings_file: + settings = json.load(settings_file) + return settings["spec"], settings["model_dir"] + + +def _init_worker(settings_path: str) -> None: global _WORKER - _WORKER = _Worker(spec, model_dir) + # Before the wait, so that only the build is left once the settings arrive. + _import_vllm() + _WORKER = _Worker(*_wait_for_settings(settings_path)) def _render(body: dict[str, Any], measure: bool) -> RenderedCall: @@ -379,30 +412,53 @@ def _info() -> dict[str, Any]: return { "turn_end_token_ids": _WORKER.turn_end_token_ids, "max_model_len": _WORKER.model_config.max_model_len, + "build_seconds": _WORKER.build_seconds, } class RenderPool: """``workers`` processes that render and answer chat calls, FIFO. - :meth:`start` starts them all. Without it the executor starts one for each - job submitted while none is idle, so the first calls wait for them. + A worker imports vLLM, then waits for the settings :meth:`build` gives the + pool, so :meth:`start` can come first. :meth:`start` starts them all; + without it the executor starts one for each job submitted while none is + idle, so the first calls wait for them. """ - def __init__(self, spec: dict[str, Any], model_dir: str, *, workers: int): - self._spec = spec - self._model_dir = model_dir + def __init__(self, *, workers: int): self._workers = workers + # Where build() writes the settings; a restarted pool's workers read + # them here too. + self._settings_dir = tempfile.mkdtemp(prefix="nemorl_render_pool_") + self._settings_path = os.path.join(self._settings_dir, "settings.json") + self._built_at: Optional[float] = None self._executor = self._start() self._info: Optional[dict[str, Any]] = None + def build(self, spec: dict[str, Any], model_dir: str) -> None: + """Give the workers what they are built from. + + ``spec`` is the engine's chat settings (``chat_runtime_data``), and + ``model_dir`` a local directory with the model's config and tokenizer + files. + """ + if self._built_at is not None: + raise RuntimeError("the render pool already has its settings") + with tempfile.NamedTemporaryFile( + "w", dir=self._settings_dir, delete=False + ) as settings_file: + json.dump({"spec": spec, "model_dir": model_dir}, settings_file) + # Renamed into place, so a worker never reads a partial file. + os.replace(settings_file.name, self._settings_path) + self._built_at = time.monotonic() + def _start(self) -> ProcessPoolExecutor: return ProcessPoolExecutor( max_workers=self._workers, # Fork would copy the frontend's Rust runtime and event loop. mp_context=multiprocessing.get_context("spawn"), initializer=_init_worker, - initargs=(self._spec, self._model_dir), + initargs=(self._settings_path,), ) async def _submit(self, fn: Callable[..., Any], *args: Any) -> Any: @@ -427,16 +483,25 @@ async def _submit(self, fn: Callable[..., Any], *args: Any) -> Any: raise async def start(self) -> None: - """Start every worker now: one job each, submitted while none is idle.""" + """Start every worker now: one job each, submitted while none is idle. + + Returns when the first is ready, which takes :meth:`build`'s settings. + """ started = time.monotonic() infos = await asyncio.gather( *(self._submit(_info) for _ in range(self._workers)) ) self._info = infos[0] + ready = time.monotonic() + assert self._built_at is not None LOGGER.info( - "The render pool's first of %d workers is ready after %.1fs", + "The render pool's first of %d workers is ready %.1fs after they " + "started and %.1fs after their settings arrived; it took %.1fs to " + "build", self._workers, - time.monotonic() - started, + ready - started, + ready - self._built_at, + self._info["build_seconds"], ) async def info(self) -> dict[str, Any]: @@ -463,3 +528,4 @@ async def respond( def shutdown(self) -> None: self._executor.shutdown(wait=False, cancel_futures=True) + shutil.rmtree(self._settings_dir, ignore_errors=True) diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 07808f28993..e5670a3f319 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -32,6 +32,8 @@ import numpy as np import ray from ray.util.placement_group import PlacementGroup +from ray.util.queue import Empty +from ray.util.queue import Queue as RayQueue from nemo_rl.data.multimodal_utils import VLLM_PROMPT_KEYS from nemo_rl.distributed.batched_data_dict import BatchedDataDict, SlicedDataDict @@ -756,11 +758,24 @@ def load_and_start(self) -> None: heavy model loading. Updates dp_openai_server_base_urls with the actual running server URLs and populates device_uuids. """ + chat_settings: Optional[RayQueue] = None + if self._dynamo_runtime is not None and self._dynamo_runtime.serves_chat: + # The frontend builds its render workers from one engine's chat + # settings while the engines load, so NeMo-Gym's first calls do + # not wait for them. + chat_settings = RayQueue(actor_options={"num_cpus": 0}) + leader = self.worker_group.workers[ + self.worker_group.get_dp_leader_worker_idx(0) + ] + ray.get(leader.announce_dynamo_chat_settings_to.remote(chat_settings)) + # Call load_model() on all model-owner workers futures = self.worker_group.run_all_workers_single_data( "load_model", run_rank_0_only_axes=["tensor_parallel", "pipeline_parallel"], ) + if chat_settings is not None: + self._publish_dynamo_chat_settings(chat_settings, futures) ray.get(futures) # Post-init (collective rpc functions needed for async engine) @@ -809,12 +824,38 @@ def _start_dynamo_services(self) -> None: # release as every engine. frontend_python=self.worker_group.py_executable, ray_address=ray.get_runtime_context().gcs_address, + # NeMo-Gym's chat calls need the flag, and the native worker starts + # its chat server on it. + serves_chat=bool(self.cfg["vllm_cfg"].get("expose_http_server")), ) runtime.start() self._dynamo_runtime = runtime self.dynamo_frontend_url = runtime.frontend_url self.dynamo_generation_cut_url = runtime.generation_cut_control_url + def _publish_dynamo_chat_settings( + self, chat_settings: RayQueue, loads: list[ray.ObjectRef] + ) -> None: + """Hand the frontend the chat settings an engine sends before it loads. + + Returns without them once any of ``loads`` has ended: registration + then gives the frontend its settings, as without this. + """ + assert self._dynamo_runtime is not None + try: + while True: + try: + settings = chat_settings.get(timeout=1) + except Empty: + ended, _ = ray.wait(loads, num_returns=1, timeout=0) + if ended: + return + continue + self._dynamo_runtime.publish_chat_settings(settings) + return + finally: + chat_settings.shutdown() + def _register_dynamo_engines(self) -> None: """Serve every shard's engine through Dynamo; wait until it routes to all.""" runtime = self._dynamo_runtime diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index fc89a375fdc..ad06f289509 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -17,7 +17,6 @@ import copy import gc import logging -import os import secrets import threading import time @@ -31,6 +30,7 @@ import uvicorn from fastapi import FastAPI, Request, Response from fastapi.responses import JSONResponse +from ray.util.queue import Queue as RayQueue from nemo_rl.data.captured_media import ( CapturedMedia, @@ -364,6 +364,8 @@ def __init__( self._http_engine_client = None # The task serving this engine through Dynamo, when it is registered. self._dynamo_endpoint_task: Optional[asyncio.Task] = None + # Where this engine sends its chat settings before it loads, when asked to. + self._dynamo_chat_settings_queue: Optional[RayQueue] = None self._init_token_capture_state() @@ -543,6 +545,15 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: ) self.llm_async_engine_args = AsyncEngineArgs(**llm_kwargs) + if self._dynamo_chat_settings_queue is not None: + # ai-dynamo is optional: import it only for engines served through Dynamo. + from nemo_rl.models.generation.dynamo.vllm.adapter import ( + chat_runtime_data, + ) + + # Sent before the engine is built, which takes far longer than the + # frontend's render workers do. + self._dynamo_chat_settings_queue.put(chat_runtime_data(self)) self.llm = AsyncLLM.from_engine_args( self.llm_async_engine_args, stat_loggers=self.stat_loggers ) @@ -723,6 +734,15 @@ async def post_init_async(self): self._setup_vllm_server() ) + def announce_dynamo_chat_settings_to(self, queue: RayQueue) -> None: + """Have this engine send ``queue`` its chat settings before it loads. + + It sends what it publishes at registration, as soon as it has its engine + args. Dynamo's frontend builds its render workers from them while the + engine loads. + """ + self._dynamo_chat_settings_queue = queue + async def start_dynamo_endpoint_async( self, *, namespace: str, engine_env: dict[str, str], leader_worker_idx: int ) -> None: @@ -734,20 +754,15 @@ async def start_dynamo_endpoint_async( engine serves report it, as ``generate_async`` results do. """ # ai-dynamo is optional: import it only for engines served through Dynamo. - from dynamo._core import fetch_model - from nemo_rl.models.generation.dynamo.engine_adapter import ( start_dynamo_endpoint, + weight_free_model_path, ) from nemo_rl.models.generation.dynamo.vllm.adapter import DynamoVllmAdapter # Dynamo fetches a Hub ID's whole checkpoint before it registers; the engine's # weights come from refit, so register the config and tokenizer files only. - model_path = ( - self.model_name - if os.path.exists(self.model_name) - else await fetch_model(self.model_name, ignore_weights=True) - ) + model_path = await weight_free_model_path(self.model_name) kv_events = self.llm.vllm_config.kv_events_config publishes = kv_events is not None and kv_events.enable_kv_cache_events # The Dynamo functional overlays check the router applied these. diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index 5511ded3005..f591fc79fd7 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -30,6 +30,7 @@ from dynamo.llm.exceptions import HttpError, InvalidArgument # noqa: E402 +from nemo_rl.models.generation.dynamo import managed_runtime # noqa: E402 from nemo_rl.models.generation.dynamo.vllm import chat_engine # noqa: E402 from nemo_rl.models.generation.dynamo.vllm.render_pool import ( # noqa: E402 RenderedCall, @@ -489,9 +490,12 @@ def test_render_workers_start_at_registration_only_where_chat_is_served( started: list[int] = [] class _StartedPool: - def __init__(self, spec: dict, model_dir: str, *, workers: int) -> None: + def __init__(self, *, workers: int) -> None: self._workers = workers + def build(self, spec: dict, model_dir: str) -> None: + pass + async def start(self) -> None: started.append(self._workers) @@ -514,3 +518,78 @@ async def register() -> None: asyncio.run(register()) assert started == ([3] if exposes_chat else []) + + +@pytest.mark.parametrize( + "sent_early, kept, built_from", + [ + # The engine registers the policy.generation it sent before it loaded. + ("the same generation", True, "/snapshots/early"), + ("another generation", False, "/snapshots/registered"), + # Nothing came before registration: the waiting workers build then. + (None, True, "/snapshots/registered"), + ], +) +def test_registration_keeps_render_workers_built_from_the_same_settings( + monkeypatch, tmp_path, sent_early, kept: bool, built_from: str +) -> None: + pools: list = [] + + class _Pool: + def __init__(self, *, workers: int) -> None: + self.built: list[tuple[dict, str]] = [] + self.shut_down = False + pools.append(self) + + def build(self, spec: dict, model_dir: str) -> None: + self.built.append((spec, model_dir)) + + async def start(self) -> None: + pass + + def shutdown(self) -> None: + self.shut_down = True + + async def early_model_dir(model: str) -> str: + return "/snapshots/early" + + monkeypatch.setattr(chat_engine, "RenderPool", _Pool) + monkeypatch.setattr(chat_engine, "weight_free_model_path", early_model_dir) + monkeypatch.setenv(chat_engine.RENDER_WORKERS_ENV, "3") + settings_file = tmp_path / "chat_settings.json" + monkeypatch.setenv(managed_runtime.CHAT_SETTINGS_FILE_ENV, str(settings_file)) + generation = { + **GENERATION, + "model_name": "model", + "vllm_cfg": {"expose_http_server": True}, + } + spec = {"engine_args": {}, "generation": generation} + if sent_early is not None: + early = { + # vLLM fills in more engine args while it builds the engine. + "engine_args": {"max_num_seqs": None}, + "generation": { + **generation, + "top_p": 0.9 if sent_early == "the same generation" else 0.5, + }, + } + settings_file.write_text(json.dumps({chat_engine.RUNTIME_DATA_KEY: early})) + model = SimpleNamespace( + runtime_config=lambda: {"runtime_data": {chat_engine.RUNTIME_DATA_KEY: spec}}, + name=lambda: "model", + local_dir=lambda: "/snapshots/registered", + ) + + async def launch_then_register() -> None: + factory = chat_engine.ChatEngineFactory() + factory.start_render_workers() + while sent_early is not None and not pools[0].built: + await asyncio.sleep(0.01) + await factory.chat_engine_factory(0, model, _Router()) + + asyncio.run(launch_then_register()) + + assert pools[0].shut_down is not kept + assert len(pools) == (1 if kept else 2) + ((built_spec, model_dir),) = pools[-1].built + assert (built_spec["generation"], model_dir) == (generation, built_from) diff --git a/tests/unit/models/generation/test_dynamo_generation_cut.py b/tests/unit/models/generation/test_dynamo_generation_cut.py index deff97621c3..7c960fc6f3f 100644 --- a/tests/unit/models/generation/test_dynamo_generation_cut.py +++ b/tests/unit/models/generation/test_dynamo_generation_cut.py @@ -147,6 +147,7 @@ def cut(call: GenerationCutPrefix) -> dict: model_name="model", frontend_python="python", ray_address="10.0.0.1:6379", + serves_chat=True, ) driver._chat_dir = str(tmp_path) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 9d96422e1d9..446c6cc77f8 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -29,7 +29,7 @@ ) -def _runtime(**frontend_args) -> ManagedDynamoRuntime: +def _runtime(serves_chat: bool = False, **frontend_args) -> ManagedDynamoRuntime: return ManagedDynamoRuntime( dynamo_cfg=DynamoCfg( startup_timeout_s=5, frontend_args=DynamoFrontendArgs(**frontend_args) @@ -37,6 +37,7 @@ def _runtime(**frontend_args) -> ManagedDynamoRuntime: model_name="model", frontend_python="/venvs/vllm/bin/python", ray_address="10.0.0.1:6379", + serves_chat=serves_chat, ) @@ -288,10 +289,11 @@ def check_engines() -> None: runtime.wait_for_engines(1, check_engines=check_engines) +@pytest.mark.parametrize("serves_chat", [True, False]) def test_frontend_launches_nemo_rls_chat_engine_from_the_given_interpreter( - monkeypatch, + monkeypatch, serves_chat: bool ) -> None: - runtime = _runtime(render_workers=3) + runtime = _runtime(serves_chat, render_workers=3) runtime._frontend_port = 3000 runtime._generation_cut_port = 2400 runtime._namespace = "nemo-rl-test" @@ -332,6 +334,19 @@ def test_frontend_launches_nemo_rls_chat_engine_from_the_given_interpreter( "generation_cut_control_token": "secret", "generation_cut_control_timeout_s": 60.0, } + # Only a run that serves chat names the file an engine's chat settings + # arrive in before it loads; the frontend starts its render workers on it. + assert (managed_runtime.CHAT_SETTINGS_FILE_ENV in env) is serves_chat + if serves_chat: + monkeypatch.setenv( + managed_runtime.CHAT_SETTINGS_FILE_ENV, + env[managed_runtime.CHAT_SETTINGS_FILE_ENV], + ) + assert managed_runtime.read_chat_settings() is None + runtime.publish_chat_settings({"nemo_rl_chat": {"engine_args": {}}}) + assert managed_runtime.read_chat_settings() == { + "nemo_rl_chat": {"engine_args": {}} + } runtime.shutdown() diff --git a/tests/unit/models/generation/test_dynamo_render_pool.py b/tests/unit/models/generation/test_dynamo_render_pool.py index 640483eae67..47bd9d664aa 100644 --- a/tests/unit/models/generation/test_dynamo_render_pool.py +++ b/tests/unit/models/generation/test_dynamo_render_pool.py @@ -22,6 +22,8 @@ import json import os import signal +import time +from concurrent.futures import ThreadPoolExecutor from concurrent.futures.process import BrokenProcessPool import pytest @@ -33,6 +35,7 @@ from nemo_rl.models.generation.dynamo.vllm.render_pool import ( # noqa: E402 RenderPool, + _wait_for_settings, _Worker, ) from nemo_rl.models.generation.prefix_recovery.restore import ( # noqa: E402 @@ -110,6 +113,12 @@ def _template_ids(tokenizer, messages, *, add_generation_prompt: bool) -> list[i return tokenizer.encode(text, add_special_tokens=False) +def _pool(spec: dict, model_dir: str) -> RenderPool: + pool = RenderPool(workers=1) + pool.build(spec, model_dir) + return pool + + def test_the_pool_renders_and_answers_with_the_checkpoints_template_and_parsers( tiny_qwen3_model_path, ) -> None: @@ -121,7 +130,7 @@ def test_the_pool_renders_and_answers_with_the_checkpoints_template_and_parsers( reply_ids = tokenizer.encode(reply_text, add_special_tokens=False) async def serve(): - pool = RenderPool(SPEC, tiny_qwen3_model_path, workers=1) + pool = _pool(SPEC, tiny_qwen3_model_path) try: info = await pool.info() rendered = await pool.render( @@ -170,16 +179,38 @@ async def serve(): assert json.loads(tool_call["function"]["arguments"]) == {"city": "Lyon"} -def test_start_brings_up_every_worker_before_any_call(tiny_qwen3_model_path) -> None: +def test_workers_start_before_their_settings_and_build_when_they_arrive( + tiny_qwen3_model_path, +) -> None: async def serve(): - pool = RenderPool(SPEC, tiny_qwen3_model_path, workers=2) + pool = RenderPool(workers=2) try: - await pool.start() - return len(pool._executor._processes) + start = asyncio.ensure_future(pool.start()) + # Both come up before any settings exist, as at the frontend's launch. + async with asyncio.timeout(60): + while len(pool._executor._processes) < 2: + await asyncio.sleep(0.01) + assert not start.done() + pool.build(SPEC, tiny_qwen3_model_path) + await start + return (await pool.info())["max_model_len"] finally: pool.shutdown() - assert asyncio.run(serve()) == 2 + assert asyncio.run(serve()) == MAX_MODEL_LEN + + +def test_a_worker_waits_for_its_settings_until_its_pool_is_shut_down(tmp_path) -> None: + path = tmp_path / "settings.json" + with ThreadPoolExecutor(max_workers=1) as worker: + waiting = worker.submit(_wait_for_settings, str(path)) + time.sleep(0.3) + assert not waiting.done() + path.write_text(json.dumps({"spec": SPEC, "model_dir": "/snapshots/model"})) + assert waiting.result(timeout=10) == (SPEC, "/snapshots/model") + # Shutdown removes the pool's directory; a worker left waiting would leak. + with pytest.raises(SystemExit): + _wait_for_settings(str(tmp_path / "removed" / "settings.json")) def test_a_dead_worker_fails_only_its_calls(tiny_qwen3_model_path) -> None: @@ -192,7 +223,7 @@ def test_a_dead_worker_fails_only_its_calls(tiny_qwen3_model_path) -> None: } async def serve(): - pool = RenderPool(SPEC, tiny_qwen3_model_path, workers=1) + pool = _pool(SPEC, tiny_qwen3_model_path) try: await pool.info() (pid,) = pool._executor._processes @@ -215,7 +246,7 @@ def test_workers_that_cannot_start_fail_the_call_by_name(tiny_qwen3_model_path): spec = {**SPEC, "generation": {**SPEC["generation"], "vllm_cfg": vllm_cfg}} async def serve(): - pool = RenderPool(spec, tiny_qwen3_model_path, workers=1) + pool = _pool(spec, tiny_qwen3_model_path) try: await pool.info() finally: @@ -245,7 +276,7 @@ def test_workers_decode_videos_with_nemo_rls_loader(tiny_qwen3_model_path) -> No spec = {**SPEC, "generation": {**SPEC["generation"], "vllm_cfg": vllm_cfg}} async def serve(): - pool = RenderPool(spec, tiny_qwen3_model_path, workers=1) + pool = _pool(spec, tiny_qwen3_model_path) try: return await pool._submit(_nemotron_vl_loader) finally: diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index 9cc05442ebb..bdb3b0ff96c 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -418,6 +418,27 @@ async def call_utility_async(method): assert config.runtime_data == vllm_adapter.chat_runtime_data(host) +def test_an_engine_sends_its_chat_settings_before_it_builds_the_engine( + monkeypatch, +) -> None: + host = _host(_FakeLLM()) + sent: list[dict] = [] + host.announce_dynamo_chat_settings_to(SimpleNamespace(put=sent.append)) + sent_before_the_build: list[list[dict]] = [] + + def build(engine_args, stat_loggers): + sent_before_the_build.append(list(sent)) + return _FakeLLM() + + monkeypatch.setattr("vllm.v1.engine.async_llm.AsyncLLM.from_engine_args", build) + + host._create_engine({"model": "/models/m", "max_model_len": MAX_MODEL_LEN}) + + # What it publishes at registration, while the frontend can still build its + # render workers ahead of it. + assert sent_before_the_build == [[vllm_adapter.chat_runtime_data(host)]] + + def test_engines_publish_what_the_frontends_chat_engine_renders_with() -> None: host = _host(_FakeLLM()) host.llm_async_engine_args = vllm.AsyncEngineArgs( diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 992168e3302..9d38e49d537 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -653,6 +653,8 @@ def _build_kit(kit_spec: _KitSpec, path: str) -> _Kit: ) ) )[chat_engine.RUNTIME_DATA_KEY] + pool = RenderPool(workers=1) + pool.build(spec, path) return _Kit( path=path, generation=generation, @@ -662,7 +664,7 @@ def _build_kit(kit_spec: _KitSpec, path: str) -> _Kit: model_config=model_config, vllm_config=vllm_config, renderer=renderer_from_config(vllm_config), - pool=RenderPool(spec, path, workers=1), + pool=pool, ) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index d1486b9cf55..9fa9e2cba73 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -3183,6 +3183,69 @@ def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error VllmGeneration.validate_settings(master_config) +@pytest.mark.parametrize("announced", [True, False]) +def test_vllm_dynamo_chat_settings_reach_the_frontend_while_the_engines_load( + monkeypatch, announced: bool +): + from ray.util.queue import Empty + + from nemo_rl.models.generation.vllm import vllm_generation + + events: list = [] + settings = {"nemo_rl_chat": {"engine_args": {}}} + + class _Queue: + def __init__(self, actor_options): + self.shut_down = False + + def get(self, timeout): + if not announced: + raise Empty + return settings + + def shutdown(self): + self.shut_down = True + + monkeypatch.setattr(vllm_generation, "RayQueue", _Queue) + monkeypatch.setattr(vllm_generation.ray, "get", lambda refs: refs) + # A load that ends before any settings arrive: the driver stops waiting. + monkeypatch.setattr( + vllm_generation.ray, "wait", lambda refs, num_returns, timeout: (refs, []) + ) + for step in ( + "_post_init", + "_report_dp_openai_server_base_urls", + "_report_device_id", + "_register_dynamo_engines", + ): + monkeypatch.setattr(VllmGeneration, step, lambda self: None) + leader = types.SimpleNamespace( + announce_dynamo_chat_settings_to=types.SimpleNamespace( + remote=lambda queue: events.append(("announce to", queue)) + ) + ) + generation = object.__new__(VllmGeneration) + generation.worker_group = types.SimpleNamespace( + workers=[leader], + get_dp_leader_worker_idx=lambda shard: 0, + run_all_workers_single_data=lambda method, **kwargs: ( + events.append(method) or ["load"] + ), + ) + generation._dynamo_runtime = types.SimpleNamespace( + serves_chat=True, + publish_chat_settings=lambda published: events.append(("publish", published)), + ) + + generation.load_and_start() + + # The leader is asked before it loads, and the frontend gets what it sent. + (_, queue), load, *published = events + assert load == "load_model" + assert published == ([("publish", settings)] if announced else []) + assert queue.shut_down + + def test_vllm_dynamo_rollouts_go_through_the_frontend(monkeypatch): from nemo_rl.experience.failures import GenerationUnavailable from nemo_rl.models.generation.dynamo import rollout_client From 34a41d89994e822d239ae7563a6f9964d51a6c2c Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 13 Nov 2026 09:22:04 -0800 Subject: [PATCH 74/82] build(gym): pin the Dynamo dialect with its review fixes Moves the Gym submodule from 234bb55da to 6dc68d2fa, the head of NVIDIA-NeMo/Gym#4817. The four new Gym commits answer that PR's review: - reasoning_field "both" is rejected with dynamo_frontend only when the server replays reasoning; - a top-k capture call that also sets logprob_token_ids keeps its HTTP 400 under Dynamo; - forward_session_id_as_conversation_id is rejected with dynamo_frontend, and the output-budget refit no longer asks the frontend for a /tokenize count; - the vLLM guide documents the dialect, and its tests cover every rejected setting, the empty annotation and the Dynamo capture handler. Gym's dependencies are the same at both commits, so uv.lock is unchanged. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 234bb55da38..6dc68d2fa4a 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 234bb55da38f7069e3d1eefa6cf9b95401008573 +Subproject commit 6dc68d2fa4a264f578b8f98bc90546a9a5b94b52 From 0d95075213de0d0f989a68296fafe406246211d4 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 13 Nov 2026 09:56:08 -0800 Subject: [PATCH 75/82] fix(dynamo): reject request fields the frontend refuses or drops in sampling_overrides too The Dynamo startup check read only vllm_model.extra_body on the policy model. Gym sends sampling_overrides keys on every request as well, so the same fields there went unnoticed: bad_words made the frontend answer every call with HTTP 400, and stop_token_ids, allowed_token_ids, bad_words_token_ids, cache_salt, logprob_token_ids and detokenize were accepted by the frontend and dropped before the chat engine. Both blocks are now checked. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 6 +-- nemo_rl/environments/nemo_gym.py | 48 +++++++++---------- .../test_nemo_gym_token_capture.py | 6 +++ 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index e482a9de526..b232f8619fd 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -158,12 +158,12 @@ Startup rejects these with `dynamo: true`: and a restarted shard does not register with it yet. - `async_rl.generation_router`: the frontend already routes the fleet behind one URL. -- `extra_body.bad_words` on the policy model +- `bad_words` in `extra_body` or `sampling_overrides` on the policy model (`env.nemo_gym.policy_model.responses_api_models.vllm_model`): the frontend answers every request carrying it with HTTP 400. Startup checks only this overlay, not Gym's own YAMLs. -- `extra_body` `stop_token_ids`, `allowed_token_ids`, `bad_words_token_ids`, - `cache_salt`, `logprob_token_ids` or `detokenize` on the policy model: the +- `stop_token_ids`, `allowed_token_ids`, `bad_words_token_ids`, `cache_salt`, + `logprob_token_ids` or `detokenize` in either block on the policy model: the frontend accepts them but drops them before NeMo RL's chat engine sees them. - `vllm_cfg.skip_tokenizer_init: true` with NeMo-Gym: the engine detokenizes the replies Gym's calls get through the frontend. diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index e7e0d7d4b2d..def7823cab0 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -3046,32 +3046,32 @@ def _apply_dynamo_dialect(nemo_gym_dict: dict[str, Any]) -> None: policy_model = dict(nemo_gym_dict.get("policy_model") or {}) models = dict(policy_model.get("responses_api_models") or {}) vllm_model = dict(models.get("vllm_model") or {}) - extra_body = vllm_model.get("extra_body") or {} - if "bad_words" in extra_body: - raise ValueError( - "vllm_model.extra_body.bad_words is not supported with " - "policy.generation.dynamo=true: the Dynamo frontend answers every " - "request carrying it with HTTP 400." - ) # Dynamo accepts these vLLM fields (its PASSTHROUGH_EXTRA_FIELDS) but never # serializes them to a Python chat engine, so ours would never see them. - dropped = sorted( - { - "allowed_token_ids", - "bad_words_token_ids", - "cache_salt", - "detokenize", - "logprob_token_ids", - "stop_token_ids", - } - & extra_body.keys() - ) - if dropped: - raise ValueError( - f"vllm_model.extra_body {dropped} is not supported with " - "policy.generation.dynamo=true: the Dynamo frontend accepts these " - "fields but drops them before NeMo RL's chat engine sees them." - ) + dropped_by_frontend = { + "allowed_token_ids", + "bad_words_token_ids", + "cache_salt", + "detokenize", + "logprob_token_ids", + "stop_token_ids", + } + # Gym puts the keys of both blocks on every request as top-level fields. + for block in ("extra_body", "sampling_overrides"): + fields = vllm_model.get(block) or {} + if "bad_words" in fields: + raise ValueError( + f"vllm_model.{block}.bad_words is not supported with " + "policy.generation.dynamo=true: the Dynamo frontend answers every " + "request carrying it with HTTP 400." + ) + dropped = sorted(dropped_by_frontend & fields.keys()) + if dropped: + raise ValueError( + f"vllm_model.{block} {dropped} is not supported with " + "policy.generation.dynamo=true: the Dynamo frontend accepts these " + "fields but drops them before NeMo RL's chat engine sees them." + ) vllm_model["dynamo_frontend"] = True # The frontend reads `reasoning` as another name for `reasoning_content` and # refuses an echoed assistant turn that carries both, Gym's default. diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index 8dfcb415536..2fd0049a06a 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -932,6 +932,12 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> ({"extra_body": {"bad_words": ["x"]}}, "bad_words"), # The frontend accepts stop_token_ids but drops it before our chat engine. ({"extra_body": {"stop_token_ids": [7]}}, "stop_token_ids"), + # Gym sends sampling_overrides keys on every request too. + ({"sampling_overrides": {"bad_words": ["x"]}}, "sampling_overrides.bad_words"), + ( + {"sampling_overrides": {"logprob_token_ids": [7]}}, + "sampling_overrides.*logprob_token_ids", + ), ], ) def test_gym_actor_config_dynamo_turns_on_gyms_dynamo_dialect( From 76111dbe0233ea5059c4050b7ba5643b828da6de Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 13 Nov 2026 12:56:32 -0800 Subject: [PATCH 76/82] test(dynamo): run Gym prefix recovery behind Dynamo Add the prefix-recovery test to the Dynamo L1 list, in the full tier as in the native list. Phase one ends with a SIGKILL, which orphans the etcd, NATS and frontend processes NeMo RL started. stop_phase1 now stops the processes that carry phase one's DYN_NAMESPACE, as the turn-recovery test does, so phase two's overlay check finds no leftover service. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- tests/functional/L1_Functional_Tests_Dynamo.sh | 3 +++ .../grpo_async_gym_single_controller_prefix_recovery.sh | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index 49b43d2f6c2..0dc93ed50d0 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -46,6 +46,9 @@ run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_con run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true # Turn recovery: Gym checkpoint + TQ snapshot, kill, restore, re-sent calls. run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller_turn_recovery.sh ++policy.generation.dynamo=true +# Prefix recovery: a checkpoint cuts running calls; restore resumes them from the saved tokens. +# Full tier, as in the native list. +run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh b/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh index 474eae65b5f..4eb4baa0186 100755 --- a/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh +++ b/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh @@ -141,6 +141,15 @@ stop_phase1() { fi sleep 1 done + # Behind Dynamo, the kill orphans etcd, NATS and the frontend: they run in + # their own sessions. Each carries phase one's DYN_NAMESPACE in its environment. + local namespace pid + namespace=$(grep -aoE "'DYN_NAMESPACE': '[^']+'" "$BASE_RUN_LOG" | head -1 | cut -d"'" -f4 || true) + if [[ -n "$namespace" ]]; then + for pid in $(grep -lsxz "DYN_NAMESPACE=$namespace" /proc/[0-9]*/environ | cut -d/ -f3 || true); do + kill -KILL "$pid" 2>/dev/null || true + done + fi } cleanup() { From 766f81d004ced81f457adf4d05eee298754c4a23 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 13 Nov 2026 12:57:03 -0800 Subject: [PATCH 77/82] build(gym): pin Gym main, which has the Dynamo dialect NVIDIA-NeMo/Gym#4817 merged as 440104210. Move the pin from that PR's branch (6dc68d2fa) to Gym main at 9e2e5d0ad. Gym's dependencies are the same at both commits, so uv.lock does not change. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 6dc68d2fa4a..9e2e5d0ad0d 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 6dc68d2fa4a264f578b8f98bc90546a9a5b94b52 +Subproject commit 9e2e5d0ad0d1723acef1b63beb28292da16f7c5b From 011fe52100a51b4e5e61cf36dc7ea98c39943659 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 13 Nov 2026 15:57:53 -0800 Subject: [PATCH 78/82] test(nightly): raise the GPU-hour cap to 5318 for the swe1 Dynamo nightly The swe1 Dynamo nightly this branch adds costs 96 GPU-hours and filled the cap exactly. Main has since added an 8-hour nightly under the same cap (#4648), so the merged list totals 5318. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- tests/unit/test_recipes_and_test_suites.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_recipes_and_test_suites.py b/tests/unit/test_recipes_and_test_suites.py index 6a834c8a04d..36e07f5e61e 100644 --- a/tests/unit/test_recipes_and_test_suites.py +++ b/tests/unit/test_recipes_and_test_suites.py @@ -260,7 +260,7 @@ def test_all_recipe_yamls_accounted_for_in_test_suites( ) -def test_nightly_compute_stays_below_5310_hours(nightly_test_suite, tracker): +def test_nightly_compute_stays_below_5318_hours(nightly_test_suite, tracker): command = f"DRYRUN=1 HF_HOME=... HF_DATASETS_CACHE=... CONTAINER= ACCOUNT= PARTITION= ./tools/launch {' '.join(nightly_test_suite)}" print(f"Running command: {command}") @@ -292,8 +292,8 @@ def test_nightly_compute_stays_below_5310_hours(nightly_test_suite, tracker): f"Last line of output was not as expected: '{last_line}'" ) total_gpu_hours = float(last_line.split(":")[-1].strip()) - assert total_gpu_hours <= 5310, ( - f"Total GPU hours exceeded 5310: {last_line}. We should revisit the test suites to reduce the total GPU hours." + assert total_gpu_hours <= 5318, ( + f"Total GPU hours exceeded 5318: {last_line}. We should revisit the test suites to reduce the total GPU hours." ) tracker.track("total_nightly_gpu_hours", total_gpu_hours) From 4f97060315300e1e7faacac0e141131e54211359 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 13 Nov 2026 17:18:33 -0800 Subject: [PATCH 79/82] test(dynamo): give the render-pool test's parser the method vLLM 0.33 calls vLLM 0.33's reply builder hands the parser the prompt's token IDs (set_prompt_token_ids) before it parses the reply. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- tests/unit/models/generation/test_dynamo_render_pool.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/models/generation/test_dynamo_render_pool.py b/tests/unit/models/generation/test_dynamo_render_pool.py index 47bd9d664aa..5a2a4513232 100644 --- a/tests/unit/models/generation/test_dynamo_render_pool.py +++ b/tests/unit/models/generation/test_dynamo_render_pool.py @@ -300,6 +300,10 @@ def parse( def count_reasoning_tokens(self, token_ids) -> int: return 0 + def set_prompt_token_ids(self, prompt_token_ids) -> None: + # vLLM hands the parser the prompt before it parses the reply. + pass + @pytest.fixture(scope="module") def reply_worker(tiny_qwen3_model_path) -> _Worker: From e449a141417e070d1aa1c35281642bfcda9a1d3f Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 13 Nov 2026 17:18:33 -0800 Subject: [PATCH 80/82] fix(gym): keep the native chat endpoint serving what it serves on main This branch made both serving paths refuse three kinds of chat call that the native server on main serves. The native route is back to main's behaviour for all three: - logprobs without top_logprobs (omitted or null): served on both paths. vLLM 0.33 reads a null as 0, like an omitted field, so it no longer matters that Dynamo's frontend hands both over alike. - stream: true: the native route streams the reply again and fails a captured call's capture with streaming_response. - n other than 1: the native route serves it. The Dynamo chat engine still refuses stream: true and n other than 1, which it cannot serve; the guide lists that among the differences from the native endpoint. on_policy_sampling keeps only the rules main's route has, and test_vllm_http_logprobs_contract is main's again. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 3 +- .../generation/dynamo/vllm/chat_engine.py | 20 +++++++++---- nemo_rl/models/generation/vllm/chat_server.py | 27 +++++++---------- nemo_rl/models/generation/vllm/utils.py | 29 +++--------------- tests/unit/data/test_captured_media.py | 1 - tests/unit/environments/test_nemo_gym.py | 30 +++++++++---------- .../generation/test_dynamo_chat_engine.py | 1 + .../models/generation/test_gym_chat_path.py | 10 ++----- .../models/generation/test_vllm_generation.py | 27 +++++++++++++++-- 9 files changed, 71 insertions(+), 77 deletions(-) diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 6185ccf0386..8c631d1a3bd 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -64,12 +64,11 @@ The frontend serves chat with NeMo RL's chat engine instead of Dynamo's own rend `tests/unit/models/generation/test_gym_chat_path.py` sends the same calls down both paths and checks they run the same prompt and `SamplingParams`, stage the same record and return the same reply. -Both paths refuse, with HTTP 400 and before anything runs, a call that streams (`stream: true`), asks for more than one choice (`n`), or sets `logprobs` without `top_logprobs`. NeMo-Gym sends none of these. - Where the behavior differs from the native endpoint: - The frontend refuses more than 32 `stop` strings, and vLLM request fields its own request type lacks (for example `structured_outputs`, `return_token_ids`, `truncate_prompt_tokens`, `priority`, `chat_template`), with HTTP 400. - gpt-oss (Harmony) chat calls are refused with HTTP 400. +- A call that streams (`stream: true`) or asks for more than one choice (`n`) is refused with HTTP 400. The native endpoint serves both, and fails the capture of a streamed token-capture call. NeMo-Gym sends neither. - A token-capture call refused for its context length acknowledges the refusal at the end of the error message instead of in the native endpoint's `x-ng-capture-refused` header, which Dynamo's errors cannot carry. Gym's dialect moves it back to the header, so the harness sees the native error. - A call's processed media travels to the engine inside the request. diff --git a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py index 0737e4c2225..2c65bc9b05e 100644 --- a/nemo_rl/models/generation/dynamo/vllm/chat_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm/chat_engine.py @@ -325,13 +325,26 @@ async def _serve( ) -> dict[str, Any]: # Deferred: prompt_and_capture imports vLLM. from nemo_rl.models.generation.vllm.prompt_and_capture import prepare_prompt - from nemo_rl.models.generation.vllm.utils import on_policy_sampling + from nemo_rl.models.generation.vllm.utils import ( + OffPolicySamplingError, + on_policy_sampling, + ) body = _drop_none( {k: v for k, v in request.items() if k not in _DYNAMO_ONLY_FIELDS} ) if request.get("chat_template_args"): body["chat_template_kwargs"] = request["chat_template_args"] + if (body.get("n") or 1) != 1: + # Capture stages one choice per call, and the trainer replays one. + raise OffPolicySamplingError( + f"request n={body['n']} is not supported; NeMo-Gym calls take n=1" + ) + if body.get("stream"): + # vLLM's delta outputs would leave this path only the last piece. + raise OffPolicySamplingError( + "request stream=true is not supported; NeMo-Gym calls do not stream" + ) body.update( on_policy_sampling( temperature=body.get("temperature"), @@ -342,11 +355,6 @@ async def _serve( presence_penalty=body.get("presence_penalty"), frequency_penalty=body.get("frequency_penalty"), logit_bias=body.get("logit_bias"), - n=body.get("n"), - logprobs=body.get("logprobs"), - # From the raw request: _drop_none drops an explicit null. - top_logprobs=request.get("top_logprobs"), - stream=body.get("stream"), generation_config=self._generation, ) ) diff --git a/nemo_rl/models/generation/vllm/chat_server.py b/nemo_rl/models/generation/vllm/chat_server.py index d534e644e94..4a5fe0cc135 100644 --- a/nemo_rl/models/generation/vllm/chat_server.py +++ b/nemo_rl/models/generation/vllm/chat_server.py @@ -32,7 +32,7 @@ import torch from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, StreamingResponse from pydantic import PrivateAttr from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, @@ -913,15 +913,6 @@ async def create_chat_completion( presence_penalty=request.presence_penalty, frequency_penalty=request.frequency_penalty, logit_bias=request.logit_bias, - n=request.n, - logprobs=request.logprobs, - # vLLM defaults an unset top_logprobs to 0; Gym must set it. - top_logprobs=( - request.top_logprobs - if "top_logprobs" in request.model_fields_set - else None - ), - stream=request.stream, generation_config=generation_config, ) for field, value in sampling.items(): @@ -947,13 +938,15 @@ async def create_chat_completion( content=generator.model_dump(), status_code=generator.error.code ) - # on_policy_sampling refused stream: true, so the reply is whole. - assert isinstance(generator, ChatCompletionResponse) - content = model_dump_chat_response_with_dynamic_message_fields(generator) - # Token capture: stage the delta and ride the coords on the - # response; strips logprobs/ids (no-op when capture is off). - content = await save_capture(host, request, content) - return JSONResponse(content=content) + if isinstance(generator, ChatCompletionResponse): + content = model_dump_chat_response_with_dynamic_message_fields(generator) + # Token capture: stage the delta and ride the coords on the + # response; strips logprobs/ids (no-op when capture is off). + content = await save_capture(host, request, content) + return JSONResponse(content=content) + + await abort_request_capture(host, request, "streaming_response") + return StreamingResponse(content=generator, media_type="text/event-stream") @app.post("/tokenize") async def tokenize(request: NeMoRLTokenizeRequest, raw_request: Request): diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index 9e3b61328f3..724f0c3c80c 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -552,23 +552,18 @@ def on_policy_sampling( presence_penalty: Optional[float], frequency_penalty: Optional[float], logit_bias: Optional[Mapping[Any, float]], - n: Optional[int], - logprobs: Optional[bool], - top_logprobs: Optional[int], - stream: Optional[bool], generation_config: Any, ) -> dict[str, Any]: """The sampling a NeMo-Gym chat call is served with, on every serving path. The request must match the train or validation profile - (:func:`check_sampling_param`), set no sampling the trainer does not - replay, sample one choice, set ``top_logprobs`` when it asks for token - information, and not stream. Floats are compared at f32: Dynamo's frontend - carries them as f32, so 0.7 arrives as 0.69999998. + (:func:`check_sampling_param`) and set no sampling the trainer does not + replay. Floats are compared at f32: Dynamo's frontend carries them as f32, + so 0.7 arrives as 0.69999998. Args: temperature, top_p, top_k, min_p, repetition_penalty, presence_penalty, - frequency_penalty, logit_bias, n, logprobs, top_logprobs, stream: + frequency_penalty, logit_bias: The request's values; None (or missing) when it did not set them. generation_config: ``policy.generation``. @@ -609,22 +604,6 @@ def on_policy_sampling( ) if logit_bias: raise OffPolicySamplingError("request logit_bias is not supported") - if (n or 1) != 1: - # Capture stages one choice per call, and the trainer replays one. - raise OffPolicySamplingError( - f"request n={n} is not supported; NeMo-Gym calls take n=1" - ) - if logprobs and top_logprobs is None: - raise OffPolicySamplingError( - "`top_logprobs` must be set when requesting token information from " - "NeMo RL's chat endpoint" - ) - if stream: - # Capture refuses a streamed call, and vLLM's delta outputs would leave - # the Dynamo path only the last piece of an uncaptured one. - raise OffPolicySamplingError( - "request stream=true is not supported; NeMo-Gym calls do not stream" - ) is_train = (_f32(temperature), _f32(top_p)) == ( profile["temperature"], profile["top_p"], diff --git a/tests/unit/data/test_captured_media.py b/tests/unit/data/test_captured_media.py index 51a77aef518..ebe4f82adaf 100644 --- a/tests/unit/data/test_captured_media.py +++ b/tests/unit/data/test_captured_media.py @@ -1455,7 +1455,6 @@ async def create_chat_completion(request, raw_request): n=1, logprobs=False, top_logprobs=None, - model_fields_set=set(), stream=False, ng_capture=admission.model_dump(), required_prefix_token_ids=None, diff --git a/tests/unit/environments/test_nemo_gym.py b/tests/unit/environments/test_nemo_gym.py index e1277c671a1..eab38f35c6f 100644 --- a/tests/unit/environments/test_nemo_gym.py +++ b/tests/unit/environments/test_nemo_gym.py @@ -2409,15 +2409,14 @@ def _standardize(l: list[dict]): def test_vllm_http_logprobs_contract(nemo_gym_vllm_generation): """Pin the vLLM OpenAI HTTP logprobs contract that NeMo-Gym capture depends on. - NeMo-Gym's vllm_model sets logprobs=True, top_logprobs=0 and - return_tokens_as_token_ids=True to extract per-token ids and logprobs for training. - A reply without per-token logprobs silently empties the captured token ids. This - exercises the real HTTP path where that translation lives (the offline LLM API does - not), so a vLLM bump that changes the contract fails here instead of silently - freezing training. - - NeMo RL's server requires top_logprobs with logprobs: both serving paths refuse an - omitted field and an explicit null, which vLLM alone would serve like 0. + NeMo-Gym's vllm_model sets logprobs=True and return_tokens_as_token_ids=True to extract + per-token ids and logprobs for training (Gym omits top_logprobs on the capture path, so + vLLM applies its default; Gym PR #1612 additionally pins top_logprobs=0, which is + equivalent). Omitting top_logprobs (default 0), sending 0 and sending an explicit null + must all return per-token logprobs; a request that returns none silently empties the + captured token ids. This exercises the real HTTP path where that translation lives (the + offline LLM API does not), so a vLLM bump that changes the contract fails here instead + of silently freezing training. All three cases share the (expensive) vLLM fixture, so they run in a single test rather than as separate parametrized cases. @@ -2462,15 +2461,14 @@ def _assert_has_token_ids(resp, label): token_ids = [int(c["token"].removeprefix("token_id:")) for c in content] assert len(token_ids) == len(content) - # Sending 0 (what Gym sends) must yield per-token logprobs whose tokens decode to ints. + # Omitting top_logprobs (what Gym does on the capture path; vLLM default 0) and sending 0 + # (the equivalent explicit pin) must both yield per-token logprobs whose tokens decode to ints. + _assert_has_token_ids(_chat(_OMIT_TOP_LOGPROBS), "omitted top_logprobs") _assert_has_token_ids(_chat(0), "top_logprobs=0") - # An omitted field and an explicit null are refused before anything runs, so capture - # never gets a reply without token ids. - for field in (_OMIT_TOP_LOGPROBS, None): - refused = _chat(field) - assert refused.status_code == 400, refused.text - assert "top_logprobs" in refused.json()["error"]["message"] + # An explicit null used to return no logprobs, which is why Gym pins top_logprobs=0. + # vLLM 0.33 treats null like 0. + _assert_has_token_ids(_chat(None), "top_logprobs=null") @pytest.mark.nemo_gym diff --git a/tests/unit/models/generation/test_dynamo_chat_engine.py b/tests/unit/models/generation/test_dynamo_chat_engine.py index f591fc79fd7..20545d9263d 100644 --- a/tests/unit/models/generation/test_dynamo_chat_engine.py +++ b/tests/unit/models/generation/test_dynamo_chat_engine.py @@ -353,6 +353,7 @@ def test_a_measure_call_returns_the_prompt_length_without_generating() -> None: ({"temperature": 0.5}, {}, "matches neither"), ({"presence_penalty": 0.5}, {}, "presence_penalty=0.5"), ({"n": 2}, {}, "n=1"), + ({"stream": True}, {}, "stream=true"), # The echoed prefix, spliced in, leaves no room for output. ( {}, diff --git a/tests/unit/models/generation/test_gym_chat_path.py b/tests/unit/models/generation/test_gym_chat_path.py index 9d38e49d537..bd90d96d0b3 100644 --- a/tests/unit/models/generation/test_gym_chat_path.py +++ b/tests/unit/models/generation/test_gym_chat_path.py @@ -886,13 +886,9 @@ def _off_policy(name: str, value: Any) -> _Case: _off_policy("presence_penalty", 0.5), _off_policy("frequency_penalty", 0.5), _off_policy("logit_bias", {"10": 5.0}), - # Capture stages one choice, and the trainer replays one. - _Case("n_2", fields={"n": 2}, status=400), - # Token information without top_logprobs, unset or sent as null. - _Case("top_logprobs_unset", omit=("top_logprobs",), status=400), - _Case("top_logprobs_null", fields={"top_logprobs": None}, status=400), - # Neither path stages a streamed call, and Gym never sends one. - _Case("stream", fields={"stream": True}, status=400), + # Token information without top_logprobs, unset or sent as null, is served as 0. + _Case("top_logprobs_unset", omit=("top_logprobs",)), + _Case("top_logprobs_null", fields={"top_logprobs": None}), # A field vLLM refuses while the request is parsed, before any step runs. _Case( "field_error", diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 02d6dc410d1..669e6d55432 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -35,6 +35,7 @@ import requests import torch from fastapi import FastAPI, HTTPException, Request, Response +from fastapi.responses import StreamingResponse from pydantic import BaseModel, ConfigDict, ValidationError from nemo_rl.algorithms.grpo import refit_policy_generation @@ -1384,8 +1385,6 @@ def _fake_chat_request(): return_tokens_as_token_ids=None, logprobs=False, top_logprobs=None, - # The fields the client set, as pydantic records them. - model_fields_set=set(), # NeMoRLChatCompletionRequest's private defaults for a call that # restores no generation prefix. _restored_generation_token_ids=(), @@ -1463,6 +1462,29 @@ def _track_request_capture(worker, request): return worker.token_capture, call +def test_vllm_chat_endpoint_serves_a_streamed_call_and_fails_its_capture( + monkeypatch, +) -> None: + """vLLM streams the reply, several choices included; capture cannot stage it.""" + worker, serving_chat, chat_handler = _setup_fake_vllm_chat_handler(monkeypatch) + + async def chunks(): + yield "data: [DONE]\n\n" # pragma: no cover - never iterated + + stream = chunks() + serving_chat.create_chat_completion = AsyncMock(return_value=stream) + request = _fake_chat_request() + request.stream = True + request.n = 2 + token_capture, call = _track_request_capture(worker, request) + + response = asyncio.run(chat_handler(request, MagicMock())) + + assert isinstance(response, StreamingResponse) + assert response.body_iterator is stream + token_capture.fail_call.assert_called_once_with(call, reason="streaming_response") + + def _stalled_engine_generator(started: asyncio.Event, cancelled: asyncio.Event): """Engine result generator for a generation that never finishes. @@ -1938,7 +1960,6 @@ async def test_chat_completion_cancelled_on_client_disconnect(monkeypatch, logpr request.logprobs = logprobs request.return_tokens_as_token_ids = True request.top_logprobs = 0 - request.model_fields_set = {"top_logprobs"} token_capture, call = _track_request_capture(worker, request) task = asyncio.create_task(chat_handler(request, MagicMock())) try: From 8904a7e221ec03b26c238412b76e44fbcb0f42e1 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 14 Nov 2026 16:29:39 -0800 Subject: [PATCH 81/82] fix(dynamo): refuse several vLLM HTTP frontends behind Dynamo vllm_cfg.http_api_server_count (#5074) adds HTTP frontend processes to each engine's native server. With policy.generation.dynamo=true NeMo-Gym's calls go through the Dynamo frontend, so those processes would serve none of them. Startup now rejects the combination and points at dynamo_cfg.frontend_args.render_workers. The native path is unchanged. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 1 + nemo_rl/models/generation/__init__.py | 8 ++++++++ .../unit/models/generation/test_dynamo_config.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 8c631d1a3bd..2f454ec4a01 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -92,6 +92,7 @@ Startup rejects these with `dynamo: true`: - `vllm_cfg.skip_tokenizer_init: true` with NeMo-Gym: the engine detokenizes the replies Gym's calls get through the frontend. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data parallelism inside vLLM, which Dynamo does not serve yet. Above data-parallel rank 0, vLLM rejects each engine's `ipc://` KV-event endpoint. - `vllm_cfg.http_server_api_key_required`: the frontend serves the same engines on 0.0.0.0 and checks no bearer token. +- `vllm_cfg.http_api_server_count` above 1: NeMo-Gym's calls go through the Dynamo frontend, so the extra vLLM HTTP frontends would serve none of them. `frontend_args.render_workers` sets how many processes render chat calls behind Dynamo. - `frontend_args.router_mode: direct`: it needs a worker ID on every request, and NeMo RL's rollouts name none. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index b5fb07139f2..0d682cae973 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -223,6 +223,14 @@ def configure_generation_config( "frontend serves the same engines on 0.0.0.0 and checks no " "bearer token." ) + if config["vllm_cfg"].get("http_api_server_count", 1) > 1: + raise ValueError( + "policy.generation.vllm_cfg.http_api_server_count > 1 is not " + "supported with policy.generation.dynamo=true: NeMo-Gym's calls " + "go through the Dynamo frontend, so the extra vLLM HTTP " + "frontends would serve none of them. Size " + "dynamo_cfg.frontend_args.render_workers instead." + ) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 7ab5fba52a3..1ad2e98a007 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -97,6 +97,21 @@ def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: ), "checks no bearer token", ), + # Gym's calls go through the Dynamo frontend, not vLLM's HTTP frontends. + ( + _vllm_config( + dynamo=True, + vllm_cfg={ + "async_engine": True, + "expose_http_server": True, + "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 1, + "http_api_server_count": 2, + }, + ), + "would serve none of them", + ), # Direct routing needs a worker ID that rollouts never carry. ( _vllm_config( From 027b75a1f1c943719a921d7e17f06d67030dc571 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 14 Nov 2026 16:29:57 -0800 Subject: [PATCH 82/82] test(dynamo): run prefix recovery behind Dynamo in the fast set A run_test line without `fast` is skipped in every CI lane that reuses a pre-built image, so CI would not show whether this test passes. Co-Authored-By: Claude Fable 5.2 Signed-off-by: jthomson04 --- tests/functional/L1_Functional_Tests_Dynamo.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index 0dc93ed50d0..68e34b8986f 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -47,8 +47,7 @@ run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_con # Turn recovery: Gym checkpoint + TQ snapshot, kill, restore, re-sent calls. run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller_turn_recovery.sh ++policy.generation.dynamo=true # Prefix recovery: a checkpoint cuts running calls; restore resumes them from the saved tokens. -# Full tier, as in the native list. -run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh ++policy.generation.dynamo=true +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then