From 27791a4f435efb7c1beac71fe2f6c557fca03a47 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 24 Oct 2026 13:04:57 -0700 Subject: [PATCH 01/19] feat(dynamo)!: serve any vLLM generation config through Dynamo's backend SDK `policy.generation.dynamo: true` on a vLLM generation config is now the whole Dynamo switch. NeMo RL's own vLLM workers register their AsyncLLM with a managed dynamo.frontend through Dynamo's engine-agnostic backend SDK (dynamo.common.backend), from inside each worker's actor loop. No Dynamo-owned vLLM code runs, so refit, pause, quantization, speculative decoding and parallelism are the native backend's own, and a vLLM bump never waits on Dynamo. - One request pipeline (vllm/request_pipeline.py) serves native generate_async, the native OpenAI server and the Dynamo adapter: the sampling-profile check, clamps, R3, token fields, capture, prefix splice and media. - Rollouts go through the frontend's KV-aware router. The engine publishes KV events, and functional overlays fail if the router applies none. - NeMo-Gym calls go through the frontend in Gym's Dynamo dialect (NVIDIA-NeMo/Gym#4817), with engine-side token and media capture. - Startup rejects what Dynamo cannot serve: colocated generation, fleet health, the generation router, a bearer token, R3 with Gym but no capture, parser plugins and chat settings with no frontend counterpart. - Removed: backend: dynamo, DynamoGeneration, the dynamo.vllm launcher, its refit routes, token wrapper, metrics scraper and isolated venv. Old keys are rejected with the migration. - ai-dynamo and ai-dynamo-runtime 1.5.1 join the vllm extra, without the planner deps. BUILD_DYNAMO installs only etcd and nats-server. - CI: the Dynamo lane runs native SingleController tests as overlays: math, Gym token capture, reload refit, and R3 with deferred-route capture. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .github/workflows/cicd-main.yml | 4 +- .github/workflows/lockfile-check.yml | 5 - 3rdparty/Gym-workspace/Gym | 2 +- docker/Dockerfile | 11 +- docker/dynamo/install.sh | 51 +- docker/dynamo/pyproject.toml | 9 - docker/dynamo/uv.lock | 4572 ----------------- docs/design-docs/dynamo-integration.md | 169 +- docs/guides/async-grpo.md | 2 +- docs/guides/dynamo-generation.md | 389 +- docs/guides/single-controller.md | 4 +- docs/index.md | 4 +- docs/observability/configuration.md | 2 +- docs/observability/metrics.md | 2 +- examples/configs/grpo_math_1B.yaml | 2 +- examples/configs/grpo_math_1B_dynamo.yaml | 97 - ...grpo_math_1B_dynamo_single_controller.yaml | 68 +- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 21 +- .../async_utils/trajectory_collector.py | 23 +- nemo_rl/algorithms/distillation.py | 2 +- nemo_rl/algorithms/grpo.py | 80 +- .../single_controller_utils/config.py | 4 +- .../single_controller_utils/setup.py | 58 +- nemo_rl/distributed/actor_environments.py | 1 - nemo_rl/distributed/virtual_cluster.py | 4 +- nemo_rl/environments/nemo_gym.py | 40 +- nemo_rl/experience/rollouts.py | 3 - nemo_rl/models/generation/__init__.py | 66 +- nemo_rl/models/generation/constants.py | 1 - nemo_rl/models/generation/dynamo/__init__.py | 8 +- nemo_rl/models/generation/dynamo/arguments.py | 167 +- nemo_rl/models/generation/dynamo/config.py | 507 +- .../generation/dynamo/dynamo_generation.py | 767 --- .../models/generation/dynamo/dynamo_worker.py | 304 -- .../models/generation/dynamo/http_client.py | 41 +- .../generation/dynamo/managed_runtime.py | 259 +- nemo_rl/models/generation/dynamo/metrics.py | 238 - .../models/generation/dynamo/native_engine.py | 930 ++++ nemo_rl/models/generation/dynamo/refit.py | 320 -- .../generation/dynamo/rollout_client.py | 114 + .../models/generation/dynamo/token_wrapper.py | 556 -- .../dynamo/validate_dynamo_vllm_args.py | 86 - nemo_rl/models/generation/dynamo/venv.py | 62 +- .../models/generation/dynamo/worker_pool.py | 246 - nemo_rl/models/generation/interfaces.py | 3 - .../models/generation/openai_server_utils.py | 72 +- nemo_rl/models/generation/vllm/config.py | 8 + .../models/generation/vllm/metric_names.py | 2 +- .../generation/vllm/request_pipeline.py | 217 + nemo_rl/models/generation/vllm/utils.py | 243 +- .../models/generation/vllm/vllm_generation.py | 200 + .../generation/vllm/vllm_worker_async.py | 425 +- nemo_rl/utils/outdated_config_checks.py | 52 + nemo_rl/utils/routed_experts_codec.py | 8 + .../collective_weight_synchronizer.py | 10 +- nemo_rl/weight_sync/factory.py | 4 +- pyproject.toml | 11 +- pyrefly.toml | 10 +- .../functional/L1_Functional_Tests_Dynamo.sh | 9 +- tests/functional/check_dynamo_overlay.sh | 53 + .../grpo_async_gym_single_controller.sh | 8 + tests/functional/grpo_dp_single_controller.sh | 4 + tests/functional/grpo_dynamo.sh | 79 - .../grpo_dynamo_single_controller.sh | 86 - ...lm_token_capture_single_controller_1n2g.sh | 4 + ...nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh | 4 +- tests/unit/algorithms/test_async_utils.py | 28 - tests/unit/algorithms/test_distillation.py | 2 +- tests/unit/algorithms/test_grpo.py | 280 +- .../test_nemo_gym_token_capture.py | 79 +- .../unit/environments/test_nemo_gym_utils.py | 7 +- tests/unit/experience/test_rollouts.py | 29 - .../generation/test_dynamo_arguments.py | 424 -- .../models/generation/test_dynamo_config.py | 202 + .../generation/test_dynamo_generation.py | 667 --- .../generation/test_dynamo_http_client.py | 123 +- .../generation/test_dynamo_managed_runtime.py | 810 +-- .../generation/test_dynamo_native_engine.py | 746 +++ .../test_dynamo_native_engine_capture.py | 232 + .../generation/test_dynamo_rollout_client.py | 67 + .../generation/test_dynamo_token_wrapper.py | 646 --- .../generation/test_dynamo_vllm_decoupling.py | 91 + .../generation/test_openai_server_utils.py | 92 + .../generation/test_swe1_dynamo_config.py | 21 +- .../models/generation/test_vllm_generation.py | 61 +- .../generation/test_vllm_request_pipeline.py | 146 + .../test_vllm_token_capture_hosting.py | 1 + .../unit/models/generation/test_vllm_utils.py | 27 + .../policy/test_worker_refit_signatures.py | 1 - tests/unit/single_controller/test_setup.py | 194 +- tests/unit/test_dependency_pins.py | 18 + .../unit/utils/test_outdated_config_checks.py | 43 + .../test_reconcile_communicator.py | 7 +- .../weight_sync/test_weight_synchronizer.py | 12 - uv.lock | 185 +- 95 files changed, 5006 insertions(+), 12048 deletions(-) delete mode 100644 docker/dynamo/pyproject.toml delete mode 100644 docker/dynamo/uv.lock delete mode 100644 examples/configs/grpo_math_1B_dynamo.yaml delete mode 100644 nemo_rl/models/generation/dynamo/dynamo_generation.py delete mode 100644 nemo_rl/models/generation/dynamo/dynamo_worker.py delete mode 100644 nemo_rl/models/generation/dynamo/metrics.py create mode 100644 nemo_rl/models/generation/dynamo/native_engine.py delete mode 100644 nemo_rl/models/generation/dynamo/refit.py create mode 100644 nemo_rl/models/generation/dynamo/rollout_client.py delete mode 100644 nemo_rl/models/generation/dynamo/token_wrapper.py delete mode 100644 nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py delete mode 100644 nemo_rl/models/generation/dynamo/worker_pool.py create mode 100644 nemo_rl/models/generation/vllm/request_pipeline.py create mode 100755 tests/functional/check_dynamo_overlay.sh delete mode 100644 tests/functional/grpo_dynamo.sh delete mode 100644 tests/functional/grpo_dynamo_single_controller.sh delete mode 100644 tests/unit/models/generation/test_dynamo_arguments.py create mode 100644 tests/unit/models/generation/test_dynamo_config.py delete mode 100644 tests/unit/models/generation/test_dynamo_generation.py create mode 100644 tests/unit/models/generation/test_dynamo_native_engine.py create mode 100644 tests/unit/models/generation/test_dynamo_native_engine_capture.py create mode 100644 tests/unit/models/generation/test_dynamo_rollout_client.py delete mode 100644 tests/unit/models/generation/test_dynamo_token_wrapper.py create mode 100644 tests/unit/models/generation/test_dynamo_vllm_decoupling.py create mode 100644 tests/unit/models/generation/test_vllm_request_pipeline.py diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index c7bad64967a..ca7de9393e7 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -238,8 +238,8 @@ jobs: h100_functional_test_scripts=$(jq -c \ '[.functional[] | select((ascii_downcase | contains("gb200")) | not)]' \ <<< "$test_plan") - # The managed Dynamo runtime is currently amd64-only and is not - # installed in the arm64 GB200 image. + # The GB200 image is built without BUILD_DYNAMO, so it has no etcd or + # nats-server for the managed Dynamo runtime. gb200_functional_test_scripts=$(jq -c \ '[.functional[] | select(. != "L1_Functional_Tests_Dynamo")]' \ <<< "$test_plan") diff --git a/.github/workflows/lockfile-check.yml b/.github/workflows/lockfile-check.yml index 7ecaf0d2684..7d54bf94049 100644 --- a/.github/workflows/lockfile-check.yml +++ b/.github/workflows/lockfile-check.yml @@ -26,8 +26,6 @@ on: - "pyproject.toml" - "uv.lock" - "3rdparty/**" - - "docker/dynamo/pyproject.toml" - - "docker/dynamo/uv.lock" - ".github/workflows/lockfile-check.yml" jobs: @@ -50,6 +48,3 @@ jobs: # uv.lock is slow (~5 min); cached runs are fast. - name: Check lockfile is up to date run: uv lock --check - - - name: Check Dynamo lockfile is up to date - run: uv lock --check --directory docker/dynamo diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 3f6f20593a1..5e2fb03bbab 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 3f6f20593a126ca9484793769a51d186aa1486c3 +Subproject commit 5e2fb03bbabe89543291cf99c471e3d32effdd83 diff --git a/docker/Dockerfile b/docker/Dockerfile index 4e8393a3eb5..cfa1afa949b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,8 @@ # --build-arg SKIP_VLLM_BUILD=1 # Skip vLLM dependencies # --build-arg SKIP_SGLANG_BUILD=1 # Skip SGLang dependencies # --build-arg SKIP_TRTLLM_BUILD=1 # Skip TRT-LLM dependencies -# Optional isolated managed Dynamo runtime (Dynamo + vLLM only): +# Optional etcd and nats-server binaries for Dynamo serving +# (policy.generation.dynamo=true; ai-dynamo itself ships in the vLLM venvs): # --build-arg BUILD_DYNAMO=1 ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.05-cuda13.2-devel-ubuntu24.04 @@ -587,7 +588,6 @@ FROM hermetic AS release # Re-declare build args for this stage ARG BUILD_DYNAMO -ARG DYNAMO_PYTHON_VERSION=3.12.11 ARG ETCD_VERSION=v3.5.21 ARG NATS_VERSION=v2.11.6 # Space-separated config paths whose NeMo Gym venvs are prefetched into the image @@ -606,7 +606,7 @@ ENV NVIDIA_BUILD_REF=${NVIDIA_BUILD_REF:-} LABEL com.nvidia.build.id="${NVIDIA_BUILD_ID}" LABEL com.nvidia.build.ref="${NVIDIA_BUILD_REF}" -ENV NEMO_RL_DYNAMO_VENV_DIR=/opt/dynamo_venv +ENV NEMO_RL_DYNAMO_BIN_DIR=/opt/dynamo/bin # AWS EFA OFI plugin discovery (p4d / p5 / p5en multi-node NCCL via SRD). # The host's aws-efa-installer places libnccl-net-ofi.so under @@ -623,11 +623,10 @@ ENV LD_LIBRARY_PATH="/opt/amazon/ofi-nccl/lib:/opt/amazon/efa/lib:${LD_LIBRARY_P # Exclude pyproject.toml and uv.lock since those may be altered by build-custom-vllm.sh COPY --from=nemo-rl --exclude=pyproject.toml --exclude=uv.lock . /opt/nemo-rl -# Keep Dynamo's Python 3.12 dependency graph isolated from NeMo-RL's normal -# Ray/vLLM environments. Default image builds do not create this environment. +# The control-plane binaries Dynamo serving starts. Default image builds skip them. RUN <<"EOF" bash -exu -o pipefail if [[ "${BUILD_DYNAMO:-0}" != "1" ]]; then - echo "BUILD_DYNAMO is not 1; skipping the managed Dynamo runtime" + echo "BUILD_DYNAMO is not 1; skipping the Dynamo service binaries" exit 0 fi bash docker/dynamo/install.sh diff --git a/docker/dynamo/install.sh b/docker/dynamo/install.sh index 12c4ecf99e1..7797a308988 100644 --- a/docker/dynamo/install.sh +++ b/docker/dynamo/install.sh @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Install the etcd and nats-server binaries the managed Dynamo services run. +# ai-dynamo itself ships in NeMo RL's vLLM worker venvs (the vllm extra), so +# Dynamo serves the same vLLM release NeMo RL pins; nothing here touches Python. + set -euo pipefail -script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -project_dir=${NEMO_RL_DYNAMO_PROJECT_DIR:-${script_dir}} -repo_root=$(realpath "${script_dir}/../..") -dynamo_venv_dir=${NEMO_RL_DYNAMO_VENV_DIR:-${repo_root}/venvs/dynamo} -dynamo_python_version=${DYNAMO_PYTHON_VERSION:-3.12.11} +bin_dir=${NEMO_RL_DYNAMO_BIN_DIR:-/opt/dynamo/bin} etcd_version=${ETCD_VERSION:-v3.5.21} nats_version=${NATS_VERSION:-v2.11.6} @@ -42,36 +42,7 @@ case "${target_arch}" in ;; esac -uv python install "${dynamo_python_version}" -uv venv --python "${dynamo_python_version}" "${dynamo_venv_dir}" -UV_PROJECT_ENVIRONMENT="${dynamo_venv_dir}" uv sync \ - --directory "${project_dir}" \ - --locked \ - --no-dev \ - --no-install-project \ - --link-mode copy - -dynamo_python=${dynamo_venv_dir}/bin/python -vllm_version=$("${dynamo_python}" -c \ - 'from importlib.metadata import version; print(version("vllm"))') -if [[ "${vllm_version}" != "0.28.0" ]]; then - echo "Expected vllm==0.28.0 from ai-dynamo[vllm]==1.5.1; got ${vllm_version}" >&2 - exit 1 -fi - -# Layerwise weight reload must count only the elements each composed loader -# actually writes (vLLM PR #44814). Without it, trailing NemotronH/Mamba2 -# parameters such as mixer.D stay unloaded after a refit and corrupt logits. -# vLLM 0.28.0 ships the fix; fail the build if a future pin drops it. -"${dynamo_python}" - <<'PY' -import inspect - -import vllm.model_executor.model_loader.reload.meta as meta - -if "min(numel," not in inspect.getsource(meta): - raise SystemExit("vLLM is missing the PR #44814 layerwise-reload fix") -PY - +mkdir -p "${bin_dir}" download_dir=$(mktemp -d "${TMPDIR:-/tmp}/nemorl-dynamo-install.XXXXXX") trap 'rm -rf "${download_dir}"' EXIT @@ -81,7 +52,7 @@ curl --fail --location --retry 3 \ tar -xzf "${download_dir}/etcd.tgz" -C "${download_dir}" install -m 0755 \ "${download_dir}/etcd-${etcd_version}-linux-${target_arch}/etcd" \ - "${dynamo_venv_dir}/bin/etcd" + "${bin_dir}/etcd" curl --fail --location --retry 3 \ "https://github.com/nats-io/nats-server/releases/download/${nats_version}/nats-server-${nats_version}-linux-${target_arch}.tar.gz" \ @@ -89,9 +60,7 @@ curl --fail --location --retry 3 \ tar -xzf "${download_dir}/nats.tgz" -C "${download_dir}" install -m 0755 \ "${download_dir}/nats-server-${nats_version}-linux-${target_arch}/nats-server" \ - "${dynamo_venv_dir}/bin/nats-server" + "${bin_dir}/nats-server" -"${dynamo_python}" -c \ - 'import importlib.metadata as m; assert m.version("ai-dynamo") == "1.5.1"; assert m.version("vllm") == "0.28.0"; assert m.version("nvidia-nccl-cu13") == "2.30.7"' -"${dynamo_venv_dir}/bin/etcd" --version -"${dynamo_venv_dir}/bin/nats-server" --version +"${bin_dir}/etcd" --version +"${bin_dir}/nats-server" --version diff --git a/docker/dynamo/pyproject.toml b/docker/dynamo/pyproject.toml deleted file mode 100644 index f1eff7a2aaf..00000000000 --- a/docker/dynamo/pyproject.toml +++ /dev/null @@ -1,9 +0,0 @@ -[project] -name = "nemo-rl-dynamo-runtime" -version = "0.0.0" -requires-python = "==3.12.*" -dependencies = ["ai-dynamo[vllm]==1.5.1"] - -[tool.uv] -package = false -override-dependencies = ["nvidia-nccl-cu13==2.30.7; sys_platform == 'linux'"] diff --git a/docker/dynamo/uv.lock b/docker/dynamo/uv.lock deleted file mode 100644 index b00d829e528..00000000000 --- a/docker/dynamo/uv.lock +++ /dev/null @@ -1,4572 +0,0 @@ -version = 1 -revision = 3 -requires-python = "==3.12.*" -resolution-markers = [ - "sys_platform == 'darwin'", - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] - -[manifest] -overrides = [{ name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'", specifier = "==2.30.7" }] - -[[package]] -name = "absl-py" -version = "2.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/1d/58e2b5a6e4d703ccb2a029943d665974cb3d5a4fb2b3e3675dd03a9df10e/absl_py-2.5.1.tar.gz", hash = "sha256:286e71c82c1a38e75bbcf185f9b37d0305ad7786535107cb49bf4df9ff2e1f95", size = 118387, upload-time = "2026-10-09T08:38:58.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/7d/01e62f59e4166af1be6238d9f2b7f3453630c51b8044b412d05b5606532a/absl_py-2.5.1-py3-none-any.whl", hash = "sha256:721200f2f0e9960f2ca9dc3a2a706b201f5f75d812c158f059cbbe29eeafbdb8", size = 137673, upload-time = "2026-10-09T08:38:56.629Z" }, -] - -[[package]] -name = "ai-dynamo" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ai-dynamo-runtime" }, - { name = "aiohttp" }, - { name = "aisimulate" }, - { name = "kubernetes" }, - { name = "msgspec" }, - { name = "prometheus-client" }, - { name = "pyzmq" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "typing-extensions" }, - { name = "zstandard" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/19/bc00c468b1d8093abafa7f3ab88362d12cfe06f9be2e65baf0ef03d370d1/ai_dynamo-1.5.1-py3-none-any.whl", hash = "sha256:86d21cb7fbebbf1c3d0933303729b1be222ad0fb4a50b17a03316165d33377b1", size = 3512837, upload-time = "2026-10-07T05:08:09.942Z" }, -] - -[package.optional-dependencies] -vllm = [ - { name = "blake3" }, - { name = "librosa" }, - { name = "nixl", extra = ["cu13"] }, - { name = "ray" }, - { name = "soundfile" }, - { name = "uvloop" }, - { name = "vllm", extra = ["otel", "runai"] }, -] - -[[package]] -name = "ai-dynamo-runtime" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "uvloop" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/63/f1cde7bb2d74aebc954251fe40ee093298f0975d6f22c47f1f3c66b11949/ai_dynamo_runtime-1.5.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:88e5580daf32056dbe2da9d4189ddd2f4e4cc2472ddfebdd19412d223ac4a596", size = 65897955, upload-time = "2026-10-07T05:08:55.817Z" }, - { url = "https://files.pythonhosted.org/packages/df/05/777f6f79e8c9359d259cf500ec62fcaf6bd9ce148d3bf47910ec02d996e3/ai_dynamo_runtime-1.5.1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b5297625a4f407b3871e3039a53f49e09102d67039fc1339349802038b84ec3e", size = 67290732, upload-time = "2026-10-07T05:09:45.438Z" }, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, -] - -[[package]] -name = "aiohttp" -version = "3.14.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohappyeyeballs" }, - { name = "aiosignal" }, - { name = "attrs" }, - { name = "frozenlist" }, - { name = "multidict" }, - { name = "propcache" }, - { name = "typing-extensions" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, - { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, - { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, - { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, - { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, - { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, - { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, - { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, - { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, - { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, - { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, - { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, - { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, - { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, - { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, -] - -[[package]] -name = "aiosignal" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "frozenlist" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, -] - -[[package]] -name = "aisimulate" -version = "0.12.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "bokeh" }, - { name = "chex" }, - { name = "flax" }, - { name = "google-vizier", extra = ["jax"] }, - { name = "jax" }, - { name = "jaxlib" }, - { name = "jinja2" }, - { name = "matplotlib" }, - { name = "munch" }, - { name = "numpy" }, - { name = "nvidia-ml-py" }, - { name = "optax" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "plotext" }, - { name = "plotly" }, - { name = "prettytable" }, - { name = "pyarrow" }, - { name = "pydantic" }, - { name = "pyyaml" }, - { name = "scipy" }, - { name = "tfp-nightly", extra = ["jax"] }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/75/8031a245717bcd8eb742fb59405576c296a3c8f554bc5cea0b4a9e25992a/aisimulate-0.12.0.tar.gz", hash = "sha256:131665696adf122f94a7e8ba50dd853718c0009070f3f5057b7680d3d75ebf7b", size = 20458, upload-time = "2026-09-17T18:00:29.629Z" } - -[[package]] -name = "annotated-doc" -version = "0.0.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, -] - -[[package]] -name = "annotated-types" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, -] - -[[package]] -name = "anthropic" -version = "0.120.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "docstring-parser" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d7/10/4ca013cb166f226bd89e0aeb0fcaff94f45ddf716d4925ce89475d3c587b/anthropic-0.120.2.tar.gz", hash = "sha256:9722efc10c27a30a69f5338ddacdb35bc6a64297a4e4ba729bf83af873d5fb3a", size = 1008421, upload-time = "2026-07-28T17:38:26.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/af/0f5db57b9397a0f3b7fc204cbef143401a7cadaf982330f97f1ce3d39f34/anthropic-0.120.2-py3-none-any.whl", hash = "sha256:0f0bc2b381dc0eb41c8d886b815d79c2041cd2374f83aed36f574b6dc9c579c1", size = 1022851, upload-time = "2026-07-28T17:38:25.466Z" }, -] - -[[package]] -name = "anyio" -version = "4.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, -] - -[[package]] -name = "apache-tvm-ffi" -version = "0.1.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/3d/4b9226cd45aa800a6904603dda9b323d728f3c3869952a673f3483b78b19/apache_tvm_ffi-0.1.11.tar.gz", hash = "sha256:153cd2c5a9717804cb0bcd9b2709f22a1e5f80ed05b5a490faf5949b136eedba", size = 2798354, upload-time = "2026-05-04T17:48:43.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/9d/0f81ca556e5836b3ca64818cdae3f47dc7822bd35d22ddef7a54106d801d/apache_tvm_ffi-0.1.11-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:6ae51cc7df415b5f373a9df4baa1165a65608e519bea81e7dd23428f00eeb689", size = 2418793, upload-time = "2026-05-04T17:47:57.879Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a9/f48e5dd4ae1f6f0c5ffac259c0a9531b7d6a7c0a4c45bc2229d55de6adf8/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da2c8d07fdc737d1ba75f4de25c29f156905b9dc980f1da90c395b4db525f522", size = 2605176, upload-time = "2026-05-04T17:47:59.676Z" }, - { url = "https://files.pythonhosted.org/packages/36/99/2848df4e8ed5bf51df1d286d1718510584fa61e88adbc9c5b23d71b38f7c/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78aa1857b04a2ea718317041ab3f01288b3d496e6036eb1b99ebdc9da0fdaef5", size = 2725887, upload-time = "2026-05-04T17:48:01.381Z" }, - { url = "https://files.pythonhosted.org/packages/7d/80/963c991934a4eb0fa0c0178f51963333fe14a96b732009da642b6bf6b42e/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a8b845c8dff498fb981c1dda36c954549204191b485a385845e604966594d0b2", size = 2513121, upload-time = "2026-05-04T17:48:03.43Z" }, - { url = "https://files.pythonhosted.org/packages/4d/18/95569107ee83619d61a3bb0d28743a0599f85c5161981e3e098c82c2b185/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2843f084cdc94dedacd8b257a395a2b71b8a3dc7fc99711b148bf1d161983128", size = 2697683, upload-time = "2026-05-04T17:48:05.222Z" }, - { url = "https://files.pythonhosted.org/packages/dc/99/f352cf1cce8f6f05584c4adf11de9eca07e6d217229bad6af35fb372926c/apache_tvm_ffi-0.1.11-cp312-abi3-win_amd64.whl", hash = "sha256:bd67e03759d25ff59f4e0ed9c8630a16872afc9dd8792f46ac3c927554015e60", size = 2365545, upload-time = "2026-05-04T17:48:07.295Z" }, -] - -[[package]] -name = "astor" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, -] - -[[package]] -name = "attrs" -version = "26.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, -] - -[[package]] -name = "audioread" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/4a/874ecf9b472f998130c2b5e145dcdb9f6131e84786111489103b66772143/audioread-3.1.0.tar.gz", hash = "sha256:1c4ab2f2972764c896a8ac61ac53e261c8d29f0c6ccd652f84e18f08a4cab190", size = 20082, upload-time = "2025-10-26T19:44:13.484Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/16/fbe8e1e185a45042f7cd3a282def5bb8d95bb69ab9e9ef6a5368aa17e426/audioread-3.1.0-py3-none-any.whl", hash = "sha256:b30d1df6c5d3de5dcef0fb0e256f6ea17bdcf5f979408df0297d8a408e2971b4", size = 23143, upload-time = "2025-10-26T19:44:12.016Z" }, -] - -[[package]] -name = "azure-core" -version = "1.41.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, -] - -[[package]] -name = "azure-identity" -version = "1.25.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "msal" }, - { name = "msal-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/0e/3a63efb48aa4a5ae2cfca61ee152fbcb668092134d3eb8bfda472dd5c617/azure_identity-1.25.3.tar.gz", hash = "sha256:ab23c0d63015f50b630ef6c6cf395e7262f439ce06e5d07a64e874c724f8d9e6", size = 286304, upload-time = "2026-03-13T01:12:20.892Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/9a/417b3a533e01953a7c618884df2cb05a71e7b68bdbce4fbdb62349d2a2e8/azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c", size = 192138, upload-time = "2026-03-13T01:12:22.951Z" }, -] - -[[package]] -name = "azure-storage-blob" -version = "12.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "isodate" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/48/84a820d898267f662b5c06f7cd76fdb8a9e272b44aa9376cef3ec0f6a294/azure_storage_blob-12.30.0.tar.gz", hash = "sha256:2cd74d4d5731e5eb6b8d5c5056ee115a5e88f8fdf22517b739836fda685018be", size = 618229, upload-time = "2026-06-08T11:45:35.575Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/0b/e106f0fd7fa785867d9ffcc47dc9e6237c0e58f51058473b777487a98edc/azure_storage_blob-12.30.0-py3-none-any.whl", hash = "sha256:d415ac50b67a8da6b3ae7e9f1014b1b55cd7aafa0b8d4ca9b380568dc7360423", size = 435610, upload-time = "2026-06-08T11:45:37.213Z" }, -] - -[[package]] -name = "blake3" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/6a/4cc5a9dd40fd8a6d283fd3761e5f59c490109571ef8e3c73245417e5a305/blake3-1.0.9.tar.gz", hash = "sha256:5fa374fa5070ca084368776c19b420157eb0f2d3f091343d6bc59189929d62e2", size = 116872, upload-time = "2026-06-22T18:02:25.366Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/d2/9bdf8345c70993aaef635398f52edfb915d6e8ad2c000c801204e387c456/blake3-1.0.9-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a70c20542d5e7960983a0ff32999049a2b0e5ef1f22dbbbdfb51cf04828a4156", size = 344587, upload-time = "2026-06-22T18:00:34.244Z" }, - { url = "https://files.pythonhosted.org/packages/36/9d/be8b1f7f85b12bb45a0fade6ca7bdbf83a507d23d0b6141ba29fe69c8cea/blake3-1.0.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:72cdecf088a9d25e6ec79948a578995649b0dbee407e7a46c543a9ecc0f6f281", size = 328864, upload-time = "2026-06-22T18:00:35.59Z" }, - { url = "https://files.pythonhosted.org/packages/f2/78/66580635d744c826671fd219938caffb16281a26f62c4f856695d4233677/blake3-1.0.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42fa57bf462285ef16400601b0fd32214c248ba92505bbb94b1221ab9af5a092", size = 373795, upload-time = "2026-06-22T18:00:36.887Z" }, - { url = "https://files.pythonhosted.org/packages/b1/79/b5b17d3004bb81a5732c0b176c812703d200ed8c652b3b7713b9633bbe10/blake3-1.0.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b25ccde5a64be070f20e5c7a81da70292db40b164b6c77588cbd6230856badbb", size = 374183, upload-time = "2026-06-22T18:00:38.205Z" }, - { url = "https://files.pythonhosted.org/packages/3c/63/0d209c44b2041bbe130ced12a23c92dd995fbfe5bce7ee77fffea16f5cb0/blake3-1.0.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a800b87433955f37691b5f361ad29c7dd3ee089c9cd109adc5aea8e24bc4c1f", size = 446783, upload-time = "2026-06-22T18:00:39.493Z" }, - { url = "https://files.pythonhosted.org/packages/c5/51/efd1f9b8a9d3e9a0e235f3ced99a738529a1019fe78b3988e29d9c2fbba6/blake3-1.0.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6879739e7904b9c42afbedbcc2e8c36cebe140fb3fc3f5c492993579cf5cd516", size = 487369, upload-time = "2026-06-22T18:00:40.875Z" }, - { url = "https://files.pythonhosted.org/packages/8d/3f/a8dcaea9e0b26e419a540ca0cd6203c9fbb505e85b02b03c5a59bf9e6a45/blake3-1.0.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6edeb3d49a24c307995899b70dd47aa901d0e9ad51d2f8a79aba4f074f32d8c5", size = 383845, upload-time = "2026-06-22T18:00:42.251Z" }, - { url = "https://files.pythonhosted.org/packages/f6/10/e9907f5b86410d5071982aaf05d149ca4d4fd8acab7e77eebbc9a333c7b4/blake3-1.0.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd56a7a972c4185070f7042ccc20166927eec3c0f98b8405f375d007b604a0b", size = 383851, upload-time = "2026-06-22T18:00:43.715Z" }, - { url = "https://files.pythonhosted.org/packages/34/cf/c7863a185550706a9624f6aa7b6d46470aaed0bb46a827c5cda2a7d03151/blake3-1.0.9-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:a288664d08dee154cc496e06e62517fc9e655ecec12b0d7db538d244ac79edf1", size = 380067, upload-time = "2026-06-22T18:00:45.249Z" }, - { url = "https://files.pythonhosted.org/packages/54/0a/e7af679c719368b400c9ba9c3460072aac2ba077ddbd4bc806fef28cda03/blake3-1.0.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:91db52a809b68b5bebe7c413ddcd230e1f759398e7fa7a873104595a4fa648b6", size = 549471, upload-time = "2026-06-22T18:00:46.793Z" }, - { url = "https://files.pythonhosted.org/packages/2c/3c/37c1dd3539b7bd9b6d2eef019802aacdb4a3d48ab484b140603bbf9c5b5a/blake3-1.0.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfaa671b07eb73883162ca940442193868358b0b904cfa266e4b74131ce966da", size = 591396, upload-time = "2026-06-22T18:00:48.122Z" }, - { url = "https://files.pythonhosted.org/packages/ae/55/4f0a23b72795292e74084834130900ea778c0583004519c86698dfffe1a5/blake3-1.0.9-cp312-cp312-win32.whl", hash = "sha256:ae47c3d5729ff89baa6ddf6de47fcfcc915985d39eb1bfcd6db653331f3c6fcc", size = 229271, upload-time = "2026-06-22T18:00:49.377Z" }, - { url = "https://files.pythonhosted.org/packages/12/91/7db93e4689f0f145bcb954dc62936e5f5090548a9fa20c6bbebfaeaa648a/blake3-1.0.9-cp312-cp312-win_amd64.whl", hash = "sha256:15566065ff90ab3da46ec0be1417406f00507af902b6fb0fbc6563e77f02fc42", size = 218220, upload-time = "2026-06-22T18:00:50.659Z" }, -] - -[[package]] -name = "bokeh" -version = "3.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "narwhals" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyyaml" }, - { name = "tornado", marker = "sys_platform != 'emscripten'" }, - { name = "xyzservices" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/ab/19f33e7442fe2f09378bdafa95d088a1ed0dd078ea1f3d676c28d3cf2ed7/bokeh-3.10.1.tar.gz", hash = "sha256:e1dc21c4fa02f793b7c6acb2b439311924f46912b1d1ae1c7b058f8245b54fe8", size = 6053211, upload-time = "2026-10-13T16:53:00.445Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/9c/aaf2f11e31a75f0fffc79659ad7f0be01f39bb018bbc83c2a8a02657589e/bokeh-3.10.1-py3-none-any.whl", hash = "sha256:d73412b6317f70df0ffe6796956af609617523dfdfbc624d4f7f496cd09e8101", size = 6721259, upload-time = "2026-10-13T16:52:58.309Z" }, -] - -[[package]] -name = "boto3" -version = "1.43.58" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/95/bd6276870084c9c1a94e17d1dc73b2de275e59bd7a0ad8bfff0a1598cec4/boto3-1.43.58.tar.gz", hash = "sha256:12871fb50c383f1b9aa4ed6dd386ba689062baef730552e79d5a9cd782b53058", size = 112685, upload-time = "2026-07-28T19:35:09.336Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/85/b0709066efb4ce7b86aba4092c739ad0e458be9d62cf32550fc4c130c93f/boto3-1.43.58-py3-none-any.whl", hash = "sha256:ce1a20cbcfaa1d0b3c8f568e2b6c7fbd34b842ea00e729d49a4b4de522828db9", size = 140026, upload-time = "2026-07-28T19:35:06.993Z" }, -] - -[[package]] -name = "botocore" -version = "1.43.58" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jmespath" }, - { name = "python-dateutil" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/64/5cd46a7e72b0647e6d78fc8da016259ad66b9ae0818f4c5d629c75e7ca49/botocore-1.43.58.tar.gz", hash = "sha256:e110ca53f65c128fe98df4d6d36a459b1db17c6114671c8a18becf151bf20909", size = 15742412, upload-time = "2026-07-28T19:34:57.468Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/82/6f8fbbea47b773734ba0199643d2d851e5c2f75bc3699fe99db8af344d96/botocore-1.43.58-py3-none-any.whl", hash = "sha256:f516159f0732da8249206163ccea3bd1f82ad2a9d184fe6ed447e1abdba4330e", size = 15426503, upload-time = "2026-07-28T19:34:53.508Z" }, -] - -[[package]] -name = "cachetools" -version = "7.1.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/af/861ebc2e318a5c3300e3eb63bc4d30f3d70a46d13b360093728ac0705eed/cachetools-7.1.6.tar.gz", hash = "sha256:c7a79e7f30ba9943c1cefd08cc36f006aaae086e017af9166f1d59d6170c47e1", size = 40572, upload-time = "2026-07-23T22:47:53.737Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/f2/2086ba18a925a73586c4d4e61d25f4a6058e56fd00d77ce8f1d361ab4c9b/cachetools-7.1.6-py3-none-any.whl", hash = "sha256:2c12e255780330af28b91bb7fb96cce4c766f04e38396b9a24510190a5827096", size = 16954, upload-time = "2026-07-23T22:47:52.397Z" }, -] - -[[package]] -name = "cbor2" -version = "6.1.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3a/6f/07b4af8da8bd27f640362b1ac8271d80895407f2ede0c2bcc9433c06e1ca/cbor2-6.1.3.tar.gz", hash = "sha256:8d70680acb55c04ea5b5ad86da094f9612b53d5a8a65d0f5b3aafc3ce917ecbb", size = 89503, upload-time = "2026-07-04T10:36:48.793Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/16/cff14259c3d19a7f0ae88b6996fe4c85f6ff1764dad889ac8a39e843e39c/cbor2-6.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3d939f55097c21e032f5a2d67592fcc57298986281f219356e2f519e4466f4ea", size = 412779, upload-time = "2026-07-04T10:36:04.975Z" }, - { url = "https://files.pythonhosted.org/packages/50/6c/f3641d19b7b85a63cb2756c10164131489c2cb46b379ec51ae22283fefb9/cbor2-6.1.3-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b025009478d644dab407164fd60e3ef4381af284f5af6966df94c663756d949e", size = 457781, upload-time = "2026-07-04T10:36:06.349Z" }, - { url = "https://files.pythonhosted.org/packages/55/85/0c55a66f3037056bfb8e1c7184168085fdea67ae5830404498bcf466233b/cbor2-6.1.3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:2226d32e102e375737656ad5d141ad8c6ae3e705e04e263f24756f0eb379c6c1", size = 468373, upload-time = "2026-07-04T10:36:07.769Z" }, - { url = "https://files.pythonhosted.org/packages/46/74/40f7db3e0d880560193916a5c9b744fcf299558bed7113f77c28237c7c29/cbor2-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e61d465244d66ffed36492eef3b44d43795d76a2bba0663a2f15c186af7f7513", size = 523844, upload-time = "2026-07-04T10:36:09.404Z" }, - { url = "https://files.pythonhosted.org/packages/b5/a1/b5e07d6a08441c3a552fe2ae48ccb7e9dfc5065b9f6a3bae9879b4f0fbc0/cbor2-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87fe7be8fab6ec4796aa127c1a52e09e79dbafd2aa31caf809cf04b8080a5975", size = 536238, upload-time = "2026-07-04T10:36:10.914Z" }, - { url = "https://files.pythonhosted.org/packages/c9/99/e166be0fd74bf3a91f5a0d103e34883efbc438d970f72cc8200e274787e5/cbor2-6.1.3-cp312-cp312-win32.whl", hash = "sha256:da25d345f01e6a40b2e5c57ef96b4dcff7be69394fb62f0f70e07f437f2376a9", size = 279858, upload-time = "2026-07-04T10:36:12.247Z" }, - { url = "https://files.pythonhosted.org/packages/d2/1b/90b4a121e40aba189c55a5822dd3c698eaf487e1d4a780ab18c804a5ef1c/cbor2-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:d5514f693db6fa6f433b4096e9b604e6a7bf151c9ef1d2db86d0858e4c5e768f", size = 300929, upload-time = "2026-07-04T10:36:13.564Z" }, - { url = "https://files.pythonhosted.org/packages/19/db/52c58a8d33464927389dde8103997b3fa51b081ce29b347ac2cc4fd0dfbf/cbor2-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:3d43183d7beb3d3cd198d69b31bd2ee487ed704a1150c75cb0a66d6ad63d8c1a", size = 290908, upload-time = "2026-07-04T10:36:14.857Z" }, -] - -[[package]] -name = "certifi" -version = "2026.7.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, -] - -[[package]] -name = "cffi" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, - { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, - { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, - { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, - { url = "https://files.pythonhosted.org/packages/58/0c/f528df19cc94b675087324d4760d9e6d5bfae97d6217aa4fac43de4f5fcc/cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7", size = 208859, upload-time = "2026-07-06T21:32:52.512Z" }, - { url = "https://files.pythonhosted.org/packages/62/f2/c9522a81c32132799a1972c39f5c5f8b4c8b9f00488a23feaa6c06f07741/cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66", size = 221844, upload-time = "2026-07-06T21:32:53.704Z" }, - { url = "https://files.pythonhosted.org/packages/6e/28/bd53988b9833e8f8ad539d26f4c07a6b3f6bcb1e9e02e7ca038250b3428d/cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe", size = 225287, upload-time = "2026-07-06T21:32:54.907Z" }, - { url = "https://files.pythonhosted.org/packages/79/99/0d0fd37f055224085f42bbb2c022d002e17dde4a97972822327b07d84101/cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b", size = 223681, upload-time = "2026-07-06T21:32:56.329Z" }, - { url = "https://files.pythonhosted.org/packages/b0/80/c138990aa2a70b1a269f6e06348729836d733d6f970867943f61d367f8cc/cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a", size = 175269, upload-time = "2026-07-06T21:32:57.777Z" }, - { url = "https://files.pythonhosted.org/packages/a8/eb/f636456ff21a83fc13c032b58cc5dde061691546ac79efa284b2989b7982/cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384", size = 185881, upload-time = "2026-07-06T21:32:59.253Z" }, - { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, - { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, - { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, - { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, - { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, - { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, - { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, - { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, - { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, - { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, - { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, - { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, - { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, - { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, -] - -[[package]] -name = "chex" -version = "0.1.87" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "jax" }, - { name = "jaxlib" }, - { name = "numpy" }, - { name = "setuptools" }, - { name = "toolz" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ba/8a/857474810b64ab135a0c3e594b0453c7f39f140757c4cd26a32bccadcbc4/chex-0.1.87.tar.gz", hash = "sha256:0096d89cc8d898bb521ef4bfbf5c24549022b0e5b301f529ab57238896fe6c5d", size = 90063, upload-time = "2024-09-30T14:40:56.93Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/dd/c1ff2eb8fbf95a8ca804abb1cc3ce70b283ee7b4bc653c3abac245670400/chex-0.1.87-py3-none-any.whl", hash = "sha256:ce536475661fd96d21be0c1728ecdbedd03f8ff950c662dfc338c92ea782cb16", size = 99369, upload-time = "2024-09-30T14:40:54.862Z" }, -] - -[[package]] -name = "click" -version = "8.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, -] - -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "compressed-tensors" -version = "0.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "loguru" }, - { name = "pydantic" }, - { name = "torch" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2c/9e/d7f18bd9a0354088abc11a0c1f2c7698f7c49e5a709faedf6a46e388f693/compressed_tensors-0.17.0.tar.gz", hash = "sha256:15c20d06bdbcf35b51fc99fd125e7b9be1e1855567c33b7a46dfac26ad6fb126", size = 257091, upload-time = "2026-06-03T16:49:17.208Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/63/6edf0415b072fff0bf8b546074dea3f0f9b148e49b601ac98bdc60a76c68/compressed_tensors-0.17.0-py3-none-any.whl", hash = "sha256:4a1b89b508f7efb8ffb4eee8a6e69e0452d9b080cae130146025c64fbe9fa9aa", size = 211714, upload-time = "2026-06-03T16:49:15.672Z" }, -] - -[[package]] -name = "contourpy" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/5a/a55177dd22553a277388e8a1b3220e92de91bacb28356cdc73caa240121d/contourpy-1.4.0.tar.gz", hash = "sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1", size = 13323726, upload-time = "2026-09-11T19:05:05.808Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/ba/01bde9753bdbba04a6da9d2bff3881f021bc708f654655e95fae26d3b3a3/contourpy-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:186ba929df36d61b6127da2e89cd1357e4cb79aca647381ab1a6feb0b152877b", size = 297302, upload-time = "2026-09-11T19:02:45.103Z" }, - { url = "https://files.pythonhosted.org/packages/dd/f5/c5bf67522d49a2222f3154fe46879451d26f0e35ece41770758a64ea78cb/contourpy-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c76f3a5318164db7d9401132fc1b5364b784c613c93fa506e3b5e6d1bf353ec8", size = 284921, upload-time = "2026-09-11T19:02:48.049Z" }, - { url = "https://files.pythonhosted.org/packages/0f/cc/cb989599eec12fda312e127eb8e04a8b21a7a6c94bf7ff1bc68273334a42/contourpy-1.4.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:43c3ccbb32c6294b183dcc8e8c46dacf5ecef809497e3d48a5be298eef185ad0", size = 356332, upload-time = "2026-09-11T19:02:51.408Z" }, - { url = "https://files.pythonhosted.org/packages/47/fb/6f620d7602507817b0b4c21dd790ed68688f1f4ae9c29aacf21f08db5cf1/contourpy-1.4.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86d05cec773c9507a3950122e0e40ce77c23c75ceeb2fc189514e71893cbb34b", size = 405878, upload-time = "2026-09-11T19:02:54.093Z" }, - { url = "https://files.pythonhosted.org/packages/32/4e/693c6d6bdece679f0953775eef3a1b66d56be55478d936e7deb1f3004422/contourpy-1.4.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2deb580178ca19437a84bd77e4bc2cd91a8ccad212413a83c273900868b5978c", size = 408296, upload-time = "2026-09-11T19:02:56.528Z" }, - { url = "https://files.pythonhosted.org/packages/78/f9/b6831508960d559581c448532ba4df312217072975486dbf2b24fcc5b76f/contourpy-1.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:875f42444c9cf48d56f724f2637e60d0f73b3b12c9041e1484580a233edf9591", size = 383243, upload-time = "2026-09-11T19:02:58.638Z" }, - { url = "https://files.pythonhosted.org/packages/f2/21/52903825a0ae7bb625e8bd30a09816d4c3c5d6174a469c10a616166cf780/contourpy-1.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f863c6100bf926cf47d13f3cd75f9bb8ebd98aaab230eeb4468b91f98f39a6b3", size = 1357306, upload-time = "2026-09-11T19:03:01.036Z" }, - { url = "https://files.pythonhosted.org/packages/d8/79/d68b1ce8539e4071518fed9523f558398f34dcd078b8927b109c72dad2ef/contourpy-1.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3863ef2e2b13fe93f8c0ebb08ee400cb07153b8b0e91c5acb26e4537f283634f", size = 1427243, upload-time = "2026-09-11T19:03:03.866Z" }, - { url = "https://files.pythonhosted.org/packages/ac/f0/b75a10e9d0616b97a30de277b6dfe83f1879880854c7330337309530eb48/contourpy-1.4.0-cp312-cp312-win32.whl", hash = "sha256:5450f091ac1be0be3ad3a2a3b3f23b5e443e78c670ced4fd347d626f92a28fd2", size = 347345, upload-time = "2026-09-11T19:03:06.192Z" }, - { url = "https://files.pythonhosted.org/packages/50/a9/dab08786bb4d77ef9a046b3c1be923be4e73c5d07d91f5a54e8ea9b09e41/contourpy-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e697d94e69f499ff6bebb899cae97a58d5d14f0e1fe9568b43a0248d2f9af8c", size = 234083, upload-time = "2026-09-11T19:03:08.122Z" }, - { url = "https://files.pythonhosted.org/packages/47/b9/3ba509755a970dbde5948e7142ac21f266be72f38017cc4087a05fdceee1/contourpy-1.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:0c7a4c2716a4e98342221954416a836cca77c996c14ddbf22b5f01d5d93ca09c", size = 559521, upload-time = "2026-09-11T19:03:10.047Z" }, -] - -[[package]] -name = "cryptography" -version = "49.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" }, - { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" }, - { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" }, - { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" }, - { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" }, - { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" }, - { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" }, - { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" }, - { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" }, - { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" }, - { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" }, - { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" }, - { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" }, - { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" }, - { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" }, - { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" }, - { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" }, - { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" }, - { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" }, - { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" }, - { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" }, - { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, - { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, -] - -[[package]] -name = "cuda-bindings" -version = "13.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-pathfinder" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, - { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" }, - { url = "https://files.pythonhosted.org/packages/7c/95/872a0392122f1fb43fcb06869790ef3171f37beee9f7db8f441739113570/cuda_bindings-13.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:b134dd8c5c66ae4c4ad814f7aee88fd215353c077010cbc47e3b55ed35ec9eff", size = 5875099, upload-time = "2026-05-29T23:11:54.635Z" }, -] - -[[package]] -name = "cuda-core" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-pathfinder" }, - { name = "numpy" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/a0/1daeae599cadd612689dbbf70d7da1c01883964fc2fbc7386f3c630a68cf/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6816dc020aee6103d8071bc02d8e4e1d91f2b49596f666896d608d92224d79d1", size = 4789856, upload-time = "2026-05-12T20:11:30.862Z" }, - { url = "https://files.pythonhosted.org/packages/a1/4d/603557ab3cb171cc2a61d3678a39cb4dae3fd21275078bfbd1c0b0b5230b/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be7b65311bf78964b7905adbf3c0f8f717d432f2854dc45169277729bf60f1e2", size = 5106023, upload-time = "2026-05-12T20:11:33.509Z" }, - { url = "https://files.pythonhosted.org/packages/c2/1a/ae079963c9df7f4274227eb63cf8f6083a532a6443adb340d951fd21c626/cuda_core-1.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:1a5c1aa3b738a7599ea289498d038fe625d259fd7ab795394541eee58a8e29bc", size = 4663076, upload-time = "2026-05-12T20:11:35.784Z" }, -] - -[[package]] -name = "cuda-pathfinder" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/b4/d088047afe39827556df21118cac9ffd20cc3f968c99a7681494d1eb333c/cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51", size = 54591, upload-time = "2026-07-21T15:03:56.224Z" }, -] - -[[package]] -name = "cuda-python" -version = "13.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings" }, - { name = "cuda-core" }, - { name = "cuda-pathfinder" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/31/7ff3f7768eded7535c621abc2fecb9d181a34ea4cae3afe682feb796f242/cuda_python-13.3.1-py3-none-any.whl", hash = "sha256:280b014139ab447b6dd70a377db1596f310d6e887d9d342e6651b919ec145fb3", size = 8295, upload-time = "2026-05-29T23:28:47.012Z" }, -] - -[[package]] -name = "cuda-tile" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/6d/cc2fb5a25689a501564a2eced4acf654f307e801a2c1506be97c0d100491/cuda_tile-1.5.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:87652483baa9c81a9a24e4450f016e4ee78fd205d8422dad8996571bd1f2622e", size = 322641, upload-time = "2026-07-08T01:49:23.388Z" }, - { url = "https://files.pythonhosted.org/packages/1c/f5/b4ba9d0fc71198d939ebf9a090228179995d8411ee9def8f638a0e3ccdc5/cuda_tile-1.5.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:cef6d30acc37557643ece0de3770fc4c33497c4af40209e424f72fbfcbe6ea5a", size = 324990, upload-time = "2026-07-08T01:49:17.739Z" }, - { url = "https://files.pythonhosted.org/packages/a5/6e/7a60f317c503580ab7946dbb7fd080438fe953d0ddfdc81904beb9a1fab7/cuda_tile-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:16d97a60ed1d33388abbca85ea08cdae6325cc700476b3135f190d0fb50329f4", size = 304817, upload-time = "2026-07-08T01:49:38.853Z" }, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.3.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, -] - -[package.optional-dependencies] -cublas = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cudart = [ - { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -curand = [ - { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cusolver = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -nvrtc = [ - { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] -nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, -] - -[[package]] -name = "cycler" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, -] - -[[package]] -name = "decorator" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, -] - -[[package]] -name = "depyf" -version = "0.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "astor" }, - { name = "dill" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/88/35/83fb0178212279aa0af031031905804c6de5618435d229f41ed21bb9ad2c/depyf-0.20.0.tar.gz", hash = "sha256:fb7683bd72c44f67b56029df2c47721e9a02ffa4d7b19095f1c54c4ebf797a98", size = 6168761, upload-time = "2025-10-13T12:33:38.589Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/65/4df6936130b56e1429114e663e7c1576cf845f3aef1b2dd200c0a5d19dba/depyf-0.20.0-py3-none-any.whl", hash = "sha256:d31effad4261cebecb58955d832e448ace88f432328f95f82fd99c30fd9308d4", size = 39381, upload-time = "2025-10-13T12:33:33.647Z" }, -] - -[[package]] -name = "detect-installer" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5f/ce/6897d812825e9d4c53e3c7112726e800cc5231b013b2223bf64f653ff362/detect_installer-0.1.0.tar.gz", hash = "sha256:00ad7ba0a36e3cf7d08a40d3643011746dbc112597c7d475cc91c416710ca4e7", size = 3049, upload-time = "2026-02-23T10:40:22.567Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/34/8cc73273414405086c58852916e4031812a6a30fe04c057e37ad99397b7f/detect_installer-0.1.0-py3-none-any.whl", hash = "sha256:034fb20fd665c36e6ba52b8821525ea07fb4f7f938cac459df889fb33801528a", size = 4539, upload-time = "2026-02-23T10:40:23.807Z" }, -] - -[[package]] -name = "dill" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, -] - -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - -[[package]] -name = "dm-tree" -version = "0.1.10" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "attrs" }, - { name = "numpy" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/66/a3ec619d22b6baffa5ab853e8dc6ec9d0c837127948af59bb15b988d7312/dm_tree-0.1.10.tar.gz", hash = "sha256:22f37b599e01cc3402a17f79c257a802aebd8d326de05b54657650845956208a", size = 35748, upload-time = "2026-03-31T17:35:39.03Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/a1/17e0d68eec978c483db4712b14d083ee01484381b29ea85edb2b20210bd0/dm_tree-0.1.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94af18e4fd22ce69eccae89eeed8ed498b6b4cc4957f4ed10b4160e59f620e1d", size = 315976, upload-time = "2026-03-31T17:35:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/cc/6f/ed603715fbc29c887a8985252e2cfe0d449497aea96bac51010159771617/dm_tree-0.1.10-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b442a0c1e9d0960e0314a2e4af81fd328a87921b6d6db6dc41bfa420536884d6", size = 184053, upload-time = "2026-03-31T17:35:16.512Z" }, - { url = "https://files.pythonhosted.org/packages/83/eb/1d55c679cee9a54e552480d308535753c72e2250cf720d7aa777bff2a4fe/dm_tree-0.1.10-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:012c2b376e88d3685c73a4b5c23be41fe933e14e380dcd90172971690b0e02d2", size = 186506, upload-time = "2026-03-31T17:35:17.593Z" }, - { url = "https://files.pythonhosted.org/packages/89/2d/adef6924f8dc7f1665eea4ce066387820c14a629d0e1005568892d56ea6a/dm_tree-0.1.10-cp312-cp312-win_amd64.whl", hash = "sha256:da8d5b8995bea1b6bb93f457e0dad5d16e6e2344a6488ced55320e7f3fd50f56", size = 112708, upload-time = "2026-03-31T17:35:18.699Z" }, -] - -[[package]] -name = "dnspython" -version = "2.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, -] - -[[package]] -name = "docstring-parser" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, -] - -[[package]] -name = "durationpy" -version = "0.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/a4/e44218c2b394e31a6dd0d6b095c4e1f32d0be54c2a4b250032d717647bab/durationpy-0.10.tar.gz", hash = "sha256:1fa6893409a6e739c9c72334fc65cca1f355dbdd93405d30f726deb5bde42fba", size = 3335, upload-time = "2025-05-17T13:52:37.26Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/0d/9feae160378a3553fa9a339b0e9c1a048e147a4127210e286ef18b730f03/durationpy-0.10-py3-none-any.whl", hash = "sha256:3b41e1b601234296b4fb368338fdcd3e13e0b4fb5b67345948f4f2bf9868b286", size = 3922, upload-time = "2025-05-17T13:52:36.463Z" }, -] - -[[package]] -name = "einops" -version = "0.8.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/77/850bef8d72ffb9219f0b1aac23fbc1bf7d038ee6ea666f331fa273031aa2/einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827", size = 56261, upload-time = "2026-01-26T04:13:17.638Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193", size = 65638, upload-time = "2026-01-26T04:13:18.546Z" }, -] - -[[package]] -name = "email-validator" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dnspython" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, -] - -[[package]] -name = "equinox" -version = "0.11.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jax" }, - { name = "jaxtyping" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/48/ed7ba0849bea18b8be49f471b8e597ca6e5975f75ed05c3be84c1e0ef7c4/equinox-0.11.7.tar.gz", hash = "sha256:96e0216a9d822ec4b1465b0cbfbab14a36fb7e7d62c55f521287db3aaaa251be", size = 139909, upload-time = "2024-09-18T17:10:15.415Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/3f/2fa4627a902a38cd836aa8f7e5a0753114bb23d6b4d2a73784dd9f5ff2e6/equinox-0.11.7-py3-none-any.whl", hash = "sha256:1177354e1795061fbad71535fe54096d8887dc059b4ab7d400fea2d47dfaa283", size = 178416, upload-time = "2024-09-18T17:10:13.699Z" }, -] - -[[package]] -name = "etils" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/ce/6e067242fde898841922ac6fc82b0bb2fe35c38e995880bdffdfbe30182a/etils-1.14.0.tar.gz", hash = "sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4", size = 108127, upload-time = "2026-03-04T17:41:36.291Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/3d/589663aeeacd59bb2f3e8596bfd3e81cf0fb18d70bb433199041f469771b/etils-1.14.0-py3-none-any.whl", hash = "sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4", size = 172934, upload-time = "2026-03-04T17:41:35.01Z" }, -] - -[package.optional-dependencies] -epath = [ - { name = "fsspec" }, - { name = "typing-extensions" }, - { name = "zipp" }, -] -epy = [ - { name = "typing-extensions" }, -] - -[[package]] -name = "fastapi" -version = "0.136.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "pydantic" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/2d/ff8d91d7b564d464629a0fd50a4489c97fcb836ac230bf3a7269232a9b1f/fastapi-0.136.3.tar.gz", hash = "sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab", size = 396410, upload-time = "2026-05-23T18:53:15.192Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl", hash = "sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620", size = 117481, upload-time = "2026-05-23T18:53:16.924Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "email-validator" }, - { name = "fastapi-cli", extra = ["standard"] }, - { name = "fastar" }, - { name = "httpx" }, - { name = "jinja2" }, - { name = "pydantic-extra-types" }, - { name = "pydantic-settings" }, - { name = "python-multipart" }, - { name = "uvicorn", extra = ["standard"] }, -] - -[[package]] -name = "fastapi-cli" -version = "0.0.32" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "rich-toolkit" }, - { name = "typer" }, - { name = "uvicorn", extra = ["standard"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/33/eb/3b534c6f8e157f9ddbf2a153512307c886cad0b258739c200dd8ff8c4452/fastapi_cli-0.0.32.tar.gz", hash = "sha256:38024d2345275e1b37ce8848727a580d84901b570e96b3256d9d36a9a5039424", size = 26636, upload-time = "2026-07-16T12:16:58.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/53/56ae5ae17bb0a5d89d1d31e5320eb1865553ebbfbde91cdc4c221245f2a8/fastapi_cli-0.0.32-py3-none-any.whl", hash = "sha256:8dcc286fa32f01bbd3f65dd09cfd5a2540ed5f2230b77db7fd30978d6165f3c4", size = 14670, upload-time = "2026-07-16T12:16:57.297Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "fastapi-cloud-cli" }, - { name = "uvicorn", extra = ["standard"] }, -] - -[[package]] -name = "fastapi-cloud-cli" -version = "0.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "detect-installer" }, - { name = "fastar" }, - { name = "httpx" }, - { name = "pydantic", extra = ["email"] }, - { name = "rich-toolkit" }, - { name = "rignore" }, - { name = "sentry-sdk" }, - { name = "typer" }, - { name = "uvicorn", extra = ["standard"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/00/dc/63aaf9913f455e39a7027c27140edd887a87d47d65ac43532d77a51718e5/fastapi_cloud_cli-0.23.0.tar.gz", hash = "sha256:840895bb8d14309aeffc905e0dcd1334d18c6f5da54b735413a8f1cb385e581e", size = 95295, upload-time = "2026-07-28T14:03:33.463Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/96/7e9aba6fabce3cb05f320abeef5b81efd5134823ae85d1a517872cb83cbc/fastapi_cloud_cli-0.23.0-py3-none-any.whl", hash = "sha256:1cd2ffa56e92e92c1fc63acc426c214dd928cbeed2a4c7c6a9a5fc85ea73de16", size = 78058, upload-time = "2026-07-28T14:03:34.386Z" }, -] - -[[package]] -name = "fastar" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/0f/0aeb3fc50046617702acc0078b277b58367fd62eb727b9ec733ae0e8bbcc/fastar-0.11.0.tar.gz", hash = "sha256:aa7f100f7313c03fdb20f1385927ba95671071ba308ad0c1763fef295e1895ce", size = 70238, upload-time = "2026-04-13T17:11:17.143Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/06/a5773706afc8bd496769786590bbc56d2d0ee419a299cc12ea3f5717fcf3/fastar-0.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3c51f1c2cdddbd1420d2897ace7738e36c65e17f6ae84e0bfe763f8d1068bb97", size = 708394, upload-time = "2026-04-13T17:09:57.269Z" }, - { url = "https://files.pythonhosted.org/packages/cc/a6/d5e2a4e48495616440a21eed07558219ca90243ad00b0502586f95bd4833/fastar-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0d9d6b052baf5380baea866675dab6ccd04ec2460d12b1c46f10ce3f4ee6a820", size = 628417, upload-time = "2026-04-13T17:09:42.145Z" }, - { url = "https://files.pythonhosted.org/packages/ab/69/9816d69ac8265c9e50456637a487ccfb7a9c566efd9dbcd673df9c2558c2/fastar-0.11.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bd2f05666d4df7e14885b5c38fefd92a785917387513d33d837ff42ec143a22f", size = 863950, upload-time = "2026-04-13T17:09:11.506Z" }, - { url = "https://files.pythonhosted.org/packages/5b/0d/f88daad53aff2e754b6b5ff2a7113f72447a34f6ef17cc23ca99988117b7/fastar-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e6e74aba1ae77ca4aedcaf1697cd413319f4c88a5ccbe5b42c709517c5097e", size = 760737, upload-time = "2026-04-13T17:07:55.958Z" }, - { url = "https://files.pythonhosted.org/packages/2f/a6/82ef4ecd969d50d92ed3ed9dbd8fe77faa24be5e5736f716edc9f4ce8d62/fastar-0.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38ef77fe940bbc9b37a98bd838727f844b11731cd39358a2640ff864fb385086", size = 757603, upload-time = "2026-04-13T17:08:10.623Z" }, - { url = "https://files.pythonhosted.org/packages/03/35/50249f0d827251f8ac511495e2eacccebda80a00a0ad73e9615b8113b84f/fastar-0.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8955e61b32d6aff82c983217abf80933fd823b0e727586fc72f08043d996fd59", size = 923952, upload-time = "2026-04-13T17:08:25.526Z" }, - { url = "https://files.pythonhosted.org/packages/7b/d8/faee41659e9c379d906d24eaee6d6833ac8cfef0a5df480e5c2a8d3efb33/fastar-0.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:483532442cdb08fbff0169510224eae0836f2f672cea6aacb52847d90fefdc46", size = 816574, upload-time = "2026-04-13T17:08:56.076Z" }, - { url = "https://files.pythonhosted.org/packages/22/47/0448ea7992b997dad2bf004bfd98eca74b5858630eae080b50c7b17d9ddc/fastar-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef5a6071121e05d8287fc75bccb054bcbac8bb0501200a0c0a8feeace5303ea4", size = 819382, upload-time = "2026-04-13T17:09:26.66Z" }, - { url = "https://files.pythonhosted.org/packages/33/ef/0d63eb43586831b7a6f8b22c4d77125a7c594423af1f4f090fa9541b9b40/fastar-0.11.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e45e598af5afe8412197d4786efd6cf29be02e7d3d4f6a3461149eae5d7e94f1", size = 885254, upload-time = "2026-04-13T17:08:40.9Z" }, - { url = "https://files.pythonhosted.org/packages/01/25/edd584675d69e49a165052c3ee886df1c5d574f3e7d813c990306387c623/fastar-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2e160919b1c47ddb8538e7e8eb4cd527281b40f0bf75110a75993838ef61f286", size = 971239, upload-time = "2026-04-13T17:10:12.997Z" }, - { url = "https://files.pythonhosted.org/packages/a5/37/e8bb24f506ba2b08fbaf36c5800e843bd4d542954e9331f00418e2d23349/fastar-0.11.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4bb4dc0fc8f7a6807febcebce8a2f3626ba4955a9263d81ecc630aad83be84c0", size = 1035185, upload-time = "2026-04-13T17:10:30.207Z" }, - { url = "https://files.pythonhosted.org/packages/9a/bf/be753736296338149ee4cb3e92e2b5423d6ba17c7b951d15218fd7e99bbf/fastar-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4ec95af56aa173f6e320e1183001bf108ba59beaf13edd1fc8200648db203588", size = 1072191, upload-time = "2026-04-13T17:10:47.072Z" }, - { url = "https://files.pythonhosted.org/packages/d2/cd/a81c1aaafb5a22ce57c98ae22f39c89413ed53e4ee6e1b1444b0bd666a6c/fastar-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:136cf342735464091c39dc3708168f9fdeb9ebea40b1ead937c61afaf46143d9", size = 1028054, upload-time = "2026-04-13T17:11:04.293Z" }, - { url = "https://files.pythonhosted.org/packages/ec/88/1ce4eed3d70627c95f49ca017f6bbbf2ddcc4b0c601d293259de7689bc20/fastar-0.11.0-cp312-cp312-win32.whl", hash = "sha256:35f23c11b556cc4d3704587faacbc0037f7bdf6c4525cd1d09c70bda4b1c6809", size = 454198, upload-time = "2026-04-13T17:11:45.168Z" }, - { url = "https://files.pythonhosted.org/packages/8f/1d/26ce92f4331cd61a69840db9ca6115829805eec24f285481a854f578e917/fastar-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:920bc56c3c0b8a8ca492904941d1883c1c947c858cd93343356c29122a38f44c", size = 486697, upload-time = "2026-04-13T17:11:31.084Z" }, - { url = "https://files.pythonhosted.org/packages/ed/96/e6eda4480559c69b05d466e7b5ea9170e81fef3795a73e059959a3258319/fastar-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:395248faf89e8a6bd5dc1fd544c8465113b627cb6d7c8b296796b60ebea33593", size = 462591, upload-time = "2026-04-13T17:11:20.577Z" }, -] - -[[package]] -name = "fastsafetensors" -version = "0.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/07/4c/f17bd54c933fd23648ce1e4272adf27d8d7e99b788c8859544bbd39f02e7/fastsafetensors-0.3.3.tar.gz", hash = "sha256:ba4fb59be8a6adbc91723848c3c6f57a9a9a5d2247d9768f84511380d01e554c", size = 77817, upload-time = "2026-07-07T07:21:47.121Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/3a/f95f7fc099ac1fc4c22aa46257d159eac88e29dd0765d21c4fc91caedb01/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c2f788a936ffd17938484360339645812e14cf1b2bdf4c18c035c713218e5a9", size = 1887326, upload-time = "2026-07-07T07:21:34.624Z" }, - { url = "https://files.pythonhosted.org/packages/92/8c/e3347b2a44a8ab9aced94fa450df4f309baa21f7f2981a8a7bd6a977f4d3/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3587bc66b8dec560ad903becf9540889013d4d47f0e10cf45f19bedc7b7bffa7", size = 1915478, upload-time = "2026-07-07T07:21:35.901Z" }, - { url = "https://files.pythonhosted.org/packages/80/65/388a55e6b2b3023fb732843335de14803f16a6d92f0cd47f1125d28b77ac/fastsafetensors-0.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:39c252a5528fa8653366f979d2ab8fa81159db4456b7c72cb5c288adb7699079", size = 424934, upload-time = "2026-07-07T07:21:37.216Z" }, -] - -[[package]] -name = "filelock" -version = "3.32.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, -] - -[[package]] -name = "flashinfer-python" -version = "0.6.16.post3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "click" }, - { name = "cuda-python" }, - { name = "cuda-tile" }, - { name = "einops" }, - { name = "nccl4py" }, - { name = "ninja" }, - { name = "numpy" }, - { name = "nvidia-cudnn-frontend" }, - { name = "nvidia-cutlass-dsl" }, - { name = "nvidia-ml-py" }, - { name = "packaging" }, - { name = "requests" }, - { name = "tabulate" }, - { name = "torch" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c2/46/17045cb47b7b93fcb4e5303e398e6b129b86239930ac0875942a83d8b962/flashinfer_python-0.6.16.post3.tar.gz", hash = "sha256:9b146cfcc1454c80f3f99444db48013b3eadaeb32f2a399bd76c9471ecb72f04", size = 11076656, upload-time = "2026-08-08T01:14:16.408Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/dc/5367cb601fc9190cc75b4c141f5f7e9a224d1c26a1e983151823e02a25b3/flashinfer_python-0.6.16.post3-py3-none-any.whl", hash = "sha256:caf686b9b079abe1c9d65ab505698bd325e8072de40afd822f2c74f2ac3bc601", size = 15836034, upload-time = "2026-08-08T01:14:13.709Z" }, -] - -[[package]] -name = "flax" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jax" }, - { name = "msgpack" }, - { name = "numpy" }, - { name = "optax" }, - { name = "orbax-checkpoint" }, - { name = "pyyaml" }, - { name = "rich" }, - { name = "tensorstore" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e6/d8/767804f923b473790e283b6e5d0812d792195e140cacfad08a2760e0afe8/flax-0.10.0.tar.gz", hash = "sha256:8025607abc6077e29554af00996041454e89d10d6d2939d7cb4b86ecbc3fe360", size = 3623416, upload-time = "2024-10-16T23:26:25.812Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/00/c19d0108f3ce4da076f41e59019149d53f3319ee96a3db880c5f01ba40ac/flax-0.10.0-py3-none-any.whl", hash = "sha256:b3025c5c7d33ba400e123656ad66b0a1900ec67685365bf322ffa47c40a1d16c", size = 420218, upload-time = "2024-10-16T23:26:23.225Z" }, -] - -[[package]] -name = "fonttools" -version = "4.66.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/87/b6/126c659ab7e0e03e01a5f5d223abf7b2c0691ae92718085a212a3924a2a3/fonttools-4.66.1.tar.gz", hash = "sha256:64967c6ddb0d4c610dfd8cb1485981b2d27972ddfb7d4bbbd9e199d2a089c450", size = 3694174, upload-time = "2026-09-29T16:11:53.706Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/1b/fcb22638f2f5918c855abfbab203701e4b03739953d26a59f3e6e59b8f30/fonttools-4.66.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8526b2b7ec4db6b81efb83438be52b1264eda9a4994d867163cfe8c65581ce8d", size = 3097433, upload-time = "2026-09-29T16:10:07.869Z" }, - { url = "https://files.pythonhosted.org/packages/7f/0d/f51141407f9a64efc9fb39b94b0194c4a99c1ffff50834e7ca7a3f1cdd53/fonttools-4.66.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6946fe7bfb28590a1fd4061a17609c9a843952deb65dcf30d1fe725070c3e7a4", size = 2587566, upload-time = "2026-09-29T16:10:10.099Z" }, - { url = "https://files.pythonhosted.org/packages/75/c0/5810d73f9102eb1a08f26b8f7a6c22498622b43e05e684cd5ec602be8ffd/fonttools-4.66.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09ae73bd219e1245debd8376077a0fa6e03175e255c4f51bae5f6a271bfe384a", size = 5416460, upload-time = "2026-09-29T16:10:12.182Z" }, - { url = "https://files.pythonhosted.org/packages/a6/6e/babde908b879a3b51ffc230919d06a804912559dd4cf8c94f3fa2af69fec/fonttools-4.66.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b8ff9e0edbcee2fbf7dff0c41b9041c1901c26acf64e23adb67495012df11de", size = 5397326, upload-time = "2026-09-29T16:10:14.133Z" }, - { url = "https://files.pythonhosted.org/packages/e6/98/8522cc7a5e5ad64a2b2b6e9598489809ed4956c532b887c65b131b8500b7/fonttools-4.66.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38ce8f5fbd5c17dd2153d47d7c8d4108f3deda3f2b4a79b60ddc470a58faded3", size = 5353857, upload-time = "2026-09-29T16:10:16.375Z" }, - { url = "https://files.pythonhosted.org/packages/15/f9/ab87d67c23178886e57f24397b3113b4ac35297f086467c2c4d4231671a8/fonttools-4.66.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8aed2bbcd6216253ef1b015763593365ee8084f621dfa53bb957c19d5e05f7cd", size = 5517604, upload-time = "2026-09-29T16:10:18.791Z" }, - { url = "https://files.pythonhosted.org/packages/d4/20/e126062610aea31919310b0b3de4d0bbe73ccbdcb3d0409cc52027db93ee/fonttools-4.66.1-cp312-cp312-win32.whl", hash = "sha256:9ea6c93091cbf83161a544388746a0911550bd98cb911faca3591cf5ead166ac", size = 2435084, upload-time = "2026-09-29T16:10:21.145Z" }, - { url = "https://files.pythonhosted.org/packages/f0/af/5c245a0587e5b7b3dd209f640b9f70d8e63404ad8dc93819ba578d685985/fonttools-4.66.1-cp312-cp312-win_amd64.whl", hash = "sha256:261d8dc95845e751f975fe8d6075600593ee253470d46d1b84801688051b09f6", size = 2486987, upload-time = "2026-09-29T16:10:22.924Z" }, - { url = "https://files.pythonhosted.org/packages/f6/10/d45b74135d5d642cb3a4fb0a957c1613ef93de4c8548671dfc3a5bf38299/fonttools-4.66.1-py3-none-any.whl", hash = "sha256:7234ae9e28db64273fbbfa72caebd0a97e3bdba6b05064114741b9539ef339d0", size = 1202222, upload-time = "2026-09-29T16:11:51.678Z" }, -] - -[[package]] -name = "frozenlist" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, - { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, - { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, - { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, - { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, - { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, - { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, - { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, - { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, - { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, - { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, - { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, - { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, - { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, - { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, -] - -[[package]] -name = "fsspec" -version = "2026.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, -] - -[[package]] -name = "gast" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/91/f6/e73969782a2ecec280f8a176f2476149dd9dba69d5f8779ec6108a7721e6/gast-0.7.0.tar.gz", hash = "sha256:0bb14cd1b806722e91ddbab6fb86bba148c22b40e7ff11e248974e04c8adfdae", size = 33630, upload-time = "2025-11-29T15:30:05.266Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/33/f1c6a276de27b7d7339a34749cc33fa87f077f921969c47185d34a887ae2/gast-0.7.0-py3-none-any.whl", hash = "sha256:99cbf1365633a74099f69c59bd650476b96baa5ef196fec88032b00b31ba36f7", size = 22966, upload-time = "2025-11-29T15:30:03.983Z" }, -] - -[[package]] -name = "google-api-core" -version = "2.33.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "googleapis-common-protos" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/87/62/8fb1fb647d2788c950d69d6a769cd9d55c918ac1fc57be2f90b7e4029787/google_api_core-2.33.0.tar.gz", hash = "sha256:3a36bcc3e319783f4c97da41f6f45ea6ffcaa55848e341de16e09cb70243c2bb", size = 181607, upload-time = "2026-07-22T16:28:28.027Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/31/5056a347bb934ea04583c8b27916ef1501729c72638629545bce26ff4223/google_api_core-2.33.0-py3-none-any.whl", hash = "sha256:a2e22a0c1d0f03eafff1858b38cf46f832d5902b0c052235bf0ab8402929fbdc", size = 176462, upload-time = "2026-07-22T16:28:22.447Z" }, -] - -[[package]] -name = "google-auth" -version = "2.56.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyasn1-modules" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c7/33/dbc946a407401b975f0719658f18e664ece2109f79ffd1ff3bf226c205f4/google_auth-2.56.2.tar.gz", hash = "sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051", size = 365820, upload-time = "2026-07-21T21:53:28.188Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/63/50636aae68c9bf17c891c7eb18b49baa9bd6b31d2a97b8de4813a9fc8d1c/google_auth-2.56.2-py3-none-any.whl", hash = "sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6", size = 258588, upload-time = "2026-07-21T21:53:26.399Z" }, -] - -[[package]] -name = "google-cloud-core" -version = "2.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a8/dd/1eef226e470369b26824a505c34482c0b493bc35fe8e0c6b003b5feca21a/google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83", size = 36001, upload-time = "2026-05-07T08:04:04.124Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390, upload-time = "2026-05-07T08:02:34.672Z" }, -] - -[[package]] -name = "google-cloud-storage" -version = "3.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e3/25/355ed97c1723c787dfaa888808d55db18371f82c38ff862357b1e902cd19/google_cloud_storage-3.13.0.tar.gz", hash = "sha256:d11d8706ea1520fba0f21043bcb7897caf7015d76ce1ad9a4f60237e4d7a9f6c", size = 17340960, upload-time = "2026-07-13T19:10:07.524Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/e8/b3678a0931ee7d4b3fdaf0813e6206d66e0922b2c26d912f308728b5b95a/google_cloud_storage-3.13.0-py3-none-any.whl", hash = "sha256:648af3ef8a6acc674e1359d3c920c67eb89a7a5ab66b336bd3ac43fed6b5ab84", size = 341428, upload-time = "2026-07-13T19:09:52.39Z" }, -] - -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, - { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, -] - -[[package]] -name = "google-resumable-media" -version = "2.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-crc32c" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/f8/1ca5781d6be9cb9f73f7d40f4958c4bd1226a60598e3e39e1d6aaf838c4b/google_resumable_media-2.10.0.tar.gz", hash = "sha256:e324bc9d0fdae4c52a08ae90456edc4e71ece858399e1217ac0eb3a51d6bc6ee", size = 2164570, upload-time = "2026-06-03T16:14:26.103Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/d8/00c6854ac1512bb9eaf13bd3f8f28222f7674947fc510a4ff7616f2efc80/google_resumable_media-2.10.0-py3-none-any.whl", hash = "sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c", size = 81533, upload-time = "2026-06-03T16:13:12.51Z" }, -] - -[[package]] -name = "google-vizier" -version = "0.1.21" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "attrs" }, - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "grpcio-tools" }, - { name = "numpy" }, - { name = "portpicker" }, - { name = "protobuf" }, - { name = "sqlalchemy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/89/25/38c4cd3a3cc9786fd87998016a709cae88ebacc11eb3b5395737b5eccebb/google-vizier-0.1.21.tar.gz", hash = "sha256:b0ef8514675ee618ae126c4c418c7c38c2934f4f0ddfb1a3670c01d6499e17f3", size = 515482, upload-time = "2025-01-07T21:02:04.894Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/b6/00c7d87a565d98727dcd857e65b3bf6e2c09d060e680fbea6e44abc90fc5/google_vizier-0.1.21-py3-none-any.whl", hash = "sha256:ae457cb42ab55f531223dcbc2b973f2050850eba40808d4927afb3ab0d7e4d2c", size = 799598, upload-time = "2025-01-07T21:02:03.047Z" }, -] - -[package.optional-dependencies] -jax = [ - { name = "chex" }, - { name = "equinox" }, - { name = "flax" }, - { name = "jax" }, - { name = "jaxlib" }, - { name = "jaxopt" }, - { name = "jaxtyping" }, - { name = "optax" }, - { name = "tfp-nightly", extra = ["jax"] }, - { name = "typeguard" }, -] - -[[package]] -name = "googleapis-common-protos" -version = "1.75.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, -] - -[[package]] -name = "grpcio" -version = "1.83.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, - { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, - { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, - { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, - { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, - { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, - { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, - { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" }, - { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" }, -] - -[[package]] -name = "grpcio-tools" -version = "1.81.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/b3/1c5951352d6777fd7f99a0ccee04617fdfd8a5dbf2918a1f58c8b2b280b8/grpcio_tools-1.81.1.tar.gz", hash = "sha256:a22a3870180927fdd84e2b27d079ef5b7f5f8c6110181b6736afc17a463481f1", size = 6236155, upload-time = "2026-06-11T12:51:21.235Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/8a/824a9ca20bcdce8a568bb8c9f98bfeb7fad62129235e6d2ae7576fd1250a/grpcio_tools-1.81.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:353b1fafcc739c31ed42271052709595b340d34f27c459beeb78a32938305bb5", size = 2585927, upload-time = "2026-06-11T12:50:05.671Z" }, - { url = "https://files.pythonhosted.org/packages/2f/35/e5f9f671378b1b89a896150d3e4fa2c6ec61a5e1e9e5107ce4c140ccc931/grpcio_tools-1.81.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:768f584c2423cbeb6cb6867817a39365b987ff16b8259a3adbc6546b9e303a4e", size = 5815665, upload-time = "2026-06-11T12:50:08.178Z" }, - { url = "https://files.pythonhosted.org/packages/c6/02/631b628e4072e988c669bd8f1b2406ef3c9a4cfcb2625bbf2a308a07b71d/grpcio_tools-1.81.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1680b35a84f4694401819ac4acac42dda6dbc7bb8fc74112fd1a60425a07adf4", size = 2635518, upload-time = "2026-06-11T12:50:10.391Z" }, - { url = "https://files.pythonhosted.org/packages/de/7c/2e3537e3ea3d1c0ddd6766cf6a7c62b487d89fb005713df2781d5f21483a/grpcio_tools-1.81.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f64e665c8ec639278ecf009beb92cbdcc5994f617c1af3d58036e1f70b1423ec", size = 2958252, upload-time = "2026-06-11T12:50:12.677Z" }, - { url = "https://files.pythonhosted.org/packages/35/68/14013cb2942bdac354746b643b4c37dd91906da8dce00f41c616e88bf33d/grpcio_tools-1.81.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba8f1ae82ad199f43448995715445cc623fb20d3882382e4be61f0da8ccb3f0e", size = 2698439, upload-time = "2026-06-11T12:50:15.017Z" }, - { url = "https://files.pythonhosted.org/packages/bd/45/000c14c0338a7ad36054b9f17ea41842deb7841c05c067dd36cc831bc0f4/grpcio_tools-1.81.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7b6d1e986d5923751bfe2b5cca9c4cb3d5653446e4fa4aacd438033e2dc360a", size = 3152160, upload-time = "2026-06-11T12:50:17.3Z" }, - { url = "https://files.pythonhosted.org/packages/41/97/881930ca3967d2c8a95649bea8ebc991a7cf2331bc96679fd3600450dccc/grpcio_tools-1.81.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7f208c207aca639dcb34648d3826c38d7cf3485118fb2065117e9fc4827406b3", size = 3710468, upload-time = "2026-06-11T12:50:19.479Z" }, - { url = "https://files.pythonhosted.org/packages/c7/b5/67baeba7366162652cdc1dbd962289accde07241bc8f42f6f02b305efcc6/grpcio_tools-1.81.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:724ecb69af63d2f6d4ccea3e6fa0ca110ed9c5824d48c2f887c631bbb03c1c3c", size = 3370797, upload-time = "2026-06-11T12:50:21.501Z" }, - { url = "https://files.pythonhosted.org/packages/f2/5d/34f2dce2125ccb107e32b57f5a9c1257edcc0793b0d2fef1e8b13a6bac3c/grpcio_tools-1.81.1-cp312-cp312-win32.whl", hash = "sha256:895a6782cec86beac71ccebb4b9848259c6f04a3028b8e42fa8d40cfe5146593", size = 1008453, upload-time = "2026-06-11T12:50:23.358Z" }, - { url = "https://files.pythonhosted.org/packages/8a/be/09da8256ec8d2a5ce8a1acc51cbbc4ca52a462d78ed3412778440a56502e/grpcio_tools-1.81.1-cp312-cp312-win_amd64.whl", hash = "sha256:0265fd1386b7458302f79542558345880d484f8fa92ae196c0c0268242c5f23a", size = 1174857, upload-time = "2026-06-11T12:50:25.685Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "hf-xet" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9e/27/06d899ea7bd721d272f84aac98bdb238de98af4cc767a69056d967d68c71/hf_xet-1.7.0.tar.gz", hash = "sha256:d406ec79053c0871817f700c2ac8c36ba0d87f9c34b7458b0f0063bb218b0466", size = 985689, upload-time = "2026-10-06T20:18:43.89Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/0b/b03be21ffaada749ba0d3197d8aefbf1aa698bac149580421c15239b299e/hf_xet-1.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e88a7a75d7d95cbee1f37dc31341d6201124cf21c6c4b1dfab8ccba9b09e0f", size = 3796096, upload-time = "2026-10-06T20:18:28.43Z" }, - { url = "https://files.pythonhosted.org/packages/c3/47/a26ebdce7056a61e931f228439bc0ab08cbec239d1690f965e5e637cba79/hf_xet-1.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:59fba37039233c7fcbe196817d6cdcf1b40dfb17b410f229d85b0cf0a1848da4", size = 3560352, upload-time = "2026-10-06T20:18:30.365Z" }, - { url = "https://files.pythonhosted.org/packages/a3/4c/2bf3b66c215d409655f28de1622393dde04c9461280d48c7924bb3b2decd/hf_xet-1.7.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2814a6e999d13464c4d679b788cc5d784eb5a4edfc638a31f10e9a11ab531ef8", size = 4212180, upload-time = "2026-10-06T20:18:32.292Z" }, - { url = "https://files.pythonhosted.org/packages/49/0c/a2f703a5a78267556e89e03316fa0805c86b72b50829bc67665746e8ebf0/hf_xet-1.7.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fcfd6c22418e57dd5b3aea649e813b2e2cfb2aebf317b210d90f1fe4b3018b52", size = 3990011, upload-time = "2026-10-06T20:18:34.21Z" }, - { url = "https://files.pythonhosted.org/packages/a4/77/e52e4201b1cbf571530a61cc57f70182045a39a230089ee5f1df182a4de2/hf_xet-1.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80f79dae613ce9e0ea1fd1ae15616ca9ac74aed4c770aabc199c4f03ebecc863", size = 4190628, upload-time = "2026-10-06T20:18:36.062Z" }, - { url = "https://files.pythonhosted.org/packages/6c/dc/03a21b89f118664a0926ff25b0f8e44a519bf22724a6a8fc7a9abbc188b6/hf_xet-1.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0a9e802f33bf50c851abe45fc5380e61f959e2d369647d6742b79ad9d6c27cab", size = 4418814, upload-time = "2026-10-06T20:18:37.888Z" }, - { url = "https://files.pythonhosted.org/packages/4d/59/b35106dfa71b6eef605dc88bd038fe99c7f86fb132a15b60d0bf2f235b2c/hf_xet-1.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:2b7bb5727889b0f2436dbaaad8fc4c3e66b8240d992716989e0c086b4278b1bc", size = 3822644, upload-time = "2026-10-06T20:18:40.052Z" }, - { url = "https://files.pythonhosted.org/packages/48/cd/072313585f74fe9d441e2eb5e0a4703c30586cd709810ea369675f61b74e/hf_xet-1.7.0-cp38-abi3-win_arm64.whl", hash = "sha256:acc3851cf2576a8fb2ae926da863f4efabe21303cf292e9a44332802ab0dcc6a", size = 3662436, upload-time = "2026-10-06T20:18:42.205Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpcore2" -version = "2.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "h11" }, - { name = "truststore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/a8/20ed1ed79cbc2ecdf5301c0968ab7c85547212e2a7bd126ddd2d986e206e/httpcore2-2.9.1.tar.gz", hash = "sha256:4d8acbf8b306f48c9d6046591fd5ba4037d1b1b1000d140fc2c3eab1e9a0c0e2", size = 67089, upload-time = "2026-07-24T09:21:03.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/fb/46c52b781975c335a2bcf1072c7bbc007cbdc8d674217f5ee1daba2c848b/httpcore2-2.9.1-py3-none-any.whl", hash = "sha256:6182472379e855fe4221246a2bb7ecede403bc61c6798062ae1787d051ccde26", size = 82809, upload-time = "2026-07-24T09:21:01.178Z" }, -] - -[[package]] -name = "httptools" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/88/1d21a36da8f5cb0fa49eafd4b169eba5608d57e75bbcf61845cbc6243216/httptools-0.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:880490234c10f70a9830743097e8958d6e4b9f5a0ffc24515023afeef984054d", size = 208247, upload-time = "2026-05-25T22:17:07.843Z" }, - { url = "https://files.pythonhosted.org/packages/a5/42/cc4feea2945cb3051038f090c9b36bd5b8a9d7f5a894a506a8983e33fd1c/httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5", size = 113064, upload-time = "2026-05-25T22:17:09.136Z" }, - { url = "https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2", size = 523851, upload-time = "2026-05-25T22:17:10.106Z" }, - { url = "https://files.pythonhosted.org/packages/b7/e4/f90a0df0b83beff265b7e3b65f2a4cefd95792d4be0ac3e16049f2acd3c2/httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09", size = 518842, upload-time = "2026-05-25T22:17:11.218Z" }, - { url = "https://files.pythonhosted.org/packages/9e/2d/0c9ac76dd2c893841fbf6498d6acec4f2442e1b7067f6e3e316a80e494e8/httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a", size = 501238, upload-time = "2026-05-25T22:17:12.728Z" }, - { url = "https://files.pythonhosted.org/packages/ca/42/906adc91ae3a5fa9c59c0a2f21c139725bd7e5b41ae6acd485cd14123ebf/httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745", size = 509567, upload-time = "2026-05-25T22:17:13.842Z" }, - { url = "https://files.pythonhosted.org/packages/05/0b/4240efeb672751ee5b9b380cb0e3fdc050bc05f68adc7a8aefc4fcd9a69a/httptools-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd96f29b4bab1d42fa6e3d008711c75e0f79e94e06827330160e3a304227f150", size = 90918, upload-time = "2026-05-25T22:17:15.155Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[[package]] -name = "httpx2" -version = "2.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "httpcore2" }, - { name = "idna" }, - { name = "truststore" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/14/38128fbafd7e0ed41d874df6c9a653d47c2d111cfe59e2b4ac95161b4abd/httpx2-2.9.1.tar.gz", hash = "sha256:1932a768737e3666291582833da748cc4e563c337cf96706fccc04fa6e58764a", size = 95458, upload-time = "2026-07-24T09:21:04.972Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/b8/cfd91c4ab9134d386d48f0b6ac662ff3d4be6efdee59ee1c67ebc3c0487c/httpx2-2.9.1-py3-none-any.whl", hash = "sha256:1820fe14a9ab1107bfeff39259987429450b070ec0ff38cc87eb0d8c97fdc71a", size = 91191, upload-time = "2026-07-24T09:21:02.6Z" }, -] - -[[package]] -name = "huggingface-hub" -version = "1.33.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "httpx" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/2a/484d112c0d8fc5f665d7b65137ac9cdb2953c982391598c3597968a12ee7/huggingface_hub-1.33.0.tar.gz", hash = "sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e", size = 1043548, upload-time = "2026-09-24T09:49:29.971Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/16/963096d224b80909432dc16561a615fd33d2d13beef3ce4c63fa25e40867/huggingface_hub-1.33.0-py3-none-any.whl", hash = "sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899", size = 846435, upload-time = "2026-09-24T09:49:28.059Z" }, -] - -[[package]] -name = "humanize" -version = "4.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/ea/13a1ef3c12d12662905801495283530251918b70d62d368f1d2e0272c70d/humanize-4.16.0.tar.gz", hash = "sha256:7dc2244a2f84a4bfb1d36c37bac80cd78e35cdc5c119206d87b018e1445f3a3f", size = 89515, upload-time = "2026-06-30T16:17:29.859Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/aa/0b7365d30fed43e7a3449aba1fe20a0a7174d9cf13e282af4e69ac825441/humanize-4.16.0-py3-none-any.whl", hash = "sha256:353eb2f34c09d098b2880eee8bef21832eae6d174f48c5762fff7e5fcb74d01d", size = 137209, upload-time = "2026-06-30T16:17:28.36Z" }, -] - -[[package]] -name = "humming-kernels" -version = "0.1.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings" }, - { name = "jinja2" }, - { name = "numpy" }, - { name = "nvidia-ml-py" }, - { name = "safetensors" }, - { name = "tabulate" }, - { name = "torch" }, - { name = "tqdm" }, - { name = "triton" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/b9/53797fff4b1b31b4567453a365646d724469b1bb0d76c07f8746617d0cb6/humming_kernels-0.1.12.tar.gz", hash = "sha256:c96c37c95f74379067d2d2d0c5154d12ee9a8cfa29a7c3370a7f3e34f207ca72", size = 259323, upload-time = "2026-07-31T14:28:23.124Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/0f/01f871dc26f8d7e1683df9464890d48cfd5c4e46c4a264d7ee0868749197/humming_kernels-0.1.12-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:988e8b9da41e3679ae2816bdfb51acab53c05cc56254b5ca98e847f4efcf24fd", size = 318019, upload-time = "2026-07-31T14:28:19.925Z" }, - { url = "https://files.pythonhosted.org/packages/ec/a2/f96912ee7d68f56e61c15555657ffa057b8afbaa21241c63342b5969bb74/humming_kernels-0.1.12-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:cd3ef712a93f3a9075ea99de2c72bcd3ec89dab3759b3a248d869f5507b60331", size = 322656, upload-time = "2026-07-31T14:28:21.511Z" }, -] - -[package.optional-dependencies] -cu13 = [ - { name = "nvidia-cuda-cccl" }, - { name = "nvidia-cuda-nvcc", version = "13.2.86", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, - { name = "nvidia-cuda-nvcc", version = "13.3.73", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, - { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, - { name = "nvidia-cuda-nvrtc", version = "13.3.33", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, - { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, - { name = "nvidia-cuda-runtime", version = "13.3.29", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, -] - -[[package]] -name = "idna" -version = "3.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, -] - -[[package]] -name = "ijson" -version = "3.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3a/06/b31f040a8764336a11152e474a7abcb3782fedb0d1cdf78f442b82878c56/ijson-3.5.1.tar.gz", hash = "sha256:af40bd1a85f55db0b8b30715c858761306bd92d5590148636f75c3309e6e76bd", size = 69913, upload-time = "2026-07-06T17:37:42.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/6e/f3ded1ebb85ccc89a30f7b10a0076f30db70ae1d1e0b6423ff93c57b7539/ijson-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ee60c7741012671867678eae71c51872cac938b76f3d4ca40a778e6c361774d2", size = 88643, upload-time = "2026-07-06T17:36:28.529Z" }, - { url = "https://files.pythonhosted.org/packages/ee/f2/18f14a1d79ef4898e746b4f50dcdbe60abab317cc2bd8390f043b9553c4e/ijson-3.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:11c1d7d36a13054b5872ecd5d745dc4009d9abdbcba2312de69e66c2f92a46d2", size = 60611, upload-time = "2026-07-06T17:36:29.597Z" }, - { url = "https://files.pythonhosted.org/packages/30/c7/6e3e591324fd4c7a7a9e1bc23548bacbd84c0d91766b71f09f13e945e7e9/ijson-3.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b9517efbe6604bce16f3e50d49b0cd1bdc58917f98cf2eab026599c5c0422991", size = 60447, upload-time = "2026-07-06T17:36:30.747Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a5/9af7be670381ddac26dd55107ed0110b50f5161673b053311db67f510dcc/ijson-3.5.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ea4fd7bec203a600b1cc88a492dfe6b75ce4b1b87488a66adcd5406022213f64", size = 139092, upload-time = "2026-07-06T17:36:31.749Z" }, - { url = "https://files.pythonhosted.org/packages/41/fb/f9c1664d75467453e6bd4e5f9cd2211b730b09e049445ab64cbac68cc6a3/ijson-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350caea815e53151994b597abc80cf669454276b5ac6aadcec69ef6d48f7e90b", size = 149921, upload-time = "2026-07-06T17:36:32.912Z" }, - { url = "https://files.pythonhosted.org/packages/43/80/d20b1c49c4aa7cc6644131e2e57192b45346ef4816566ed1cd9fd05bae38/ijson-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4fcebfe1685bb7ba06a8255a5d428ea6b4b895d7acf979cb637d8bbc9db2f47", size = 149848, upload-time = "2026-07-06T17:36:34.032Z" }, - { url = "https://files.pythonhosted.org/packages/fd/fc/5baa710869f5ab939e6233583ced1546889b55c35f35b844c518ac10abc3/ijson-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d78f362f51c8691798758a9e6ac3c9d385ee1228cb82987c91562a2fae235cd3", size = 150810, upload-time = "2026-07-06T17:36:35.19Z" }, - { url = "https://files.pythonhosted.org/packages/54/16/a12b3d987a5c1677b04557c6f9b9feb7e04b7d4171e9a344856cb9136e9b/ijson-3.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0b184180d45f85fd4479659582749b109e49f4a29c21ac700ccc9c2280fe015e", size = 142989, upload-time = "2026-07-06T17:36:36.23Z" }, - { url = "https://files.pythonhosted.org/packages/ed/63/1026c535671fc334fc85aeb78f0945c825e7a338575edc753c0f455459ae/ijson-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e353891d33a2e6aa5caf72c2a5fbadd7a46f5f9b32dcfd0c84113b2444c255b8", size = 151702, upload-time = "2026-07-06T17:36:37.296Z" }, - { url = "https://files.pythonhosted.org/packages/cb/af/b58aa3a2bf4d31c388ea78b49826605f60932891ce97e404d196766b4ea3/ijson-3.5.1-cp312-cp312-win32.whl", hash = "sha256:936f28671f018f8ac4d3f003ae9fa01d0467ab4ef4cfd0c97f23beda485b61c6", size = 52613, upload-time = "2026-07-06T17:36:38.345Z" }, - { url = "https://files.pythonhosted.org/packages/04/66/ce70a92949c2a753dad91fdd5761dc14f3a44517e80cfc3c26612982ed61/ijson-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:322c783f3ee0c6b383bbd4db88370b10172168808cc2a0bf811f1253f7435602", size = 54729, upload-time = "2026-07-06T17:36:39.337Z" }, - { url = "https://files.pythonhosted.org/packages/a5/ff/e17784240c9cf1d58de2f2853ebaf9cc54f6bce117a1f12a6150bbb4a5aa/ijson-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:e2ac204b59f09e38e16d277f906240e9fd38780e42076599419265af183dc4b4", size = 53714, upload-time = "2026-07-06T17:36:40.308Z" }, -] - -[[package]] -name = "interegular" -version = "0.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/9d/8b6dde58a028a3962ce17e84d5fe73758df61378e00ef8ac3d85da34b0ff/interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600", size = 24705, upload-time = "2024-01-06T23:01:22.372Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/01/72d6472f80651673716d1deda2a5bbb633e563ecf94f4479da5519d69d25/interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c", size = 23635, upload-time = "2024-01-06T23:01:20.829Z" }, -] - -[[package]] -name = "isodate" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, -] - -[[package]] -name = "jax" -version = "0.4.38" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jaxlib" }, - { name = "ml-dtypes" }, - { name = "numpy" }, - { name = "opt-einsum" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/e5/c4aa9644bb96b7f6747bd7c9f8cda7665ca5e194fa2542b2dea3ff730701/jax-0.4.38.tar.gz", hash = "sha256:43bae65881628319e0a2148e8f81a202fbc2b8d048e35c7cb1df2416672fa4a8", size = 1930034, upload-time = "2024-12-17T23:03:47.623Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/49/b4418a7a892c0dd64442bbbeef54e1cdfe722dfc5a7bf0d611d3f5f90e99/jax-0.4.38-py3-none-any.whl", hash = "sha256:78987306f7041ea8500d99df1a17c33ed92620c2268c4c3677fb24e06712be64", size = 2236864, upload-time = "2024-12-17T23:03:44.433Z" }, -] - -[[package]] -name = "jaxlib" -version = "0.4.38" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ml-dtypes" }, - { name = "numpy" }, - { name = "scipy" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/df/08b94c593c0867c7eaa334592807ba74495de4be90580f360db8b96221dc/jaxlib-0.4.38-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:3fefea985f0415816f3bbafd3f03a437050275ef9bac9a72c1314e1644ac57c1", size = 99737849, upload-time = "2024-12-17T23:06:20.406Z" }, - { url = "https://files.pythonhosted.org/packages/ab/b1/c9d2a7ba9ebeabb7ac37082f4c466364f475dc7550a79358c0f0aa89fdf2/jaxlib-0.4.38-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f33bcafe32c97a562ecf6894d7c41674c80c0acdedfa5423d49af51147149874", size = 79509242, upload-time = "2024-12-17T23:06:33.73Z" }, - { url = "https://files.pythonhosted.org/packages/53/25/dd670d8bdf3799ece76d12cfe6a6a250ea256057aa4b0fcace4753a99d2d/jaxlib-0.4.38-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:496f45b0e001a2341309cd0c74af0b670537dced79c168cb230cfcc773f0aa86", size = 93251503, upload-time = "2024-12-17T23:06:48.243Z" }, - { url = "https://files.pythonhosted.org/packages/f9/cc/37fce5162f6b9070203fd76cc0f298d9b3bfdf01939a78935a6078d63621/jaxlib-0.4.38-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:dad6c0a96567c06d083c0469fec40f201210b099365bd698be31a6d2ec88fd59", size = 101792792, upload-time = "2024-12-17T23:07:01.615Z" }, - { url = "https://files.pythonhosted.org/packages/6f/7a/8515950a60a4ea5b13cc98fc0a42e36553b2db5a6eedc00d3bd7836f77b5/jaxlib-0.4.38-cp312-cp312-win_amd64.whl", hash = "sha256:966cdec36cfa978f5b4582bcb4147fe511725b94c1a752dac3a5f52ce46b6fa3", size = 64288223, upload-time = "2024-12-17T23:07:15.024Z" }, -] - -[[package]] -name = "jaxopt" -version = "0.8.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jax" }, - { name = "jaxlib" }, - { name = "numpy" }, - { name = "scipy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/da/ff7d7fbd13b8ed5e8458e80308d075fc649062b9f8676d3fc56f2dc99a82/jaxopt-0.8.5.tar.gz", hash = "sha256:2790bd68ef132b216c083a8bc7a2704eceb35a92c0fc0a1e652e79dfb1e9e9ab", size = 121709, upload-time = "2025-04-14T17:59:01.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/d8/55e0901103c93d57bab3b932294c216f0cbd49054187ce29f8f13808d530/jaxopt-0.8.5-py3-none-any.whl", hash = "sha256:ff221d1a86908ec759eb1e219ee1d12bf208a70707e961bf7401076fe7cf4d5e", size = 172434, upload-time = "2025-04-14T17:59:00.342Z" }, -] - -[[package]] -name = "jaxtyping" -version = "0.3.11" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wadler-lindig" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/c1/091b8852bd7cbf50bd655543c8506033cf4029300c67f8c176c1286879a9/jaxtyping-0.3.11.tar.gz", hash = "sha256:b09c14acf6686feb9e0df5b0d8c6e7c5b6f8d36bf059ee54cd522a186c2ef050", size = 46489, upload-time = "2026-06-13T18:35:23.167Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/38/c66bbdc5047f4776c2bd3e47e5295a350e3fa44d5b8942105e71c2a876a0/jaxtyping-0.3.11-py3-none-any.whl", hash = "sha256:8a4bedc4e3f963fa82df41bd13c7ebc2bad925601eb48614c65798f21329d4e3", size = 56593, upload-time = "2026-06-13T18:35:22.01Z" }, -] - -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - -[[package]] -name = "jiter" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, - { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, - { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, - { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, - { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, - { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, - { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, - { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, - { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, - { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, - { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, - { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, - { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, - { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, - { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, - { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, -] - -[[package]] -name = "jmespath" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, -] - -[[package]] -name = "joblib" -version = "1.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, -] - -[[package]] -name = "jsonschema" -version = "4.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rpds-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, -] - -[[package]] -name = "jsonschema-specifications" -version = "2025.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "referencing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, -] - -[[package]] -name = "kiwisolver" -version = "1.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/07/bd78e6a8fae171ea041ef5bba3ed21a003522fa088834b069b1909981f30/kiwisolver-1.5.1.tar.gz", hash = "sha256:f1303ef2eec81262a4b708c3e858afe58d7c75ad91c1c05266eda7673369859a", size = 104395, upload-time = "2026-08-28T10:28:27.153Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/9b/65b302742389c6f96f2956bef5decf26011309feb2fc5d79613af18adea4/kiwisolver-1.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:63fb7294b768f444eb4b068965f2662f28c2fd4161e23bd60fcf3ff27b74c046", size = 123876, upload-time = "2026-08-28T10:25:27.44Z" }, - { url = "https://files.pythonhosted.org/packages/71/74/c21f339956f6f691b2ed7e31d5f3ae767304df6c460192739fc830853051/kiwisolver-1.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0ebdef3eae5336568147c39a55be6a2036ffde53faa9ca2d978989ae7c2da12c", size = 66487, upload-time = "2026-08-28T10:25:28.728Z" }, - { url = "https://files.pythonhosted.org/packages/84/e5/bdb34e21523e01dceda064d63713f3bdec91388af24fba1eca7ea5e85864/kiwisolver-1.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1798e83840c3f627246104c4d8a9639c60fa068adf9ce92b61791781fa8a68c1", size = 64660, upload-time = "2026-08-28T10:25:30.071Z" }, - { url = "https://files.pythonhosted.org/packages/fc/f4/dadfec469313c7f428efa7e84b4aba9732f813c13ea7131a24b7b008ef57/kiwisolver-1.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34633ecf50d16187ab8e5528b7a2530f2feb4e23f300db4672538b51cfc5cd38", size = 1477929, upload-time = "2026-08-28T10:25:31.495Z" }, - { url = "https://files.pythonhosted.org/packages/6f/35/09c58daac34e6f6ea5c6dee0094b422118e5a7c265586008a95fd135ac5f/kiwisolver-1.5.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d27c2123977cb9269c30a49ba45f03a4323017ef693e19db4ec9dbe1299a3002", size = 1278499, upload-time = "2026-08-28T10:25:33.375Z" }, - { url = "https://files.pythonhosted.org/packages/19/32/739765e24fbad29d13f83e546ea4abc215a78cea9d677ca09025b027724d/kiwisolver-1.5.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6a797a1cefc8b9c93170db580337e1fe3d011ad18b1299943231279406342048", size = 1296677, upload-time = "2026-08-28T10:25:35.059Z" }, - { url = "https://files.pythonhosted.org/packages/df/32/03304d1010e2cc45e5b3b52cef7e43fed3a2a5cd6c87a89b4a88e1d85b5d/kiwisolver-1.5.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2551cf9917af48ee7c4b29cc82320489508cf96fd26a51f6fc124de661cd44c7", size = 1346037, upload-time = "2026-08-28T10:25:36.705Z" }, - { url = "https://files.pythonhosted.org/packages/3e/57/4c49377bfd274450dd72ecaa13eaac32ea804a03363e4d1db0c5aa999ceb/kiwisolver-1.5.1-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:38f6e0deb4d0a4615efe0c4efc5990b06ae450ab50a0b321c0b078b6d238c083", size = 988248, upload-time = "2026-08-28T10:25:38.299Z" }, - { url = "https://files.pythonhosted.org/packages/cb/c3/38df144a08b6c5d75ca4504e5cc3141bb3bfef64c04f4ef48204f42711b6/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bfd1de989b3330420e29de39352f5c049905c9e3ee67233a50d550e3d652c148", size = 2228722, upload-time = "2026-08-28T10:25:40.038Z" }, - { url = "https://files.pythonhosted.org/packages/e7/11/3221838a89cd64d9b386353e000cd8a296069a20fbe3584507fdfd5bebae/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1209042a623ddfda5497e4066c7b77651dde8e1d3a9dd97599dc7e97f3b9b78c", size = 2325216, upload-time = "2026-08-28T10:25:41.699Z" }, - { url = "https://files.pythonhosted.org/packages/83/d4/075c219230697bb5db910d37262b9bacf880f92b4811a02ab81ed073a253/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:26e8268480be5061d509e29669d59103c067a26377a56491630ece11762e3858", size = 1977689, upload-time = "2026-08-28T10:25:43.559Z" }, - { url = "https://files.pythonhosted.org/packages/bb/08/1d219c3c2dd960983d0d4da623d916e9de6385df2b0bab3d1af0e9b8fccc/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d79308fa689fac89cbcfbd4dbfc80b5f95c54c5a7fd4d194be221f9d33d026e6", size = 2491443, upload-time = "2026-08-28T10:25:45.242Z" }, - { url = "https://files.pythonhosted.org/packages/ba/d3/024208ec1079d273f1047468d1bdffbf38bb75b7b268090fd3a0301b9d9a/kiwisolver-1.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b03af77d77e50edba2030fd5f7c352ff209314b09030a3cba7c14edf9a09a444", size = 2295200, upload-time = "2026-08-28T10:25:46.984Z" }, - { url = "https://files.pythonhosted.org/packages/6e/7c/7b210498f9f92e1cd7855f260fa69ef056881087b199ee20c208f0e4189a/kiwisolver-1.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:06a6917674de9e0fe3f66f5430787f59a9f2ddb64af9b714eaec547e29ef5c19", size = 70748, upload-time = "2026-08-28T10:25:48.444Z" }, - { url = "https://files.pythonhosted.org/packages/94/61/ef0daa157c8bb23672f7423e0d14c39db1dc6ef8ed47e6bc54c9c1bef3bf/kiwisolver-1.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:ad8b9671348d7c8716715652ae11f85ed0eb99e265a2df2ca490577d69860b2c", size = 68324, upload-time = "2026-08-28T10:25:49.81Z" }, - { url = "https://files.pythonhosted.org/packages/a9/c4/1407df7512a5b36cc79840e01710dc575733c461b13ab866cae77eaf87f3/kiwisolver-1.5.1-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:482676e5bd48d70ac99d9fc78863469845421e01184fa83f1f9366dc49f7e974", size = 134002, upload-time = "2026-08-28T10:28:08.881Z" }, - { url = "https://files.pythonhosted.org/packages/16/45/c37a21ad5c0ab581a93c55ad544721aaa1f0ae94edb29c6a678a23d013e6/kiwisolver-1.5.1-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:072bdb15a3c19a5b5dbc8f8fb1f4e1884bf4f3507eeb4cc6334401274d37a5c0", size = 194292, upload-time = "2026-08-28T10:28:11.06Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c1/69f00d627949580e43d57af0aa465df46868d7c29801c137a55374101294/kiwisolver-1.5.1-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:a5a00665d1a0e26763a7338d7e911d4598fbc1d50dd0d6b7919b7dc6c5d6569f", size = 73362, upload-time = "2026-08-28T10:28:12.449Z" }, -] - -[[package]] -name = "kubernetes" -version = "32.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "durationpy" }, - { name = "google-auth" }, - { name = "oauthlib" }, - { name = "python-dateutil" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "requests-oauthlib" }, - { name = "six" }, - { name = "urllib3" }, - { name = "websocket-client" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/e8/0598f0e8b4af37cd9b10d8b87386cf3173cb8045d834ab5f6ec347a758b3/kubernetes-32.0.1.tar.gz", hash = "sha256:42f43d49abd437ada79a79a16bd48a604d3471a117a8347e87db693f2ba0ba28", size = 946691, upload-time = "2025-02-18T21:06:34.148Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/10/9f8af3e6f569685ce3af7faab51c8dd9d93b9c38eba339ca31c746119447/kubernetes-32.0.1-py2.py3-none-any.whl", hash = "sha256:35282ab8493b938b08ab5526c7ce66588232df00ef5e1dbe88a419107dc10998", size = 1988070, upload-time = "2025-02-18T21:06:31.391Z" }, -] - -[[package]] -name = "lark" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, -] - -[[package]] -name = "lazy-loader" -version = "0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/ac/21a1f8aa3777f5658576777ea76bfb124b702c520bbe90edf4ae9915eafa/lazy_loader-0.5.tar.gz", hash = "sha256:717f9179a0dbed357012ddad50a5ad3d5e4d9a0b8712680d4e687f5e6e6ed9b3", size = 15294, upload-time = "2026-03-06T15:45:09.054Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/a1/8d812e53a5da1687abb10445275d41a8b13adb781bbf7196ddbcf8d88505/lazy_loader-0.5-py3-none-any.whl", hash = "sha256:ab0ea149e9c554d4ffeeb21105ac60bed7f3b4fd69b1d2360a4add51b170b005", size = 8044, upload-time = "2026-03-06T15:45:07.668Z" }, -] - -[[package]] -name = "librosa" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "audioread" }, - { name = "decorator" }, - { name = "joblib" }, - { name = "lazy-loader" }, - { name = "msgpack" }, - { name = "numba" }, - { name = "numpy" }, - { name = "pooch" }, - { name = "scikit-learn" }, - { name = "scipy" }, - { name = "soundfile" }, - { name = "soxr" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/36/360b5aafa0238e29758729e9486c6ed92a6f37fa403b7875e06c115cdf4a/librosa-0.11.0.tar.gz", hash = "sha256:f5ed951ca189b375bbe2e33b2abd7e040ceeee302b9bbaeeffdfddb8d0ace908", size = 327001, upload-time = "2025-03-11T15:09:54.884Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/ba/c63c5786dfee4c3417094c4b00966e61e4a63efecee22cb7b4c0387dda83/librosa-0.11.0-py3-none-any.whl", hash = "sha256:0b6415c4fd68bff4c29288abe67c6d80b587e0e1e2cfb0aad23e4559504a7fa1", size = 260749, upload-time = "2025-03-11T15:09:52.982Z" }, -] - -[[package]] -name = "llguidance" -version = "1.7.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/91/6bc8bb503dc259e46d253b5424385a54fe06c38a4c7a12befe69a3c2455a/llguidance-1.7.6.tar.gz", hash = "sha256:db7febbe412ed2015501904646750071d7e00e6df7f85c4b956ad4f206fd2df7", size = 1156574, upload-time = "2026-06-03T20:13:25.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/1d/5a9a13421b1f3f1c1acf82beb63ed72fa4d302e65099b72f4a4fe5a098ab/llguidance-1.7.6-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:eabf4572c8731734c0444c353b9ea06bc5c156986d2ff0a4ec0499159271381f", size = 3227892, upload-time = "2026-06-03T20:13:09.533Z" }, - { url = "https://files.pythonhosted.org/packages/46/fe/bb185f11bad82f2637e3cd8cbf6b200cbb6ed56ac395de47ea05a60d4649/llguidance-1.7.6-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:9c54c899db8cb4b4fba128a7d844730066576c70d806c95ada92b2bd2d6ab498", size = 3138127, upload-time = "2026-06-03T20:13:11.649Z" }, - { url = "https://files.pythonhosted.org/packages/51/b9/dc76d7716e04dc7b3427cae52eaa32bd20771382d4d1dd9f4538a9dd2086/llguidance-1.7.6-cp39-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:e70fa25ed550c2b50c2fd70baa9e2808b4ecb859d01e453bd5459aff62ba38c3", size = 2899993, upload-time = "2026-06-03T20:13:13.563Z" }, - { url = "https://files.pythonhosted.org/packages/1a/64/d74336f22242ef94356a456057d4ff1be7c1bc9c7dbc867171c6982a5512/llguidance-1.7.6-cp39-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:ceec951d29a74309984e3be0fe7f5f56c1362434cd937abd517b259a60908b1e", size = 3074809, upload-time = "2026-06-03T20:13:15.498Z" }, - { url = "https://files.pythonhosted.org/packages/49/37/99d700f0e2c83acf25a8d8946b2bee9f5eac47bc530bfbd53ba3126c667f/llguidance-1.7.6-cp39-abi3-win_amd64.whl", hash = "sha256:ace7e81cd31950a87186356ab24bd7f75fbc10a05ca9d9f7f8748f931963f763", size = 2879207, upload-time = "2026-06-03T20:13:23.341Z" }, -] - -[[package]] -name = "llvmlite" -version = "0.47.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/88/a8952b6d5c21e74cbf158515b779666f692846502623e9e3c39d8e8ba25f/llvmlite-0.47.0.tar.gz", hash = "sha256:62031ce968ec74e95092184d4b0e857e444f8fdff0b8f9213707699570c33ccc", size = 193614, upload-time = "2026-03-31T18:29:53.497Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/48/4b7fe0e34c169fa2f12532916133e0b219d2823b540733651b34fdac509a/llvmlite-0.47.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:306a265f408c259067257a732c8e159284334018b4083a9e35f67d19792b164f", size = 37232769, upload-time = "2026-03-31T18:28:43.735Z" }, - { url = "https://files.pythonhosted.org/packages/e6/4b/e3f2cd17822cf772a4a51a0a8080b0032e6d37b2dbe8cfb724eac4e31c52/llvmlite-0.47.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5853bf26160857c0c2573415ff4efe01c4c651e59e2c55c2a088740acfee51cd", size = 56275178, upload-time = "2026-03-31T18:28:48.342Z" }, - { url = "https://files.pythonhosted.org/packages/b6/55/a3b4a543185305a9bdf3d9759d53646ed96e55e7dfd43f53e7a421b8fbae/llvmlite-0.47.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:003bcf7fa579e14db59c1a1e113f93ab8a06b56a4be31c7f08264d1d4072d077", size = 55128632, upload-time = "2026-03-31T18:28:52.901Z" }, - { url = "https://files.pythonhosted.org/packages/2f/f5/d281ae0f79378a5a91f308ea9fdb9f9cc068fddd09629edc0725a5a8fde1/llvmlite-0.47.0-cp312-cp312-win_amd64.whl", hash = "sha256:f3079f25bdc24cd9d27c4b2b5e68f5f60c4fdb7e8ad5ee2b9b006007558f9df7", size = 38138692, upload-time = "2026-03-31T18:28:57.147Z" }, -] - -[[package]] -name = "lm-format-enforcer" -version = "0.11.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "interegular" }, - { name = "packaging" }, - { name = "pydantic" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/84/d5/41cd417ba7dfdbbcfe46cebf81fb3dfd7c591b89897560ad05bb410a465d/lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da", size = 40258, upload-time = "2025-08-24T19:37:47.527Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/ef/11292bb0b85cf4c93447cab5a29f64576ed14d3ab4280e35ddd23486594a/lm_format_enforcer-0.11.3-py3-none-any.whl", hash = "sha256:cf586350875def1ae7a8fba84fcbbfc8371424b6c9d05c1fcba70aa233fbf06f", size = 45418, upload-time = "2025-08-24T19:37:46.325Z" }, -] - -[[package]] -name = "loguru" -version = "0.7.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "win32-setctime", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, -] - -[[package]] -name = "markupsafe" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, - { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, - { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, - { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, - { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, - { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, - { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, - { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, -] - -[[package]] -name = "matplotlib" -version = "3.11.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "contourpy" }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e7/c8/9aa712a0afb882649424dd8de8ad9aa6235e796e84c6052e8f6dc1598d0d/matplotlib-3.11.2.tar.gz", hash = "sha256:cec596316640f2b394b8f0daa0ea61a8eae82d017b620b9f202befb972a59ea4", size = 32660610, upload-time = "2026-09-11T19:05:31.214Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/ce/1bfcc4873b121597791ad74032943b123218c0613af0b97e8dd05e916fb1/matplotlib-3.11.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ef752769cd962f39ea0b6ffc82d1ea43a0012c5a6157c7a075212fa509cfcff2", size = 9476466, upload-time = "2026-09-11T19:03:19.45Z" }, - { url = "https://files.pythonhosted.org/packages/a6/c4/7f5f3601ee69baf072c0c7d3ce60c03e0618621c5a56c34a62a460e29d11/matplotlib-3.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ef31985c4dedb5f1424e1aec6849a47dd37689cb7fa3c20b1b82187f26806261", size = 9305583, upload-time = "2026-09-11T19:03:22.39Z" }, - { url = "https://files.pythonhosted.org/packages/b8/90/2b3fd67ee273163faeda6d514be70b5596eaa0fd77b60ffc294ad0b34f5f/matplotlib-3.11.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df4f7784aca81a94f254c0a2767d592ee25f407e488f5fa7203e51093fb6ca27", size = 9860353, upload-time = "2026-09-11T19:03:25.049Z" }, - { url = "https://files.pythonhosted.org/packages/f4/84/32549e7a462dc311aed2ab62e5d2538028840b5c53a8be0195c789937c3d/matplotlib-3.11.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b9a7ad579856284135e401ecc918c5f8a017ee30539298862a109f51b971710", size = 10670348, upload-time = "2026-09-11T19:03:28.108Z" }, - { url = "https://files.pythonhosted.org/packages/84/39/02e21b74f7439bd643d717ea846006d46e309e6d29b632b57751265311d6/matplotlib-3.11.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3aa4b8516fd26659e4363abbf317c703d9116496c5db2e9d0609a2866dd39dd2", size = 10810580, upload-time = "2026-09-11T19:03:31.402Z" }, - { url = "https://files.pythonhosted.org/packages/04/93/0d239bde12b308262265c2d98909b2f0ecad2b5deeae96241642e822e9c7/matplotlib-3.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:c5c1c68ee401fc98271263410f0e5ce88285abacf7627132914e8adf3d70ff43", size = 9349409, upload-time = "2026-09-11T19:03:34.195Z" }, - { url = "https://files.pythonhosted.org/packages/49/a8/06baf901c02246c8a222b655cc4540ef9c15e1549a04e07927f9cde716c3/matplotlib-3.11.2-cp312-cp312-win_arm64.whl", hash = "sha256:643ff850d8e0f5b8319337f87ed3cb59506afb3df3cc48de777d85871233be7b", size = 9028084, upload-time = "2026-09-11T19:03:37.032Z" }, -] - -[[package]] -name = "mcp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "httpx2" }, - { name = "jsonschema" }, - { name = "mcp-types" }, - { name = "opentelemetry-api" }, - { name = "pydantic" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "python-multipart" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, - { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/33/32d4dff2c95bb5d897c3ef4c83649a08996b17b58f0a326d2495d4c81179/mcp-2.0.0.tar.gz", hash = "sha256:0f440e735c13ece8bb19bc62cf0b86f4313448432fbb77d35e14034f4e050728", size = 1662284, upload-time = "2026-07-28T13:45:32.346Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/72/7d7897418912c1d12e87556630dfb7bf0eac71160e9bef8b447960804ee3/mcp-2.0.0-py3-none-any.whl", hash = "sha256:1cb4c75d2d2c7b8c1d756355e5d82a39f2822cc7f13e22a2051d7ca3592349d6", size = 349980, upload-time = "2026-07-28T13:45:28.853Z" }, -] - -[[package]] -name = "mcp-types" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bb/56/9b8e1c152f61f6c6b07c4b5896c88c7d0ae90bac6ee6306f852fcc5c1eb0/mcp_types-2.0.0.tar.gz", hash = "sha256:d7d939b9285c9961ae8866ba75ef85da34d12bafe276efbf4eb6a131786d8379", size = 66632, upload-time = "2026-07-28T13:45:33.804Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/4c/c78d78c3d52b0ac594ad7cc8ef5972adfe070e3597a8a4c6ce0cd39196ea/mcp_types-2.0.0-py3-none-any.whl", hash = "sha256:6b2de797ca2797f568b79529e1b25948e34de511bcc0bd82fef1039a6d1b8eb0", size = 69649, upload-time = "2026-07-28T13:45:30.713Z" }, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - -[[package]] -name = "mistral-common" -version = "1.11.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jsonschema" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "pydantic" }, - { name = "pydantic-extra-types", extra = ["pycountry"] }, - { name = "requests" }, - { name = "tiktoken" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/d0/61b2c24be62a8e2f0e46a1c16de23de386c8644408da249bc66768a6681b/mistral_common-1.11.7.tar.gz", hash = "sha256:d3b79583595cf6d96a2ab33e42cb8449768383147b8c56cac5a4f193be19d20d", size = 6387178, upload-time = "2026-07-23T09:21:17.206Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/a4/bc2850eb33cc2d633a21f51530756350dca325ee69b07c9202552f2bbadb/mistral_common-1.11.7-py3-none-any.whl", hash = "sha256:a9511b88eacacbe7dacddd9d3498c1739f56847b7fdddbd5a22e7844fd9def95", size = 6553583, upload-time = "2026-07-23T09:21:19.818Z" }, -] - -[package.optional-dependencies] -image = [ - { name = "opencv-python-headless" }, -] - -[[package]] -name = "ml-dtypes" -version = "0.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, - { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, - { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, - { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, - { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, -] - -[[package]] -name = "model-hosting-container-standards" -version = "0.1.16" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fastapi" }, - { name = "httpx" }, - { name = "jmespath" }, - { name = "pydantic" }, - { name = "setuptools" }, - { name = "starlette" }, - { name = "supervisor" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2d/5f/bc0d0fce1bd0a35378696aa13b21feffa18d9cda837f4e1be124e45ee090/model_hosting_container_standards-0.1.16.tar.gz", hash = "sha256:d34589633900e53c3ee5f7c78280a7cf7e4f6532c35e763341a262fc85cbe84a", size = 94130, upload-time = "2026-06-15T21:29:34.771Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/ef/6eabeb251d2a0598cb5f9a274159e05ae07a1e3fe6a1473bf6035793252a/model_hosting_container_standards-0.1.16-py3-none-any.whl", hash = "sha256:47f4f65713120bc3a69feb022981a38db9e557aedf88dbd72077f20588caa12b", size = 125666, upload-time = "2026-06-15T21:29:33.415Z" }, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, -] - -[[package]] -name = "msal" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9a/99/d840198ecf6e8057bbc937f129ae940404485d736cda73253bbff9537f01/msal-1.37.0.tar.gz", hash = "sha256:1b1672a33ee467c1d70b341bb16cafd51bb3c817147a95b93263794b03971bec", size = 182444, upload-time = "2026-05-29T19:49:05.561Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/b0/d807279f4b55d16d1f120d5ac4344c6e39b56732e2a224d40bded7fd67ad/msal-1.37.0-py3-none-any.whl", hash = "sha256:dd17e95a7c71bce75e8108113438ba7c4a086b3bcad4f57a8c09b7af3d753c2d", size = 123725, upload-time = "2026-05-29T19:49:04.335Z" }, -] - -[[package]] -name = "msal-extensions" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, -] - -[[package]] -name = "msgpack" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/f9/c0a1c127f9049db9155afc316952ea571720dd01833ff5e4d7e8e6352dbb/msgpack-1.2.1.tar.gz", hash = "sha256:04c721c2c7448767e9e3f2520a475663d8ee0f09c31890f6d2bd70fd636a9647", size = 183960, upload-time = "2026-06-18T16:13:52.594Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/dd/9e8cbd8f5582ca4b590336f2b91ee5662f6a6ca562b565abaf696a0f81ff/msgpack-1.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2ef59c659f289eddf8aa6623823f19fa2f40a4029266889eac7a2505dd210c35", size = 83531, upload-time = "2026-06-18T16:12:58.249Z" }, - { url = "https://files.pythonhosted.org/packages/50/2e/ebdb85a8da151397a2790363676b7ed7c125924fe618e4c6d8befb0cc62c/msgpack-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3567748a5107cb40cdf66a275430c2f87c07777698f4bfd25c35f44d533258c", size = 82657, upload-time = "2026-06-18T16:12:59.396Z" }, - { url = "https://files.pythonhosted.org/packages/26/aa/753ad8b007b464e1d8aa0c8e650b9c5f4f725e658fc5ac8a7635c55b7f6e/msgpack-1.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60926b75d00c8e816ef98f3034f484a8bc64242d66839cef4cf7e503142316a0", size = 410634, upload-time = "2026-06-18T16:13:00.383Z" }, - { url = "https://files.pythonhosted.org/packages/6a/fd/6adabd4f6d5e686f97dd02ce7fce3fe4cf672cbac36b8f67ff4040e8ad8b/msgpack-1.2.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:020e881a764b20d8d7ca1a54fc01b8175519d108e3c3f194fddc200bda95951a", size = 419989, upload-time = "2026-06-18T16:13:01.776Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cc/85039b7b0eb168aaad7383a23c97e291a11f08351cb45a606ce865e4e3f1/msgpack-1.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4202c74688ca06591f78cb18988228bd4cca2cc75d57b60008372892d2f1e6e6", size = 377544, upload-time = "2026-06-18T16:13:03.637Z" }, - { url = "https://files.pythonhosted.org/packages/ed/bf/35963899493b32030c85fc513b723ae66144ac70c11ebc52e889e16e3d99/msgpack-1.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8b267ce94efb76fbd1b3373511420074ee3187f0f7811bf394531de13294735a", size = 400842, upload-time = "2026-06-18T16:13:05.012Z" }, - { url = "https://files.pythonhosted.org/packages/a6/df/8e2ac970c8f99264cd9997d1c73df5466bc19da3301d7dc5500862a9b089/msgpack-1.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e4f1d0f8f98ade9634e01fb704a408f9336c0a8f1117b369f5db83dc7551d8b1", size = 374108, upload-time = "2026-06-18T16:13:06.232Z" }, - { url = "https://files.pythonhosted.org/packages/17/dd/fa8bd265110dfa51c20cb529f9e6d240a16fafe7e645004c6af2d01353ba/msgpack-1.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f02cf17a6ca1abe29b5f980644f7551f94d71f2011509b26d8625ce038f0df64", size = 414939, upload-time = "2026-06-18T16:13:07.478Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b9/8377a5ad8953fc0437c70cc98d9ae29f27fe5ac5109fbec0812085865735/msgpack-1.2.1-cp312-cp312-win32.whl", hash = "sha256:0c0d9802354507bcba62af19c17918e3eb437cc25e6f50657d511b5856a77aac", size = 64504, upload-time = "2026-06-18T16:13:08.822Z" }, - { url = "https://files.pythonhosted.org/packages/57/7f/ce1e377df7e62461fefd9eb23bfb93a4a523f40a517b377b8f844d836828/msgpack-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:5c24aa15d5963051e1a5c62b12c50cd705992502b5ec1f3bece6046f33c9fc24", size = 71421, upload-time = "2026-06-18T16:13:09.828Z" }, - { url = "https://files.pythonhosted.org/packages/8f/32/ebfe84c9929f08f188d56c7a2fd913406a9ddad76a634697c1c43b8112e6/msgpack-1.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:4227224aaec8f7fbcbfbd4272319347b2bb4030366502600f8c45588c5187b07", size = 64775, upload-time = "2026-06-18T16:13:11.056Z" }, -] - -[[package]] -name = "msgspec" -version = "0.21.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/60/f79b9b013a16fa3a58350c9295ddc6789f2e335f36ea61ed10a21b215364/msgspec-0.21.1.tar.gz", hash = "sha256:2313508e394b0d208f8f56892ca9b2799e2561329de9763b19619595a6c0f72c", size = 319193, upload-time = "2026-04-12T21:44:50.394Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/cf/317224852c00248c620a9bcf4b26e2e4ab8afd752f18d2a6ef73ebd423b6/msgspec-0.21.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4248cf0b6129b7d230eacd493c17cc2d4f3989f3bb7f633a928a85b7dcfa251", size = 196188, upload-time = "2026-04-12T21:44:07.181Z" }, - { url = "https://files.pythonhosted.org/packages/6d/81/074612945c0666078f7366f40000013de9f6ba687491d450df699bceebc9/msgspec-0.21.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5102c7e9b3acff82178449b85006d96310e690291bb1ea0142f1b24bcb8aabcb", size = 188473, upload-time = "2026-04-12T21:44:08.736Z" }, - { url = "https://files.pythonhosted.org/packages/8a/37/655101799590bcc5fddb2bd3fe0e6194e816c2d1da7c361725f5eb89a910/msgspec-0.21.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:846758412e9518252b2ac9bffd6f0e54d9ff614f5f9488df7749f81ff5c80920", size = 218871, upload-time = "2026-04-12T21:44:09.917Z" }, - { url = "https://files.pythonhosted.org/packages/b5/d1/d4cd9fe89c7d400d7a18f86ccc94daa3f0927f53558846fcb60791dce5d6/msgspec-0.21.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21995e74b5c598c2e004110ad66ec7f1b8c20bf2bcf3b2de8fd9a3094422d3ff", size = 225025, upload-time = "2026-04-12T21:44:11.191Z" }, - { url = "https://files.pythonhosted.org/packages/24/bf/e20549e602b9edccadeeff98760345a416f9cce846a657e8b18e3396b212/msgspec-0.21.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6129f0cca52992e898fd5344187f7c8127b63d810b2fd73e36fca73b4c6475ee", size = 222672, upload-time = "2026-04-12T21:44:12.481Z" }, - { url = "https://files.pythonhosted.org/packages/b4/68/04d7a8f0f786545cf9b8c280c57aa6befb5977af6e884b8b54191cbe44b3/msgspec-0.21.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ef3ec2296248d1f8b9231acb051b6d471dfde8f21819e86c9adaaa9f42918521", size = 227303, upload-time = "2026-04-12T21:44:13.709Z" }, - { url = "https://files.pythonhosted.org/packages/cc/4d/619866af2840875be408047bf9e70ceafbae6ab50660de7134ed1b25eb86/msgspec-0.21.1-cp312-cp312-win_amd64.whl", hash = "sha256:d4ab834a054c6f0cbeef6df9e7e1b33d5f1bc7b86dea1d2fd7cad003873e783d", size = 190017, upload-time = "2026-04-12T21:44:14.977Z" }, - { url = "https://files.pythonhosted.org/packages/5e/2e/a8f9eca8fd00e097d7a9e99ba8a4685db994494448e3d4f0b7f6e9a3c0f7/msgspec-0.21.1-cp312-cp312-win_arm64.whl", hash = "sha256:628aaa35c74950a8c59da330d7e98917e1c7188f983745782027748ee4ca573e", size = 175345, upload-time = "2026-04-12T21:44:16.431Z" }, -] - -[[package]] -name = "multidict" -version = "6.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, - { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, - { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, - { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, - { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, - { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, - { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, - { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, - { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, - { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, - { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, - { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, - { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, - { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, - { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, - { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, - { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, - { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, -] - -[[package]] -name = "munch" -version = "4.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/2b/45098135b5f9f13221820d90f9e0516e11a2a0f55012c13b081d202b782a/munch-4.0.0.tar.gz", hash = "sha256:542cb151461263216a4e37c3fd9afc425feeaf38aaa3025cd2a981fadb422235", size = 19089, upload-time = "2023-07-01T09:49:35.98Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/b3/7c69b37f03260a061883bec0e7b05be7117c1b1c85f5212c72c8c2bc3c8c/munch-4.0.0-py2.py3-none-any.whl", hash = "sha256:71033c45db9fb677a0b7eb517a4ce70ae09258490e419b0e7f00d1e386ecb1b4", size = 9950, upload-time = "2023-07-01T09:49:34.472Z" }, -] - -[[package]] -name = "narwhals" -version = "2.24.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/58946e5aab18393e793bd4add6985b95d0e01c3a2d832f38f54468b10dcd/narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d", size = 661143, upload-time = "2026-07-13T10:49:19.086Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, -] - -[[package]] -name = "nccl4py" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-core" }, - { name = "cuda-pathfinder" }, - { name = "numpy" }, - { name = "packaging" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/2d/86a96d31ad8092f86f15f45d051d599a3fa7a7044b0176f27a50ff16260c/nccl4py-0.6.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f5b6cac4a7edaeb1b626afabbf29b0d316b847738dace109fb9dd71f2c6b56", size = 3626706, upload-time = "2026-09-23T17:14:14.74Z" }, - { url = "https://files.pythonhosted.org/packages/bf/c5/bd7d2f1251215881233580f402606e5a480b60f469f7dbea01d30fb99066/nccl4py-0.6.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4006cf8c12c26ed29811b80858a58f8f31e4b42ea85b6c1d23401981449ab9d", size = 3695883, upload-time = "2026-09-23T17:14:37.411Z" }, -] - -[[package]] -name = "nemo-rl-dynamo-runtime" -version = "0.0.0" -source = { virtual = "." } -dependencies = [ - { name = "ai-dynamo", extra = ["vllm"] }, -] - -[package.metadata] -requires-dist = [{ name = "ai-dynamo", extras = ["vllm"], specifier = "==1.5.1" }] - -[[package]] -name = "nest-asyncio" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, -] - -[[package]] -name = "networkx" -version = "3.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, -] - -[[package]] -name = "ninja" -version = "1.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/73/79a0b22fc731989c708068427579e840a6cf4e937fe7ae5c5d0b7356ac22/ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978", size = 242558, upload-time = "2025-08-11T15:10:19.421Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/74/d02409ed2aa865e051b7edda22ad416a39d81a84980f544f8de717cab133/ninja-1.13.0-py3-none-macosx_10_9_universal2.whl", hash = "sha256:fa2a8bfc62e31b08f83127d1613d10821775a0eb334197154c4d6067b7068ff1", size = 310125, upload-time = "2025-08-11T15:09:50.971Z" }, - { url = "https://files.pythonhosted.org/packages/8e/de/6e1cd6b84b412ac1ef327b76f0641aeb5dcc01e9d3f9eee0286d0c34fd93/ninja-1.13.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630", size = 177467, upload-time = "2025-08-11T15:09:52.767Z" }, - { url = "https://files.pythonhosted.org/packages/c8/83/49320fb6e58ae3c079381e333575fdbcf1cca3506ee160a2dcce775046fa/ninja-1.13.0-py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:be7f478ff9f96a128b599a964fc60a6a87b9fa332ee1bd44fa243ac88d50291c", size = 187834, upload-time = "2025-08-11T15:09:54.115Z" }, - { url = "https://files.pythonhosted.org/packages/56/c7/ba22748fb59f7f896b609cd3e568d28a0a367a6d953c24c461fe04fc4433/ninja-1.13.0-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:60056592cf495e9a6a4bea3cd178903056ecb0943e4de45a2ea825edb6dc8d3e", size = 202736, upload-time = "2025-08-11T15:09:55.745Z" }, - { url = "https://files.pythonhosted.org/packages/79/22/d1de07632b78ac8e6b785f41fa9aad7a978ec8c0a1bf15772def36d77aac/ninja-1.13.0-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1c97223cdda0417f414bf864cfb73b72d8777e57ebb279c5f6de368de0062988", size = 179034, upload-time = "2025-08-11T15:09:57.394Z" }, - { url = "https://files.pythonhosted.org/packages/ed/de/0e6edf44d6a04dabd0318a519125ed0415ce437ad5a1ec9b9be03d9048cf/ninja-1.13.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa", size = 180716, upload-time = "2025-08-11T15:09:58.696Z" }, - { url = "https://files.pythonhosted.org/packages/54/28/938b562f9057aaa4d6bfbeaa05e81899a47aebb3ba6751e36c027a7f5ff7/ninja-1.13.0-py3-none-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4be9c1b082d244b1ad7ef41eb8ab088aae8c109a9f3f0b3e56a252d3e00f42c1", size = 146843, upload-time = "2025-08-11T15:10:00.046Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fb/d06a3838de4f8ab866e44ee52a797b5491df823901c54943b2adb0389fbb/ninja-1.13.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6739d3352073341ad284246f81339a384eec091d9851a886dfa5b00a6d48b3e2", size = 154402, upload-time = "2025-08-11T15:10:01.657Z" }, - { url = "https://files.pythonhosted.org/packages/31/bf/0d7808af695ceddc763cf251b84a9892cd7f51622dc8b4c89d5012779f06/ninja-1.13.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:11be2d22027bde06f14c343f01d31446747dbb51e72d00decca2eb99be911e2f", size = 552388, upload-time = "2025-08-11T15:10:03.349Z" }, - { url = "https://files.pythonhosted.org/packages/9d/70/c99d0c2c809f992752453cce312848abb3b1607e56d4cd1b6cded317351a/ninja-1.13.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:aa45b4037b313c2f698bc13306239b8b93b4680eb47e287773156ac9e9304714", size = 472501, upload-time = "2025-08-11T15:10:04.735Z" }, - { url = "https://files.pythonhosted.org/packages/9f/43/c217b1153f0e499652f5e0766da8523ce3480f0a951039c7af115e224d55/ninja-1.13.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f8e1e8a1a30835eeb51db05cf5a67151ad37542f5a4af2a438e9490915e5b72", size = 638280, upload-time = "2025-08-11T15:10:06.512Z" }, - { url = "https://files.pythonhosted.org/packages/8c/45/9151bba2c8d0ae2b6260f71696330590de5850e5574b7b5694dce6023e20/ninja-1.13.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:3d7d7779d12cb20c6d054c61b702139fd23a7a964ec8f2c823f1ab1b084150db", size = 642420, upload-time = "2025-08-11T15:10:08.35Z" }, - { url = "https://files.pythonhosted.org/packages/3c/fb/95752eb635bb8ad27d101d71bef15bc63049de23f299e312878fc21cb2da/ninja-1.13.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:d741a5e6754e0bda767e3274a0f0deeef4807f1fec6c0d7921a0244018926ae5", size = 585106, upload-time = "2025-08-11T15:10:09.818Z" }, - { url = "https://files.pythonhosted.org/packages/c1/31/aa56a1a286703800c0cbe39fb4e82811c277772dc8cd084f442dd8e2938a/ninja-1.13.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:e8bad11f8a00b64137e9b315b137d8bb6cbf3086fbdc43bf1f90fd33324d2e96", size = 707138, upload-time = "2025-08-11T15:10:11.366Z" }, - { url = "https://files.pythonhosted.org/packages/34/6f/5f5a54a1041af945130abdb2b8529cbef0cdcbbf9bcf3f4195378319d29a/ninja-1.13.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b4f2a072db3c0f944c32793e91532d8948d20d9ab83da9c0c7c15b5768072200", size = 581758, upload-time = "2025-08-11T15:10:13.295Z" }, - { url = "https://files.pythonhosted.org/packages/95/97/51359c77527d45943fe7a94d00a3843b81162e6c4244b3579fe8fc54cb9c/ninja-1.13.0-py3-none-win32.whl", hash = "sha256:8cfbb80b4a53456ae8a39f90ae3d7a2129f45ea164f43fadfa15dc38c4aef1c9", size = 267201, upload-time = "2025-08-11T15:10:15.158Z" }, - { url = "https://files.pythonhosted.org/packages/29/45/c0adfbfb0b5895aa18cec400c535b4f7ff3e52536e0403602fc1a23f7de9/ninja-1.13.0-py3-none-win_amd64.whl", hash = "sha256:fb8ee8719f8af47fed145cced4a85f0755dd55d45b2bddaf7431fa89803c5f3e", size = 309975, upload-time = "2025-08-11T15:10:16.697Z" }, - { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, -] - -[[package]] -name = "nixl" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nixl-cu12" }, - { name = "nixl-cu13" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/b3/c7f635c70c8ae7b0b920b1c82ecabf67617614bcb2abd8df0cd727ac8520/nixl-1.3.2-py3-none-any.whl", hash = "sha256:4324569b19aaf6e16d9d7b27ef1965a2e36a170fb66c6d1b6597a76bf4761cbe", size = 10303, upload-time = "2026-07-24T20:19:58.019Z" }, -] - -[package.optional-dependencies] -cu13 = [ - { name = "nixl-cu13" }, -] - -[[package]] -name = "nixl-cu12" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/28/8ba9eab9faa5f9455d8ebd322630398573f4e109b84ef8615d85d4bca3b4/nixl_cu12-1.3.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ea79ec205168cf819614127265ce1fff1e61d9d126d6d56cbd1ecaa29c980723", size = 80286069, upload-time = "2026-07-24T20:14:48.56Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d3/2964339654b3fe85e7aa62fdce4da3b97ee40337f3b72466aa79251f1196/nixl_cu12-1.3.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ef8ccdffcd54978e8a799de59287efcb3af0b7ba3bf02e04bc4df4c842f1f569", size = 82188539, upload-time = "2026-07-24T20:12:52.953Z" }, -] - -[[package]] -name = "nixl-cu13" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/2f/154a40352b7b6cb5afc41e1b794821c9631974712a6536ade78ef681a0b6/nixl_cu13-1.3.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5c3f5c1265fcec50f608a56fe828fad892322a1170755cdade9cc37c6545bb73", size = 64425110, upload-time = "2026-07-24T20:18:51.96Z" }, - { url = "https://files.pythonhosted.org/packages/99/d8/5768b907b85d8856c07674ddd0ffeb736ed987ff530a12e8f17a273cab3b/nixl_cu13-1.3.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:22fcd7183b2cd831b3da781c9e9991c5f6ef77a238ee6b7ac05d42558ea469a9", size = 66330361, upload-time = "2026-07-24T20:16:38.296Z" }, -] - -[[package]] -name = "numba" -version = "0.65.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llvmlite" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/61/7299643b9c18d669e04be7c5bcb64d985070d07553274817b45b049e7bfe/numba-0.65.0.tar.gz", hash = "sha256:edad0d9f6682e93624c00125a471ae4df186175d71fd604c983c377cdc03e68b", size = 2764131, upload-time = "2026-04-01T03:52:01.946Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/2f/8bd31a1ea43c01ac215283d83aa5f8d5acbe7a36c85b82f1757bfe9ccb31/numba-0.65.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:b27ee4847e1bfb17e9604d100417ee7c1d10f15a6711c6213404b3da13a0b2aa", size = 2680705, upload-time = "2026-04-01T03:51:32.597Z" }, - { url = "https://files.pythonhosted.org/packages/73/36/88406bd58600cc696417b8e5dd6a056478da808f3eaf48d18e2421e0c2d9/numba-0.65.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a52d92ffd297c10364bce60cd1fcb88f99284ab5df085f2c6bcd1cb33b529a6f", size = 3801411, upload-time = "2026-04-01T03:51:34.321Z" }, - { url = "https://files.pythonhosted.org/packages/0c/61/ce753a1d7646dd477e16d15e89473703faebb8995d2f71d7ad69a540b565/numba-0.65.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da8e371e328c06d0010c3d8b44b21858652831b85bcfba78cb22c042e22dbd8e", size = 3501622, upload-time = "2026-04-01T03:51:36.348Z" }, - { url = "https://files.pythonhosted.org/packages/7d/86/db87a5393f1b1fabef53ac3ba4e6b938bb27e40a04ad7cc512098fcae032/numba-0.65.0-cp312-cp312-win_amd64.whl", hash = "sha256:59bb9f2bb9f1238dfd8e927ba50645c18ae769fef4f3d58ea0ea22a2683b91f5", size = 2749979, upload-time = "2026-04-01T03:51:37.88Z" }, -] - -[[package]] -name = "numpy" -version = "2.3.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/65/21b3bc86aac7b8f2862db1e808f1ea22b028e30a225a34a5ede9bf8678f2/numpy-2.3.5.tar.gz", hash = "sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0", size = 20584950, upload-time = "2025-11-16T22:52:42.067Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/37/e669fe6cbb2b96c62f6bbedc6a81c0f3b7362f6a59230b23caa673a85721/numpy-2.3.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:74ae7b798248fe62021dbf3c914245ad45d1a6b0cb4a29ecb4b31d0bfbc4cc3e", size = 16733873, upload-time = "2025-11-16T22:49:49.84Z" }, - { url = "https://files.pythonhosted.org/packages/c5/65/df0db6c097892c9380851ab9e44b52d4f7ba576b833996e0080181c0c439/numpy-2.3.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee3888d9ff7c14604052b2ca5535a30216aa0a58e948cdd3eeb8d3415f638769", size = 12259838, upload-time = "2025-11-16T22:49:52.863Z" }, - { url = "https://files.pythonhosted.org/packages/5b/e1/1ee06e70eb2136797abe847d386e7c0e830b67ad1d43f364dd04fa50d338/numpy-2.3.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:612a95a17655e213502f60cfb9bf9408efdc9eb1d5f50535cc6eb365d11b42b5", size = 5088378, upload-time = "2025-11-16T22:49:55.055Z" }, - { url = "https://files.pythonhosted.org/packages/6d/9c/1ca85fb86708724275103b81ec4cf1ac1d08f465368acfc8da7ab545bdae/numpy-2.3.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3101e5177d114a593d79dd79658650fe28b5a0d8abeb8ce6f437c0e6df5be1a4", size = 6628559, upload-time = "2025-11-16T22:49:57.371Z" }, - { url = "https://files.pythonhosted.org/packages/74/78/fcd41e5a0ce4f3f7b003da85825acddae6d7ecb60cf25194741b036ca7d6/numpy-2.3.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b973c57ff8e184109db042c842423ff4f60446239bd585a5131cc47f06f789d", size = 14250702, upload-time = "2025-11-16T22:49:59.632Z" }, - { url = "https://files.pythonhosted.org/packages/b6/23/2a1b231b8ff672b4c450dac27164a8b2ca7d9b7144f9c02d2396518352eb/numpy-2.3.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28", size = 16606086, upload-time = "2025-11-16T22:50:02.127Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c5/5ad26fbfbe2012e190cc7d5003e4d874b88bb18861d0829edc140a713021/numpy-2.3.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:51c1e14eb1e154ebd80e860722f9e6ed6ec89714ad2db2d3aa33c31d7c12179b", size = 16025985, upload-time = "2025-11-16T22:50:04.536Z" }, - { url = "https://files.pythonhosted.org/packages/d2/fa/dd48e225c46c819288148d9d060b047fd2a6fb1eb37eae25112ee4cb4453/numpy-2.3.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b46b4ec24f7293f23adcd2d146960559aaf8020213de8ad1909dba6c013bf89c", size = 18542976, upload-time = "2025-11-16T22:50:07.557Z" }, - { url = "https://files.pythonhosted.org/packages/05/79/ccbd23a75862d95af03d28b5c6901a1b7da4803181513d52f3b86ed9446e/numpy-2.3.5-cp312-cp312-win32.whl", hash = "sha256:3997b5b3c9a771e157f9aae01dd579ee35ad7109be18db0e85dbdbe1de06e952", size = 6285274, upload-time = "2025-11-16T22:50:10.746Z" }, - { url = "https://files.pythonhosted.org/packages/2d/57/8aeaf160312f7f489dea47ab61e430b5cb051f59a98ae68b7133ce8fa06a/numpy-2.3.5-cp312-cp312-win_amd64.whl", hash = "sha256:86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa", size = 12782922, upload-time = "2025-11-16T22:50:12.811Z" }, - { url = "https://files.pythonhosted.org/packages/78/a6/aae5cc2ca78c45e64b9ef22f089141d661516856cf7c8a54ba434576900d/numpy-2.3.5-cp312-cp312-win_arm64.whl", hash = "sha256:f28620fe26bee16243be2b7b874da327312240a7cdc38b769a697578d2100013", size = 10194667, upload-time = "2025-11-16T22:50:16.16Z" }, -] - -[[package]] -name = "nvidia-cublas" -version = "13.1.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cuda-nvrtc", version = "13.0.88", source = { registry = "https://pypi.org/simple" } }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, -] - -[[package]] -name = "nvidia-cuda-cccl" -version = "13.3.3.4.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/bd/572971ffc14bd36676c821fc15d991b08fe6179cb09368250147475f954d/nvidia_cuda_cccl-13.3.3.4.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:067d19b4b3c9d0f2ebec9f29a311b2863db96bf98e058bbc331597d51ce818cf", size = 3454030, upload-time = "2026-06-29T16:41:49.092Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ab/049726d90147865a3ea53bae6cb7c35b98bf1fdf96cdb967101329625f83/nvidia_cuda_cccl-13.3.3.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc0adc188d570b09f4d606c7dc05a42aa3d8aa082e0d60f7bbfc5b6435f627c6", size = 3454034, upload-time = "2026-06-29T16:42:07.435Z" }, - { url = "https://files.pythonhosted.org/packages/24/d3/b1afcd9c40ceca72022579215fcaf5318cd747fd896cb928d4a1de924ff8/nvidia_cuda_cccl-13.3.3.4.1-py3-none-win_amd64.whl", hash = "sha256:d7c92cc03047031fa7af30866636d35ce4af409c28fc7dd8f69cb17053741399", size = 3454014, upload-time = "2026-06-29T17:09:09.012Z" }, -] - -[[package]] -name = "nvidia-cuda-crt" -version = "13.3.73" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/41/2089e411507d66458d67208bdd1bc562d492bb6458c3d2aea4603072a219/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:60aacc0b5e1e8b40c62abe4d1ab16440add91b99bd2f17f62dd091586b73d166", size = 157353, upload-time = "2026-06-29T16:42:38.163Z" }, - { url = "https://files.pythonhosted.org/packages/7e/ce/16d76f4b5b3f7460f5ebd17516685495c149c66651ffdd381f90e4d4e65c/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df14a17ae1c5c3171265411212246654d780f89344ea85344466c6b955247543", size = 157352, upload-time = "2026-06-29T16:43:09.209Z" }, - { url = "https://files.pythonhosted.org/packages/49/b3/6791ffba6f4b8e0d3ed875285aad8078ee407afa464ecd934ae298c205b1/nvidia_cuda_crt-13.3.73-py3-none-win_amd64.whl", hash = "sha256:af04e75148db1f0eea30958f33a9ec5a5a2dc2afa99ca4323f9a93b840602ca5", size = 158286, upload-time = "2026-06-29T17:09:28.621Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, -] - -[[package]] -name = "nvidia-cuda-nvcc" -version = "13.2.86" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "nvidia-cuda-crt" }, - { name = "nvidia-cuda-runtime", version = "13.0.96", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-nvvm", version = "13.2.86", source = { registry = "https://pypi.org/simple" } }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/a3/403638f80960e677ae8ecc78059d8c85dc2519b85ed8c0229840dc6f3b54/nvidia_cuda_nvcc-13.2.86-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:909140a1f942b943982b2eff120e618c94e29d75d9e33f5cd074f0e64eb411e8", size = 38716270, upload-time = "2026-07-16T09:37:12.754Z" }, - { url = "https://files.pythonhosted.org/packages/38/7b/ad9b5f84e8820af24afa8db66b5b2785b9bdc3b8724dffcd5e6ef2d40e53/nvidia_cuda_nvcc-13.2.86-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d3f56c8d705bad35bbba69eb0470d90856fd5d4dc48c6a6173aaf1e9f887cf5", size = 44042846, upload-time = "2026-07-16T09:37:47.98Z" }, - { url = "https://files.pythonhosted.org/packages/03/c1/01dad23300993e52933773751d0dd8e795175daa830b798ce324db377e3a/nvidia_cuda_nvcc-13.2.86-py3-none-win_amd64.whl", hash = "sha256:4171face482ef8ca35b5b2d59cfd25d2d0e2a2e2534fe34d2ce20695437868d9", size = 31816369, upload-time = "2026-07-16T10:01:34.327Z" }, -] - -[[package]] -name = "nvidia-cuda-nvcc" -version = "13.3.73" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'darwin'", -] -dependencies = [ - { name = "nvidia-cuda-crt" }, - { name = "nvidia-cuda-runtime", version = "13.3.29", source = { registry = "https://pypi.org/simple" } }, - { name = "nvidia-nvvm", version = "13.3.73", source = { registry = "https://pypi.org/simple" } }, -] - -[[package]] -name = "nvidia-cuda-nvdisasm" -version = "13.4.92" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/88/2382bc8b0200cd5a39e4ccc4d3930310c4699ac24bf91761ba780c00bb85/nvidia_cuda_nvdisasm-13.4.92-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b007c08c8248b29281ed9a174b478d557b58ea8a921ab78369a2843f0718f77d", size = 5163390, upload-time = "2026-09-16T20:38:33.02Z" }, - { url = "https://files.pythonhosted.org/packages/59/ea/b83b98288fd3d37b16c0da8846987d9c8719e9ff49ab974c1988d9e63f5a/nvidia_cuda_nvdisasm-13.4.92-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f35a5b6ddb64b6758c22c0b078c5b6ff6d54f7c240cd74bf131ab5bf13b6c3fe", size = 5203296, upload-time = "2026-09-16T20:38:39.558Z" }, - { url = "https://files.pythonhosted.org/packages/bf/73/acabcf7b9ac28e9bcd8326ae72f958f5ec05e32fb8683263868e1990bedf/nvidia_cuda_nvdisasm-13.4.92-py3-none-win_amd64.whl", hash = "sha256:600c43bde6b323be4d950e33a248aa97ff9cdea28a4b7dae083680f092bdd230", size = 5470680, upload-time = "2026-09-16T20:53:29.634Z" }, - { url = "https://files.pythonhosted.org/packages/25/26/2edceaba240f549c7a4d5acc3ccde21085fd76913abc854aa68825c98e52/nvidia_cuda_nvdisasm-13.4.92-py3-none-win_arm64.whl", hash = "sha256:624a3406572a983079edce35284cc5b6b1a3f23b02a62057b49a79be33d76252", size = 5422156, upload-time = "2026-09-16T20:57:06.408Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, - { url = "https://files.pythonhosted.org/packages/4a/af/345fedb9f4c76c84ab4fa445b36bd4048a4d9db60e6bc76b4f913ff4b852/nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872", size = 76807835, upload-time = "2025-09-04T08:39:15.274Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.3.33" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'darwin'", -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, - { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, - { url = "https://files.pythonhosted.org/packages/b7/94/6b867483bec07da24ffa32736c79fabb94ef3a7af4d787a9d4a974868576/nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492", size = 2927037, upload-time = "2025-10-09T09:04:23.782Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.3.29" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'darwin'", -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.20.0.48" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, - { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, -] - -[[package]] -name = "nvidia-cudnn-frontend" -version = "1.26.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/c4/3f587b73ac2eb6e391aebffb7a7a9ac9ed70e1e0e6a1d90ec0fdcb1a516f/nvidia_cudnn_frontend-1.26.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee50df3468f672aa31402fde4c911ad545d08e1d5e133e15bc55c3679d9fd9ca", size = 3471242, upload-time = "2026-07-07T20:55:30.929Z" }, - { url = "https://files.pythonhosted.org/packages/9b/31/64818fbaa117456349241b42ddfaef3ae6b050f5e99bb0e9d78eb831279b/nvidia_cudnn_frontend-1.26.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a1223c4e2e8bbe6f620148d6848f4eb773dd94bef534d5b748e91232b5be618", size = 3627033, upload-time = "2026-07-07T20:55:47.089Z" }, - { url = "https://files.pythonhosted.org/packages/f9/99/04a37f34b271ed157c6108c3191bd1993ab3fad8d5d66516970bd1e7c12d/nvidia_cudnn_frontend-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f104a40cb6f25cf01b7d7e84cb99af7dedd32b1cb8264384c2f363b7a3f7fb6", size = 2998730, upload-time = "2026-07-07T20:56:09.848Z" }, -] - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, - { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, -] - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, - { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, -] - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, - { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, - { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, -] - -[[package]] -name = "nvidia-cutlass-dsl" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cutlass-dsl-libs-base" }, - { name = "nvidia-cutlass-dsl-libs-cu12" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/ec/14e6ecbfed31ec35bbd1bb6965ae61f879370906a8f9c2e851e292704ba4/nvidia_cutlass_dsl-4.6.2-py3-none-any.whl", hash = "sha256:06ac62deb182a852dfb053032cf945bf02b9aa1bf502a18b129d280d7babb26c", size = 10460, upload-time = "2026-08-05T14:39:41.657Z" }, -] - -[package.optional-dependencies] -cu13 = [ - { name = "nvidia-cutlass-dsl-libs-cu13" }, -] - -[[package]] -name = "nvidia-cutlass-dsl-libs-base" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-python" }, - { name = "numpy" }, - { name = "nvidia-cuda-nvdisasm" }, - { name = "nvidia-cutlass-dsl-libs-core" }, - { name = "protobuf" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/f3/4be98f2faa283471e355a42ecbc20956fb2c3b6dbc71861f483be277e99a/nvidia_cutlass_dsl_libs_base-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e129db7155d56ba4478be098c0f819e37690a6ce5d43a9a9a83bf1300d32c57f", size = 3321894, upload-time = "2026-08-05T14:12:30.139Z" }, - { url = "https://files.pythonhosted.org/packages/54/1b/cf4a0837054bb4b9dc36cdd86f9d2e7fead3629e3373aaa7f7ec1e5e1542/nvidia_cutlass_dsl_libs_base-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:24dfaddad6077fd0de14eecaf66a72762f2f5f30ae1872231f5bf8b243ac2eac", size = 2824987, upload-time = "2026-08-05T14:12:49.618Z" }, -] - -[[package]] -name = "nvidia-cutlass-dsl-libs-core" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-python" }, - { name = "numpy" }, - { name = "nvidia-cuda-nvdisasm" }, - { name = "protobuf" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/ef/10c0089234a32f1379e1f213f28f8c7521a9ff0154e3acc58f613d5ef3be/nvidia_cutlass_dsl_libs_core-4.6.2-py3-none-any.whl", hash = "sha256:571d4b46fca1bfbb123d0dc348eaa7fd80af0014ddffc07b849e8195b2364333", size = 772393, upload-time = "2026-08-05T14:09:50.76Z" }, -] - -[[package]] -name = "nvidia-cutlass-dsl-libs-cu12" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-python" }, - { name = "numpy" }, - { name = "nvidia-cuda-nvdisasm" }, - { name = "nvidia-cutlass-dsl-libs-base" }, - { name = "protobuf" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/b7/019a8f74ad30ad9b6190dd5963d8921fc2b03777316e47260eb822624a0a/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ea3b96dd6b627fe20637acfa7876ea1adfa8d81a2b106ffff0bb2d042c0c2731", size = 87013242, upload-time = "2026-08-05T14:18:57.703Z" }, - { url = "https://files.pythonhosted.org/packages/80/6e/70c545a600b52b3f0fbbe01608aefc4e675c924ba886485ad72541f5c88e/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7f510307369d522da8e7d666557b9b9e7df06b94748bd5dc0198d59dfd0d918a", size = 88454261, upload-time = "2026-08-05T14:21:14.733Z" }, -] - -[[package]] -name = "nvidia-cutlass-dsl-libs-cu13" -version = "4.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-python" }, - { name = "numpy" }, - { name = "nvidia-cuda-nvdisasm" }, - { name = "nvidia-cutlass-dsl-libs-base" }, - { name = "protobuf" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/38/bbe5b8f1ef1a15e7bd1e2ea86686c38f15eeee0a4ba440a5d00d8222cb8e/nvidia_cutlass_dsl_libs_cu13-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b9543257622ce70d696c6a2bda7e9617ef5b1cc9f29bfe66f4b38d1d1e29a8cb", size = 86712108, upload-time = "2026-08-05T14:27:35.488Z" }, - { url = "https://files.pythonhosted.org/packages/ca/05/7874c37d50e112177a64a96dd6c7eaeacb1661ebb864e0fe5cfd9f706767/nvidia_cutlass_dsl_libs_cu13-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a299a623fad75eb752c1a4e447b924e9049ed396b037509c41a48da82d1f340a", size = 88036030, upload-time = "2026-08-05T14:27:58.991Z" }, -] - -[[package]] -name = "nvidia-ml-py" -version = "13.610.43" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f0/b5/a8fbc356f768fa5c9cfd646668fd7d34bf55bdd1c6e20754642a64d930d4/nvidia_ml_py-13.610.43.tar.gz", hash = "sha256:65437eb73d68d0c62c931ca4d45038472faff03bd0b8729abba4b899f70d60f2", size = 52109, upload-time = "2026-06-01T18:54:08.829Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/45/caa600acfab94560807a20a64b5830d2cd3c3202b7f1328644d70b7d6bd8/nvidia_ml_py-13.610.43-py3-none-any.whl", hash = "sha256:f13c72698edef492f985cc225f14faafe68ae065a2e407f45bdf6f4b9b43fde8", size = 53163, upload-time = "2026-06-01T18:54:07.704Z" }, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.30.7" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/21/a73174c6157101bdf1ffc22b517f76ff0082613989dd9bc8f43e8034caac/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:ca786ffa5a647c75d4d1f5cc72a6c4f537947e2ba8823d7c8aaf768e7a7b9f77", size = 215983881, upload-time = "2026-06-09T03:23:15.633Z" }, - { url = "https://files.pythonhosted.org/packages/3f/34/c500f90c7ae641b8e0f98965b36b8a7ac79cc8b296e8d251fe3eb592ee54/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:cefa7fdb9710efd0f39c5f1be1d61ff6fc9a996c451265bd7fbdcf9455ed4b50", size = 215965170, upload-time = "2026-06-09T03:23:39.73Z" }, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, - { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, -] - -[[package]] -name = "nvidia-nvvm" -version = "13.2.86" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'win32'", - "sys_platform == 'emscripten'", - "sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/1d/b50325516e259d4de9fca78b069840d39b6a172c5ba012b88d9985e01fa3/nvidia_nvvm-13.2.86-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a9244f3209922d655612c11ebb4117ea1f80983cb2b215c9cccded127282cebc", size = 64280112, upload-time = "2026-07-16T09:58:27.625Z" }, - { url = "https://files.pythonhosted.org/packages/e6/64/039ad70d68355634581d3b66f99b7aa8f75ca6078e1a6a2c9223677bbee3/nvidia_nvvm-13.2.86-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a502dcc2859f17a925adba495c222d44a221b9eb10e7d111a7046dc2cc883b69", size = 61886756, upload-time = "2026-07-16T09:57:51.818Z" }, - { url = "https://files.pythonhosted.org/packages/c8/1c/a1e27ca53d767f5043eab9332564581e976953a437e1bc8c3642679a622f/nvidia_nvvm-13.2.86-py3-none-win_amd64.whl", hash = "sha256:7d140c2dd2d177af71240a31bbca20f40b4aee9a4e331156083e387855222b49", size = 56750837, upload-time = "2026-07-16T10:11:39.216Z" }, -] - -[[package]] -name = "nvidia-nvvm" -version = "13.3.73" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "sys_platform == 'darwin'", -] - -[[package]] -name = "nvtx" -version = "0.2.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/dd/692765e87de30bae1522cdffaa0f2b52949658a92a0fa6d96b1a01eae9d2/nvtx-0.2.15.tar.gz", hash = "sha256:2287d3be05b85661deb386f878d1f536c2e532774aa9ec7a50c434942ed81ae5", size = 121230, upload-time = "2026-03-18T10:01:25.547Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/07/698355285a03a366ef63ea9762fc1feef3f9f25483e1655408f72d827090/nvtx-0.2.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2cc530cd0f1a2c14a3a7e683833db509888ac5ed4ead94e5c9e2c7317c6937a7", size = 807159, upload-time = "2026-03-18T10:09:49.232Z" }, - { url = "https://files.pythonhosted.org/packages/c0/d1/08f22448d83481408d663065764ba583df091a7de629ed38fc97e522f1af/nvtx-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ca8030a6d197952318013dd1c12c22da1d4b9feb76ba72e0fcd449961183c2c", size = 806187, upload-time = "2026-03-18T10:13:32.972Z" }, - { url = "https://files.pythonhosted.org/packages/54/23/c97c39e3b7ba256aa343cb828ca0d1c8421f705ca84795658ecd14ca95ed/nvtx-0.2.15-cp312-cp312-win_amd64.whl", hash = "sha256:70a1e768964e0520b68ccabc4df391cc227537c45936a7eba6507bc65e617e00", size = 129178, upload-time = "2026-03-18T10:02:55.299Z" }, -] - -[[package]] -name = "oauthlib" -version = "3.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, -] - -[[package]] -name = "openai" -version = "2.49.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d3/9a/275bee8349b766906741223b98c668fcd4e17e1982a87597778743b4156f/openai-2.49.0.tar.gz", hash = "sha256:80f934333b5b83cef2fde9af7151dacaa72e150f43f92b7675f7647ca6157f48", size = 1080973, upload-time = "2026-07-27T22:51:40.014Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/ca/53357e460a1172e831ecbe43dd0c37342b7211a1eb09f4cf21a412adbbdf/openai-2.49.0-py3-none-any.whl", hash = "sha256:b694201eaa42a1ccf2aa125fe29458150108fb22df1abfb55d7188599da81d8c", size = 1648589, upload-time = "2026-07-27T22:51:38Z" }, -] - -[[package]] -name = "openai-harmony" -version = "0.0.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3e/92/2d038d096f29179c7c9571b431f9e739f87a487121901725e23fe338dd9d/openai_harmony-0.0.8.tar.gz", hash = "sha256:6e43f98e6c242fa2de6f8ea12eab24af63fa2ed3e89c06341fb9d92632c5cbdf", size = 284777, upload-time = "2025-11-05T19:07:06.727Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/c6/2502f416d46be3ec08bb66d696cccffb57781a499e3ff2e4d7c174af4e8f/openai_harmony-0.0.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:029ec25ca74abe48fdb58eb9fdd2a8c1618581fc33ce8e5653f8a1ffbfbd9326", size = 2627806, upload-time = "2025-11-05T19:06:57.063Z" }, - { url = "https://files.pythonhosted.org/packages/d3/d2/ce6953ca87db9cae3e775024184da7d1c5cb88cead19a2d75b42f00a959c/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4f709815924ec325b9a890e6ab2bbb0ceec8e319a4e257328eb752cf36b2efc", size = 2948463, upload-time = "2025-11-05T19:06:48.17Z" }, - { url = "https://files.pythonhosted.org/packages/fa/4c/b553c9651662d6ce102ca7f3629d268b23df1abe5841e24bed81e8a8e949/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5cfcfd963b50a41fc656c84d3440ca6eecdccd6c552158ce790b8f2e33dfb5a9", size = 2704083, upload-time = "2025-11-05T19:06:50.205Z" }, - { url = "https://files.pythonhosted.org/packages/9b/af/4eec8f9ab9c27bcdb444460c72cf43011d176fc44c79d6e113094ca1e152/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a3a16972aa1cee38ea958470cd04ac9a2d5ac38fdcf77ab686611246220c158", size = 2959765, upload-time = "2025-11-05T19:06:53.62Z" }, - { url = "https://files.pythonhosted.org/packages/11/3c/33f3374e4624e0e776f6b13b73c45a7ead7f9c4529f8369ed5bfcaa30cac/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4d5cfa168e74d08f8ba6d58a7e49bc7daef4d58951ec69b66b0d56f4927a68d", size = 3427031, upload-time = "2025-11-05T19:06:51.829Z" }, - { url = "https://files.pythonhosted.org/packages/25/3f/1a192b93bb47c6b44cd98ba8cc1d3d2a9308f1bb700c3017e6352da11bda/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c007d277218a50db8839e599ed78e0fffe5130f614c3f6d93ae257f282071a29", size = 2953260, upload-time = "2025-11-05T19:06:55.406Z" }, - { url = "https://files.pythonhosted.org/packages/5b/f8/93b582cad3531797c3db7c2db5400fd841538ccddfd9f5e3df61be99a630/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8565d4f5a0638da1bffde29832ed63c9e695c558611053add3b2dc0b56c92dbc", size = 3127044, upload-time = "2025-11-05T19:06:59.553Z" }, - { url = "https://files.pythonhosted.org/packages/1d/10/4327dbf87f75ae813405fd9a9b4a5cde63d506ffed0a096a440a4cabd89c/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:cbaa3bda75ef0d8836e1f8cc84af62f971b1d756d740efc95c38c3e04c0bfde2", size = 2932931, upload-time = "2025-11-05T19:07:01.437Z" }, - { url = "https://files.pythonhosted.org/packages/8a/c8/1774eec4f6f360ef57618fb8f52e3d3af245b2491bd0297513aa09eec04b/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:772922a9bd24e133950fad71eb1550836f415a88e8c77870e12d0c3bd688ddc2", size = 2996140, upload-time = "2025-11-05T19:07:03.438Z" }, - { url = "https://files.pythonhosted.org/packages/60/c3/3d1e01e2dba517a91760e4a03e4f20ffc75039a6fe584d0e6f9b5c78fd15/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:007b0476a1f331f8130783f901f1da6f5a7057af1a4891f1b6a31dec364189b5", size = 3205080, upload-time = "2025-11-05T19:07:05.078Z" }, - { url = "https://files.pythonhosted.org/packages/14/63/119de431572d7c70a7bf1037034a9be6ed0a7502a7498ba7302bca5b3242/openai_harmony-0.0.8-cp38-abi3-win32.whl", hash = "sha256:a9b5f893326b28d9e935ade14b4f655f5a840942473bc89b201c25f7a15af9cf", size = 2082457, upload-time = "2025-11-05T19:07:09.631Z" }, - { url = "https://files.pythonhosted.org/packages/40/1f/c83cf5a206c263ee70448a5ae4264682555f4d0b5bed0d2cc6ca1108103d/openai_harmony-0.0.8-cp38-abi3-win_amd64.whl", hash = "sha256:39d44f0d8f466bd56698e7ead708bead3141e27b9b87e3ab7d5a6d0e4a869ee5", size = 2438369, upload-time = "2025-11-05T19:07:08.1Z" }, -] - -[[package]] -name = "opencv-python-headless" -version = "5.0.0.93" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/99/76b7c80252aa83c1af16393454aafd125a0287101afe8deb0a6821af0e30/opencv_python_headless-5.0.0.93.tar.gz", hash = "sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c", size = 81817738, upload-time = "2026-07-02T07:01:06.039Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/7c/8c8097891c509d98cd128493835c95631c80be6a8f37ed9d25716c2e16f1/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f", size = 48322581, upload-time = "2026-07-02T05:50:34.207Z" }, - { url = "https://files.pythonhosted.org/packages/90/8c/eab2ad388c3cbab2a350c10c2ef19ce6bd099240afc31789032c996bab52/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:1e55af3abfb462eeeabe5c775f12bdb36216d8a93a3583d69e6bd6e1d6ba7d00", size = 34782894, upload-time = "2026-07-02T05:51:39.856Z" }, - { url = "https://files.pythonhosted.org/packages/ec/78/afca939f40ffe2b2380bfa86f812b2f7d4acc5a27b27dc41b49cad7ce7b4/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10818d91510e05c04568ae12b5cd120779c70c01bf897b001a6221fe430df80f", size = 36521085, upload-time = "2026-07-02T06:55:24.429Z" }, - { url = "https://files.pythonhosted.org/packages/2b/97/8170e9819764c47e436c130d3ff6cfb73b58f923eae9d3a03d8982b04aec/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09a872a157c1376ab922a69bbf22f9a95bcc7b658a9d8b436a60212b02b2eeb4", size = 56563598, upload-time = "2026-07-02T06:55:47.355Z" }, - { url = "https://files.pythonhosted.org/packages/3a/98/1a28a7101e31801042b3098871a74b76c61581d328ef40774ff4edb53a56/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:840bd717c21e5c11cadadc022a823315ea417f961213d06b4df010e019eb16f4", size = 39648433, upload-time = "2026-07-02T06:56:04.255Z" }, - { url = "https://files.pythonhosted.org/packages/9b/21/f6ef335f6e65724aa78b8d792b48d40a48c381715f1e62f5a5049e09d07e/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed709fdf9aa0bd1f2ed8549e71d19449b03a675bb581eb292285f6861953be37", size = 61204038, upload-time = "2026-07-02T06:56:41.823Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8f/b8756467ea991449a293797f6b3fa80fcfdd29598a0a60d1cd5715b96e61/opencv_python_headless-5.0.0.93-cp37-abi3-win32.whl", hash = "sha256:c6bcd96b185975ea240d22cfdb15a1f6d080cc95264cfbe2621f21bb144d89b9", size = 35411237, upload-time = "2026-07-02T05:50:12.901Z" }, - { url = "https://files.pythonhosted.org/packages/b8/88/763b967f7efd7226b82c9fae16d560cba049b1f0c036647e65c610fd636e/opencv_python_headless-5.0.0.93-cp37-abi3-win_amd64.whl", hash = "sha256:829717b6a95554f273e49e357cee3b3a2a26b6f4842fbc1bed2b45bdd8f87e0e", size = 43825962, upload-time = "2026-07-02T05:50:09.627Z" }, -] - -[[package]] -name = "opentelemetry-api" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f2/45/7af37fe54e5d3e66e7dcd7ba8b8aeee73f202bfac909cc94b8c4e428f9ac/opentelemetry_exporter_otlp-1.44.0.tar.gz", hash = "sha256:af1cde7c33ea8ed624bf04ac49a885730fe44c1f1ad698656e592c38f70ce106", size = 6090, upload-time = "2026-07-16T15:25:34.585Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/c3/7b466a9463944e70b37b744072a0c1b88a425dade3fff0631adec66c9bcc/opentelemetry_exporter_otlp-1.44.0-py3-none-any.whl", hash = "sha256:4a498fa8d8fd8be9e8e2d175fe5524a3fe581ccffadd8509db86526a5fb97051", size = 6727, upload-time = "2026-07-16T15:25:14.445Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-proto" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/47/80d9e9d468dc5de3af5096f5ccdb065fa4dd1470f74495cc53e59e397f47/opentelemetry_exporter_otlp_proto_grpc-1.44.0.tar.gz", hash = "sha256:40d1ae9e03fcc36de3cbac610cc99f35894938bff9cfd90fc4ec68bd85448463", size = 27225, upload-time = "2026-07-16T15:25:38.308Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/29/6ae42ba32b153ae0a44ae125f0caff2188bbe62d99c82d1768da30864e72/opentelemetry_exporter_otlp_proto_grpc-1.44.0-py3-none-any.whl", hash = "sha256:6a1a645ea182a2f59440c51fa8301d309f3324a8f9d65f8395584b064b67ee4e", size = 19624, upload-time = "2026-07-16T15:25:19.096Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-http" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, -] - -[[package]] -name = "opentelemetry-proto" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, -] - -[[package]] -name = "opentelemetry-sdk" -version = "1.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.65b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions-ai" -version = "0.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/24/02/10aeacc37a38a3a8fa16ff67bec1ae3bf882539f6f9efb0f70acf802ca2d/opentelemetry_semantic_conventions_ai-0.5.1.tar.gz", hash = "sha256:153906200d8c1d2f8e09bd78dbef526916023de85ac3dab35912bfafb69ff04c", size = 26533, upload-time = "2026-03-26T14:20:38.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/55/22/41fb05f1dc5fda2c468e05a41814c20859016c85117b66c8a257cae814f6/opentelemetry_semantic_conventions_ai-0.5.1-py3-none-any.whl", hash = "sha256:25aeb22bd261543b4898a73824026d96770e5351209c7d07a0b1314762b1f6e4", size = 11250, upload-time = "2026-03-26T14:20:37.108Z" }, -] - -[[package]] -name = "opt-einsum" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" }, -] - -[[package]] -name = "optax" -version = "0.2.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "chex" }, - { name = "etils", extra = ["epy"] }, - { name = "jax" }, - { name = "jaxlib" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/af/b5/f88a0d851547b2e6b2c7e7e6509ad66236b3e7019f1f095bb03dbaa61fa1/optax-0.2.4.tar.gz", hash = "sha256:4e05d3d5307e6dde4c319187ae36e6cd3a0c035d4ed25e9e992449a304f47336", size = 229717, upload-time = "2024-11-12T21:52:17.439Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/24/28d0bb21600a78e46754947333ec9a297044af884d360092eb8561575fe9/optax-0.2.4-py3-none-any.whl", hash = "sha256:db35c04e50b52596662efb002334de08c2a0a74971e4da33f467e84fac08886a", size = 319212, upload-time = "2024-11-12T21:52:15.446Z" }, -] - -[[package]] -name = "orbax-checkpoint" -version = "0.11.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "etils", extra = ["epath", "epy"] }, - { name = "humanize" }, - { name = "jax" }, - { name = "msgpack" }, - { name = "nest-asyncio" }, - { name = "numpy" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "simplejson" }, - { name = "tensorstore" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7e/b4/262439a3fe00064b53a5182c0149447304f5a2d5a328a92d64390c18d189/orbax_checkpoint-0.11.5.tar.gz", hash = "sha256:8331ff594980a241ba43eb59dd683e5b590b339cff32a7b72d78cb5a350030b4", size = 249258, upload-time = "2025-02-11T19:33:45.364Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/80/e659696b5b1c2ced427efedd2d9d29c1bc31d841ac8a031215aa38f6b2ae/orbax_checkpoint-0.11.5-py3-none-any.whl", hash = "sha256:b55a7a254ea0ab18237e8234a6ca8bf5522f589fcc2ac698cf6893d5e7ae3500", size = 342800, upload-time = "2025-02-11T19:33:43.507Z" }, -] - -[[package]] -name = "outlines-core" -version = "0.2.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/04/4a0812eb27c086cfd2e66e7ec9150f33e105912a9b7f8b335e3479f03a06/outlines_core-0.2.14.tar.gz", hash = "sha256:64808deed1591ca3029ff64346ceb974cd5d780c916ea82504951fe83523039e", size = 191539, upload-time = "2026-01-09T15:59:10.016Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/93/30b9188648a479b32be429a24166db47a7bfdb0f9a8aac4c6dcf569e0a52/outlines_core-0.2.14-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:95e6476d9702d2fcc4e85370dbbfb6933a46c816e9c90107f6ce36eb68b5d64a", size = 2049651, upload-time = "2026-01-09T15:58:28.549Z" }, - { url = "https://files.pythonhosted.org/packages/0d/06/f3557daa8e87d5b95f64de269a301d73ec3c2202ab897c3e1f1cb93eb1db/outlines_core-0.2.14-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:f04731a5e29a190e2cc9f692a1f3fb2414a645355ca7d01b83df43439c38bea8", size = 2201046, upload-time = "2026-01-09T15:58:29.958Z" }, - { url = "https://files.pythonhosted.org/packages/0c/67/d8acf778990964c951080d568284e858d466f27dfd6f2674781927faba1c/outlines_core-0.2.14-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:0e4c69f0a8565edb56464c4c9b6c291a10805f3a96dff84182980e90ae1a5e2f", size = 2049558, upload-time = "2026-01-09T15:58:31.003Z" }, - { url = "https://files.pythonhosted.org/packages/17/e1/0320b14b49b8379ced1ab195ecf5875dbd2267b90148847541f43bfde6c1/outlines_core-0.2.14-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:63f53cfd9614e754499ae86dd699f3abcecf42d6a4e58d80fd80347881d85960", size = 2197854, upload-time = "2026-01-09T15:58:32.39Z" }, - { url = "https://files.pythonhosted.org/packages/29/29/3a04944407207a5d214879ca5ca33c2bd3e65199a4e927051c1bdaaa4d50/outlines_core-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bb2060c240c4507f334965a8948dbeeb22007560d797f6debd92346c0b620cb", size = 2341426, upload-time = "2026-01-09T15:58:33.553Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a7/a77f746272504bac3f628047d56ea1731b61549a3e1d9bbfd226f2968246/outlines_core-0.2.14-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1de34681c7e0e7e1551fc9036e4fa3c57986336c905a10536591ceb6d869c258", size = 2236941, upload-time = "2026-01-09T15:58:35.118Z" }, - { url = "https://files.pythonhosted.org/packages/99/0d/9f599d938923ab8ceeff26fdf2f9ea53bea3c962085c4927a08338a32349/outlines_core-0.2.14-cp312-cp312-win32.whl", hash = "sha256:870e8e038853818cb202ccc8cde92251f300f96805bfcc3be1c883adda7b5297", size = 1842940, upload-time = "2026-01-09T15:58:36.544Z" }, - { url = "https://files.pythonhosted.org/packages/f8/df/0f145c52ebd156d80273e2f5278227ea57e0275b2aa863bed33f44f77923/outlines_core-0.2.14-cp312-cp312-win_amd64.whl", hash = "sha256:87b42440478764cce1353a87d8560ef82f3b39b9d753bfe93195ea3584f369e3", size = 2137266, upload-time = "2026-01-09T15:58:37.831Z" }, -] - -[[package]] -name = "packaging" -version = "26.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, -] - -[[package]] -name = "pandas" -version = "3.0.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e2/17/d7b106e05bfa642e8694451e7d3d759c6a241c5386a5d962e4f66c047e06/pandas-3.0.6.tar.gz", hash = "sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10", size = 4667686, upload-time = "2026-09-17T23:23:18.345Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/4c/597d588c055d4373cff19cbbc32d4dd046c7be8fadee957585b5ba9e5b24/pandas-3.0.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7dac2d65e9087e8e7b5a45fe15c4920911a221df061ab629943ce016489145c7", size = 10397411, upload-time = "2026-09-17T23:20:49.465Z" }, - { url = "https://files.pythonhosted.org/packages/18/8f/48907c7c707b61a8e5018c32e1a3f70623209bfb59020a2f6196159d3aa7/pandas-3.0.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9dab635a549e58a053c7b0fa054dc0bd7be22f0ed9a720f4a85d5fb993276172", size = 10051279, upload-time = "2026-09-17T23:20:52.409Z" }, - { url = "https://files.pythonhosted.org/packages/67/fa/613d867c3d9554a61bafdec6f79565c8a3e73235feb52cc4a72ad2e0fa6a/pandas-3.0.6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3dccb584123b399c07562ac4d62543e90ede49ddf8ce3c13ffc64cbe828c281", size = 10303429, upload-time = "2026-09-17T23:20:55.597Z" }, - { url = "https://files.pythonhosted.org/packages/cb/67/0c0f18e38d7f2d2af8c24b3315bc4046e73bbdd4a5540405506671ad0c0d/pandas-3.0.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0704044b676496b8350e023b09f174a26772456c974a2b11c36bebb558c9490d", size = 10788193, upload-time = "2026-09-17T23:20:58.617Z" }, - { url = "https://files.pythonhosted.org/packages/39/53/1b57f3162501fe36687e4870e1b918a6458ca7386b173af75663ace95857/pandas-3.0.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e7c1905ef02c3d6d43d9dbd5b6ccb4da4870a0b0c821bbc103fbdb6f3ad2707b", size = 11323501, upload-time = "2026-09-17T23:21:01.911Z" }, - { url = "https://files.pythonhosted.org/packages/f2/d2/b1182e8d39100369d7f13f4a125a3fb6b096fef112c46d0566c25781ff68/pandas-3.0.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:569e114072b24fc4970c12e2b4bab252671668a40b324318903380cab0254c0c", size = 11838173, upload-time = "2026-09-17T23:21:04.85Z" }, - { url = "https://files.pythonhosted.org/packages/c7/33/5b717af24d2f27995e51e0875a269dddd373045216e34cf62e3aa764eaa1/pandas-3.0.6-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:2a8fc94be2ee5f1d86f97aacd8cc566f81680b6498e76f3007421bb5d98151bf", size = 7118499, upload-time = "2026-09-17T23:21:07.661Z" }, - { url = "https://files.pythonhosted.org/packages/bd/2a/14b3b17cd75cef4a1ee1af4234eb41cc1afe4103b98c2d80e8916abfd42b/pandas-3.0.6-cp312-cp312-win_amd64.whl", hash = "sha256:3ef908d28590b3f42d7070e7ad8f9b34b442b260b7f3c1afb57e0040c58cdb1b", size = 9661439, upload-time = "2026-09-17T23:21:10.959Z" }, - { url = "https://files.pythonhosted.org/packages/3b/11/3d580a604a1e35d69f6676847bd12db7d14344bc677b717f4413e79c5d0d/pandas-3.0.6-cp312-cp312-win_arm64.whl", hash = "sha256:f4e7c52eb108d752e7592268108fd3e98efd76d83a3125cdd06c621c2e44359b", size = 8959806, upload-time = "2026-09-17T23:21:13.851Z" }, -] - -[[package]] -name = "partial-json-parser" -version = "0.2.1.1.post7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/6d/eed37d7ebc1e0bcd27b831c0cf1fe94881934316187c4b30d23f29ea0bd4/partial_json_parser-0.2.1.1.post7.tar.gz", hash = "sha256:86590e1ba6bcb6739a2dfc17d2323f028cb5884f4c6ce23db376999132c9a922", size = 10296, upload-time = "2025-11-17T07:27:41.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/32/658973117bf0fd82a24abbfb94fe73a5e86216e49342985e10acce54775a/partial_json_parser-0.2.1.1.post7-py3-none-any.whl", hash = "sha256:145119e5eabcf80cbb13844a6b50a85c68bf99d376f8ed771e2a3c3b03e653ae", size = 10877, upload-time = "2025-11-17T07:27:40.457Z" }, -] - -[[package]] -name = "pillow" -version = "12.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, - { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, - { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, - { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, - { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, - { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, - { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, - { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, -] - -[[package]] -name = "platformdirs" -version = "4.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/78/9b/560e4be8e26f6fd133a03630a8df0c663b9e8d61b4ade152b72005aec83b/platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0", size = 31953, upload-time = "2026-07-21T13:09:36.565Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/68/d8d58938dfb1370b266a1a729e6d77a985be23689a0496498ee17b2cbf90/platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74", size = 23247, upload-time = "2026-07-21T13:09:35.422Z" }, -] - -[[package]] -name = "plotext" -version = "5.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/d7/f75f397af966fe252d0d34ffd3cae765317fce2134f925f95e7d6725d1ce/plotext-5.3.2.tar.gz", hash = "sha256:52d1e932e67c177bf357a3f0fe6ce14d1a96f7f7d5679d7b455b929df517068e", size = 61967, upload-time = "2024-09-24T15:13:37.728Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/1e/12fe7c40cd2099a1f454518754ed229b01beaf3bbb343127f0cc13ce6c22/plotext-5.3.2-py3-none-any.whl", hash = "sha256:394362349c1ddbf319548cfac17ca65e6d5dfc03200c40dfdc0503b3e95a2283", size = 64047, upload-time = "2024-09-24T15:13:36.296Z" }, -] - -[[package]] -name = "plotly" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "narwhals" }, - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/c3/72b369f5ed7701b04ab0ea3dcf83e9bbce71c0b3bc6f07f87568550d09ea/plotly-7.1.0.tar.gz", hash = "sha256:f860166a4a3d78c69cb1f4a15f28a5c8283eade98a282a698f3bb853a449ace5", size = 6689315, upload-time = "2026-09-15T19:21:21.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/7d/905a3a3d51087515719058c94cfbda2ff0fc14417c20d557ae3e82d8b250/plotly-7.1.0-py3-none-any.whl", hash = "sha256:dbb7fa18afce40d0a8e80d1bf162eceb3faa0ce5a77fe741ad09a74cf78f53f3", size = 9692368, upload-time = "2026-09-15T19:21:18.331Z" }, -] - -[[package]] -name = "pooch" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "platformdirs" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/43/85ef45e8b36c6a48546af7b266592dc32d7f67837a6514d111bced6d7d75/pooch-1.9.0.tar.gz", hash = "sha256:de46729579b9857ffd3e741987a2f6d5e0e03219892c167c6578c0091fb511ed", size = 61788, upload-time = "2026-01-30T19:15:09.649Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl", hash = "sha256:f265597baa9f760d25ceb29d0beb8186c243d6607b0f60b83ecf14078dbc703b", size = 67175, upload-time = "2026-01-30T19:15:08.36Z" }, -] - -[[package]] -name = "portpicker" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "psutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4d/d0/cda2fc582f09510c84cd6b7d7b9e22a02d4e45dbad2b2ef1c6edd7847e00/portpicker-1.6.0.tar.gz", hash = "sha256:bd507fd6f96f65ee02781f2e674e9dc6c99bbfa6e3c39992e3916204c9d431fa", size = 25676, upload-time = "2023-08-15T04:37:08.865Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/2d/440e4d7041fff89f28f483733eb617127aa866135c2dc719e05893f089e1/portpicker-1.6.0-py3-none-any.whl", hash = "sha256:b2787a41404cf7edbe29b07b9e0ed863b09f2665dcc01c1eb0c2261c1e7d0755", size = 16613, upload-time = "2023-08-15T04:37:07.327Z" }, -] - -[[package]] -name = "prettytable" -version = "3.18.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/74/ba08d81e668ccfe8658d7520a307e63c19862c08eb4ccb26f356c5239a7a/prettytable-3.18.0.tar.gz", hash = "sha256:439217116152244369caf3d9f1caf2f9fe29b03bd79e88d2928c8e718c95d680", size = 76373, upload-time = "2026-06-22T16:07:50.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/be/2e6798ace5cc036f5d05d36b7b2fd85346f1a708c87060890b070d0ec607/prettytable-3.18.0-py3-none-any.whl", hash = "sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a", size = 37357, upload-time = "2026-06-22T16:07:48.595Z" }, -] - -[[package]] -name = "prometheus-client" -version = "0.26.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/73/f1334c29c2af4cd9dba6c7817e61b611bd0215e2eb5565c6064a4de18802/prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b", size = 92910, upload-time = "2026-07-24T19:36:41.893Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" }, -] - -[[package]] -name = "prometheus-fastapi-instrumentator" -version = "8.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "prometheus-client" }, - { name = "starlette" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/f4/cdcebf7094b03b99fba71ac8f56bd6f227973642662f49d272332d8419b3/prometheus_fastapi_instrumentator-8.1.0.tar.gz", hash = "sha256:b77f3043665e8d28e2bbd21017506195a43d9adf1d402d01bf95b494b7e560e1", size = 20492, upload-time = "2026-07-26T11:12:44.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/b9/91a2246e6cf01b7ccb14479803c8a50f9c258ae5c6a0f16ba3294820632b/prometheus_fastapi_instrumentator-8.1.0-py3-none-any.whl", hash = "sha256:b9f40b2cff3f7891ca0610b3ae4fc6ec723fd326b04bb659819aaeb821a0fc7d", size = 19649, upload-time = "2026-07-26T11:12:45.168Z" }, -] - -[[package]] -name = "propcache" -version = "0.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, - { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, - { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, - { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, - { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, - { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, - { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, - { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, - { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, - { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, - { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, - { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, - { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, - { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, - { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, - { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, - { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, -] - -[[package]] -name = "proto-plus" -version = "1.28.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/73/3e/29e0d6a2c5adde6ab5772253fd16ab346324026b89a66e354689c86d0584/proto_plus-1.28.2.tar.gz", hash = "sha256:26d843eb99c1e32fdf1d20ff0faae56607f7748fe774acf9ecd5cfe6c6472501", size = 58063, upload-time = "2026-07-22T16:28:29.119Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/84/4e9a53a062d4073c74897a6bd20fff74d55307341b3e85c081002462b3ef/proto_plus-1.28.2-py3-none-any.whl", hash = "sha256:b874236fcac2358f601e4330bcb76cb8b89c851303ccf4078408b3d4774d1c52", size = 50693, upload-time = "2026-07-22T16:28:24.059Z" }, -] - -[[package]] -name = "protobuf" -version = "6.33.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, -] - -[[package]] -name = "psutil" -version = "7.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, - { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, - { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, - { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, - { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, - { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, - { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, - { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, -] - -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - -[[package]] -name = "pyarrow" -version = "26.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/34/17c34cb38e5d940e38f0f0d9fdfa0e8a506676409ea9b85aff7e3079f831/pyarrow-26.0.0.tar.gz", hash = "sha256:0cccd36e00ea3afeb52ded61f2721ce71f604853d70c45365c58324eb773d6ae", size = 1239433, upload-time = "2026-10-09T08:26:25.315Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/60/6793778f2617cce469383dac0ba08c4f2401cf342df0c7b9ca53939d9b46/pyarrow-26.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:90ddaf7c625307ad52f31a9b25c34fe5e4897c7529ee3481135822b2b6842ff1", size = 36333953, upload-time = "2026-10-09T08:14:00.387Z" }, - { url = "https://files.pythonhosted.org/packages/db/81/f944cc63ce8a753e5fbff25de6d1d475ebd7fffdf9cf98c65130294fc896/pyarrow-26.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee341973f78a0b46e073d065e88e75026a9c584051e97f98a0d05d96c6bac7dd", size = 38688456, upload-time = "2026-10-09T08:14:04.344Z" }, - { url = "https://files.pythonhosted.org/packages/f5/2d/7e5c722fa5d5d9f3b75e62fe11694b34217664d4f05ac88031197166b277/pyarrow-26.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:01c863a18bd9c8412453dd0d92de6d0ee7b2b3d6fb079d9734a4b2a3c8bd4453", size = 50867603, upload-time = "2026-10-09T08:14:09.115Z" }, - { url = "https://files.pythonhosted.org/packages/88/e4/9cd356d906e71bd79b0c3fc5c9a54e01a0020dcf14c152ccfbcb503c7298/pyarrow-26.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6a628922ba20705fa964ca73e4ef959c2fb2f14b9bbec5589a6a1e68e6257c85", size = 53931932, upload-time = "2026-10-09T08:14:24.051Z" }, - { url = "https://files.pythonhosted.org/packages/bb/e4/5bae3133b7fe04c24907a20f3bc1fba388cbbde659199e7b76445982047a/pyarrow-26.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:954d971b363b16ee41f89389a4053315dc71265f2ce5c2468eb0a910b1166268", size = 54444720, upload-time = "2026-10-09T08:14:31.214Z" }, - { url = "https://files.pythonhosted.org/packages/ba/b4/ee422493bb6dafdbef776cfe2c2a73106a1063a79bf4e78d1e5f51176885/pyarrow-26.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5d5768d03426abe6526d5274adefa00abf00a7f81118c46e98b5a46390f5549e", size = 57388949, upload-time = "2026-10-09T08:14:38.964Z" }, - { url = "https://files.pythonhosted.org/packages/54/3c/1783aab1dac28e175dcf26dfc7123725efc474caecaed91e8a34cb89cad0/pyarrow-26.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cc903e1069e9dd5e9dcf780324c0112e27e051e422ecfaff574fb33ed65d9160", size = 28567581, upload-time = "2026-10-09T08:14:44.279Z" }, -] - -[[package]] -name = "pyasn1" -version = "0.6.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - -[[package]] -name = "pybase64" -version = "1.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/b8/4ed5c7ad5ec15b08d35cc79ace6145d5c1ae426e46435f4987379439dfea/pybase64-1.4.3.tar.gz", hash = "sha256:c2ed274c9e0ba9c8f9c4083cfe265e66dd679126cd9c2027965d807352f3f053", size = 137272, upload-time = "2025-12-06T13:27:04.013Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/a7/efcaa564f091a2af7f18a83c1c4875b1437db56ba39540451dc85d56f653/pybase64-1.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:18d85e5ab8b986bb32d8446aca6258ed80d1bafe3603c437690b352c648f5967", size = 38167, upload-time = "2025-12-06T13:23:16.821Z" }, - { url = "https://files.pythonhosted.org/packages/db/c7/c7ad35adff2d272bf2930132db2b3eea8c44bb1b1f64eb9b2b8e57cde7b4/pybase64-1.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3f5791a3491d116d0deaf4d83268f48792998519698f8751efb191eac84320e9", size = 31673, upload-time = "2025-12-06T13:23:17.835Z" }, - { url = "https://files.pythonhosted.org/packages/43/1b/9a8cab0042b464e9a876d5c65fe5127445a2436da36fda64899b119b1a1b/pybase64-1.4.3-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f0b3f200c3e06316f6bebabd458b4e4bcd4c2ca26af7c0c766614d91968dee27", size = 68210, upload-time = "2025-12-06T13:23:18.813Z" }, - { url = "https://files.pythonhosted.org/packages/62/f7/965b79ff391ad208b50e412b5d3205ccce372a2d27b7218ae86d5295b105/pybase64-1.4.3-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb632edfd132b3eaf90c39c89aa314beec4e946e210099b57d40311f704e11d4", size = 71599, upload-time = "2025-12-06T13:23:20.195Z" }, - { url = "https://files.pythonhosted.org/packages/03/4b/a3b5175130b3810bbb8ccfa1edaadbd3afddb9992d877c8a1e2f274b476e/pybase64-1.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:356ef1d74648ce997f5a777cf8f1aefecc1c0b4fe6201e0ef3ec8a08170e1b54", size = 59922, upload-time = "2025-12-06T13:23:21.487Z" }, - { url = "https://files.pythonhosted.org/packages/da/5d/c38d1572027fc601b62d7a407721688b04b4d065d60ca489912d6893e6cf/pybase64-1.4.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:c48361f90db32bacaa5518419d4eb9066ba558013aaf0c7781620279ecddaeb9", size = 56712, upload-time = "2025-12-06T13:23:22.77Z" }, - { url = "https://files.pythonhosted.org/packages/e7/d4/4e04472fef485caa8f561d904d4d69210a8f8fc1608ea15ebd9012b92655/pybase64-1.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:702bcaa16ae02139d881aeaef5b1c8ffb4a3fae062fe601d1e3835e10310a517", size = 59300, upload-time = "2025-12-06T13:23:24.543Z" }, - { url = "https://files.pythonhosted.org/packages/86/e7/16e29721b86734b881d09b7e23dfd7c8408ad01a4f4c7525f3b1088e25ec/pybase64-1.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:53d0ffe1847b16b647c6413d34d1de08942b7724273dd57e67dcbdb10c574045", size = 60278, upload-time = "2025-12-06T13:23:25.608Z" }, - { url = "https://files.pythonhosted.org/packages/b1/02/18515f211d7c046be32070709a8efeeef8a0203de4fd7521e6b56404731b/pybase64-1.4.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9a1792e8b830a92736dae58f0c386062eb038dfe8004fb03ba33b6083d89cd43", size = 54817, upload-time = "2025-12-06T13:23:26.633Z" }, - { url = "https://files.pythonhosted.org/packages/e7/be/14e29d8e1a481dbff151324c96dd7b5d2688194bb65dc8a00ca0e1ad1e86/pybase64-1.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d468b1b1ac5ad84875a46eaa458663c3721e8be5f155ade356406848d3701f6", size = 58611, upload-time = "2025-12-06T13:23:27.684Z" }, - { url = "https://files.pythonhosted.org/packages/b4/8a/a2588dfe24e1bbd742a554553778ab0d65fdf3d1c9a06d10b77047d142aa/pybase64-1.4.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e97b7bdbd62e71898cd542a6a9e320d9da754ff3ebd02cb802d69087ee94d468", size = 52404, upload-time = "2025-12-06T13:23:28.714Z" }, - { url = "https://files.pythonhosted.org/packages/27/fc/afcda7445bebe0cbc38cafdd7813234cdd4fc5573ff067f1abf317bb0cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b33aeaa780caaa08ffda87fc584d5eab61e3d3bbb5d86ead02161dc0c20d04bc", size = 68817, upload-time = "2025-12-06T13:23:30.079Z" }, - { url = "https://files.pythonhosted.org/packages/d3/3a/87c3201e555ed71f73e961a787241a2438c2bbb2ca8809c29ddf938a3157/pybase64-1.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c0efcf78f11cf866bed49caa7b97552bc4855a892f9cc2372abcd3ed0056f0d", size = 57854, upload-time = "2025-12-06T13:23:31.17Z" }, - { url = "https://files.pythonhosted.org/packages/fd/7d/931c2539b31a7b375e7d595b88401eeb5bd6c5ce1059c9123f9b608aaa14/pybase64-1.4.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66e3791f2ed725a46593f8bd2761ff37d01e2cdad065b1dceb89066f476e50c6", size = 54333, upload-time = "2025-12-06T13:23:32.422Z" }, - { url = "https://files.pythonhosted.org/packages/de/5e/537601e02cc01f27e9d75f440f1a6095b8df44fc28b1eef2cd739aea8cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:72bb0b6bddadab26e1b069bb78e83092711a111a80a0d6b9edcb08199ad7299b", size = 56492, upload-time = "2025-12-06T13:23:33.515Z" }, - { url = "https://files.pythonhosted.org/packages/96/97/2a2e57acf8f5c9258d22aba52e71f8050e167b29ed2ee1113677c1b600c1/pybase64-1.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5b3365dbcbcdb0a294f0f50af0c0a16b27a232eddeeb0bceeefd844ef30d2a23", size = 70974, upload-time = "2025-12-06T13:23:36.27Z" }, - { url = "https://files.pythonhosted.org/packages/75/2e/a9e28941c6dab6f06e6d3f6783d3373044be9b0f9a9d3492c3d8d2260ac0/pybase64-1.4.3-cp312-cp312-win32.whl", hash = "sha256:7bca1ed3a5df53305c629ca94276966272eda33c0d71f862d2d3d043f1e1b91a", size = 33686, upload-time = "2025-12-06T13:23:37.848Z" }, - { url = "https://files.pythonhosted.org/packages/83/e3/507ab649d8c3512c258819c51d25c45d6e29d9ca33992593059e7b646a33/pybase64-1.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:9f2da8f56d9b891b18b4daf463a0640eae45a80af548ce435be86aa6eff3603b", size = 35833, upload-time = "2025-12-06T13:23:38.877Z" }, - { url = "https://files.pythonhosted.org/packages/bc/8a/6eba66cd549a2fc74bb4425fd61b839ba0ab3022d3c401b8a8dc2cc00c7a/pybase64-1.4.3-cp312-cp312-win_arm64.whl", hash = "sha256:0631d8a2d035de03aa9bded029b9513e1fee8ed80b7ddef6b8e9389ffc445da0", size = 31185, upload-time = "2025-12-06T13:23:39.908Z" }, - { url = "https://files.pythonhosted.org/packages/17/45/92322aec1b6979e789b5710f73c59f2172bc37c8ce835305434796824b7b/pybase64-1.4.3-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:2baaa092f3475f3a9c87ac5198023918ea8b6c125f4c930752ab2cbe3cd1d520", size = 38746, upload-time = "2025-12-06T13:26:25.869Z" }, - { url = "https://files.pythonhosted.org/packages/11/94/f1a07402870388fdfc2ecec0c718111189732f7d0f2d7fe1386e19e8fad0/pybase64-1.4.3-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:cde13c0764b1af07a631729f26df019070dad759981d6975527b7e8ecb465b6c", size = 32573, upload-time = "2025-12-06T13:26:27.792Z" }, - { url = "https://files.pythonhosted.org/packages/fa/8f/43c3bb11ca9bacf81cb0b7a71500bb65b2eda6d5fe07433c09b543de97f3/pybase64-1.4.3-graalpy312-graalpy250_312_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5c29a582b0ea3936d02bd6fe9bf674ab6059e6e45ab71c78404ab2c913224414", size = 43461, upload-time = "2025-12-06T13:26:28.906Z" }, - { url = "https://files.pythonhosted.org/packages/2d/4c/2a5258329200be57497d3972b5308558c6de42e3749c6cc2aa1cbe34b25a/pybase64-1.4.3-graalpy312-graalpy250_312_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6b664758c804fa919b4f1257aa8cf68e95db76fc331de5f70bfc3a34655afe1", size = 36058, upload-time = "2025-12-06T13:26:30.092Z" }, - { url = "https://files.pythonhosted.org/packages/ea/6d/41faa414cde66ec023b0ca8402a8f11cb61731c3dc27c082909cbbd1f929/pybase64-1.4.3-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:f7537fa22ae56a0bf51e4b0ffc075926ad91c618e1416330939f7ef366b58e3b", size = 36231, upload-time = "2025-12-06T13:26:31.656Z" }, -] - -[[package]] -name = "pycountry" -version = "26.2.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/1d/061b9e7a48b85cfd69f33c33d2ef784a531c359399ad764243399673c8f5/pycountry-26.2.16.tar.gz", hash = "sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801", size = 7711342, upload-time = "2026-02-17T03:42:52.367Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/42/7703bd45b62fecd44cd7d3495423097e2f7d28bc2e99e7c1af68892ab157/pycountry-26.2.16-py3-none-any.whl", hash = "sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a", size = 8044600, upload-time = "2026-02-17T03:42:49.777Z" }, -] - -[[package]] -name = "pycparser" -version = "3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, -] - -[[package]] -name = "pydantic" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/84/6b/69fd5c7194b21ebde0f8637e2a4ddc766ada29d472bfa6a5ca533d79549a/pydantic-2.13.0.tar.gz", hash = "sha256:b89b575b6e670ebf6e7448c01b41b244f471edd276cd0b6fe02e7e7aca320070", size = 843468, upload-time = "2026-04-13T10:51:35.571Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/01/d7/c3a52c61f5b7be648e919005820fbac33028c6149994cd64453f49951c17/pydantic-2.13.0-py3-none-any.whl", hash = "sha256:ab0078b90da5f3e2fd2e71e3d9b457ddcb35d0350854fbda93b451e28d56baaf", size = 471872, upload-time = "2026-04-13T10:51:33.343Z" }, -] - -[package.optional-dependencies] -email = [ - { name = "email-validator" }, -] - -[[package]] -name = "pydantic-core" -version = "2.46.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/0a/9414cddf82eda3976b14048cc0fa8f5b5d1aecb0b22e1dcd2dbfe0e139b1/pydantic_core-2.46.0.tar.gz", hash = "sha256:82d2498c96be47b47e903e1378d1d0f770097ec56ea953322f39936a7cf34977", size = 471441, upload-time = "2026-04-13T09:06:33.813Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/d2/206c72ad47071559142a35f71efc29eb16448a4a5ae9487230ab8e4e292b/pydantic_core-2.46.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:66ccedb02c934622612448489824955838a221b3a35875458970521ef17b2f9c", size = 2117060, upload-time = "2026-04-13T09:04:47.443Z" }, - { url = "https://files.pythonhosted.org/packages/17/2c/7a53b33f91c8b77e696b1a6aa3bed609bf9374bdc0f8dcda681bc7d922b8/pydantic_core-2.46.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a44f27f4d2788ef9876ec47a43739b118c5904d74f418f53398f6ced3bbcacf2", size = 1951802, upload-time = "2026-04-13T09:05:34.591Z" }, - { url = "https://files.pythonhosted.org/packages/fc/20/90e548c1f6d38800ef11c915881525770ce270d8e5e887563ff046a08674/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f26a1032bcce6ca4b4670eb3f7d8195bd0a8b8f255f1307823e217ca3cfa7c27", size = 1976621, upload-time = "2026-04-13T09:04:03.909Z" }, - { url = "https://files.pythonhosted.org/packages/20/3c/9c5810ca70b60c623488cdd80f7e9ee1a0812df81e97098b64788719860f/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b8d1412f725060527e56675904b17a2d421dddcf861eecf7c75b9dda47921a4", size = 2056721, upload-time = "2026-04-13T09:04:40.992Z" }, - { url = "https://files.pythonhosted.org/packages/1a/a3/d6e5f4cdec84278431c75540f90838c9d0a4dfe9402a8f3902073660ff28/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc3d1569edd859cabaa476cabce9eecd05049a7966af7b4a33b541bfd4ca1104", size = 2239634, upload-time = "2026-04-13T09:03:52.478Z" }, - { url = "https://files.pythonhosted.org/packages/46/42/ef58aacf330d8de6e309d62469aa1f80e945eaf665929b4037ac1bfcebc1/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:38108976f2d8afaa8f5067fd1390a8c9f5cc580175407cda636e76bc76e88054", size = 2315739, upload-time = "2026-04-13T09:05:04.971Z" }, - { url = "https://files.pythonhosted.org/packages/8b/86/c63b12fafa2d86a515bfd1840b39c23a49302f02b653161bf9c3a0566c50/pydantic_core-2.46.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5a06d8ed01dad5575056b5187e5959b336793c6047920a3441ee5b03533836", size = 2098169, upload-time = "2026-04-13T09:07:27.151Z" }, - { url = "https://files.pythonhosted.org/packages/76/19/b5b33a2f6be4755b21a20434293c4364be255f4c1a108f125d101d4cc4ee/pydantic_core-2.46.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:04017ace142da9ce27cafd423a480872571b5c7e80382aec22f7d715ca8eb870", size = 2170830, upload-time = "2026-04-13T09:04:39.448Z" }, - { url = "https://files.pythonhosted.org/packages/99/ae/7559f99a29b7d440012ddb4da897359304988a881efaca912fd2f655652e/pydantic_core-2.46.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2629ad992ed1b1c012e6067f5ffafd3336fcb9b54569449fabb85621f1444ed3", size = 2203901, upload-time = "2026-04-13T09:04:01.048Z" }, - { url = "https://files.pythonhosted.org/packages/dd/0e/b0ef945a39aeb4ac58da316813e1106b7fbdfbf20ac141c1c27904355ac5/pydantic_core-2.46.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3068b1e7bd986aebc88f6859f8353e72072538dcf92a7fb9cf511a0f61c5e729", size = 2191789, upload-time = "2026-04-13T09:06:39.915Z" }, - { url = "https://files.pythonhosted.org/packages/90/f4/830484e07188c1236b013995818888ab93bab8fd88aa9689b1d8fd22220d/pydantic_core-2.46.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:1e366916ff69ff700aa9326601634e688581bc24c5b6b4f8738d809ec7d72611", size = 2344423, upload-time = "2026-04-13T09:05:12.252Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ba/e455c18cbdc333177af754e740be4fe9d1de173d65bbe534daf88da02ac0/pydantic_core-2.46.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:485a23e8f4618a1b8e23ac744180acde283fffe617f96923d25507d5cade62ec", size = 2384037, upload-time = "2026-04-13T09:06:24.503Z" }, - { url = "https://files.pythonhosted.org/packages/78/1f/b35d20d73144a41e78de0ae398e60fdd8bed91667daa1a5a92ab958551ba/pydantic_core-2.46.0-cp312-cp312-win32.whl", hash = "sha256:520940e1b702fe3b33525d0351777f25e9924f1818ca7956447dabacf2d339fd", size = 1967068, upload-time = "2026-04-13T09:05:23.374Z" }, - { url = "https://files.pythonhosted.org/packages/d1/84/4b6252e9606e8295647b848233cc4137ee0a04ebba8f0f9fb2977655b38c/pydantic_core-2.46.0-cp312-cp312-win_amd64.whl", hash = "sha256:90d2048e0339fa365e5a66aefe760ddd3b3d0a45501e088bc5bc7f4ed9ff9571", size = 2071008, upload-time = "2026-04-13T09:05:21.392Z" }, - { url = "https://files.pythonhosted.org/packages/39/95/d08eb508d4d5560ccbd226ee5971e5ef9b749aba9b413c0c4ed6e406d4f6/pydantic_core-2.46.0-cp312-cp312-win_arm64.whl", hash = "sha256:a70247649b7dffe36648e8f34be5ce8c5fa0a27ff07b071ea780c20a738c05ce", size = 2036634, upload-time = "2026-04-13T09:05:48.299Z" }, - { url = "https://files.pythonhosted.org/packages/74/0c/106ed5cc50393d90523f09adcc50d05e42e748eb107dc06aea971137f02d/pydantic_core-2.46.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:bc0e2fefe384152d7da85b5c2fe8ce2bf24752f68a58e3f3ea42e28a29dfdeb2", size = 2104968, upload-time = "2026-04-13T09:06:26.967Z" }, - { url = "https://files.pythonhosted.org/packages/f5/71/b494cef3165e3413ee9bbbb5a9eedc9af0ea7b88d8638beef6c2061b110e/pydantic_core-2.46.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:a2ab0e785548be1b4362a62c4004f9217598b7ee465f1f420fc2123e2a5b5b02", size = 1940442, upload-time = "2026-04-13T09:06:29.332Z" }, - { url = "https://files.pythonhosted.org/packages/7e/3e/a4d578c8216c443e26a1124f8c1e07c0654264ce5651143d3883d85ff140/pydantic_core-2.46.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16d45aecb18b8cba1c68eeb17c2bb2d38627ceed04c5b30b882fc9134e01f187", size = 1999672, upload-time = "2026-04-13T09:04:42.798Z" }, - { url = "https://files.pythonhosted.org/packages/cd/c1/9114560468685525a21770138382fd0cb849aaf351ff2c7b97f760d121e0/pydantic_core-2.46.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5078f6c377b002428e984259ac327ef8902aacae6c14b7de740dd4869a491501", size = 2154533, upload-time = "2026-04-13T09:04:50.868Z" }, -] - -[[package]] -name = "pydantic-extra-types" -version = "2.11.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/71/dba38ee2651f84f7842206adbd2233d8bbdb59fb85e9fa14232486a8c471/pydantic_extra_types-2.11.1.tar.gz", hash = "sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049", size = 172002, upload-time = "2026-03-16T08:08:03.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl", hash = "sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1", size = 79526, upload-time = "2026-03-16T08:08:02.533Z" }, -] - -[package.optional-dependencies] -pycountry = [ - { name = "pycountry" }, -] - -[[package]] -name = "pydantic-settings" -version = "2.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, -] - -[[package]] -name = "pygments" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, -] - -[[package]] -name = "pyjwt" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, -] - -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - -[[package]] -name = "pynvvideocodec" -version = "2.0.4" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/1c/78f6fdf85133157a6a3405eab5ef4c2bc8048194dbda1c91bb9b8645bb36/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bad9e25f494abdcfa8f9dffa33a840509eda3ffcdf6e7cf6465d73be307c0c82", size = 28630316, upload-time = "2026-07-08T04:25:54.596Z" }, - { url = "https://files.pythonhosted.org/packages/ca/49/98da271686e00676f41b1197ba5431ddc341b96d8efb68ea9d68e2b0d870/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b59cec7a1a3f78fad13fead78cad8b6d9686827f9ff4477080245457675a01d0", size = 43176147, upload-time = "2026-05-27T04:04:08.297Z" }, - { url = "https://files.pythonhosted.org/packages/42/80/7b13c12fd5f3243b01190130ce098a44ddf62e030e6ed712911cbfe40311/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:a0daa28b09705806c8c6b26326df217c45e60c0a12a673ea3ea6ee5e2e7193b0", size = 35754893, upload-time = "2026-05-27T04:04:43.866Z" }, - { url = "https://files.pythonhosted.org/packages/76/50/eb1571ab1cee8ebb8a7bdfc355078beebe4b2bb2e5c6ad5d0e18ab8585db/pynvvideocodec-2.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:46e2adb82dc6ac333d3535cc76e4e25c7e8d80dd272b1aba0c28702b861d5261", size = 25692590, upload-time = "2026-05-27T04:05:17.164Z" }, -] - -[[package]] -name = "pyparsing" -version = "3.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/11/b213bebff182584360cb8d17c72c1677fec5c5c228de439e63bcf8ab1c8f/pyparsing-3.3.3.tar.gz", hash = "sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36", size = 1050487, upload-time = "2026-09-20T20:59:05.609Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4", size = 126420, upload-time = "2026-09-20T20:59:04.025Z" }, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - -[[package]] -name = "python-dotenv" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, -] - -[[package]] -name = "python-json-logger" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/ff/3cc9165fd44106973cd7ac9facb674a65ed853494592541d339bdc9a30eb/python_json_logger-4.1.0.tar.gz", hash = "sha256:b396b9e3ed782b09ff9d6e4f1683d46c83ad0d35d2e407c09a9ebbf038f88195", size = 17573, upload-time = "2026-03-29T04:39:56.805Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/be/0631a861af4d1c875f096c07d34e9a63639560a717130e7a87cbc82b7e3f/python_json_logger-4.1.0-py3-none-any.whl", hash = "sha256:132994765cf75bf44554be9aa49b06ef2345d23661a96720262716438141b6b2", size = 15021, upload-time = "2026-03-29T04:39:55.266Z" }, -] - -[[package]] -name = "python-multipart" -version = "0.0.32" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, -] - -[[package]] -name = "pywin32" -version = "312" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" }, - { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" }, - { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" }, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, -] - -[[package]] -name = "pyzmq" -version = "27.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "implementation_name == 'pypy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:452631b640340c928fa343801b0d07eb0c3789a5ffa843f6e1a9cee0ba4eb4fc", size = 1306279, upload-time = "2025-09-08T23:08:03.807Z" }, - { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, - { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, - { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, - { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, - { url = "https://files.pythonhosted.org/packages/e6/2f/104c0a3c778d7c2ab8190e9db4f62f0b6957b53c9d87db77c284b69f33ea/pyzmq-27.1.0-cp312-abi3-win32.whl", hash = "sha256:250e5436a4ba13885494412b3da5d518cd0d3a278a1ae640e113c073a5f88edd", size = 559184, upload-time = "2025-09-08T23:08:15.163Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7f/a21b20d577e4100c6a41795842028235998a643b1ad406a6d4163ea8f53e/pyzmq-27.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:9ce490cf1d2ca2ad84733aa1d69ce6855372cb5ce9223802450c9b2a7cba0ccf", size = 619480, upload-time = "2025-09-08T23:08:17.192Z" }, - { url = "https://files.pythonhosted.org/packages/78/c2/c012beae5f76b72f007a9e91ee9401cb88c51d0f83c6257a03e785c81cc2/pyzmq-27.1.0-cp312-abi3-win_arm64.whl", hash = "sha256:75a2f36223f0d535a0c919e23615fc85a1e23b71f40c7eb43d7b1dedb4d8f15f", size = 552993, upload-time = "2025-09-08T23:08:18.926Z" }, -] - -[[package]] -name = "quack-kernels" -version = "0.6.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "einops" }, - { name = "nvidia-cutlass-dsl" }, - { name = "torch" }, - { name = "torch-c-dlpack-ext" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/75/5c/67c4a0d54cbb8d4d05af73132d620d1900a193b33e0d5ea6a25829b941d2/quack_kernels-0.6.4.tar.gz", hash = "sha256:8bf08a0e1a85aecc892ce84f4b6ed7abb6a44ec7c68b83040abe174bc8c2b936", size = 845485, upload-time = "2026-08-07T23:25:31.94Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/f5/36ec7e075e90bd92bdf30fee0606f189348c2bbb0af6aebaf7382927c622/quack_kernels-0.6.4-py3-none-any.whl", hash = "sha256:e77c5d1f1299b0b38487fe8737df6c6975daa16bca7f7eb883bd1a74d09e7e78", size = 728458, upload-time = "2026-08-07T23:25:30.549Z" }, -] - -[[package]] -name = "ray" -version = "2.56.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "jsonschema" }, - { name = "msgpack" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "requests" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/50/d9/a17feef16a123f5d32d4c5fa7de853c59ba702f8404cf452a7ce20faca13/ray-2.56.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:44bc0000c5bfad85b2ff6e0ef91e95f901d1a2d2fdd72f94f08a046eb494cd61", size = 66346989, upload-time = "2026-07-17T21:28:39.4Z" }, - { url = "https://files.pythonhosted.org/packages/05/19/c3b1bcccd09decaf2a2e3370041ae67070bb1a8638f2665d36edfbb0261d/ray-2.56.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:8fdd6b096215906cf1f9acdc7898c9d6140606f2d27245778b8385a9f19e6cb0", size = 73319289, upload-time = "2026-07-17T21:28:44.502Z" }, - { url = "https://files.pythonhosted.org/packages/d2/7f/577a61bf2c8eff26e942afe53a6b03f4dbf5b4f233b832c228417d0c954e/ray-2.56.1-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:e5d3173696831134c76bd09451dfe95c32d72c271253b0d6b09d2df9994aa660", size = 74194147, upload-time = "2026-07-17T21:28:50.567Z" }, - { url = "https://files.pythonhosted.org/packages/00/e9/0fd1223597f9ca98ec496ec726043918a5301547789b1be95a635ec82649/ray-2.56.1-cp312-cp312-win_amd64.whl", hash = "sha256:8052573ee5ef8c4fdd7aeb6a257c80542e69c48f3f6d117101f95c970ffdc7e2", size = 28373294, upload-time = "2026-07-17T21:28:55.122Z" }, -] - -[[package]] -name = "referencing" -version = "0.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "rpds-py" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, -] - -[[package]] -name = "regex" -version = "2026.7.19" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, - { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, - { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, - { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, - { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, - { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, - { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, - { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, - { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, - { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, - { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, - { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, - { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, - { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, - { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, -] - -[[package]] -name = "requests" -version = "2.34.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, -] - -[[package]] -name = "requests-oauthlib" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "oauthlib" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, -] - -[[package]] -name = "rich" -version = "15.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, -] - -[[package]] -name = "rich-toolkit" -version = "0.20.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "rich" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/3a/a258c2fbc6c6bdf428611388f5698ba5d57ffdf0755e1cab474d9cc47813/rich_toolkit-0.20.3.tar.gz", hash = "sha256:223dd2cfba325ed55e94933b9e53f3aca13e9fdf76622bd564c18109a2273c1b", size = 205355, upload-time = "2026-07-13T14:38:06.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/ce/639d0d0ce3d25c5edbd1afecd308bb35dc04883a45ac9f0855c8aee4e919/rich_toolkit-0.20.3-py3-none-any.whl", hash = "sha256:419aa87516d5f3849cca553c6dcf707c02a36d508fcf996946606725d34a3002", size = 36195, upload-time = "2026-07-13T14:38:05.687Z" }, -] - -[[package]] -name = "rignore" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c9/77/6ba90ab4a538d3ec244329c57c4d26a78c8313ea6fa72c8768d46f11c1c9/rignore-0.8.0.tar.gz", hash = "sha256:2e5ad6b19834f04a877d26fe863fd77ed851ed4019fdca097fb1b744311e3562", size = 55358, upload-time = "2026-07-17T19:01:21.257Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/7e/0d270c1ed723b82bea8ccd504185acb5d71830975260e8de02af7acff728/rignore-0.8.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a5e285ec58b3b66284a7f48805e4db0ea948370deb484a4935b147187ecf1e25", size = 847184, upload-time = "2026-07-17T18:58:31.348Z" }, - { url = "https://files.pythonhosted.org/packages/ff/dc/841941f8b0883a8038f9d540607238c456df20e98243a61142fd15699806/rignore-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:367d3cf401b477a5ba7eb05b5b94c491b0d704507d9eaf80378a8d843fe00674", size = 816590, upload-time = "2026-07-17T18:58:32.571Z" }, - { url = "https://files.pythonhosted.org/packages/57/53/9e047a6cd95b553519703350f7a3530ddd31d309c9ade1ec913db5882f74/rignore-0.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d49870c032abcc28db2210ad92b3bd5706ec792ab25c9295a6c536a4b92226ce", size = 884130, upload-time = "2026-07-17T18:58:33.921Z" }, - { url = "https://files.pythonhosted.org/packages/07/5a/ae444f30dfa47716ccadf9ed9512736494c0bf5222f330845a637f1c569c/rignore-0.8.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d50632823c273ee19212fd939b04e9b55d5a6e0c9b998fadef6dc9a0c2f6aecc", size = 857230, upload-time = "2026-07-17T18:58:35.32Z" }, - { url = "https://files.pythonhosted.org/packages/ba/67/b2ddfbf5a42a8ea89e6a0330189f7f76f9113e0d8c6a4c68b68ff68f139c/rignore-0.8.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b3337e69856ba18c9d079e0ce8b342dc256f48585ae3e1c7aac67682b672e83a", size = 1133331, upload-time = "2026-07-17T18:58:36.895Z" }, - { url = "https://files.pythonhosted.org/packages/8e/76/291be16f7260ac87dedab41394f7738834bb1cd05d6e1b878eea8d598520/rignore-0.8.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1264c48aaff09c6431b428144e9e9fbd0d7864673f7bd752b51d2409971275e", size = 912520, upload-time = "2026-07-17T18:58:38.178Z" }, - { url = "https://files.pythonhosted.org/packages/0c/99/cd0df2def95ecfa93c781431ce7c58ea52f90742550ff327ed0f1f715f14/rignore-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8912e3cb4034b972ba75c9ecabb79bc41a1076cdf726e53019e45b71b82da56", size = 927109, upload-time = "2026-07-17T18:58:40Z" }, - { url = "https://files.pythonhosted.org/packages/e4/58/ce8af7214b903a2f86c3223dabd23e03a0c499df6559be5c9e2b15b4344c/rignore-0.8.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:43b20be0f1c8ae3f3a2575dbee4a38bd406608855f6e79cde8fda1c6bff9e203", size = 892314, upload-time = "2026-07-17T18:58:41.391Z" }, - { url = "https://files.pythonhosted.org/packages/47/3f/e8f4ec6cdf04d0cb9915590679771b55a6f579a7478704d1e6be45a2f792/rignore-0.8.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02cf38d96b6f54b69719b726877e308b34ac2c9c9f636dc8872ede2ff6825579", size = 962528, upload-time = "2026-07-17T18:58:42.736Z" }, - { url = "https://files.pythonhosted.org/packages/58/57/d10a43221644177f776d658ed9a03204e49d9bfd6805fbece841ea6720ac/rignore-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5c820d3fa597ee419f3643f8c098a51249a868ca8515296a56a1505c01a8083d", size = 1060799, upload-time = "2026-07-17T18:58:44.279Z" }, - { url = "https://files.pythonhosted.org/packages/a9/0e/d769dfa933861dd469c9158ffc9bc2cb3bc6d46f88000fb6f41cf2905a3a/rignore-0.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e3e863b2cb1db481384bd43a25730ef00f7eccdd49c82a0e1d481a6412dc2653", size = 1132462, upload-time = "2026-07-17T18:58:45.583Z" }, - { url = "https://files.pythonhosted.org/packages/c1/3d/57fc6264ebf8d9b95850589899e6192d9f92b67ea48e56fc32969b75474a/rignore-0.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:90cc363aedc7a93b4b15933f2104bfcae1c6c1635a9bc479665cae83044577d9", size = 1139679, upload-time = "2026-07-17T18:58:46.928Z" }, - { url = "https://files.pythonhosted.org/packages/fa/be/24b12a8464e19d348aeb388267f897ede0fae6618052c3148b747acae214/rignore-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:303a9fd02d3612d15e6dc3474ae7d3eae1d07b668b7b4943fd4df305be4d2f76", size = 1138125, upload-time = "2026-07-17T18:58:48.42Z" }, - { url = "https://files.pythonhosted.org/packages/d6/8d/89f7cba3491164c04f8a64056ca973494536ffdf48bfba41565f54dfa122/rignore-0.8.0-cp312-cp312-win32.whl", hash = "sha256:e17a0914378fa15d1e29effce4b39fd2031f253e52d307ff4b71c443d1d5d30e", size = 637663, upload-time = "2026-07-17T18:58:49.76Z" }, - { url = "https://files.pythonhosted.org/packages/3f/ff/823a5ead8bba0a2054cbf2dbef99989204557904ad8578f514bec1b9b4e7/rignore-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:d5c8f1dff84c4d4114f1553564b5d909a0c4c09dbe1b5916a0506d719f1fc3b5", size = 728265, upload-time = "2026-07-17T18:58:51.028Z" }, - { url = "https://files.pythonhosted.org/packages/8f/ce/73c919505f9f270ee3e34205ff2dbe0b0d73e945f8c569922acff148bcf4/rignore-0.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:6cdea3f85de8286a38ae75a0f9092cd3afc4d33ce6ee2e3f6005f97f7da9d249", size = 665216, upload-time = "2026-07-17T18:58:52.308Z" }, -] - -[[package]] -name = "rpds-py" -version = "2026.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, - { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, - { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, - { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, - { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, - { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, - { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, - { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, - { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, - { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, - { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, - { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, - { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, -] - -[[package]] -name = "runai-model-streamer" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "humanize" }, - { name = "numpy" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/55/ebf29e73c7c4a66d093e0018b212ef8ccc828231f37c3362edda2535ecb3/runai_model_streamer-0.16.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:035cc282935d7d55032ce2da97507b3d94201e647d622bd44476a6ef9f04f824", size = 649145, upload-time = "2026-07-13T07:51:59.982Z" }, - { url = "https://files.pythonhosted.org/packages/ee/3d/32656788027d72d6bb54804a1479429ad1ccb36d212ff408c4f513bdd4a5/runai_model_streamer-0.16.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:0f9958ecdef6786347b855f345d3076a039201f6f2c0e22dd52c74eef8941430", size = 650434, upload-time = "2026-07-13T07:52:01.341Z" }, -] - -[package.optional-dependencies] -azure = [ - { name = "runai-model-streamer-azure" }, -] -gcs = [ - { name = "runai-model-streamer-gcs" }, -] -s3 = [ - { name = "runai-model-streamer-s3" }, -] - -[[package]] -name = "runai-model-streamer-azure" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-identity" }, - { name = "azure-storage-blob" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/13/16cf17e039ce7f7bc93a32c87876917a6cd946d668a3195990ce9f1494d3/runai_model_streamer_azure-0.16.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:77e00e797852167d9036c36a952a6cb46cabbb7d78ddf30a159ef424803f1d20", size = 5905985, upload-time = "2026-07-13T07:52:17.805Z" }, - { url = "https://files.pythonhosted.org/packages/cc/d4/39c97a540199a0f48a522bf44e5cb8bcc4e9f5a6bd1f051b14677c8c8fec/runai_model_streamer_azure-0.16.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:3c66172e3c22405d68cb47bccf022d7cab299f1571400ade684771b860420483", size = 5627580, upload-time = "2026-07-13T07:52:19.411Z" }, -] - -[[package]] -name = "runai-model-streamer-gcs" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "google-cloud-storage" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/f5/347ae314c8a76c76976c7edfc82764e1fefc9c75b393cc16c492e96dcfb0/runai_model_streamer_gcs-0.16.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4a536d2058c025a4452afa2cf869514ccd77d8b1437edd7b5edb9f881fc8ca25", size = 23302179, upload-time = "2026-07-13T07:52:11.08Z" }, - { url = "https://files.pythonhosted.org/packages/7a/ca/b34853e30bab59285d7b742af3d5e840213ebea816187b14d0b142c23b4f/runai_model_streamer_gcs-0.16.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:295f1af31452f47c32ae02d07787ab0922393971988070c0b9c618528d8ac8ec", size = 23249177, upload-time = "2026-07-13T07:52:13.385Z" }, -] - -[[package]] -name = "runai-model-streamer-s3" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "boto3" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/c9/36d39711463f48ce753e8fb7e21facff97a9789773a8f7d67f25897bed7d/runai_model_streamer_s3-0.16.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:b7ac88beda87b8f2864e8666c4b8974a0c1e886f9d307ae9de6fbfe18e3b9253", size = 6189831, upload-time = "2026-07-13T07:52:04.955Z" }, - { url = "https://files.pythonhosted.org/packages/58/40/275cf91d178abfc6e1feda301b7bc4ac85d8dbafeebbd77ffdb370938cf6/runai_model_streamer_s3-0.16.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:fb225b5dfbecf8087fe9073f5eb6916b56d19b7f7b49625269053f7205d7c0fd", size = 5931724, upload-time = "2026-07-13T07:52:06.589Z" }, -] - -[[package]] -name = "s3transfer" -version = "0.19.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, -] - -[[package]] -name = "safetensors" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, - { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, - { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, - { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, - { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, - { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, - { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, - { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, - { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, - { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, - { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, - { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, - { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, - { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, - { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, -] - -[[package]] -name = "scikit-learn" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy" }, - { name = "scipy" }, - { name = "threadpoolctl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/20/75f915ff375d6249e6550ac740fdbbd66159a068fd3af1400ff62036b07a/scikit_learn-1.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2bd41b0d201bc81575531b96b713d3eb5e5f50fb0b82101ff0f92294fdc236ac", size = 8741122, upload-time = "2026-06-02T11:53:24.08Z" }, - { url = "https://files.pythonhosted.org/packages/cc/d5/2b5148f2279196775e1db2aeb85d14b70ac80e7e32b3b28e7ebeafb0901d/scikit_learn-1.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5be45aa4a42a68a533913a6ed736cf309de2226411c79ef8d609a5456f1939b1", size = 8261512, upload-time = "2026-06-02T11:53:27.183Z" }, - { url = "https://files.pythonhosted.org/packages/a0/ee/5adbc77656b71f9456a2f5a7a9fdb4bcf9207a6b962889f1c2f9323afa4e/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e50ed4da51974e86e940690e9a3d82e729b62b5a49f7c9bac534d515d39d86f", size = 8837603, upload-time = "2026-06-02T11:53:30.328Z" }, - { url = "https://files.pythonhosted.org/packages/6c/c2/63fdda36c56437eeb44aaf9493c8bcd62ce230ab1598924fc626ffbfa943/scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8", size = 9132097, upload-time = "2026-06-02T11:53:33.456Z" }, - { url = "https://files.pythonhosted.org/packages/83/a4/c8e67227c680e2259c8864ae72ff48b06e16a6f51253a22167aa02a8aa4e/scikit_learn-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:4306775fad04cc4b472a1b15af1ae9cede1540fbfcc17fbce3767cd8dc7ae283", size = 8211173, upload-time = "2026-06-02T11:53:36.602Z" }, - { url = "https://files.pythonhosted.org/packages/cf/fd/3c0863792e98e67e9184aa4029288a175935eb65443afcd30d4f143450cf/scikit_learn-1.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:26e22435f63bcdcf396b574273f29f13dd531f5ea035801f5be10ba1540a4e60", size = 7867451, upload-time = "2026-06-02T11:53:39.075Z" }, -] - -[[package]] -name = "scipy" -version = "1.18.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a", size = 31056519, upload-time = "2026-06-19T14:59:45.203Z" }, - { url = "https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b", size = 28681889, upload-time = "2026-06-19T14:59:48.103Z" }, - { url = "https://files.pythonhosted.org/packages/9b/2d/11dd93d21e147a73ba22bd75c0b9208d3a2e0ec76d53170ce7d9029b1015/scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9", size = 20423580, upload-time = "2026-06-19T14:59:50.665Z" }, - { url = "https://files.pythonhosted.org/packages/9c/01/93552f75e0d2a7dd115a45e59209c51e8d514daff02fc887d2623be06fe1/scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8", size = 23054441, upload-time = "2026-06-19T14:59:53.564Z" }, - { url = "https://files.pythonhosted.org/packages/3c/23/21f5e703643d66f21faa6b4c73195bfcad70c55efcb4f1ab327cd7c4101a/scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab", size = 33968720, upload-time = "2026-06-19T14:59:56.415Z" }, - { url = "https://files.pythonhosted.org/packages/dd/aa/1b939f6c67ed68635bb538e6752d3dacc02f66535182e939a89581a44e9c/scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2", size = 35287115, upload-time = "2026-06-19T14:59:59.411Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ff/eec46be7e9234208f801062b53e1983085eddebd693f6c9bfb03b459830d/scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11", size = 35577989, upload-time = "2026-06-19T15:00:02.235Z" }, - { url = "https://files.pythonhosted.org/packages/84/ca/210d4759c7210bb7d269437421959b39a33434e2776b60c5cb8a763bb30a/scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de", size = 37421717, upload-time = "2026-06-19T15:00:05.102Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/9a9edb45345bd6744da5ddfb6628e5d5185920494c6a67ec45b6381004cb/scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132", size = 36597428, upload-time = "2026-06-19T15:00:08.112Z" }, - { url = "https://files.pythonhosted.org/packages/99/0e/33f32a2a58987e26aec0f7df252cbbad1e90ae77bdbc76f40dd4ed0cf0ea/scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76", size = 24351481, upload-time = "2026-06-19T15:00:11.103Z" }, -] - -[[package]] -name = "sentencepiece" -version = "0.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/33/ea3cb3839607eb175da835244a798f797f478c5ddf0e8ecdf57ea85a4c70/sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6", size = 8218435, upload-time = "2026-07-12T08:39:34.701Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/13/7a562289c8d5b49ebdf3f9c1e8ab67cf14a8743b1d90c8f406bfdec36b72/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b", size = 2188384, upload-time = "2026-07-12T08:38:28.486Z" }, - { url = "https://files.pythonhosted.org/packages/85/d1/912f14fd5eae168aba726ffb6a9a2dc1c71fe7676c53da6f5c442b886d4a/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820", size = 1441553, upload-time = "2026-07-12T08:38:30.552Z" }, - { url = "https://files.pythonhosted.org/packages/bd/44/caa9cab5f261a019e2808bc5046152775dc57352ba9cbae7525e9e7a1ed4/sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838", size = 1347176, upload-time = "2026-07-12T08:38:32.211Z" }, - { url = "https://files.pythonhosted.org/packages/19/90/cd798935668cff71d309d8ff10385844ecf216b1fe454f1993ed8bf2cb91/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a", size = 1325200, upload-time = "2026-07-12T08:38:33.689Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2d/37e3da037318a70066ded0d51bc2a7f35491ae6338dd993d5eb1503fc3b5/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da", size = 1397736, upload-time = "2026-07-12T08:38:35.771Z" }, - { url = "https://files.pythonhosted.org/packages/8d/11/753fca2e6b109be3ab7867abf357dfe48677fe726ae5a5363d0b54ca9450/sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383", size = 1248030, upload-time = "2026-07-12T08:38:37.604Z" }, - { url = "https://files.pythonhosted.org/packages/e2/0a/70efbe861ca182d7d4b6e1a20f58e043400848fa9f2915229f082e221648/sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d", size = 1187325, upload-time = "2026-07-12T08:38:39.348Z" }, -] - -[[package]] -name = "sentry-sdk" -version = "2.66.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7f/6f/d59cad0889d15fde85254cf58e701484de3f3f0406003b3197746910b19b/sentry_sdk-2.66.1.tar.gz", hash = "sha256:f882fb08710c5f8bfc603aafa3e901b384009a19cc3f76a572b863392ee81cdc", size = 940543, upload-time = "2026-07-22T12:26:54.553Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/d3/726bd88f0eece09ddf431bea4c9191c18e7a8d070b854eb0014d447712ee/sentry_sdk-2.66.1-py3-none-any.whl", hash = "sha256:86002793161d9a95ef04bdd8d442e9bfece5d989b755f05d6360215094a7aff6", size = 505555, upload-time = "2026-07-22T12:26:52.71Z" }, -] - -[[package]] -name = "setproctitle" -version = "1.3.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/48/49393a96a2eef1ab418b17475fb92b8fcfad83d099e678751b05472e69de/setproctitle-1.3.7.tar.gz", hash = "sha256:bc2bc917691c1537d5b9bca1468437176809c7e11e5694ca79a9ca12345dcb9e", size = 27002, upload-time = "2025-09-05T12:51:25.278Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/f0/2dc88e842077719d7384d86cc47403e5102810492b33680e7dadcee64cd8/setproctitle-1.3.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2dc99aec591ab6126e636b11035a70991bc1ab7a261da428491a40b84376654e", size = 18049, upload-time = "2025-09-05T12:49:36.241Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b4/50940504466689cda65680c9e9a1e518e5750c10490639fa687489ac7013/setproctitle-1.3.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdd8aa571b7aa39840fdbea620e308a19691ff595c3a10231e9ee830339dd798", size = 13079, upload-time = "2025-09-05T12:49:38.088Z" }, - { url = "https://files.pythonhosted.org/packages/d0/99/71630546b9395b095f4082be41165d1078204d1696c2d9baade3de3202d0/setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2906b6c7959cdb75f46159bf0acd8cc9906cf1361c9e1ded0d065fe8f9039629", size = 32932, upload-time = "2025-09-05T12:49:39.271Z" }, - { url = "https://files.pythonhosted.org/packages/50/22/cee06af4ffcfb0e8aba047bd44f5262e644199ae7527ae2c1f672b86495c/setproctitle-1.3.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6915964a6dda07920a1159321dcd6d94fc7fc526f815ca08a8063aeca3c204f1", size = 33736, upload-time = "2025-09-05T12:49:40.565Z" }, - { url = "https://files.pythonhosted.org/packages/5c/00/a5949a8bb06ef5e7df214fc393bb2fb6aedf0479b17214e57750dfdd0f24/setproctitle-1.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff72899861c765bd4021d1ff1c68d60edc129711a2fdba77f9cb69ef726a8b6", size = 35605, upload-time = "2025-09-05T12:49:42.362Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3a/50caca532a9343828e3bf5778c7a84d6c737a249b1796d50dd680290594d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7cb05bd446687ff816a3aaaf831047fc4c364feff7ada94a66024f1367b448c", size = 33143, upload-time = "2025-09-05T12:49:43.515Z" }, - { url = "https://files.pythonhosted.org/packages/ca/14/b843a251296ce55e2e17c017d6b9f11ce0d3d070e9265de4ecad948b913d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3a57b9a00de8cae7e2a1f7b9f0c2ac7b69372159e16a7708aa2f38f9e5cc987a", size = 34434, upload-time = "2025-09-05T12:49:45.31Z" }, - { url = "https://files.pythonhosted.org/packages/c8/b7/06145c238c0a6d2c4bc881f8be230bb9f36d2bf51aff7bddcb796d5eed67/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8828b356114f6b308b04afe398ed93803d7fca4a955dd3abe84430e28d33739", size = 32795, upload-time = "2025-09-05T12:49:46.419Z" }, - { url = "https://files.pythonhosted.org/packages/ef/dc/ef76a81fac9bf27b84ed23df19c1f67391a753eed6e3c2254ebcb5133f56/setproctitle-1.3.7-cp312-cp312-win32.whl", hash = "sha256:b0304f905efc845829ac2bc791ddebb976db2885f6171f4a3de678d7ee3f7c9f", size = 12552, upload-time = "2025-09-05T12:49:47.635Z" }, - { url = "https://files.pythonhosted.org/packages/e2/5b/a9fe517912cd6e28cf43a212b80cb679ff179a91b623138a99796d7d18a0/setproctitle-1.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:9888ceb4faea3116cf02a920ff00bfbc8cc899743e4b4ac914b03625bdc3c300", size = 13247, upload-time = "2025-09-05T12:49:49.16Z" }, -] - -[[package]] -name = "setuptools" -version = "80.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/95/faf61eb8363f26aa7e1d762267a8d602a1b26d4f3a1e758e92cb3cb8b054/setuptools-80.10.2.tar.gz", hash = "sha256:8b0e9d10c784bf7d262c4e5ec5d4ec94127ce206e8738f29a437945fbc219b70", size = 1200343, upload-time = "2026-01-25T22:38:17.252Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" }, -] - -[[package]] -name = "shellingham" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, -] - -[[package]] -name = "simplejson" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/f0/ea064bba6c9afda0168ddb834f1c75a93351031e25aee35c046108e7f292/simplejson-4.2.0.tar.gz", hash = "sha256:55b121b70a560f4610bd3a355ab2015aca4f39978f6a82353f24d2013fe85861", size = 123986, upload-time = "2026-10-03T03:34:23.27Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/24/87a310dcd8bd02876bbc33164d41f1d7b78a14ae98210b187cae22f560fc/simplejson-4.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:aa067739b28c661deb4421ee9ec1d7bad5ee06b7c50f8cf0d009e7945abe7d52", size = 117330, upload-time = "2026-10-03T03:32:06.623Z" }, - { url = "https://files.pythonhosted.org/packages/4d/cf/b1fc78e122ab98a2bae6fd6d66dec18d1ed436cafb875278f4d9e7677cbd/simplejson-4.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f458e7a2dd3d1b8b90dc12900c9e5a0f8b863fa7b02286fee13086962244f70a", size = 95482, upload-time = "2026-10-03T03:32:07.969Z" }, - { url = "https://files.pythonhosted.org/packages/27/ec/bad733020b3eef7e8414385fdfd4ba6afd5b8a56a0c4eb4eaa21be380f2a/simplejson-4.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c490ec62ed1b66a27afd5085e743e7f93b745c515257373de8433f4d51e5c3bb", size = 95297, upload-time = "2026-10-03T03:32:09.864Z" }, - { url = "https://files.pythonhosted.org/packages/2d/29/fb579920d8ec86ebb1f9a8cc8c9d17f3dcdb32a885dbd8332d21aa855575/simplejson-4.2.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8c1e156ad810704994439719b9c03694e267052d4938ca188d91a1769d6f742b", size = 198576, upload-time = "2026-10-03T03:32:11.003Z" }, - { url = "https://files.pythonhosted.org/packages/8e/7d/11fee9bebb22944c9e294139d5035cfa584f692192dabf3659e9c9bc0148/simplejson-4.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8910997afb7bae918b1ccf766e106e37707c8f8b4c61ac6ce433c4c86c5848f", size = 196424, upload-time = "2026-10-03T03:32:12.299Z" }, - { url = "https://files.pythonhosted.org/packages/a4/9e/cfb4d64d68e589f93aa83ecdf0ab9e62c787d7a5ef41c8a31f74eb62857c/simplejson-4.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f0767e82c062486211af7ee88cbe4732ca24250ce8127ffebd47732455439b69", size = 203804, upload-time = "2026-10-03T03:32:13.758Z" }, - { url = "https://files.pythonhosted.org/packages/7f/12/0e752142cdfbbd442f9ac614eb92b61c3def9a6fdd7aad4771ec9acc4abe/simplejson-4.2.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:797f086f589395e701ab077e9996dc0522a0b60158993e703e42749c4a17127c", size = 188048, upload-time = "2026-10-03T03:32:15.5Z" }, - { url = "https://files.pythonhosted.org/packages/4f/ff/4da29e068b788803681f17693153ed7f472c4190400eb724698b8e3dcd4b/simplejson-4.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2f53916dc840f4424dbafca0da7e8a3bafa7372ce7e1866c764966e36271f7bb", size = 193244, upload-time = "2026-10-03T03:32:16.694Z" }, - { url = "https://files.pythonhosted.org/packages/3b/4a/8e4167770b595ad5f7d2e20f0605e83f3467fa0ca6b85d50ee09019c4607/simplejson-4.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:703f532ec018562bb0c8eaf4b4851f5736c0f60d02e23ba8736ce885fa361eda", size = 201201, upload-time = "2026-10-03T03:32:18.001Z" }, - { url = "https://files.pythonhosted.org/packages/c6/5a/b8095e99e96a0f33f49d4da723d3a8f3b4295a13362c6fc26810f5dc51c4/simplejson-4.2.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2c1772c43537c7cc616fc217344acb00dec8312cfa76e725b6c5a6a4d5f80fb5", size = 186277, upload-time = "2026-10-03T03:32:19.326Z" }, - { url = "https://files.pythonhosted.org/packages/3d/87/fd79ac0e3841cc173f4b44d9158d595eaac072603691b505dafab6bde0aa/simplejson-4.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:471f30cd51ffdda1a0c421dc9963ada31e9d29bd688a3198041d2c69d18d65c4", size = 195937, upload-time = "2026-10-03T03:32:20.615Z" }, - { url = "https://files.pythonhosted.org/packages/c5/3f/45f19753465fa2ec337259b608236620e44a6aa410c1167edda010f30ce1/simplejson-4.2.0-cp312-cp312-win32.whl", hash = "sha256:85bde07e265b39be9593c0dd5e144c2308aa51d2dd1c18f495b46fa942f336d7", size = 92129, upload-time = "2026-10-03T03:32:21.839Z" }, - { url = "https://files.pythonhosted.org/packages/78/f0/08a6cffc4545112ca4c9918110e8c5227146a69fd270b0c64ff22a445ac6/simplejson-4.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:733acb0a25795becbbb6c5564f5c1c2e839889a931a72249fb0cc1c176659d83", size = 93961, upload-time = "2026-10-03T03:32:22.996Z" }, - { url = "https://files.pythonhosted.org/packages/e9/4c/9acdf4ae4f41c09a09ad17427e5ee912f35aa56ea1d1723a9d927d659d4e/simplejson-4.2.0-py3-none-any.whl", hash = "sha256:c2a2e5f43287cbe3413f7b73b04d5a6f75c7bd93d783e628f5978853a2ef738d", size = 72826, upload-time = "2026-10-03T03:34:21.667Z" }, -] - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "sniffio" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, -] - -[[package]] -name = "soundfile" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d2/db/949331952a6fb1c5b12e9de80fd08747966c2039d1a61db4764fbd3981c2/soundfile-0.14.0.tar.gz", hash = "sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11", size = 47842, upload-time = "2026-06-06T08:58:47.869Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/d1/5e338af9ca6ed0786cd5bb03f6d60de1c325728c1189014f3b59aae7403c/soundfile-0.14.0-py2.py3-none-any.whl", hash = "sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8", size = 26799, upload-time = "2026-06-06T08:58:33.269Z" }, - { url = "https://files.pythonhosted.org/packages/7e/72/c6b21e58d3113596e7e8de0a08d6f1d95173492cfbca0a4db14148cbba2a/soundfile-0.14.0-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4", size = 1144568, upload-time = "2026-06-06T08:58:35.231Z" }, - { url = "https://files.pythonhosted.org/packages/63/7a/dfdd6f8c748988427119f75eb860a3cedd858d1aea1fe28f39ad8559ef22/soundfile-0.14.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c", size = 1103726, upload-time = "2026-06-06T08:58:37.948Z" }, - { url = "https://files.pythonhosted.org/packages/4a/f8/fc39fad6f879633461d27394cd1ddaf1f769ffa0597dca35872f51b16461/soundfile-0.14.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377", size = 1238050, upload-time = "2026-06-06T08:58:39.932Z" }, - { url = "https://files.pythonhosted.org/packages/7b/a2/70fd4432b924684c372df8b0a45708c36c057ef3596c9eb53e0a806b980b/soundfile-0.14.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d", size = 1315963, upload-time = "2026-06-06T08:58:41.716Z" }, - { url = "https://files.pythonhosted.org/packages/d9/34/c9e80783d83eab739a9531fdee03675d53e0bf1b2ccb4bb3af5844675046/soundfile-0.14.0-py2.py3-none-win32.whl", hash = "sha256:0a6ae43c50c71b4e020cc55382925cb89451c1ed1a0c3d0f5d802da269226849", size = 902199, upload-time = "2026-06-06T08:58:43.289Z" }, - { url = "https://files.pythonhosted.org/packages/ed/97/b39c18ac1df45e755ca22b8b00e872929da5d107998a207a5e4ac831bfda/soundfile-0.14.0-py2.py3-none-win_amd64.whl", hash = "sha256:299491d3499460fb1b74bb4bd78b57ffc2d243a5fafa7b6ec1b264875c78453e", size = 1021480, upload-time = "2026-06-06T08:58:45.016Z" }, - { url = "https://files.pythonhosted.org/packages/f4/83/55c65e61cf457805ce2ec157c1c6ae17715d0851aa2374422de0538838ca/soundfile-0.14.0-py2.py3-none-win_arm64.whl", hash = "sha256:e090704718e124e7c844695236f1fce8d18a5e761eaf7c82dfcd124620805f98", size = 888858, upload-time = "2026-06-06T08:58:46.593Z" }, -] - -[[package]] -name = "soxr" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ed/11/27cebce4a108f77afea7c80545115536b45e3f11ebfb914f638fdd9ba847/soxr-1.1.0.tar.gz", hash = "sha256:9f228ae21c78fa9359ca98d8a5e8e91f30639e438e574133dace62c5b5309e44", size = 173067, upload-time = "2026-05-03T00:15:18.214Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/8a/f3da7973b5f1b05d2d7e94d5376b881dcbc05297900cae6c3d33d95b209b/soxr-1.1.0-cp312-abi3-macosx_10_14_x86_64.whl", hash = "sha256:e0e09fa633ce2e67df08b298afced4d184f6e753fc330f241022250f1d0d61da", size = 204124, upload-time = "2026-05-03T00:14:54.505Z" }, - { url = "https://files.pythonhosted.org/packages/03/dc/200013a74641f8774664bbcd2346c695c05c2e300ea792adcb40a293eed0/soxr-1.1.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d6a7ad82b8d5f3fcc04b1d2ca055562b96af571e1d4fa7c6c61d0fb509ac43b4", size = 165457, upload-time = "2026-05-03T00:14:56.007Z" }, - { url = "https://files.pythonhosted.org/packages/88/2b/2e5eba817a762a2ec589ff165b8bc5955b25a0ad140045f7cd8e45410543/soxr-1.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf98c0d7b7d5ef5bf072fee8d3020e8b664f2d195933ea7bc5089267c2e22a06", size = 206529, upload-time = "2026-05-03T00:14:57.646Z" }, - { url = "https://files.pythonhosted.org/packages/5c/f1/0e55195893228609c9a08c3b13b7a83a46c3a992cd00d3304f0f320cfb07/soxr-1.1.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b033078e86f3c4a658e5697fac8995764fad9e799563616b630136b613167f1", size = 240413, upload-time = "2026-05-03T00:14:59.363Z" }, - { url = "https://files.pythonhosted.org/packages/b0/4d/621e4150e4815246ad552d215a8a294a90143fedd19ee442cf82d3b3abc8/soxr-1.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:6ae2a174bffea94e8ead857dad85999d3f49f091774dbad5b046c0417d7092f4", size = 174357, upload-time = "2026-05-03T00:15:00.724Z" }, -] - -[[package]] -name = "sqlalchemy" -version = "2.1.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/44/311bac6b6ef81e4dfd0287d04900108b1f5c00c9761dd3c0a2b7b9d0f86b/sqlalchemy-2.1.4.tar.gz", hash = "sha256:7bd7ad604487daa7eab8716471c29a7185f17b5287ce73bb7bc79fea050d8cfd", size = 10544216, upload-time = "2026-10-07T17:33:59.116Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/5e/cb5b078e007340661b010fa8bd31ce27468f88e09b35266544df4e0c52ca/sqlalchemy-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f953be9ba26039a24a5205c65d33518b608ce6f4f0f4e9b9c14eaf42a10dfc52", size = 2466469, upload-time = "2026-10-07T18:17:24.049Z" }, - { url = "https://files.pythonhosted.org/packages/b1/98/44e2fdc5bc053dae559bf4f4eb7967ceecbad162299ecfc8de2edc3fcbe7/sqlalchemy-2.1.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ac64fce94c5b389062d2e3806db5dc780447591e0dfd5ead218c884f0703f2e", size = 4668213, upload-time = "2026-10-07T18:37:42.294Z" }, - { url = "https://files.pythonhosted.org/packages/08/25/ed2262f964687b06f10c2c98b2dc9c9ed211f7cc11702879969a9ac217e4/sqlalchemy-2.1.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e5045fb6aadbb0f978ab9b9d8822f7b7a97d2281814e7d13d791155664eace3", size = 4720857, upload-time = "2026-10-07T18:24:46.842Z" }, - { url = "https://files.pythonhosted.org/packages/4d/d4/fab64c61d5d22ddbb077afd1e6b29b498bdacdf6406a03f53566e7e01686/sqlalchemy-2.1.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e3a026436c51f296aa1d01243909a3b76490950e927824b10899a083cc26e7c3", size = 4369351, upload-time = "2026-10-07T18:59:45.483Z" }, - { url = "https://files.pythonhosted.org/packages/d9/e4/33413f0fafbcf3b332320aac2c1e40f3b4f17e56359a9474cb10de4bee8b/sqlalchemy-2.1.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:71040390ef01c85e9d26e5c83cb0c5942dcc8725c49186430af160ce2f54234d", size = 4591496, upload-time = "2026-10-07T18:37:44.433Z" }, - { url = "https://files.pythonhosted.org/packages/bb/65/19821440cbd5c93da053d627b3e402eff11ff252bfae37700645b3c155a4/sqlalchemy-2.1.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:07c60abaffb980b7382f2c75be8a5279c2b5df2626a0f5d751dd942799bf3b5c", size = 4362379, upload-time = "2026-10-07T18:59:48.278Z" }, - { url = "https://files.pythonhosted.org/packages/01/e3/168a0f93efd6ec40f59645a7e45ab08918e0bc8ecf07656e4ca09acdcc30/sqlalchemy-2.1.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a577e2127e52b0fe2bc54c73abb375a20ffe6f59fbc5568ccafc233f5bfcf8ef", size = 4667997, upload-time = "2026-10-07T18:24:48.72Z" }, - { url = "https://files.pythonhosted.org/packages/54/79/0a852ef65864acd8d577d7aa6f67146167382bd6faee7a7586b9e6e28275/sqlalchemy-2.1.4-cp312-cp312-win32.whl", hash = "sha256:6c79e0c824d51c586757ecd342160bbdede9010df04bb71b9bbfffd5c7b6ee29", size = 2376664, upload-time = "2026-10-07T18:25:00.637Z" }, - { url = "https://files.pythonhosted.org/packages/27/b9/a5934263bb1d712f743289ca224ab3b87e3570ac157802291e37ab85d365/sqlalchemy-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:dffa69d2f3ba1933c1c1882dbef8fb3231b33eb19263e8b8c5cea24995071f06", size = 2429357, upload-time = "2026-10-07T18:25:02.565Z" }, - { url = "https://files.pythonhosted.org/packages/a5/fa/a2323d81384ff214aa189057b7455b63623e66f28208b982e86c3cb042f5/sqlalchemy-2.1.4-cp312-cp312-win_arm64.whl", hash = "sha256:e30524ae24e31d83e1b5f734862882c442f4158e3566f2c5f5e9bd3c659bb517", size = 2388756, upload-time = "2026-10-07T18:22:36.025Z" }, - { url = "https://files.pythonhosted.org/packages/f7/62/dbf11a262f6fbb41390cab2d8e47a30ec0961018b68201607b599dd489f5/sqlalchemy-2.1.4-py3-none-any.whl", hash = "sha256:0b96edcc2cd60fe1e35f67a46f4eb076e57297841b9eae949ac5f196593f00a7", size = 2054935, upload-time = "2026-10-07T18:01:16.403Z" }, -] - -[[package]] -name = "sse-starlette" -version = "3.4.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "starlette" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6c/10/a34c656829ffc1c4b22ef36d70d9ebb6b99c020e2aeb17cee5485099f028/sse_starlette-3.4.6.tar.gz", hash = "sha256:725f8a1bd6d26ae1b2c9610c0ef5065dfdd496f3988d28adcf8c4b49dc25c627", size = 32542, upload-time = "2026-07-20T14:16:32.201Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/36/e10c1d1b7ca881d2625db2ec28508578499187bb1c389952c398474e1834/sse_starlette-3.4.6-py3-none-any.whl", hash = "sha256:56217ab4c9a9f9c5db7b21e08732d3e7c2b807f45231ad23de0551a24c4a41f6", size = 16516, upload-time = "2026-07-20T14:16:30.978Z" }, -] - -[[package]] -name = "starlette" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, -] - -[[package]] -name = "supervisor" -version = "4.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/b5/37e7a3706de436a8a2d75334711dad1afb4ddffab09f25e31d89e467542f/supervisor-4.3.0.tar.gz", hash = "sha256:4a2bf149adf42997e1bb44b70c43b613275ec9852c3edacca86a9166b27e945e", size = 468912, upload-time = "2025-08-23T18:25:02.418Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/65/5e726c372da8a5e35022a94388b12252710aad0c2351699c3d76ae8dba78/supervisor-4.3.0-py2.py3-none-any.whl", hash = "sha256:0bcb763fddafba410f35cbde226aa7f8514b9fb82eb05a0c85f6588d1c13f8db", size = 320736, upload-time = "2025-08-23T18:25:00.767Z" }, -] - -[[package]] -name = "sympy" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mpmath" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, -] - -[[package]] -name = "tabulate" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, -] - -[[package]] -name = "tensorstore" -version = "0.1.85" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ml-dtypes" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/16/a6530387ed22d07e974af2ef55e34fb19a7251a6911c85a0be029a4dcf9c/tensorstore-0.1.85.tar.gz", hash = "sha256:26698bded0278e98e0988eb8f7d76ad248762afc5f55a38b9122ffd1474e505f", size = 7317948, upload-time = "2026-08-05T15:35:56.098Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/81/612d3f58009ea1380f9befe2dced9d98c0a0fd608cd3e9d89389f9a04757/tensorstore-0.1.85-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:6b9453f28de853c804339bf882d0bc3eb523c4c55600b764239240bdec4460a8", size = 17015828, upload-time = "2026-08-05T15:35:03.426Z" }, - { url = "https://files.pythonhosted.org/packages/fa/3c/74050e09a490c0cfed6016430226710d29fe4e165e12145368c7de4c6426/tensorstore-0.1.85-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af9189ca9f692bf83ce53e760a8b53fcf46ad0bf365930726faa0e301d0ab147", size = 15411026, upload-time = "2026-08-05T15:35:05.89Z" }, - { url = "https://files.pythonhosted.org/packages/88/7e/a1100ea66dbb028e0ea02f6d2e413e05285534d82d560f1f7d6145603640/tensorstore-0.1.85-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2f42d1e57df7e07655b819dd07f3de258fde97e39b896a53f089d7d7067e21", size = 19973651, upload-time = "2026-08-05T15:35:09.016Z" }, - { url = "https://files.pythonhosted.org/packages/78/49/c311f02b82299c6d79e18c7012c8b816141c54721253cbcba1c7c59550d4/tensorstore-0.1.85-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de27f2d6e063050f4283d1fdfe41e6c0ebc6ceacaab10ca2f7e9ecbf35221962", size = 21580735, upload-time = "2026-08-05T15:35:11.727Z" }, - { url = "https://files.pythonhosted.org/packages/88/da/3d66562291808e1edd433704bfdd4f212c1a6c53920293fb5f7296d644a5/tensorstore-0.1.85-cp312-cp312-win_amd64.whl", hash = "sha256:a5421480cd6ab144039c88fc7018ea5cefb8d00038e6dd2470cf776ef1bf45a5", size = 14129526, upload-time = "2026-08-05T15:35:14.98Z" }, -] - -[[package]] -name = "tfp-nightly" -version = "0.26.0.dev20260626" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "cloudpickle" }, - { name = "decorator" }, - { name = "dm-tree" }, - { name = "gast" }, - { name = "numpy" }, - { name = "six" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/5c/e24a6f85b2038cbf209844944774102bb37e3a1e67ae524c71a7aad0e0a6/tfp_nightly-0.26.0.dev20260626-py2.py3-none-any.whl", hash = "sha256:57d024ecbc1da39e33af522f98bdfc95cb118af04e27a0b3391618f29be546ee", size = 6975580, upload-time = "2026-06-26T08:45:22.208Z" }, -] - -[package.optional-dependencies] -jax = [ - { name = "jax" }, - { name = "jaxlib" }, -] - -[[package]] -name = "threadpoolctl" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, -] - -[[package]] -name = "tiktoken" -version = "0.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "regex" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/8e/144bde4e01df66b34bb865557c7cd754ed08b036217ebd79c9db5e9048a9/tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791", size = 1034888, upload-time = "2026-05-15T04:50:31.579Z" }, - { url = "https://files.pythonhosted.org/packages/36/18/d4ac9d20956cdebca04841316660ed584c2fecdc2b81722a28bc7ad3b1e4/tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b", size = 982970, upload-time = "2026-05-15T04:50:32.961Z" }, - { url = "https://files.pythonhosted.org/packages/74/ed/6bb8d05b9f731f749fee5c6f5ca63e981143c826a5985877330507bd13b7/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7", size = 1115741, upload-time = "2026-05-15T04:50:34.475Z" }, - { url = "https://files.pythonhosted.org/packages/34/de/2ca96b07a82d972b74fe4b46de055b79c904e45c7eab699354a0bfa697dc/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649", size = 1136523, upload-time = "2026-05-15T04:50:35.782Z" }, - { url = "https://files.pythonhosted.org/packages/ee/dc/9dafec002c2d4424378563cf4cf5c7fb93631d2a55013c8b87554ee4012c/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b", size = 1181954, upload-time = "2026-05-15T04:50:36.99Z" }, - { url = "https://files.pythonhosted.org/packages/a1/d0/1f8578c45b2f24759b46f0b50d31878c63c73e6bf0f2227e10ec5c5408dc/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91", size = 1240069, upload-time = "2026-05-15T04:50:38.221Z" }, - { url = "https://files.pythonhosted.org/packages/aa/90/28d7f154888610aa9237e541986beb62b479df29d193a5a0617dbb1514d0/tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41", size = 874748, upload-time = "2026-05-15T04:50:39.587Z" }, -] - -[[package]] -name = "tilelang" -version = "0.1.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "cloudpickle" }, - { name = "ml-dtypes" }, - { name = "numpy" }, - { name = "psutil" }, - { name = "setuptools", marker = "sys_platform == 'darwin'" }, - { name = "torch" }, - { name = "torch-c-dlpack-ext" }, - { name = "tqdm" }, - { name = "typing-extensions" }, - { name = "z3-solver" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/9a/5776adb5b6e03e141c8cff020e16c45b87c9b7a7a5bdfe83416d6cc23933/tilelang-0.1.12.tar.gz", hash = "sha256:595b39581d099a53f875bd8d553863e7e4f58998052b58764f5472cbfbb87043", size = 93565090, upload-time = "2026-07-08T04:22:52.901Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/51/4ae1fb532406082a4ebf16c890f5f98db14b8588f2c486334ba23578b4bc/tilelang-0.1.12-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:8741d6cd519782645d55c17abd8986402c62948bc2122d3417c445d11de5dd16", size = 38347295, upload-time = "2026-07-08T04:22:35.691Z" }, - { url = "https://files.pythonhosted.org/packages/d1/53/f281a0bd9ee7e03d6a97828fc0e443321ed26ea2b0bd74bf9f1d9451d30f/tilelang-0.1.12-cp38-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbeb5573cbe2544a51a5c9a6cc737c5e3b5c2d95411caa3687fd9b08eb9d5f97", size = 50501074, upload-time = "2026-07-08T04:22:39.973Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a4/ff0b14d00d29fe418063cd97490f55aa9cad6f981a6fbbf29124b024637d/tilelang-0.1.12-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:a1701eba4ad603d73f1b128b88f6a2350f1b1161729e5b2d1e540a1ec7a82d52", size = 46281933, upload-time = "2026-07-08T04:22:44.244Z" }, - { url = "https://files.pythonhosted.org/packages/cb/15/1a8a5b5df0a23ece80b4da8e49c43418244e5ab165ed1938a1cc2656cda2/tilelang-0.1.12-cp38-abi3-win_amd64.whl", hash = "sha256:daeb3d529ab9e7665adb94c4bf0d47732294709aa9b8b9bd673fa48001385422", size = 34020025, upload-time = "2026-07-08T04:22:48.066Z" }, -] - -[[package]] -name = "tokenizers" -version = "0.22.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, - { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, - { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, - { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, - { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, - { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, - { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, - { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, - { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, - { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, - { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, - { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, - { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, - { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, - { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, -] - -[[package]] -name = "tokenspeed-mla" -version = "0.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "nvidia-cutlass-dsl" }, - { name = "tokenspeed-triton" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/65/81d7e9f14472bc4c6abb576c9b1edd8e40ab01832027c4e647bfe2890749/tokenspeed_mla-0.1.8-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:952209cf4b29a54e6b6e7e088be9d4a40f24792b06fdfa330de8077b9926a7d9", size = 752341, upload-time = "2026-06-24T03:36:28.619Z" }, - { url = "https://files.pythonhosted.org/packages/27/df/0037ade72b165ac97859040919e006aa3d80cb8cc3a79420fb6c03eb16a0/tokenspeed_mla-0.1.8-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6a7526d7327746893f8c20d24aa63ba5b8a123d0dfd6e66388e13b768b6452c6", size = 755827, upload-time = "2026-06-24T03:36:29.96Z" }, -] - -[[package]] -name = "tokenspeed-triton" -version = "3.8.10.post20260721" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/44/89740db8951918c9acd8731243eef8b44d0eb92ea423552639265c46018e/tokenspeed_triton-3.8.10.post20260721-cp312-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d793ad0eaebb1d08272c97a2b8f2c31304231748b03de9a08e70a362de92a6e0", size = 82966664, upload-time = "2026-07-21T17:14:38.568Z" }, - { url = "https://files.pythonhosted.org/packages/91/53/f46b401e8ec8998f5b9c39cff0614b796bf49113a09f588cfdfa342789a3/tokenspeed_triton-3.8.10.post20260721-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66cba8d32a1539afd0ff3eec1782b082d01b4db6824d68017d1d789a03d0be37", size = 87210295, upload-time = "2026-07-21T17:14:42.173Z" }, -] - -[[package]] -name = "toolz" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/6f/ae20c212a07aa2d156c787383d8088a5e045ee39628661edb190c97e1659/toolz-1.2.0.tar.gz", hash = "sha256:9667a038e9d6ecba37995e26cb2f59ec6420b6ad8dd9677de59db9b956b08490", size = 55442, upload-time = "2026-10-07T04:16:25.639Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/17/4c8beb6c8c4176c6bf143bfd7e1e4dd6719b00ced90738c7ac471b71c1df/toolz-1.2.0-py3-none-any.whl", hash = "sha256:890f820b1cb8152785aaf9386d8707770110809035800985ca65cb24ce1120ef", size = 60789, upload-time = "2026-10-07T04:16:24.173Z" }, -] - -[[package]] -name = "torch" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/3a/ed0f4d4d1dcde03bced7aac9a28e800abcdc0cbd06b6775044c9fbd877b7/torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027", size = 111213045, upload-time = "2026-07-08T16:05:22.997Z" }, - { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" }, - { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" }, - { url = "https://files.pythonhosted.org/packages/21/d6/e8f3c6f7e01f626f77259de9860d2a78bc84c40539e28e79b7e98b0bb659/torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d", size = 122057313, upload-time = "2026-07-08T16:03:53.43Z" }, -] - -[[package]] -name = "torch-c-dlpack-ext" -version = "0.1.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "torch" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/37/de/921b6491efce5c389a5ef9bbed3d2d6660005840dae488124173180859ab/torch_c_dlpack_ext-0.1.5.tar.gz", hash = "sha256:d06f0357d575d22a168cc77acb9020fc4bae30968ceb6718a055dcbe92bacabe", size = 12913, upload-time = "2026-01-12T11:25:08.484Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/67/10d236698525d7b7db4d74ec0a4b01f5b2db33968995fdd9ac6b4635e327/torch_c_dlpack_ext-0.1.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:c0f2bd51fcd99c0e5b50314e1985f2728c4941bfa821f065e6c30951d1f995ca", size = 5291237, upload-time = "2026-01-12T11:24:44.011Z" }, - { url = "https://files.pythonhosted.org/packages/87/06/8d760997307a5c3be4384424667bf31aae0a42060838c532c7d846516175/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3562ee411258676f9c38b8ad39306d1c8d027b6a86f6a87c920d2d009a9d1510", size = 443069, upload-time = "2026-01-12T11:24:45.451Z" }, - { url = "https://files.pythonhosted.org/packages/e2/79/a914539b4785f3e44f891aa012a886edb8bc10fe081c440981c57543ce21/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6f9da4bb9af70e27facc777458be62e10dbbbddda7672d16138db0553c5a524", size = 897846, upload-time = "2026-01-12T11:24:48.168Z" }, - { url = "https://files.pythonhosted.org/packages/3a/e6/7d7a97a3953208d6d6ce749180c34d1dab48464ded9a76cecabe9d021ce6/torch_c_dlpack_ext-0.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:670fbbab70123cc228bed41693a3720757af57a0ad22669063c9db25321e8f55", size = 1482855, upload-time = "2026-01-12T11:24:49.581Z" }, -] - -[[package]] -name = "torchaudio" -version = "2.11.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/b1/77658817acacd01a72b714440c62f419efc4d90170e704e8e7a2c0918988/torchaudio-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a1cf1acc883bee9cb906a933572fed6a8a933f86ef34e9ea7d803f72317e8c1b", size = 684226, upload-time = "2026-03-23T18:13:40.023Z" }, - { url = "https://files.pythonhosted.org/packages/78/28/c7adc053039f286c2aca0038b766cbe3294e66fec6b29a820e95128f9ede/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bc653defca1c16154398517a1adc98d0fb7f1dd08e58ced217558d213c2c6e29", size = 1626670, upload-time = "2026-03-23T18:13:42.162Z" }, - { url = "https://files.pythonhosted.org/packages/88/d8/d6d0f896e064aa67377484efef4911cdcc07bce2929474e1417cc0af18c2/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6503c0bdb29daf2e6281bb70ea2dfe2c3553b782b619eb5d73bdadd8a3f7cecf", size = 1771992, upload-time = "2026-03-23T18:13:33.188Z" }, - { url = "https://files.pythonhosted.org/packages/23/a8/941277ecc39f7a0a169d554302a1f1afd87c1d94a8aec828891916cea59a/torchaudio-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:478110f981e5d40a8d82221732c57a56c85a1d5895fb8fe646e86ee15eded3bd", size = 328663, upload-time = "2026-03-23T18:13:19.218Z" }, -] - -[[package]] -name = "torchcodec" -version = "0.17.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/53/33acd60b2880ed4b0d6526436f691a0f447de7b53aceb42eb58a9719e474/torchcodec-0.17.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3d69d30fbd2d5f9cc6d63c6d970acb5fa3aeddb5524a059f0ca538a83972f65c", size = 4219893, upload-time = "2026-09-30T12:52:54.86Z" }, - { url = "https://files.pythonhosted.org/packages/75/20/5692a1177b4be33086770970dcf83effa821376f447899cea3ba850ea2fa/torchcodec-0.17.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:81655cf25e168e74f27aee29db9fc16e37a9f5b2a188a0f7fe25cdfaab2ddd2e", size = 8854331, upload-time = "2026-09-30T12:52:56.445Z" }, - { url = "https://files.pythonhosted.org/packages/05/44/d00a4df9c8985c6ce9e6d1e3f861496cdcde8b5b4b7543358d49e03f7337/torchcodec-0.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:defd5f0901999c9e981f6465a0f4aac8b5a5147e38e539b77c0711c015cf5349", size = 9972897, upload-time = "2026-09-30T12:52:59.059Z" }, - { url = "https://files.pythonhosted.org/packages/f4/e4/0f2a356d6d08b1e2ece096288a02f2978806d077cac7a8340adf312b21ed/torchcodec-0.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:0771bc114ff75a30f0ea8aa82cc7b259c518214eafe73fee2e606d745d77942b", size = 6810114, upload-time = "2026-09-30T12:53:00.966Z" }, -] - -[[package]] -name = "torchvision" -version = "0.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pillow" }, - { name = "torch" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/49/c1cab1ecbb3ff1a380a3f99283db1dee61b8afe354f6352c643b65937130/torchvision-0.28.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:e9f54c30cd52e3ef7fd034cc69b7bb7e0964e1c8f8743e018ab92e95b40f9eee", size = 1856020, upload-time = "2026-07-08T16:07:52.182Z" }, - { url = "https://files.pythonhosted.org/packages/f0/4c/95233776e2def960e5abb7a07931230a545f43717a56a1e1140162033598/torchvision-0.28.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5cf78ebc401ce64ae19b8c55de866bb836797d559a4de9c25ccbe74cfa642d3a", size = 7842127, upload-time = "2026-07-08T16:07:53.446Z" }, - { url = "https://files.pythonhosted.org/packages/93/e4/e9b2495d0d57b9f60d63c57d0a910410a81b4b073bf70917bef815291119/torchvision-0.28.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:028a3d481b37d785605620d7cdad897064c5a55bae2aa1f2658766333e291940", size = 7675040, upload-time = "2026-07-08T16:07:58.017Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9c/55ed9cb6dfe3ee9c837df5cd0e758372e5829aa38b8dd71343aa632cc4e2/torchvision-0.28.0-cp312-cp312-win_amd64.whl", hash = "sha256:87dc16b2df427c1318ad335f1e2be2b3b15b2cf20f7934c83b0505a48425ee5d", size = 4085785, upload-time = "2026-07-08T16:07:50.928Z" }, -] - -[[package]] -name = "tornado" -version = "6.5.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/61/53d562a57b28c08eda40b258c0f975e360541943ad7c7bef897a40caafda/tornado-6.5.10.tar.gz", hash = "sha256:a6b1ccd08c04b4a06fb5aeb381be99de5ad1e5375c1785e31d78c880feb57687", size = 537910, upload-time = "2026-09-15T13:47:48.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/5b/ff5fc58fa2427c30dea74c90053f4fc5eda1e7f3833ed3ecc7147fe2b311/tornado-6.5.10-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9261783640e23258694a9ff0795df430a5a7b0a651d3dd53dd0969ad6be16da7", size = 465883, upload-time = "2026-09-15T13:47:35.463Z" }, - { url = "https://files.pythonhosted.org/packages/ad/f5/cd7be26c34a3315532f3aef5f092465da8f59c334dd439d3c14aaef16461/tornado-6.5.10-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:83e6cf438b106c6b3852d70960967bb1b70c87438050dca0981e4b9aa751a4c1", size = 464046, upload-time = "2026-09-15T13:47:37.178Z" }, - { url = "https://files.pythonhosted.org/packages/60/33/df6d7d04854a58619f8349a51e3edb138324130a7562b0bb21f115bb940f/tornado-6.5.10-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bdf942448169e5336451d0494d7e3d81cfa726d5aa312affdc4682dd62a62f6d", size = 467096, upload-time = "2026-09-15T13:47:38.559Z" }, - { url = "https://files.pythonhosted.org/packages/29/17/cc35dff68272d685cffd8600ffafbd8067e7d05e7348d9f80caddffbbd5f/tornado-6.5.10-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69acca6501eed74582b76dbbceee2a91613f54728e3e418346000d7103101676", size = 468067, upload-time = "2026-09-15T13:47:40.085Z" }, - { url = "https://files.pythonhosted.org/packages/c3/01/6e5349b4e1a53a4b4972a6716785e1fe7407f312063c3972690af8ff301b/tornado-6.5.10-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:66aaa3f57d30c6e6becee83ff28055d5930ac724214bde99393eefda83d5e015", size = 467901, upload-time = "2026-09-15T13:47:41.576Z" }, - { url = "https://files.pythonhosted.org/packages/28/5e/b4facf94370dba006819c8d304376f8b9fbec6b935b5e51bf45823a9790b/tornado-6.5.10-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4bd192b959f9128fb99b8898148070ba4574c9589b78bce42d1851131fe85828", size = 467308, upload-time = "2026-09-15T13:47:43.145Z" }, - { url = "https://files.pythonhosted.org/packages/56/ae/047938e828cafc8eca4c908fafb6588fee944e3af39a0af9d7b602499ae5/tornado-6.5.10-cp39-abi3-win32.whl", hash = "sha256:302eb1e0e3e159314eb591920529fdea80acca92df5510a2cec5bbd4f099ec72", size = 468387, upload-time = "2026-09-15T13:47:44.556Z" }, - { url = "https://files.pythonhosted.org/packages/d8/d4/5901517f05affd752490f6a654ba31b7474664e8dd80bd045a00c220bd88/tornado-6.5.10-cp39-abi3-win_amd64.whl", hash = "sha256:37ae8f150cecfdbf747fc4e12f5e9a97ecd8cf1d4cdb3f119e2de84b11196918", size = 468828, upload-time = "2026-09-15T13:47:45.961Z" }, - { url = "https://files.pythonhosted.org/packages/f3/1a/fd497f3a7f7b74bb04f4b94536b5c9f80742b5d50501fd27977652ddec16/tornado-6.5.10-cp39-abi3-win_arm64.whl", hash = "sha256:ce045d3c298fddd30e89a2777f97039d1b641eb9518ac7b26a4721903539c694", size = 467847, upload-time = "2026-09-15T13:47:47.283Z" }, -] - -[[package]] -name = "tqdm" -version = "4.70.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, -] - -[[package]] -name = "transformers" -version = "5.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "regex" }, - { name = "safetensors" }, - { name = "tokenizers" }, - { name = "tqdm" }, - { name = "typer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, -] - -[[package]] -name = "triton" -version = "3.7.1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/fa/f856e24deb462d5f18bd4b5a746957862ab9b6ee5834bda60605ec348366/triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1", size = 184692359, upload-time = "2026-06-17T20:03:48.288Z" }, - { url = "https://files.pythonhosted.org/packages/c4/6f/fb96d15db6f36d6eae4cafb998c2e0353bf59d7c4ea1662d7497f269134a/triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728", size = 197719725, upload-time = "2026-06-17T19:53:20.419Z" }, -] - -[[package]] -name = "truststore" -version = "0.10.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, -] - -[[package]] -name = "typeguard" -version = "2.13.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3a/38/c61bfcf62a7b572b5e9363a802ff92559cb427ee963048e1442e3aef7490/typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4", size = 40604, upload-time = "2021-12-10T21:09:39.158Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/bb/d43e5c75054e53efce310e79d63df0ac3f25e34c926be5dffb7d283fb2a8/typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1", size = 17605, upload-time = "2021-12-10T21:09:37.844Z" }, -] - -[[package]] -name = "typer" -version = "0.27.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "rich" }, - { name = "shellingham" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, -] - -[[package]] -name = "typing-inspection" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, -] - -[[package]] -name = "tzdata" -version = "2026.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, -] - -[[package]] -name = "uvicorn" -version = "0.51.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a2/65/b7c6c443ccc58678c91e1e973bbe2a878591538655d6e1d47f24ba1c51f3/uvicorn-0.51.0.tar.gz", hash = "sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0", size = 94412, upload-time = "2026-07-08T10:59:05.962Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/ec/dbb7e5a6b91f86bfb9eb7d2988a2730907b6a729875b949c7f022e8b88fa/uvicorn-0.51.0-py3-none-any.whl", hash = "sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b", size = 73219, upload-time = "2026-07-08T10:59:04.44Z" }, -] - -[package.optional-dependencies] -standard = [ - { name = "httptools" }, - { name = "python-dotenv" }, - { name = "pyyaml" }, - { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, - { name = "watchfiles" }, - { name = "websockets" }, -] - -[[package]] -name = "uvloop" -version = "0.22.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, - { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, - { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, - { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, - { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, -] - -[[package]] -name = "vllm" -version = "0.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "anthropic" }, - { name = "apache-tvm-ffi" }, - { name = "blake3" }, - { name = "cachetools" }, - { name = "cbor2" }, - { name = "cloudpickle" }, - { name = "compressed-tensors" }, - { name = "depyf" }, - { name = "einops" }, - { name = "fastapi", extra = ["standard"] }, - { name = "fastsafetensors" }, - { name = "filelock" }, - { name = "flashinfer-python" }, - { name = "huggingface-hub" }, - { name = "humming-kernels", extra = ["cu13"] }, - { name = "ijson" }, - { name = "jsonschema" }, - { name = "lark" }, - { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, - { name = "lm-format-enforcer" }, - { name = "mcp" }, - { name = "mistral-common", extra = ["image"] }, - { name = "model-hosting-container-standards" }, - { name = "msgspec" }, - { name = "ninja" }, - { name = "numba" }, - { name = "numpy" }, - { name = "nvidia-cudnn-frontend" }, - { name = "nvidia-cutlass-dsl", extra = ["cu13"] }, - { name = "nvtx" }, - { name = "openai" }, - { name = "openai-harmony" }, - { name = "opencv-python-headless" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp" }, - { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions-ai" }, - { name = "outlines-core" }, - { name = "partial-json-parser" }, - { name = "pillow" }, - { name = "prometheus-client" }, - { name = "prometheus-fastapi-instrumentator" }, - { name = "protobuf" }, - { name = "psutil" }, - { name = "py-cpuinfo" }, - { name = "pybase64" }, - { name = "pydantic" }, - { name = "pynvvideocodec" }, - { name = "python-json-logger" }, - { name = "pyyaml" }, - { name = "pyzmq" }, - { name = "quack-kernels" }, - { name = "regex" }, - { name = "requests" }, - { name = "safetensors" }, - { name = "sentencepiece" }, - { name = "setproctitle" }, - { name = "setuptools" }, - { name = "six" }, - { name = "starlette" }, - { name = "tiktoken" }, - { name = "tilelang" }, - { name = "tokenizers" }, - { name = "tokenspeed-mla", marker = "sys_platform == 'linux'" }, - { name = "torch" }, - { name = "torchaudio" }, - { name = "torchcodec" }, - { name = "torchvision" }, - { name = "tqdm" }, - { name = "transformers" }, - { name = "typing-extensions" }, - { name = "watchfiles" }, - { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/39/096d88cfe34ee43a8b6e7c957d1ad6338fa5256474d6ff9a32008c6f8cd6/vllm-0.28.0.tar.gz", hash = "sha256:ac96dd0ec5be9c13f2aa4bfb50498c4727110406f6e4980b58a4097e4d18634a", size = 39991441, upload-time = "2026-08-26T10:08:30.56Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/31/007ac11dc756f7e69600b90b08e39f573eed9aaee6258198f9d9a2b06355/vllm-0.28.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:817b8181f7f61b4a62dc1d5d9ab39f2bfb60a6cb86c29879a78a147b85756787", size = 308873410, upload-time = "2026-08-26T10:08:13.738Z" }, - { url = "https://files.pythonhosted.org/packages/87/d7/97f6ecc2ae883e601e08d7cef87cb54ececeefcfe6b5e12d5d92f8d06d6b/vllm-0.28.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:addb0ffdaafd8155d75e9b3f5ddb3da28fdee9e8a7097ede91f7db2e9e1a3889", size = 314500103, upload-time = "2026-08-26T10:07:25.091Z" }, -] - -[package.optional-dependencies] -otel = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp" }, - { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions-ai" }, -] -runai = [ - { name = "runai-model-streamer", extra = ["azure", "gcs", "s3"] }, -] - -[[package]] -name = "wadler-lindig" -version = "0.1.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1e/67/cbae4bf7683a64755c2c1778c418fea96d00e34395bb91743f08bd951571/wadler_lindig-0.1.7.tar.gz", hash = "sha256:81d14d3fe77d441acf3ebd7f4aefac20c74128bf460e84b512806dccf7b2cd55", size = 15842, upload-time = "2025-06-18T07:00:42.843Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/96/04e7b441807b26b794da5b11e59ed7f83b2cf8af202bd7eba8ad2fa6046e/wadler_lindig-0.1.7-py3-none-any.whl", hash = "sha256:e3ec83835570fd0a9509f969162aeb9c65618f998b1f42918cfc8d45122fe953", size = 20516, upload-time = "2025-06-18T07:00:41.684Z" }, -] - -[[package]] -name = "watchfiles" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/2f/e42c992d2afda3108ea1c02acecc991b9f31d05c14adc2a7cee9ee211fc4/watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26", size = 400115, upload-time = "2026-05-18T04:32:02.06Z" }, - { url = "https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c", size = 393659, upload-time = "2026-05-18T04:30:50.951Z" }, - { url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" }, - { url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" }, - { url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" }, - { url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" }, - { url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" }, - { url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" }, - { url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" }, - { url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" }, - { url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" }, - { url = "https://files.pythonhosted.org/packages/fa/08/d9e2e0f9e8e6791d33aefc694ad7eefa7f901f63caff84a81ded38692f9c/watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898", size = 275480, upload-time = "2026-05-18T04:30:31.307Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e6/9d42569c0102645cc8cea5d8c7d8a1e9d4ada2cb7f05f75e554b8aa2202a/watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379", size = 288718, upload-time = "2026-05-18T04:32:10.745Z" }, - { url = "https://files.pythonhosted.org/packages/0a/26/88e0dc6ee3898169d7fa22bb6a69cabf2502d2ee25cb8c876d1262d204f8/watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5", size = 281026, upload-time = "2026-05-18T04:30:22.23Z" }, -] - -[[package]] -name = "wcwidth" -version = "0.9.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f0/b4/7830542634bb2d3e62aa3b586a72d5b3b6c91c3168929e7000ef3fed041d/wcwidth-0.9.2.tar.gz", hash = "sha256:ae0ef90b90f6af38b54f1fe6d58662ec33b3cb4b8391958a62416d654231727b", size = 955039, upload-time = "2026-10-05T00:24:05.521Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/1e/4532a81fb9dfbf4114a816775e0a36c3a64ee1d1f4bba2094e2da50be5dc/wcwidth-0.9.2-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:7ef5a940bd5e30bac6e721f1a48fce0cd7bb3ece19e9c5d139e72c76c35cfd07", size = 605343, upload-time = "2026-10-05T00:23:22.649Z" }, - { url = "https://files.pythonhosted.org/packages/a0/07/cb6940e81134b7ed25fa312ee9ab536a63db0793b149f88a90e603ceace9/wcwidth-0.9.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ae0800c5339423cc53d33a266ad264b42ba8aaa16d4464f6e6b1bee607f50b17", size = 609960, upload-time = "2026-10-05T00:23:27.049Z" }, - { url = "https://files.pythonhosted.org/packages/a4/80/15ad05d40bfa99155639fb9e13b3d77083aa0fab893c816db2543d29005c/wcwidth-0.9.2-cp310-abi3-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:9e542f1f8475b78452a295495d7a5bc3ead565112e9446a64dc93462a41c2a79", size = 762566, upload-time = "2026-10-05T00:23:38.322Z" }, - { url = "https://files.pythonhosted.org/packages/bc/f0/b8ef7758003d66b60f093695831a86dcc726aac01ee6446ffcbda27b61e3/wcwidth-0.9.2-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:674b518af28d38ee645ff97b74f5760abee5fad4bac74413bfc4b881ef2ce724", size = 771632, upload-time = "2026-10-05T00:23:32.448Z" }, - { url = "https://files.pythonhosted.org/packages/db/6c/f940133c71427c208575910e981942bd78c98b1f7cd0d1425ca4b7457c04/wcwidth-0.9.2-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:751bef0ab404b6a1dc028b56b4b85d46486be1c55833f80da533e42dc691f389", size = 769763, upload-time = "2026-10-05T00:23:40.175Z" }, - { url = "https://files.pythonhosted.org/packages/92/8f/285f862826f721964ec7c42f81dc53d23afbd723a0f4cd989651f8218e25/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c3d80f39ba4653a595edae9aa46a509d14883790a8fc23c5db221ceb207f64b7", size = 782134, upload-time = "2026-10-05T00:23:33.926Z" }, - { url = "https://files.pythonhosted.org/packages/c2/2d/64aa54882a5d556d3654c1f926d9118b797461033e23a158409941a37c8f/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:0a47e03d8293590ecce66c45dc20ff7b4b885e3c78093722239585eca0d77ab2", size = 782851, upload-time = "2026-10-05T00:23:41.974Z" }, - { url = "https://files.pythonhosted.org/packages/59/39/52389f6de7fe2e9c14ceb8253dd99034bd86e1c87847ea3c100a97dded9a/wcwidth-0.9.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:67d901a4ad99249eb775b4ee4769ca97fa405d35a75f46e83166910a47003f04", size = 784621, upload-time = "2026-10-05T00:23:43.449Z" }, - { url = "https://files.pythonhosted.org/packages/b3/8b/20225500a076ace27bbcc8a6fd7c55125133c57a618816c7b7b8b73070b1/wcwidth-0.9.2-cp310-abi3-win32.whl", hash = "sha256:ee1fd0db9d9fd711a70f3e7765e0e04c05d26982fa05361456163062549d7da4", size = 593123, upload-time = "2026-10-05T00:23:55.953Z" }, - { url = "https://files.pythonhosted.org/packages/5a/d6/b0690f55ea0483530a18bac917fbadbf54f35122510446fc370f5f1c2453/wcwidth-0.9.2-cp310-abi3-win_amd64.whl", hash = "sha256:2a9746de704242bd4fdaabb31dd46b82f694a56a8d21081ad89b679a89da9fec", size = 597827, upload-time = "2026-10-05T00:23:57.489Z" }, - { url = "https://files.pythonhosted.org/packages/e5/11/6ecf4e9e268ab1a4ec617ffcccc2ee4a71301625f5490912dbaba462fa9c/wcwidth-0.9.2-cp310-abi3-win_arm64.whl", hash = "sha256:b9c6ab615e03723b7f8760ea2f27758d656e7e13b51515c9dca5c3e8b04612fa", size = 598415, upload-time = "2026-10-05T00:23:51.517Z" }, - { url = "https://files.pythonhosted.org/packages/9c/b4/0bfa065af506540d9d558e3e5548cff00bc1f9b24e6e2a8512498e8628de/wcwidth-0.9.2-py3-none-any.whl", hash = "sha256:89ca642c5bf0101157a09366be69fad0379db1f700ae39a920e103234573670e", size = 301667, upload-time = "2026-10-05T00:23:21.097Z" }, -] - -[[package]] -name = "websocket-client" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, -] - -[[package]] -name = "websockets" -version = "16.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/f7/bc3a25c5ec26ce62ce487690becc2f3710bbc7b33338f005ad390db0b986/websockets-16.1.1.tar.gz", hash = "sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57", size = 182204, upload-time = "2026-07-17T22:51:05.858Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/9d/681cda21c9eee743203a6cb79b9d3d05adad9aa60ec660c6c9bf4dd619ca/websockets-16.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00", size = 179600, upload-time = "2026-07-17T22:49:13.92Z" }, - { url = "https://files.pythonhosted.org/packages/fb/8d/6195a88b45e8d2a8f745fc2046e36f885a3c9763e6767d2c46229bf9510c/websockets-16.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b", size = 177272, upload-time = "2026-07-17T22:49:15.453Z" }, - { url = "https://files.pythonhosted.org/packages/73/e3/fe2d498c64dea0095c9a9f9a351af4cd6eef31b618395582bc1f38ba45ff/websockets-16.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175", size = 177542, upload-time = "2026-07-17T22:49:16.875Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ed/f1831681fce0e3242346e5458486003c5f124ed69e5e0b847fd029db4973/websockets-16.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1", size = 187137, upload-time = "2026-07-17T22:49:18.323Z" }, - { url = "https://files.pythonhosted.org/packages/6f/79/4ff9dcc1bb46f6b4c536936dde1fd60f9b564f3304307274db97f4c9496d/websockets-16.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15", size = 188374, upload-time = "2026-07-17T22:49:19.65Z" }, - { url = "https://files.pythonhosted.org/packages/62/c3/5c49b6efb36cab733d23773f6de575e1dba65736ead17d5d2b2a1daef779/websockets-16.1.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa", size = 191155, upload-time = "2026-07-17T22:49:21.331Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f6/56ccceda3a4838d18f1d40821480da4775397e8b1eecf4031e20c50e2e90/websockets-16.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab", size = 189011, upload-time = "2026-07-17T22:49:22.889Z" }, - { url = "https://files.pythonhosted.org/packages/86/d6/ad5286241a2bce1107e2798d3bfbd62cf79aee167bdb654f8cb1e9dbf949/websockets-16.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847", size = 187766, upload-time = "2026-07-17T22:49:24.339Z" }, - { url = "https://files.pythonhosted.org/packages/bc/67/d65c970b7e347fdca69479beb7811c2060529956730a7a4e3ae7c66b0e31/websockets-16.1.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428", size = 185173, upload-time = "2026-07-17T22:49:25.743Z" }, - { url = "https://files.pythonhosted.org/packages/1d/5b/14af3cd4ee69d8ea9baca58f3dc3cfb1ba78332a347fd478cb096549d60e/websockets-16.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf", size = 187809, upload-time = "2026-07-17T22:49:27.147Z" }, - { url = "https://files.pythonhosted.org/packages/7b/11/be301710d70de97e3e7b3586e6d492c9c06d6a61bf1c2202c36cf0c75607/websockets-16.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751", size = 186412, upload-time = "2026-07-17T22:49:28.611Z" }, - { url = "https://files.pythonhosted.org/packages/db/07/fe1435bf6fe738a3d3b54dbe0c18dabf12cba4d909ac8b58b539ce27c1f4/websockets-16.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f", size = 188290, upload-time = "2026-07-17T22:49:29.965Z" }, - { url = "https://files.pythonhosted.org/packages/8a/0a/81f394aff8efcbb01208c1ced77df0a3c7fcce584a88c7273663697946c2/websockets-16.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2", size = 185844, upload-time = "2026-07-17T22:49:31.447Z" }, - { url = "https://files.pythonhosted.org/packages/39/5c/dd485b995473f415510251fe9bd708f2d24458f439fce958daf8d66dc7c6/websockets-16.1.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383", size = 186823, upload-time = "2026-07-17T22:49:33.104Z" }, - { url = "https://files.pythonhosted.org/packages/9d/0b/f78de76ff446f1e66af12b43c48a35f31744de93cfdec2f4ea67d5d7bbf1/websockets-16.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3", size = 187102, upload-time = "2026-07-17T22:49:34.616Z" }, - { url = "https://files.pythonhosted.org/packages/37/a1/4cf892007778eaf84ad162bfc98046e0ed89b63ac55949e3236626b2a23f/websockets-16.1.1-cp312-cp312-win32.whl", hash = "sha256:1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747", size = 179943, upload-time = "2026-07-17T22:49:36.213Z" }, - { url = "https://files.pythonhosted.org/packages/d9/de/6abe251d28c3a3f217096575400b27750b18e0b1d2fff3a2a239960fea07/websockets-16.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7", size = 180243, upload-time = "2026-07-17T22:49:37.626Z" }, - { url = "https://files.pythonhosted.org/packages/be/4d/2d0d67834092e354d2b0498f014a41249a89556bc406cf86f3e1557bb463/websockets-16.1.1-py3-none-any.whl", hash = "sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3", size = 173814, upload-time = "2026-07-17T22:51:04.184Z" }, -] - -[[package]] -name = "win32-setctime" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, -] - -[[package]] -name = "wrapt" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3e/d2/a254a26d8ceaea87e0eee2e89fcfe53ddc1858418647493bb2937549ab6f/wrapt-2.5.0.tar.gz", hash = "sha256:c48cdb6c904dca76d9915a579e4a5fab6b0c25f650c1019ce78a78effaf7a345", size = 183288, upload-time = "2026-09-27T01:41:56.874Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/a6/44589f9b34160280a1fbccbcf206b18df034a1b3e9584336d3a7f039a33c/wrapt-2.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b898caea081303006decc562c7fca5126f7c96507e78dd8f1ae3285dfa50ddc7", size = 106192, upload-time = "2026-09-27T01:40:00.941Z" }, - { url = "https://files.pythonhosted.org/packages/ce/2b/94db3ba2e9528400e4173cbb67a56822a4644d4c2b319feb0cc638094b36/wrapt-2.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8837fbe708cb9d8a2d32a37dee836d24a531f02560db26418e2b181986fa21cb", size = 105828, upload-time = "2026-09-27T01:40:02.494Z" }, - { url = "https://files.pythonhosted.org/packages/11/0e/3ce67af67525c0068680637c0eba823ce2f0fc6eff2d1779cab656fb8c3d/wrapt-2.5.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cabb9c17ab79b2549d1f23b36f436473ad9253ef53995a817feba26fae69d5b", size = 250177, upload-time = "2026-09-27T01:40:03.819Z" }, - { url = "https://files.pythonhosted.org/packages/d3/3b/262b2c3c38aca6fa32cfea6407dbb346ce26ff788179b98137b3b3f3d8ca/wrapt-2.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6761765cc520ff9616fb035c02a85d1d744f7f70edd4649718fd0d09c589eacf", size = 250332, upload-time = "2026-09-27T01:40:05.15Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ec/d54d273a2223d1964ecd5b5954aca18e8fbf2c5d8471e60ef3cbaaed9325/wrapt-2.5.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a145a7826eddea3eb5814903f98f93756042b919bb5305544cb1331daa2705b1", size = 229600, upload-time = "2026-09-27T01:40:06.47Z" }, - { url = "https://files.pythonhosted.org/packages/8e/88/33c75ac47b13b0bf77b44d96d02b1d0a2e6db66180909bacce86ef8fcd8e/wrapt-2.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6a9ee62a970075738909909bdbef3a7da9f7ae03dfca584db283547a29503b56", size = 246890, upload-time = "2026-09-27T01:40:07.838Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0e/c3a3a158801e5bee7a0d9aa3561a127bd80f166b02b7b520a39e45d8f9b9/wrapt-2.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:691671ea05684f921ffc2e935fd3f9311c1795a10fbbfa006b46269733668f66", size = 226463, upload-time = "2026-09-27T01:40:09.206Z" }, - { url = "https://files.pythonhosted.org/packages/7c/2c/4c48ba51698a87e2e8299fc04856bbef391cb725f195bc93f0bc6196e1f2/wrapt-2.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e716f47c7f61e11709d3c0904213c94fc22999abf0c41461276cef886c1e8b4d", size = 238583, upload-time = "2026-09-27T01:40:10.631Z" }, - { url = "https://files.pythonhosted.org/packages/33/d2/077835618ed96b131730f74301a7236efd841ec2f341afe36303389b56ef/wrapt-2.5.0-cp312-cp312-win32.whl", hash = "sha256:5421acb5c363a9bc959122a8645e3f1f42010c932dc53885b11a5ff5b5a6d730", size = 100221, upload-time = "2026-09-27T01:40:11.91Z" }, - { url = "https://files.pythonhosted.org/packages/96/9f/9e56db2a3492275be809082c6f63a7a063f52b3e1812aae1eb03ec33f64c/wrapt-2.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:ab45839c912777e2738fed369636589c8b2a6d9c44ca56de0fd0814581d467c2", size = 105786, upload-time = "2026-09-27T01:40:13.208Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e4/e37ff75e5254564aa50e13bc4313383d02d5ec78f5bd3b9b1994dd7052d5/wrapt-2.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:ce4cab32c37ef71e69cf88f909b7febd0dd79543e5ae3650e2b874e0f3d3b975", size = 102937, upload-time = "2026-09-27T01:40:14.534Z" }, - { url = "https://files.pythonhosted.org/packages/87/7d/5ed859fad4b5eddd598a846150aaab2703730ed4886c5c5e03b0df0cfdd5/wrapt-2.5.0-py3-none-any.whl", hash = "sha256:107eea1a511e98a3a5033b0c2cb403fbb37f05dee6ac1fb85c0460d311ec278c", size = 81586, upload-time = "2026-09-27T01:41:55.479Z" }, -] - -[[package]] -name = "xgrammar" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "apache-tvm-ffi" }, - { name = "numpy" }, - { name = "pydantic" }, - { name = "torch" }, - { name = "transformers" }, - { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/81/f4/e71693d8cec60b7e36dab660784ecc5a6aa51e478a83b556011645c58c87/xgrammar-0.2.3.tar.gz", hash = "sha256:f76423630ae3ac4e090cb38ce1e30e7bcc69b3dee4d22d94353944386a4c6f18", size = 2447704, upload-time = "2026-06-27T04:45:24.759Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/1c/0cdb22fc799e6d158b3243eeb895ae2e086825487b57767838c98d4864ee/xgrammar-0.2.3-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:173e167d43a5cf4171eee2be86097decff8803b0a0853d7baaf446c732a7d3a9", size = 23284489, upload-time = "2026-06-27T04:44:23.927Z" }, - { url = "https://files.pythonhosted.org/packages/cf/bc/994dc6f222189174840c29a1f5b4c175e69dfe13ed2e25b6dbbe9f200a29/xgrammar-0.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aa35f24835a59c822e249ecc80912eea4de03fc8b04afb2f82c8b950a56be6ef", size = 23240027, upload-time = "2026-06-27T04:44:26.255Z" }, - { url = "https://files.pythonhosted.org/packages/e4/79/0bb37937bf847c738c64b64dc50ddc12e7c526b34c5ab82cebe58da5ec8f/xgrammar-0.2.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11255f184971489fc72b948b096e2917f482ba2dca975177f5411562cedb9c6d", size = 44314481, upload-time = "2026-06-27T04:44:28.875Z" }, - { url = "https://files.pythonhosted.org/packages/d4/fd/5ebd5d14b8993cb225151bbb8f2011742fc7a7d94a3bdbc3ec3954b9b62d/xgrammar-0.2.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdf081fab29694302d41d61dcf52fad7d253879a718bc6afc68db0a0dabd7f19", size = 44855110, upload-time = "2026-06-27T04:44:31.586Z" }, - { url = "https://files.pythonhosted.org/packages/17/66/67239f43b0244f65aec4639f51ab95905db42eb66e532ee2a4e5cdce32de/xgrammar-0.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:e7787dd8321a04f86116b756aa3dadd622e3607a3559b1e986cc5f77da00d68e", size = 15780277, upload-time = "2026-06-27T04:44:34.081Z" }, -] - -[[package]] -name = "xyzservices" -version = "2026.9.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c4/bd/b75d5c2312caec607ec995200fa1eb8453225fd869947ec743919c279544/xyzservices-2026.9.1.tar.gz", hash = "sha256:8d1a39bf6b192940cc5db5264eeefc1b20dd184f8b83b1303b078743744f5943", size = 1134047, upload-time = "2026-09-04T10:35:41.399Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/71/9219fe65ef32bddc0b5d14e1d98dfefe9d2c9c1b1d1e93efca1f29fcbd4d/xyzservices-2026.9.1-py3-none-any.whl", hash = "sha256:af4fddac0f1fa5f7951834e2c58e10109e722be548cb1fe4f1a8e8b18cef4c09", size = 98829, upload-time = "2026-09-04T10:35:40.08Z" }, -] - -[[package]] -name = "yarl" -version = "1.24.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "multidict" }, - { name = "propcache" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/31/33/ebe9e3d1f86c7a0b51094c0a146392045ca1631d2664889539dec8088a33/yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f", size = 228679, upload-time = "2026-07-20T02:07:45.435Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/84/71d051c850b5af41d168c679d9eb67eb7c55283ac4ee131673edf134bc4e/yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d", size = 136035, upload-time = "2026-07-20T02:05:25.489Z" }, - { url = "https://files.pythonhosted.org/packages/03/4d/8ad27f9a1b7e69313cca5d695b925b48efe51208d3490e0844bae97cabc0/yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec", size = 97642, upload-time = "2026-07-20T02:05:27.429Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b4/05b4131c407006cd1e410e9c6539f16a0945724677e5364447313c15ea3e/yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c", size = 97323, upload-time = "2026-07-20T02:05:29.441Z" }, - { url = "https://files.pythonhosted.org/packages/20/16/e618c875c73e0e39611f20a581b3d5e8d59b8857bf001bee3263044c6deb/yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54", size = 107741, upload-time = "2026-07-20T02:05:31.367Z" }, - { url = "https://files.pythonhosted.org/packages/d9/9a/c4defeaf3ed33fcb346aacf9c6e971a8d4e2bde04a0310e79abb208e7965/yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12", size = 103570, upload-time = "2026-07-20T02:05:33.303Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e7/0e0e0de5865ebd5914537ef486f36c727a59865c3ac0cf5ff1b32aececbf/yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d", size = 115815, upload-time = "2026-07-20T02:05:35.292Z" }, - { url = "https://files.pythonhosted.org/packages/2b/27/ca56b700cb170aba25a3893b75355b213935657dc5714d2383354a270e62/yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1", size = 116025, upload-time = "2026-07-20T02:05:37.503Z" }, - { url = "https://files.pythonhosted.org/packages/d6/d0/d56c859b8222116f5d68459199f48359e0bf121b6f65a69bf329b3602ba0/yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9", size = 109835, upload-time = "2026-07-20T02:05:39.506Z" }, - { url = "https://files.pythonhosted.org/packages/70/a2/3a35557e4d1a79425040eba202ccaf08bdc8717680fc77e2498a1ad2e0a5/yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027", size = 108884, upload-time = "2026-07-20T02:05:41.584Z" }, - { url = "https://files.pythonhosted.org/packages/e4/35/ef4c26356b7913c68983bac2d72a4212b3347af551cb8d250b99b5ed7b7f/yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b", size = 107308, upload-time = "2026-07-20T02:05:43.697Z" }, - { url = "https://files.pythonhosted.org/packages/d5/91/ff0dc66c2ccf3e0153ab97ff61eabab4400e6a5264af427ab30cd69f1857/yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293", size = 103646, upload-time = "2026-07-20T02:05:45.895Z" }, - { url = "https://files.pythonhosted.org/packages/74/f0/33b9271c7f881766359d58266fa0811d2e5210ed860e28da7dc6d7786344/yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e", size = 115305, upload-time = "2026-07-20T02:05:47.832Z" }, - { url = "https://files.pythonhosted.org/packages/ef/65/fd79fb1868c4a80db8661091de525bf430f63c3bea1b20e8b6a84fc7d359/yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b", size = 108404, upload-time = "2026-07-20T02:05:49.604Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ba/dbabe6b262f17a816c70cfc09558dbf03ece3ec76684d02f911a3d3a189c/yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce", size = 115940, upload-time = "2026-07-20T02:05:51.741Z" }, - { url = "https://files.pythonhosted.org/packages/a5/43/fab2d1dad9d340a268cdde63756a123d069723efff6a372d123fa74a9517/yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba", size = 110006, upload-time = "2026-07-20T02:05:53.554Z" }, - { url = "https://files.pythonhosted.org/packages/c4/27/41eb51bbd1b8d89546b83897cfb0164f1e109304fd408dbb151b639eec0f/yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b", size = 97618, upload-time = "2026-07-20T02:05:55.57Z" }, - { url = "https://files.pythonhosted.org/packages/3c/25/b2553764b3d65db711d8f45416351ec4f420847558eb669edcbcaadf5780/yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c", size = 93018, upload-time = "2026-07-20T02:05:57.554Z" }, - { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" }, -] - -[[package]] -name = "z3-solver" -version = "4.15.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/8e/0c8f17309549d2e5cde9a3ccefa6365437f1e7bafe71878eaf9478e47b18/z3_solver-4.15.4.0.tar.gz", hash = "sha256:928c29b58c4eb62106da51c1914f6a4a55d0441f8f48a81b9da07950434a8946", size = 5018600, upload-time = "2025-10-29T18:12:03.062Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/33/a3d5d2eaeb0f7b3174d57d405437eabb2075d4d50bd9ea0957696c435c7b/z3_solver-4.15.4.0-py3-none-macosx_13_0_arm64.whl", hash = "sha256:407e825cc9211f95ef46bdc8d151bf630e7ab2d62a21d24cd74c09cc5b73f3aa", size = 37052538, upload-time = "2025-10-29T18:11:46.233Z" }, - { url = "https://files.pythonhosted.org/packages/47/84/fd7ffac1551cd9f8d44fe41358f738be670fc4c24dfd514fab503f2cf3e7/z3_solver-4.15.4.0-py3-none-macosx_13_0_x86_64.whl", hash = "sha256:00bd10c5a6a5f6112d3a9a810d0799227e52f76caa860dafa5e00966bb47eb13", size = 39807925, upload-time = "2025-10-29T18:11:49.81Z" }, - { url = "https://files.pythonhosted.org/packages/21/c9/bb51a96af0091324c81b803f16c49f719f9f6ea0b0bb52200f5c97ec4892/z3_solver-4.15.4.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e103a6f203f505b8b8b8e5c931cc407c95b61556512d4921c1ddc0b3f41b08e", size = 29268352, upload-time = "2025-10-29T18:11:53.032Z" }, - { url = "https://files.pythonhosted.org/packages/bf/2e/0b49f7e4e53817cfb09a0f6585012b782dfe0b666e8abefcb4fac0570606/z3_solver-4.15.4.0-py3-none-manylinux_2_34_aarch64.whl", hash = "sha256:62c7e9cbdd711932301f29919ad9158de9b2f58b4d281dd259bbcd0a2f408ba1", size = 27226534, upload-time = "2025-10-29T18:11:55.59Z" }, - { url = "https://files.pythonhosted.org/packages/26/91/33de49538444d4aafbe47415c450c2f9abab1733e1226f276b496672f46c/z3_solver-4.15.4.0-py3-none-win32.whl", hash = "sha256:be3bc916545c96ffbf89e00d07104ff14f78336e55db069177a1bfbcc01b269d", size = 13191672, upload-time = "2025-10-29T18:11:58.424Z" }, - { url = "https://files.pythonhosted.org/packages/03/d6/a0b135e4419df475177ae78fc93c422430b0fd8875649486f9a5989772e6/z3_solver-4.15.4.0-py3-none-win_amd64.whl", hash = "sha256:00e35b02632ed085ea8199fb230f6015e6fc40554a6680c097bd5f060e827431", size = 16259597, upload-time = "2025-10-29T18:12:01.14Z" }, -] - -[[package]] -name = "zipp" -version = "4.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/23/655a1802fe8041302c959774ca7c80b53bc24737ff3ef45cb50ef11bd96c/zipp-4.1.1.tar.gz", hash = "sha256:7ebb7a44c021b29fd8dbd7cce6812d0d7b5b454521f93cc71af6ccd155aaa70b", size = 27649, upload-time = "2026-10-03T17:03:03.452Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/98/df615823cd9419131ce19fba00de53a663794369e198aade064a244b385d/zipp-4.1.1-py3-none-any.whl", hash = "sha256:8979f52d874162f485ff2981e3891f3a3317b7a3dd43ff1e1775b9304f307a9c", size = 10582, upload-time = "2026-10-03T17:03:02.506Z" }, -] - -[[package]] -name = "zstandard" -version = "0.25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, - { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, - { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, - { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, - { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, - { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, - { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, - { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, - { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, - { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, - { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, - { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, - { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, - { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, - { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, - { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, - { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, -] diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 955d0d07b49..8aa388e138d 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -1,63 +1,106 @@ -# Managed Dynamo generation design - -The managed Dynamo backend owns a fixed vLLM fleet inside the Ray allocation. -It is deliberately narrower than Dynamo itself: there is no external-runtime, -Kubernetes, DGD, multi-node engine-group, or non-vLLM mode. - -## Ownership and placement - -Constructing `ManagedDynamoRuntime` is inert. Its explicit `start()` method -allocates ports, launches etcd and NATS JetStream, creates one Ray-managed -`dynamo.vllm` process per model-parallel group, and starts the frontend. A -worker group must fit on one node. Its world size is derived from vLLM tensor -parallelism times pipeline parallelism; expert parallelism must be either one -or equal to tensor parallelism. - -Startup completes only after the frontend sees the same fixed membership at -the generation and RL endpoints and advertises the configured model. Worker -handles are recorded before readiness checks so partial startup failures can -be torn down. Shutdown is idempotent and guards the frontend, worker pool, -NATS, etcd, and temporary state independently. - -## Generation state - -Both GRPO trainers use `DynamoGeneration.generate_async()` against the managed -frontend. NeMo-Gym traffic passes through a process-local token wrapper. It -uses the policy tokenizer, preserves caller `nvext.extra_fields`, adds Dynamo -engine metadata, and translates rendered multi-turn prefixes back to the exact -caller token IDs. - -Serialized rollout copies contain only frontend URLs, immutable worker admin -endpoints, and the prepared refit metadata. They cannot own or stop services. -That is enough for the SingleController actor to issue weight updates and for -AREAL-style post-refit cache invalidation; Magistral keeps its existing -driver-side invalidation lifecycle. - -## Weight refit - -Dynamo uses `CollectiveWeightSynchronizer`. If each engine has world size `E`, -worker `i` starts at rank `training_world_size + i * E`. The policy sender uses -vLLM's peer initialization and a packed transfer with two 1-GiB buffers. vLLM -reads the packing mode and geometry once, from the init info that NeMo-RL sends -to `init_weight_transfer_engine`; each update round carries only tensor names, -dtypes, and shapes. Before a worker starts, the isolated vLLM environment checks -that its weight-transfer API accepts that init and update shape. - -Generation is drained before refit. The worker then runs vLLM's native -`start_weight_update`, `update_weights`, and `finish_weight_update` transaction. -KV-cache invalidation stays outside the generic synchronizer because GRPO's -cache mode determines where it runs. - -## Dependency isolation - -`BUILD_DYNAMO=1` adds a Python 3.12 `/opt/dynamo_venv` to the standard image. -It contains only `ai-dynamo[vllm]==1.5.1`, its pinned vLLM 0.28.0, etcd, -and NATS. NeMo-RL's normal Ray and engine environments are unchanged; the -standard NeMo-RL vLLM environment currently uses vLLM 0.29.0. - -Layerwise reload depends on vLLM PR #44814, which fixes reload accounting for -composed loaders. vLLM 0.28.0 includes it, and the installer fails if the -installed vLLM does not. - -See [Managed Dynamo generation on Slurm](../guides/dynamo-generation.md) for -build, configuration, and launch instructions. +# Dynamo integration design + +`policy.generation.dynamo: true` serves NeMo RL's own vLLM engines through +Dynamo's frontend and KV-aware router. Two requirements shape the design: + +- A vLLM bump must never be blocked by Dynamo. +- Dynamo CI coverage must stay consistent with native vLLM's. + +Both hold only if the engine process contains no Dynamo-owned vLLM code. Then a +vLLM bump can break Dynamo serving only where it also breaks native vLLM. + +## Engine hosting + +`VllmGeneration` builds its workers exactly as without Dynamo. Each data-parallel +leader (`VllmAsyncGenerationWorkerImpl`) owns an `AsyncLLM`. With Dynamo on, it +also runs `dynamo.common.backend.Worker` as a task on its own actor event loop, +which is the loop that owns the `AsyncLLM`. + +`dynamo.common.backend` is Dynamo's engine-agnostic backend SDK and imports no +vLLM. The adapter, `nemo_rl/models/generation/dynamo/native_engine.py`, +implements its `LLMEngine` over the worker's `AsyncLLM`: + +- `generate` maps Dynamo's token request onto vLLM `SamplingParams` the way + vLLM's OpenAI server does, and streams token deltas back. Rollouts that NeMo RL + itself sends carry an annotation. For those, the engine uses the same code as + native `generate_async`: + - the worker's `_rollout_max_new_tokens` and `_rollout_sampling_params` build + the request; + - `request_pipeline.rollout_result` builds the result, routed experts + included. + + The result travels in `engine_data`, and the driver assembles the same + output with `request_pipeline.rollout_output`. + + NeMo-Gym's calls also carry an annotation (Gym's Dynamo dialect). The engine + applies the native chat endpoint's rules to them: the on-policy sampling check + (`vllm/utils.off_policy_sampling_error`), the prefix splice + (`openai_server_utils.splice_prefix_tokens`) and the context clamp. A + token-capture call stages through the worker's own capture lifecycle + (`_begin_request_capture` / `_finish_request_capture`), and the token fields + and routed experts come from the native endpoint's helpers + (`chat_generation_tokens`, `chat_routed_experts`). + + A request with media carries the frontend's tokens, with one unexpanded + placeholder per item, and the media URLs. The adapter loads the media with + vLLM's media connector. It then builds the processed prompt with the engine's + renderer (`process_for_engine_async`), the same step that follows rendering + on the native endpoint. The splice, the placeholder remap + (`remap_multimodal_placeholders`) and media capture (`_capture_request_media`) + all run on that processed prompt. +- `abort` cancels the vLLM request on client disconnect. +- Registration advertises the context length, the KV block size (from vLLM's + KV-cache group metadata), the block count and the scheduler limits. It + publishes no token budget. The engine owns the context check and the output + cap, because a prefix splice changes the prompt after the frontend has seen it. +- `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ + publisher. It is set only when the engine caches prefixes, and it binds a + wildcard endpoint, because vLLM binds only wildcard endpoints. A vLLM stat + logger publishes the router's per-engine load signal. + +Because the Worker runs on the actor's own loop, the worker's other methods +interleave with Dynamo requests exactly as with native requests: +- refit (every transport); +- `pause_generation_async` / `resume_generation_async`. + +Colocated generation is rejected at startup: a sleeping engine stays +registered, so the frontend would keep routing to it. + +No HTTP control routes are involved. The Rust Worker installs a SIGTERM handler +that chains to Ray's, so actor shutdown is unchanged, and discovery drops an +engine when its process exits. + +## Driver + +`VllmGeneration` starts `ManagedDynamoRuntime`, which runs etcd, NATS and +`dynamo.frontend`. After the engines load, it asks every DP leader to register +and waits until the frontend routes to all of them. Each poll also asks the +leaders whether their endpoint task stopped, so a failed registration fails +with its cause instead of a timeout. + +`generate_async` on text prompts posts a token-prompt completion to the frontend +(`dynamo/rollout_client.py`). It sets top-level `ignore_eos` and a context-sized +`max_tokens`, so the frontend's own stop rules never end a stream before the +engine reports its result. Everything else keeps native dispatch; the guide +lists what goes where. + +## Dependency coupling + +The vLLM workers' venv (the `vllm` extra) carries the released `ai-dynamo` and +`ai-dynamo-runtime` wheels, never `ai-dynamo[vllm]`. The engine process only +imports `dynamo.common.backend`, `dynamo.llm.exceptions` and `dynamo._core`. + +What a Dynamo upgrade has to re-check: +- the `dynamo.common.backend` contract (`LLMEngine`, `Worker`, `WorkerConfig`, + publisher and logprob helpers); +- the request fields of Dynamo's preprocessed request; +- the frontend's `nvext` extensions and annotation forwarding; +- the parser names the native settings map to (`dynamo/config.py`). + +The one vLLM↔Dynamo wire coupling is vLLM's KV-event msgpack, which Dynamo's +router decodes. A change there degrades routing, not serving. The managed +runtime logs the router's applied-event counts at shutdown, and the Dynamo +functional overlays fail if the engines published events but none were +applied (`tests/functional/check_dynamo_overlay.sh`). + +`ai-dynamo-runtime` caps pydantic, which pins the vLLM venvs' pydantic version. diff --git a/docs/guides/async-grpo.md b/docs/guides/async-grpo.md index 9a76ceae772..f4d3562e023 100644 --- a/docs/guides/async-grpo.md +++ b/docs/guides/async-grpo.md @@ -221,7 +221,7 @@ If no `replay_buffer.pt` file is found in the latest checkpoint directory, train 3. **Resource Allocation**: Ensure sufficient GPU memory for both the training and generation clusters -4. **In-Flight Weight Updates**: Enable `in_flight_weight_updates: true` to refit without waiting for the longest in-flight generation to finish. Except for managed Dynamo, the collector requests a generation pause and resume from every async backend around the weight transfer. Async vLLM implements this contract while preserving request state. SGLang acknowledges these hooks through its weight synchronizer, which pauses, flushes KV, transfers weights, and resumes generation; its configured pause mode determines whether requests are requeued or aborted. A backend that does not implement the hook emits a warning once per backend type per process and refits without a collector-side pause or drain. Managed Dynamo always drains active trajectories before refit. vLLM requires `async_engine: true`; the Megatron backend is always async-engine. +4. **In-Flight Weight Updates**: Enable `in_flight_weight_updates: true` to refit without waiting for the longest in-flight generation to finish. The collector requests a generation pause and resume from every async backend around the weight transfer. Async vLLM implements this contract while preserving request state. SGLang acknowledges these hooks through its weight synchronizer, which pauses, flushes KV, transfers weights, and resumes generation; its configured pause mode determines whether requests are requeued or aborted. A backend that does not implement the hook emits a warning once per backend type per process and refits without a collector-side pause or drain. vLLM requires `async_engine: true`; the Megatron backend is always async-engine. 5. **Recompute KV Cache After Weight Updates**: `recompute_kv_cache_after_weight_updates` defaults to `true` to invalidate reusable KV/prefix caches when weights change. On the native async vLLM in-flight path, caches are cleared while generation is paused, so preserved requests recompute their KV after resuming. This repeats prefill for those requests and can reduce throughput. SGLang always flushes KV inside its weight synchronizer before the transfer, regardless of this flag; on its in-flight path, the collector skips a second flush after generation resumes. Other refit paths keep their existing post-update invalidation behavior. Set the flag to `false` to retain pre-update KV caches and avoid this recomputation cost on backends that support doing so. On the Megatron generation backend, the flag must agree with `policy.generation.mcore_generation_config.kv_cache_management_mode`: use `true` with `recompute`, or explicitly set `false` with `persist`/`offload`; setup errors on a mismatch. diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index da804901718..09915d3f276 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -1,20 +1,165 @@ -# Managed Dynamo generation on Slurm +# Dynamo serving for vLLM generation -NeMo RL can launch and own Dynamo's control plane, frontend, and a fixed vLLM -worker fleet inside a Slurm-backed Ray allocation. This mode supports direct -GRPO and NeMo-Gym rollouts, NCCL weight refits, cache invalidation, and -`generation_metrics/*` telemetry sent to enabled loggers such as W&B. -See the [Dynamo integration design](../design-docs/dynamo-integration.md) for -the service ownership, startup, and weight-refit architecture. +Any vLLM generation config can serve its rollouts through +[Dynamo](https://github.com/ai-dynamo/dynamo)'s OpenAI frontend and KV-aware +router. Add one line: + +```yaml +policy: + generation: + backend: vllm + dynamo: true +``` -This integration is managed and vLLM-only. It does not connect to an external -Dynamo deployment and does not support Kubernetes, DGD, SGLang, TensorRT-LLM, -speculative decoding, quantized generation, or model-parallel engine groups -that span nodes. +The same NeMo RL vLLM workers run the same engines, with the same refit +transports, pause and resume, quantization, speculative decoding and metrics. +Each engine also registers with a managed `dynamo.frontend`, and NeMo RL's own +rollouts (`generate_async`) go through that frontend, so each request lands on +the engine that already holds most of its prompt's KV cache. The few settings +Dynamo cannot serve fail at startup; see [Not supported](#not-supported). + +See the [Dynamo integration design](../design-docs/dynamo-integration.md) for +how the engines are hosted and why Dynamo upgrades and vLLM bumps stay +independent. + +## What happens at startup + +With `dynamo: true`: + +- `vllm_cfg.async_engine` is turned on, because Dynamo drives the engine's + `AsyncLLM`. Every rollout path the sync engine supports also runs on the async + one. +- The driver starts etcd, NATS and `dynamo.frontend`. The frontend runs from the + vLLM workers' own venv, so it is the same ai-dynamo release. +- Each engine turns on vLLM's KV-cache events for `router_mode: kv` and + registers with the frontend from inside its own worker process. Setup waits + until the frontend routes to every engine. +- vLLM publishes KV-cache events only for blocks its prefix cache stores. With + `vllm_cfg.enable_prefix_caching: false`, the engines publish none, and the + KV router routes on load alone. +- Shutdown stops the services after the engines. + +## Configure + +`dynamo: true` is a complete configuration. `policy.generation.dynamo_cfg` only +tunes the frontend and registration. Its fields and defaults live in +`nemo_rl/models/generation/dynamo/config.py`, and +[`examples/configs/grpo_math_1B_dynamo_single_controller.yaml`](../../examples/configs/grpo_math_1B_dynamo_single_controller.yaml) +shows them all: + +| Key | Default | Meaning | +|---|---|---| +| `startup_timeout_s` | 600 | Bounds service start-up and engine registration | +| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timed-out rollout is not retried; NeMo-Gym's calls keep Gym's timeouts | +| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | +| `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | +| `frontend_args.router_mode` | `kv` | Frontend routing policy | +| `frontend_args.session_affinity_ttl_secs` | 1800 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it | +| `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | +| `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | + +The etcd and nats-server binaries are found in `NEMO_RL_DYNAMO_BIN_DIR` when it +is set (the image sets it to `/opt/dynamo/bin`), and on `PATH` otherwise. + +## Which requests go through the frontend + +| Request | Path | +|---|---| +| `generate_async` on text token prompts, router replay (R3) included | Frontend (KV-aware); the engine samples and assembles it exactly as native `generate_async` does | +| NeMo RL's own multimodal rollouts | Native shard dispatch. Those rollouts send media to the engine directly rather than as OpenAI content | +| `generate_text_async`, which evaluation uses | Native shard dispatch | +| NeMo-Gym's OpenAI requests | Frontend, in Gym's Dynamo dialect; the engine serves them as the native chat endpoint does. See [NeMo-Gym](#nemo-gym) | + +A rollout through the frontend has the same `GenerationOutputSpec` as the +native path: token IDs, log probabilities, truncation, routed experts with +their `r3_routed_experts_*` counts, and `gen_leader_worker_idx`. Both paths +build it with `nemo_rl/models/generation/vllm/request_pipeline.py`. + +OpenAI clients of the frontend get what vLLM's own OpenAI server applies: +- the model's generation-config sampling defaults; +- `n > 1`; +- guided decoding (`guided_*`, `response_format`); +- `bad_words_token_ids` and `allowed_token_ids`; +- prompt log probabilities (`nvext.extra_fields: ["prompt_logprobs"]`); +- vLLM's queue limits, rejected with HTTP 503; +- a 400 naming the "maximum context length" for a prompt that leaves no room + for output. The engine, not the frontend, checks the context, so it checks + the prompt it actually runs. + +## NeMo-Gym + +NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect +(`vllm_model.dynamo_frontend`, from +[NVIDIA-NeMo/Gym#4817](https://github.com/NVIDIA-NeMo/Gym/pull/4817)). Gym marks +every call with an annotation, and the engine serves a marked call as the native +chat endpoint does: + +- Sampling must match `temperature`/`top_p` or `val_temperature`/`val_top_p`, + with `top_k` unset. vLLM samples with the configured values. +- The token prefix Gym echoes replaces the history the frontend rendered. +- Output is clamped to the context, and a prompt that leaves no room is refused + with the native endpoint's "maximum context length" error. +- Stop strings stop vLLM itself, so the token IDs match the native endpoint's. +- The response carries `prompt_token_ids`, `completion_token_ids` and + `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. +- Images, video and audio in the messages are processed as on the native + endpoint. The frontend passes each media URL through with one unexpanded + placeholder per item. The engine loads the media with vLLM's media connector + and runs vLLM's multimodal processor on the frontend's tokens. The splice, + placeholder remap and media capture then work on that processed prompt. + +With `token_capture.enabled`, Gym uses its `dynamo_worker` capture backend. The +same vLLM workers stage each call's tokens, log probabilities, routed experts and +processed media from their own process, exactly as the native endpoint does, and +only the commit coordinates travel back through the frontend. The frontend keeps a rollout on +one engine (`session_affinity_ttl_secs`), as the native backend pins each Gym +session to one server. + +The native server's chat settings carry over: + +| Native setting (`vllm_cfg`) | Under Dynamo | +|---|---| +| `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | +| `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | +| `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | +| `http_server_serving_chat_kwargs.default_chat_template_kwargs` | Sent by Gym under its own `chat_template_kwargs` | +| `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | + +Where the behavior differs from the native endpoint: + +- The engine sees the rendered prompt, not the messages. It finds where the + echoed prefix ends by counting the prefix's EOS tokens, not by rendering the + history. +- When a request sets `chat_template_kwargs.force_nonempty_content`, Dynamo + reports a reasoning-only turn as content. + +## Not supported + +Startup rejects these with `dynamo: true`: + +- `colocated.enabled: true`: a colocated engine sleeps while the policy trains, + and the frontend would keep routing to it. The SingleController never runs + colocated vLLM; the other entrypoints need non-colocated generation. +- `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, + and a restarted shard does not register with it yet. +- `async_rl.generation_router`: the frontend already routes the fleet behind one + URL. +- `vllm_cfg.http_server_api_key_required`: the frontend checks no bearer token. +- On the SingleController, router replay with Gym but without token capture: + Gym's dialect does not return routed experts in responses. Other entrypoints + do not check this, and such rollouts replay no routes. +- Parser plugins with no Dynamo parser, and the chat settings above that the + frontend has no counterpart for. +- `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without + `dynamo: true`. + +A Gym call with `n > 1` is refused when it arrives. ## Build the image -The normal image is unchanged unless `BUILD_DYNAMO` is set: +ai-dynamo is part of the `vllm` extra, so every vLLM worker venv carries it. +`BUILD_DYNAMO=1` adds only the etcd and nats-server binaries, under +`/opt/dynamo/bin`: ```bash docker buildx build \ @@ -26,210 +171,36 @@ docker buildx build \ . ``` -The opt-in layer installs `ai-dynamo[vllm]==1.5.1` in isolated Python -3.12 under `/opt/dynamo_venv`, along with etcd v3.5.21 and NATS Server v2.11.6. -It does not replace NeMo RL's normal Ray or vLLM dependencies: the standard -NeMo RL vLLM environment currently uses vLLM 0.29.0, while this isolated -Dynamo environment uses Dynamo's vLLM 0.28.0 pin. Both environments pin -`nvidia-nccl-cu13==2.30.7` so their NCCL communicators use the same release. -For a local source checkout, the same environment can be installed under -`venvs/dynamo`: +For a local checkout, install the binaries with: ```bash -bash docker/dynamo/install.sh +NEMO_RL_DYNAMO_BIN_DIR=$PWD/venvs/dynamo-bin bash docker/dynamo/install.sh ``` -Set `NEMO_RL_DYNAMO_VENV_DIR` to choose another location. The installer checks -that Dynamo resolved vLLM 0.28.0 and NCCL 2.30.7, and that the installed vLLM -contains the PR #44814 layerwise-reload fix. - -A Dynamo upgrade must update and reverify these coupled locations: - -- `docker/dynamo/pyproject.toml` and `docker/dynamo/uv.lock`: the Dynamo pin - and resolved dependency set -- root `pyproject.toml` and `uv.lock`, plus the isolated Dynamo project and - lockfile: the `nvidia-nccl-cu13` pins must remain identical -- `docker/dynamo/install.sh`, `tests/functional/grpo_dynamo.sh`, and - `tests/functional/grpo_dynamo_single_controller.sh`: the vLLM version, the - #44814 fix check, and runtime assertions -- `nemo_rl/models/generation/dynamo/refit.py` and - `validate_dynamo_vllm_args.py`: the vLLM weight-transfer payloads (packing is - declared in the init info; update rounds carry names, dtypes, and shapes; - `start_weight_update` takes no arguments) -- `tests/unit/distributed/test_stateless_process_group.py`: vLLM's - `broadcast_from/0/0` weight-transfer wire key -- `nemo_rl/models/generation/dynamo/token_wrapper.py`: the real Dynamo response - keys `nvext.engine_data.{prompt_token_ids,completion_token_ids,completion_logprobs}`; - reverify them against real Dynamo output because unit tests validate only the - expected local response shape -- `nemo_rl/models/generation/dynamo/managed_runtime.py`: the managed - `DYN_ENABLE_EXPERIMENTAL_PARSERS_V2=1` setting. Dynamo 1.3.0's legacy tool - jail removes `nvext.engine_data`; remove this setting only after an upgraded - Dynamo preserves the token metadata for `tool_choice=auto` -- this guide and the Dynamo design document: the stated versions - -## Configure Dynamo - -Start with [`examples/configs/grpo_math_1B_dynamo.yaml`](../../examples/configs/grpo_math_1B_dynamo.yaml). -The important boundary is: +## Run -```yaml -policy: - generation: - backend: dynamo - dynamo_cfg: - engine: vllm - frontend_args: - router_mode: kv - vllm_cfg: - tensor_parallel_size: 1 - pipeline_parallel_size: 1 - expert_parallel_size: 1 - colocated: - enabled: false - resources: - gpus_per_node: 1 - num_nodes: 1 -``` - -NeMo RL derives each engine's world size from TP times PP. EP must be one or -equal to TP. Parser settings belong under `dynamo_cfg.worker_args`; inherited -vLLM HTTP-parser settings are rejected with the corresponding Dynamo field. -Service ports and the namespace are runtime-owned rather than public config. - -`vllm_cfg` settings are handled in five explicit classes: - -| Class | Behavior | Examples | -| --- | --- | --- | -| Translated | Forwarded to `dynamo.vllm` | TP, PP, EP, dtype, model length | -| Moved | Startup error naming the Dynamo replacement | tool and reasoning parsers, HTTP serving chat kwargs | -| Unsupported | Warning when active, or an error when it requests unsupported low precision | tokenizer skipping, MX and mixed BF16/FP8 helpers | -| Managed runtime | Consumed or enforced by NeMo RL rather than forwarded | HTTP-wrapper enablement, metrics sampling, processed rollout logprobs | -| Inapplicable | Ignored because the managed path owns that behavior | async mode, progress bars, NeMo RL HTTP/ZMQ refit ports | - -The shared GRPO base config also supplies `mcore_generation_config` and -`refit_cfg`. Dynamo accepts these inherited sections but does not use them; -worker arguments come from `vllm_cfg`, and refit uses the collective weight -synchronizer. Any other active top-level `policy.generation` setting that -Dynamo does not honor, such as `bad_words` or `ignore_eos`, draws a startup -warning. - -Enable and filter worker telemetry with both managed configuration sections: - -```yaml -policy: - generation: - vllm_cfg: - enable_vllm_metrics_logger: true - vllm_metrics_logger_interval: 1.0 - dynamo_cfg: - metrics_include_prefixes: null # null selects the curated defaults - metrics_exclude_prefixes: null # null excludes python_ and process_ -``` - -The NCCL sender also selects vLLM's peer protocol: the policy publishes both -the raw NeMo RL unique ID and vLLM's pickled `ncclUniqueId`, then uses the -all-reduce warmup expected by `PyNcclCommunicator`. This protocol choice and -the packed 1-GiB/two-buffer geometry come from the generation backend rather -than GRPO-specific branches. - -The fixed port layout is: - -- `1313-1399`: driver-local etcd and NATS control plane -- `3000-3999`: frontend and token-wrapper HTTP endpoints -- `4000-4099`: node-local `DYN_SYSTEM_PORT` -- `7000 + slot * 100`: node-local vLLM rendezvous ports - -## Run the two-GPU smoke - -Convert the image to the format required by the Slurm site, then submit from -the repository root: +Use any non-colocated vLLM recipe and add the switch: ```bash -export CONTAINER=/shared/images/nemo-rl-dynamo.sqsh -export MOUNTS="$PWD:$PWD" -export GPUS_PER_NODE=2 -export BASE_LOG_DIR="$PWD/results/dynamo-smoke/logs" -printf -v COMMAND '%q ' \ - /opt/nemo_rl_venv/bin/python -u "$PWD/examples/run_grpo.py" \ - --config "$PWD/examples/configs/grpo_math_1B_dynamo.yaml" -export COMMAND - -sbatch \ - --nodes=1 \ - --gres=gpu:2 \ - --exclusive \ - --account= \ - --partition= \ - ray.sub +uv run examples/run_grpo_single_controller.py \ + --config examples/configs/grpo_math_1B_megatron_single_controller.yaml \ + ++policy.generation.dynamo=true ``` -The recipe assigns one GPU to training and one to a TP1 Dynamo worker. Its two -steps exercise generation, refit, post-refit cache invalidation, telemetry, -and cleanup. For a matched control, run the same seed/model/batch settings with -the standard non-colocated vLLM backend and compare post-refit output validity. - -## Run under the SingleController - -The async SingleController trainer can also use managed Dynamo. It follows the -same data path as the native vLLM backend: the SingleController sends each turn -to the Dynamo frontend, builds the rollout record from the response, and commits -it to the TransferQueue data plane, from which the policy workers read their -shards. No Dynamo-side component writes to TransferQueue. - -Start from [`examples/configs/grpo_math_1B_dynamo_single_controller.yaml`](../../examples/configs/grpo_math_1B_dynamo_single_controller.yaml) -and launch it with `examples/run_grpo_single_controller.py`. Before every weight -update the SingleController drains and pauses each Dynamo worker (requests the -engine is running finish on the old weights; queued and new requests wait and -run on the new weights), runs the NCCL refit, and then resumes the workers. With -`async_rl.recompute_kv_cache_after_weight_updates` (the default), the drain also -clears each worker's KV and prefix cache. The drain is bounded by -`dynamo_cfg.control_timeout_s`. - -Current limits on this path: - -- Native rollouts only. NeMo-Gym under the SingleController supports only - `backend: vllm` and `backend: megatron`. -- Text policies only, and no MoE router replay: `policy.is_vlm` and - `policy.router_replay.enabled` must be false. -- Fleet health is not supported: keep `async_rl.generation_fleet_health.enabled` - false (the Dynamo frontend owns request routing) and set - `async_rl.generation_fleet_health.refit_timeout_s: null` (default 300), since - Dynamo workers cannot abort their refit collective. -- `dynamo_cfg.control_timeout_s` must be at least `dynamo_cfg.request_timeout_s`, - because it bounds the drain. -- `vllm_cfg.enable_vllm_metrics_logger` requires - `rollout_checkpointing.telemetry_interval_s`: the rollout telemetry pump drains - the Dynamo metrics sampler and reports each engine's generated tokens, running - and waiting requests, and KV-cache usage. - -Setup rejects any of these before allocating clusters. - -## Run SWE1 with W&B - -The three-node nightly recipe targets -`nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16`. It uses two 8-GPU training -nodes and one 8-GPU inference node. The inference node runs two TP4/EP4 Dynamo -engines. Download the standard SWE1 split under -`${HF_HOME}/superv3_data/swe1`, then run the registered test-suite driver: - -```bash -HF_HOME=/shared/huggingface \ -WANDB_API_KEY= \ -bash tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh -``` - -A successful acceptance run completes four training steps, produces valid -generations after refit, and records worker timelines under -`generation_metrics/*` in TensorBoard and W&B. - -## Operational notes - -- The driver owns all services. Do not start a separate etcd, NATS, frontend, - or worker fleet for this mode. -- Startup validates fixed worker membership; a dead or replaced worker fails - refit instead of serving mixed model versions. -- Shutdown is idempotent and terminates whole subprocess groups, including - partial-startup failures. -- Fault tolerance and a multi-controller architecture remain follow-up work. +is the same run with every `dynamo_cfg` default spelled out. + +## Migrating from `backend: dynamo` + +`backend: dynamo` launched Dynamo's own vLLM engine and was removed. Startup +rejects it with the migration: + +- Set `backend: vllm` and `dynamo: true`. `vllm_cfg` and `vllm_kwargs` keep their + meaning, and every vLLM feature is now available. +- Remove these from `dynamo_cfg`: + - `engine`; + - `control_timeout_s`: refit and pause use the vLLM backend's controls; + - `metrics_include_prefixes` / `metrics_exclude_prefixes`: use + `vllm_cfg.enable_vllm_metrics_logger`; + - `worker_args.extra_cli_args`: engines are configured through `vllm_cfg` and + `vllm_kwargs`. diff --git a/docs/guides/single-controller.md b/docs/guides/single-controller.md index db23125c7f4..e9f1cd4f72b 100644 --- a/docs/guides/single-controller.md +++ b/docs/guides/single-controller.md @@ -542,12 +542,12 @@ The SC path is still under active development. Feature gaps are tracked in [issu Gym rollouts; multimodal/VLM MOPD is not yet supported. See [Multi-Teacher On-Policy Distillation](../about/algorithms/mopd.md#running-mopd). - Train backend: only Megatron is supported and validated; the AutoModel training path has not been tested on SC. -- Generation backend: vLLM and Megatron generation are supported (Megatron in both non-colocated and colocated modes). Managed Dynamo (`backend: dynamo`) is supported on the native rollout path; see [Managed Dynamo generation](dynamo-generation.md#run-under-the-singlecontroller). SGLang and TRT-LLM have not been tested on SC. +- Generation backend: vLLM and Megatron generation are supported (Megatron in both non-colocated and colocated modes). vLLM generation can also serve through Dynamo (`policy.generation.dynamo: true`); see [Dynamo serving](dynamo-generation.md). SGLang and TRT-LLM have not been tested on SC. - Validation is not yet supported (setup raises on `val_period > 0`, `val_at_start`, or `val_at_end`); checkpointing is. - (PPO) Rollout drop budgets — `async_rl.rollout_failure.max_skipped_prompts` and `max_consecutive_dropped_prompts` must both be `0`. A drop shortens the step, and the critic shards it against the configured `value.train_global_batch_size` rather than its actual size, so setup rejects a non-zero budget. The resiliency layer stays available on GRPO. - Reward shaping and sample filtering — `reward_shaping`, `reward_scaling`, and `use_dynamic_sampling` are implemented on neither algorithm block, so setup rejects them rather than silently skipping the shaping. Environment-flagged sample masking and `overlong_filtering` are supported; truncated completions are excluded from the loss through `sample_mask`, and a step in which every completion is filtered is rejected rather than skipped. - The `windowed` sampler has no `over_sampling_ratio` cap — over-produced groups aged past the window are evicted, wasting rollout compute. -- The drain gate in refit is not yet supported. Managed Dynamo pauses its engines before every refit because its weight update is unsafe while requests run, but that is not the drain gate: running requests finish on the old weights, while queued requests and later turns of a multi-turn rollout run on the new ones. +- The drain gate in refit is not yet supported. ### Full training-data dumps diff --git a/docs/index.md b/docs/index.md index e88dfe3c699..9cfda03a275 100644 --- a/docs/index.md +++ b/docs/index.md @@ -79,11 +79,11 @@ Measure model and agent capabilities with repeatable benchmarks and custom evalu Configure and launch NeMo RL on multi-node Slurm or Kubernetes clusters for distributed computing. ::: -:::{grid-item-card} {octicon}`workflow` Managed Dynamo Generation +:::{grid-item-card} {octicon}`workflow` Dynamo Serving :link: guides/dynamo-generation :link-type: doc -Run a fixed Dynamo vLLM fleet with NCCL refit and W&B telemetry inside a Slurm Ray allocation. +Serve any vLLM generation config through Dynamo's frontend and KV-aware router with `dynamo: true`. ::: :::: diff --git a/docs/observability/configuration.md b/docs/observability/configuration.md index 9fab877b536..36e42c3b9e1 100644 --- a/docs/observability/configuration.md +++ b/docs/observability/configuration.md @@ -100,7 +100,7 @@ Each process sets stable-for-the-run values on its OTel `Resource` — `init_tel | `nv.dl.training.config.optimizer` | on Megatron `policy.megatron_cfg.optimizer.optimizer` (`adam`), otherwise `policy.optimizer.name` (`torch.optim.AdamW`) — the backends both spell it and store it differently, and only the enabled one is read | | `nv.dl.training.config.recompute_granularity` | Megatron runs only: `policy.megatron_cfg.recompute_granularity`, and only when `activation_checkpointing` is on. Megatron ignores the granularity otherwise, so reporting it would describe recompute that is not happening | | `nv.dl.training.target.train_iters` | `.max_num_steps`, when positive. This is the configured cap, not a prediction: many configs set a placeholder (`grpo_math_1B` uses `1000000` with `max_num_epochs: 1`) and stop earlier on epochs. Looked up by algorithm name, so an entry point that keeps its settings under a differently named section reports no target at all — `sft_v2` reads `master_config.sft`, and omits this. `rm` spells "train for one epoch" as `-1`, which is omitted rather than reported as a target | -| `nv.dl.software.torch`, `.cuda`, `.nccl` | versions as loaded in the process that opened the span, not as pinned in the lockfile — they are per process, so filter on `rl.worker_group` to read one backend's versions. CUDA and NCCL are omitted on a CPU-only build. SGLang, TRT-LLM and Dynamo workers do not report versions yet | +| `nv.dl.software.torch`, `.cuda`, `.nccl` | versions as loaded in the process that opened the span, not as pinned in the lockfile — they are per process, so filter on `rl.worker_group` to read one backend's versions. CUDA and NCCL are omitted on a CPU-only build. SGLang and TRT-LLM workers do not report versions yet | | `nv.dl.software.transformer_engine` | the installed distribution's version, read from package metadata rather than by importing it — importing transformer-engine loads its PyTorch extension in every process that sets up telemetry. Omitted when it is not installed, which is every process except Megatron and Automodel workers: its absence on a driver, controller or vLLM span does not mean the run lacked TE | | `nv.dl.rank`, `nv.dl.world_size` | this process's rank and group size (`RANK` / `WORLD_SIZE`, or `0` / `1` for the driver and singleton actors) | | `nv.dl.local_rank` | worker processes only: `LOCAL_RANK`, which `RayWorkerGroup` exports beside `RANK` as the worker's bundle index in its placement group. Node-local only when each node has its own placement group — a generation backend spanning nodes requests a single unified one (SGLang always; vLLM, TRT-LLM and Megatron when their parallelism crosses nodes) and the index then runs across nodes. Not a physical device index either, since Ray gives each worker its own `CUDA_VISIBLE_DEVICES` | diff --git a/docs/observability/metrics.md b/docs/observability/metrics.md index ce58a5f7b31..3824e9bf1d3 100644 --- a/docs/observability/metrics.md +++ b/docs/observability/metrics.md @@ -187,7 +187,7 @@ The counters carry the step's delta, so summing them across steps reconstructs t **What counts as failed.** vLLM's finish reasons are `stop`, `length`, `abort`, `error` and `repetition`. Only `stop` and `length` leave a usable sample behind, and `length` is a normal RL outcome — a rollout routinely runs to `max_tokens` — so those two are `ok` and *everything else* is `failed`. Counting the remainder rather than an explicit deny-list keeps `ok + failed` equal to the engine's own total even if a future vLLM adds a reason we have not heard of. -These are the family names vLLM registers. The reader returns family names, which `prometheus_client` strips of `_total`, so no suffixed alias is needed on this path — unlike the HTTP-scrape path in `nemo_rl/models/generation/dynamo/metrics.py`, where sample names do carry it. `test_engine_metric_names_match_vllm` checks the copies against vLLM itself. A series that matches nothing is **omitted**, leaving a visible gap in the dashboard rather than a plausible-looking zero, and so is one whose total went backwards, which means an engine restarted mid-step. +These are the family names vLLM registers. The reader returns family names, which `prometheus_client` strips of `_total`, so no suffixed alias is needed on this path. `test_engine_metric_names_match_vllm` checks the copies against vLLM itself. A series that matches nothing is **omitted**, leaving a visible gap in the dashboard rather than a plausible-looking zero, and so is one whose total went backwards, which means an engine restarted mid-step. ## Driver-side generation metrics (`gen_ai.*`) diff --git a/examples/configs/grpo_math_1B.yaml b/examples/configs/grpo_math_1B.yaml index c55645114c8..d354b952783 100644 --- a/examples/configs/grpo_math_1B.yaml +++ b/examples/configs/grpo_math_1B.yaml @@ -504,7 +504,7 @@ policy: # Non-colocated vLLM also supports vllm_s3_sparse, vllm_zmq_sparse, and nixl. # Non-colocated vLLM or Megatron generation supports nccl_reshard (M-to-N). refit_transport: null - # vLLM backend only - sglang/megatron/trtllm/dynamo require this to remain null. + # vLLM backend only - sglang/megatron/trtllm require this to remain null. # Selects the NeMo-RL Ray actor class, NOT vLLM's own `worker_extension_cls`. worker_extension_cls_fqn: null # optional registered vLLM generation-worker subclass; mutually exclusive with quant_cfg # Keep real-quant export tensors on CPU by default. Set false only for diff --git a/examples/configs/grpo_math_1B_dynamo.yaml b/examples/configs/grpo_math_1B_dynamo.yaml deleted file mode 100644 index fcf30538306..00000000000 --- a/examples/configs/grpo_math_1B_dynamo.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Two-step, two-GPU smoke for the Ray-managed Dynamo vLLM backend. -defaults: grpo_math_1B.yaml - -grpo: - num_prompts_per_step: 2 - num_generations_per_prompt: 2 - max_num_steps: 2 - val_period: 0 - val_at_start: false - val_at_end: false - async_grpo: - enabled: true - in_flight_weight_updates: false - recompute_kv_cache_after_weight_updates: true - -loss_fn: - use_importance_sampling_correction: true - truncated_importance_sampling_type: tis - truncated_importance_sampling_ratio: 2.0 - -checkpointing: - enabled: false - -policy: - train_global_batch_size: 4 - train_micro_batch_size: 1 - logprob_batch_size: 1 - max_total_sequence_length: 512 - sequence_packing: - enabled: false - automodel_cfg: - enabled: true - cpu_offload: false - activation_checkpointing: false - megatron_cfg: - enabled: false - generation: - backend: dynamo - max_new_tokens: 128 - temperature: 0.7 - top_p: 0.95 - top_k: 50 - dynamo_cfg: - engine: vllm - startup_timeout_s: 600 - request_timeout_s: 900 - control_timeout_s: 600 - metrics_include_prefixes: null - metrics_exclude_prefixes: null - worker_args: - tool_call_parser: null - reasoning_parser: null - exclude_tools_when_tool_choice_none: true - enable_structural_tag: false - structural_tag_scope: auto - structural_tag_schema: auto - custom_jinja_template: null - endpoint_types: [chat, completions] - extra_cli_args: [] - frontend_args: - tokenizer: default - tokenizer_cache: false - tokenizer_cache_bytes: 52428800 - router_mode: kv - extra_cli_args: [] - vllm_cfg: - async_engine: true - tensor_parallel_size: 1 - pipeline_parallel_size: 1 - expert_parallel_size: 1 - precision: bfloat16 - gpu_memory_utilization: 0.6 - max_model_len: ${policy.max_total_sequence_length} - kv_cache_dtype: auto - enforce_eager: true - expose_http_server: false - enable_vllm_metrics_logger: true - vllm_metrics_logger_interval: 1.0 - env_vars: - NCCL_MNNVL_ENABLE: "0" - vllm_kwargs: - max_num_seqs: 16 - colocated: - enabled: false - resources: - gpus_per_node: 1 - num_nodes: 1 - -logger: - log_dir: results/grpo-math-1b-dynamo - wandb_enabled: false - tensorboard_enabled: true - monitor_gpus: false - -cluster: - gpus_per_node: 2 - num_nodes: 1 diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index da974c294c4..291aac186b6 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -1,73 +1,45 @@ -# SingleController variant with the Ray-managed Dynamo vLLM backend. +# grpo_math_1B_megatron_single_controller.yaml served through Dynamo. +# `policy.generation.dynamo: true` is the whole switch: the same vLLM workers +# register with a managed dynamo.frontend, and rollouts go through its KV-aware +# router. Every other vLLM setting keeps its meaning; Dynamo implies +# vllm_cfg.async_engine and needs non-colocated generation. # -# Rollouts follow the native-backend path: the SingleController sends each turn -# to the Dynamo frontend over HTTP, builds the rollout record from the response, -# and commits it to the TransferQueue data plane. Before every weight update the -# SingleController drains and pauses the Dynamo workers (running requests -# finish on the old weights; queued and new requests wait and run on the new -# weights), then resumes them afterwards. +# dynamo_cfg is optional. The values below are its defaults +# (nemo_rl/models/generation/dynamo/config.py). defaults: grpo_math_1B_megatron_single_controller.yaml -async_rl: - generation_fleet_health: - # Dynamo workers cannot abort their refit collective, so no refit deadline. - refit_timeout_s: null - policy: - # Training uses the Megatron backend inherited from - # grpo_math_1B_megatron_single_controller.yaml; the SingleController's split - # training API is implemented by the Megatron policy worker. generation: - backend: dynamo + dynamo: true dynamo_cfg: - engine: vllm + # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: 600 + # Timeout for one rollout routed through the frontend (in place of + # NRL_VLLM_ASYNC_TIMEOUT_SECONDS); a timed-out rollout is not retried. request_timeout_s: 900 - # Bounds the pre-refit drain, which waits for running requests to finish, - # so it must be at least request_timeout_s. - control_timeout_s: 900 - metrics_include_prefixes: null - metrics_exclude_prefixes: null worker_args: + # Dynamo's built-in parser names. null follows the native server's + # vllm_cfg.http_server_serving_chat_kwargs (tool_parser, reasoning_parser). tool_call_parser: null reasoning_parser: null - exclude_tools_when_tool_choice_none: true + exclude_tools_when_tool_choice_none: false enable_structural_tag: false structural_tag_scope: auto structural_tag_schema: auto + # A chat template that replaces the model's own. null follows + # vllm_cfg.http_server_serving_chat_kwargs.chat_template. custom_jinja_template: null endpoint_types: [chat, completions] - extra_cli_args: [] frontend_args: tokenizer: default tokenizer_cache: false tokenizer_cache_bytes: 52428800 + # kv: route each request to the engine holding most of its prefix. router_mode: kv + # Keep a NeMo-Gym token-capture rollout on one engine while it idles up + # to this many seconds (null disables it). + session_affinity_ttl_secs: 1800 extra_cli_args: [] - vllm_cfg: - async_engine: true - tensor_parallel_size: 1 - pipeline_parallel_size: 1 - expert_parallel_size: 1 - precision: bfloat16 - gpu_memory_utilization: 0.6 - max_model_len: ${policy.max_total_sequence_length} - kv_cache_dtype: auto - enforce_eager: true - expose_http_server: false - # Under the SingleController the rollout telemetry pump reports these - # metrics, so enabling this requires rollout_checkpointing.telemetry_interval_s. - enable_vllm_metrics_logger: false - vllm_metrics_logger_interval: 1.0 - env_vars: - NCCL_MNNVL_ENABLE: "0" - vllm_kwargs: - max_num_seqs: 16 - colocated: - enabled: false - resources: - gpus_per_node: 1 - num_nodes: 1 logger: log_dir: results/grpo-math-1b-dynamo-single-controller diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml index 75869d90917..010fee9cb49 100644 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml @@ -82,26 +82,11 @@ policy: sequence_packing: fuse_loss: true generation: - backend: dynamo + backend: vllm + dynamo: true dynamo_cfg: - engine: vllm startup_timeout_s: 1800 request_timeout_s: 7200 - control_timeout_s: 600 - metrics_include_prefixes: null - metrics_exclude_prefixes: null - worker_args: - tool_call_parser: qwen3_coder - reasoning_parser: nemotron_nano - exclude_tools_when_tool_choice_none: true - enable_structural_tag: false - structural_tag_scope: auto - structural_tag_schema: auto - custom_jinja_template: null - endpoint_types: - - chat - - completions - extra_cli_args: [] frontend_args: tokenizer: fastokens tokenizer_cache: true @@ -115,8 +100,6 @@ policy: enable_vllm_metrics_logger: true vllm_metrics_logger_interval: 0.5 env_vars: null - reasoning_parser_plugin: null - http_server_serving_chat_kwargs: null vllm_kwargs: attention_backend: FLASH_ATTN moe_backend: triton diff --git a/nemo_rl/algorithms/async_utils/trajectory_collector.py b/nemo_rl/algorithms/async_utils/trajectory_collector.py index fc8fd1c8eee..8986e8dda90 100644 --- a/nemo_rl/algorithms/async_utils/trajectory_collector.py +++ b/nemo_rl/algorithms/async_utils/trajectory_collector.py @@ -1048,10 +1048,10 @@ def resume(self) -> None: def prepare_for_refit(self) -> None: """Pause new generation starts and optionally wait for pending generations. - Every async backend configured for in-flight weight updates, except managed - Dynamo, is asked to pause generation. vLLM preserves in-flight request state - with its native keep-mode pause. Backends without pause support warn and - retain their existing behavior. Managed Dynamo drains active trajectories. + Every async backend configured for in-flight weight updates is asked to + pause generation. vLLM preserves in-flight request state with its native + keep-mode pause. Backends without pause support warn and retain their + existing behavior. For non-async engines, waits for all pending generations to complete before refit. """ @@ -1067,16 +1067,7 @@ def prepare_for_refit(self) -> None: # Check if we're using async engine generation_cfg = self.master_config.policy["generation"] backend = generation_cfg["backend"] - if backend == "dynamo": - # Dynamo's native layerwise reload temporarily materializes model - # parameters while the NCCL update is in progress. It is not safe - # to execute an already-issued vLLM request concurrently with that - # reload (in particular for NemotronH/Mamba parameters), even when - # the update route accepts allow_unpaused=True. Stop new trajectory - # starts above and drain every active trajectory before refitting. - is_async_engine = False - else: - is_async_engine = should_use_async_rollouts(generation_cfg) + is_async_engine = should_use_async_rollouts(generation_cfg) in_flight_weight_updates = self.async_config.in_flight_weight_updates if is_async_engine and in_flight_weight_updates: @@ -1144,10 +1135,6 @@ def resume_after_refit(self) -> None: ) except Exception as e: print(f"⚠️ Failed to invalidate generation backend KV caches: {e}") - if self.master_config.policy["generation"]["backend"] == "dynamo": - raise RuntimeError( - "Managed Dynamo KV cache invalidation failed after refit" - ) from e finally: self._generation_pause_requested_for_refit = False self._generation_paused_for_refit = False diff --git a/nemo_rl/algorithms/distillation.py b/nemo_rl/algorithms/distillation.py index b861d73522c..74a2691af88 100644 --- a/nemo_rl/algorithms/distillation.py +++ b/nemo_rl/algorithms/distillation.py @@ -549,7 +549,7 @@ def init_vllm_deferred(): def init_nemo_gym(): shard_set = build_nemo_gym_actors( env_configs, - base_urls=cast(list[str], deferred_vllm.dp_openai_server_base_urls), + base_urls=cast(list[str], deferred_vllm.client_base_urls), model_name=generation_config["model_name"], tokenizer=tokenizer, # Distillation does not configure vLLM for router replay. diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 5a96be314a3..9d808421d49 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -126,7 +126,6 @@ run_nemo_gym_rollout_sync, should_mask_flagged_samples, ) -from nemo_rl.models.generation.dynamo import DynamoConfig, DynamoGeneration from nemo_rl.models.generation.interfaces import ( GenerationConfig, GenerationInterface, @@ -628,16 +627,11 @@ def _needs_hf_refit_handshake( def _uses_managed_noncolocated_refit( - generation_backend: str, nccl_reshard_refit_enabled: bool, offload_policy_before_refit: bool, ) -> bool: """Whether non-colocated setup must attach a weight synchronizer.""" - return ( - nccl_reshard_refit_enabled - or generation_backend == "dynamo" - or offload_policy_before_refit - ) + return nccl_reshard_refit_enabled or offload_policy_before_refit def _shutdown_completed_nemo_gym_startup( @@ -707,20 +701,6 @@ def setup( if generation_config["backend"] == "vllm": normalize_nvfp4_pertoken_policy_config(policy_config, entry_point="grpo") normalize_vllm_refit_config(cast(VllmConfig, generation_config)) - elif generation_config["backend"] == "dynamo": - # Validate the complete managed-Dynamo boundary before allocating Ray - # placement groups or starting any external services. - if grpo_config.async_grpo.in_flight_weight_updates: - raise ValueError( - "grpo.async_grpo.in_flight_weight_updates must be false when " - "policy.generation.backend='dynamo'; managed Dynamo drains " - "rollouts before layerwise weight refit" - ) - generation_config.setdefault("vllm_kwargs", {})["hf_overrides"] = ( - policy_config.get("hf_config_overrides") or {} - ) - generation_config = DynamoConfig.model_validate(generation_config).model_dump() - policy_config["generation"] = generation_config # generation.disaggregation is written in backend-agnostic terms, but only # the TRT-LLM backend implements it today: every other backend would @@ -1267,10 +1247,6 @@ def _spinup_nemo_gym(base_urls, model_name, policy_api_key=None): gpus_per_instance = replica_width( cast(TrtllmConfig, generation_config) ) - elif generation_config["backend"] == "dynamo": - gpus_per_instance = DynamoConfig.model_validate( - generation_config - ).engine_world_size else: sglang_cfg = generation_config.get("sglang_cfg", {}) gpus_per_instance = sglang_cfg.get("gpus_per_server", 1) @@ -1344,10 +1320,6 @@ def _spinup_nemo_gym(base_urls, model_name, policy_api_key=None): MegatronGeneration.init_cluster_placement_groups( inference_cluster, policy_config ) - elif generation_config["backend"] == "dynamo": - # Managed Dynamo creates one single-node engine per placement - # group and does not need a backend-specific PG strategy. - inference_cluster.get_placement_groups() else: { "vllm": VllmGeneration, @@ -1776,8 +1748,8 @@ def init_megatron_generation_task(policy_future): ) vllm_reserve_time = time.perf_counter() - vllm_reserve_t0 print( - f" ✓ Reserved {len(deferred_vllm.dp_openai_server_base_urls)} vLLM server URLs: " - f"{deferred_vllm.dp_openai_server_base_urls}", + f" ✓ Reserved {len(deferred_vllm.client_base_urls)} vLLM server URLs: " + f"{deferred_vllm.client_base_urls}", flush=True, ) @@ -1791,7 +1763,7 @@ def init_vllm_deferred(): def init_nemo_gym(): """Spin up NeMo Gym servers with the pre-assigned vLLM URLs.""" return _spinup_nemo_gym( - deferred_vllm.dp_openai_server_base_urls, + deferred_vllm.client_base_urls, generation_config["model_name"], policy_api_key=deferred_vllm.http_server_api_key, ) @@ -1901,37 +1873,6 @@ def init_trtllm(): ) setup_timing_metrics.nemo_gym_init_time_s = nemo_gym_time - elif backend == "dynamo": - # Managed Dynamo owns a fixed worker fleet on the inference virtual cluster. - - def init_dynamo(): - t0 = time.perf_counter() - generation = DynamoGeneration( - cluster=inference_cluster, - config=generation_config, - tokenizer=tokenizer, - tokenizer_config=policy_config["tokenizer"], - ) - return generation, time.perf_counter() - t0 - - policy_generation, policy = initialize_generation_with_policy( - init_generation_fn=init_dynamo, - colocated_inference=False, - setup_timing_metrics=setup_timing_metrics, - ) - - if enable_nemo_gym: - nemo_gym_actor, nemo_gym_time = _spinup_nemo_gym( - policy_generation.dp_openai_server_base_urls, - generation_config["model_name"], - ) - setup_timing_metrics.nemo_gym_init_time_s = nemo_gym_time - - print( - f" ✓ Using Dynamo backend (frontend: {policy_generation.frontend_url})", - flush=True, - ) - # Record when worker initialization completes (for calculating other setup time) worker_init_complete_time = time.perf_counter() - setup_start_time @@ -1973,7 +1914,6 @@ def init_dynamo(): t0 = time.perf_counter() # init collective if _uses_managed_noncolocated_refit( - generation_backend=backend, nccl_reshard_refit_enabled=nccl_reshard_refit_enabled, offload_policy_before_refit=offload_policy_before_refit, ): @@ -4910,17 +4850,17 @@ def async_grpo_train( media to NeMo Gym prompt rows. """ # Ensure we are running with a compatible async generation backend. - # Async GRPO supports vLLM, Megatron, TRT-LLM, and Dynamo; + # Async GRPO supports vLLM, Megatron, and TRT-LLM; # SGLang async rollouts do not support the async GRPO replay path. generation_config = master_config.policy["generation"] backend = generation_config.get("backend", "") if generation_config else "" - assert backend in ("vllm", "megatron", "trtllm", "dynamo"), ( - "Async GRPO supports the vLLM, Megatron, TRT-LLM, and Dynamo generation backends; " + assert backend in ("vllm", "megatron", "trtllm"), ( + "Async GRPO supports the vLLM, Megatron, and TRT-LLM generation backends; " f"got policy.generation.backend={backend!r}." ) assert should_use_async_rollouts(generation_config), ( - "Async GRPO requires Dynamo, Megatron, or an async vLLM or TRT-LLM " - "generation engine. Set policy.generation.backend=dynamo, " + "Async GRPO requires Megatron, or an async vLLM or TRT-LLM " + "generation engine. Set " "policy.generation.vllm_cfg.async_engine=true (vLLM), or " "policy.generation.trtllm_cfg.async_engine=true (TRT-LLM). " "Megatron Inference always uses its async engine." @@ -5217,7 +5157,7 @@ def _flush_collector_telemetry() -> None: raise # Generation must hold the policy's real weights before any backend starts - # collecting. In particular, vLLM and Dynamo start with dummy weights when + # collecting. In particular, vLLM starts with dummy weights when # the first refit supplies model parameters. ray.get(trajectory_collector.set_weight_version.remote(weight_version)) ray.get(trajectory_collector.start_collection.remote(CyclingDataLoader(dataloader))) diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py index 6a61eb838e2..bd5f9fa7f3a 100644 --- a/nemo_rl/algorithms/single_controller_utils/config.py +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -664,7 +664,9 @@ class TokenCaptureConfig(BaseModel, extra="allow"): capture_dir: Optional[str] = None # Generation backend hosting token capture. This is derived from # policy.generation.backend during setup; users should not set it separately. - generation_backend: Optional[Literal["vllm", "megatron"]] = None + # "dynamo" is the vLLM backend served through Dynamo (policy.generation.dynamo): + # the same workers host capture, and NeMo-Gym reaches them through the frontend. + generation_backend: Optional[Literal["vllm", "megatron", "dynamo"]] = None # Keep routed_experts out of canonical rows and assemble them on policy # workers from strict staged-fragment plans. defer_routed_experts_to_policy: bool = False diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index e3ffbb1ba11..302783c2b18 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -120,7 +120,7 @@ should_mask_flagged_samples, ) from nemo_rl.models.generation import resolve_generation_class -from nemo_rl.models.generation.dynamo import DynamoConfig, DynamoGeneration +from nemo_rl.models.generation.dynamo.config import dynamo_enabled from nemo_rl.models.generation.fleet_health import ( FleetHealthPolicy, GenerationFleetHealth, @@ -130,7 +130,6 @@ GenerationRouterActor, GenerationRouterImpl, ) -from nemo_rl.models.generation.interfaces import GenerationConfig from nemo_rl.models.generation.megatron.megatron_generation import MegatronGeneration from nemo_rl.models.generation.sglang.config import SGLangConfig from nemo_rl.models.generation.sglang.sglang_generation import SGLangGeneration @@ -517,10 +516,6 @@ def _build_clusters( gpus_per_instance = vllm_cfg["tensor_parallel_size"] * vllm_cfg.get( "pipeline_parallel_size", 1 ) - elif backend == "dynamo": - gpus_per_instance = DynamoConfig.model_validate( - generation_config_dict - ).engine_world_size elif backend == "sglang": gpus_per_instance = generation_config_dict["sglang_cfg"].get( "gpus_per_server", 1 @@ -532,7 +527,7 @@ def _build_clusters( else: raise ValueError( "single_controller_utils.setup only supports vllm, sglang, " - f"megatron, or dynamo generation; got {backend!r}" + f"or megatron generation; got {backend!r}" ) nodes_per_instance = ( gpus_per_instance + inference_gpus_per_node - 1 @@ -595,7 +590,7 @@ def _build_generation( tokenizer: Optional[PreTrainedTokenizerBase] = None, processor: Optional[AutoProcessor] = None, ) -> tuple[Any, float]: - """Spin up the generation backend (vLLM, SGLang, Megatron, or managed Dynamo). + """Spin up the generation backend (vLLM, SGLang, or Megatron). Args: inference_cluster: Ray virtual cluster the generation workers run on. @@ -608,8 +603,8 @@ def _build_generation( Returns: A tuple of (generation object, wall time spent in this call). The - generation object is a VllmGeneration, SGLangGeneration, MegatronGeneration, - or DynamoGeneration. + generation object is a VllmGeneration, SGLangGeneration, or + MegatronGeneration. """ t0 = time.perf_counter() generation_config = master_config.policy["generation"] @@ -659,21 +654,10 @@ def _build_generation( skip_weight_load=True, ) - elif backend == "dynamo": - assert not defer_model_load, ( - "defer_model_load is only supported for the vllm backend" - ) - # setup_single_controller validated and normalized this config through - # DynamoConfig before allocating clusters. - gen = DynamoGeneration( - cluster=inference_cluster, - config=cast(dict[str, Any], generation_config), - ) - else: raise ValueError( - "single_controller_utils.setup only supports vllm, sglang, megatron, or " - f"dynamo generation; got {backend!r}" + "single_controller_utils.setup only supports vllm, sglang, or megatron " + f"generation; got {backend!r}" ) if not defer_model_load: @@ -825,12 +809,12 @@ def _spinup_gym( def _generation_max_seq_len(generation_config) -> int: """Return the per-backend max sequence length. - vllm and dynamo (which manages vLLM workers) use vllm_cfg.max_model_len; + vllm uses vllm_cfg.max_model_len; sglang uses sglang_cfg.context_length; megatron uses mcore_generation_config.max_model_len. """ backend = generation_config["backend"] - if backend in ("vllm", "dynamo"): + if backend == "vllm": return generation_config["vllm_cfg"]["max_model_len"] if backend == "sglang": return generation_config["sglang_cfg"]["context_length"] @@ -1213,7 +1197,7 @@ def setup_single_controller( telemetry_interval_s = master_config.rollout_checkpointing.telemetry_interval_s if telemetry_interval_s is not None: generation_backend = generation_config["backend"] - if generation_backend not in ("vllm", "dynamo"): + if generation_backend != "vllm": warnings.warn( "rollout_checkpointing.telemetry_interval_s is enabled with " f"policy.generation.backend={generation_backend!r}. Canonical " @@ -1231,7 +1215,7 @@ def setup_single_controller( "vLLM token, request, and KV-cache signals will be absent.", stacklevel=2, ) - elif generation_backend == "vllm" and not vllm_cfg["async_engine"]: + elif not vllm_cfg["async_engine"]: warnings.warn( "rollout_checkpointing.telemetry_interval_s and " "policy.generation.vllm_cfg.enable_vllm_metrics_logger are " @@ -1257,18 +1241,6 @@ def setup_single_controller( if checkpointing_pretrained is not None: policy_config["pretrained_checkpoint"] = checkpointing_pretrained - if generation_config["backend"] == "dynamo": - # Validate the complete managed-Dynamo boundary before allocating Ray - # placement groups or starting any external services, as grpo.setup does. - dynamo_config = cast(dict[str, Any], generation_config) - dynamo_config.setdefault("vllm_kwargs", {})["hf_overrides"] = ( - policy_config.get("hf_config_overrides") or {} - ) - generation_config = cast( - GenerationConfig, DynamoConfig.model_validate(dynamo_config).model_dump() - ) - policy_config["generation"] = generation_config - # Token capture: validate the supported combination loudly at setup # (NeMo-Gym rollout path; vLLM with async_engine=true, or Megatron with # expose_http_server=true). The serving worker's venv already carries @@ -1368,7 +1340,11 @@ def setup_single_controller( # Fill the derived ledger-hosting fields (see TokenCaptureConfig): a # per-run control-plane bearer token, the process-shared capture # directory used by every Gym worker, and the capture-host backend. - token_capture_cfg.generation_backend = generation_config["backend"] + token_capture_cfg.generation_backend = ( + "dynamo" + if dynamo_enabled(generation_config) + else generation_config["backend"] + ) if token_capture_cfg.control_auth_token is None: # Deferred import: only needed on the capture path. import secrets @@ -1852,7 +1828,7 @@ def _build_trainer_then_megatron_generation() -> tuple[ defer_model_load=True, ) defer_generation_model_load = True - gym_base_urls = generation.dp_openai_server_base_urls + gym_base_urls = generation.client_base_urls # Before the Gym task is built, so Gym can be handed the router's single URL. # These two statements are the only failable ones related to the port holder's creation. try: diff --git a/nemo_rl/distributed/actor_environments.py b/nemo_rl/distributed/actor_environments.py index 8bd9ccaf943..25e2cc4791a 100644 --- a/nemo_rl/distributed/actor_environments.py +++ b/nemo_rl/distributed/actor_environments.py @@ -54,7 +54,6 @@ "nemo_gym", ], "nemo_rl.models.generation.sglang.sglang_worker.SGLangGenerationWorker": ["sglang"], - "nemo_rl.models.generation.dynamo.dynamo_worker.DynamoVllmWorker": None, "nemo_rl.models.policy.workers.automodel_policy_worker.AutomodelPolicyWorker": [ "automodel" ], diff --git a/nemo_rl/distributed/virtual_cluster.py b/nemo_rl/distributed/virtual_cluster.py index f87fe68f224..049ddde289f 100644 --- a/nemo_rl/distributed/virtual_cluster.py +++ b/nemo_rl/distributed/virtual_cluster.py @@ -157,8 +157,8 @@ def uv_py_executable(extras: Sequence[str]) -> str: # 1313-1399 Dynamo etcd/NATS control plane (driver-local allocation) # 1400-1999 Master address / TCPStore (cluster.master_port_range_low/high) # [3000, 4999) Shared NeMo RL generation range (policy.generation.port_range_low/high) -# [3000, 4000) Dynamo frontend/token-wrapper HTTP endpoints -# [4000, 4100) Dynamo worker system endpoints (node-local free-port selection) +# [3000, 4000) Dynamo frontend HTTP endpoint +# [4000, 4100) Dynamo engine KV-event endpoints (node-local free-port selection) # 5000-5999 NeMo Gym HTTP servers (env.nemo_gym.port_range_low/high) # 6000 NeMo-Skills sandbox Nginx (NEMO_SKILLS_SANDBOX_PORT; ray.sub starts one # sidecar per allocated node, driver included) diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 0900200b4b1..017f5d152dd 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -92,6 +92,10 @@ RolloutDataFailure, http_status_is_infra, ) +from nemo_rl.models.generation.dynamo.config import ( + dynamo_enabled, + native_default_chat_template_kwargs, +) from nemo_rl.models.generation.interfaces import ( resolve_routed_experts_dtype_name_for_model, should_use_async_rollouts, @@ -300,10 +304,6 @@ def should_use_nemo_gym(master_config: NemoGymCompatibleConfig) -> bool: should_expose_http_server = generation_config.get("trtllm_cfg", {}).get( "expose_http_server" ) - elif generation_config["backend"] == "dynamo": - should_expose_http_server = generation_config.get("vllm_cfg", {}).get( - "expose_http_server" - ) else: should_expose_http_server = False assert should_expose_http_server, ( @@ -446,9 +446,11 @@ def _external_staging_backend(token_capture: Dict[str, Any]) -> str: return "vllm_worker" if generation_backend == "megatron": return "megatron_worker" + if generation_backend == "dynamo": + return "dynamo_worker" raise ValueError( "token_capture.enabled requires setup-derived generation_backend to be " - f"'vllm' or 'megatron'; got {generation_backend!r}" + f"'vllm', 'megatron' or 'dynamo'; got {generation_backend!r}" ) @@ -1870,8 +1872,7 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config = config.policy["generation"] # Enable the backend's OpenAI-compatible server. - if generation_config["backend"] in ("vllm", "dynamo"): - # Dynamo uses these flags to expose its token wrapper to Gym. + if generation_config["backend"] == "vllm": generation_config["vllm_cfg"]["async_engine"] = True generation_config["vllm_cfg"]["expose_http_server"] = True elif generation_config["backend"] == "megatron": @@ -1887,7 +1888,7 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["trtllm_cfg"]["expose_http_server"] = True else: raise ValueError( - "NeMo-Gym setup supports vllm, dynamo, megatron, or trtllm " + "NeMo-Gym setup supports vllm, megatron, or trtllm " f"generation; got {generation_config['backend']!r}" ) @@ -1895,6 +1896,29 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_strings"] = None generation_config["stop_token_ids"] = None + if dynamo_enabled(generation_config): + # Gym's policy model calls the Dynamo frontend, which takes token data in + # nvext (Gym's Dynamo dialect) instead of vLLM's request extensions. + vllm_model = ( + config.env.setdefault("nemo_gym", {}) + .setdefault("policy_model", {}) + .setdefault("responses_api_models", {}) + .setdefault("vllm_model", {}) + ) + vllm_model["dynamo_frontend"] = True + # The frontend reads `reasoning` as another name for `reasoning_content` and + # refuses an echoed assistant turn that carries both, Gym's default. + vllm_model.setdefault("reasoning_field", "reasoning_content") + # The native server merges these under every request's own template + # kwargs; the frontend has no such default, so Gym sends them. They sit + # under the kwargs Gym sends: vllm_model's own, else extra_body's. + default_kwargs = native_default_chat_template_kwargs(generation_config) + if default_kwargs: + gym_kwargs = vllm_model.get("chat_template_kwargs") or ( + (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} + ) + vllm_model["chat_template_kwargs"] = {**default_kwargs, **gym_kwargs} + # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for # multi-turn multimodal postprocessing). diff --git a/nemo_rl/experience/rollouts.py b/nemo_rl/experience/rollouts.py index 0384ef6f111..486fccfffaa 100644 --- a/nemo_rl/experience/rollouts.py +++ b/nemo_rl/experience/rollouts.py @@ -753,7 +753,6 @@ async def generate_responses_async( # SGLang exposes ``sglang_cfg`` and gates on ``use_async_rollouts``; # vLLM exposes ``cfg`` and gates on ``vllm_cfg.async_engine``; # TRT-LLM requires its flag; the Megatron backend is always async. - # Managed Dynamo always exposes its rollout frontend asynchronously. vllm_cfg = getattr(policy_generation, "cfg", None) sglang_cfg = getattr(policy_generation, "sglang_cfg", None) generation_config = vllm_cfg or sglang_cfg or {} @@ -765,8 +764,6 @@ async def generate_responses_async( use_async_generation = bool( generation_config.get("vllm_cfg", {}).get("async_engine", False) ) - elif backend == "dynamo": - use_async_generation = True elif backend == "trtllm": assert generation_config.get("trtllm_cfg", {}).get("async_engine", False), ( "TRT-LLM backend requires trtllm_cfg.async_engine=true; the " diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index e043d018150..888015266e3 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -17,6 +17,12 @@ from transformers import PreTrainedTokenizerBase +from nemo_rl.models.generation.dynamo.config import ( + check_dynamo_serving_chat_kwargs, + dynamo_cfg, + dynamo_enabled, + dynamo_parsers, +) from nemo_rl.models.generation.interfaces import GenerationConfig, GenerationInterface from nemo_rl.models.generation.trtllm.config import ( TrtllmConfig, @@ -106,10 +112,6 @@ def resolve_generation_class( from nemo_rl.models.generation.trtllm import TrtllmGeneration return TrtllmGeneration - if backend == "dynamo": - from nemo_rl.models.generation.dynamo import DynamoGeneration - - return DynamoGeneration raise ValueError(f"Unknown generation backend: {backend!r}") @@ -153,6 +155,19 @@ def configure_generation_config( "extend the training worker instead." ) + if dynamo_enabled(config) and config["backend"] != "vllm": + raise ValueError( + "policy.generation.dynamo=true serves NeMo RL's vLLM workers through " + f"Dynamo and does not apply to backend={config['backend']!r}. Set " + "policy.generation.backend=vllm, or remove policy.generation.dynamo." + ) + if config.get("dynamo_cfg") is not None and not dynamo_enabled(config): + raise ValueError( + "policy.generation.dynamo_cfg is set but policy.generation.dynamo is not " + "true, so it would be ignored. Set policy.generation.dynamo=true to " + "serve generation through Dynamo, or remove dynamo_cfg." + ) + # tokenizer setting if "_pad_token_id" in config: warnings.warn( @@ -164,19 +179,42 @@ def configure_generation_config( if config["stop_token_ids"] is None: config["stop_token_ids"] = [tokenizer.eos_token_id] - # vLLM setting shared by the standard and managed Dynamo backends. - if config["backend"] in ("vllm", "dynamo"): - vllm_backed_config = cast(VllmConfig, config) + if config["backend"] == "vllm": + config = cast(VllmConfig, config) # VLM training can require real startup weights for modules that are # frozen and therefore never refitted (for example, the vision tower). # Preserve an explicit recipe override while retaining the lightweight # dummy default for ordinary training. - vllm_backed_config["vllm_cfg"].setdefault( - "load_format", "auto" if is_eval else "dummy" - ) - - if config["backend"] == "vllm": - config = cast(VllmConfig, config) + config["vllm_cfg"].setdefault("load_format", "auto" if is_eval else "dummy") + if dynamo_enabled(config): + # Validate before any Ray allocation; dynamo_parsers also rejects + # parser plugins that Dynamo's Rust frontend cannot run. + dynamo_cfg(config) + dynamo_parsers(config) + check_dynamo_serving_chat_kwargs(config) + if config["vllm_cfg"].get("http_server_api_key_required"): + raise ValueError( + "policy.generation.vllm_cfg.http_server_api_key_required is not " + "supported with policy.generation.dynamo=true: NeMo-Gym calls the " + "Dynamo frontend, which checks no bearer token." + ) + colocated = config.get("colocated") + if colocated is not None and colocated["enabled"]: + raise ValueError( + "policy.generation.dynamo=true requires non-colocated generation " + "(policy.generation.colocated.enabled=false): a colocated engine " + "sleeps while the policy trains, and the Dynamo frontend would " + "keep routing requests to it." + ) + if not config["vllm_cfg"]["async_engine"]: + # Dynamo serves the engine's AsyncLLM. The async engine runs every + # rollout path the sync one does, so turning it on is the switch. + print( + " [Dynamo] policy.generation.dynamo=true: using the async " + "vLLM engine (vllm_cfg.async_engine=true)", + flush=True, + ) + config["vllm_cfg"]["async_engine"] = True validate_vllm_quantization_config(config) validate_nvfp4_pertoken_generation(config, is_eval=is_eval) if config.get("real_quant"): @@ -232,6 +270,8 @@ def configure_generation_config( is_eval or config["stop_strings"] is not None or config["vllm_cfg"].get("expose_http_server", None) + # Dynamo serves OpenAI clients as the exposed server does. + or dynamo_enabled(config) ): config["vllm_cfg"]["skip_tokenizer_init"] = False else: diff --git a/nemo_rl/models/generation/constants.py b/nemo_rl/models/generation/constants.py index 06564f055ef..c00f2297b56 100644 --- a/nemo_rl/models/generation/constants.py +++ b/nemo_rl/models/generation/constants.py @@ -21,5 +21,4 @@ VLLM_BACKEND = "vllm" SGLANG_BACKEND = "sglang" MEGATRON_BACKEND = "megatron" -DYNAMO_BACKEND = "dynamo" TRTLLM_BACKEND = "trtllm" diff --git a/nemo_rl/models/generation/dynamo/__init__.py b/nemo_rl/models/generation/dynamo/__init__.py index 082e6260c19..40376929afa 100644 --- a/nemo_rl/models/generation/dynamo/__init__.py +++ b/nemo_rl/models/generation/dynamo/__init__.py @@ -13,16 +13,16 @@ # limitations under the License. from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, - DynamoConfig, DynamoFrontendArgs, DynamoWorkerArgs, + dynamo_cfg, + dynamo_enabled, ) -from nemo_rl.models.generation.dynamo.dynamo_generation import DynamoGeneration __all__ = [ "DynamoCfg", - "DynamoConfig", "DynamoFrontendArgs", - "DynamoGeneration", "DynamoWorkerArgs", + "dynamo_cfg", + "dynamo_enabled", ] diff --git a/nemo_rl/models/generation/dynamo/arguments.py b/nemo_rl/models/generation/dynamo/arguments.py index 07041493f71..c08d9873200 100644 --- a/nemo_rl/models/generation/dynamo/arguments.py +++ b/nemo_rl/models/generation/dynamo/arguments.py @@ -12,23 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Deterministic argument and environment construction for managed Dynamo.""" +"""Argument construction and log redaction for the managed Dynamo frontend.""" import json from collections.abc import Mapping, Sequence from typing import Any -from nemo_rl.models.generation.dynamo.config import ( - DYNAMO_VLLM_FLAGS, - DynamoCfg, -) - -_MANAGED_FLAGS = { - "--component", - "--model-name", - "--model-path", - "--endpoint", -} +from nemo_rl.models.generation.dynamo.config import DynamoCfg _CREDENTIAL_FLAGS = { "--access-token", @@ -49,10 +39,6 @@ def _normalise_flag(flag: str) -> str: return flag -def _flag_for_key(key: str) -> str: - return "--" + key.replace("_", "-") - - def _serialise_value(value: Any) -> str: if isinstance(value, (dict, list, tuple)): return json.dumps(value, separators=(",", ":"), sort_keys=True) @@ -69,7 +55,7 @@ def add(self, flag: str, value: Any = None, *, source: str) -> None: prior = self.sources.get(normalised) if prior is not None: raise ValueError( - f"Dynamo worker option {normalised} is set by both {prior} and {source}." + f"Dynamo frontend option {normalised} is set by both {prior} and {source}." ) self.sources[normalised] = source if isinstance(value, bool): @@ -89,13 +75,9 @@ def add_raw(self, args: Sequence[str], *, source: str) -> None: current_flag = token current_has_value = "=" in token normalised = _normalise_flag(token) - if normalised in _MANAGED_FLAGS: - raise ValueError( - f"{source} may not override managed option {normalised}." - ) if normalised in self.sources: raise ValueError( - f"Dynamo worker option {normalised} is set by both " + f"Dynamo frontend option {normalised} is set by both " f"{self.sources[normalised]} and {source}." ) self.sources[normalised] = source @@ -109,111 +91,6 @@ def add_raw(self, args: Sequence[str], *, source: str) -> None: self.argv.extend(args) -def build_dynamo_vllm_argv( - *, - model_name: str, - namespace: str, - seed: int, - vllm_cfg: Mapping[str, Any], - vllm_kwargs: Mapping[str, Any], - dynamo_cfg: DynamoCfg, -) -> list[str]: - """Build a conflict-free ``python -m dynamo.vllm`` argument list.""" - builder = _ArgvBuilder() - builder.add("--model", model_name, source="managed runtime") - builder.add("--served-model-name", model_name, source="managed runtime") - builder.add("--namespace", namespace, source="managed runtime") - builder.add("--discovery-backend", "etcd", source="managed runtime") - builder.add("--request-plane", "tcp", source="managed runtime") - builder.add("--event-plane", "nats", source="managed runtime") - builder.add("--enable-rl", source="managed runtime") - builder.add( - "--weight-transfer-config", - {"backend": "nccl"}, - source="managed runtime", - ) - builder.add("--trust-remote-code", source="managed runtime") - builder.add("--seed", seed, source="managed runtime") - - for key, flag in DYNAMO_VLLM_FLAGS.items(): - value = vllm_cfg.get(key) - if value is not None: - builder.add(flag, value, source=f"vllm_cfg.{key}") - - if int(vllm_cfg["expert_parallel_size"]) > 1: - builder.add( - "--enable-expert-parallel", - source="vllm_cfg.expert_parallel_size", - ) - - worker_args = dynamo_cfg.worker_args - if worker_args.tool_call_parser is not None: - builder.add( - "--dyn-tool-call-parser", - worker_args.tool_call_parser, - source="dynamo_cfg.worker_args.tool_call_parser", - ) - if worker_args.reasoning_parser is not None: - builder.add( - "--dyn-reasoning-parser", - worker_args.reasoning_parser, - source="dynamo_cfg.worker_args.reasoning_parser", - ) - builder.add( - "--exclude-tools-when-tool-choice-none", - worker_args.exclude_tools_when_tool_choice_none, - source="dynamo_cfg.worker_args.exclude_tools_when_tool_choice_none", - ) - builder.add( - "--dyn-enable-structural-tag", - worker_args.enable_structural_tag, - source="dynamo_cfg.worker_args.enable_structural_tag", - ) - builder.add( - "--dyn-structural-tag-scope", - worker_args.structural_tag_scope, - source="dynamo_cfg.worker_args.structural_tag_scope", - ) - builder.add( - "--dyn-structural-tag-schema", - worker_args.structural_tag_schema, - source="dynamo_cfg.worker_args.structural_tag_schema", - ) - if worker_args.custom_jinja_template is not None: - builder.add( - "--custom-jinja-template", - worker_args.custom_jinja_template, - source="dynamo_cfg.worker_args.custom_jinja_template", - ) - builder.add( - "--endpoint-types", - ",".join(worker_args.endpoint_types), - source="dynamo_cfg.worker_args.endpoint_types", - ) - - for key, value in vllm_kwargs.items(): - if value is None: - continue - if key == "speculative_config": - raise ValueError( - "policy.generation.vllm_kwargs.speculative_config is not " - "supported by backend='dynamo' because draft weights are not " - "refit after step 0" - ) - flag = _flag_for_key(key) - normalised = _normalise_flag(flag) - source = f"vllm_kwargs.{key}" - if normalised in _MANAGED_FLAGS: - raise ValueError(f"{source} may not override managed option {normalised}.") - builder.add(flag, value, source=source) - - builder.add_raw( - worker_args.extra_cli_args, - source="dynamo_cfg.worker_args.extra_cli_args", - ) - return builder.argv - - def build_dynamo_frontend_argv( *, host: str, @@ -234,6 +111,12 @@ def build_dynamo_frontend_argv( dynamo_cfg.frontend_args.router_mode, source="dynamo_cfg.frontend_args.router_mode", ) + if dynamo_cfg.frontend_args.session_affinity_ttl_secs is not None: + builder.add( + "--router-session-affinity-ttl-secs", + dynamo_cfg.frontend_args.session_affinity_ttl_secs, + source="dynamo_cfg.frontend_args.session_affinity_ttl_secs", + ) builder.add_raw( dynamo_cfg.frontend_args.extra_cli_args, source="dynamo_cfg.frontend_args.extra_cli_args", @@ -241,36 +124,6 @@ def build_dynamo_frontend_argv( return builder.argv -def build_managed_worker_env( - *, - base_env: Mapping[str, str], - configured_env: Mapping[str, str], - manager_env: Mapping[str, str], -) -> dict[str, str]: - """Return a reproducible worker environment with semantic overrides blocked.""" - reserved_env_keys = set(manager_env) - forbidden = sorted( - key - for key in configured_env - if key.startswith("DYN_") or key in reserved_env_keys - ) - if forbidden: - raise ValueError( - "vllm_cfg.env_vars may not override managed Dynamo settings: " - + ", ".join(forbidden) - ) - - env = { - key: value - for key, value in base_env.items() - if not key.startswith(("DYN_", "ETCD_", "NATS_")) - and key not in reserved_env_keys - } - env.update(configured_env) - env.update(manager_env) - return env - - def redact_argv(argv: Sequence[str]) -> list[str]: """Redact values following explicit credential options for safe logging.""" redacted = list(argv) diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 2bf99ba4e12..d08758ce3ed 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -12,143 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Validated configuration for the managed Dynamo generation backend.""" - -import warnings -from typing import Annotated, Any, Literal - -from pydantic import ( - BaseModel, - BeforeValidator, - PositiveFloat, - PositiveInt, - model_validator, -) - -# Packed weight-transfer geometry. NeMo RL declares it to each Dynamo worker in -# the NCCL init info, and the trainer packs with the same values, so both sides -# recompute identical chunk boundaries. -VLLM_PACKED_BUFFER_SIZE_BYTES = 1024**3 -VLLM_PACKED_NUM_BUFFERS = 2 - -DYNAMO_VLLM_FLAGS: dict[str, str] = { - "tensor_parallel_size": "--tensor-parallel-size", - "pipeline_parallel_size": "--pipeline-parallel-size", - "gpu_memory_utilization": "--gpu-memory-utilization", - "max_model_len": "--max-model-len", - "kv_cache_dtype": "--kv-cache-dtype", - "load_format": "--load-format", - "precision": "--dtype", - "enforce_eager": "--enforce-eager", -} - -_VLLM_CFG_STRUCTURAL = { - "env_vars", - "expert_parallel_size", -} - -_VLLM_CFG_MOVED = { - "http_server_serving_chat_kwargs": ( - "dynamo_cfg.worker_args.custom_jinja_template and tool_call_parser" - ), - "reasoning_parser_plugin": "dynamo_cfg.worker_args.reasoning_parser", - "tool_parser_plugin": "dynamo_cfg.worker_args.tool_call_parser", -} - -_VLLM_CFG_UNSUPPORTED = { - # Applied inside NeMo RL's in-process vLLM worker; it cannot configure the - # managed ``dynamo.vllm`` subprocess. - "cap_max_tokens_to_context", - # Enforced by NeMo RL's own HTTP server; the managed Dynamo frontend checks - # no bearer token, so a truthy value is rejected below rather than warned. - "http_server_api_key_required", - "is_mx", - "num_first_layers_in_bf16", - "num_last_layers_in_bf16", - "skip_tokenizer_init", - "use_deep_gemm", -} - -_VLLM_CFG_MANAGED_RUNTIME = { - "enable_vllm_metrics_logger", - "expose_http_server", - "logprobs_mode", - "vllm_metrics_logger_interval", -} - -_VLLM_CFG_INAPPLICABLE = { - "async_engine", - "enable_return_routed_experts", - "http_refit_api_key_env_var", - "http_refit_server_port", - "use_tqdm", - "zmq_refit_server_port", -} - -_VLLM_SINGLE_RANK_ONLY_FIELDS = { - "data_parallel_size", - "decode_context_parallel_size", - "prefill_context_parallel_size", -} - -# Top-level ``policy.generation`` keys outside the declared fields that are accounted -# for; any other active key would be silently dropped, so it is warned about. -_GENERATION_CLASSIFIED = { - # Read by Dynamo generation, or validated in DynamoConfig. - "model_name", - "max_new_tokens", - "temperature", - "top_p", - "top_k", - "stop_token_ids", - "stop_strings", - "_pad_token_id", - "colocated", - "refit_transport", - "quant_cfg", - "real_quant", - "sglang_cfg", - "trtllm_cfg", - # Read by the algorithm, rollouts, or NeMo-Gym rather than the engine. - "val_temperature", - "val_top_p", - "val_top_k", - "port_range_low", - "port_range_high", - "use_async_rollouts", - "_debug_payload_metrics", - "_draft_weights_from_refit", - "_mtp_weights_from_refit", - "_draft_full_refit", - # Shared-YAML sections for other backends and refit paths. - "mcore_generation_config", - "refit_cfg", - "real_quant_export_cpu_offload", - "real_quant_ignore", -} - - -def _is_active(value: Any) -> bool: - """Whether a setting is on; a block with an ``enabled`` switch counts only then.""" - if isinstance(value, BaseModel): - value = value.model_dump() - if isinstance(value, dict) and "enabled" in value: - return bool(value["enabled"]) - return bool(value) +"""Configuration for serving NeMo RL's vLLM workers behind Dynamo. + +``policy.generation.dynamo: true`` on any vLLM config is the whole switch; +``policy.generation.dynamo_cfg`` only tunes the frontend and registration, and +every field below carries its default. +""" + +from typing import Any, Literal, Mapping + +from pydantic import BaseModel, Field, PositiveFloat, PositiveInt, model_validator class DynamoWorkerArgs(BaseModel, extra="forbid"): - """Structured arguments passed to every managed ``dynamo.vllm`` worker.""" - - tool_call_parser: str | None - reasoning_parser: str | None - exclude_tools_when_tool_choice_none: bool - enable_structural_tag: bool - structural_tag_scope: Literal["auto", "always"] - structural_tag_schema: Literal["auto", "strict"] - custom_jinja_template: str | None - endpoint_types: list[Literal["chat", "completions"]] - extra_cli_args: list[str] + """How each vLLM engine registers with Dynamo (``WorkerConfig`` fields).""" + + # Dynamo's built-in parser names (dynamo-parsers). None follows the native + # server's vllm_cfg.http_server_serving_chat_kwargs (see dynamo_parsers). + tool_call_parser: str | None = None + reasoning_parser: str | None = None + # False, as vLLM's OpenAIServingChat (the native server) defaults it. + exclude_tools_when_tool_choice_none: bool = False + enable_structural_tag: bool = False + structural_tag_scope: Literal["auto", "always"] = "auto" + structural_tag_schema: Literal["auto", "strict"] = "auto" + # A Jinja chat template file that replaces the model's own. None follows + # vllm_cfg.http_server_serving_chat_kwargs.chat_template. + custom_jinja_template: str | None = None + endpoint_types: list[Literal["chat", "completions"]] = ["chat", "completions"] @model_validator(mode="after") def _validate_endpoint_types(self) -> "DynamoWorkerArgs": @@ -160,11 +51,13 @@ def _validate_endpoint_types(self) -> "DynamoWorkerArgs": class DynamoFrontendArgs(BaseModel, extra="forbid"): - """Structured arguments passed to the managed Dynamo frontend.""" + """Arguments for the managed ``dynamo.frontend`` process.""" - tokenizer: Literal["default", "fastokens"] - tokenizer_cache: bool - tokenizer_cache_bytes: PositiveInt + tokenizer: Literal["default", "fastokens"] = "default" + tokenizer_cache: bool = False + tokenizer_cache_bytes: PositiveInt = 50 * 1024**2 + # "kv" routes each request to the engine holding most of its prefix, fed by + # the engines' vLLM KV-cache events. router_mode: Literal[ "round-robin", "random", @@ -173,222 +66,154 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "direct", "least-loaded", "device-aware-weighted", - ] - extra_cli_args: list[str] + ] = "kv" + # Idle TTL in seconds for keeping one session on one engine, or None to + # disable it. NeMo-Gym names the session on token-capture calls only (its + # rollout id), as the native backend pins each Gym session to one server; the + # engine splices a call's staged prefix after routing, so the KV router alone + # cannot see where that prefix is cached. + session_affinity_ttl_secs: PositiveInt | None = 1800 + extra_cli_args: list[str] = Field(default_factory=list) class DynamoCfg(BaseModel, extra="forbid"): - """Driver-owned Dynamo service and worker-fleet configuration.""" - - engine: Literal["vllm"] - startup_timeout_s: PositiveFloat - request_timeout_s: PositiveFloat - control_timeout_s: PositiveFloat - worker_args: DynamoWorkerArgs - frontend_args: DynamoFrontendArgs - metrics_include_prefixes: list[str] | None - metrics_exclude_prefixes: list[str] | None + """``policy.generation.dynamo_cfg``: the frontend and engine registration.""" + + # Bounds etcd/NATS/frontend start-up and every engine's registration. + startup_timeout_s: PositiveFloat = 600.0 + # Timeout for one NeMo RL rollout routed through the frontend, in place of + # NRL_VLLM_ASYNC_TIMEOUT_SECONDS; a timed-out rollout is not retried. + # NeMo-Gym's own calls keep Gym's timeouts. + request_timeout_s: PositiveFloat = 900.0 + worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) + frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) + + +# vLLM parser names whose Dynamo built-in parser goes by another name. nano_v3 is +# NeMo RL's vLLM plugin: DeepSeek-R1 reasoning parsing, with the whole output as +# content when chat_template_kwargs.enable_thinking is false, as Dynamo's +# nemotron_v3 parser does. +_DYNAMO_REASONING_PARSER_NAMES = {"nano_v3": "nemotron_v3"} +_DYNAMO_TOOL_PARSER_NAMES = {"openai": "harmony"} + + +def _dynamo_parser( + configured: str | None, + native: str | None, + plugin: str | None, + renames: Mapping[str, str], + *, + kind: str, + worker_arg: str, +) -> str | None: + if configured is not None or native is None: + return configured + if native in renames: + return renames[native] + if plugin: + raise ValueError( + f"policy.generation.vllm_cfg.{kind}_parser_plugin loads a Python parser " + "into vLLM's OpenAI server, which Dynamo's Rust frontend cannot run. Set " + f"policy.generation.dynamo_cfg.worker_args.{worker_arg} to the Dynamo " + f"built-in parser for {native!r}." + ) + return native -class DynamoVllmConfig(BaseModel, extra="allow"): - """Known vLLM settings consumed by ``dynamo.vllm``. +def dynamo_parsers( + generation_config: Mapping[str, Any], +) -> tuple[str | None, str | None]: + """The ``(tool_call_parser, reasoning_parser)`` Dynamo's frontend applies. - Additional fields remain visible in ``model_extra`` so argument construction - can warn rather than silently dropping an inherited vLLM setting. + ``dynamo_cfg.worker_args`` names them explicitly. Otherwise they follow the + native server's ``vllm_cfg.http_server_serving_chat_kwargs``, renamed where + Dynamo calls the same parser differently. A parser plugin without such a + rename has no Dynamo equivalent and is rejected. """ + worker_args = dynamo_cfg(generation_config).worker_args + vllm_cfg = generation_config["vllm_cfg"] + serving = vllm_cfg.get("http_server_serving_chat_kwargs") or {} + tool_call_parser = _dynamo_parser( + worker_args.tool_call_parser, + serving.get("tool_parser"), + vllm_cfg.get("tool_parser_plugin"), + _DYNAMO_TOOL_PARSER_NAMES, + kind="tool", + worker_arg="tool_call_parser", + ) + reasoning_parser = _dynamo_parser( + worker_args.reasoning_parser, + serving.get("reasoning_parser"), + vllm_cfg.get("reasoning_parser_plugin"), + _DYNAMO_REASONING_PARSER_NAMES, + kind="reasoning", + worker_arg="reasoning_parser", + ) + return tool_call_parser, reasoning_parser + + +def check_dynamo_serving_chat_kwargs(generation_config: Mapping[str, Any]) -> None: + """Reject native chat-server settings Dynamo's frontend has no counterpart for. + + The frontend renders templates and parses tool calls itself: it always + parses ``tool_choice="auto"`` calls when a tool parser is set, and has no + content-format setting. The native defaults (``enable_auto_tools: true``, + ``chat_template_content_format: auto``) match it; other values would + silently change what the policy sees. + """ + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + content_format = serving.get("chat_template_content_format") + if content_format not in (None, "auto"): + raise ValueError( + "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." + f"chat_template_content_format={content_format!r} is not supported " + "with policy.generation.dynamo=true: Dynamo's frontend renders chat " + "templates itself and has no content-format setting. Remove it or " + "set it to 'auto'." + ) + if serving.get("enable_auto_tools") is False: + raise ValueError( + "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." + "enable_auto_tools=false is not supported with " + "policy.generation.dynamo=true: Dynamo's frontend always parses " + "tool_choice='auto' calls when a tool parser is set. Remove it, or " + "remove the tool parser." + ) - async_engine: bool - tensor_parallel_size: PositiveInt - pipeline_parallel_size: PositiveInt - expert_parallel_size: PositiveInt - gpu_memory_utilization: float - max_model_len: PositiveInt - kv_cache_dtype: str - load_format: str - precision: str - enforce_eager: bool - expose_http_server: bool - enable_vllm_metrics_logger: bool - vllm_metrics_logger_interval: PositiveFloat - env_vars: dict[str, str] | None - @model_validator(mode="after") - def _validate_parallelism_and_precision(self) -> "DynamoVllmConfig": - if self.expert_parallel_size not in (1, self.tensor_parallel_size): - raise ValueError( - "backend='dynamo' requires expert_parallel_size to be 1 or " - "equal tensor_parallel_size" - ) - if self.precision.lower() not in { - "bf16", - "bfloat16", - }: - raise ValueError( - f"policy.generation.vllm_cfg.precision={self.precision!r} is not " - "supported by backend='dynamo'; managed weight refit currently " - "supports BF16 generation only" - ) - if self.kv_cache_dtype != "auto": - raise ValueError( - f"policy.generation.vllm_cfg.kv_cache_dtype={self.kv_cache_dtype!r} " - "is not supported by backend='dynamo'; use 'auto'" - ) - extra = self.model_extra or {} - if extra.get("is_mx"): - raise ValueError( - "policy.generation.vllm_cfg.is_mx is not supported by " - "backend='dynamo'; use backend='vllm' for MXFP8 generation" - ) - if extra.get("http_server_api_key_required"): - raise ValueError( - "policy.generation.vllm_cfg.http_server_api_key_required is not " - "supported by backend='dynamo'; the managed Dynamo frontend checks " - "no bearer token, so the exposed server would stay open" - ) - if ( - int(extra.get("num_first_layers_in_bf16") or 0) != 0 - or int(extra.get("num_last_layers_in_bf16") or 0) != 0 - ): - raise ValueError( - "mixed BF16/FP8 generation is not supported by backend='dynamo'; " - "use backend='vllm'" - ) - logprobs_mode = extra.get("logprobs_mode") - if logprobs_mode not in (None, "processed_logprobs"): - raise ValueError( - "policy.generation.vllm_cfg.logprobs_mode must be " - "'processed_logprobs' when backend='dynamo'; the managed " - "--enable-rl option selects processed rollout log probabilities" - ) - - configured_fields = self.model_fields_set | set(extra) - for key, replacement in _VLLM_CFG_MOVED.items(): - if extra.get(key) is not None: - raise ValueError( - f"policy.generation.vllm_cfg.{key} is not read by the " - f"Dynamo backend; set {replacement} instead" - ) - for key in sorted(_VLLM_CFG_UNSUPPORTED & configured_fields): - if not extra.get(key): - continue - warnings.warn( - f"policy.generation.vllm_cfg.{key} is ignored by backend='dynamo'", - stacklevel=2, - ) - classified = ( - set(DYNAMO_VLLM_FLAGS) - | _VLLM_CFG_STRUCTURAL - | set(_VLLM_CFG_MOVED) - | _VLLM_CFG_UNSUPPORTED - | _VLLM_CFG_MANAGED_RUNTIME - | _VLLM_CFG_INAPPLICABLE - | _VLLM_SINGLE_RANK_ONLY_FIELDS - ) - unclassified = { - key for key in configured_fields if getattr(self, key, None) is not None - } - classified - if unclassified: - warnings.warn( - "vllm_cfg keys ignored by backend='dynamo': " - f"{sorted(unclassified)}. Add them to DYNAMO_VLLM_FLAGS or an " - "explicit not-forwarded classification.", - stacklevel=2, - ) - return self +def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: + """The chat template the frontend renders with: a file path or literal Jinja. + ``dynamo_cfg.worker_args.custom_jinja_template``, else the native server's + ``http_server_serving_chat_kwargs.chat_template``; None uses the model's own. + """ + configured = dynamo_cfg(generation_config).worker_args.custom_jinja_template + if configured is not None: + return configured + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + return serving.get("chat_template") -def _require_nonempty_vllm_config(value: Any) -> Any: - if not isinstance(value, dict) or not value: - raise ValueError( - "policy.generation.vllm_cfg must be a nonempty mapping when " - "backend='dynamo'" - ) - return value +def native_default_chat_template_kwargs( + generation_config: Mapping[str, Any], +) -> dict[str, Any]: + """The template kwargs the native server merges under every request's own. -class DynamoConfig(BaseModel, extra="allow"): - """Validated boundary for ``policy.generation.backend=dynamo``.""" + ``default_chat_template_kwargs``, or the older ``chat_template_kwargs`` + spelling, of ``vllm_cfg.http_server_serving_chat_kwargs``. + """ + serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} + kwargs = serving.get("default_chat_template_kwargs") + if kwargs is None: + kwargs = serving.get("chat_template_kwargs") + return dict(kwargs or {}) - backend: Literal["dynamo"] - dynamo_cfg: DynamoCfg - vllm_cfg: Annotated[ - DynamoVllmConfig, BeforeValidator(_require_nonempty_vllm_config) - ] - vllm_kwargs: dict[str, Any] - @property - def engine_world_size(self) -> int: - """Return the derived ranks in each single-node vLLM engine.""" - return self.vllm_cfg.tensor_parallel_size * self.vllm_cfg.pipeline_parallel_size +def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: + """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" + return bool(generation_config.get("dynamo")) - @model_validator(mode="after") - def _validate_backend_boundary(self) -> "DynamoConfig": - extra = self.model_extra or {} - # Shared GRPO YAML inheritance supplies mcore_generation_config and - # refit_cfg. Managed Dynamo uses vLLM arguments and the collective - # synchronizer instead, so these two sections are intentionally ignored. - for backend_cfg in ("sglang_cfg", "trtllm_cfg"): - if extra.get(backend_cfg): - raise ValueError( - f"policy.generation.{backend_cfg} is not valid when " - "backend='dynamo'; Dynamo manages vLLM only" - ) - colocated = extra.get("colocated") - if isinstance(colocated, dict) and colocated.get("enabled"): - raise ValueError( - "policy.generation.colocated.enabled must be false when " - "backend='dynamo'" - ) - if extra.get("refit_transport") is not None: - raise ValueError( - "policy.generation.refit_transport must be null when " - "backend='dynamo'; managed Dynamo supports NCCL collective refit only" - ) - for quantization_field in ("quant_cfg", "real_quant"): - if extra.get(quantization_field): - raise ValueError( - f"policy.generation.{quantization_field} is not supported " - "when backend='dynamo'" - ) - speculative_config = self.vllm_kwargs.get("speculative_config") or ( - self.vllm_cfg.model_extra or {} - ).get("speculative_config") - if speculative_config: - raise ValueError( - "policy.generation.vllm_kwargs.speculative_config is not " - "supported by backend='dynamo' because draft weights are not " - "refit after step 0" - ) - if self.vllm_kwargs.get("quantization") is not None: - raise ValueError( - "policy.generation.vllm_kwargs.quantization is not supported " - "when backend='dynamo'" - ) - vllm_extra = self.vllm_cfg.model_extra or {} - for field in sorted(_VLLM_SINGLE_RANK_ONLY_FIELDS): - for source, value in ( - ("vllm_cfg", vllm_extra.get(field)), - ("vllm_kwargs", self.vllm_kwargs.get(field)), - ): - if value is not None and int(value) != 1: - raise ValueError( - f"policy.generation.{source}.{field} must be 1 when " - "backend='dynamo'; managed refit rank geometry is TP × PP" - ) - stop_strings = extra.get("stop_strings") - if stop_strings is not None and len(stop_strings) > 32: - raise ValueError( - "policy.generation.stop_strings supports at most 32 values when " - "backend='dynamo'" - ) - ignored = sorted( - key - for key, value in extra.items() - if key not in _GENERATION_CLASSIFIED and _is_active(value) - ) - if ignored: - warnings.warn( - f"policy.generation keys ignored by backend='dynamo': {ignored}", - stacklevel=2, - ) - return self + +def dynamo_cfg(generation_config: Mapping[str, Any]) -> DynamoCfg: + """The validated ``dynamo_cfg`` block; an absent block means all defaults.""" + return DynamoCfg.model_validate(generation_config.get("dynamo_cfg") or {}) diff --git a/nemo_rl/models/generation/dynamo/dynamo_generation.py b/nemo_rl/models/generation/dynamo/dynamo_generation.py deleted file mode 100644 index f94439feeb1..00000000000 --- a/nemo_rl/models/generation/dynamo/dynamo_generation.py +++ /dev/null @@ -1,767 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Generation and NCCL refit through a driver-owned Dynamo vLLM fleet.""" - -import asyncio -import logging -from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, cast - -import ray -import torch - -from nemo_rl.data.multimodal_utils import VLLM_PROMPT_KEYS -from nemo_rl.distributed.batched_data_dict import BatchedDataDict -from nemo_rl.distributed.virtual_cluster import RayVirtualCluster -from nemo_rl.models.generation.dynamo.config import DynamoConfig -from nemo_rl.models.generation.dynamo.http_client import ( - async_http_post_json, - format_dynamo_error, -) -from nemo_rl.models.generation.dynamo.managed_runtime import ManagedDynamoRuntime -from nemo_rl.models.generation.dynamo.metrics import DynamoMetricsSampler -from nemo_rl.models.generation.dynamo.refit import DynamoRefitChannel -from nemo_rl.models.generation.dynamo.token_wrapper import DynamoTokenWrapperServer -from nemo_rl.models.generation.interfaces import ( - CollectiveSenderSpec, - GenerationDatumSpec, - GenerationInterface, - GenerationOutputSpec, - reject_unenforceable_refit_deadline, - verify_right_padding, -) -from nemo_rl.models.megatron.router_replay import router_replay_enabled - -if TYPE_CHECKING: - from nemo_rl.algorithms.single_controller_utils.config import MasterConfig - -LOGGER = logging.getLogger(__name__) - -_HTTP_MAX_ATTEMPTS = 3 -_HTTP_RETRY_DELAY_S = 1.0 -_RETRYABLE_HTTP_STATUS_CODES = {408, 429} - - -def _is_retryable_http_response(response: Any) -> bool: - """Return whether an internal HTTP error shape represents a transient error.""" - if not isinstance(response, dict): - return False - if "transport_error" in response or response.get("json_decode_error") is True: - return True - status = response.get("http_status") - return isinstance(status, int) and ( - status in _RETRYABLE_HTTP_STATUS_CODES or 500 <= status < 600 - ) - - -def _parse_dynamo_completion_response( - response: dict[str, Any], *, request_url: str -) -> tuple[list[int], list[float], bool]: - """Parse the Dynamo OpenAI completion response for direct generation.""" - if not isinstance(response, dict): - raise RuntimeError( - f"Dynamo completion response from {request_url} was not a JSON object." - ) - if response.get("status") == "error": - raise RuntimeError( - f"Dynamo completion request to {request_url} failed: " - f"{format_dynamo_error(response)}" - ) - - choices = response.get("choices") - if not isinstance(choices, list) or not choices: - raise RuntimeError( - f"Dynamo completion response from {request_url} did not include choices." - ) - choice = choices[0] - if not isinstance(choice, dict): - raise RuntimeError( - f"Dynamo completion response from {request_url} has invalid choice shape." - ) - - nvext = response.get("nvext") - if not isinstance(nvext, dict): - raise RuntimeError( - f"Dynamo completion response from {request_url} did not include nvext." - ) - completion_token_ids = nvext.get("completion_token_ids") - if not isinstance(completion_token_ids, list): - raise RuntimeError( - "Dynamo completion response did not include " - "nvext.completion_token_ids. Ensure the Dynamo frontend is " - "configured to return completion token IDs." - ) - generated_token_ids = [int(token_id) for token_id in completion_token_ids] - - if not generated_token_ids: - return ( - generated_token_ids, - [], - choice.get("finish_reason") == "length", - ) - - logprobs = choice.get("logprobs") - if not isinstance(logprobs, dict): - raise RuntimeError( - f"Dynamo completion response from {request_url} did not include " - "choice.logprobs." - ) - token_logprobs = logprobs.get("token_logprobs") - if not isinstance(token_logprobs, list): - raise RuntimeError( - f"Dynamo completion response from {request_url} did not include " - "choice.logprobs.token_logprobs." - ) - if len(token_logprobs) != len(generated_token_ids): - raise RuntimeError( - f"Dynamo completion response from {request_url} returned " - f"{len(token_logprobs)} token logprobs for " - f"{len(generated_token_ids)} generated tokens." - ) - - generated_logprobs = [] - for idx, logprob in enumerate(token_logprobs): - if not isinstance(logprob, (int, float)) or isinstance(logprob, bool): - raise RuntimeError( - f"Dynamo completion response from {request_url} returned invalid " - f"logprob {logprob!r} for generated token {idx}." - ) - generated_logprobs.append(float(logprob)) - - return ( - generated_token_ids, - generated_logprobs, - choice.get("finish_reason") == "length", - ) - - -class DynamoGeneration(GenerationInterface): - """Own a fixed Dynamo service fleet and expose it for NeMo-RL rollouts.""" - - @classmethod - def validate_settings(cls, master_config: "MasterConfig") -> None: - """Reject SingleController settings the managed Dynamo backend cannot honor.""" - # Both would otherwise fail only after the engines are up: R3 in the Megatron - # worker's setup, a VLM policy at its first multimodal prompt. - if router_replay_enabled(master_config.policy): - raise ValueError( - "policy.router_replay.enabled=true requires " - "policy.generation.backend='vllm'; managed Dynamo does not return " - "routed experts." - ) - if master_config.policy.get("is_vlm"): - raise ValueError( - "policy.is_vlm=true is not supported with " - "policy.generation.backend='dynamo'; managed Dynamo serves text " - "prompts only." - ) - fleet_health = master_config.async_rl.generation_fleet_health - if fleet_health.enabled: - # The Dynamo frontend routes every request across its worker fleet, so - # there are no NeMo RL-selected shards for fleet health to track. - raise ValueError( - "async_rl.generation_fleet_health.enabled=true is not supported with " - "policy.generation.backend='dynamo'; the Dynamo frontend owns request " - "routing. Set async_rl.generation_fleet_health.enabled=false." - ) - if fleet_health.refit_timeout_s is not None: - # Dynamo workers cannot abort their refit collective, so a deadline would - # be refused at the first weight update; fail before anything builds. - raise ValueError( - "async_rl.generation_fleet_health.refit_timeout_s must be null with " - "policy.generation.backend='dynamo'; Dynamo workers cannot enforce a " - "refit deadline." - ) - # Setup has already normalized this through DynamoConfig. - generation = cast(dict[str, Any], master_config.policy["generation"]) - dynamo_cfg = generation["dynamo_cfg"] - if dynamo_cfg["control_timeout_s"] < dynamo_cfg["request_timeout_s"]: - raise ValueError( - "policy.generation.dynamo_cfg.control_timeout_s " - f"({dynamo_cfg['control_timeout_s']}) must be >= request_timeout_s " - f"({dynamo_cfg['request_timeout_s']}): it bounds the pre-refit drain, " - "which waits for running requests to finish." - ) - if ( - generation["vllm_cfg"]["enable_vllm_metrics_logger"] - and master_config.rollout_checkpointing.telemetry_interval_s is None - ): - # The rollout telemetry pump is the only reader of Dynamo's metrics on - # this path; without it the flag would silently do nothing. - raise ValueError( - "policy.generation.vllm_cfg.enable_vllm_metrics_logger requires " - "rollout_checkpointing.telemetry_interval_s with " - "policy.generation.backend='dynamo' under the SingleController; " - "the rollout telemetry pump is what reports Dynamo's metrics." - ) - - def __init__( - self, - cluster: Optional[RayVirtualCluster], - config: dict[str, Any], - tokenizer: Any | None = None, - tokenizer_config: Optional[dict[str, Any]] = None, - ): - validated_config = DynamoConfig.model_validate(config) - self.cfg = validated_config.model_dump() - self._dynamo_cfg = validated_config.dynamo_cfg - dynamo_cfg = self._dynamo_cfg - vllm_cfg = validated_config.vllm_cfg - expose_http_server = vllm_cfg.expose_http_server - tokenizer_chat_template_kwargs: Optional[dict[str, Any]] = None - if expose_http_server: - if tokenizer is None: - raise RuntimeError( - "DynamoGeneration requires a tokenizer when exposing an " - "OpenAI-compatible rollout server." - ) - if ( - tokenizer_config is not None - and "chat_template_kwargs" in tokenizer_config - and tokenizer_config["chat_template_kwargs"] is not None - ): - chat_template_kwargs = tokenizer_config["chat_template_kwargs"] - if not isinstance(chat_template_kwargs, dict): - raise RuntimeError( - "policy.tokenizer.chat_template_kwargs must be a dictionary." - ) - tokenizer_chat_template_kwargs = dict(chat_template_kwargs) - if cluster is None: - raise RuntimeError( - "Managed Dynamo requires a non-colocated inference RayVirtualCluster." - ) - self._managed_runtime: Optional[ManagedDynamoRuntime] = ManagedDynamoRuntime( - cluster=cluster, - config=self.cfg, - ) - self._token_wrapper_server: Optional[DynamoTokenWrapperServer] = None - self._dynamo_frontend_base_url = "" - self.dp_openai_server_base_urls: list[Optional[str]] = [] - self._refit_channel: DynamoRefitChannel | None = None - self._metrics_sampler: DynamoMetricsSampler | None = None - try: - self._managed_runtime.start() - url = self._managed_runtime.frontend_url - self._dynamo_frontend_base_url = url - workers = self._managed_runtime.refit_workers() - self._refit_channel = DynamoRefitChannel( - workers, - engine_world_size=validated_config.engine_world_size, - control_timeout_s=dynamo_cfg.control_timeout_s, - validate_workers=self._managed_runtime.validate_workers, - ) - - if expose_http_server: - self._token_wrapper_server = DynamoTokenWrapperServer( - dynamo_frontend_base_url=url, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=( - dynamo_cfg.worker_args.exclude_tools_when_tool_choice_none - ), - request_timeout_s=dynamo_cfg.request_timeout_s, - ) - wrapper_url = self._token_wrapper_server.start() - self.dp_openai_server_base_urls = [wrapper_url] - print( - " [Dynamo] Forwarding rollout chat requests through token " - f"wrapper {wrapper_url} -> {url}", - flush=True, - ) - else: - self.dp_openai_server_base_urls = [None] - print(f" [Dynamo] Forwarding rollouts to {url}", flush=True) - except Exception: - self.shutdown() - raise - - # ------------------------------------------------------------------ - # GenerationInterface — lifecycle - # ------------------------------------------------------------------ - - def prepare_for_generation(self, *args: Any, **kwargs: Any) -> bool: - return True - - @property - def frontend_url(self) -> str: - """Return the internal managed Dynamo OpenAI frontend URL.""" - if not self._dynamo_frontend_base_url: - raise RuntimeError("DynamoGeneration does not have a frontend URL.") - return self._dynamo_frontend_base_url - - def finish_generation(self, *args: Any, **kwargs: Any) -> bool: - """Invalidate cached rollout state after synchronous generation.""" - return self.invalidate_kv_cache() - - def _logger_sampler(self) -> DynamoMetricsSampler | None: - """Return this process's metrics sampler, starting it on first use. - - The sampler owns a thread and locks, so it cannot travel with a pickled - copy: it starts in whichever process reads the metrics (the driver for - the legacy loops, the SingleController actor for its telemetry pump). - """ - vllm_cfg = self.cfg["vllm_cfg"] - channel = self._refit_channel - if ( - self._metrics_sampler is None - and vllm_cfg["enable_vllm_metrics_logger"] - and channel is not None - ): - self._metrics_sampler = DynamoMetricsSampler( - channel.workers, - interval_s=vllm_cfg["vllm_metrics_logger_interval"], - include_prefixes=self._dynamo_cfg.metrics_include_prefixes, - exclude_prefixes=self._dynamo_cfg.metrics_exclude_prefixes, - ) - self._metrics_sampler.start() - return self._metrics_sampler - - def get_logger_metrics(self) -> dict[str, Any]: - """Return per-worker Dynamo metric timelines for generation logging.""" - sampler = self._logger_sampler() - return {} if sampler is None else sampler.snapshot() - - def clear_logger_metrics(self) -> None: - """Clear the Dynamo metric timelines for the next logging window.""" - sampler = self._logger_sampler() - if sampler is not None: - sampler.clear() - - def drain_latest_logger_metrics(self) -> dict[str, Any]: - """Return each worker's latest Dynamo metrics for rollout telemetry.""" - sampler = self._logger_sampler() - return {} if sampler is None else sampler.drain_latest() - - def get_inference_world_size(self) -> int: - """Return the number of vLLM ranks across all discovered workers.""" - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.inference_world_size - - def get_collective_sender_spec(self) -> CollectiveSenderSpec: - """Return vLLM's NCCL protocol and packed-transfer geometry.""" - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.sender_spec - - def shutdown(self) -> bool: - """Stop process-local helpers and any driver-owned managed runtime.""" - sampler = self._metrics_sampler - self._metrics_sampler = None - if sampler is not None: - try: - sampler.shutdown() - except Exception: - LOGGER.exception("Failed to stop the Dynamo metrics sampler") - token_wrapper_server = self._token_wrapper_server - self._token_wrapper_server = None - if token_wrapper_server is not None: - try: - token_wrapper_server.shutdown() - except Exception: - LOGGER.exception("Failed to stop the Dynamo token wrapper") - managed_runtime = self._managed_runtime - self._managed_runtime = None - if managed_runtime is not None: - try: - managed_runtime.shutdown() - except Exception: - LOGGER.exception("Failed to stop the managed Dynamo runtime") - self._refit_channel = None - return True - - # ------------------------------------------------------------------ - # Pickling — async rollouts ship the GenerationInterface across Ray actors - # ------------------------------------------------------------------ - - def __getstate__(self) -> dict[str, Any]: - """Serialize only HTTP clients needed by Ray rollout actors. - - Driver-owned subprocesses, threads, and Ray worker handles are excluded. - The refit channel's client copy is retained so weight updates and - AREAL-style cache invalidation still reach every managed worker after - deserialization. - """ - refit_channel = self._refit_channel - return { - "cfg": self.cfg, - "dp_openai_server_base_urls": self.dp_openai_server_base_urls, - "_dynamo_frontend_base_url": self._dynamo_frontend_base_url, - "_refit_channel": ( - None if refit_channel is None else refit_channel.client_copy() - ), - } - - def __setstate__(self, state: dict[str, Any]) -> None: - """Restore a client-only rollout copy with no service ownership.""" - self.cfg = state["cfg"] - validated_config = DynamoConfig.model_validate(self.cfg) - self._dynamo_cfg = validated_config.dynamo_cfg - self.dp_openai_server_base_urls = state["dp_openai_server_base_urls"] - frontend_url = state["_dynamo_frontend_base_url"] - if not isinstance(frontend_url, str) or not frontend_url: - raise RuntimeError("Pickled DynamoGeneration has no frontend URL.") - self._dynamo_frontend_base_url = frontend_url - self._token_wrapper_server = None - self._managed_runtime = None - self._metrics_sampler = None - self._refit_channel = state["_refit_channel"] - - def _completion_url(self) -> str: - base_url = self._dynamo_frontend_base_url - if not base_url: - raise RuntimeError("DynamoGeneration does not have a frontend URL.") - return f"{base_url.rstrip('/')}/completions" - - def _request_timeout_s(self) -> float: - return self._dynamo_cfg.request_timeout_s - - def _merge_stop_strings(self, batch_stop_strings: Any) -> Optional[list[str]]: - stop_set: set[str] = set() - - configured_stop_strings = self.cfg.get("stop_strings") - if configured_stop_strings is not None: - stop_set.update(configured_stop_strings) - - if batch_stop_strings is not None: - for sample_stop_strings in batch_stop_strings: - if not sample_stop_strings: - continue - if isinstance(sample_stop_strings, str): - stop_set.add(sample_stop_strings) - else: - stop_set.update(sample_stop_strings) - - if len(stop_set) > 32: - raise ValueError( - "Dynamo supports at most 32 stop strings after merging configured " - "and per-sample values" - ) - return list(stop_set) if stop_set else None - - def _prompt_token_ids( - self, - data: BatchedDataDict["GenerationDatumSpec"], - sample_idx: int, - ) -> list[int]: - if VLLM_PROMPT_KEYS & data.keys(): - raise NotImplementedError( - "DynamoGeneration direct generate() supports token-ID LLM " - "prompts only; multimodal vLLM prompt data is not supported." - ) - - input_length = int(data["input_lengths"][sample_idx].item()) - return data["input_ids"][sample_idx, :input_length].tolist() - - def _build_completion_request( - self, - *, - prompt_token_ids: list[int], - greedy: bool, - stop_strings: Optional[list[str]], - max_new_tokens: int, - ) -> dict[str, Any]: - top_k_cfg = self.cfg["top_k"] - top_k_val = 1 if greedy else (top_k_cfg if top_k_cfg is not None else -1) - - payload: dict[str, Any] = { - "model": self.cfg["model_name"], - "prompt": prompt_token_ids, - "max_tokens": int(max_new_tokens), - "temperature": 0.0 if greedy else self.cfg["temperature"], - "top_p": self.cfg["top_p"], - "top_k": top_k_val, - "n": 1, - "logprobs": 0, - "include_stop_str_in_output": True, - "nvext": {"extra_fields": ["completion_token_ids"]}, - } - - if self.cfg["stop_token_ids"] is not None: - payload["stop_token_ids"] = self.cfg["stop_token_ids"] - if stop_strings is not None: - payload["stop"] = stop_strings - - return payload - - def _allowed_new_tokens(self, input_length: int) -> int: - """Return the generation budget for a prompt.""" - remaining_ctx = int(self.cfg["vllm_cfg"]["max_model_len"]) - input_length - if remaining_ctx <= 0: - raise ValueError( - f"Dynamo prompt length {input_length} must be less than " - f"vllm_cfg.max_model_len={self.cfg['vllm_cfg']['max_model_len']}" - ) - return min(self.cfg["max_new_tokens"], remaining_ctx) - - def _assert_response_within_context( - self, *, input_length: int, generated_length: int - ) -> None: - response_length = input_length + generated_length - max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) - if response_length > max_model_len: - raise AssertionError( - "Dynamo response length exceeded " - f"vllm_cfg.max_model_len: {response_length} > {max_model_len}" - ) - - async def _post_completion_request( - self, - *, - prompt_token_ids: list[int], - greedy: bool, - stop_strings: Optional[list[str]], - max_new_tokens: int, - ) -> tuple[list[int], list[float], bool]: - request_url = self._completion_url() - payload = self._build_completion_request( - prompt_token_ids=prompt_token_ids, - greedy=greedy, - stop_strings=stop_strings, - max_new_tokens=max_new_tokens, - ) - response: dict[str, Any] = {} - for attempt in range(1, _HTTP_MAX_ATTEMPTS + 1): - response = await async_http_post_json( - request_url, - payload, - self._request_timeout_s(), - ) - if not _is_retryable_http_response(response): - break - if attempt == _HTTP_MAX_ATTEMPTS: - break - LOGGER.warning( - "Dynamo completion attempt %d/%d failed; retrying in %.1fs: %s", - attempt, - _HTTP_MAX_ATTEMPTS, - _HTTP_RETRY_DELAY_S, - format_dynamo_error(response), - ) - await asyncio.sleep(_HTTP_RETRY_DELAY_S) - return _parse_dynamo_completion_response(response, request_url=request_url) - - def _single_sample_output( - self, - *, - input_ids: torch.Tensor, - input_length: int, - generated_token_ids: list[int], - generated_logprobs: list[float], - truncated: bool, - ) -> BatchedDataDict["GenerationOutputSpec"]: - output_length = input_length + len(generated_token_ids) - self._assert_response_within_context( - input_length=input_length, - generated_length=len(generated_token_ids), - ) - output_ids = torch.full( - (output_length,), - self.cfg["_pad_token_id"], - dtype=input_ids.dtype, - device=input_ids.device, - ) - output_ids[:input_length] = input_ids[:input_length] - if generated_token_ids: - output_ids[input_length:output_length] = torch.tensor( - generated_token_ids, - dtype=input_ids.dtype, - device=input_ids.device, - ) - - logprobs = torch.zeros( - (1, output_length), - dtype=torch.float32, - device=input_ids.device, - ) - for idx, logprob in enumerate(generated_logprobs[: len(generated_token_ids)]): - logprobs[0, input_length + idx] = logprob - - return BatchedDataDict[GenerationOutputSpec]( - { - "output_ids": output_ids.unsqueeze(0), - "logprobs": logprobs, - "generation_lengths": torch.tensor( - [len(generated_token_ids)], - dtype=torch.long, - device=input_ids.device, - ), - "unpadded_sequence_lengths": torch.tensor( - [output_length], - dtype=torch.long, - device=input_ids.device, - ), - "truncated": torch.tensor( - [truncated], - dtype=torch.bool, - device=input_ids.device, - ), - } - ) - - def generate( - self, - data: BatchedDataDict["GenerationDatumSpec"], - greedy: bool = False, - ) -> BatchedDataDict["GenerationOutputSpec"]: - """Reject the unused blocking interface. - - Both synchronous and asynchronous GRPO trainers use ``generate_async`` - for the managed HTTP frontend. - """ - raise NotImplementedError( - "Dynamo generation uses generate_async() for both synchronous and " - "asynchronous GRPO trainers" - ) - - async def generate_async( - self, - data: BatchedDataDict["GenerationDatumSpec"], - greedy: bool = False, - ) -> AsyncGenerator[tuple[int, BatchedDataDict["GenerationOutputSpec"]], None]: - """Generate one token-ID prompt asynchronously through the managed frontend.""" - assert isinstance(data, BatchedDataDict), ( - f"data must be a BatchedDataDict, got type: {type(data)}" - ) - assert "input_ids" in data and "input_lengths" in data, ( - "input_ids and input_lengths are required in data for Dynamo generation" - ) - if len(data["input_ids"]) == 0: - return - - verify_right_padding(data, pad_value=self.cfg["_pad_token_id"]) - - input_ids_batch = data["input_ids"] - input_lengths_batch = data["input_lengths"] - batch_size = input_ids_batch.shape[0] - assert batch_size == 1, ( - "generate_async is restricted to handle only single samples, " - f"but received batch_size={batch_size}. Please handle batching " - "outside this method." - ) - sample_idx = 0 - input_length = int(input_lengths_batch[sample_idx].item()) - batch_stop_strings = data.get("stop_strings", [[] for _ in range(batch_size)]) - per_sample_stop_strings = None - if batch_stop_strings and sample_idx < len(batch_stop_strings): - per_sample_stop_strings = batch_stop_strings[sample_idx] - final_stop_strings = self._merge_stop_strings( - [per_sample_stop_strings] if per_sample_stop_strings else None - ) - - allowed_new_tokens = self._allowed_new_tokens(input_length) - input_ids = input_ids_batch[sample_idx] - ( - generated_token_ids, - generated_logprobs, - truncated, - ) = await self._post_completion_request( - prompt_token_ids=self._prompt_token_ids(data, sample_idx), - greedy=greedy, - stop_strings=final_stop_strings, - max_new_tokens=allowed_new_tokens, - ) - - yield ( - sample_idx, - self._single_sample_output( - input_ids=input_ids, - input_length=input_length, - generated_token_ids=generated_token_ids, - generated_logprobs=generated_logprobs, - truncated=truncated, - ), - ) - - def init_collective( - self, - ip: str, - port: int, - world_size: int, - *, - train_world_size: int, - ) -> list[ray.ObjectRef]: - """Initialize native vLLM NCCL transfer on every managed worker.""" - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.init_collective( - ip, - port, - world_size, - train_world_size=train_world_size, - ) - - def prepare_refit_info( - self, state_dict_info: Optional[dict[str, Any]] - ) -> Optional[list[str]]: - """Serialize checkpoint-format tensor metadata for native vLLM refit.""" - if state_dict_info is None: - return None - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - channel.prepare(state_dict_info) - return None - - def update_weights_via_ipc_zmq(self) -> list[ray.ObjectRef]: - raise NotImplementedError( - "DynamoGeneration only supports NCCL weight transfer." - ) - - def update_weights_from_collective( - self, refit_timeout_s: Optional[float] = None - ) -> list[ray.ObjectRef]: - """Receive packed checkpoint-format weights on every Dynamo worker.""" - reject_unenforceable_refit_deadline("Dynamo", refit_timeout_s) - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.update_weights() - - def invalidate_kv_cache(self) -> bool: - """Flush every fixed Dynamo worker's prefix/KV cache.""" - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - return channel.flush_cache() - - def pause_generation_for_refit( - self, *, clear_cache: bool, timeout_s: Optional[float] = None - ) -> bool: - """Drain and pause every worker; layerwise reload is unsafe mid-request. - - Unlike vLLM's frozen pause, running requests finish on the current - weights; queued and new requests wait and run on the new ones. Because - the drain waits for running requests, ``dynamo_cfg.control_timeout_s`` - bounds it rather than ``timeout_s``. - """ - del timeout_s - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - channel.pause(clear_cache=clear_cache) - return True - - def resume_generation_after_refit( - self, *, timeout_s: Optional[float] = None - ) -> bool: - """Resume every worker after a weight update.""" - del timeout_s - channel = self._refit_channel - if channel is None: - raise RuntimeError("Dynamo refit channel is unavailable") - channel.resume() - return True diff --git a/nemo_rl/models/generation/dynamo/dynamo_worker.py b/nemo_rl/models/generation/dynamo/dynamo_worker.py deleted file mode 100644 index 624d58ca09a..00000000000 --- a/nemo_rl/models/generation/dynamo/dynamo_worker.py +++ /dev/null @@ -1,304 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Ray actors used by the fixed, managed Dynamo vLLM fleet.""" - -import json -import os -import signal -import socket -import subprocess -import time -from pathlib import Path -from typing import Any - -import ray - -from nemo_rl.distributed.virtual_cluster import _get_free_port_local, _get_node_ip_local -from nemo_rl.models.generation.dynamo.arguments import ( - build_dynamo_vllm_argv, - build_managed_worker_env, - redact_argv, - redact_environment, -) -from nemo_rl.models.generation.dynamo.config import ( - VLLM_PACKED_BUFFER_SIZE_BYTES, - VLLM_PACKED_NUM_BUFFERS, - DynamoConfig, -) -from nemo_rl.models.generation.dynamo.venv import ( - get_dynamo_python, - get_dynamo_venv_dir, -) - - -@ray.remote(num_cpus=0) -class DynamoGpuReservation: # pragma: no cover - """Hold one placement-group GPU while a sibling actor owns the engine.""" - - def __init__(self) -> None: - self._process_pid: int | None = None - - def metadata(self) -> dict[str, Any]: - gpu_ids = [int(float(gpu_id)) for gpu_id in ray.get_gpu_ids()] - if len(gpu_ids) != 1: - raise RuntimeError( - f"Expected one GPU for Dynamo reservation, got {gpu_ids}." - ) - return {"node_ip": _get_node_ip_local(), "gpu_id": gpu_ids[0]} - - def select_free_port( - self, - *, - port_range_low: int, - port_range_high: int, - excluded_ports: list[int], - ) -> int: - """Select an unused node-local port from a half-open range.""" - return _get_free_port_local( - port_range_low, - port_range_high, - max_retries=None, - excluded_ports=set(excluded_ports), - ) - - def register_process_group(self, pid: int) -> bool: - """Record the colocated worker process group for failure cleanup.""" - if self._process_pid not in (None, pid): - raise RuntimeError( - "Dynamo GPU reservation already owns process group " - f"{self._process_pid}; cannot register {pid}." - ) - self._process_pid = pid - return True - - def cleanup_process_group(self) -> bool: - """Best-effort cleanup if the subprocess-owning actor died first.""" - pid = self._process_pid - if pid is None: - return True - try: - os.killpg(pid, signal.SIGTERM) - except ProcessLookupError: - self._process_pid = None - return True - time.sleep(2) - try: - os.killpg(pid, signal.SIGKILL) - except ProcessLookupError: - pass - self._process_pid = None - return True - - -@ray.remote(num_cpus=0) -class DynamoVllmWorker: # pragma: no cover - """Own one ``dynamo.vllm`` subprocess for a model-parallel GPU group.""" - - def __init__( - self, - config: dict[str, Any], - *, - namespace: str, - group_name: str, - cuda_devices: list[int], - system_port: int, - vllm_port: int, - manager_env: dict[str, str], - startup_timeout_s: float, - seed: int, - cleanup_reservation: ray.actor.ActorHandle, - ) -> None: - self._group_name = group_name - self._node_ip = _get_node_ip_local() - self._system_port = system_port - self._vllm_port = vllm_port - self._process: subprocess.Popen | None = None - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: - probe.bind(("0.0.0.0", system_port)) - except OSError as exc: - raise RuntimeError( - f"DYN_SYSTEM_PORT {system_port} is unavailable for {group_name}." - ) from exc - - validated_config = DynamoConfig.model_validate(config) - dynamo_cfg = validated_config.dynamo_cfg - dynamo_python = get_dynamo_python() - dynamo_venv = str(get_dynamo_venv_dir()) - vllm_cfg = validated_config.vllm_cfg.model_dump() - vllm_kwargs = dict(validated_config.vllm_kwargs) - configured_env = dict(vllm_cfg.get("env_vars") or {}) - worker_env = build_managed_worker_env( - base_env=os.environ, - configured_env=configured_env, - manager_env={ - **manager_env, - "CUDA_VISIBLE_DEVICES": ",".join(str(gpu) for gpu in cuda_devices), - "DYN_SYSTEM_PORT": str(system_port), - "PYTHONHASHSEED": "0", - "VLLM_PORT": str(vllm_port), - "VLLM_SKIP_P2P_CHECK": "1", - "VIRTUAL_ENV": dynamo_venv, - "UV_PROJECT_ENVIRONMENT": dynamo_venv, - }, - ) - argv = build_dynamo_vllm_argv( - model_name=config["model_name"], - namespace=namespace, - seed=seed, - vllm_cfg=vllm_cfg, - vllm_kwargs=vllm_kwargs, - dynamo_cfg=dynamo_cfg, - ) - self._validate_argv( - dynamo_python, - argv, - worker_env, - timeout_s=startup_timeout_s, - ) - - command = [dynamo_python, "-m", "dynamo.vllm", *argv] - relevant_env = { - key: value - for key, value in worker_env.items() - if key.startswith(("CUDA_", "DYN_", "ETCD_", "NATS_", "NCCL_", "VLLM_")) - } - print( - f" [Dynamo:{group_name}] launching argv={redact_argv(command)!r} " - f"env={redact_environment(relevant_env)!r} " - f"system_url={self.system_url}", - flush=True, - ) - try: - self._process = subprocess.Popen( - command, env=worker_env, start_new_session=True - ) - ray.get( - cleanup_reservation.register_process_group.remote(self._process.pid) - ) - self._wait_for_system_port(startup_timeout_s) - except Exception: - self._stop_process() - raise - - @property - def system_url(self) -> str: - host = f"[{self._node_ip}]" if ":" in self._node_ip else self._node_ip - return f"http://{host}:{self._system_port}" - - @staticmethod - def _validate_argv( - dynamo_python: str, - argv: list[str], - env: dict[str, str], - *, - timeout_s: float, - ) -> None: - validator = Path(__file__).with_name("validate_dynamo_vllm_args.py") - command = [ - dynamo_python, - str(validator), - json.dumps(argv), - json.dumps( - { - "buffer_size_bytes": VLLM_PACKED_BUFFER_SIZE_BYTES, - "num_buffers": VLLM_PACKED_NUM_BUFFERS, - } - ), - ] - try: - result = subprocess.run( - command, - env=env, - capture_output=True, - text=True, - check=False, - timeout=timeout_s, - ) - except subprocess.TimeoutExpired as error: - raise RuntimeError( - "Resolved dynamo.vllm argument validation exceeded " - f"startup_timeout_s={timeout_s}." - ) from error - if result.returncode != 0: - raise RuntimeError( - "Resolved dynamo.vllm arguments failed validation: " - f"stdout={result.stdout!r}, stderr={result.stderr!r}" - ) - - def _wait_for_system_port(self, timeout_s: float) -> None: - deadline = time.monotonic() + timeout_s - while time.monotonic() < deadline: - process = self._process - if process is not None and process.poll() is not None: - raise RuntimeError( - f"dynamo.vllm for {self._group_name} exited with " - f"code {process.returncode} before its system endpoint was ready." - ) - try: - with socket.create_connection( - ("127.0.0.1", self._system_port), timeout=1 - ): - return - except OSError: - time.sleep(0.5) - raise RuntimeError( - f"dynamo.vllm for {self._group_name} did not open DYN_SYSTEM_PORT " - f"{self._system_port} within {timeout_s}s." - ) - - def metadata(self) -> dict[str, Any]: - process = self._process - if process is None: - raise RuntimeError(f"dynamo.vllm for {self._group_name} is not running.") - return { - "instance_id": self._group_name, - "system_url": self.system_url, - "process_pid": process.pid, - "vllm_port": self._vllm_port, - } - - def is_alive(self) -> bool: - return self._process is not None and self._process.poll() is None - - def _stop_process(self) -> None: - process = self._process - if process is None: - return - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - pass - if process.poll() is None: - try: - process.wait(timeout=15) - except subprocess.TimeoutExpired: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - process.wait(timeout=5) - self._process = None - - def shutdown(self) -> bool: - process = self._process - if process is None: - return True - self._stop_process() - print( - f" [Dynamo:{self._group_name}] stopped pid={process.pid}", - flush=True, - ) - return True diff --git a/nemo_rl/models/generation/dynamo/http_client.py b/nemo_rl/models/generation/dynamo/http_client.py index 8f69bee6b4d..b7d81f3de15 100644 --- a/nemo_rl/models/generation/dynamo/http_client.py +++ b/nemo_rl/models/generation/dynamo/http_client.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Small HTTP client helpers shared by managed Dynamo components.""" +"""HTTP helpers for requests to the managed Dynamo frontend.""" import json -import urllib.error -import urllib.request from typing import Any import aiohttp @@ -37,31 +35,6 @@ def _decode_json_object(body: bytes) -> dict[str, Any]: return decoded -def http_post_json( - url: str, payload: dict[str, Any], timeout_s: float -) -> dict[str, Any]: - """POST JSON and return either the decoded object or an error object.""" - data = json.dumps(payload).encode("utf-8") - request = urllib.request.Request( - url, - data=data, - headers={"Content-Type": "application/json"}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=timeout_s) as response: - body = response.read() - except urllib.error.HTTPError as error: - raw = error.read().decode("utf-8", "replace") if error.fp else "" - return {"status": "error", "http_status": error.code, "raw": raw} - except (urllib.error.URLError, TimeoutError) as error: - return { - "status": "error", - "transport_error": f"{type(error).__name__}: {error}", - } - return _decode_json_object(body) - - async def async_http_post_json( url: str, payload: dict[str, Any], timeout_s: float ) -> dict[str, Any]: @@ -77,7 +50,13 @@ async def async_http_post_json( "http_status": response.status, "raw": body.decode("utf-8", "replace"), } - except (aiohttp.ClientError, TimeoutError) as error: + except TimeoutError: + # Before ClientError: aiohttp's ServerTimeoutError is both. + return { + "status": "error", + "timeout_error": f"no response within {timeout_s}s", + } + except aiohttp.ClientError as error: return { "status": "error", "transport_error": f"{type(error).__name__}: {error}", @@ -86,11 +65,13 @@ async def async_http_post_json( def format_dynamo_error(response: dict[str, Any]) -> str: - """Format an error object returned by :func:`http_post_json`.""" + """Format an error object returned by :func:`async_http_post_json`.""" if "http_status" in response: return f"HTTP {response['http_status']}: {response.get('raw', '')}" if "transport_error" in response: return str(response["transport_error"]) + if "timeout_error" in response: + return str(response["timeout_error"]) if "raw" in response: return str(response["raw"]) return str(response) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index 34e0eb77696..df75a7d9f12 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Driver-owned lifecycle for a fixed Ray-managed Dynamo deployment.""" +"""Driver-owned Dynamo services: etcd, NATS and the OpenAI frontend. + +The vLLM engines are NeMo RL's own workers; each registers itself with these +services (see ``native_engine``), so the runtime only waits to see them. +""" import atexit import json @@ -27,14 +31,13 @@ import time import urllib.error import urllib.request -from typing import Any +from typing import Callable from nemo_rl.distributed.virtual_cluster import ( DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH, DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, - RayVirtualCluster, _get_free_port_local, _get_node_ip_local, ) @@ -43,14 +46,21 @@ redact_argv, redact_environment, ) -from nemo_rl.models.generation.dynamo.config import DynamoConfig -from nemo_rl.models.generation.dynamo.venv import ( - get_dynamo_executable, - get_dynamo_python, -) -from nemo_rl.models.generation.dynamo.worker_pool import FixedDynamoWorkerPool +from nemo_rl.models.generation.dynamo.config import DynamoCfg +from nemo_rl.models.generation.dynamo.venv import get_dynamo_service_executable + +# The engines register on this component and endpoint (see native_engine). +DYNAMO_COMPONENT = "backend" +DYNAMO_ENDPOINT = "generate" LOGGER = logging.getLogger(__name__) +# The KV router's applied-event counter (Dynamo's kvrouter.KV_CACHE_EVENTS_APPLIED), +# one sample per label set. +_KV_EVENTS_APPLIED = re.compile( + r'^\w*kv_cache_events_applied(?:_total)?\{(?=[^}]*status="(?P[^"]*)")' + r"[^}]*\}\s+(?P\S+)$", + re.MULTILINE, +) def _managed_namespace() -> str: @@ -62,25 +72,22 @@ def _managed_namespace() -> str: class ManagedDynamoRuntime: - """Own etcd, NATS, frontend, and a fixed Ray actor worker fleet.""" + """Own etcd, NATS and the frontend that NeMo RL's vLLM engines register with.""" def __init__( - self, - *, - cluster: RayVirtualCluster, - config: dict[str, Any], + self, *, dynamo_cfg: DynamoCfg, model_name: str, frontend_python: str ) -> None: - validated_config = DynamoConfig.model_validate(config) - self._cluster = cluster - self._config = validated_config.model_dump() - self._dynamo_cfg = validated_config.dynamo_cfg - self._engine_world_size = validated_config.engine_world_size - if self._engine_world_size > cluster.num_gpus_per_node: - raise ValueError( - "Managed Dynamo requires each TP/PP engine group to fit on one " - f"node: tp*pp={self._engine_world_size} exceeds " - f"cluster.num_gpus_per_node={cluster.num_gpus_per_node}" - ) + """Describe the services; nothing starts until :meth:`start`. + + Args: + dynamo_cfg: The validated ``policy.generation.dynamo_cfg``. + model_name: The served model; the frontend is ready once it lists it. + frontend_python: An interpreter whose environment has ``ai-dynamo``: + the vLLM workers' own. + """ + self._dynamo_cfg = dynamo_cfg + self._model_name = model_name + self._frontend_python = frontend_python self._namespace = _managed_namespace() self._host = "" self._etcd_port = 0 @@ -95,7 +102,10 @@ def __init__( self._frontend_process: subprocess.Popen | None = None self._etcd_data_dir: str | None = None self._nats_data_dir: str | None = None - self._pool: FixedDynamoWorkerPool | None = None + + @property + def namespace(self) -> str: + return self._namespace @property def frontend_url(self) -> str: @@ -104,13 +114,20 @@ def frontend_url(self) -> str: host = f"[{self._host}]" if ":" in self._host else self._host return f"http://{host}:{self._frontend_port}/v1" + @property + def engine_env(self) -> dict[str, str]: + """The environment each engine's Dynamo worker needs to register.""" + if not self._started: + raise RuntimeError("Managed Dynamo runtime has not been started") + return dict(self._manager_env) + def start(self) -> None: - """Start the complete managed service fleet.""" + """Start etcd, NATS and the frontend.""" if self._started: raise RuntimeError("Managed Dynamo runtime is already started") # Managed services use new process groups and would survive an - # exception that escapes GRPO setup. Keep an interpreter-exit fallback - # in addition to the normal explicit shutdown path. + # exception that escapes setup. Keep an interpreter-exit fallback in + # addition to the normal explicit shutdown path. atexit.register(self.shutdown) self._atexit_registered = True self._host = _get_node_ip_local() @@ -147,15 +164,16 @@ def allocate_port(*, low: int, high: int) -> int: "NATS_SERVER": f"nats://{self._host}:{self._nats_port}", "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", - # Dynamo 1.3.0's legacy tool jail rebuilds response chunks with - # nvext=None. Its qwen3/deepseek v2 path preserves engine_data, - # which NeMo-Gym needs for exact token IDs and log probabilities. + # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. + # Its v2 parser path preserves engine_data, which NeMo-Gym needs for + # exact token IDs and log probabilities. "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", + # A canary request would stall behind a refit pause; engine liveness + # is the Ray actor's, as for every other vLLM worker. "DYN_HEALTH_CHECK_ENABLED": "false", "DYN_SDK_DISABLE_ANSI_LOGGING": "1", - "DYN_RL_INIT_WEIGHTS_TIMEOUT_S": str(self._dynamo_cfg.control_timeout_s), } print( f" [Dynamo] managed environment={redact_environment(self._manager_env)!r}", @@ -164,17 +182,7 @@ def allocate_port(*, low: int, high: int) -> int: try: self._start_etcd() self._start_nats() - self._pool = FixedDynamoWorkerPool( - cluster=self._cluster, - config=self._config, - namespace=self._namespace, - engine_world_size=self._engine_world_size, - manager_env=self._manager_env, - startup_timeout_s=self._dynamo_cfg.startup_timeout_s, - ) - self._pool.start() self._start_frontend() - self._wait_for_frontend(self._pool.size) self._started = True except Exception: self.shutdown() @@ -223,7 +231,7 @@ def _start_etcd(self) -> None: self._etcd_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_etcd_") peer_url = f"http://{self._host}:{self._etcd_peer_port}" command = [ - get_dynamo_executable("etcd"), + get_dynamo_service_executable("etcd"), "--listen-client-urls", f"http://0.0.0.0:{self._etcd_port}", "--advertise-client-urls", @@ -263,7 +271,7 @@ def _start_nats(self) -> None: self._nats_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_nats_") self._nats_process = subprocess.Popen( [ - get_dynamo_executable("nats-server"), + get_dynamo_service_executable("nats-server"), "-js", "-sd", self._nats_data_dir, @@ -298,7 +306,7 @@ def _start_frontend(self) -> None: dynamo_cfg=self._dynamo_cfg, ) command = [ - get_dynamo_python(), + self._frontend_python, "-m", "dynamo.frontend", *argv, @@ -307,6 +315,9 @@ def _start_frontend(self) -> None: f" [Dynamo] launching frontend argv={redact_argv(command)!r}", flush=True ) frontend_env = self._frontend_env() + dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) + frontend_env["VIRTUAL_ENV"] = dynamo_venv + frontend_env["UV_PROJECT_ENVIRONMENT"] = dynamo_venv tokenizer_env = { key: value for key, value in frontend_env.items() @@ -320,102 +331,66 @@ def _start_frontend(self) -> None: command, env=frontend_env, start_new_session=True ) - def _wait_for_frontend(self, expected_workers: int) -> None: + def wait_for_engines( + self, expected_engines: int, check_engines: Callable[[], None] + ) -> None: + """Block until the frontend routes to ``expected_engines`` registered engines. + + ``/health`` lists discovery registrations before the frontend's model + watcher has installed its OpenAI routes, so readiness also needs the + served model on ``/v1/models``; otherwise an immediate request can race + the watcher and get a transient 404. + + ``check_engines`` runs on every poll and raises when an engine exited or + its endpoint stopped, so a failed registration fails at once. + """ health_url = f"http://127.0.0.1:{self._frontend_port}/health" models_url = f"http://127.0.0.1:{self._frontend_port}/v1/models" - expected_model = str(self._config["model_name"]) deadline = time.monotonic() + self._dynamo_cfg.startup_timeout_s - last_counts = (0, 0) - last_models: set[str] = set() + registered = 0 + served: set[str] = set() while time.monotonic() < deadline: - if self._pool is None or not self._pool.is_alive(): - raise RuntimeError( - "A Ray-managed Dynamo vLLM worker exited while the frontend " - "was waiting for model registration." - ) - if ( - self._frontend_process is not None - and self._frontend_process.poll() is not None - ): - raise RuntimeError( - f"Dynamo frontend exited with code {self._frontend_process.returncode}." - ) + self._assert_services_alive() + check_engines() try: with urllib.request.urlopen(health_url, timeout=5) as response: - payload = json.loads(response.read()) + health = json.loads(response.read()) + with urllib.request.urlopen(models_url, timeout=5) as response: + models = json.loads(response.read()) except (urllib.error.URLError, TimeoutError, json.JSONDecodeError): time.sleep(1) continue - generate_ids: set[str] = set() - rl_ids: set[str] = set() - for instance in payload.get("instances", []): - if not isinstance(instance, dict): - continue - if instance.get("namespace") != self._namespace: - continue - if instance.get("component") != "backend": - continue - instance_id = instance.get("instance_id") - if instance_id is None: - continue - if instance.get("endpoint") == "generate": - generate_ids.add(str(instance_id)) - elif instance.get("endpoint") == "rl": - rl_ids.add(str(instance_id)) - last_counts = (len(generate_ids), len(rl_ids)) - if ( - last_counts == (expected_workers, expected_workers) - and generate_ids == rl_ids - ): - # /health reflects discovery registrations before the frontend's - # model watcher has necessarily installed its OpenAI routes. Do - # not expose frontend_url until the served model is visible; - # otherwise an immediate /v1/completions request can race with - # watcher setup and receive a transient 404. - try: - with urllib.request.urlopen(models_url, timeout=5) as response: - models_payload = json.loads(response.read()) - except ( - urllib.error.URLError, - TimeoutError, - json.JSONDecodeError, - ): - time.sleep(1) - continue - last_models = { - str(model["id"]) - for model in models_payload.get("data", []) - if isinstance(model, dict) and model.get("id") is not None + registered = len( + { + str(instance["instance_id"]) + for instance in health.get("instances", []) + if isinstance(instance, dict) + and instance.get("namespace") == self._namespace + and instance.get("component") == DYNAMO_COMPONENT + and instance.get("endpoint") == DYNAMO_ENDPOINT + and instance.get("instance_id") is not None } - if expected_model not in last_models: - time.sleep(1) - continue + ) + served = { + str(model["id"]) + for model in models.get("data", []) + if isinstance(model, dict) and model.get("id") is not None + } + if registered >= expected_engines and self._model_name in served: print( - f" [Dynamo] frontend ready with {expected_workers} generation " - "and RL workers", + f" [Dynamo] frontend {self.frontend_url} routes to " + f"{registered} engine(s)", flush=True, ) return time.sleep(1) raise RuntimeError( - "Dynamo frontend did not observe the fixed worker fleet within " - f"{self._dynamo_cfg.startup_timeout_s}s: expected={expected_workers}, " - f"last_generate={last_counts[0]}, last_rl={last_counts[1]}, " - f"expected_model={expected_model!r}, last_models={sorted(last_models)!r}." + "Dynamo frontend did not see every vLLM engine within " + f"{self._dynamo_cfg.startup_timeout_s}s: expected={expected_engines}, " + f"registered={registered}, model={self._model_name!r}, " + f"served={sorted(served)!r}." ) - def refit_workers(self) -> list[dict[str, Any]]: - self._assert_services_alive() - if self._pool is None: - raise RuntimeError("Managed Dynamo worker pool is not running.") - return self._pool.refit_workers() - - def validate_workers(self, expected: list[dict[str, Any]]) -> list[dict[str, Any]]: - self._assert_services_alive() - if self._pool is None: - raise RuntimeError("Managed Dynamo worker pool is not running.") - return self._pool.validate(expected) - def _assert_services_alive(self) -> None: for label, process in ( ("etcd", self._etcd_process), @@ -428,20 +403,42 @@ def _assert_services_alive(self) -> None: f"Managed Dynamo {label} is not alive (code={code})." ) + def _log_router_kv_events(self) -> None: + """Log how many engine KV-cache events the KV router applied, by status. + + Read from the frontend's metrics before it stops. A KV router that never + applied one routed on load alone. + """ + if self._frontend_process is None or self._frontend_process.poll() is not None: + return + url = f"http://127.0.0.1:{self._frontend_port}/metrics" + try: + with urllib.request.urlopen(url, timeout=5) as response: + metrics = response.read().decode() + except (urllib.error.URLError, TimeoutError) as error: + print( + f" [Dynamo] router KV events: metrics unavailable ({error})", + flush=True, + ) + return + applied: dict[str, float] = {} + for match in _KV_EVENTS_APPLIED.finditer(metrics): + applied[match["status"]] = applied.get(match["status"], 0.0) + float( + match["value"] + ) + print(f" [Dynamo] router applied KV events: {json.dumps(applied)}", flush=True) + def shutdown(self) -> None: """Best-effort, idempotent teardown of every owned resource.""" + try: + self._log_router_kv_events() + except Exception: + LOGGER.exception("Failed to read the Dynamo router's KV-event metrics") try: self._stop_process(self._frontend_process, "frontend") except Exception: LOGGER.exception("Failed to stop the managed Dynamo frontend") self._frontend_process = None - pool = self._pool - self._pool = None - if pool is not None: - try: - pool.shutdown() - except Exception: - LOGGER.exception("Failed to stop the managed Dynamo worker pool") try: self._stop_process(self._nats_process, "NATS") except Exception: diff --git a/nemo_rl/models/generation/dynamo/metrics.py b/nemo_rl/models/generation/dynamo/metrics.py deleted file mode 100644 index daf7a611c32..00000000000 --- a/nemo_rl/models/generation/dynamo/metrics.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Prometheus polling lifecycle for managed Dynamo workers.""" - -import logging -import threading -import urllib.error -import urllib.request -from collections.abc import Sequence -from typing import Any - -from nemo_rl.models.generation.dynamo.refit import DynamoWorkerEndpoint - -LOGGER = logging.getLogger(__name__) - -DEFAULT_METRICS_EXCLUDE_PREFIXES = ("python_", "process_") -CURATED_METRICS_INCLUDE_PREFIXES = ( - "dynamo_component_gpu_cache_usage", - "dynamo_component_inflight_requests", - "dynamo_work_handler_queue_depth", - "dynamo_component_requests_total", - "dynamo_work_handler_time_to_first_response", - "vllm:num_requests_running", - "vllm:num_requests_waiting", - "vllm:kv_cache_usage_perc", - "vllm:generation_tokens", - "vllm:prompt_tokens_total", - "vllm:inter_token_latency", -) -CANONICAL_LOGGER_ALIASES = { - "inflight_batch_sizes": [ - "dynamo_component_inflight_requests", - "vllm_num_requests_running", - ], - "num_pending_samples": [ - "dynamo_work_handler_queue_depth", - "vllm_num_requests_waiting", - ], - "kv_cache_usage_perc": [ - "dynamo_component_gpu_cache_usage_percent", - "vllm_kv_cache_usage_perc", - "vllm_gpu_cache_usage_perc", - ], - "generation_tokens": [ - "vllm_generation_tokens_total", - "vllm_generation_tokens", - ], -} - - -def _http_get_text(url: str, timeout_s: float) -> str | None: - try: - with urllib.request.urlopen(url, timeout=timeout_s) as response: - return response.read().decode("utf-8", "replace") - except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError): - return None - - -def parse_prometheus_metrics( - text: str, - include_prefixes: tuple[str, ...] | None = None, - exclude_prefixes: tuple[str, ...] = DEFAULT_METRICS_EXCLUDE_PREFIXES, -) -> dict[str, float]: - """Parse Prometheus text exposition into summed scalar values.""" - metrics: dict[str, float] = {} - for raw_line in text.splitlines(): - line = raw_line.strip() - if not line or line.startswith("#"): - continue - if "{" in line: - name = line[: line.index("{")] - try: - value_text = line[line.rindex("}") + 1 :] - except ValueError: - continue - else: - parts = line.split(None, 1) - if len(parts) != 2: - continue - name, value_text = parts - if name.endswith(("_bucket", "_created")): - continue - if include_prefixes and not name.startswith(include_prefixes): - continue - if exclude_prefixes and name.startswith(exclude_prefixes): - continue - value_parts = value_text.split() - if not value_parts: - continue - try: - value = float(value_parts[0]) - except ValueError: - continue - key = name.replace(":", "_") - metrics[key] = metrics.get(key, 0.0) + value - return metrics - - -def _with_canonical_aliases(metrics: dict[str, Any]) -> dict[str, Any]: - """Rename the first available source of each canonical logger metric.""" - for alias, sources in CANONICAL_LOGGER_ALIASES.items(): - if alias in metrics: - continue - source = next((name for name in sources if name in metrics), None) - metrics[alias] = dict(metrics[source]) if source is not None else {} - if source is not None: - del metrics[source] - return metrics - - -class DynamoMetricsSampler: - """Own polling thread, samples, and deterministic sampler shutdown.""" - - def __init__( - self, - workers: Sequence[dict[str, Any] | DynamoWorkerEndpoint], - *, - interval_s: float, - include_prefixes: list[str] | None, - exclude_prefixes: list[str] | None, - ) -> None: - self._workers = tuple( - DynamoWorkerEndpoint.from_metadata(worker) - if isinstance(worker, dict) - else worker - for worker in workers - ) - if not self._workers: - raise ValueError("Managed Dynamo metrics require at least one worker") - self._interval_s = interval_s - self._include_prefixes = ( - tuple(include_prefixes) - if include_prefixes is not None - else CURATED_METRICS_INCLUDE_PREFIXES - ) - self._exclude_prefixes = ( - tuple(exclude_prefixes) - if exclude_prefixes is not None - else DEFAULT_METRICS_EXCLUDE_PREFIXES - ) - self._samples: dict[str, dict[int, list[float]]] = {} - self._lock = threading.Lock() - self._stop = threading.Event() - self._thread: threading.Thread | None = None - - def start(self) -> None: - if self._thread is not None: - raise RuntimeError("Dynamo metrics sampler is already started") - self._thread = threading.Thread( - target=self._run, - name="dynamo-metrics-sampler", - daemon=True, - ) - self._thread.start() - - def _run(self) -> None: - self._stop.wait(min(2.0, self._interval_s)) - while not self._stop.is_set(): - for ordinal, worker in enumerate(self._workers): - text = _http_get_text( - f"{worker.system_url}/metrics", - timeout_s=self._interval_s + 2.0, - ) - if self._stop.is_set(): - break - if not text: - continue - metrics = parse_prometheus_metrics( - text, - self._include_prefixes, - self._exclude_prefixes, - ) - with self._lock: - for name, value in metrics.items(): - self._samples.setdefault(name, {}).setdefault( - ordinal, [] - ).append(value) - self._stop.wait(self._interval_s) - - def snapshot(self) -> dict[str, Any]: - with self._lock: - metrics = self._copy_samples() - return _with_canonical_aliases(metrics) - - def drain_latest(self) -> dict[str, Any]: - """Return each worker's latest sample of every metric and drop older ones. - - The SingleController's rollout telemetry polls this, as it does the vLLM - backend's ``drain_latest_logger_metrics``, which keeps histories bounded. - """ - with self._lock: - self._samples = { - name: { - worker_id: samples[-1:] - for worker_id, samples in worker_metrics.items() - } - for name, worker_metrics in self._samples.items() - } - metrics = self._copy_samples() - return _with_canonical_aliases(metrics) - - def _copy_samples(self) -> dict[str, dict[int, list[float]]]: - return { - name: { - worker_id: list(samples) - for worker_id, samples in worker_metrics.items() - } - for name, worker_metrics in self._samples.items() - } - - def clear(self) -> None: - with self._lock: - self._samples = {} - - def shutdown(self) -> None: - self._stop.set() - thread = self._thread - if ( - thread is not None - and thread is not threading.current_thread() - and thread.is_alive() - ): - thread.join(timeout=max(5.0, self._interval_s + 3.0)) - if thread.is_alive(): - LOGGER.warning("Dynamo metrics sampler did not stop before shutdown") - self._thread = None diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py new file mode 100644 index 00000000000..2eb085d3e5d --- /dev/null +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -0,0 +1,930 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serve a NeMo RL vLLM worker's own AsyncLLM through Dynamo's backend SDK. + +The engine process holds no Dynamo-owned vLLM code. This adapter implements +``dynamo.common.backend.LLMEngine``, which imports no vLLM, over the AsyncLLM +that ``VllmAsyncGenerationWorkerImpl`` owns, so refit, pause, sleep, +quantization and speculative decoding are that worker's own. +""" + +from __future__ import annotations + +import asyncio +import functools +import hashlib +import json +import logging +import os +import tempfile +import time +from collections.abc import AsyncGenerator +from typing import TYPE_CHECKING, Any, Optional + +import numpy as np +import torch +from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names +from dynamo.common.backend import ( + EngineConfig, + GenerateChunk, + GenerateRequest, + LLMEngine, + LlmRegistration, + Worker, + WorkerConfig, +) +from dynamo.common.backend.logprobs import ( + extract_from_completion_output, + extract_prompt_logprobs_from_completion_output, + parse_logprob_options, +) +from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource +from dynamo.llm.exceptions import HttpError, InvalidArgument +from vllm import envs as vllm_envs +from vllm.entrypoints.chat_utils import load_chat_template +from vllm.exceptions import GracefulHTTPError +from vllm.inputs import TokensPrompt +from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY +from vllm.sampling_params import ( + RequestOutputKind, + SamplingParams, + StructuredOutputsParams, +) +from vllm.transformers_utils.processor import get_video_processor_cls_name +from vllm.v1.metrics.loggers import StatLoggerBase + +from nemo_rl.data.captured_media import MediaCaptureRejected +from nemo_rl.distributed.virtual_cluster import ( + DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH, + DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, + _get_free_port_local, +) +from nemo_rl.models.generation.dynamo.config import ( + DynamoCfg, + dynamo_cfg, + dynamo_chat_template, + dynamo_parsers, +) +from nemo_rl.models.generation.dynamo.managed_runtime import ( + DYNAMO_COMPONENT, + DYNAMO_ENDPOINT, +) +from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION +from nemo_rl.models.generation.openai_server_utils import ( + derive_prefix_eos_count, + splice_prefix_tokens, +) +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_result, + rollout_result_to_wire, +) +from nemo_rl.models.generation.vllm.utils import ( + chat_generation_tokens, + chat_routed_experts, + off_policy_sampling_error, + remap_multimodal_placeholders, + validate_rollout_prompt, +) + +if TYPE_CHECKING: + from nemo_rl.models.generation.vllm.vllm_worker_async import ( + VllmAsyncGenerationWorkerImpl, + ) + +LOGGER = logging.getLogger(__name__) + +# Annotations NeMo-Gym's Dynamo dialect (vllm_model with dynamo_frontend, Gym#4817) +# sets on its requests; Dynamo forwards nvext.annotations to the engine verbatim. +GYM_ROLLOUT_ANNOTATION = "nemo_gym:rollout" +# A token-capture call's admission JSON. +GYM_CAPTURE_ANNOTATION_PREFIX = "ng_capture:" +# Without capture: the last echoed assistant turn's prompt + generation token +# IDs, which the native server reads from the messages instead. +GYM_PREFIX_ANNOTATION_PREFIX = "ng_prefix:" +_GYM_SAMPLING_KEYS = ("temperature", "top_p", "val_temperature", "val_top_p") + +# The vLLM modality each key of Dynamo's multi_modal_data holds. +_DYNAMO_MEDIA_MODALITIES = { + "image_url": "image", + "video_url": "video", + "audio_url": "audio", +} + +# Filled from the model's generation config when a request leaves them unset, +# as vLLM's OpenAI server fills them. +_GENERATION_CONFIG_KEYS = ( + "repetition_penalty", + "temperature", + "top_k", + "top_p", + "min_p", +) +_SAMPLING_KEYS = ( + *_GENERATION_CONFIG_KEYS, + "seed", + "presence_penalty", + "frequency_penalty", +) + + +def dynamo_kv_events_config( + cfg: DynamoCfg, *, prefix_caching: bool +) -> Optional[dict[str, Any]]: + """The ``kv_events_config`` that feeds Dynamo's KV router. + + None unless the frontend routes by KV and the engine caches prefixes: vLLM + publishes block events only for the blocks its prefix cache stores. + """ + if cfg.frontend_args.router_mode != "kv" or not prefix_caching: + return None + port = _get_free_port_local( + DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH + ) + return { + "enable_kv_cache_events": True, + "publisher": "zmq", + # A wildcard host makes vLLM's publisher bind; any other TCP endpoint + # makes it connect, and Dynamo's ZMQ source connects too, so no event + # would flow. kv_event_sources subscribes on 127.0.0.1. + "endpoint": f"tcp://*:{port}", + } + + +def _annotation_value(request: GenerateRequest, prefix: str) -> Optional[str]: + """The value of the first ```` annotation, or None.""" + for annotation in request.get("annotations") or (): + if annotation.startswith(prefix): + return annotation[len(prefix) :] + return None + + +def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: + """The rollout spec if ``request`` is one of NeMo RL's own rollouts. + + Those come from ``VllmGeneration.generate_async`` and are built here + exactly as the native path builds them. + """ + rollout = _annotation_value(request, ROLLOUT_ANNOTATION) + return None if rollout is None else json.loads(rollout) + + +def _as_f32(value: Optional[float]) -> Optional[float]: + """Dynamo's frontend carries sampling floats as f32 (0.7 arrives as 0.69999998).""" + return None if value is None else float(np.float32(value)) + + +def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: + """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. + + Dynamo sets at most one constraint. The decoding backend is the engine's + own (``structured_outputs_config``), as for the native server. + """ + structural_tag = guided_decoding.get("structural_tag") + if isinstance(structural_tag, dict): + structural_tag = json.dumps(structural_tag) + return StructuredOutputsParams( + json=guided_decoding.get("json"), + regex=guided_decoding.get("regex"), + choice=guided_decoding.get("choice"), + grammar=guided_decoding.get("grammar"), + whitespace_pattern=guided_decoding.get("whitespace_pattern"), + structural_tag=structural_tag, + ) + + +def build_sampling_params( + request: GenerateRequest, + defaults: dict[str, Any], + *, + output_room: int, + overrides: Optional[dict[str, Any]] = None, + output_kind: RequestOutputKind = RequestOutputKind.DELTA, +) -> SamplingParams: + """Map an OpenAI-shaped Dynamo request onto vLLM SamplingParams. + + As vLLM's OpenAI server builds them: the model's generation-config + ``defaults`` for what the request leaves unset, and ``max_tokens`` capped + by the context. Stop strings are applied by Dynamo's Rust detokenizer, so + only token-level stops reach vLLM unless ``overrides`` sets them. + + Args: + request: Dynamo's preprocessed request. + defaults: ``ModelConfig.get_diff_sampling_param()``. + output_room: The context left after the prompt the engine runs, which + media expansion or a prefix splice can make longer than the + request's ``token_ids``. + overrides: SamplingParams fields that replace the request's. + output_kind: How vLLM reports outputs. + """ + sampling = request.get("sampling_options") or {} + passthrough = (request.get("extra_args") or {}).get("sampling_options") or {} + stop = request.get("stop_conditions") or {} + n = sampling.get("n") or 1 + if sampling.get("best_of") not in (None, n): + raise InvalidArgument("best_of is not supported by vLLM's V1 engine") + + kwargs: dict[str, Any] = { + key: defaults[key] for key in _GENERATION_CONFIG_KEYS if key in defaults + } + kwargs.update( + {key: sampling[key] for key in _SAMPLING_KEYS if sampling.get(key) is not None} + ) + kwargs["n"] = n + kwargs["max_tokens"] = min( + limit + for limit in ( + max(1, output_room), + stop.get("max_tokens"), + defaults.get("max_tokens"), + ) + if limit is not None + ) + if stop.get("min_tokens") is not None: + kwargs["min_tokens"] = stop["min_tokens"] + if stop.get("ignore_eos") is not None: + kwargs["ignore_eos"] = stop["ignore_eos"] + stop_ids = set(stop.get("stop_token_ids") or ()) | set( + stop.get("stop_token_ids_hidden") or () + ) + if stop_ids: + kwargs["stop_token_ids"] = sorted(stop_ids) + if sampling.get("guided_decoding"): + kwargs["structured_outputs"] = structured_outputs(sampling["guided_decoding"]) + for key in ("allowed_token_ids", "logprob_token_ids"): + if passthrough.get(key) is not None: + kwargs[key] = passthrough[key] + + logprobs, prompt_logprobs = parse_logprob_options( + request.get("output_options") or {} + ) + # Explicit logprob_token_ids replace top-k selection; vLLM rejects both. + if logprobs is not None and "logprob_token_ids" not in kwargs: + kwargs["logprobs"] = logprobs + if prompt_logprobs is not None: + kwargs["prompt_logprobs"] = prompt_logprobs + kwargs["detokenize"] = False + kwargs.update(overrides or {}) + params = SamplingParams(**kwargs, output_kind=output_kind) + if passthrough.get("bad_words_token_ids") is not None: + # vLLM only tokenizes bad_words strings into this field; a token-in + # client sends the token IDs themselves. + if not hasattr(params, "_bad_words_token_ids"): + raise RuntimeError( + "vLLM SamplingParams has no _bad_words_token_ids; update the " + "bad_words_token_ids passthrough for this vLLM" + ) + params._bad_words_token_ids = passthrough["bad_words_token_ids"] + return params + + +class SnapshotStatLogger(StatLoggerBase): + """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" + + publisher: Optional[Any] = None + num_gpu_blocks: int = 0 + + def __init__(self, vllm_config: Any, engine_index: int = 0) -> None: + self.dp_rank = engine_index + + def record(self, scheduler_stats, iteration_stats, *args, **kwargs) -> None: + cls = type(self) + if scheduler_stats is None or cls.publisher is None: + return + total = max(1, cls.num_gpu_blocks) + usage = scheduler_stats.kv_cache_usage + hit_rate = None + prefix = getattr(scheduler_stats, "prefix_cache_stats", None) + if prefix is not None and prefix.queries > 0: + hit_rate = prefix.hits / prefix.queries + cls.publisher.publish( + self.dp_rank, + ComponentSnapshot( + kv_used_blocks=int(total * usage), + kv_total_blocks=total, + gpu_cache_usage=usage, + kv_cache_hit_rate=hit_rate, + dp_rank=self.dp_rank, + ), + ) + + def log_engine_initialized(self) -> None: + pass + + +class NativeVllmEngine(LLMEngine): + """LLMEngine over a running NeMo RL vLLM worker. Does not own its lifecycle.""" + + def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int): + self._host = host + self._leader_worker_idx = leader_worker_idx + self._llm = host.llm + self._max_model_len = self._llm.vllm_config.model_config.max_model_len + self._defaults = self._llm.model_config.get_diff_sampling_param() + # The train and validation sampling NeMo-Gym rollouts must match, at the + # f32 precision the frontend delivers. + self._gym_sampling_profile = { + key: _as_f32(host.cfg[key]) for key in _GYM_SAMPLING_KEYS + } + + @classmethod + async def from_args(cls, argv=None): + raise NotImplementedError("constructed in-process by the vLLM worker") + + async def start(self, worker_id: int) -> EngineConfig: + del worker_id + vllm_config = self._llm.vllm_config + cache = vllm_config.cache_config + scheduler = vllm_config.scheduler_config + # The KV-event block size is the main-attention group's, which can + # differ from cache_config.block_size on hybrid models. + block_size = cache.block_size + for group in await self._llm.engine_core.call_utility_async( + "get_kv_cache_group_metadata" + ): + if group.get("kind") in ("full_attention", "mla_attention"): + block_size = group["block_size"] + break + SnapshotStatLogger.num_gpu_blocks = cache.num_gpu_blocks or 0 + model = self._host.model_name + return EngineConfig( + model=model, + served_model_name=model, + # No token budget: the engine owns prompt overflow and the output + # cap, as the native server does. A budget would make the frontend + # fill an omitted max_tokens from the prompt it rendered, which a + # NeMo-Gym prefix splice then changes, and refuse NeMo RL's own + # rollouts that fill the context. + llm=LlmRegistration( + context_length=self._max_model_len, + kv_cache_block_size=block_size, + total_kv_blocks=cache.num_gpu_blocks, + max_num_seqs=scheduler.max_num_seqs, + max_num_batched_tokens=scheduler.max_num_batched_tokens, + ), + ) + + async def generate( + self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + if request.get("prompt_embeds"): + raise InvalidArgument( + "Embedding prompts are not served through Dynamo by NeMo RL's " + "vLLM engines." + ) + rollout = _rollout_request(request) + if rollout is not None: + async for chunk in self._generate_rollout(request, rollout, context): + yield chunk + return + annotations = request.get("annotations") or () + if GYM_ROLLOUT_ANNOTATION in annotations: + async for chunk in self._generate_gym(request, context): + yield chunk + return + engine_input = await self._engine_input(request) + prompt_len = len(engine_input["prompt_token_ids"]) + self._check_prompt_fits(prompt_len) + sampling_params = build_sampling_params( + request, self._defaults, output_room=self._max_model_len - prompt_len + ) + try: + # vLLM's queue limits (max_num_queued_reqs / max_num_queued_tokens), + # checked before every request as the native server checks them. + self._llm.check_admission(sampling_params.n, context.id()) + except GracefulHTTPError as error: + raise HttpError(int(error.http_status), error.message) from error + logprobs_requested, prompt_logprobs_requested = parse_logprob_options( + request.get("output_options") or {} + ) + tokenizer = self._llm.tokenizer + completion_tokens: dict[int, int] = {} + prompt_output = None + async for output in self._llm.generate( + engine_input, sampling_params, context.id() + ): + # Under DELTA, vLLM sends every prompt logprob once, at the end of + # prefill; later outputs carry an empty list. + if getattr(output, "prompt_logprobs", None): + prompt_output = output + for completion in output.outputs: + token_ids = list(completion.token_ids) + completion_tokens[completion.index] = completion_tokens.get( + completion.index, 0 + ) + len(token_ids) + if not token_ids and completion.finish_reason is None: + continue + chunk: GenerateChunk = { + "token_ids": token_ids, + "index": completion.index, + } + log_probs, top_logprobs = extract_from_completion_output( + completion, 0, tokenizer=tokenizer, include_bytes=True + ) + if log_probs is not None: + chunk["log_probs"] = log_probs + if logprobs_requested: + chunk["top_logprobs"] = top_logprobs + if completion.finish_reason is not None: + chunk["finish_reason"] = completion.finish_reason + chunk["completion_usage"] = _usage( + output, sum(completion_tokens.values()) + ) + if prompt_logprobs_requested is not None and prompt_output: + prompt_logprobs = ( + extract_prompt_logprobs_from_completion_output( + prompt_output, tokenizer=tokenizer + ) + ) + if prompt_logprobs is not None: + chunk["engine_data"] = {"prompt_logprobs": prompt_logprobs} + yield chunk + + async def _generate_rollout( + self, request: GenerateRequest, rollout: dict[str, Any], context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + """One NeMo RL rollout, served exactly as ``generate_async`` serves it. + + The worker builds the SamplingParams and the shared request pipeline + builds the result, routed experts included, from vLLM's final output. + One chunk carries it all in ``engine_data``; the driver assembles the + same output ``generate_async`` returns. + """ + host = self._host + prompt_token_ids = list(request["token_ids"]) + max_new_tokens = host._rollout_max_new_tokens(len(prompt_token_ids)) + final_output = None + if max_new_tokens == 0: + result = RolloutResult.nothing_generated() + else: + sampling_params = host._rollout_sampling_params( + max_new_tokens=max_new_tokens, + greedy=rollout["greedy"], + stop_strings=rollout["stop_strings"], + ) + async for output in self._llm.generate( + TokensPrompt(prompt_token_ids=prompt_token_ids), + sampling_params, + context.id(), + ): + final_output = output + if final_output is None: + raise RuntimeError(f"No output received for request {context.id()}") + validate_rollout_prompt(prompt_token_ids, final_output.prompt_token_ids) + result = rollout_result( + final_output, + input_length=len(prompt_token_ids), + return_routed_experts=host._return_routed_experts_enabled(), + routed_experts_dtype=host.routed_experts_dtype, + device=torch.device("cpu"), + ) + chunk: GenerateChunk = { + "token_ids": result.token_ids, + "index": 0, + "finish_reason": ( + "length" + if final_output is None + else final_output.outputs[0].finish_reason + ), + "engine_data": { + "rollout": rollout_result_to_wire(result), + "gen_leader_worker_idx": self._leader_worker_idx, + }, + } + if final_output is not None: + chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) + yield chunk + + async def _generate_gym( + self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + """One NeMo-Gym model call, served as the native chat endpoint serves it. + + The request was rendered by the frontend. As on the native server, it must + sample on-policy, its echoed or staged prefix replaces the rendered + history, output is clamped to the context, and a token-capture call stages + its delta from this process. Tokens, log probabilities and routed experts + come from vLLM's final output through the native endpoint's helpers. One + chunk carries the whole completion and, in ``engine_data``, the token + arrays Gym's dialect reads, or a captured call's commit coordinates. + """ + host = self._host + sampling: dict[str, Any] = request.get("sampling_options") or {} + sampling_error = off_policy_sampling_error( + temperature=_as_f32(sampling.get("temperature")), + top_p=_as_f32(sampling.get("top_p")), + top_k=sampling.get("top_k"), + generation_config=self._gym_sampling_profile, + ) + if sampling_error is not None: + raise InvalidArgument(sampling_error) + if (sampling.get("n") or 1) != 1: + raise InvalidArgument( + "NeMo-Gym calls through Dynamo take n=1: engine_data carries one " + "choice's tokens." + ) + # The profile's own values, so vLLM samples exactly as natively. + validation = ( + _as_f32(sampling["temperature"]) + != self._gym_sampling_profile["temperature"] + or _as_f32(sampling["top_p"]) != self._gym_sampling_profile["top_p"] + ) + temperature, top_p = ( + (host.cfg["val_temperature"], host.cfg["val_top_p"]) + if validation + else (host.cfg["temperature"], host.cfg["top_p"]) + ) + + capture_context = _annotation_value(request, GYM_CAPTURE_ANNOTATION_PREFIX) + admission = ( + None + if capture_context is None + else host._parse_capture_admission(json.loads(capture_context)) + ) + prefix_token_ids = None + if admission is not None: + if admission.mode == "token_in": + # A blocking data-plane read; keep it off the engine loop. + prefix_token_ids = await asyncio.to_thread( + host._resolve_admission_prefix, admission + ) + elif ( + echoed := _annotation_value(request, GYM_PREFIX_ANNOTATION_PREFIX) + ) is not None: + prefix_token_ids = json.loads(echoed) + # As the native endpoint does: a captured call reads the processor's own + # pixels, not cache references (#4124). + engine_input = await self._engine_input( + request, skip_mm_cache=admission is not None and host._capture_media + ) + template_token_ids = engine_input["prompt_token_ids"] + self._check_prompt_fits(len(template_token_ids)) + token_ids, splice = template_token_ids, None + if prefix_token_ids is not None: + # The frontend rendered the whole conversation; replace its history + # with the exact tokens the model generated for it. The engine sees no + # messages, so the prefix's EOS count stands in for the native + # endpoint's render of the history. + tokenizer = self._llm.tokenizer + splice = splice_prefix_tokens( + tokenizer=tokenizer, + model_prefix_token_ids=prefix_token_ids, + template_token_ids=template_token_ids, + template_prefix_eos_count=derive_prefix_eos_count( + prefix_token_ids, eos_token_id=tokenizer.eos_token_id + ), + ) + token_ids = splice.token_ids + self._check_prompt_fits(len(token_ids)) + try: + # Off the loop: a retained-media check reads staged calls. + media = await asyncio.to_thread( + host._capture_request_media, + engine_input, + admission=admission, + splice=splice, + ) + except MediaCaptureRejected as error: + raise InvalidArgument(f"{error} ({error.code})") from error + if ( + media is None + and splice is not None + and (placeholders := engine_input.get("mm_placeholders")) + ): + # Media capture already moved them to spliced coordinates. + engine_input["mm_placeholders"] = remap_multimodal_placeholders( + template_token_ids=template_token_ids, + final_token_ids=token_ids, + mm_placeholders=placeholders, + splice=splice, + ) + engine_input["prompt_token_ids"] = token_ids + + stop = (request.get("stop_conditions") or {}).get("stop") + overrides: dict[str, Any] = { + "temperature": temperature, + "top_p": top_p, + # As the native endpoint does: Gym requests leave top_k to NeMo RL. + "top_k": -1, + } + if stop: + # vLLM stops on them as natively, so the token arrays match; the + # frontend then cuts the text at the same place. + overrides.update(stop=stop, detokenize=True) + if admission is not None: + # A captured call stages the sampled tokens' log probabilities. + overrides["logprobs"] = 0 + sampling_params = build_sampling_params( + request, + self._defaults, + output_room=self._max_model_len - len(token_ids), + overrides=overrides, + output_kind=RequestOutputKind.FINAL_ONLY, + ) + try: + self._llm.check_admission(1, context.id()) + except GracefulHTTPError as error: + raise HttpError(int(error.http_status), error.message) from error + + if admission is not None: + host._begin_request_capture( + request, + token_ids, + admission=admission, + prefix_token_ids=prefix_token_ids, + media=media, + ) + staged = False + try: + final_output = None + async for output in self._llm.generate( + engine_input, sampling_params, context.id() + ): + final_output = output + if final_output is None: + raise RuntimeError(f"No output received for request {context.id()}") + completion = final_output.outputs[0] + chunk: GenerateChunk = { + "token_ids": list(completion.token_ids), + "index": 0, + "finish_reason": completion.finish_reason, + "completion_usage": _usage(final_output, len(completion.token_ids)), + } + if admission is not None: + chunk["engine_data"] = { + "ng_commit_coords": await self._stage_gym_call( + request, final_output + ) + } + staged = True + else: + chunk["engine_data"] = self._gym_token_data(final_output) + log_probs, top_logprobs = extract_from_completion_output( + completion, 0, tokenizer=self._llm.tokenizer, include_bytes=True + ) + if log_probs is not None: + chunk["log_probs"] = log_probs + chunk["top_logprobs"] = top_logprobs + yield chunk + finally: + if admission is not None and not staged: + host._abort_request_capture( + request, + reason=( + "client_disconnect" if context.is_stopped() else "engine_error" + ), + ) + + async def _load_media( + self, request: GenerateRequest + ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: + """The request's media and their UUIDs by modality, as vLLM's chat parser loads them. + + Dynamo passes each media URL through (data: URIs included); the engine + fetches and decodes it with vLLM's own media connector and settings. + """ + media = request.get("multi_modal_data") + if not media: + return None + model_config = self._llm.model_config + mm_config = model_config.multimodal_config + connector = MEDIA_CONNECTOR_REGISTRY.load( + vllm_envs.VLLM_MEDIA_CONNECTOR, + media_io_kwargs=request.get("media_io_kwargs") + or (mm_config.media_io_kwargs if mm_config else None), + allowed_local_media_path=model_config.allowed_local_media_path, + allowed_media_domains=model_config.allowed_media_domains, + ) + fetch = { + "image": connector.fetch_image_async, + "audio": connector.fetch_audio_async, + } + if "video_url" in media: + fetch["video"] = functools.partial( + connector.fetch_video_async, + video_processor=get_video_processor_cls_name(model_config), + ) + uuids = request.get("multi_modal_uuids") or {} + mm_data: dict[str, list[Any]] = {} + mm_uuids: dict[str, list[Any]] = {} + for key, items in media.items(): + modality = _DYNAMO_MEDIA_MODALITIES.get(key) + if modality is None: + raise InvalidArgument(f"Unsupported media type {key!r}") + if any("Url" not in item for item in items): + raise InvalidArgument( + f"{key} items must carry their URL: NeMo RL's vLLM engines load " + "media themselves" + ) + try: + mm_data[modality] = list( + await asyncio.gather( + *(fetch[modality](item["Url"]) for item in items) + ) + ) + except ValueError as error: + raise InvalidArgument(f"Cannot load {key}: {error}") from error + mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) + return mm_data, mm_uuids + + async def _engine_input( + self, request: GenerateRequest, *, skip_mm_cache: bool = False + ) -> dict[str, Any]: + """The processed vLLM prompt for the request, as the native chat endpoint builds it. + + The frontend's tokens carry one unexpanded placeholder per media item; + vLLM's processor expands them and processes the media, exactly as after + the native endpoint renders and tokenizes the messages. + """ + prompt = TokensPrompt(prompt_token_ids=list(request["token_ids"])) + media = await self._load_media(request) + if media is not None: + prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media + if request.get("mm_processor_kwargs"): + prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] + return await self._llm.renderer.process_for_engine_async( + prompt, time.time(), skip_mm_cache=skip_mm_cache + ) + + def _check_prompt_fits(self, prompt_len: int) -> None: + """Refuse a prompt that leaves no room for output, as the native endpoint does. + + NeMo-Gym reads "context length" in a 400 as a length refusal rather than an + engine failure. + """ + if prompt_len >= self._max_model_len: + raise InvalidArgument( + f"Prompt length ({prompt_len}) fills or exceeds this model's " + f"maximum context length ({self._max_model_len}). No room for " + "output tokens." + ) + + @staticmethod + def _gym_token_data(final_output: Any) -> dict[str, Any]: + """The token arrays Gym's dialect lifts onto the message, as the native endpoint reports them.""" + token_data: dict[str, Any] = { + "prompt_token_ids": list(final_output.prompt_token_ids), + "completion_token_ids": list(final_output.outputs[0].token_ids), + } + if final_output.outputs[0].logprobs is not None: + _, token_data["completion_logprobs"] = chat_generation_tokens( + final_output.outputs[0] + ) + return token_data + + async def _stage_gym_call(self, request: GenerateRequest, final_output: Any) -> Any: + """Stage a captured call from this process; return its commit coordinates.""" + host = self._host + completion = final_output.outputs[0] + generation_token_ids, generation_log_probs = chat_generation_tokens(completion) + message: dict[str, Any] = { + "generation_token_ids": generation_token_ids, + "generation_log_probs": generation_log_probs, + } + if host._return_routed_experts_enabled(): + message["routed_experts"] = chat_routed_experts( + final_output, + completion, + prompt_token_count=len(final_output.prompt_token_ids), + device=torch.device("cpu"), + logger=LOGGER, + routed_experts_dtype=host.routed_experts_dtype, + ) + # The native endpoint's response shape; the prompt is the one recorded at + # begin_call. + content = { + "choices": [ + { + "index": 0, + "finish_reason": completion.finish_reason, + "message": message, + } + ] + } + # The sink write is a blocking data-plane round trip. + content = await asyncio.to_thread( + host._finish_request_capture, request, content + ) + return content["ng_commit_coords"] + + async def abort(self, context: Context) -> None: + await self._llm.abort(context.id()) + + async def kv_event_sources(self) -> list[KvEventSource]: + kv_events = self._llm.vllm_config.kv_events_config + if kv_events is None or not kv_events.enable_kv_cache_events: + return [] + return [ZmqSource(endpoint=kv_events.endpoint.replace("*", "127.0.0.1"))] + + def component_metrics_dp_ranks(self) -> list[int]: + return [0] + + def attach_snapshot_publisher(self, publisher: Any) -> None: + SnapshotStatLogger.publisher = publisher + + async def cleanup(self) -> None: + # The worker owns the AsyncLLM; its shutdown() tears the engine down. + SnapshotStatLogger.publisher = None + + +def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: + prompt_tokens = len(output.prompt_token_ids or ()) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": num_output_tokens, + "total_tokens": prompt_tokens + num_output_tokens, + "prompt_tokens_details": {"cached_tokens": output.num_cached_tokens or 0}, + } + + +def _check_parser_name(name: Optional[str], known: list[str], field: str) -> None: + if name is not None and name not in known: + raise ValueError( + f"Dynamo has no {field.replace('_', ' ')} named {name!r}; set " + f"policy.generation.dynamo_cfg.worker_args.{field} to one of " + f"{sorted(known)}." + ) + + +def _chat_template_file(chat_template: Optional[str]) -> Optional[str]: + """A file holding ``chat_template``, which Dynamo reads by path. + + Resolved as the native server resolves it (vLLM's ``load_chat_template``): + a file, a vLLM built-in template name, or literal Jinja. Named by content, + so every engine on a node, and every restart, shares one file. + """ + if chat_template is None: + return None + content = load_chat_template(chat_template) + digest = hashlib.sha256(content.encode()).hexdigest()[:16] + path = os.path.join(tempfile.gettempdir(), f"nemo_rl_chat_template_{digest}.jinja") + if not os.path.exists(path): + # Written aside and renamed, so a concurrent reader never sees a part. + with tempfile.NamedTemporaryFile( + "w", dir=os.path.dirname(path), suffix=".jinja", delete=False + ) as template_file: + template_file.write(content) + os.replace(template_file.name, path) + return path + + +def start_dynamo_endpoint( + host: "VllmAsyncGenerationWorkerImpl", + *, + namespace: str, + engine_env: dict[str, str], + leader_worker_idx: int, +) -> asyncio.Task: + """Register ``host``'s engine with Dynamo and serve it on the running loop. + + The Rust Worker serves until the process gets SIGTERM. It installs a chained + handler, so Ray's actor handler still runs and the actor exits as usual; + discovery drops the engine with its process. + """ + # The Rust runtime reads these when the first Worker is constructed. + os.environ.update(engine_env) + kv_events = host.llm.vllm_config.kv_events_config + publishes = kv_events is not None and kv_events.enable_kv_cache_events + # The Dynamo functional overlays check the router applied these. + print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) + worker_args = dynamo_cfg(host.cfg).worker_args + tool_call_parser, reasoning_parser = dynamo_parsers(host.cfg) + _check_parser_name(tool_call_parser, get_tool_parser_names(), "tool_call_parser") + _check_parser_name( + reasoning_parser, get_reasoning_parser_names(), "reasoning_parser" + ) + config = WorkerConfig( + namespace=namespace, + component=DYNAMO_COMPONENT, + endpoint=DYNAMO_ENDPOINT, + model_name=host.model_name, + served_model_name=host.model_name, + endpoint_types=",".join(worker_args.endpoint_types), + discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], + request_plane=engine_env["DYN_REQUEST_PLANE"], + event_plane=engine_env["DYN_EVENT_PLANE"], + custom_jinja_template=_chat_template_file(dynamo_chat_template(host.cfg)), + tool_call_parser=tool_call_parser, + reasoning_parser=reasoning_parser, + exclude_tools_when_tool_choice_none=( + worker_args.exclude_tools_when_tool_choice_none + ), + structural_tag_mode="on" if worker_args.enable_structural_tag else "off", + structural_tag_scope=worker_args.structural_tag_scope, + structural_tag_schema=worker_args.structural_tag_schema, + ) + return asyncio.get_running_loop().create_task( + Worker(NativeVllmEngine(host, leader_worker_idx), config).run() + ) diff --git a/nemo_rl/models/generation/dynamo/refit.py b/nemo_rl/models/generation/dynamo/refit.py deleted file mode 100644 index 4df856747b6..00000000000 --- a/nemo_rl/models/generation/dynamo/refit.py +++ /dev/null @@ -1,320 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""NCCL refit protocol for a fixed managed Dynamo worker fleet.""" - -from collections.abc import Callable, Sequence -from dataclasses import dataclass -from typing import Any - -import ray - -from nemo_rl.models.generation.dynamo.config import ( - VLLM_PACKED_BUFFER_SIZE_BYTES, - VLLM_PACKED_NUM_BUFFERS, -) -from nemo_rl.models.generation.dynamo.http_client import ( - format_dynamo_error, - http_post_json, -) -from nemo_rl.models.generation.interfaces import CollectiveSenderSpec - - -@dataclass(frozen=True) -class DynamoWorkerEndpoint: - """Serializable identity and admin endpoint for one Dynamo vLLM engine.""" - - instance_id: str - system_url: str - - @classmethod - def from_metadata(cls, metadata: dict[str, Any]) -> "DynamoWorkerEndpoint": - return cls( - instance_id=str(metadata["instance_id"]), - system_url=str(metadata["system_url"]), - ) - - -@ray.remote(num_cpus=0) -def _post_worker_route( # pragma: no cover - *, - system_url: str, - route: str, - payload: dict[str, Any], - timeout_s: float, -) -> bool: - response = http_post_json( - f"{system_url}/engine/{route}", - payload, - timeout_s, - ) - if response.get("status") != "ok": - raise RuntimeError( - f"Dynamo worker {system_url} route {route} failed: " - f"{format_dynamo_error(response)}" - ) - return True - - -@ray.remote(num_cpus=0) -def _update_worker_weights( # pragma: no cover - *, - system_url: str, - update_info: dict[str, Any], - timeout_s: float, -) -> bool: - common = {"allow_unpaused": True, "reset_prefix_cache": False} - steps: tuple[tuple[str, dict[str, Any]], ...] = ( - ("start_weight_update", {}), - ("update_weights", {"update_info": update_info}), - ("finish_weight_update", {}), - ) - for engine_rpc, kwargs in steps: - response = http_post_json( - f"{system_url}/engine/update_weights_from_distributed", - {"engine_rpc": engine_rpc, **common, **kwargs}, - timeout_s, - ) - if response.get("status") != "ok": - raise RuntimeError( - f"Dynamo worker {system_url} RPC {engine_rpc} failed: " - f"{format_dynamo_error(response)}" - ) - return True - - -class DynamoRefitChannel: - """Closed refit protocol shared by driver and serialized rollout copies.""" - - def __init__( - self, - workers: Sequence[dict[str, Any] | DynamoWorkerEndpoint], - *, - engine_world_size: int, - control_timeout_s: float, - validate_workers: Callable[[list[dict[str, Any]]], list[dict[str, Any]]] - | None = None, - ) -> None: - self._worker_metadata = [ - dict(worker) for worker in workers if isinstance(worker, dict) - ] - self._workers = tuple( - DynamoWorkerEndpoint.from_metadata(worker) - if isinstance(worker, dict) - else worker - for worker in workers - ) - if not self._workers: - raise ValueError("Dynamo refit requires at least one worker endpoint") - self._engine_world_size = engine_world_size - self._control_timeout_s = control_timeout_s - self._validate_workers = validate_workers - self._update_info: dict[str, Any] | None = None - - def client_copy(self) -> "DynamoRefitChannel": - """Return a serializable copy without the driver-owned worker validator. - - The prepared update metadata travels with the copy: SingleController - prepares refit on the driver and then pickles the synchronizer (and - this generation client) into its actor, which issues every update. - """ - copy = DynamoRefitChannel( - self._workers, - engine_world_size=self._engine_world_size, - control_timeout_s=self._control_timeout_s, - ) - copy._update_info = self._update_info - return copy - - def _validated_workers(self) -> tuple[DynamoWorkerEndpoint, ...]: - if self._validate_workers is None: - return self._workers - current = self._validate_workers(self._worker_metadata) - endpoints = tuple(DynamoWorkerEndpoint.from_metadata(item) for item in current) - if endpoints != self._workers: - raise RuntimeError( - "Managed Dynamo worker membership changed after collective setup" - ) - return endpoints - - @property - def workers(self) -> tuple[DynamoWorkerEndpoint, ...]: - """Return the managed workers' endpoints, as fixed at collective setup.""" - return self._workers - - @property - def inference_world_size(self) -> int: - return len(self._workers) * self._engine_world_size - - @property - def sender_spec(self) -> CollectiveSenderSpec: - """Return the peer and packing geometry declared to workers at init.""" - return CollectiveSenderSpec( - nccl_peer="vllm", - buffer_size_bytes=VLLM_PACKED_BUFFER_SIZE_BYTES, - num_buffers=VLLM_PACKED_NUM_BUFFERS, - use_legacy_packing=True, - ) - - def prepare(self, state_dict_info: dict[str, Any] | None) -> None: - if state_dict_info is None: - raise ValueError("state_dict_info must not be None for Dynamo refit") - names: list[str] = [] - dtype_names: list[str] = [] - shapes: list[list[int]] = [] - for name, (shape, dtype) in state_dict_info.items(): - names.append(name) - dtype_names.append(str(dtype).removeprefix("torch.")) - shapes.append(list(shape)) - self._update_info = { - "names": names, - "dtype_names": dtype_names, - "shapes": shapes, - } - - def init_collective( - self, - ip: str, - port: int, - world_size: int, - *, - train_world_size: int, - ) -> list[ray.ObjectRef]: - expected_world_size = train_world_size + self.inference_world_size - if world_size != expected_world_size: - raise ValueError( - f"NCCL world_size={world_size} does not match expected " - f"{expected_world_size}" - ) - workers = self._validated_workers() - return [ - _post_worker_route.remote( - system_url=worker.system_url, - route="init_weights_update_group", - payload={ - "engine_rpc": "init_weight_transfer_engine", - "init_info": { - "master_address": ip, - "master_port": port, - "rank_offset": train_world_size - + worker_index * self._engine_world_size, - "world_size": world_size, - # vLLM reads packing once at init and defaults it off; - # omitting it desynchronizes the packed broadcast. - "packed": True, - "packed_buffer_size_bytes": VLLM_PACKED_BUFFER_SIZE_BYTES, - "packed_num_buffers": VLLM_PACKED_NUM_BUFFERS, - }, - }, - timeout_s=self._control_timeout_s, - ) - for worker_index, worker in enumerate(workers) - ] - - def update_weights(self) -> list[ray.ObjectRef]: - if self._update_info is None: - raise RuntimeError( - "prepare_refit_info() must be called before Dynamo weight updates" - ) - return [ - _update_worker_weights.remote( - system_url=worker.system_url, - update_info=self._update_info, - timeout_s=self._control_timeout_s, - ) - for worker in self._validated_workers() - ] - - def _pause_workers( - self, *, clear_cache: bool - ) -> tuple[list[DynamoWorkerEndpoint], list[str]]: - """Drain in-flight requests, optionally clear caches; new requests wait.""" - pause_futures = [ - _post_worker_route.remote( - system_url=worker.system_url, - route="pause_generation", - payload={"mode": "wait", "clear_cache": clear_cache}, - timeout_s=self._control_timeout_s, - ) - for worker in self._workers - ] - paused_workers: list[DynamoWorkerEndpoint] = [] - pause_errors: list[str] = [] - for worker, future in zip(self._workers, pause_futures, strict=True): - try: - ray.get(future) - except Exception as error: - pause_errors.append(f"{worker.system_url}: {error}") - else: - paused_workers.append(worker) - return paused_workers, pause_errors - - def _resume_workers(self, workers: Sequence[DynamoWorkerEndpoint]) -> list[str]: - resume_futures = [ - ( - worker, - _post_worker_route.remote( - system_url=worker.system_url, - route="resume_generation", - payload={}, - timeout_s=self._control_timeout_s, - ), - ) - for worker in workers - ] - resume_errors: list[str] = [] - for worker, future in resume_futures: - try: - ray.get(future) - except Exception as error: - resume_errors.append(f"{worker.system_url}: {error}") - return resume_errors - - def pause(self, *, clear_cache: bool) -> None: - """Drain and pause every worker ahead of a weight update. - - Workers that confirmed the pause are resumed if any peer fails. - """ - paused_workers, pause_errors = self._pause_workers(clear_cache=clear_cache) - if pause_errors: - resume_errors = self._resume_workers(paused_workers) - details = "pause/clear failed for " + "; ".join(pause_errors) - if resume_errors: - details += "; resume failed for " + "; ".join(resume_errors) - raise RuntimeError(f"Dynamo pause before weight update failed: {details}") - - def resume(self) -> None: - """Resume every worker after a weight update.""" - resume_errors = self._resume_workers(self._workers) - if resume_errors: - raise RuntimeError( - "Dynamo resume after weight update failed for " - + "; ".join(resume_errors) - ) - - def flush_cache(self) -> bool: - """Drain, clear, and resume every worker using immutable endpoints.""" - paused_workers, pause_errors = self._pause_workers(clear_cache=True) - resume_errors = self._resume_workers(paused_workers) - - if pause_errors or resume_errors: - details = [] - if pause_errors: - details.append("pause/clear failed for " + "; ".join(pause_errors)) - if resume_errors: - details.append("resume failed for " + "; ".join(resume_errors)) - raise RuntimeError( - "Dynamo KV cache invalidation failed: " + "; ".join(details) - ) - return True diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py new file mode 100644 index 00000000000..82f5b897e77 --- /dev/null +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -0,0 +1,114 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Send NeMo RL rollouts through the Dynamo frontend's KV-aware router.""" + +import asyncio +import json +import logging +from typing import Any, Optional + +from nemo_rl.models.generation.dynamo.http_client import ( + async_http_post_json, + format_dynamo_error, +) + +LOGGER = logging.getLogger(__name__) + +# Marks a request as a NeMo RL rollout for the engine (native_engine), which +# then samples it from the generation config. Defined here, not in the engine +# module, so the driver need not import ai-dynamo. +ROLLOUT_ANNOTATION = "nemo_rl:rollout:" + +_HTTP_MAX_ATTEMPTS = 3 +_HTTP_RETRY_DELAY_S = 1.0 +_RETRYABLE_HTTP_STATUS_CODES = {408, 429} + + +def _is_retryable(response: dict[str, Any]) -> bool: + # A timeout is final: the request may still be running, and a retry would + # let one hung rollout hold its slot for several request_timeout_s. + if "transport_error" in response or response.get("json_decode_error") is True: + return True + status = response.get("http_status") + return isinstance(status, int) and ( + status in _RETRYABLE_HTTP_STATUS_CODES or 500 <= status < 600 + ) + + +def rollout_payload( + *, + model_name: str, + prompt_token_ids: list[int], + max_model_len: int, + greedy: bool, + stop_strings: Optional[list[str]], +) -> dict[str, Any]: + """The completion request for one rollout. + + Sampling is the engine's: the annotation tells it to build the request as + ``generate_async`` does, from the generation config. ``ignore_eos`` and a + context-sized ``max_tokens`` keep the frontend from ending the stream on + its own stop rules before the engine reports the result. + """ + rollout = {"greedy": greedy, "stop_strings": stop_strings or None} + return { + "model": model_name, + "prompt": prompt_token_ids, + "max_tokens": max(1, max_model_len - len(prompt_token_ids)), + "ignore_eos": True, + "nvext": { + "annotations": [ROLLOUT_ANNOTATION + json.dumps(rollout)], + "extra_fields": ["engine_data"], + }, + } + + +def parse_rollout_response(response: dict[str, Any], *, url: str) -> dict[str, Any]: + """Return the engine's ``engine_data``, or raise with the frontend's error. + + It holds the rollout result (``request_pipeline.rollout_result_to_wire``) + and the serving worker's index. + """ + if response.get("status") == "error": + raise RuntimeError( + f"Dynamo rollout request to {url} failed: {format_dynamo_error(response)}" + ) + engine_data = (response.get("nvext") or {}).get("engine_data") + if not isinstance(engine_data, dict) or "rollout" not in engine_data: + raise RuntimeError( + f"Dynamo rollout response from {url} carried no rollout engine_data; " + "is every engine NeMo RL's vLLM worker?" + ) + return engine_data + + +async def post_rollout( + url: str, payload: dict[str, Any], timeout_s: float +) -> dict[str, Any]: + """POST one rollout, retrying transient errors; returns its ``engine_data``.""" + response: dict[str, Any] = {} + for attempt in range(1, _HTTP_MAX_ATTEMPTS + 1): + response = await async_http_post_json(url, payload, timeout_s) + if not _is_retryable(response) or attempt == _HTTP_MAX_ATTEMPTS: + break + LOGGER.warning( + "Dynamo rollout attempt %d/%d failed; retrying in %.1fs: %s", + attempt, + _HTTP_MAX_ATTEMPTS, + _HTTP_RETRY_DELAY_S, + format_dynamo_error(response), + ) + await asyncio.sleep(_HTTP_RETRY_DELAY_S) + return parse_rollout_response(response, url=url) diff --git a/nemo_rl/models/generation/dynamo/token_wrapper.py b/nemo_rl/models/generation/dynamo/token_wrapper.py deleted file mode 100644 index eab81fafa35..00000000000 --- a/nemo_rl/models/generation/dynamo/token_wrapper.py +++ /dev/null @@ -1,556 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""OpenAI-compatible token wrapper for NeMo-Gym traffic to Dynamo.""" - -import asyncio -import json -import threading -from contextlib import asynccontextmanager -from copy import deepcopy -from typing import Any, Optional - -from nemo_rl.distributed.virtual_cluster import ( - DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH, - DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, - _get_free_port_local, - _get_node_ip_local, -) -from nemo_rl.models.generation.openai_server_utils import replace_prefix_tokens - -_GYM_TOKEN_METADATA_FIELDS = ( - "prompt_token_ids", - "generation_token_ids", - "generation_log_probs", -) -_TOOL_ARGUMENT_MAPPING_ERROR = "Can only get item pairs from a mapping." - - -def _coerce_token_id_list(value: Any, field_name: str) -> list[int]: - if not isinstance(value, list): - raise ValueError(f"{field_name} must be a list of token IDs.") - try: - return [int(token_id) for token_id in value] - except (TypeError, ValueError) as e: - raise ValueError(f"{field_name} must contain only integer token IDs.") from e - - -def _coerce_logprob_list(value: Any, field_name: str) -> list[float]: - if not isinstance(value, list): - raise ValueError(f"{field_name} must be a list of numeric log probabilities.") - logprobs: list[float] = [] - for index, logprob in enumerate(value): - if not isinstance(logprob, (int, float)) or isinstance(logprob, bool): - raise ValueError(f"{field_name}[{index}] must be numeric.") - logprobs.append(float(logprob)) - return logprobs - - -def _strip_gym_token_metadata(messages: list[Any]) -> list[Any]: - stripped_messages = deepcopy(messages) - for message in stripped_messages: - if isinstance(message, dict): - for field in _GYM_TOKEN_METADATA_FIELDS: - message.pop(field, None) - return stripped_messages - - -def _chat_template_kwargs( - request_body: dict[str, Any], - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], -) -> dict[str, Any]: - kwargs: dict[str, Any] = {} - if tokenizer_chat_template_kwargs is not None: - if not isinstance(tokenizer_chat_template_kwargs, dict): - raise ValueError("tokenizer chat_template_kwargs must be a JSON object.") - kwargs.update(tokenizer_chat_template_kwargs) - - request_kwargs = request_body.get("chat_template_kwargs") - if request_kwargs is not None: - if not isinstance(request_kwargs, dict): - raise ValueError("chat_template_kwargs must be a JSON object.") - kwargs.update(request_kwargs) - - if "reasoning_effort" in request_body: - kwargs["reasoning_effort"] = request_body["reasoning_effort"] - return kwargs - - -def _request_add_generation_prompt(request_body: dict[str, Any]) -> bool: - if "add_generation_prompt" in request_body: - return bool(request_body["add_generation_prompt"]) - return not bool(request_body.get("continue_final_message", False)) - - -def _apply_chat_template( - *, - tokenizer: Any, - request_body: dict[str, Any], - messages: list[Any], - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], - exclude_tools_when_tool_choice_none: bool, - add_generation_prompt: bool, - tokenize: bool, -) -> Any: - tools = request_body.get("tools") - if ( - exclude_tools_when_tool_choice_none - and request_body.get("tool_choice") == "none" - ): - tools = None - - apply_chat_template = type(tokenizer).apply_chat_template - return apply_chat_template( - tokenizer, - messages, - tools=tools, - documents=request_body.get("documents"), - chat_template=request_body.get("chat_template"), - add_generation_prompt=add_generation_prompt, - continue_final_message=bool(request_body.get("continue_final_message", False)), - tokenize=tokenize, - return_tensors=None, - return_dict=False, - **_chat_template_kwargs(request_body, tokenizer_chat_template_kwargs), - ) - - -def _render_prompt_token_ids( - *, - tokenizer: Any, - request_body: dict[str, Any], - messages: list[Any], - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], - exclude_tools_when_tool_choice_none: bool, - add_generation_prompt: bool, -) -> list[int]: - token_ids = _apply_chat_template( - tokenizer=tokenizer, - request_body=request_body, - messages=messages, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=add_generation_prompt, - tokenize=True, - ) - - if isinstance(token_ids, list) and ( - not token_ids or not isinstance(token_ids[0], list) - ): - return _coerce_token_id_list(token_ids, "prompt token IDs") - if isinstance(token_ids, list) and len(token_ids) == 1: - return _coerce_token_id_list(token_ids[0], "prompt token IDs") - raise ValueError( - "Dynamo token wrapper expected chat template rendering to return one " - "list of prompt token IDs." - ) - - -def _render_prompt_token_ids_with_optional_prefix( - *, - tokenizer: Any, - request_body: dict[str, Any], - messages: list[Any], - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], - exclude_tools_when_tool_choice_none: bool, - add_generation_prompt: bool, - assistant_index: int | None, -) -> tuple[list[int], list[int] | None]: - full_prompt_token_ids = _render_prompt_token_ids( - tokenizer=tokenizer, - request_body=request_body, - messages=messages, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=add_generation_prompt, - ) - if assistant_index is None: - return full_prompt_token_ids, None - - template_prefix_token_ids = _render_prompt_token_ids( - tokenizer=tokenizer, - request_body=request_body, - messages=messages[: assistant_index + 1], - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=False, - ) - return full_prompt_token_ids, template_prefix_token_ids - - -def _latest_tokenized_assistant_index(messages: list[Any]) -> Optional[int]: - for index in reversed(range(len(messages))): - message = messages[index] - if not isinstance(message, dict) or message.get("role") != "assistant": - continue - if ( - message.get("prompt_token_ids") is not None - and message.get("generation_token_ids") is not None - ): - return index - return None - - -def _derive_required_prefix_token_ids(messages: list[Any]) -> list[int] | None: - """Return the exact prompt and generation IDs from the latest model turn.""" - assistant_index = _latest_tokenized_assistant_index(messages) - if assistant_index is None: - return None - message = messages[assistant_index] - if not isinstance(message, dict): - return None - return _coerce_token_id_list( - message["prompt_token_ids"], "prompt_token_ids" - ) + _coerce_token_id_list(message["generation_token_ids"], "generation_token_ids") - - -def _normalize_tool_arguments_for_template( - messages: list[Any], *, before_index: int -) -> None: - """Make OpenAI tool calls renderable by model chat templates. - - OpenAI chat messages carry ``function.arguments`` as a JSON string, while - some model templates iterate those arguments as a mapping. Normalize only - the local template copy; the request forwarded to Dynamo retains its - original OpenAI payload. - """ - for message in messages[:before_index]: - if not isinstance(message, dict) or message.get("role") != "assistant": - continue - tool_calls = message.get("tool_calls") - if not isinstance(tool_calls, list): - continue - for tool_call in tool_calls: - if not isinstance(tool_call, dict): - continue - function = tool_call.get("function", tool_call) - if not isinstance(function, dict): - continue - arguments = function.get("arguments") - if not isinstance(arguments, str): - continue - try: - parsed_arguments = json.loads(arguments) - except json.JSONDecodeError: - parsed_arguments = {} - function["arguments"] = ( - parsed_arguments if isinstance(parsed_arguments, dict) else {} - ) - - -def _validate_engine_data( - response_body: dict[str, Any], -) -> tuple[list[int], list[int], list[float]]: - nvext = response_body.get("nvext") - engine_data = nvext.get("engine_data") if isinstance(nvext, dict) else None - if not isinstance(engine_data, dict): - raise ValueError("Dynamo response did not include nvext.engine_data.") - - prompt_token_ids = _coerce_token_id_list( - engine_data.get("prompt_token_ids"), - "nvext.engine_data.prompt_token_ids", - ) - completion_token_ids = _coerce_token_id_list( - engine_data.get("completion_token_ids"), - "nvext.engine_data.completion_token_ids", - ) - completion_logprobs = _coerce_logprob_list( - engine_data.get("completion_logprobs"), - "nvext.engine_data.completion_logprobs", - ) - if len(completion_logprobs) != len(completion_token_ids): - raise ValueError( - "Dynamo response returned " - f"{len(completion_logprobs)} generation log probabilities for " - f"{len(completion_token_ids)} generation token IDs." - ) - return prompt_token_ids, completion_token_ids, completion_logprobs - - -def _inject_gym_token_metadata(response_body: dict[str, Any]) -> None: - """Expose Dynamo engine metadata on the message fields consumed by Gym.""" - ( - prompt_token_ids, - generation_token_ids, - generation_logprobs, - ) = _validate_engine_data(response_body) - choices = response_body.get("choices") - if not isinstance(choices, list) or not choices: - raise ValueError("Dynamo response did not include choices[0].") - choice = choices[0] - if not isinstance(choice, dict): - raise ValueError("Dynamo response choices[0] must be a JSON object.") - message = choice.get("message") - if not isinstance(message, dict): - raise ValueError("Dynamo response choices[0].message must be a JSON object.") - message["prompt_token_ids"] = prompt_token_ids - message["generation_token_ids"] = generation_token_ids - message["generation_log_probs"] = generation_logprobs - - -def prepare_dynamo_chat_completion_request( - request_body: dict[str, Any], - *, - tokenizer: Any, - tokenizer_chat_template_kwargs: Optional[dict[str, Any]] = None, - exclude_tools_when_tool_choice_none: bool, -) -> dict[str, Any]: - """Prepare a NeMo-Gym chat-completion request for Dynamo token input.""" - if request_body.get("stream"): - raise ValueError("Dynamo native token wrapper does not support stream=True.") - - n = request_body.get("n", 1) - if n is not None and int(n) != 1: - raise ValueError("Dynamo native token wrapper currently supports only n=1.") - - messages = request_body.get("messages") - if not isinstance(messages, list): - raise ValueError("Dynamo token wrapper requires chat-completion messages.") - - prepared_body = deepcopy(request_body) - stripped_messages = _strip_gym_token_metadata(messages) - prepared_body["messages"] = stripped_messages - prepared_body.pop("required_prefix_token_ids", None) - - required_prefix_token_ids = _derive_required_prefix_token_ids(messages) - add_generation_prompt = _request_add_generation_prompt(prepared_body) - template_messages = deepcopy(stripped_messages) - assistant_index: int | None = None - if required_prefix_token_ids is not None: - assistant_index = _latest_tokenized_assistant_index(messages) - if assistant_index is None: - raise ValueError( - "Dynamo prefix token metadata must be attached to an assistant message." - ) - - try: - ( - full_prompt_token_ids, - template_prefix_token_ids, - ) = _render_prompt_token_ids_with_optional_prefix( - tokenizer=tokenizer, - request_body=prepared_body, - messages=template_messages, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=add_generation_prompt, - assistant_index=assistant_index, - ) - except TypeError as e: - if str(e) != _TOOL_ARGUMENT_MAPPING_ERROR: - raise - _normalize_tool_arguments_for_template( - template_messages, before_index=len(template_messages) - ) - ( - full_prompt_token_ids, - template_prefix_token_ids, - ) = _render_prompt_token_ids_with_optional_prefix( - tokenizer=tokenizer, - request_body=prepared_body, - messages=template_messages, - tokenizer_chat_template_kwargs=tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, - add_generation_prompt=add_generation_prompt, - assistant_index=assistant_index, - ) - - if required_prefix_token_ids is not None: - assert template_prefix_token_ids is not None - full_prompt_token_ids = replace_prefix_tokens( - tokenizer, - model_prefix_token_ids=required_prefix_token_ids, - template_prefix_token_ids=template_prefix_token_ids, - template_token_ids=full_prompt_token_ids, - ) - - nvext = prepared_body.get("nvext") - if nvext is None: - nvext = {} - if not isinstance(nvext, dict): - raise ValueError("nvext must be a JSON object.") - nvext = dict(nvext) - extra_fields = nvext.get("extra_fields", []) - if not isinstance(extra_fields, list): - raise ValueError("nvext.extra_fields must be a JSON list.") - nvext["extra_fields"] = list(dict.fromkeys([*extra_fields, "engine_data"])) - nvext["token_data"] = full_prompt_token_ids - prepared_body["nvext"] = nvext - - return prepared_body - - -class DynamoTokenWrapperServer: - """Small HTTP server that supplies tokenized chat prompts to Dynamo.""" - - def __init__( - self, - *, - dynamo_frontend_base_url: str, - tokenizer: Any, - tokenizer_chat_template_kwargs: Optional[dict[str, Any]], - exclude_tools_when_tool_choice_none: bool, - request_timeout_s: Optional[float], - ) -> None: - self.dynamo_frontend_base_url = dynamo_frontend_base_url - self.tokenizer = tokenizer - self.tokenizer_chat_template_kwargs = tokenizer_chat_template_kwargs - self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none - self.request_timeout_s = request_timeout_s - self.base_url: Optional[str] = None - self.server: Any = None - self.thread: Optional[threading.Thread] = None - self._client_session: Any = None - - def start(self) -> str: - """Start the wrapper in a background uvicorn thread.""" - import aiohttp - import uvicorn - from fastapi import FastAPI, HTTPException, Request - from fastapi.responses import JSONResponse - - @asynccontextmanager - async def lifespan(_: FastAPI): - timeout = ( - aiohttp.ClientTimeout(total=self.request_timeout_s) - if self.request_timeout_s is not None - else aiohttp.ClientTimeout(total=None) - ) - async with aiohttp.ClientSession( - timeout=timeout, - # Explicit: the default TCPConnector(limit=100) would cap the whole - # rollout path at 100 concurrent requests to the frontend. - connector=aiohttp.TCPConnector(limit=0), - ) as session: - self._client_session = session - try: - yield - finally: - self._client_session = None - - app = FastAPI(lifespan=lifespan) - - @app.get("/health") - async def health() -> dict[str, str]: - return { - "status": "ok", - "dynamo_frontend_base_url": self.dynamo_frontend_base_url, - } - - @app.post("/v1/chat/completions") - async def chat_completions(request: Request) -> JSONResponse: - try: - request_body = await request.json() - except json.JSONDecodeError as e: - raise HTTPException(status_code=400, detail="Invalid JSON body.") from e - if not isinstance(request_body, dict): - raise HTTPException( - status_code=400, - detail="Chat completion body must be a JSON object.", - ) - - try: - prepared_body = await asyncio.to_thread( - prepare_dynamo_chat_completion_request, - request_body, - tokenizer=self.tokenizer, - tokenizer_chat_template_kwargs=self.tokenizer_chat_template_kwargs, - exclude_tools_when_tool_choice_none=self.exclude_tools_when_tool_choice_none, - ) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - status_code, response_body = await self._forward_chat_completion( - prepared_body, - authorization=request.headers.get("authorization"), - ) - if 200 <= status_code < 300: - try: - _inject_gym_token_metadata(response_body) - except ValueError as e: - return JSONResponse( - content={"error": {"message": str(e)}}, - status_code=502, - ) - return JSONResponse(content=response_body, status_code=status_code) - - node_ip = _get_node_ip_local() - free_port = _get_free_port_local( - DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, - DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH, - ) - self.base_url = f"http://{node_ip}:{free_port}/v1" - - config = uvicorn.Config( - app, - host="0.0.0.0", - port=free_port, - timeout_keep_alive=120, - ) - self.server = uvicorn.Server(config=config) - self.thread = threading.Thread( - target=self.server.run, - name="dynamo-token-wrapper", - daemon=True, - ) - self.thread.start() - return self.base_url - - async def _forward_chat_completion( - self, - request_body: dict[str, Any], - *, - authorization: Optional[str], - ) -> tuple[int, dict[str, Any]]: - import aiohttp - - url = f"{self.dynamo_frontend_base_url.rstrip('/')}/chat/completions" - headers = {"Content-Type": "application/json"} - if authorization: - headers["Authorization"] = authorization - - session = self._client_session - if session is None: - return 503, {"error": {"message": "Dynamo token wrapper is not ready."}} - try: - async with session.post( - url, - json=request_body, - headers=headers, - ) as response: - response_text = await response.text() - if not response_text: - return response.status, {} - try: - response_body = json.loads(response_text) - except json.JSONDecodeError: - response_body = {"raw": response_text} - if not isinstance(response_body, dict): - response_body = {"response": response_body} - return response.status, response_body - except asyncio.TimeoutError: - return 504, {"error": {"message": f"Timed out forwarding to {url}."}} - except aiohttp.ClientError as e: - return 502, { - "error": { - "message": f"Failed to forward request to {url}: {type(e).__name__}: {e}" - } - } - - def shutdown(self) -> None: - """Stop the background uvicorn server.""" - if self.server is not None: - self.server.should_exit = True - if self.thread is not None: - self.thread.join(timeout=10) diff --git a/nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py b/nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py deleted file mode 100644 index 612c75cc7d0..00000000000 --- a/nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Validate a resolved ``dynamo.vllm`` argv without starting an engine.""" - -import dataclasses -import json -import sys - -# Fields NeMo RL sends in each ``update_weights`` round (see refit.py). -_UPDATE_INFO_FIELDS = {"names", "dtype_names", "shapes"} - - -def main() -> None: - if len(sys.argv) != 3: - raise SystemExit( - "usage: validate_dynamo_vllm_args.py '' ''" - ) - - # Parser construction evaluates vLLM device defaults. Image builds and - # unit preflights may run on CPU-only Slurm nodes, where a CUDA wheel leaves - # current_platform unspecified even though no engine will be constructed. - # Match Dynamo frontend's parser-only fallback without changing GPU runs. - import vllm.platforms - - if vllm.platforms.current_platform.device_type == "": - from vllm.platforms.cpu import CpuPlatform - - vllm.platforms.current_platform = CpuPlatform() - - from dynamo.vllm.args import parse_args - from vllm.distributed.weight_transfer.nccl_common import ( - NCCLWeightTransferInitInfo, - ) - from vllm.distributed.weight_transfer.nccl_engine import ( - NCCLWeightTransferUpdateInfo, - ) - - argv = json.loads(sys.argv[1]) - if not isinstance(argv, list) or not all(isinstance(item, str) for item in argv): - raise TypeError("resolved Dynamo argv must be a JSON list of strings") - parse_args(argv) - - # NeMo RL declares the packed geometry in the init info and sends only - # per-round tensor metadata in the update info. Fail before engine startup - # if the installed vLLM weight-transfer API no longer accepts that shape. - geometry = json.loads(sys.argv[2]) - try: - NCCLWeightTransferInitInfo( - master_address="127.0.0.1", - master_port=1, - rank_offset=1, - world_size=2, - packed=True, - packed_buffer_size_bytes=geometry["buffer_size_bytes"], - packed_num_buffers=geometry["num_buffers"], - ) - except TypeError as error: - raise SystemExit( - "vLLM NCCLWeightTransferInitInfo does not accept NeMo RL's packed " - f"init info: {error}. Update the Dynamo refit protocol in refit.py." - ) from error - update_fields = { - field.name for field in dataclasses.fields(NCCLWeightTransferUpdateInfo) - } - if update_fields != _UPDATE_INFO_FIELDS: - raise SystemExit( - "vLLM NCCLWeightTransferUpdateInfo fields changed: engine has " - f"{sorted(update_fields)}, NeMo RL sends {sorted(_UPDATE_INFO_FIELDS)}. " - "Update the Dynamo refit protocol in refit.py." - ) - - -if __name__ == "__main__": - main() diff --git a/nemo_rl/models/generation/dynamo/venv.py b/nemo_rl/models/generation/dynamo/venv.py index 020c923bb34..9a60b194e53 100644 --- a/nemo_rl/models/generation/dynamo/venv.py +++ b/nemo_rl/models/generation/dynamo/venv.py @@ -12,33 +12,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Resolve executables installed by the optional Dynamo environment.""" +"""Resolve the interpreter and service binaries the managed Dynamo runtime launches.""" import os +import shutil from pathlib import Path +from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env +from nemo_rl.utils.venvs import create_local_venv_on_each_node -def get_dynamo_venv_dir() -> Path: - """Return the configured Dynamo virtual-environment directory.""" - configured = os.environ.get("NEMO_RL_DYNAMO_VENV_DIR") - if configured: - return Path(configured).expanduser().resolve() - repo_root = Path(__file__).resolve().parents[4] - return repo_root / "venvs" / "dynamo" +# Every vLLM worker venv carries the locked ai-dynamo release, so the frontend +# runs from the async worker's venv on the same release as every engine. +VLLM_ASYNC_WORKER_FQN = ( + "nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker" +) -def get_dynamo_executable(name: str) -> str: - """Return an executable in the Dynamo environment, failing if absent.""" - executable = get_dynamo_venv_dir() / "bin" / name - if not executable.is_file() or not os.access(executable, os.X_OK): - raise FileNotFoundError( - f"Dynamo executable {executable} is unavailable. Build with " - "BUILD_DYNAMO=1 or run docker/dynamo/install.sh with " - "NEMO_RL_DYNAMO_VENV_DIR set." - ) - return str(executable) +def get_dynamo_python() -> str: + """Return the vLLM async worker's Python, which carries ``ai-dynamo``. + The driver launches ``dynamo.frontend`` with it. A uv-managed venv is + resolved as the workers' own is: an existing one is reused as-is, and a + missing one is built once on each node. Re-syncing it here would change a + venv the engines are already running from. + """ + py_executable = get_actor_python_env(VLLM_ASYNC_WORKER_FQN) + if py_executable.startswith("uv"): + return create_local_venv_on_each_node(py_executable, VLLM_ASYNC_WORKER_FQN) + return py_executable -def get_dynamo_python() -> str: - """Return the validated Python interpreter for ``ai-dynamo``.""" - return get_dynamo_executable("python") + +def get_dynamo_service_executable(name: str) -> str: + """Return the etcd or nats-server binary the runtime starts. + + Looks in ``NEMO_RL_DYNAMO_BIN_DIR`` when it is set, otherwise on ``PATH``. + """ + bin_dir = os.environ.get("NEMO_RL_DYNAMO_BIN_DIR") + if bin_dir: + executable = Path(bin_dir).expanduser() / name + if executable.is_file() and os.access(executable, os.X_OK): + return str(executable) + raise FileNotFoundError( + f"{name} is not an executable file in NEMO_RL_DYNAMO_BIN_DIR={bin_dir}." + ) + found = shutil.which(name) + if found is None: + raise FileNotFoundError( + f"{name} is not on PATH. Install it, or set NEMO_RL_DYNAMO_BIN_DIR to " + "the directory that holds etcd and nats-server." + ) + return found diff --git a/nemo_rl/models/generation/dynamo/worker_pool.py b/nemo_rl/models/generation/dynamo/worker_pool.py deleted file mode 100644 index 1042e542f64..00000000000 --- a/nemo_rl/models/generation/dynamo/worker_pool.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Fixed Ray-managed pool of Dynamo vLLM subprocess owners.""" - -import os -from typing import Any - -import ray -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy - -from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env -from nemo_rl.distributed.virtual_cluster import ( - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH, - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, - DEFAULT_VLLM_PORT_RANGE_LOW, - DEFAULT_VLLM_PORTS_PER_ENGINE, - RayVirtualCluster, -) -from nemo_rl.models.generation.dynamo.dynamo_worker import ( - DynamoGpuReservation, - DynamoVllmWorker, -) - -_WORKER_FQN = "nemo_rl.models.generation.dynamo.dynamo_worker.DynamoVllmWorker" - - -def _vllm_port_for_node_slot(node_slot: int) -> int: - """Return vLLM's node-local scan base with 100 ports of headroom.""" - return DEFAULT_VLLM_PORT_RANGE_LOW + node_slot * DEFAULT_VLLM_PORTS_PER_ENGINE - - -class FixedDynamoWorkerPool: - """Reserve inference GPUs and launch one worker per model-parallel group.""" - - def __init__( - self, - *, - cluster: RayVirtualCluster, - config: dict[str, Any], - namespace: str, - engine_world_size: int, - manager_env: dict[str, str], - startup_timeout_s: float, - ) -> None: - self._cluster = cluster - self._config = config - self._namespace = namespace - self._engine_world_size = engine_world_size - self._manager_env = manager_env - self._startup_timeout_s = startup_timeout_s - self._workers: list[ray.actor.ActorHandle] = [] - self._reservations: list[ray.actor.ActorHandle] = [] - self._cleanup_reservations: list[ray.actor.ActorHandle] = [] - self._reservation_metadata: list[dict[str, Any]] = [] - self._metadata: list[dict[str, Any]] = [] - - @property - def size(self) -> int: - return len(self._workers) - - def is_alive(self) -> bool: - """Return whether every managed vLLM subprocess is still alive.""" - return bool(self._workers) and all( - ray.get([worker.is_alive.remote() for worker in self._workers]) - ) - - def start(self) -> None: - if self._workers: - raise RuntimeError("Managed Dynamo worker pool is already started.") - placement_groups = self._cluster.get_placement_groups() - python_env = get_actor_python_env(_WORKER_FQN) - runtime_env: dict[str, Any] = { - "py_executable": python_env, - "env_vars": dict(os.environ), - } - - group_index = 0 - engine_slots_by_node: dict[str, int] = {} - system_ports_by_node: dict[str, set[int]] = {} - metadata_refs: list[ray.ObjectRef] = [] - for pg_index, placement_group in enumerate(placement_groups): - bundle_count = placement_group.bundle_count - if bundle_count % self._engine_world_size != 0: - raise ValueError( - f"Inference placement group {pg_index} has {bundle_count} GPU " - f"bundles, which is not divisible by engine_world_size=" - f"{self._engine_world_size}." - ) - for start in range(0, bundle_count, self._engine_world_size): - bundle_indices = list(range(start, start + self._engine_world_size)) - reservation_handles = [] - for bundle_index in bundle_indices: - strategy = PlacementGroupSchedulingStrategy( - placement_group=placement_group, - placement_group_bundle_index=bundle_index, - placement_group_capture_child_tasks=True, - ) - reservation_handles.append( - DynamoGpuReservation.options( - num_gpus=1, - runtime_env=runtime_env, - scheduling_strategy=strategy, - ).remote() - ) - self._reservations.extend(reservation_handles) - reservation_metadata = ray.get( - [handle.metadata.remote() for handle in reservation_handles] - ) - self._reservation_metadata.extend(reservation_metadata) - node_ips = {item["node_ip"] for item in reservation_metadata} - if len(node_ips) != 1: - raise RuntimeError( - "A managed Dynamo engine group spans multiple nodes. " - "Multi-node TP/PP is not supported in the fixed-fleet milestone." - ) - cuda_devices = [item["gpu_id"] for item in reservation_metadata] - node_ip = next(iter(node_ips)) - node_slot = engine_slots_by_node.get(node_ip, 0) - engine_slots_by_node[node_ip] = node_slot + 1 - system_ports = system_ports_by_node.setdefault(node_ip, set()) - system_port = ray.get( - reservation_handles[0].select_free_port.remote( - port_range_low=DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, - port_range_high=DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH, - excluded_ports=sorted(system_ports), - ) - ) - system_ports.add(system_port) - vllm_port = _vllm_port_for_node_slot(node_slot) - group_name = f"{self._namespace}-dynamo-vllm-{pg_index}-{group_index}" - leader_strategy = PlacementGroupSchedulingStrategy( - placement_group=placement_group, - placement_group_bundle_index=bundle_indices[0], - placement_group_capture_child_tasks=True, - ) - worker = DynamoVllmWorker.options( - num_gpus=0, - runtime_env={ - **runtime_env, - "env_vars": { - **runtime_env["env_vars"], - "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1", - }, - }, - scheduling_strategy=leader_strategy, - name=group_name, - ).remote( - self._config, - namespace=self._namespace, - group_name=group_name, - cuda_devices=cuda_devices, - system_port=system_port, - vllm_port=vllm_port, - manager_env=self._manager_env, - startup_timeout_s=self._startup_timeout_s, - seed=pg_index * 1024 + group_index, - cleanup_reservation=reservation_handles[0], - ) - self._workers.append(worker) - self._cleanup_reservations.append(reservation_handles[0]) - self._metadata.append({}) - metadata_refs.append(worker.metadata.remote()) - group_index += 1 - - metadata_error: Exception | None = None - for index, metadata_ref in enumerate(metadata_refs): - try: - self._metadata[index] = dict(ray.get(metadata_ref)) - except Exception as error: - if metadata_error is None: - metadata_error = error - if metadata_error is not None: - raise metadata_error - - def refit_workers(self) -> list[dict[str, Any]]: - return [dict(item) for item in self._metadata] - - def validate(self, expected: list[dict[str, Any]]) -> list[dict[str, Any]]: - if not self._workers or not all( - ray.get([w.is_alive.remote() for w in self._workers]) - ): - raise RuntimeError("A Ray-managed Dynamo vLLM worker exited.") - try: - current_reservations = ray.get( - [reservation.metadata.remote() for reservation in self._reservations] - ) - except Exception as exc: - raise RuntimeError( - "A Ray-managed Dynamo GPU reservation actor exited." - ) from exc - if current_reservations != self._reservation_metadata: - raise RuntimeError( - "Ray-managed Dynamo GPU reservation membership changed: " - f"expected={self._reservation_metadata}, current={current_reservations}." - ) - current = ray.get([worker.metadata.remote() for worker in self._workers]) - if current != expected: - raise RuntimeError( - "Ray-managed Dynamo worker membership changed after NCCL collective " - f"initialization: expected={expected}, current={current}." - ) - return [dict(item) for item in current] - - def shutdown(self) -> None: - for worker, reservation in zip( - self._workers, - self._cleanup_reservations, - strict=True, - ): - try: - ray.get(worker.shutdown.remote(), timeout=30) - except Exception: - try: - ray.get( - reservation.cleanup_process_group.remote(), - timeout=15, - ) - except Exception: - pass - for worker in self._workers: - try: - ray.kill(worker, no_restart=True) - except Exception: - pass - for reservation in self._reservations: - try: - ray.kill(reservation, no_restart=True) - except Exception: - pass - self._workers.clear() - self._reservations.clear() - self._cleanup_reservations.clear() - self._reservation_metadata.clear() - self._metadata.clear() diff --git a/nemo_rl/models/generation/interfaces.py b/nemo_rl/models/generation/interfaces.py index 47204458008..a067106d1ef 100644 --- a/nemo_rl/models/generation/interfaces.py +++ b/nemo_rl/models/generation/interfaces.py @@ -289,9 +289,6 @@ def should_use_async_rollouts( return False backend = generation_config.get("backend", "") - if backend == "dynamo": - return True - if backend == "sglang": return bool(generation_config.get("use_async_rollouts", False)) diff --git a/nemo_rl/models/generation/openai_server_utils.py b/nemo_rl/models/generation/openai_server_utils.py index f01cd1f6091..53a757dc04f 100644 --- a/nemo_rl/models/generation/openai_server_utils.py +++ b/nemo_rl/models/generation/openai_server_utils.py @@ -58,13 +58,48 @@ def replace_prefix_tokens( ).token_ids +def _eos_token_id_set(eos_token_id: int | Collection[int]) -> frozenset[int]: + eos_token_ids = ( + frozenset([eos_token_id]) + if isinstance(eos_token_id, int) + else frozenset(eos_token_id) + ) + assert eos_token_ids, "EOS token id collection must not be empty" + return eos_token_ids + + +def derive_prefix_eos_count( + model_prefix_token_ids: list[int], *, eos_token_id: int | Collection[int] +) -> int: + """The EOS count a template render of the prefix's history would hold. + + For hosts that see only the rendered prompt, not the messages (the managed + Dynamo engine receives token ids from its frontend), this stands in for + counting EOS in ``template_prefix_token_ids``. Every EOS in the prefix + closed one message, and a template closes the final assistant turn with one + EOS even when the model was truncated before emitting it. The two counts + agree when the history up to the prefix's last assistant turn is the + history the prefix was generated from (Gym's lineage admission checks this + for captured calls; without capture it is assumed, as on the native server) + and the model emitted no EOS text as ordinary tokens. A render with fewer + EOS than this count fails the splice; an extra EOS before the boundary cuts + early instead. + """ + eos_token_ids = _eos_token_id_set(eos_token_id) + count = sum(tid in eos_token_ids for tid in model_prefix_token_ids) + if not model_prefix_token_ids or model_prefix_token_ids[-1] not in eos_token_ids: + count += 1 + return count + + def splice_prefix_tokens( *, tokenizer: Any, model_prefix_token_ids: list[int], - template_prefix_token_ids: list[int], + template_prefix_token_ids: list[int] | None = None, template_token_ids: list[int], eos_token_id: int | Collection[int] | None = None, + template_prefix_eos_count: int | None = None, ) -> PrefixSplice: """This is a subroutine used inside the OpenAI-compatible Chat Completion server. @@ -132,19 +167,24 @@ def splice_prefix_tokens( toward the boundary, and when the model's prefix ends on one of them that exact id is kept in place of the template's, so the spliced prompt still starts with the model's tokens verbatim. + + Pass exactly one of ``template_prefix_token_ids`` and + ``template_prefix_eos_count``. The prefix render is only ever used for its + EOS count, so a host without the messages passes the count directly (see + :func:`derive_prefix_eos_count`). """ + if (template_prefix_token_ids is None) == (template_prefix_eos_count is None): + raise ValueError( + "Pass exactly one of template_prefix_token_ids and " + "template_prefix_eos_count" + ) if not model_prefix_token_ids: return PrefixSplice(template_token_ids, 0, 0) if eos_token_id is None: eos_token_id = tokenizer.eos_token_id assert eos_token_id is not None, "Tokenizer must have an EOS token ID" - eos_token_ids = ( - frozenset([eos_token_id]) - if isinstance(eos_token_id, int) - else frozenset(eos_token_id) - ) - assert eos_token_ids, "EOS token id collection must not be empty" + eos_token_ids = _eos_token_id_set(eos_token_id) # The model isn't guaranteed to end on EOS (e.g. it hit max_tokens); chat # templates always add one, so cut the model input to just before its EOS. @@ -157,7 +197,11 @@ def splice_prefix_tokens( # templates may strip prior reasoning blocks when re-rendering history; # EOS counting preserves the original generated reasoning tokens without # requiring a customized chat template. - count_needed = sum(tid in eos_token_ids for tid in template_prefix_token_ids) + count_needed = ( + template_prefix_eos_count + if template_prefix_token_ids is None + else sum(tid in eos_token_ids for tid in template_prefix_token_ids) + ) count_seen = 0 template_cut_start = -1 for pos, tid in enumerate(template_token_ids): @@ -171,14 +215,14 @@ def splice_prefix_tokens( message = ( f"EOS token #{count_needed} not found in template_token_ids " f"(only found {count_seen} EOS tokens total)!\n" - f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" - f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" ) + if template_prefix_token_ids is not None: + message += f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" + message += f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" if tokenizer is not None: - message += ( - f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}\n\n" - f"Template repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" - ) + if template_prefix_token_ids is not None: + message += f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}" + message += f"\n\nTemplate repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" raise AssertionError(message) boundary = ( diff --git a/nemo_rl/models/generation/vllm/config.py b/nemo_rl/models/generation/vllm/config.py index 9bb7ca05791..5107f142983 100644 --- a/nemo_rl/models/generation/vllm/config.py +++ b/nemo_rl/models/generation/vllm/config.py @@ -25,6 +25,7 @@ PositiveInt, ) +from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.interfaces import ( GenerationConfig, get_num_routed_experts, @@ -246,6 +247,13 @@ class VllmConfig(GenerationConfig): # ACTOR_ENVIRONMENT_REGISTRY. worker_extension_cls_fqn: NotRequired[str | None] + # Serve these same workers behind Dynamo's frontend (OpenAI API and KV-aware + # router). Every other vLLM setting keeps its meaning. Implies + # vllm_cfg.async_engine, since Dynamo drives the engine's AsyncLLM. + dynamo: NotRequired[bool] + # Optional frontend and registration settings; DynamoCfg holds the defaults. + dynamo_cfg: NotRequired[DynamoCfg] + def validate_vllm_quantization_config(config: VllmConfig) -> None: """Reject quantization options that would otherwise be silently ignored.""" diff --git a/nemo_rl/models/generation/vllm/metric_names.py b/nemo_rl/models/generation/vllm/metric_names.py index 5c369d2566b..98a64634164 100644 --- a/nemo_rl/models/generation/vllm/metric_names.py +++ b/nemo_rl/models/generation/vllm/metric_names.py @@ -31,7 +31,7 @@ # Engine series surfaced per step, beyond the spec-decode family. These are the # family names vLLM registers (vllm/v1/metrics/loggers.py); its reader returns # family names, which prometheus_client strips of ``_total``, so no suffixed -# alias is needed here (unlike the HTTP-scrape path in dynamo/metrics.py). +# alias is needed here. # test_engine_metric_names_match_vllm checks these against vLLM itself. PROMPT_TOKEN_COUNTERS = ("vllm:prompt_tokens",) GENERATION_TOKEN_COUNTERS = ("vllm:generation_tokens",) diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py new file mode 100644 index 00000000000..b57808f0078 --- /dev/null +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -0,0 +1,217 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Turn a finished vLLM rollout into NeMo RL's per-sample generation output. + +Every path that serves a rollout from a NeMo RL vLLM engine goes through here: +the worker's own ``generate_async``, and Dynamo serving (the engine builds the +result, the driver assembles the output). A rollout feature added here, such +as router replay (R3), therefore behaves the same on every path. +""" + +import logging +from dataclasses import dataclass +from typing import Any, Optional, cast + +import torch + +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.models.generation.interfaces import GenerationOutputSpec +from nemo_rl.models.generation.vllm.utils import pad_and_align_routed_expert_indices +from nemo_rl.utils.routed_experts_codec import ( + decode_routed_experts, + encode_routed_experts, + routed_experts_envelope_dtype, +) + +LOGGER = logging.getLogger(__name__) + + +@dataclass +class RolloutResult: + """One rollout's outcome, independent of where its output is assembled.""" + + token_ids: list[int] + # The sampled token's log probability at each position; 0.0 where vLLM + # reported none. + logprobs: list[float] + # The engine stopped at max_tokens. + truncated: bool + # [prompt + generated, num_moe_layers, topk], aligned for router replay. + routed_experts: Optional[torch.Tensor] + # missing/expected/actual route counts; set when router replay is on. + r3_stats: Optional[dict[str, int]] + + @classmethod + def nothing_generated(cls) -> "RolloutResult": + """The result when no new token fits: the prompt alone, untruncated.""" + return cls([], [], False, None, None) + + +def selected_logprobs(completion: Any) -> list[float]: + """The sampled token's log probability at each position of a completion.""" + by_position = completion.logprobs or () + logprobs = [] + for idx, token_id in enumerate(completion.token_ids): + candidates = by_position[idx] if idx < len(by_position) else None + entry = candidates.get(token_id) if candidates else None + logprobs.append(0.0 if entry is None else float(entry.logprob)) + return logprobs + + +def rollout_result( + final_output: Any, + *, + input_length: int, + return_routed_experts: bool, + routed_experts_dtype: torch.dtype, + device: torch.device, + sample_idx: int = 0, +) -> RolloutResult: + """Build the result of a finished rollout from vLLM's final RequestOutput. + + Args: + final_output: The request's last (cumulative) ``RequestOutput``. + input_length: Prompt length in tokens. + return_routed_experts: Router replay is on, so routes are required. + routed_experts_dtype: The dtype router replay consumes. + device: Where to place the routed-experts tensor. + sample_idx: For log messages only. + """ + completion = final_output.outputs[0] + token_ids = list(completion.token_ids) + total_length = input_length + len(token_ids) + # return_stats=True returns the (routes, stats) pair. + routed_experts, r3_stats = cast( + tuple[Optional[torch.Tensor], dict[str, int]], + pad_and_align_routed_expert_indices( + final_output, + completion, + valid_length=total_length, + padded_length=total_length, + device=device, + require_complete_routed_experts=return_routed_experts, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ), + ) + if return_routed_experts and routed_experts is None: + raise RuntimeError( + "vLLM was asked to return routed experts but the generation output " + "did not include routed_experts." + ) + if return_routed_experts and r3_stats["missing_routes"] > 0: + LOGGER.warning( + "R3 router replay fallback: vLLM returned incomplete " + "routed_experts for sample_idx=%d, missing_token_routes=%d, " + "actual_routes=%d, expected_routes=%d. Megatron will use its " + "own router for those missing token routes.", + sample_idx, + r3_stats["missing_routes"], + r3_stats["actual_routes"], + r3_stats["expected_routes"], + ) + return RolloutResult( + token_ids=token_ids, + logprobs=selected_logprobs(completion), + truncated=completion.finish_reason == "length", + routed_experts=routed_experts, + r3_stats=r3_stats if return_routed_experts else None, + ) + + +def rollout_output( + result: RolloutResult, + *, + input_ids: torch.Tensor, + input_length: int, + pad_token_id: int, +) -> BatchedDataDict[GenerationOutputSpec]: + """The single-sample output ``generate_async`` yields for ``result``. + + Args: + result: The rollout's result. + input_ids: The sample's (right-padded) prompt row. + input_length: Prompt length in tokens. + pad_token_id: The tokenizer's pad token, as ``generate_async`` uses it. + """ + device = input_ids.device + generated = result.token_ids + output_length = input_length + len(generated) + output_ids = torch.full( + (output_length,), pad_token_id, dtype=input_ids.dtype, device=device + ) + output_ids[:input_length] = input_ids[:input_length] + output_ids[input_length:] = torch.tensor( + generated, dtype=input_ids.dtype, device=device + ) + logprobs = torch.zeros((1, output_length), dtype=torch.float32, device=device) + logprobs[0, input_length:] = torch.tensor( + result.logprobs, dtype=torch.float32, device=device + ) + output: dict[str, Any] = { + "output_ids": output_ids.unsqueeze(0), + "logprobs": logprobs, + "generation_lengths": torch.tensor( + [len(generated)], dtype=torch.long, device=device + ), + "unpadded_sequence_lengths": torch.tensor( + [output_length], dtype=torch.long, device=device + ), + "truncated": torch.tensor([result.truncated], dtype=torch.bool, device=device), + } + if result.r3_stats is not None: + for key in ("missing_routes", "expected_routes", "actual_routes"): + output[f"r3_routed_experts_{key}"] = torch.tensor( + [result.r3_stats[key]], dtype=torch.long, device=device + ) + if result.routed_experts is not None: + output["routed_experts"] = result.routed_experts.to(device).unsqueeze(0) + return BatchedDataDict[GenerationOutputSpec](output) + + +def rollout_result_to_wire(result: RolloutResult) -> dict[str, Any]: + """JSON-safe form of ``result``, for engines that answer over HTTP.""" + return { + "token_ids": result.token_ids, + "logprobs": result.logprobs, + "truncated": result.truncated, + "routed_experts": ( + None + if result.routed_experts is None + else encode_routed_experts(result.routed_experts) + ), + "r3_stats": result.r3_stats, + } + + +def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: + """Inverse of :func:`rollout_result_to_wire`; routes keep their own dtype.""" + routed_experts = wire["routed_experts"] + if routed_experts is not None: + routed_experts = decode_routed_experts( + routed_experts, routed_experts_envelope_dtype(routed_experts) + ) + if len(wire["logprobs"]) != len(wire["token_ids"]): + raise ValueError( + f"rollout result has {len(wire['logprobs'])} logprobs for " + f"{len(wire['token_ids'])} tokens" + ) + return RolloutResult( + token_ids=list(wire["token_ids"]), + logprobs=list(wire["logprobs"]), + truncated=bool(wire["truncated"]), + routed_experts=routed_experts, + r3_stats=wire["r3_stats"], + ) diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index b77da9c6e6f..2d871819ae7 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -318,6 +318,57 @@ def _as_routed_experts_tensor( return tensor.to(dtype=dtype) +def off_policy_sampling_error( + *, + temperature: Optional[float], + top_p: Optional[float], + top_k: Optional[int], + generation_config: Any, +) -> Optional[str]: + """Why an HTTP rollout request would sample off-policy, or None if it would not. + + Request sampling must match NeMo RL's exactly, or inference runs off-policy and + destabilizes training. Validation rollouts are the one exception: they carry the + validation profile (``val_temperature`` / ``val_top_p``), which is metric-only. + Multi-turn agents issue their own requests, so the serving HTTP endpoint is the + one chokepoint they all pass. + + Args: + temperature: The request's temperature. + top_p: The request's top_p. It must be explicit: vLLM resolves an unset top_p + from the model's generation_config.json, not to 1.0. + top_k: The request's top_k, which must be unset or -1 (disabled), matching + ``BaseVllmGenerationWorker._build_sampling_params``. + generation_config: ``policy.generation``, with the train and validation + sampling parameters. + """ + if top_k not in (None, -1): + return f"Top k sampling parameter must be unset, empty, or -1. Got `{top_k}`" + if top_p is None: + return ( + "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " + "resolved by vLLM from the model's generation_config.json and would " + "bypass the on-policy sampling check." + ) + is_train_sampling = ( + temperature == generation_config["temperature"] + and top_p == generation_config["top_p"] + ) + is_val_sampling = ( + temperature == generation_config["val_temperature"] + and top_p == generation_config["val_top_p"] + ) + if is_train_sampling or is_val_sampling: + return None + return ( + f"request sampling (temperature={temperature}, top_p={top_p}) matches neither " + f"the train sampling params (temperature={generation_config['temperature']}, " + f"top_p={generation_config['top_p']}) nor the validation sampling params " + f"(val_temperature={generation_config['val_temperature']}, " + f"val_top_p={generation_config['val_top_p']})" + ) + + def validate_rollout_prompt(expected: list[int], actual: list[int] | None) -> None: """Reject rollouts generated from a different prompt than the learner's.""" if actual != expected: @@ -496,6 +547,58 @@ def pad_and_align_routed_expert_indices( return (full, stats) if return_stats else full +def chat_routed_experts( + final_request_output: Any, + generation_details: Any, + *, + prompt_token_count: int, + device: torch.device, + logger: Any, + routed_experts_dtype: torch.dtype, +) -> str: + """The routed-experts envelope the chat endpoint attaches to one choice.""" + generation_token_count = len(getattr(generation_details, "token_ids", []) or []) + routed_result = pad_and_align_routed_expert_indices( + final_request_output, + generation_details, + valid_length=prompt_token_count + generation_token_count, + padded_length=prompt_token_count + generation_token_count, + device=device, + require_complete_routed_experts=True, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ) + if not isinstance(routed_result, tuple): + raise RuntimeError( + "Expected routed_experts alignment to return stats for the " + "OpenAI-compatible chat endpoint." + ) + routed_experts, r3_stats = routed_result + if routed_experts is None: + raise RuntimeError( + "vLLM was asked to return routed experts for the " + "OpenAI-compatible chat endpoint but the generation " + "output did not include routed_experts." + ) + if r3_stats["missing_routes"] > 0 and logger is not None: + logger.warning( + "R3 router replay fallback: vLLM returned incomplete " + "routed_experts for chat choice_idx=%d, " + "missing_token_routes=%d, actual_routes=%d, " + "expected_routes=%d. Megatron will use its own router " + "for those missing token routes.", + generation_details.index, + r3_stats["missing_routes"], + r3_stats["actual_routes"], + r3_stats["expected_routes"], + ) + # Base64 envelope instead of .tolist(): nested JSON int lists cost + # ~1s of CPU per serialize/parse hop at long context lengths and get + # re-validated at every gym HTTP hop; a single string passes through + # the gym chain opaquely. + return encode_routed_experts(routed_experts.to(dtype=routed_experts_dtype)) + + def attach_routed_experts_to_chat_response_choices( response: Any, final_request_output: Any, @@ -519,49 +622,14 @@ def attach_routed_experts_to_chat_response_choices( if generation_details is None: continue attached_choice_indices.add(choice.index) - - generation_token_count = len(getattr(generation_details, "token_ids", []) or []) - routed_result = pad_and_align_routed_expert_indices( + choice.message.routed_experts = chat_routed_experts( final_request_output, generation_details, - valid_length=prompt_token_count + generation_token_count, - padded_length=prompt_token_count + generation_token_count, + prompt_token_count=prompt_token_count, device=device, - require_complete_routed_experts=True, - return_stats=True, + logger=logger, routed_experts_dtype=routed_experts_dtype, ) - if not isinstance(routed_result, tuple): - raise RuntimeError( - "Expected routed_experts alignment to return stats for the " - "OpenAI-compatible chat endpoint." - ) - routed_experts, r3_stats = routed_result - if routed_experts is None: - raise RuntimeError( - "vLLM was asked to return routed experts for the " - "OpenAI-compatible chat endpoint but the generation " - "output did not include routed_experts." - ) - if r3_stats["missing_routes"] > 0 and logger is not None: - logger.warning( - "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for chat choice_idx=%d, " - "missing_token_routes=%d, actual_routes=%d, " - "expected_routes=%d. Megatron will use its own router " - "for those missing token routes.", - choice.index, - r3_stats["missing_routes"], - r3_stats["actual_routes"], - r3_stats["expected_routes"], - ) - # Base64 envelope instead of .tolist(): nested JSON int lists cost - # ~1s of CPU per serialize/parse hop at long context lengths and get - # re-validated at every gym HTTP hop; a single string passes through - # the gym chain opaquely. - choice.message.routed_experts = encode_routed_experts( - routed_experts.to(dtype=routed_experts_dtype) - ) if len(attached_choice_indices) != len(choices): missing_choice_indices = sorted( @@ -623,50 +691,9 @@ def attach_token_information_to_chat_response_choices( ) for choice in choices: - generation_details = outputs_by_index[choice.index] - output_token_ids = getattr(generation_details, "token_ids", None) - if output_token_ids is None: - raise RuntimeError( - "vLLM was asked to return token information for the " - "OpenAI-compatible chat endpoint but generation output " - f"choice_idx={choice.index} did not include token_ids." - ) - generation_token_ids = list(output_token_ids) - - generation_logprob_details = getattr(generation_details, "logprobs", None) - if generation_logprob_details is None: - if generation_token_ids: - raise RuntimeError( - "vLLM was asked to return token information for the " - "OpenAI-compatible chat endpoint but generation output " - f"choice_idx={choice.index} did not include logprobs." - ) - generation_log_probs = [] - else: - if len(generation_token_ids) != len(generation_logprob_details): - raise RuntimeError( - "vLLM returned mismatched generation token IDs and log " - "probabilities for the OpenAI-compatible chat endpoint: " - f"choice_idx={choice.index}, " - f"token_count={len(generation_token_ids)}, " - f"logprob_count={len(generation_logprob_details)}." - ) - generation_log_probs = [] - for token_id, position_logprobs in zip( - generation_token_ids, generation_logprob_details - ): - selected_token_logprob = position_logprobs.get(token_id) - if selected_token_logprob is None: - raise RuntimeError( - "vLLM generation log probabilities did not include the " - "selected token while attaching token information to " - "the OpenAI-compatible chat response: " - f"choice_idx={choice.index}, token_id={token_id}." - ) - generation_log_probs.append( - max(float(selected_token_logprob.logprob), VLLM_LOGPROB_FLOOR) - ) - + generation_token_ids, generation_log_probs = chat_generation_tokens( + outputs_by_index[choice.index] + ) choice.message.prompt_token_ids = list(prompt_token_ids) choice.message.generation_token_ids = generation_token_ids choice.message.generation_log_probs = generation_log_probs @@ -674,6 +701,58 @@ def attach_token_information_to_chat_response_choices( return response +def chat_generation_tokens(generation_details: Any) -> tuple[list[int], list[float]]: + """One choice's generated token IDs and their sampled-token log probabilities. + + As the chat endpoint reports them in ``generation_token_ids`` and + ``generation_log_probs``: each log probability is floored at + ``VLLM_LOGPROB_FLOOR``. + """ + choice_index = generation_details.index + output_token_ids = getattr(generation_details, "token_ids", None) + if output_token_ids is None: + raise RuntimeError( + "vLLM was asked to return token information for the " + "OpenAI-compatible chat endpoint but generation output " + f"choice_idx={choice_index} did not include token_ids." + ) + generation_token_ids = list(output_token_ids) + + generation_logprob_details = getattr(generation_details, "logprobs", None) + if generation_logprob_details is None: + if generation_token_ids: + raise RuntimeError( + "vLLM was asked to return token information for the " + "OpenAI-compatible chat endpoint but generation output " + f"choice_idx={choice_index} did not include logprobs." + ) + return generation_token_ids, [] + if len(generation_token_ids) != len(generation_logprob_details): + raise RuntimeError( + "vLLM returned mismatched generation token IDs and log " + "probabilities for the OpenAI-compatible chat endpoint: " + f"choice_idx={choice_index}, " + f"token_count={len(generation_token_ids)}, " + f"logprob_count={len(generation_logprob_details)}." + ) + generation_log_probs = [] + for token_id, position_logprobs in zip( + generation_token_ids, generation_logprob_details + ): + selected_token_logprob = position_logprobs.get(token_id) + if selected_token_logprob is None: + raise RuntimeError( + "vLLM generation log probabilities did not include the " + "selected token while attaching token information to " + "the OpenAI-compatible chat response: " + f"choice_idx={choice_index}, token_id={token_id}." + ) + generation_log_probs.append( + max(float(selected_token_logprob.logprob), VLLM_LOGPROB_FLOOR) + ) + return generation_token_ids, generation_log_probs + + def model_dump_chat_response_with_dynamic_message_fields( response: Any, ) -> dict[str, Any]: diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index eb395b4baca..7dee0203c6f 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -32,12 +32,14 @@ import ray from ray.util.placement_group import PlacementGroup +from nemo_rl.data.multimodal_utils import VLLM_PROMPT_KEYS from nemo_rl.distributed.batched_data_dict import BatchedDataDict, SlicedDataDict from nemo_rl.distributed.named_sharding import NamedSharding from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env from nemo_rl.distributed.refit_watchdog import RefitAborted from nemo_rl.distributed.virtual_cluster import NVLINK_DOMAIN_UNKNOWN, RayVirtualCluster from nemo_rl.distributed.worker_groups import RayWorkerBuilder, RayWorkerGroup +from nemo_rl.models.generation.dynamo.config import dynamo_cfg, dynamo_enabled from nemo_rl.models.generation.fleet_health import ( GenerationFleetHealth, HealthyShardSelector, @@ -47,12 +49,17 @@ GenerationInterface, GenerationOutputSpec, agreed_media_columns, + verify_right_padding, ) from nemo_rl.models.generation.vllm.config import ( REFITTABLE_FP8_KV_CACHE_DTYPES, VllmConfig, validate_vllm_quantization_config, ) +from nemo_rl.models.generation.vllm.request_pipeline import ( + rollout_output, + rollout_result_from_wire, +) from nemo_rl.models.generation.vllm.draft_utils import ( DRAFT_DISABLE_MODULE_SHARING_ENV, draft_module_sharing_disable_required, @@ -82,6 +89,7 @@ from nemo_rl.algorithms.single_controller_utils.config import MasterConfig from nemo_rl.data.captured_media import MediaColumnSpec from nemo_rl.data_plane.interfaces import DataPlaneConfig + from nemo_rl.models.generation.dynamo.managed_runtime import ManagedDynamoRuntime logger = logging.getLogger(__name__) @@ -168,11 +176,47 @@ def _record_vllm_generation_metrics( class VllmGeneration(GenerationInterface): + # Dynamo serving (policy.generation.dynamo): __init__ starts etcd, NATS and + # the frontend; each engine registers itself once its AsyncLLM runs. Class + # defaults, so shutdown() also works when __init__ failed before starting them. + _dynamo_runtime: Optional["ManagedDynamoRuntime"] = None + dynamo_frontend_url: Optional[str] = None + @classmethod def validate_settings(cls, master_config: "MasterConfig") -> None: """Reject pure-config vLLM settings the SC entrypoint cannot honor.""" generation_config = cast(VllmConfig, master_config.policy["generation"]) assert_reload_refit_config_supported(generation_config) + if dynamo_enabled(generation_config): + # Deferred: the NeMo-Gym environment module is heavy and optional. + from nemo_rl.environments.nemo_gym import should_use_nemo_gym + from nemo_rl.models.megatron.router_replay import router_replay_enabled + + if master_config.async_rl.generation_router.enabled: + raise ValueError( + "async_rl.generation_router.enabled=true does not apply with " + "policy.generation.dynamo=true: NeMo-Gym calls the Dynamo " + "frontend, which routes the whole fleet behind one URL. " + "Disable the generation router." + ) + if master_config.async_rl.generation_fleet_health.enabled: + raise ValueError( + "async_rl.generation_fleet_health.enabled=true is not supported " + "with policy.generation.dynamo=true yet: the Dynamo frontend " + "picks engines itself, and a restarted shard does not register " + "with it. Disable fleet health." + ) + if ( + should_use_nemo_gym(master_config) + and router_replay_enabled(master_config.policy) + and not master_config.token_capture.enabled + ): + raise ValueError( + "Router replay with NeMo-Gym and policy.generation.dynamo=true " + "requires token_capture.enabled=true: the engine stages routed " + "experts with the captured call, and NeMo-Gym's Dynamo dialect " + "does not return them in responses." + ) @staticmethod def init_cluster_placement_groups( @@ -262,6 +306,12 @@ def __init__( "See https://github.com/NVIDIA-NeMo/RL/issues/1101 for more details." ) + if dynamo_enabled(self.cfg) and not self.cfg["vllm_cfg"]["async_engine"]: + raise ValueError( + "policy.generation.dynamo=true serves the async vLLM engine; set " + "vllm_cfg.async_engine=true (configure_generation_config does)." + ) + # Validate sampling parameters early to avoid resource allocation with unsupported configs. top_k: int | None = self.cfg["top_k"] if top_k is not None and top_k != -1 and top_k < 1: @@ -429,6 +479,9 @@ def __init__( # and it should not be discovered with getattr at the read site. self._refit_membership: Optional["RefitMembership"] = None + if dynamo_enabled(self.cfg): + self._start_dynamo_services() + if defer_model_load: # Workers only reserved ports — collect URLs immediately and defer # the heavy model loading (and HTTP server start) to load_and_start(). @@ -443,6 +496,7 @@ def __init__( # when the http server is active. self.dp_openai_server_base_urls = self._report_dp_openai_server_base_urls() self.device_uuids = self._report_device_id() + self._register_dynamo_engines() self._step_metrics_snapshot: dict[str | tuple[str, int], float] | None = None @@ -685,6 +739,79 @@ def load_and_start(self) -> None: # Save device UUIDs self.device_uuids = self._report_device_id() + self._register_dynamo_engines() + + @property + def client_base_urls(self) -> list[Optional[str]]: + """Where OpenAI clients such as NeMo-Gym send requests. + + The Dynamo frontend when it serves this fleet (``policy.generation.dynamo``); + otherwise every data-parallel shard's own server, as + ``dp_openai_server_base_urls``. Known before a deferred model load. + """ + if self.dynamo_frontend_url is not None: + return [self.dynamo_frontend_url] + return self.dp_openai_server_base_urls + + def _start_dynamo_services(self) -> None: + """Start the etcd, NATS and frontend this fleet's engines register with.""" + from nemo_rl.models.generation.dynamo.managed_runtime import ( + ManagedDynamoRuntime, + ) + from nemo_rl.models.generation.dynamo.venv import get_dynamo_python + + runtime = ManagedDynamoRuntime( + dynamo_cfg=dynamo_cfg(self.cfg), + model_name=self.cfg["model_name"], + frontend_python=get_dynamo_python(), + ) + runtime.start() + self._dynamo_runtime = runtime + self.dynamo_frontend_url = runtime.frontend_url + + def _register_dynamo_engines(self) -> None: + """Serve every shard's engine through Dynamo; wait until it routes to all.""" + runtime = self._dynamo_runtime + if runtime is None: + return + leader_indices = [ + self.worker_group.get_dp_leader_worker_idx(shard) + for shard in range(self.dp_size) + ] + ray.get( + [ + self.worker_group.workers[idx].start_dynamo_endpoint_async.remote( + namespace=runtime.namespace, + engine_env=runtime.engine_env, + leader_worker_idx=idx, + ) + for idx in leader_indices + ] + ) + runtime.wait_for_engines( + self.dp_size, check_engines=self._check_dynamo_endpoints + ) + + def _check_dynamo_endpoints(self) -> None: + """Raise if a DP leader exited or its Dynamo endpoint stopped serving.""" + leaders = [ + self.worker_group.workers[self.worker_group.get_dp_leader_worker_idx(shard)] + for shard in range(self.dp_size) + ] + try: + errors = ray.get( + [leader.dynamo_endpoint_error.remote() for leader in leaders] + ) + except ray.exceptions.RayActorError as error: + raise RuntimeError( + "A vLLM engine exited while waiting for Dynamo registration." + ) from error + for shard, error in enumerate(errors): + if error is not None: + raise RuntimeError( + f"The Dynamo endpoint of generation shard {shard} stopped " + f"serving: {error}" + ) def _post_init(self): # Choose the appropriate method based on async_engine setting @@ -1379,11 +1506,69 @@ def validate_generate_data(data): return False # Return False for empty case to trigger early return return True + if self._dynamo_serves(data): + if validate_generate_data(data): + yield await self._generate_through_dynamo(data, greedy) + return async for result in self._async_generate_base( data, "generate_async", validate_generate_data, greedy ): yield result + def _dynamo_serves(self, data: BatchedDataDict[GenerationDatumSpec]) -> bool: + """Whether a rollout goes through the Dynamo frontend's KV-aware router. + + Media-backed prompts keep this class's own dispatch: the engine does not + yet take them through the frontend. + """ + return self.dynamo_frontend_url is not None and not ( + VLLM_PROMPT_KEYS & data.keys() + ) + + async def _generate_through_dynamo( + self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool + ) -> tuple[int, BatchedDataDict[GenerationOutputSpec]]: + """One rollout through the frontend, sampled by the engine as natively.""" + # Local import: nemo_rl.experience pulls the rollout stack, which must not + # become a load-time dependency of the generation backend. + from nemo_rl.experience.failures import GenerationUnavailable + from nemo_rl.models.generation.dynamo.rollout_client import ( + post_rollout, + rollout_payload, + ) + + assert data.size == 1, ( + "generate_async is restricted to handle only single samples, " + f"but received batch_size={data.size}. Please handle batching " + "outside this method." + ) + verify_right_padding(data, pad_value=self.cfg["_pad_token_id"]) + input_length = int(data["input_lengths"][0].item()) + input_ids = data["input_ids"][0] + stop_strings = data.get("stop_strings", [None])[0] + payload = rollout_payload( + model_name=self.cfg["model_name"], + prompt_token_ids=input_ids[:input_length].tolist(), + max_model_len=int(self.cfg["vllm_cfg"]["max_model_len"]), + greedy=greedy, + stop_strings=stop_strings, + ) + url = f"{self.dynamo_frontend_url}/completions" + try: + engine_data = await post_rollout( + url, payload, dynamo_cfg(self.cfg).request_timeout_s + ) + except RuntimeError as error: + raise GenerationUnavailable(str(error)) from error + output = rollout_output( + rollout_result_from_wire(engine_data["rollout"]), + input_ids=input_ids, + input_length=input_length, + pad_token_id=self.cfg["_pad_token_id"], + ) + output["gen_leader_worker_idx"] = [int(engine_data["gen_leader_worker_idx"])] + return 0, output + def prepare_for_generation(self, *args: Any, **kwargs: Any) -> bool: """Wake workers up for colocated inference.""" # non-colocated no need to wake up @@ -1448,6 +1633,21 @@ def shutdown(self) -> bool: except Exception as e: print(f"Error during policy shutdown: {e}") return False + finally: + # After the engines: they leave discovery as their processes exit. + if self._dynamo_runtime is not None: + self._dynamo_runtime.shutdown() + self._dynamo_runtime = None + + def __getstate__(self) -> dict[str, Any]: + """Leave the driver-owned Dynamo services behind when shipped to an actor. + + Rollout actors receive this object; a copy keeps the frontend URL and the + worker handles, but the service processes stay with the driver. + """ + state = self.__dict__.copy() + state["_dynamo_runtime"] = None + return state def prepare_refit_info( self, state_dict_info: Optional[dict[str, Any]] diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 9231aa06d22..d61641a91d8 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -49,6 +49,7 @@ ) from nemo_rl.distributed.worker_group_utils import get_nsight_config_if_pattern_matches from nemo_rl.experience.route_assembly import boundary_route_rows +from nemo_rl.models.generation.dynamo.config import dynamo_cfg, dynamo_enabled from nemo_rl.models.generation.interfaces import ( GenerationDatumSpec, GenerationOutputSpec, @@ -64,13 +65,18 @@ resolve_collective_rpc_result, ) from nemo_rl.models.generation.vllm.config import parse_nvfp4_pertoken_rollout +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result, +) from nemo_rl.models.generation.vllm.utils import ( attach_routed_experts_to_chat_response_choices, attach_token_information_to_chat_response_choices, format_prompt_for_vllm_generation, + off_policy_sampling_error, validate_rollout_prompt, model_dump_chat_response_with_dynamic_message_fields, - pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) from nemo_rl.models.generation.vllm.vllm_worker import BaseVllmGenerationWorker @@ -273,6 +279,8 @@ def __init__( self.http_server = None self._engine_loop = None self._http_engine_client = None + # The task serving this engine through Dynamo, when it is registered. + self._dynamo_endpoint_task: Optional[asyncio.Task] = None # Ledger-authoritative token capture (dormant until the # setup_token_capture fan-out runs). The weight @@ -407,6 +415,26 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: stacklevel=1, ) llm_kwargs["compilation_config"] = CompilationConfig(**compilation_config) + if dynamo_enabled(self.cfg): + from nemo_rl.models.generation.dynamo.native_engine import ( + dynamo_kv_events_config, + ) + + # Dynamo's KV-aware router indexes the engine's own KV-cache events. + llm_kwargs.setdefault( + "kv_events_config", + dynamo_kv_events_config( + dynamo_cfg(self.cfg), + prefix_caching=llm_kwargs["enable_prefix_caching"], + ), + ) + # AsyncEngineArgs leaves kv_events_config dicts unconverted. + if isinstance(llm_kwargs.get("kv_events_config"), dict): + from vllm.config import KVEventsConfig + + llm_kwargs["kv_events_config"] = KVEventsConfig( + **llm_kwargs["kv_events_config"] + ) self.llm_async_engine_args = AsyncEngineArgs(**llm_kwargs) self.stat_loggers = ( @@ -414,6 +442,13 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: if self.cfg["vllm_cfg"].get("enable_vllm_metrics_logger", False) else [] ) + if dynamo_enabled(self.cfg): + from nemo_rl.models.generation.dynamo.native_engine import ( + SnapshotStatLogger, + ) + + # Feeds the router's per-engine load signal. + self.stat_loggers.append(SnapshotStatLogger) self.llm = AsyncLLM.from_engine_args( self.llm_async_engine_args, stat_loggers=self.stat_loggers ) @@ -577,6 +612,37 @@ async def post_init_async(self): self._setup_vllm_server() ) + async def start_dynamo_endpoint_async( + self, *, namespace: str, engine_env: dict[str, str], leader_worker_idx: int + ) -> None: + """Register this engine with Dynamo and serve it from this actor's loop. + + The loop is the one that owns the AsyncLLM, so Dynamo requests, refit, + pause and sleep interleave exactly as this worker's own requests do. + ``leader_worker_idx`` is this worker's index in its group; rollouts the + engine serves report it, as ``generate_async`` results do. + """ + from nemo_rl.models.generation.dynamo.native_engine import ( + start_dynamo_endpoint, + ) + + self._dynamo_endpoint_task = start_dynamo_endpoint( + self, + namespace=namespace, + engine_env=engine_env, + leader_worker_idx=leader_worker_idx, + ) + + async def dynamo_endpoint_error(self) -> Optional[str]: + """Why this engine's Dynamo endpoint stopped serving; None while it serves.""" + task = self._dynamo_endpoint_task + if task is None or not task.done(): + return None + if task.cancelled(): + return "cancelled" + error = task.exception() + return "exited" if error is None else repr(error) + async def get_reserved_url(self) -> Optional[str]: """Return the URL from the reserved socket, available before model loading.""" if self._reserved_socket is not None: @@ -691,7 +757,10 @@ def _capture_admission(self, request: Any) -> Any | None: context. The dict itself is never mutated: the admission is the typed, read-only contract that the prefix resolution and ``begin_call`` share. """ - context = getattr(request, "ng_capture", None) + return self._parse_capture_admission(getattr(request, "ng_capture", None)) + + def _parse_capture_admission(self, context: Any) -> Any | None: + """``_capture_admission`` for a raw ``ng_capture`` context (Dynamo's annotation).""" if self.token_capture is None or not context: return None # Deferred: nemo_gym is an optional extra absent in non-gym runs. @@ -1473,46 +1542,19 @@ class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): async def create_chat_completion( request: NeMoRLChatCompletionRequest, raw_request: Request ): - # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params - # Right now we explicitly assert set this to -1. - assert request.top_k in (None, -1), ( - f"Top k sampling parameter must be unset, empty, or -1. Got `{request.top_k}`" - ) - request.top_k = -1 - # The request sampling params need to exactly match those as are set in NeMo RL. # If they do not match, the inference will be off policy and destroy training - # stability. Validation rollouts are the one exception: they are stamped with - # the validation sampling profile (generation.val_temperature / val_top_p), - # which is metric-only and safe to serve — grpo.validate() is the only - # caller that constructs a non-train GenerationSamplingParams. Multi-turn - # agents issue their own requests, so this server-side check is the one - # chokepoint they all pass. - # vLLM resolves an unset top_p from the model's generation_config.json - # (ModelConfig.generation_config defaults to "auto"), NOT to 1.0, so a - # request omitting it would sample off-policy while passing this check. - assert request.top_p is not None, ( - "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " - "resolved by vLLM from the model's generation_config.json and would " - "bypass the on-policy sampling check." - ) - request_top_p = request.top_p - is_train_sampling = ( - request.temperature == generation_config["temperature"] - and request_top_p == generation_config["top_p"] - ) - is_val_sampling = ( - request.temperature == generation_config["val_temperature"] - and request_top_p == generation_config["val_top_p"] - ) - assert is_train_sampling or is_val_sampling, ( - f"request sampling (temperature={request.temperature}, " - f"top_p={request.top_p}) matches neither the train sampling params " - f"(temperature={generation_config['temperature']}, " - f"top_p={generation_config['top_p']}) nor the validation sampling " - f"params (val_temperature={generation_config['val_temperature']}, " - f"val_top_p={generation_config['val_top_p']})" + # stability. grpo.validate() is the only caller that constructs a non-train + # GenerationSamplingParams, and the validation profile it stamps is accepted. + sampling_error = off_policy_sampling_error( + temperature=request.temperature, + top_p=request.top_p, + top_k=request.top_k, + generation_config=generation_config, ) + assert sampling_error is None, sampling_error + # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params + request.top_k = -1 try: generator = await openai_serving_chat.create_chat_completion( @@ -1813,6 +1855,55 @@ async def init_collective_async( ), ) + def _rollout_max_new_tokens(self, input_length: int) -> int: + """Token budget for one rollout prompt; 0 means nothing fits. + + The configured cap, clipped to the context window and to speculative + decoding's lookahead. Shared by ``generate_async`` and the Dynamo engine, + so a rollout gets the same budget whichever path serves it. + """ + max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) + allowed_new_tokens = max( + 0, min(self.cfg["max_new_tokens"], max_model_len - input_length) + ) + spec_cfg = self.cfg.get("vllm_kwargs", {}).get("speculative_config") or {} + spec_lookahead = int(spec_cfg.get("num_speculative_tokens", 0)) + if allowed_new_tokens > 0 and spec_lookahead > 0: + allowed_new_tokens = self._request_max_new_tokens( + configured_max_new_tokens=allowed_new_tokens, + input_length=input_length, + max_model_len=max_model_len, + cap_to_context=False, + spec_lookahead=spec_lookahead, + ) + return allowed_new_tokens + + def _rollout_sampling_params( + self, + *, + max_new_tokens: int, + greedy: bool, + stop_strings: Optional[list[str]], + ) -> Any: + """The SamplingParams for one rollout. + + Shared by ``generate_async`` and Dynamo serving, so a rollout is sampled + the same way whichever path serves it. + + Args: + max_new_tokens: The rollout's budget, from ``_rollout_max_new_tokens``. + greedy: Greedy decoding instead of the configured sampling. + stop_strings: The sample's own stop strings, merged with the + configured ones. + """ + return self._build_sampling_params( + greedy=greedy, + stop_strings=self._merge_stop_strings( + [stop_strings] if stop_strings else None + ), + max_new_tokens=max_new_tokens, + ) + async def generate_async( self, data: BatchedDataDict[GenerationDatumSpec], @@ -1865,231 +1956,49 @@ async def process_single_sample(sample_idx): ): per_sample_stop_strings = batch_specific_stop_strings_list[sample_idx] - final_stop_strings_for_sample = self._merge_stop_strings( - [per_sample_stop_strings] if per_sample_stop_strings else None - ) - - max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) - remaining_ctx = max_model_len - current_input_actual_length - allowed_new_tokens = max(0, min(self.cfg["max_new_tokens"], remaining_ctx)) - - spec_cfg = self.cfg.get("vllm_kwargs", {}).get("speculative_config") or {} - spec_lookahead = int(spec_cfg.get("num_speculative_tokens", 0)) - if allowed_new_tokens > 0 and spec_lookahead > 0: - allowed_new_tokens = self._request_max_new_tokens( - configured_max_new_tokens=allowed_new_tokens, - input_length=current_input_actual_length, - max_model_len=max_model_len, - cap_to_context=False, - spec_lookahead=spec_lookahead, - ) - - # Handle case where no tokens can be generated due to length constraints - if allowed_new_tokens == 0: - # Access the input data directly from the function parameters - input_ids_single_row = input_ids_batch[sample_idx] - - # Create output tensors with just the input (no generated tokens) - output_ids_single_item_batched = input_ids_single_row[ - :current_input_actual_length - ].unsqueeze(0) - - logprobs_single_item = torch.zeros( - (1, current_input_actual_length), - dtype=torch.float32, - device=input_ids_single_row.device, + max_new_tokens = self._rollout_max_new_tokens(current_input_actual_length) + input_ids_single_row = input_ids_batch[sample_idx] + if max_new_tokens == 0: + # No new token fits in the context: return the prompt alone. + result = RolloutResult.nothing_generated() + else: + sampling_params_for_request = self._rollout_sampling_params( + max_new_tokens=max_new_tokens, + greedy=greedy, + stop_strings=per_sample_stop_strings, ) + request_id = str(uuid.uuid4()) + final_request_output = None + async for req_output in self.llm.generate( + prompt=prompt, + sampling_params=sampling_params_for_request, + request_id=request_id, + ): + final_request_output = req_output + if final_request_output is None: + raise RuntimeError(f"No output received for request {request_id}") - generation_lengths_tensor = torch.tensor( - [0], dtype=torch.long, device=input_ids_single_row.device + validate_rollout_prompt( + input_ids_single_row[:current_input_actual_length].tolist(), + final_request_output.prompt_token_ids, ) - - unpadded_sequence_lengths_tensor = torch.tensor( - [current_input_actual_length], - dtype=torch.long, + result = rollout_result( + final_request_output, + input_length=current_input_actual_length, + return_routed_experts=self._return_routed_experts_enabled(), + routed_experts_dtype=self.routed_experts_dtype, device=input_ids_single_row.device, + sample_idx=sample_idx, ) - - # Not truncated since no generation was attempted (length constraint) - truncated_tensor = torch.tensor( - [False], dtype=torch.bool, device=input_ids_single_row.device - ) - - result_batch = BatchedDataDict[GenerationOutputSpec]( - { - "output_ids": output_ids_single_item_batched, - "logprobs": logprobs_single_item, - "generation_lengths": generation_lengths_tensor, - "unpadded_sequence_lengths": unpadded_sequence_lengths_tensor, - "truncated": truncated_tensor, - } - ) - - return (sample_idx, result_batch) - - sampling_params_for_request = self._build_sampling_params( - greedy=greedy, - stop_strings=final_stop_strings_for_sample, - max_new_tokens=allowed_new_tokens, - ) - - request_id = str(uuid.uuid4()) - - # Generate using vLLM async engine - vllm_request_generator = self.llm.generate( - prompt=prompt, - sampling_params=sampling_params_for_request, - request_id=request_id, - ) - - # Get the final result from the generator - final_request_output = None - async for req_output in vllm_request_generator: - final_request_output = req_output - - if final_request_output is None: - raise RuntimeError(f"No output received for request {request_id}") - - validate_rollout_prompt( - input_ids_batch[sample_idx, :current_input_actual_length].tolist(), - final_request_output.prompt_token_ids, - ) - - # Process the output - generation_details = final_request_output.outputs[0] - generated_token_ids = list(generation_details.token_ids) - num_generated_tokens = len(generated_token_ids) - return_routed_experts = self._return_routed_experts_enabled() - - original_input_ids_single_row = input_ids_batch[sample_idx] - final_output_tensor_len = current_input_actual_length + num_generated_tokens - - # Create output_ids tensor for this single item - output_ids_single_item = torch.full( - (final_output_tensor_len,), - self.cfg["_pad_token_id"], - dtype=original_input_ids_single_row.dtype, - device=original_input_ids_single_row.device, - ) - # Copy original input (up to its actual length) - output_ids_single_item[:current_input_actual_length] = ( - original_input_ids_single_row[:current_input_actual_length] - ) - # Add generated tokens after the actual input - output_ids_single_item[ - current_input_actual_length : current_input_actual_length - + num_generated_tokens - ] = torch.tensor( - generated_token_ids, - dtype=original_input_ids_single_row.dtype, - device=original_input_ids_single_row.device, - ) - - # Reshape to (1, seq_len) for BatchedDataDict - output_ids_single_item_batched = output_ids_single_item.unsqueeze(0) - - # Create logprobs tensor for this single item - logprobs_single_item = torch.zeros( - (1, final_output_tensor_len), - dtype=torch.float32, - device=original_input_ids_single_row.device, - ) - if hasattr(generation_details, "logprobs") and generation_details.logprobs: - for idx, logprob_dict_per_token in enumerate( - generation_details.logprobs - ): - if logprob_dict_per_token and idx < len(generated_token_ids): - token_id_at_idx = generated_token_ids[idx] - if token_id_at_idx in logprob_dict_per_token: - logprob_value = logprob_dict_per_token[ - token_id_at_idx - ].logprob - position_in_output_tensor = ( - current_input_actual_length + idx - ) - if position_in_output_tensor < final_output_tensor_len: - logprobs_single_item[0, position_in_output_tensor] = ( - logprob_value - ) - - # Generation lengths - generation_lengths_tensor = torch.tensor( - [num_generated_tokens], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - - # Unpadded sequence lengths (actual_input + actual_generated) - unpadded_total_length = current_input_actual_length + num_generated_tokens - unpadded_sequence_lengths_tensor = torch.tensor( - [unpadded_total_length], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - - # Check if response was truncated (hit max_tokens length limit) - is_truncated = generation_details.finish_reason == "length" - truncated_tensor = torch.tensor( - [is_truncated], - dtype=torch.bool, - device=original_input_ids_single_row.device, - ) - - result_dict = { - "output_ids": output_ids_single_item_batched, - "logprobs": logprobs_single_item, - "generation_lengths": generation_lengths_tensor, - "unpadded_sequence_lengths": unpadded_sequence_lengths_tensor, - "truncated": truncated_tensor, - } - routed_experts, r3_stats = pad_and_align_routed_expert_indices( - final_request_output, - generation_details, - valid_length=unpadded_total_length, - padded_length=final_output_tensor_len, - device=original_input_ids_single_row.device, - require_complete_routed_experts=return_routed_experts, - return_stats=True, - routed_experts_dtype=self.routed_experts_dtype, + return ( + sample_idx, + rollout_output( + result, + input_ids=input_ids_single_row, + input_length=current_input_actual_length, + pad_token_id=self.cfg["_pad_token_id"], + ), ) - if return_routed_experts and routed_experts is None: - raise RuntimeError( - "vLLM was asked to return routed experts but the generation output " - "did not include routed_experts." - ) - if return_routed_experts: - if r3_stats["missing_routes"] > 0: - LOGGER.warning( - "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for sample_idx=%d, missing_token_routes=%d, " - "actual_routes=%d, expected_routes=%d. Megatron will use its " - "own router for those missing token routes.", - sample_idx, - r3_stats["missing_routes"], - r3_stats["actual_routes"], - r3_stats["expected_routes"], - ) - result_dict["r3_routed_experts_missing_routes"] = torch.tensor( - [r3_stats["missing_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - result_dict["r3_routed_experts_expected_routes"] = torch.tensor( - [r3_stats["expected_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - result_dict["r3_routed_experts_actual_routes"] = torch.tensor( - [r3_stats["actual_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - if routed_experts is not None: - result_dict["routed_experts"] = routed_experts.unsqueeze(0) - - result_batch = BatchedDataDict[GenerationOutputSpec](result_dict) - - return (sample_idx, result_batch) # Create tasks for all samples and yield results as they complete sample_tasks = [ diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 2456d8e488f..5e511555b3e 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -201,6 +201,57 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: ) +# Keys of the managed dynamo.vllm backend's dynamo_cfg with no counterpart now that +# Dynamo serves NeMo RL's own vLLM workers, and what replaces each. +_REMOVED_DYNAMO_CFG_KEYS = { + "engine": "Dynamo always serves the vLLM backend's workers", + "control_timeout_s": "refit and pause use the vLLM backend's own controls", + "metrics_include_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", + "metrics_exclude_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", +} + + +def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: + """Fail on the removed ``backend: dynamo`` and its dynamo_cfg keys. + + Dynamo now serves NeMo RL's own vLLM workers: ``backend: vllm`` plus + ``dynamo: true``. One migration, so the backend is reported first and the + dynamo_cfg keys that no longer exist with it. + + Args: + config: The config as the user wrote it, resolved to plain dicts. + """ + policy = config.get("policy") + if not isinstance(policy, dict): + return + generation = policy.get("generation") + if not isinstance(generation, dict): + return + problems = [] + if generation.get("backend") == "dynamo": + problems.append( + "policy.generation.backend='dynamo' was removed: Dynamo now serves NeMo " + "RL's own vLLM workers. Set policy.generation.backend=vllm and " + "policy.generation.dynamo=true; vllm_cfg and vllm_kwargs keep their " + "meaning." + ) + dynamo_cfg = generation.get("dynamo_cfg") + if isinstance(dynamo_cfg, dict): + for key, replacement in _REMOVED_DYNAMO_CFG_KEYS.items(): + if key in dynamo_cfg: + problems.append( + f"policy.generation.dynamo_cfg.{key} was removed: {replacement}." + ) + worker_args = dynamo_cfg.get("worker_args") + if isinstance(worker_args, dict) and "extra_cli_args" in worker_args: + problems.append( + "policy.generation.dynamo_cfg.worker_args.extra_cli_args was " + "removed: engines are configured through vllm_cfg and vllm_kwargs." + ) + if problems: + raise ValueError(" ".join(problems)) + + def check_outdated_config(config: dict[str, Any]) -> None: """Fail fast on config the code no longer accepts, naming the migration to apply. @@ -218,3 +269,4 @@ def check_outdated_config(config: dict[str, Any]) -> None: reject_outdated_metric_name_format(config) reject_outdated_mcore_generation_keys(config) reject_outdated_dynamo_router_reset_states(config) + reject_outdated_dynamo_backend(config) diff --git a/nemo_rl/utils/routed_experts_codec.py b/nemo_rl/utils/routed_experts_codec.py index e7cc41be2c5..511748740c1 100644 --- a/nemo_rl/utils/routed_experts_codec.py +++ b/nemo_rl/utils/routed_experts_codec.py @@ -104,3 +104,11 @@ def decode_routed_experts(payload: Union[str, Any], dtype: torch.dtype) -> torch return torch.empty(shape, dtype=dtype) tensor = torch.frombuffer(raw, dtype=wire_dtype).reshape(shape) return tensor if tensor.dtype == dtype else tensor.to(dtype) + + +def routed_experts_envelope_dtype(payload: str) -> torch.dtype: + """The dtype a base64 envelope was encoded with.""" + parts = payload.split(":", 2) + if len(parts) != 3 or parts[0] != _MAGIC or parts[1] not in _WIRE_TORCH_DTYPES: + raise ValueError("routed_experts payload is not a valid envelope.") + return _WIRE_TORCH_DTYPES[parts[1]] diff --git a/nemo_rl/weight_sync/collective_weight_synchronizer.py b/nemo_rl/weight_sync/collective_weight_synchronizer.py index 6b8fa7d50ad..7ac9366d6ae 100644 --- a/nemo_rl/weight_sync/collective_weight_synchronizer.py +++ b/nemo_rl/weight_sync/collective_weight_synchronizer.py @@ -190,13 +190,13 @@ def _desired_membership( ) -> Optional[RefitMembership]: """The membership to track, or None for a backend that owns no DP worker group. - NOT every GenerationInterface has one. vLLM and TRT-LLM do; Dynamo, Megatron and - SGLang do not, and the interface declares nothing either way -- so reading + NOT every GenerationInterface has one. vLLM and TRT-LLM do; Megatron and SGLang + do not, and the interface declares nothing either way -- so reading ``self._generation.worker_group`` assumes a vLLM shape that this synchronizer is - not entitled to assume. It holds a GenerationInterface, and Dynamo reaches it - through the ordinary non-colocated branch of the factory. + not entitled to assume. - That assumption broke L1_Functional_Tests_Dynamo on the plain grpo.py path: + That assumption once broke L1_Functional_Tests_Dynamo on the plain grpo.py path, + under the since-removed managed Dynamo backend: grpo.py:1747 policy_generation.weight_synchronizer.init_communicator() AttributeError: 'DynamoGeneration' object has no attribute 'worker_group' diff --git a/nemo_rl/weight_sync/factory.py b/nemo_rl/weight_sync/factory.py index 3524b22c559..9a09ee550b2 100644 --- a/nemo_rl/weight_sync/factory.py +++ b/nemo_rl/weight_sync/factory.py @@ -27,7 +27,6 @@ from typing import Any, Optional from nemo_rl.models.generation.constants import ( - DYNAMO_BACKEND, MEGATRON_BACKEND, SGLANG_BACKEND, TRTLLM_BACKEND, @@ -92,7 +91,7 @@ def create_weight_synchronizer( policy: Policy object (ColocatablePolicyInterface). generation: Generation object (GenerationInterface). generation_backend: Name of the generation backend ("vllm", "sglang", - "megatron", or "dynamo"). + "megatron", or "trtllm"). colocated: Whether policy and generation share the same GPUs. train_cluster: RayVirtualCluster for training workers. Required for non-colocated deployments except SGLang, which owns its own group. @@ -114,7 +113,6 @@ def create_weight_synchronizer( VLLM_BACKEND, SGLANG_BACKEND, MEGATRON_BACKEND, - DYNAMO_BACKEND, TRTLLM_BACKEND, } if generation_backend not in _SUPPORTED_BACKENDS: diff --git a/pyproject.toml b/pyproject.toml index 4e2770faaf7..259d712d387 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,6 +177,11 @@ vllm = [ "flashinfer-cubin==0.6.18", "flashinfer-jit-cache==0.6.18", "nvidia-cutlass-dsl[cu13]==4.6.2", + # Dynamo serving (policy.generation.dynamo=true) hosts these vLLM workers behind + # dynamo.frontend through Dynamo's engine-agnostic backend SDK. Only the base + # package: its [vllm] extra pins Dynamo's own vLLM, which NeMo RL never runs. + "ai-dynamo==1.5.1", + "ai-dynamo-runtime==1.5.1", ] sglang = [ # sglang-miles (3003d70f) and sglang-kernel 0.4.5 are built against torch 2.11; @@ -453,10 +458,14 @@ override-dependencies = [ "nvidia-cudnn-frontend==1.25.0 ; sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')", # NCCL 2.30.4 fixes a context-parallel hang on GB200 when GPUs are non-contiguous; # 2.30.7 additionally fixes hangs in the NCCL m2n reshard (). - # Keep NCCL shared with the isolated Dynamo environment across torch upgrades. "nvidia-nccl-cu13==2.30.7", "nvidia-nvshmem-cu13>=3.6.5; sys_platform != 'darwin'", "opencv-python-headless; sys_platform == 'never'", + # ai-dynamo (vllm extra) requires these for its planner and simulators only + # (aisimulate brings jax, flax and tfp-nightly). The backend SDK and frontend + # that Dynamo serving uses import neither. + "aisimulate; sys_platform == 'never'", + "kubernetes; sys_platform == 'never'", "timm<=1.0.22", "fastapi[standard]>=0.133.0,<0.137.0", "nvidia-modelopt[torch]>=0.44.0a0", diff --git a/pyrefly.toml b/pyrefly.toml index ca658115f82..ae201bdbf1d 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -234,16 +234,11 @@ project-includes = [ "nemo_rl/models/generation/dynamo/__init__.py", "nemo_rl/models/generation/dynamo/arguments.py", "nemo_rl/models/generation/dynamo/config.py", - "nemo_rl/models/generation/dynamo/dynamo_generation.py", - "nemo_rl/models/generation/dynamo/dynamo_worker.py", "nemo_rl/models/generation/dynamo/http_client.py", "nemo_rl/models/generation/dynamo/managed_runtime.py", - "nemo_rl/models/generation/dynamo/metrics.py", - "nemo_rl/models/generation/dynamo/refit.py", - "nemo_rl/models/generation/dynamo/token_wrapper.py", - "nemo_rl/models/generation/dynamo/validate_dynamo_vllm_args.py", + "nemo_rl/models/generation/dynamo/native_engine.py", + "nemo_rl/models/generation/dynamo/rollout_client.py", "nemo_rl/models/generation/dynamo/venv.py", - "nemo_rl/models/generation/dynamo/worker_pool.py", "nemo_rl/models/generation/engine_supervisor.py", "nemo_rl/models/generation/fleet_health.py", "nemo_rl/models/generation/generation_router.py", @@ -280,6 +275,7 @@ project-includes = [ "nemo_rl/models/generation/vllm/quantization/nvfp4_pertoken_config.py", "nemo_rl/models/generation/vllm/quantization/utils.py", "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", + "nemo_rl/models/generation/vllm/request_pipeline.py", "nemo_rl/models/generation/vllm/refit_layout.py", "nemo_rl/models/generation/vllm/refit_loader.py", "nemo_rl/models/generation/vllm/utils.py", diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index e3b08a8b481..225bbb65919 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -33,8 +33,13 @@ run_test() { fi } -run_test fast uv run --no-sync bash ./tests/functional/grpo_dynamo.sh -run_test uv run --no-sync bash ./tests/functional/grpo_dynamo_single_controller.sh +# Dynamo overlays: native SingleController tests rerun with policy.generation.dynamo=true. +# Each keeps the native test's checks and adds check_dynamo_overlay.sh's. +run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh ++policy.generation.dynamo=true +run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true ++policy.generation.dynamo=true +run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true +# Router replay with deferred routes: the engine stages each captured call's routed experts. +run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/check_dynamo_overlay.sh b/tests/functional/check_dynamo_overlay.sh new file mode 100755 index 00000000000..ee98e2671a0 --- /dev/null +++ b/tests/functional/check_dynamo_overlay.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# What a native functional test also checks when it runs as the Dynamo overlay +# (++policy.generation.dynamo=true): every engine registered with the managed +# frontend, the frontend routed rollouts to them, the KV router applied the +# engines' KV-cache events (when they publish any: KV routing with prefix +# caching), and shutdown stopped its services. The test's own metric checks hold +# as they do natively. +# +# bash check_dynamo_overlay.sh + +set -euo pipefail + +run_log=$1 + +grep -E "\[Dynamo\] frontend .* routes to [0-9]+ engine" "${run_log}" +routed=$(grep -c 'route request completed request.outcome="success"' "${run_log}" || true) +echo "Requests the Dynamo frontend routed: ${routed}" +if [[ "${routed}" -eq 0 ]]; then + echo "No request went through the Dynamo frontend" >&2 + exit 1 +fi + +# vLLM's KV-event wire format is the one coupling between NeMo RL's vLLM and +# Dynamo's router; a format change shows up here as no applied events. +if grep -q "\[Dynamo\] engine publishes KV events: True" "${run_log}"; then + grep -E "\[Dynamo\] router applied KV events" "${run_log}" + applied=$(grep -oE '\[Dynamo\] router applied KV events: .*"ok": [0-9.]+' "${run_log}" \ + | grep -oE '[0-9.]+$' | tail -1) + if [[ -z "${applied}" || "${applied%.*}" -eq 0 ]]; then + echo "The KV router applied none of the engines' KV-cache events" >&2 + exit 1 + fi +fi + +dynamo_bin_dir=${NEMO_RL_DYNAMO_BIN_DIR:-/opt/dynamo/bin} +if pgrep -f "[d]ynamo.frontend|${dynamo_bin_dir}/[e]tcd|${dynamo_bin_dir}/[n]ats-server"; then + echo "Managed Dynamo services remain after shutdown" >&2 + exit 1 +fi diff --git a/tests/functional/grpo_async_gym_single_controller.sh b/tests/functional/grpo_async_gym_single_controller.sh index 6acd46388cb..a69e17c847f 100755 --- a/tests/functional/grpo_async_gym_single_controller.sh +++ b/tests/functional/grpo_async_gym_single_controller.sh @@ -113,6 +113,10 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then if [[ "$*" == *token_capture.enabled=true* ]]; then # Nonzero only when a finalizer actor ran, i.e. capture really was on. EXTRA_CHECKS+=('max(data["train/finalize/total_ms"]) > 0') + if [[ "$*" == *router_replay.enabled=true* ]]; then + # Every captured row carries routed experts for the policy to replay. + EXTRA_CHECKS+=('min(data["train/finalize/routed_experts_row_coverage"]) == 1') + fi fi # Observed to be between 0.8-1.3 @@ -121,3 +125,7 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then 'max(data["train/reward"]) > 0' \ "${EXTRA_CHECKS[@]}" fi + +if [[ "$*" == *generation.dynamo=true* ]]; then + bash $SCRIPT_DIR/check_dynamo_overlay.sh $RUN_LOG +fi diff --git a/tests/functional/grpo_dp_single_controller.sh b/tests/functional/grpo_dp_single_controller.sh index f20bece801d..de4656dc6cf 100755 --- a/tests/functional/grpo_dp_single_controller.sh +++ b/tests/functional/grpo_dp_single_controller.sh @@ -61,3 +61,7 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then 'min(data["train/probs_ratio_clamped_max"]) > 0.79' \ 'max(data["train/probs_ratio_clamped_max"]) < 1.21' fi + +if [[ "$*" == *generation.dynamo=true* ]]; then + bash $SCRIPT_DIR/check_dynamo_overlay.sh $RUN_LOG +fi diff --git a/tests/functional/grpo_dynamo.sh b/tests/functional/grpo_dynamo.sh deleted file mode 100644 index bce045b9df8..00000000000 --- a/tests/functional/grpo_dynamo.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) -PROJECT_ROOT=$(realpath "${SCRIPT_DIR}/../..") -EXP_NAME=$(basename "$0" .sh) -EXP_DIR=${SCRIPT_DIR}/${EXP_NAME} -LOG_DIR=${EXP_DIR}/logs -RUN_LOG=${EXP_DIR}/run.log -export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} - -rm -rf "${EXP_DIR}" -mkdir -p "${LOG_DIR}" -git config --global --add safe.directory "${PROJECT_ROOT}" - -dynamo_venv_dir=${NEMO_RL_DYNAMO_VENV_DIR:-/opt/dynamo_venv} -dynamo_python=${dynamo_venv_dir}/bin/python -"${dynamo_python}" -c \ - 'import importlib.metadata as m; assert m.version("ai-dynamo") == "1.5.1"; assert m.version("vllm") == "0.28.0"' -"${dynamo_venv_dir}/bin/etcd" --version -"${dynamo_venv_dir}/bin/nats-server" --version - -cd "${PROJECT_ROOT}" -uv run --no-sync coverage run -a \ - --data-file="${PROJECT_ROOT}/tests/.coverage" \ - --source="${PROJECT_ROOT}/nemo_rl" \ - "${PROJECT_ROOT}/examples/run_grpo.py" \ - --config "${PROJECT_ROOT}/examples/configs/grpo_math_1B_dynamo.yaml" \ - policy.model_name=Qwen/Qwen3-0.6B \ - policy.tokenizer.name=Qwen/Qwen3-0.6B \ - logger.log_dir="${LOG_DIR}" \ - 2>&1 | tee "${RUN_LOG}" - -grep -F "Performing policy generation refit" "${RUN_LOG}" -grep -F "Invalidated generation backend KV caches after weight update" "${RUN_LOG}" - -refit_count=$(grep -Fc "Performing policy generation refit" "${RUN_LOG}" || true) -cache_success_count=$(grep -Fc \ - "Invalidated generation backend KV caches after weight update" \ - "${RUN_LOG}" || true) -if [[ "${refit_count}" -eq 0 || "${cache_success_count}" -ne "${refit_count}" ]]; then - echo "Expected one successful cache invalidation per refit; refits=${refit_count}, successes=${cache_success_count}" >&2 - exit 1 -fi -if grep -Fq \ - -e "Failed to invalidate generation backend KV caches" \ - -e "KV cache invalidation not supported or only partially applied" \ - "${RUN_LOG}"; then - echo "The Dynamo run reported a cache invalidation failure" >&2 - exit 1 -fi - -metrics_json=${EXP_DIR}/metrics.json -uv run --no-sync tests/json_dump_tb_logs.py \ - "${LOG_DIR}" \ - --output_path "${metrics_json}" \ - --require-tag-prefix "generation_metrics/" -uv run --no-sync tests/check_metrics.py \ - "${metrics_json}" \ - 'max(data["train/token_mult_prob_error"]) < 1.05' - -if pgrep -f "[d]ynamo.frontend|[d]ynamo.vllm|[V]LLM::EngineCore|${dynamo_venv_dir}/bin/[e]tcd|${dynamo_venv_dir}/bin/[n]ats-server"; then - echo "Managed Dynamo processes remain after GRPO shutdown" >&2 - exit 1 -fi diff --git a/tests/functional/grpo_dynamo_single_controller.sh b/tests/functional/grpo_dynamo_single_controller.sh deleted file mode 100644 index 9e41e463e50..00000000000 --- a/tests/functional/grpo_dynamo_single_controller.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Two-GPU SingleController + TransferQueue run with managed Dynamo generation. - -set -euo pipefail - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) -PROJECT_ROOT=$(realpath "${SCRIPT_DIR}/../..") -EXP_NAME=$(basename "$0" .sh) -EXP_DIR=${SCRIPT_DIR}/${EXP_NAME} -LOG_DIR=${EXP_DIR}/logs -RUN_LOG=${EXP_DIR}/run.log -export PYTHONPATH=${PROJECT_ROOT}:${PYTHONPATH:-} - -rm -rf "${EXP_DIR}" -mkdir -p "${LOG_DIR}" -git config --global --add safe.directory "${PROJECT_ROOT}" - -dynamo_venv_dir=${NEMO_RL_DYNAMO_VENV_DIR:-/opt/dynamo_venv} -"${dynamo_venv_dir}/bin/python" -c \ - 'import importlib.metadata as m; assert m.version("ai-dynamo") == "1.5.1"; assert m.version("vllm") == "0.28.0"' - -cd "${PROJECT_ROOT}" -uv run --no-sync coverage run -a \ - --data-file="${PROJECT_ROOT}/tests/.coverage" \ - --source="${PROJECT_ROOT}/nemo_rl" \ - "${PROJECT_ROOT}/examples/run_grpo_single_controller.py" \ - --config "${PROJECT_ROOT}/examples/configs/grpo_math_1B_dynamo_single_controller.yaml" \ - policy.model_name=Qwen/Qwen3-0.6B \ - policy.tokenizer.name=Qwen/Qwen3-0.6B \ - grpo.num_prompts_per_step=2 \ - grpo.num_generations_per_prompt=4 \ - grpo.seq_logprob_error_threshold=1000 \ - grpo.max_num_steps=3 \ - policy.train_global_batch_size=8 \ - policy.train_micro_batch_size=1 \ - policy.logprob_batch_size=1 \ - policy.generation.max_new_tokens=128 \ - async_rl.sampler.name=in_order \ - async_rl.sampler.max_lookahead_versions=1 \ - async_rl.min_groups_for_streaming_train=2 \ - async_rl.max_inflight_prompts=2 \ - async_rl.max_buffered_rollouts=4 \ - data_plane.backend=simple \ - logger.log_dir="${LOG_DIR}" \ - logger.tensorboard_enabled=true \ - logger.wandb_enabled=false \ - logger.monitor_gpus=false \ - checkpointing.enabled=false \ - "$@" \ - 2>&1 | tee "${RUN_LOG}" - -# Every weight sync drains Dynamo first (pause) and restores it afterwards (resume). -# Dynamo accepts allow_unpaused updates, so a lost drain would not fail the refit. -sync_count=$(grep -Fc "_sync_weights: sync done" "${RUN_LOG}" || true) -pause_count=$(grep -Fc "[RL] Engine paused (mode=wait, clear_cache=True)" "${RUN_LOG}" || true) -resume_count=$(grep -Fc "[RL] Engine resumed" "${RUN_LOG}" || true) -if [[ "${sync_count}" -lt 2 || "${pause_count}" -lt "${sync_count}" || "${resume_count}" -lt "${sync_count}" ]]; then - echo "Expected every weight sync to pause and resume Dynamo; syncs=${sync_count}, pauses=${pause_count}, resumes=${resume_count}" >&2 - exit 1 -fi - -metrics_json=${EXP_DIR}/metrics.json -uv run --no-sync tests/json_dump_tb_logs.py "${LOG_DIR}" --output_path "${metrics_json}" -# The worker starts from dummy weights, so these hold only if every refit landed. -uv run --no-sync tests/check_metrics.py "${metrics_json}" \ - 'max(data["train/token_mult_prob_error"]) < 1.05' \ - 'max(data["train/gen_kl_error"]) < 0.002' - -if pgrep -f "[d]ynamo.frontend|[d]ynamo.vllm|[V]LLM::EngineCore|${dynamo_venv_dir}/bin/[e]tcd|${dynamo_venv_dir}/bin/[n]ats-server"; then - echo "Managed Dynamo processes remain after SingleController shutdown" >&2 - exit 1 -fi diff --git a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh index 2d265e6153b..ee3da965de3 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh @@ -157,3 +157,7 @@ uv run --no-sync tests/check_metrics.py "${JSON_METRICS}" \ 'min(data["train/finalize/media_row_rate"]) == 1' \ 'max(data["train/gen_kl_error"]) < 0.05' \ 'all_finite(data["train/reward"])' + +if [[ "$*" == *generation.dynamo=true* ]]; then + bash "${SCRIPT_DIR}/check_dynamo_overlay.sh" "${RUN_LOG}" +fi diff --git a/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh b/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh index 1dc7edd194a..e9cf7073a7c 100755 --- a/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh +++ b/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh @@ -59,8 +59,8 @@ refit_count=$(grep -c "✅ Ready for refit" "${RUN_LOG}" || true) cache_invalidation_count=$(grep -c \ "✅ Invalidated generation backend KV caches after weight update" \ "${RUN_LOG}" || true) -cache_invalidation_failure_count=$(grep -cE \ - "Failed to invalidate generation backend KV caches|Dynamo KV cache invalidation failed" \ +cache_invalidation_failure_count=$(grep -c \ + "Failed to invalidate generation backend KV caches" \ "${RUN_LOG}" || true) if [[ ${cache_invalidation_failure_count} -ne 0 ]]; then echo "[ERROR] Found ${cache_invalidation_failure_count} cache invalidation failure(s)" diff --git a/tests/unit/algorithms/test_async_utils.py b/tests/unit/algorithms/test_async_utils.py index 4753a784c80..67d3b4334f3 100644 --- a/tests/unit/algorithms/test_async_utils.py +++ b/tests/unit/algorithms/test_async_utils.py @@ -2811,34 +2811,6 @@ def test_resume_after_refit_skips_cache_invalidation_when_recompute_disabled(sel collector.policy_generation.invalidate_kv_cache.assert_not_called() - def test_dynamo_cache_invalidation_failure_is_fatal_and_unblocks_waiters(self): - collector = self.create_local_collector() - collector.master_config.policy["generation"] = {"backend": "dynamo"} - collector.master_config.grpo.async_grpo.recompute_kv_cache_after_weight_updates = True - collector.policy_generation.invalidate_kv_cache = mock.Mock( - side_effect=RuntimeError("pause failed") - ) - collector._refit_pause_cleared.clear() - - with pytest.raises(RuntimeError, match="cache invalidation failed"): - collector.resume_after_refit() - - assert collector._refit_pause_cleared.is_set() - - def test_dynamo_prepare_for_refit_drains_pending_generations(self): - """Dynamo layerwise reload never overlaps an active generation.""" - collector = self.create_local_collector() - collector.master_config.policy["generation"] = { - "backend": "dynamo", - "dynamo_cfg": {}, - } - collector.master_config.grpo.async_grpo.in_flight_weight_updates = True - collector.wait_for_pending_generations = mock.MagicMock() - - collector.prepare_for_refit() - - collector.wait_for_pending_generations.assert_called_once_with() - def test_calculate_target_weights(self): """Test target weight calculation logic.""" buffer = ReplayBuffer.remote( diff --git a/tests/unit/algorithms/test_distillation.py b/tests/unit/algorithms/test_distillation.py index e112f379de6..d82c2044179 100644 --- a/tests/unit/algorithms/test_distillation.py +++ b/tests/unit/algorithms/test_distillation.py @@ -1332,7 +1332,7 @@ class DummyVllmGeneration: def __init__(self, cluster, config, defer_model_load=False): self.cfg = config self.defer_model_load = defer_model_load - self.dp_openai_server_base_urls = ["http://reserved-vllm"] + self.client_base_urls = ["http://reserved-vllm"] # The per-job bearer token the real class generates when the # recipe requires one; Gym's model server must receive it. self.http_server_api_key = "k" diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index 90fd42415ec..7950eefff3e 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -102,7 +102,6 @@ ) from nemo_rl.experience.rollouts import calculate_rewards from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo import DynamoConfig from nemo_rl.models.generation.interfaces import should_use_async_rollouts from nemo_rl.models.generation.megatron import MegatronGeneration from nemo_rl.models.policy.draft_config import Eagle3DraftConfig @@ -2096,7 +2095,6 @@ def test_async_resume_plumbs_frontier_metadata_into_collector( @pytest.mark.parametrize( ("generation_config", "expected"), [ - ({"backend": "dynamo"}, True), ({"backend": "vllm", "vllm_cfg": {"async_engine": False}}, False), ({"backend": "vllm", "vllm_cfg": {"async_engine": True}}, True), ( @@ -2139,13 +2137,11 @@ def test_should_use_nemo_gym_accepts_megatron_always_async(): assert should_use_nemo_gym(master_config) -@pytest.mark.parametrize("backend", ["dynamo", "vllm"]) def test_initial_refit_completes_before_async_collection_starts( mock_grpo_components, - backend, ) -> None: master_config = mock_grpo_components["master_config"] - master_config.policy["generation"]["backend"] = backend + master_config.policy["generation"]["backend"] = "vllm" master_config.policy["generation"]["colocated"]["enabled"] = False master_config.grpo.max_num_steps = 1 master_config.grpo.val_period = 0 @@ -2184,7 +2180,7 @@ def record_refit(*args, **kwargs): def test_async_grpo_awaits_resume_after_refit_failure(mock_grpo_components) -> None: master_config = mock_grpo_components["master_config"] - master_config.policy["generation"]["backend"] = "dynamo" + master_config.policy["generation"]["backend"] = "vllm" master_config.policy["generation"]["colocated"]["enabled"] = False master_config.grpo.max_num_steps = 1 master_config.grpo.val_period = 0 @@ -2216,23 +2212,6 @@ def test_async_grpo_awaits_resume_after_refit_failure(mock_grpo_components) -> N ) -def test_should_use_nemo_gym_requires_dynamo_token_wrapper() -> None: - master_config = MagicMock() - master_config.env = {"should_use_nemo_gym": True} - master_config.policy = { - "generation": { - "backend": "dynamo", - "vllm_cfg": {"expose_http_server": False}, - } - } - - with pytest.raises(AssertionError, match="expose_http_server: true"): - should_use_nemo_gym(master_config) - - master_config.policy["generation"]["vllm_cfg"]["expose_http_server"] = True - assert should_use_nemo_gym(master_config) is True - - @pytest.mark.parametrize( ( "lookahead_complete", @@ -3316,247 +3295,6 @@ def test_noncolocated_inference_requires_explicit_gpus_per_node_single_node( setup(master_config, tokenizer, dataset, None) -def test_dynamo_rejects_colocated_inference_before_setup_side_effects( - mock_grpo_components, -): - from nemo_rl.algorithms.grpo import setup - - master_config = mock_grpo_components["master_config"] - master_config.policy["hf_config_overrides"] = {"rope_theta": 1_000_000.0} - master_config.policy["generation"] = { - "backend": "dynamo", - "dynamo_cfg": { - "engine": "vllm", - "startup_timeout_s": 60, - "request_timeout_s": 60, - "control_timeout_s": 30, - "metrics_include_prefixes": None, - "metrics_exclude_prefixes": None, - "worker_args": { - "tool_call_parser": None, - "reasoning_parser": None, - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": False, - "structural_tag_scope": "auto", - "structural_tag_schema": "auto", - "custom_jinja_template": None, - "endpoint_types": ["chat", "completions"], - "extra_cli_args": [], - }, - "frontend_args": { - "tokenizer": "default", - "tokenizer_cache": False, - "tokenizer_cache_bytes": 1024, - "router_mode": "kv", - "extra_cli_args": [], - }, - }, - "vllm_cfg": { - "async_engine": True, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "expert_parallel_size": 1, - "gpu_memory_utilization": 0.6, - "max_model_len": 512, - "kv_cache_dtype": "auto", - "load_format": "auto", - "precision": "bfloat16", - "enforce_eager": True, - "expose_http_server": False, - "enable_vllm_metrics_logger": True, - "vllm_metrics_logger_interval": 1.0, - "env_vars": {}, - }, - "vllm_kwargs": {}, - "colocated": { - "enabled": True, - "resources": {"gpus_per_node": None, "num_nodes": None}, - }, - } - - master_config.grpo.async_grpo.in_flight_weight_updates = True - with ( - patch("nemo_rl.algorithms.grpo.Logger") as mock_logger, - pytest.raises(ValueError, match="in_flight_weight_updates must be false"), - ): - setup( - master_config, - tokenizer=MagicMock(), - dataset=MagicMock(), - val_dataset=None, - ) - mock_logger.assert_not_called() - - master_config.grpo.async_grpo.in_flight_weight_updates = False - - with ( - patch("nemo_rl.algorithms.grpo.Logger") as mock_logger, - pytest.raises( - ValueError, - match="must be false", - ), - ): - setup( - master_config, - tokenizer=MagicMock(), - dataset=MagicMock(), - val_dataset=None, - ) - - mock_logger.assert_not_called() - assert ( - master_config.policy["generation"]["vllm_kwargs"]["hf_overrides"] - == (master_config.policy["hf_config_overrides"]) - ) - - del master_config.policy["generation"]["vllm_kwargs"] - del master_config.policy["hf_config_overrides"] - with pytest.raises(ValueError, match="must be false"): - setup( - master_config, - tokenizer=MagicMock(), - dataset=MagicMock(), - val_dataset=None, - ) - assert master_config.policy["generation"]["vllm_kwargs"]["hf_overrides"] == {} - - -def test_setup_initializes_noncolocated_dynamo_with_nemo_gym(monkeypatch) -> None: - from nemo_rl.algorithms import grpo as grpo_mod - - repo_root = Path(__file__).resolve().parents[3] - register_omegaconf_resolvers() - config = OmegaConf.to_container( - load_config(repo_root / "examples/configs/grpo_math_1B_dynamo.yaml"), - resolve=True, - ) - config["cluster"].update({"num_nodes": 2, "gpus_per_node": 4, "segment_size": 1}) - generation = config["policy"]["generation"] - generation["vllm_cfg"].update( - { - "tensor_parallel_size": 2, - "pipeline_parallel_size": 2, - "expert_parallel_size": 2, - "expose_http_server": True, - } - ) - generation["colocated"]["resources"] = { - "gpus_per_node": 4, - "num_nodes": 1, - } - config["env"]["should_use_nemo_gym"] = True - tokenizer = MagicMock() - tokenizer.pad_token_id = 0 - tokenizer.eos_token_id = 1 - config["policy"]["generation"] = configure_generation_config(generation, tokenizer) - master_config = MasterConfig.model_validate(config) - - cluster_instances = [] - - class DummyCluster: - def __init__(self, **kwargs): - self.kwargs = kwargs - self.num_gpus_per_node = kwargs["num_gpus_per_node"] - self.get_placement_groups = MagicMock(return_value=[object()]) - cluster_instances.append(self) - - class DummyLoader: - def __init__(self, *_args, **_kwargs): - pass - - def __len__(self): - return 1 - - class DummyCheckpointer: - def get_latest_checkpoint_path(self): - return None - - def load_training_info(self, _path): - return None - - def get_resume_paths(self, _path): - return None, None - - class DummyPolicy: - def print_node_ip_and_gpu_id(self): - pass - - dynamo_init = MagicMock() - - class DummyDynamoGeneration: - weight_synchronizer = None - dp_openai_server_base_urls = ["http://dynamo-wrapper.example/v1"] - frontend_url = "http://dynamo-frontend.example/v1" - - def __init__(self, *, cluster, config, tokenizer, tokenizer_config): - dynamo_init( - cluster=cluster, - config=config, - tokenizer=tokenizer, - tokenizer_config=tokenizer_config, - ) - - synchronizer = MagicMock() - nemo_gym_shard_set = MagicMock() - build_nemo_gym_actors = MagicMock(return_value=nemo_gym_shard_set) - monkeypatch.setattr(grpo_mod, "Logger", lambda *_args, **_kwargs: MagicMock()) - monkeypatch.setattr( - grpo_mod, "CheckpointManager", lambda *_args, **_kwargs: DummyCheckpointer() - ) - monkeypatch.setattr( - grpo_mod, "ClippedPGLossFn", lambda *_args, **_kwargs: MagicMock() - ) - monkeypatch.setattr(grpo_mod, "StatefulDataLoader", DummyLoader) - monkeypatch.setattr( - grpo_mod, - "get_ray_cluster_topology", - lambda: { - "train-node": ("nvlink_domain_train", 0), - "inference-node": ("nvlink_domain_inference", 1), - }, - ) - monkeypatch.setattr(grpo_mod, "RayVirtualCluster", DummyCluster) - monkeypatch.setattr(grpo_mod, "Policy", lambda *_args, **_kwargs: DummyPolicy()) - monkeypatch.setattr(grpo_mod, "DynamoGeneration", DummyDynamoGeneration) - monkeypatch.setattr( - grpo_mod, "create_weight_synchronizer", lambda **_kwargs: synchronizer - ) - monkeypatch.setattr(grpo_mod, "build_nemo_gym_actors", build_nemo_gym_actors) - # The agent-coverage check asks an unsharded job's actor for its entries - # over Ray, which a mock actor cannot answer; the check has its own tests. - monkeypatch.setattr( - grpo_mod, "validate_dataset_agent_coverage", lambda *_args, **_kwargs: None - ) - - dataset = MagicMock() - dataset.__len__.return_value = 2 - result = setup(master_config, tokenizer, dataset, None) - - train_cluster, inference_cluster = cluster_instances - assert train_cluster.kwargs["bundle_ct_per_node_list"] == [4] - assert inference_cluster.kwargs["bundle_ct_per_node_list"] == [4] - assert train_cluster.kwargs["node_resource_constraints"] == [ - {"nvlink_domain_train": 0.001} - ] - assert inference_cluster.kwargs["node_resource_constraints"] is None - assert result[1].dp_openai_server_base_urls == ["http://dynamo-wrapper.example/v1"] - assert result[2] is nemo_gym_shard_set - dynamo_config = dynamo_init.call_args.kwargs["config"] - assert dynamo_init.call_args.kwargs["cluster"] is inference_cluster - assert DynamoConfig.model_validate(dynamo_config).engine_world_size == 4 - synchronizer.init_communicator.assert_called_once_with() - build_nemo_gym_actors.assert_called_once_with( - master_config.env, - base_urls=["http://dynamo-wrapper.example/v1"], - model_name=master_config.policy["model_name"], - tokenizer=tokenizer, - enable_router_replay=False, - use_fastokens=False, - # Only the vLLM backend generates a bearer token for its HTTP server. - policy_api_key=None, - ) - - def test_setup_hands_the_vllm_http_server_api_key_to_nemo_gym(monkeypatch) -> None: """The deferred-vLLM branch is the one path that forwards a real key: the workers enforce it, so Gym's model server must send the same one.""" @@ -3632,7 +3370,7 @@ def offload_after_refit(self): class DummyVllmGeneration: weight_synchronizer = None - dp_openai_server_base_urls = ["http://vllm-worker.example/v1"] + client_base_urls = ["http://vllm-worker.example/v1"] # What the real class generates when the recipe requires a key. http_server_api_key = "per-job-key" @@ -7154,22 +6892,20 @@ def test_needs_hf_refit_handshake(backend, nccl_reshard, colocated, expected): @pytest.mark.parametrize( - ("backend", "nccl_reshard", "offload_policy", "expected"), + ("nccl_reshard", "offload_policy", "expected"), [ - ("vllm", False, False, False), - ("vllm", False, True, True), - ("vllm", True, False, True), - ("dynamo", False, False, True), + (False, False, False), + (False, True, True), + (True, False, True), ], ) def test_noncolocated_refit_synchronizer_selection( - backend, nccl_reshard, offload_policy, expected + nccl_reshard, offload_policy, expected ): from nemo_rl.algorithms import grpo as grpo_mod assert ( grpo_mod._uses_managed_noncolocated_refit( - generation_backend=backend, nccl_reshard_refit_enabled=nccl_reshard, offload_policy_before_refit=offload_policy, ) diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index e5daf577f89..3469ac88884 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -39,7 +39,11 @@ def _digest(label: str) -> str: @pytest.mark.parametrize( ("generation_backend", "expected"), - [("vllm", "vllm_worker"), ("megatron", "megatron_worker")], + [ + ("vllm", "vllm_worker"), + ("megatron", "megatron_worker"), + ("dynamo", "dynamo_worker"), + ], ) def test_external_staging_backend_maps_generation_backend( generation_backend: str, expected: str @@ -572,8 +576,11 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> assert should_use_nemo_gym(config) is True -def test_setup_nemo_gym_config_dynamo_exposes_http_server_via_vllm_cfg() -> None: - """Dynamo serves Gym through the vllm_cfg HTTP server settings.""" +@pytest.mark.parametrize("dynamo", [False, True]) +def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( + dynamo: bool, +) -> None: + """With Dynamo, Gym's policy model calls the frontend in Gym's Dynamo dialect.""" from nemo_rl.algorithms.grpo import MasterConfig from nemo_rl.environments.nemo_gym import ( setup_nemo_gym_config, @@ -584,7 +591,8 @@ def test_setup_nemo_gym_config_dynamo_exposes_http_server_via_vllm_cfg() -> None env={"should_use_nemo_gym": True}, policy={ "generation": { - "backend": "dynamo", + "backend": "vllm", + "dynamo": dynamo, "vllm_cfg": {"async_engine": False, "expose_http_server": False}, } }, @@ -596,3 +604,66 @@ def test_setup_nemo_gym_config_dynamo_exposes_http_server_via_vllm_cfg() -> None assert vllm_cfg["async_engine"] is True assert vllm_cfg["expose_http_server"] is True assert should_use_nemo_gym(config) is True + policy_model = config.env["nemo_gym"].get("policy_model") if dynamo else None + if dynamo: + assert policy_model["responses_api_models"]["vllm_model"] == { + "dynamo_frontend": True, + # The frontend refuses reasoning under both of Gym's keys. + "reasoning_field": "reasoning_content", + } + else: + assert "nemo_gym" not in config.env + + +@pytest.mark.parametrize( + ("vllm_model", "expected"), + [ + ({}, {"enable_thinking": True, "truncate_history_thinking": False}), + ( + {"chat_template_kwargs": {"enable_thinking": False}}, + {"enable_thinking": False, "truncate_history_thinking": False}, + ), + ( + {"extra_body": {"chat_template_kwargs": {"force_nonempty_content": True}}}, + { + "enable_thinking": True, + "truncate_history_thinking": False, + "force_nonempty_content": True, + }, + ), + ], +) +def test_setup_nemo_gym_config_dynamo_sends_the_native_default_template_kwargs( + vllm_model: dict, expected: dict +) -> None: + """The native server's default template kwargs sit under the ones Gym sends.""" + from nemo_rl.algorithms.grpo import MasterConfig + from nemo_rl.environments.nemo_gym import setup_nemo_gym_config + + config = MasterConfig.model_construct( + env={ + "should_use_nemo_gym": True, + "nemo_gym": { + "policy_model": {"responses_api_models": {"vllm_model": vllm_model}} + }, + }, + policy={ + "generation": { + "backend": "vllm", + "dynamo": True, + "vllm_cfg": { + "async_engine": True, + "http_server_serving_chat_kwargs": { + "default_chat_template_kwargs": { + "enable_thinking": True, + "truncate_history_thinking": False, + } + }, + }, + } + }, + ) + + setup_nemo_gym_config(config, tokenizer=None) + + assert vllm_model["chat_template_kwargs"] == expected diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index d47b492fae1..ea0b267beef 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -43,7 +43,7 @@ ) -@pytest.mark.parametrize("backend", ["vllm", "dynamo", "megatron", "trtllm"]) +@pytest.mark.parametrize("backend", ["vllm", "megatron", "trtllm"]) def test_setup_nemo_gym_config_enables_http_server(backend: str) -> None: generation = { "backend": backend, @@ -57,10 +57,9 @@ def test_setup_nemo_gym_config_enables_http_server(backend: str) -> None: setup_nemo_gym_config(config, tokenizer=None) - is_vllm_like = backend in ("vllm", "dynamo") assert generation["vllm_cfg"] == { - "async_engine": is_vllm_like, - "expose_http_server": is_vllm_like, + "async_engine": backend == "vllm", + "expose_http_server": backend == "vllm", } assert generation["mcore_generation_config"] == { "expose_http_server": backend == "megatron" diff --git a/tests/unit/experience/test_rollouts.py b/tests/unit/experience/test_rollouts.py index c87795b5df7..2858b40bba3 100644 --- a/tests/unit/experience/test_rollouts.py +++ b/tests/unit/experience/test_rollouts.py @@ -987,11 +987,6 @@ def fake_rewards(batch, task_to_env): assert [call["dedup"] for call in calls] == [deduplicate_multimodal_data] * 2 -class _DummyDynamoGeneration(_DummySGLangGeneration): - def __init__(self): - self.cfg = {"backend": "dynamo"} - - def test_generate_responses_async_requires_sglang_opt_in(): generation_input_data = BatchedDataDict( { @@ -1037,30 +1032,6 @@ def test_generate_responses_async_allows_sglang_opt_in(): assert gen_metrics["total_generated_tokens"] == 1 -def test_generate_responses_async_allows_dynamo(): - generation_input_data = BatchedDataDict( - { - "input_ids": torch.tensor([[1]]), - "input_lengths": torch.tensor([1], dtype=torch.long), - } - ) - batch = BatchedDataDict({"message_log": [[]]}) - - updated_batch, generated_ids, gen_metrics = asyncio.run( - generate_responses_async( - _DummyDynamoGeneration(), - generation_input_data, - batch, - _DummyTokenizer(), - input_lengths=generation_input_data["input_lengths"], - ) - ) - - assert updated_batch["message_log"][0][-1]["content"] == "ok" - assert generated_ids[0].tolist() == [2] - assert gen_metrics["total_generated_tokens"] == 1 - - @pytest.fixture(scope="function") def rollout_tokenizer(): """Loads the tokenizer for the tests.""" diff --git a/tests/unit/models/generation/test_dynamo_arguments.py b/tests/unit/models/generation/test_dynamo_arguments.py deleted file mode 100644 index d5e992dcdc2..00000000000 --- a/tests/unit/models/generation/test_dynamo_arguments.py +++ /dev/null @@ -1,424 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import warnings - -import pytest -from pydantic import ValidationError - -from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.arguments import ( - build_dynamo_frontend_argv, - build_dynamo_vllm_argv, - build_managed_worker_env, - redact_argv, - redact_environment, -) -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - DynamoConfig, - DynamoWorkerArgs, -) -from nemo_rl.models.generation.interfaces import DisaggConfig - - -def _config(**overrides) -> dict: - config = { - "backend": "dynamo", - "model_name": "Qwen/Qwen3-0.6B", - "dynamo_cfg": _dynamo_cfg(), - "vllm_cfg": { - "async_engine": True, - "tensor_parallel_size": 2, - "pipeline_parallel_size": 1, - "expert_parallel_size": 2, - "gpu_memory_utilization": 0.8, - "max_model_len": 512, - "precision": "bfloat16", - "kv_cache_dtype": "auto", - "load_format": "auto", - "enforce_eager": False, - "expose_http_server": False, - "enable_vllm_metrics_logger": True, - "vllm_metrics_logger_interval": 1.0, - "env_vars": None, - }, - "vllm_kwargs": {}, - "colocated": {"enabled": False}, - } - config.update(overrides) - return config - - -def _dynamo_cfg() -> dict: - return { - "engine": "vllm", - "startup_timeout_s": 600, - "request_timeout_s": 900, - "control_timeout_s": 600, - "metrics_include_prefixes": None, - "metrics_exclude_prefixes": None, - "worker_args": { - "tool_call_parser": None, - "reasoning_parser": None, - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": False, - "structural_tag_scope": "auto", - "structural_tag_schema": "auto", - "custom_jinja_template": None, - "endpoint_types": ["chat", "completions"], - "extra_cli_args": [], - }, - "frontend_args": { - "tokenizer": "default", - "tokenizer_cache": False, - "tokenizer_cache_bytes": 50 * 1024 * 1024, - "router_mode": "kv", - "extra_cli_args": [], - }, - } - - -def _flag_value(argv: list[str], flag: str) -> str: - return argv[argv.index(flag) + 1] - - -def test_config_derives_world_size_and_rejects_removed_public_fields() -> None: - assert DynamoConfig.model_validate(_config()).engine_world_size == 2 - for field in ("engine_world_size", "namespace", "dynamo_python", "etcd_port"): - with pytest.raises(ValidationError, match=field): - DynamoConfig.model_validate( - _config(dynamo_cfg={field: 1 if field.endswith("port") else "x"}) - ) - - -@pytest.mark.parametrize( - ("override", "match"), - [ - ({"vllm_cfg": {}}, "nonempty"), - ({"sglang_cfg": {"foo": 1}}, "sglang_cfg"), - ({"trtllm_cfg": {"foo": 1}}, "trtllm_cfg"), - ({"colocated": {"enabled": True}}, "must be false"), - ({"refit_transport": "nccl_reshard"}, "must be null"), - ({"quant_cfg": "nvfp4"}, "quant_cfg"), - ({"vllm_kwargs": {"speculative_config": {"model": "draft"}}}, "draft"), - ], -) -def test_config_rejects_unsupported_modes(override, match) -> None: - if override.get("vllm_cfg") == {}: - config = _config() - config["vllm_cfg"] = {} - else: - config = _config(**override) - with pytest.raises(ValidationError, match=match): - DynamoConfig.model_validate(config) - - -@pytest.mark.parametrize( - ("vllm_cfg", "match"), - [ - ( - {"tensor_parallel_size": 2, "expert_parallel_size": 3}, - "expert_parallel_size", - ), - ({"precision": "fp8"}, "precision"), - ({"kv_cache_dtype": "fp8"}, "kv_cache_dtype"), - ], -) -def test_config_rejects_unsupported_parallelism_and_precision(vllm_cfg, match) -> None: - config = _config() - config["vllm_cfg"].update(vllm_cfg) - with pytest.raises(ValidationError, match=match): - DynamoConfig.model_validate(config) - - -def test_config_classifies_managed_logprobs_and_runtime_fields() -> None: - config = _config() - config["vllm_cfg"].update( - { - "logprobs_mode": "processed_logprobs", - "cap_max_tokens_to_context": False, - "use_deep_gemm": False, - "num_first_layers_in_bf16": 0, - "num_last_layers_in_bf16": 0, - } - ) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - DynamoConfig.model_validate(config) - assert caught == [] - - config["vllm_cfg"]["logprobs_mode"] = "raw_logprobs" - with pytest.raises(ValidationError, match="processed_logprobs"): - DynamoConfig.model_validate(config) - - -@pytest.mark.parametrize("field", ["skip_tokenizer_init", "cap_max_tokens_to_context"]) -def test_config_warns_only_for_active_unsupported_fields(field) -> None: - config = _config() - config["vllm_cfg"][field] = True - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - DynamoConfig.model_validate(config) - assert [str(warning.message) for warning in caught] == [ - f"policy.generation.vllm_cfg.{field} is ignored by backend='dynamo'" - ] - - -@pytest.mark.parametrize( - ("override", "ignored"), - [ - ({"bad_words": ["foo"]}, ["bad_words"]), - ({"ignore_eos": True}, ["ignore_eos"]), - # The validator receives disaggregation as a DisaggConfig, not a dict. - ({"disaggregation": DisaggConfig(enabled=True)}, ["disaggregation"]), - # Inherited shared-YAML settings the exemplars carry stay quiet. - ({"disaggregation": DisaggConfig(enabled=False)}, []), - ({"ignore_eos": False}, []), - ({"refit_cfg": {"nixl": {}}, "mcore_generation_config": {"x": 1}}, []), - ({"real_quant_export_cpu_offload": True}, []), - ], -) -def test_config_warns_for_ignored_generation_keys(override, ignored) -> None: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - DynamoConfig.model_validate(_config(**override)) - expected = ( - [f"policy.generation keys ignored by backend='dynamo': {ignored}"] - if ignored - else [] - ) - assert [str(warning.message) for warning in caught] == expected - - -def test_config_rejects_a_required_http_server_api_key() -> None: - """The managed Dynamo frontend checks no bearer token, so asking for one - fails loudly instead of leaving the exposed server open; the exemplar's - explicit ``false`` is accepted without an "ignored" warning.""" - config = _config() - config["vllm_cfg"]["http_server_api_key_required"] = True - with pytest.raises(ValidationError, match="http_server_api_key_required"): - DynamoConfig.model_validate(config) - - config["vllm_cfg"]["http_server_api_key_required"] = False - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - DynamoConfig.model_validate(config) - assert caught == [] - - -@pytest.mark.parametrize( - "field", - [ - "data_parallel_size", - "prefill_context_parallel_size", - "decode_context_parallel_size", - ], -) -@pytest.mark.parametrize("source", ["vllm_cfg", "vllm_kwargs"]) -def test_config_rejects_parallel_dimensions_outside_tp_pp(field, source) -> None: - config = _config() - config[source][field] = 2 - - with pytest.raises(ValidationError, match=f"{source}.{field} must be 1"): - DynamoConfig.model_validate(config) - - -def test_config_rejects_more_than_32_stop_strings() -> None: - config = _config(stop_strings=[str(index) for index in range(33)]) - - with pytest.raises(ValidationError, match="stop_strings supports at most 32"): - DynamoConfig.model_validate(config) - - -def test_config_does_not_limit_stop_token_ids() -> None: - config = _config(stop_token_ids=list(range(33))) - - assert DynamoConfig.model_validate(config).model_extra["stop_token_ids"] == list( - range(33) - ) - - -def test_configure_generation_config_selects_dynamo_load_format() -> None: - class Tokenizer: - pad_token_id = 0 - eos_token_id = 1 - - training = _config(stop_token_ids=None) - evaluation = _config(stop_token_ids=None) - del training["vllm_cfg"]["load_format"] - del evaluation["vllm_cfg"]["load_format"] - - assert ( - configure_generation_config(training, Tokenizer())["vllm_cfg"]["load_format"] - == "dummy" - ) - assert ( - configure_generation_config(evaluation, Tokenizer(), is_eval=True)["vllm_cfg"][ - "load_format" - ] - == "auto" - ) - - -def test_worker_argv_translates_structured_fields_and_warns_unclassified() -> None: - config = _dynamo_cfg() - config["worker_args"].update( - {"tool_call_parser": "qwen3_coder", "reasoning_parser": "nemotron_nano"} - ) - cfg = DynamoCfg.model_validate(config) - generation_config = _config() - generation_config["vllm_cfg"]["unclassified_field"] = 1 - with pytest.warns(UserWarning, match="unclassified_field"): - validated = DynamoConfig.model_validate(generation_config) - vllm_cfg = validated.vllm_cfg.model_dump() - argv = build_dynamo_vllm_argv( - model_name="model", - namespace="nemo-rl-1", - seed=7, - vllm_cfg=vllm_cfg, - vllm_kwargs={"max_num_seqs": 16, "hf_overrides": {"rope_theta": 1e6}}, - dynamo_cfg=cfg, - ) - - assert _flag_value(argv, "--model") == "model" - assert _flag_value(argv, "--weight-transfer-config") == '{"backend":"nccl"}' - assert _flag_value(argv, "--dyn-tool-call-parser") == "qwen3_coder" - assert _flag_value(argv, "--dyn-reasoning-parser") == "nemotron_nano" - assert _flag_value(argv, "--max-num-seqs") == "16" - assert _flag_value(argv, "--hf-overrides") == '{"rope_theta":1000000.0}' - assert "--enable-expert-parallel" in argv - - -def test_worker_argv_rejects_replaced_and_managed_options() -> None: - generation_config = _config() - generation_config["vllm_cfg"]["http_server_serving_chat_kwargs"] = { - "tool_parser": "x" - } - with pytest.raises(ValueError, match="worker_args.custom_jinja_template"): - DynamoConfig.model_validate(generation_config) - config = _dynamo_cfg() - config["worker_args"]["extra_cli_args"] = ["--model", "other"] - with pytest.raises(ValueError, match="--model is set by both"): - build_dynamo_vllm_argv( - model_name="model", - namespace="namespace", - seed=0, - vllm_cfg=_config()["vllm_cfg"], - vllm_kwargs={}, - dynamo_cfg=DynamoCfg.model_validate(config), - ) - - -def test_config_accepts_inherited_unused_sections() -> None: - config = _config( - mcore_generation_config={"some_shared_setting": True}, - refit_cfg={"some_shared_setting": True}, - ) - - validated = DynamoConfig.model_validate(config) - - assert validated.model_extra["mcore_generation_config"] == { - "some_shared_setting": True - } - assert validated.model_extra["refit_cfg"] == {"some_shared_setting": True} - - -def test_frontend_argv_and_environment_are_runtime_owned() -> None: - cfg = DynamoCfg.model_validate(_dynamo_cfg()) - argv = build_dynamo_frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg - ) - assert _flag_value(argv, "--router-mode") == "kv" - assert not any(arg.startswith("--router-reset-states") for arg in argv) - assert not any(arg.startswith("--no-router-reset-states") for arg in argv) - - env = build_managed_worker_env( - base_env={ - "DYN_NAMESPACE": "stale", - "ETCD_ENDPOINTS": "http://stale-etcd:2379", - "ETCD_USERNAME": "stale-user", - "NATS_SERVER": "nats://stale-nats:4222", - "NATS_AUTH_TOKEN": "stale-token", - "NCCL_DEBUG": "INFO", - }, - configured_env={"NCCL_IB_DISABLE": "0"}, - manager_env={ - "DYN_NAMESPACE": "owned", - "DYN_SYSTEM_PORT": "4000", - "ETCD_ENDPOINTS": "http://managed-etcd:2379", - "NATS_SERVER": "nats://managed-nats:4222", - }, - ) - assert env["DYN_NAMESPACE"] == "owned" - assert env["DYN_SYSTEM_PORT"] == "4000" - assert env["ETCD_ENDPOINTS"] == "http://managed-etcd:2379" - assert env["NATS_SERVER"] == "nats://managed-nats:4222" - assert "ETCD_USERNAME" not in env - assert "NATS_AUTH_TOKEN" not in env - with pytest.raises(ValueError, match="VLLM_PORT"): - build_managed_worker_env( - base_env={}, - configured_env={"VLLM_PORT": "9999"}, - manager_env={"VLLM_PORT": "7000"}, - ) - - -def test_every_worker_config_field_reaches_argv(monkeypatch) -> None: - from nemo_rl.models.generation.dynamo import arguments - - sources: set[str] = set() - original_add = arguments._ArgvBuilder.add - - def record_source(self, flag, value=None, *, source): - sources.add(source) - return original_add(self, flag, value, source=source) - - monkeypatch.setattr(arguments._ArgvBuilder, "add", record_source) - config = _dynamo_cfg() - config["worker_args"].update( - { - "tool_call_parser": "qwen3_coder", - "reasoning_parser": "nemotron_nano", - "custom_jinja_template": "template", - } - ) - cfg = DynamoCfg.model_validate(config) - build_dynamo_vllm_argv( - model_name="model", - namespace="namespace", - seed=0, - vllm_cfg=_config()["vllm_cfg"], - vllm_kwargs={}, - dynamo_cfg=cfg, - ) - - configured_fields = { - source.rsplit(".", 1)[-1] - for source in sources - if source.startswith("dynamo_cfg.worker_args.") - } - assert set(DynamoWorkerArgs.model_fields) - {"extra_cli_args"} <= configured_fields - - -def test_redaction_hides_credentials() -> None: - assert redact_argv(["worker", "--api-key", "secret"])[2] == "" - assert redact_argv( - ["worker", "--max-num-batched-tokens", "8192", "--stop-token-ids", "1,2"] - ) == ["worker", "--max-num-batched-tokens", "8192", "--stop-token-ids", "1,2"] - assert redact_environment({"HF_TOKEN": "secret", "NCCL_DEBUG": "INFO"}) == { - "HF_TOKEN": "", - "NCCL_DEBUG": "INFO", - } diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py new file mode 100644 index 00000000000..ba6edd103e9 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -0,0 +1,202 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from pydantic import ValidationError + +from nemo_rl.models.generation import configure_generation_config +from nemo_rl.models.generation.dynamo.arguments import ( + build_dynamo_frontend_argv, + redact_argv, + redact_environment, +) +from nemo_rl.models.generation.dynamo.config import ( + DynamoCfg, + dynamo_cfg, + dynamo_chat_template, + dynamo_parsers, + native_default_chat_template_kwargs, +) + + +class _Tokenizer: + pad_token_id = 0 + eos_token_id = 1 + + +def _vllm_config(**overrides) -> dict: + config = { + "backend": "vllm", + "model_name": "Qwen/Qwen3-0.6B", + "stop_token_ids": None, + "stop_strings": None, + "vllm_cfg": {"async_engine": False, "max_model_len": 512}, + } + config.update(overrides) + return config + + +def test_dynamo_true_alone_is_a_complete_config() -> None: + config = configure_generation_config(_vllm_config(dynamo=True), _Tokenizer()) + + # Dynamo serves the AsyncLLM, so the switch selects the async engine, and it + # serves OpenAI clients, which need the engine's tokenizer. + assert config["vllm_cfg"]["async_engine"] is True + assert config["vllm_cfg"]["skip_tokenizer_init"] is False + assert dynamo_cfg(config) == DynamoCfg() + assert dynamo_cfg(config).frontend_args.router_mode == "kv" + + +def test_without_dynamo_the_vllm_config_is_untouched() -> None: + config = configure_generation_config(_vllm_config(), _Tokenizer()) + + assert config["vllm_cfg"]["async_engine"] is False + assert config["vllm_cfg"]["skip_tokenizer_init"] is True + assert "dynamo_cfg" not in config + + +@pytest.mark.parametrize( + ("config", "error"), + [ + # A colocated engine sleeps while the policy trains. + ( + _vllm_config(dynamo=True, colocated={"enabled": True}), + "requires non-colocated generation", + ), + (_vllm_config(dynamo=True, backend="sglang"), "backend=vllm"), + # dynamo_cfg alone would be ignored. + (_vllm_config(dynamo_cfg={}), "dynamo is not true"), + ], +) +def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( + config, error +) -> None: + with pytest.raises(ValueError, match=error): + configure_generation_config(config, _Tokenizer()) + + +def test_dynamo_cfg_rejects_unknown_keys() -> None: + with pytest.raises(ValidationError, match="extra_cli_args"): + dynamo_cfg({"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}) + with pytest.raises(ValidationError, match="duplicates"): + dynamo_cfg({"dynamo_cfg": {"worker_args": {"endpoint_types": ["chat"] * 2}}}) + + +def test_frontend_argv_is_runtime_owned() -> None: + cfg = DynamoCfg.model_validate( + {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} + ) + argv = build_dynamo_frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg + ) + assert argv[argv.index("--router-mode") + 1] == "kv" + # Gym names a token-capture call's session; keep the session on one engine. + assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "1800" + assert argv[-2:] == ["--foo", "1"] + no_affinity = DynamoCfg.model_validate( + {"frontend_args": {"session_affinity_ttl_secs": None}} + ) + assert "--router-session-affinity-ttl-secs" not in build_dynamo_frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=no_affinity + ) + + clashing = DynamoCfg.model_validate( + {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} + ) + with pytest.raises(ValueError, match="--http-port is set by both"): + build_dynamo_frontend_argv( + host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=clashing + ) + + +def test_redaction_hides_credentials() -> None: + assert redact_argv(["frontend", "--api-key", "secret"])[2] == "" + assert redact_environment({"HF_TOKEN": "secret", "NCCL_DEBUG": "INFO"}) == { + "HF_TOKEN": "", + "NCCL_DEBUG": "INFO", + } + + +def _serving(**serving_chat_kwargs) -> dict: + return _vllm_config( + dynamo=True, + vllm_cfg={ + "async_engine": True, + "max_model_len": 512, + "http_server_serving_chat_kwargs": serving_chat_kwargs, + }, + ) + + +def test_parsers_follow_the_native_server_under_dynamos_names() -> None: + assert dynamo_parsers(_serving(tool_parser="hermes", reasoning_parser="qwen3")) == ( + "hermes", + "qwen3", + ) + # NeMo RL's nano_v3 vLLM plugin is Dynamo's nemotron_v3 parser. + nano_v3 = _serving(reasoning_parser="nano_v3") + nano_v3["vllm_cfg"]["reasoning_parser_plugin"] = "nano_v3_reasoning_parser.py" + assert dynamo_parsers(nano_v3) == (None, "nemotron_v3") + assert dynamo_parsers(_serving(tool_parser="openai")) == ("harmony", None) + # dynamo_cfg names them explicitly. + explicit = _serving(tool_parser="hermes") + explicit["dynamo_cfg"] = {"worker_args": {"tool_call_parser": "qwen3_coder"}} + assert dynamo_parsers(explicit) == ("qwen3_coder", None) + + +def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> None: + config = _serving(tool_parser="my_parser") + config["vllm_cfg"]["tool_parser_plugin"] = "my_parser.py" + + with pytest.raises(ValueError, match="worker_args.tool_call_parser"): + configure_generation_config(config, _Tokenizer()) + + +def test_chat_template_and_default_kwargs_follow_the_native_server() -> None: + config = _serving( + chat_template="templates/chat.jinja", + default_chat_template_kwargs={"enable_thinking": False}, + ) + assert dynamo_chat_template(config) == "templates/chat.jinja" + assert native_default_chat_template_kwargs(config) == {"enable_thinking": False} + # The older spelling the native server also accepts. + legacy = _serving(chat_template_kwargs={"enable_thinking": True}) + assert native_default_chat_template_kwargs(legacy) == {"enable_thinking": True} + assert dynamo_chat_template(_serving()) is None + + +@pytest.mark.parametrize( + ("serving_chat_kwargs", "error"), + [ + ({"chat_template_content_format": "string"}, "no content-format setting"), + ({"enable_auto_tools": False}, "always parses"), + ], +) +def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( + serving_chat_kwargs, error +) -> None: + with pytest.raises(ValueError, match=error): + configure_generation_config(_serving(**serving_chat_kwargs), _Tokenizer()) + # The native defaults are what the frontend does anyway. + configure_generation_config( + _serving(chat_template_content_format="auto", enable_auto_tools=True), + _Tokenizer(), + ) + + +def test_a_bearer_token_is_rejected_because_the_frontend_checks_none() -> None: + config = _vllm_config(dynamo=True) + config["vllm_cfg"]["http_server_api_key_required"] = True + + with pytest.raises(ValueError, match="checks no bearer token"): + configure_generation_config(config, _Tokenizer()) diff --git a/tests/unit/models/generation/test_dynamo_generation.py b/tests/unit/models/generation/test_dynamo_generation.py deleted file mode 100644 index 5b13bdf110d..00000000000 --- a/tests/unit/models/generation/test_dynamo_generation.py +++ /dev/null @@ -1,667 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import pickle -from typing import Any -from unittest.mock import MagicMock - -import pytest -import torch - -from nemo_rl.distributed.batched_data_dict import BatchedDataDict -from nemo_rl.models.generation.dynamo import DynamoGeneration -from nemo_rl.models.generation.dynamo import dynamo_generation as generation_module -from nemo_rl.models.generation.dynamo import metrics as metrics_module -from nemo_rl.models.generation.dynamo import refit as refit_module -from nemo_rl.models.generation.dynamo.config import ( - VLLM_PACKED_BUFFER_SIZE_BYTES, - VLLM_PACKED_NUM_BUFFERS, -) -from nemo_rl.models.generation.dynamo.metrics import ( - DynamoMetricsSampler, - parse_prometheus_metrics, -) -from nemo_rl.models.generation.dynamo.refit import DynamoRefitChannel - - -def _config(*, tp: int = 1, expose_http_server: bool = False) -> dict[str, Any]: - return { - "backend": "dynamo", - "model_name": "Qwen/Qwen3-0.6B", - "max_new_tokens": 16, - "temperature": 1.0, - "top_p": 1.0, - "top_k": None, - "stop_token_ids": None, - "stop_strings": None, - "_pad_token_id": 0, - "colocated": {"enabled": False}, - "dynamo_cfg": { - "engine": "vllm", - "startup_timeout_s": 5, - "request_timeout_s": 30, - "control_timeout_s": 10, - "metrics_include_prefixes": None, - "metrics_exclude_prefixes": None, - "worker_args": { - "tool_call_parser": None, - "reasoning_parser": None, - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": False, - "structural_tag_scope": "auto", - "structural_tag_schema": "auto", - "custom_jinja_template": None, - "endpoint_types": ["chat", "completions"], - "extra_cli_args": [], - }, - "frontend_args": { - "tokenizer": "default", - "tokenizer_cache": False, - "tokenizer_cache_bytes": 50 * 1024 * 1024, - "router_mode": "kv", - "extra_cli_args": [], - }, - }, - "vllm_cfg": { - "async_engine": True, - "tensor_parallel_size": tp, - "pipeline_parallel_size": 1, - "expert_parallel_size": tp, - "gpu_memory_utilization": 0.8, - "precision": "bfloat16", - "kv_cache_dtype": "auto", - "max_model_len": 5, - "load_format": "auto", - "enforce_eager": False, - "expose_http_server": expose_http_server, - "enable_vllm_metrics_logger": True, - "vllm_metrics_logger_interval": 1.0, - "env_vars": None, - }, - "vllm_kwargs": {}, - } - - -def _patch_runtime( - monkeypatch: pytest.MonkeyPatch, - workers: list[dict[str, Any]] | None = None, - calls: list[str] | None = None, -) -> None: - endpoints = workers or [ - {"instance_id": "worker-0", "system_url": "http://10.0.0.2:4000"} - ] - events = calls if calls is not None else [] - - class FakeRuntime: - def __init__(self, *, cluster, config): - events.append("init") - - def start(self): - events.append("start") - - @property - def frontend_url(self): - return "http://10.0.0.1:3000/v1" - - def refit_workers(self): - return [dict(worker) for worker in endpoints] - - def validate_workers(self, expected): - return expected - - def shutdown(self): - events.append("shutdown") - - monkeypatch.setattr(generation_module, "ManagedDynamoRuntime", FakeRuntime) - - -def _data() -> BatchedDataDict: - return BatchedDataDict( - { - "input_ids": torch.tensor([[1, 2, 3, 0]], dtype=torch.long), - "input_lengths": torch.tensor([3], dtype=torch.long), - "stop_strings": [["stop"]], - } - ) - - -def _completion_response(token_ids: list[int]) -> dict[str, Any]: - return { - "choices": [ - { - "finish_reason": "stop", - "logprobs": {"token_logprobs": [-0.25] * len(token_ids)}, - } - ], - "nvext": {"completion_token_ids": token_ids}, - } - - -def test_runtime_start_world_size_sender_geometry_and_shutdown(monkeypatch) -> None: - calls: list[str] = [] - _patch_runtime( - monkeypatch, - workers=[ - {"instance_id": "a", "system_url": "http://10.0.0.2:4000"}, - {"instance_id": "b", "system_url": "http://10.0.0.3:4000"}, - ], - calls=calls, - ) - generation = DynamoGeneration(cluster=object(), config=_config(tp=2)) - - assert calls[:2] == ["init", "start"] - assert generation.frontend_url == "http://10.0.0.1:3000/v1" - assert generation.dp_openai_server_base_urls == [None] - assert generation.get_inference_world_size() == 4 - sender = generation.get_collective_sender_spec() - assert sender.nccl_peer == "vllm" - assert sender.buffer_size_bytes == VLLM_PACKED_BUFFER_SIZE_BYTES - assert sender.num_buffers == VLLM_PACKED_NUM_BUFFERS - assert sender.use_legacy_packing - assert generation.shutdown() - assert generation.shutdown() - assert calls.count("shutdown") == 1 - - -def test_blocking_generate_is_rejected_and_async_generation_uses_http( - monkeypatch, -) -> None: - _patch_runtime(monkeypatch) - requests = [] - - async def fake_post(url, payload, timeout_s): - requests.append((url, payload, timeout_s)) - return _completion_response([8, 9]) - - monkeypatch.setattr(generation_module, "async_http_post_json", fake_post) - generation = DynamoGeneration(cluster=object(), config=_config()) - with pytest.raises(NotImplementedError, match="generate_async"): - generation.generate(_data()) - - async def collect(): - return [item async for item in generation.generate_async(_data())] - - outputs = asyncio.run(collect()) - assert outputs[0][0] == 0 - assert outputs[0][1]["output_ids"].tolist() == [[1, 2, 3, 8, 9]] - assert requests[0][0].endswith("/v1/completions") - assert requests[0][1]["max_tokens"] == 2 - assert requests[0][1]["stop"] == ["stop"] - assert "return_tokens_as_token_ids" not in requests[0][1] - - -@pytest.mark.parametrize("prompt_key", ["vllm_content", "vllm_multi_modal_data"]) -def test_async_generate_rejects_multimodal_prompt_columns( - monkeypatch: pytest.MonkeyPatch, prompt_key: str -) -> None: - _patch_runtime(monkeypatch) - post = MagicMock( - side_effect=AssertionError("Multimodal prompts must not reach HTTP") - ) - monkeypatch.setattr(generation_module, "async_http_post_json", post) - generation = DynamoGeneration(cluster=object(), config=_config()) - data = _data() - data[prompt_key] = ["prompt" if prompt_key == "vllm_content" else {"image": "img"}] - - async def collect() -> list: - return [item async for item in generation.generate_async(data)] - - with pytest.raises(NotImplementedError, match="multimodal"): - asyncio.run(collect()) - post.assert_not_called() - - -def test_prompt_at_context_limit_is_rejected(monkeypatch) -> None: - _patch_runtime(monkeypatch) - generation = DynamoGeneration(cluster=object(), config=_config()) - - with pytest.raises(ValueError, match="prompt length 5 must be less than"): - generation._allowed_new_tokens(5) - - -def test_finish_generation_invalidates_sync_rollout_cache(monkeypatch) -> None: - _patch_runtime(monkeypatch) - generation = DynamoGeneration(cluster=object(), config=_config()) - generation.invalidate_kv_cache = MagicMock(return_value=True) - - assert generation.finish_generation() - generation.invalidate_kv_cache.assert_called_once_with() - - -def test_merged_stop_strings_enforce_dynamo_limit(monkeypatch) -> None: - _patch_runtime(monkeypatch) - config = _config() - config["stop_strings"] = [f"configured-{index}" for index in range(16)] - generation = DynamoGeneration(cluster=object(), config=config) - - assert ( - len( - generation._merge_stop_strings( - [[f"request-{index}" for index in range(16)]] - ) - ) - == 32 - ) - with pytest.raises(ValueError, match="at most 32 stop strings"): - generation._merge_stop_strings([[f"request-{index}" for index in range(17)]]) - - -def test_token_wrapper_is_used_for_nemo_gym(monkeypatch) -> None: - _patch_runtime(monkeypatch) - wrappers = [] - - class FakeWrapper: - def __init__(self, **kwargs): - self.kwargs = kwargs - wrappers.append(self) - - def start(self): - return "http://127.0.0.1:3001/v1" - - def shutdown(self): - pass - - monkeypatch.setattr(generation_module, "DynamoTokenWrapperServer", FakeWrapper) - tokenizer = object() - generation = DynamoGeneration( - cluster=object(), - config=_config(expose_http_server=True), - tokenizer=tokenizer, - tokenizer_config={"chat_template_kwargs": {"enable_thinking": False}}, - ) - assert generation.dp_openai_server_base_urls == ["http://127.0.0.1:3001/v1"] - assert wrappers[0].kwargs["tokenizer"] is tokenizer - - -def test_refit_rank_offsets_update_and_pickled_cache_invalidation(monkeypatch) -> None: - workers = [ - {"instance_id": "a", "system_url": "http://10.0.0.2:4000"}, - {"instance_id": "b", "system_url": "http://10.0.0.3:4000"}, - ] - _patch_runtime(monkeypatch, workers=workers) - init_calls = [] - update_calls = [] - cache_calls = [] - monkeypatch.setattr( - refit_module._post_worker_route, - "remote", - lambda **kwargs: init_calls.append(kwargs) or True, - ) - monkeypatch.setattr( - refit_module._update_worker_weights, - "remote", - lambda **kwargs: update_calls.append(kwargs) or True, - ) - monkeypatch.setattr(refit_module.ray, "get", lambda refs: refs) - - generation = DynamoGeneration(cluster=object(), config=_config(tp=2)) - generation.prepare_refit_info({"weight": (torch.Size([4, 8]), torch.bfloat16)}) - assert generation.init_collective("10.1.0.1", 1500, 7, train_world_size=3) == [ - True, - True, - ] - assert [call["payload"]["init_info"]["rank_offset"] for call in init_calls] == [ - 3, - 5, - ] - assert all(call["timeout_s"] == 10 for call in init_calls) - # vLLM reads packing from the init info only (worker default is unpacked). - for call in init_calls: - init_info = call["payload"]["init_info"] - assert init_info["packed"] is True - assert init_info["packed_buffer_size_bytes"] == 1024**3 - assert init_info["packed_num_buffers"] == 2 - assert generation.update_weights_from_collective() == [True, True] - assert update_calls[0]["update_info"] == { - "names": ["weight"], - "dtype_names": ["bfloat16"], - "shapes": [[4, 8]], - } - assert all(call["timeout_s"] == 10 for call in update_calls) - - restored = pickle.loads(pickle.dumps(generation)) - assert restored.frontend_url == generation.frontend_url - # SingleController prepares refit on the driver, then pickles the - # generation client into its actor, which issues the weight updates. - expected_update_info = update_calls[0]["update_info"] - update_calls.clear() - assert restored.update_weights_from_collective() == [True, True] - assert [call["update_info"] for call in update_calls] == [expected_update_info] * 2 - monkeypatch.setattr( - refit_module._post_worker_route, - "remote", - lambda **kwargs: cache_calls.append(kwargs) or True, - ) - assert restored.invalidate_kv_cache() - assert [call["route"] for call in cache_calls] == [ - "pause_generation", - "pause_generation", - "resume_generation", - "resume_generation", - ] - assert all(call["timeout_s"] == 10 for call in cache_calls) - assert all( - call["payload"] == {"mode": "wait", "clear_cache": True} - for call in cache_calls[:2] - ) - assert restored._managed_runtime is None - - # The metrics sampler starts in the process that reads it: the driver starts - # none, and the SingleController's telemetry pump starts it in the copy. - monkeypatch.setattr(metrics_module, "_http_get_text", lambda *args: None) - assert generation._metrics_sampler is None - assert not any(restored.drain_latest_logger_metrics().values()) - assert restored._metrics_sampler is not None - assert restored._metrics_sampler._workers == restored._refit_channel.workers - assert restored.shutdown() - assert restored._metrics_sampler is None - - -def test_cache_invalidation_resumes_workers_that_paused_before_peer_failure( - monkeypatch, -) -> None: - calls = [] - - def remote(**kwargs): - calls.append(kwargs) - return (kwargs["route"], kwargs["system_url"]) - - def get(ref): - if ref == ("pause_generation", "http://worker-b:4000"): - raise RuntimeError("pause refused") - return True - - monkeypatch.setattr(refit_module._post_worker_route, "remote", remote) - monkeypatch.setattr(refit_module.ray, "get", get) - channel = DynamoRefitChannel( - [ - {"instance_id": "a", "system_url": "http://worker-a:4000"}, - {"instance_id": "b", "system_url": "http://worker-b:4000"}, - ], - engine_world_size=1, - control_timeout_s=10, - ) - - with pytest.raises(RuntimeError, match="pause/clear failed"): - channel.flush_cache() - - assert [(call["route"], call["system_url"]) for call in calls] == [ - ("pause_generation", "http://worker-a:4000"), - ("pause_generation", "http://worker-b:4000"), - ("resume_generation", "http://worker-a:4000"), - ] - - -@pytest.mark.parametrize("clear_cache", [True, False]) -def test_refit_pause_drains_every_worker_and_resume_restores_them( - monkeypatch, clear_cache: bool -) -> None: - workers = [ - {"instance_id": "a", "system_url": "http://10.0.0.2:4000"}, - {"instance_id": "b", "system_url": "http://10.0.0.3:4000"}, - ] - _patch_runtime(monkeypatch, workers=workers) - calls = [] - monkeypatch.setattr( - refit_module._post_worker_route, - "remote", - lambda **kwargs: calls.append(kwargs) or True, - ) - monkeypatch.setattr(refit_module.ray, "get", lambda refs: refs) - - # SingleController drives the pause from its own pickled client copy. - generation = pickle.loads( - pickle.dumps(DynamoGeneration(cluster=object(), config=_config())) - ) - assert generation.pause_generation_for_refit(clear_cache=clear_cache) - assert [(call["route"], call["payload"]) for call in calls] == [ - ("pause_generation", {"mode": "wait", "clear_cache": clear_cache}), - ("pause_generation", {"mode": "wait", "clear_cache": clear_cache}), - ] - - calls.clear() - assert generation.resume_generation_after_refit() - assert [(call["route"], call["system_url"]) for call in calls] == [ - ("resume_generation", "http://10.0.0.2:4000"), - ("resume_generation", "http://10.0.0.3:4000"), - ] - - -def test_failed_pause_resumes_the_workers_that_paused(monkeypatch) -> None: - calls = [] - - def remote(**kwargs): - calls.append(kwargs) - return (kwargs["route"], kwargs["system_url"]) - - def get(ref): - if ref == ("pause_generation", "http://worker-b:4000"): - raise RuntimeError("pause refused") - return True - - monkeypatch.setattr(refit_module._post_worker_route, "remote", remote) - monkeypatch.setattr(refit_module.ray, "get", get) - channel = DynamoRefitChannel( - [ - {"instance_id": "a", "system_url": "http://worker-a:4000"}, - {"instance_id": "b", "system_url": "http://worker-b:4000"}, - ], - engine_world_size=1, - control_timeout_s=10, - ) - - with pytest.raises(RuntimeError, match="pause before weight update failed"): - channel.pause(clear_cache=True) - - assert [(call["route"], call["system_url"]) for call in calls] == [ - ("pause_generation", "http://worker-a:4000"), - ("pause_generation", "http://worker-b:4000"), - ("resume_generation", "http://worker-a:4000"), - ] - - -def test_native_refit_transaction_keeps_cache_mode_external(monkeypatch) -> None: - calls = [] - monkeypatch.setattr( - refit_module, - "http_post_json", - lambda url, payload, timeout_s: calls.append(payload) or {"status": "ok"}, - ) - assert refit_module._update_worker_weights._function( - system_url="http://worker:4000", - update_info={"names": ["weight"]}, - timeout_s=30, - ) - assert [call["engine_rpc"] for call in calls] == [ - "start_weight_update", - "update_weights", - "finish_weight_update", - ] - assert all(call["reset_prefix_cache"] is False for call in calls) - # vLLM's Worker.start_weight_update takes no arguments; Dynamo forwards - # every non-reserved key, so any extra key would raise in the engine. - assert set(calls[0]) == {"engine_rpc", "allow_unpaused", "reset_prefix_cache"} - - -def test_metrics_parser_and_sampler_aliases() -> None: - sampler = DynamoMetricsSampler( - [{"instance_id": "a", "system_url": "http://worker:4000"}], - interval_s=1, - include_prefixes=None, - exclude_prefixes=None, - ) - parsed = parse_prometheus_metrics( - 'vllm:num_requests_running{model_name="model",engine="0"} 3\n' - 'vllm:num_requests_waiting{model_name="model",engine="0"} 2\n' - 'vllm:kv_cache_usage_perc{model_name="model",engine="0"} 0.5\n' - 'vllm:generation_tokens_total{model_name="model",engine="0"} 7\n' - 'vllm:gpu_cache_usage_perc{model_name="model",engine="0"} 0.9\n' - "python_gc_objects_collected_total 10\n", - sampler._include_prefixes, - sampler._exclude_prefixes, - ) - sampler._samples = {name: {0: [value]} for name, value in parsed.items()} - metrics = sampler.snapshot() - assert metrics["inflight_batch_sizes"] == {0: [3.0]} - assert metrics["num_pending_samples"] == {0: [2.0]} - assert metrics["kv_cache_usage_perc"] == {0: [0.5]} - assert metrics["generation_tokens"] == {0: [7.0]} - assert "vllm_gpu_cache_usage_perc" not in parsed - assert "python_gc_objects_collected_total" not in parsed - - # Rollout telemetry drains to each worker's latest sample, which it keeps. - sampler._samples["vllm_generation_tokens_total"][0].append(9.0) - for _ in range(2): - assert sampler.drain_latest()["generation_tokens"] == {0: [9.0]} - - -def test_metrics_http_errors_are_ignored(monkeypatch) -> None: - monkeypatch.setattr( - metrics_module.urllib.request, - "urlopen", - lambda *args, **kwargs: (_ for _ in ()).throw( - metrics_module.urllib.error.URLError("refused") - ), - ) - assert metrics_module._http_get_text("http://worker/metrics", 1) is None - - -def test_completion_parser_rejects_misaligned_logprobs() -> None: - response = _completion_response([8, 9]) - response["choices"][0]["logprobs"]["token_logprobs"] = [-0.1] - with pytest.raises(RuntimeError, match="1 token logprobs for 2"): - generation_module._parse_dynamo_completion_response( - response, request_url="http://dynamo/v1/completions" - ) - - -@pytest.mark.parametrize( - ("response", "expected"), - [ - ({"http_status": 408}, True), - ({"http_status": 429}, True), - ({"http_status": 503}, True), - ({"transport_error": "refused"}, True), - ({"json_decode_error": True}, True), - ({"http_status": 400}, False), - ({"http_status": 404}, False), - ], -) -def test_completion_retry_predicate(response, expected) -> None: - assert generation_module._is_retryable_http_response(response) is expected - - -def test_completion_retry_eventually_succeeds(monkeypatch) -> None: - _patch_runtime(monkeypatch) - responses = iter( - [{"status": "error", "http_status": 503}, _completion_response([8])] - ) - calls = [] - - async def fake_post(*args): - calls.append(args) - return next(responses) - - async def no_sleep(_): - return None - - monkeypatch.setattr(generation_module, "async_http_post_json", fake_post) - monkeypatch.setattr(generation_module.asyncio, "sleep", no_sleep) - generation = DynamoGeneration(cluster=object(), config=_config()) - - token_ids, _, _ = asyncio.run( - generation._post_completion_request( - prompt_token_ids=[1], - greedy=False, - stop_strings=None, - max_new_tokens=1, - ) - ) - - assert token_ids == [8] - assert len(calls) == 2 - - -@pytest.mark.parametrize("status", [400, 503]) -def test_completion_retry_stops_on_nonretryable_or_exhaustion( - monkeypatch, status -) -> None: - _patch_runtime(monkeypatch) - calls = [] - - async def fake_post(*args): - calls.append(args) - return {"status": "error", "http_status": status} - - async def no_sleep(_): - return None - - monkeypatch.setattr(generation_module, "async_http_post_json", fake_post) - monkeypatch.setattr(generation_module.asyncio, "sleep", no_sleep) - generation = DynamoGeneration(cluster=object(), config=_config()) - - with pytest.raises(RuntimeError, match=f"HTTP {status}"): - asyncio.run( - generation._post_completion_request( - prompt_token_ids=[1], - greedy=False, - stop_strings=None, - max_new_tokens=1, - ) - ) - - assert len(calls) == (1 if status == 400 else generation_module._HTTP_MAX_ATTEMPTS) - - -def test_direct_completions_are_not_limited_by_default_thread_pool( - monkeypatch, -) -> None: - _patch_runtime(monkeypatch) - request_count = 40 - entered_count = 0 - all_entered = asyncio.Event() - release = asyncio.Event() - - async def fake_post(*args): - nonlocal entered_count - entered_count += 1 - if entered_count == request_count: - all_entered.set() - await release.wait() - return _completion_response([8]) - - monkeypatch.setattr(generation_module, "async_http_post_json", fake_post) - generation = DynamoGeneration(cluster=object(), config=_config()) - - async def run_requests(): - tasks = [ - asyncio.create_task( - generation._post_completion_request( - prompt_token_ids=[1], - greedy=False, - stop_strings=None, - max_new_tokens=1, - ) - ) - for _ in range(request_count) - ] - await asyncio.wait_for(all_entered.wait(), timeout=1) - release.set() - return await asyncio.gather(*tasks) - - responses = asyncio.run(run_requests()) - - assert entered_count == request_count - assert len(responses) == request_count diff --git a/tests/unit/models/generation/test_dynamo_http_client.py b/tests/unit/models/generation/test_dynamo_http_client.py index 195db9bb55a..d9002bce5c1 100644 --- a/tests/unit/models/generation/test_dynamo_http_client.py +++ b/tests/unit/models/generation/test_dynamo_http_client.py @@ -13,29 +13,12 @@ # limitations under the License. import asyncio -import io -import json -import urllib.error import aiohttp import pytest -from nemo_rl.models.generation.dynamo import dynamo_generation as generation_module from nemo_rl.models.generation.dynamo import http_client - - -class _SyncResponse: - def __init__(self, body: bytes): - self._body = body - - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def read(self) -> bytes: - return self._body +from nemo_rl.models.generation.dynamo.rollout_client import _is_retryable class _AsyncResponse: @@ -72,89 +55,6 @@ def post(self, url, *, json): return self._response -def _sync_post(monkeypatch, result): - def fake_urlopen(request, timeout): - if isinstance(result, BaseException): - raise result - return _SyncResponse(result) - - monkeypatch.setattr(http_client.urllib.request, "urlopen", fake_urlopen) - return http_client.http_post_json("http://worker/route", {"value": 1}, 3) - - -def test_http_post_json_success_preserves_request_contract(monkeypatch) -> None: - captured = {} - - def fake_urlopen(request, timeout): - captured["request"] = request - captured["timeout"] = timeout - return _SyncResponse(b'{"status":"ok"}') - - monkeypatch.setattr(http_client.urllib.request, "urlopen", fake_urlopen) - - response = http_client.http_post_json( - "http://worker/route", {"value": 1}, timeout_s=3 - ) - - request = captured["request"] - assert response == {"status": "ok"} - assert request.full_url == "http://worker/route" - assert request.get_method() == "POST" - assert json.loads(request.data) == {"value": 1} - assert captured["timeout"] == 3 - - -@pytest.mark.parametrize( - ("body", "expected"), - [ - ( - b"not-json", - {"status": "error", "json_decode_error": True, "raw": "not-json"}, - ), - (b"[1, 2]", {"status": "error", "raw": "[1, 2]"}), - ], -) -def test_http_post_json_rejects_invalid_json_shapes( - monkeypatch, body, expected -) -> None: - assert _sync_post(monkeypatch, body) == expected - - -@pytest.mark.parametrize( - ("error", "retryable", "message"), - [ - ( - urllib.error.HTTPError( - "http://worker/route", - 503, - "unavailable", - hdrs=None, - fp=io.BytesIO(b"try later"), - ), - True, - "HTTP 503: try later", - ), - ( - urllib.error.HTTPError( - "http://worker/route", 400, "bad request", hdrs=None, fp=None - ), - False, - "HTTP 400: ", - ), - (urllib.error.URLError("refused"), True, "URLError"), - (TimeoutError("slow"), True, "TimeoutError"), - ], -) -def test_http_errors_round_trip_through_consumers( - monkeypatch, error, retryable, message -) -> None: - response = _sync_post(monkeypatch, error) - - assert response["status"] == "error" - assert generation_module._is_retryable_http_response(response) is retryable - assert message in http_client.format_dynamo_error(response) - - @pytest.mark.parametrize( ("body", "status", "expected", "retryable"), [ @@ -173,7 +73,7 @@ def test_http_errors_round_trip_through_consumers( ), ], ) -def test_async_http_post_json_uses_same_error_contract( +def test_async_http_post_json_error_contract( monkeypatch, body, status, expected, retryable ) -> None: session = _AsyncSession(_AsyncResponse(body, status)) @@ -190,14 +90,22 @@ def test_async_http_post_json_uses_same_error_contract( ) assert response == expected - assert generation_module._is_retryable_http_response(response) is retryable + assert _is_retryable(response) is retryable + if response.get("status") == "error": + assert expected["raw"] in http_client.format_dynamo_error(response) assert session.requests == [("http://worker/route", {"value": 1})] @pytest.mark.parametrize( - "error", [aiohttp.ClientConnectionError("refused"), TimeoutError("slow")] + ("error", "key", "retryable"), + [ + (aiohttp.ClientConnectionError("refused"), "transport_error", True), + # One request_timeout_s bounds a rollout: a timeout is not retried. + (TimeoutError("slow"), "timeout_error", False), + (aiohttp.ServerTimeoutError("slow read"), "timeout_error", False), + ], ) -def test_async_http_transport_errors_are_retryable(monkeypatch, error) -> None: +def test_async_http_transport_errors(monkeypatch, error, key, retryable) -> None: session = _AsyncSession(error=error) monkeypatch.setattr( http_client.aiohttp, @@ -212,5 +120,6 @@ def test_async_http_transport_errors_are_retryable(monkeypatch, error) -> None: ) assert response["status"] == "error" - assert generation_module._is_retryable_http_response(response) - assert "transport_error" in response + assert _is_retryable(response) is retryable + assert key in response + assert response[key] in http_client.format_dynamo_error(response) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index e13cce6aaf2..0b56635518c 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -11,128 +11,31 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import json import signal import subprocess import sys from pathlib import Path -from types import SimpleNamespace from unittest.mock import MagicMock import pytest -from nemo_rl.models.generation.dynamo.dynamo_worker import ( - DynamoGpuReservation, - DynamoVllmWorker, -) +from nemo_rl.models.generation.dynamo import venv as dynamo_venv +from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, _managed_namespace, ) -from nemo_rl.models.generation.dynamo.venv import get_dynamo_venv_dir -from nemo_rl.models.generation.dynamo.worker_pool import ( - FixedDynamoWorkerPool, - _vllm_port_for_node_slot, -) - -class _Cluster: - num_gpus_per_node = 4 - -def test_dynamo_package_directory_does_not_shadow_stdlib_http() -> None: - package_dir = ( - Path(__file__).resolve().parents[4] - / "nemo_rl" - / "models" - / "generation" - / "dynamo" - ) - result = subprocess.run( - [sys.executable, "-c", "import http.client"], - cwd=package_dir, - capture_output=True, - text=True, - check=False, +def _runtime(**frontend_args) -> ManagedDynamoRuntime: + return ManagedDynamoRuntime( + dynamo_cfg=DynamoCfg.model_validate( + {"startup_timeout_s": 5, "frontend_args": frontend_args} + ), + model_name="model", + frontend_python="/venvs/vllm/bin/python", ) - assert result.returncode == 0, result.stderr - - -def _config(*, tp: int = 1) -> dict: - return { - "backend": "dynamo", - "model_name": "model", - "colocated": {"enabled": False}, - "dynamo_cfg": { - "engine": "vllm", - "startup_timeout_s": 5, - "request_timeout_s": 30, - "control_timeout_s": 10, - "metrics_include_prefixes": None, - "metrics_exclude_prefixes": None, - "worker_args": { - "tool_call_parser": None, - "reasoning_parser": None, - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": False, - "structural_tag_scope": "auto", - "structural_tag_schema": "auto", - "custom_jinja_template": None, - "endpoint_types": ["chat", "completions"], - "extra_cli_args": [], - }, - "frontend_args": { - "tokenizer": "default", - "tokenizer_cache": False, - "tokenizer_cache_bytes": 50 * 1024 * 1024, - "router_mode": "kv", - "extra_cli_args": [], - }, - }, - "vllm_cfg": { - "async_engine": True, - "tensor_parallel_size": tp, - "pipeline_parallel_size": 1, - "expert_parallel_size": tp, - "gpu_memory_utilization": 0.8, - "max_model_len": 512, - "precision": "bfloat16", - "kv_cache_dtype": "auto", - "load_format": "auto", - "enforce_eager": False, - "expose_http_server": False, - "enable_vllm_metrics_logger": True, - "vllm_metrics_logger_interval": 1.0, - "env_vars": None, - }, - "vllm_kwargs": {}, - } - - -class _RemoteMethod: - def __init__(self, result): - self._result = result - self.calls = [] - - def remote(self, *args, **kwargs): - self.calls.append((args, kwargs)) - return self._result - - -class _FakeWorker: - def __init__(self, alive=True, metadata=None): - self.is_alive = _RemoteMethod(alive) - self.metadata = _RemoteMethod(metadata or {}) - self.shutdown = _RemoteMethod(True) - - -class _FakeReservation: - def __init__(self, metadata=None, system_port=4000): - self.metadata = _RemoteMethod(metadata or {}) - self.select_free_port = _RemoteMethod(system_port) - self.register_process_group = _RemoteMethod(True) - self.cleanup_process_group = _RemoteMethod(True) class _FakeProcess: @@ -159,35 +62,55 @@ def read(self): return json.dumps(self._payload).encode() +def _started(runtime: ManagedDynamoRuntime) -> ManagedDynamoRuntime: + runtime._started = True + runtime._host = "10.0.0.1" + runtime._frontend_port = 3000 + runtime._namespace = "nemo-rl-test" + runtime._etcd_process = _FakeProcess() + runtime._nats_process = _FakeProcess() + runtime._frontend_process = _FakeProcess() + return runtime + + +def test_dynamo_package_directory_does_not_shadow_stdlib_http() -> None: + package_dir = ( + Path(__file__).resolve().parents[4] + / "nemo_rl" + / "models" + / "generation" + / "dynamo" + ) + result = subprocess.run( + [sys.executable, "-c", "import http.client"], + cwd=package_dir, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + def test_runtime_construction_is_inert_and_namespace_is_driver_owned( monkeypatch, ) -> None: monkeypatch.setenv("SLURM_JOB_ID", "Job/123.4") assert _managed_namespace() == "nemo-rl-job-123-4" - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) + runtime = _runtime() assert runtime._started is False assert runtime._etcd_process is None with pytest.raises(RuntimeError, match="not been started"): _ = runtime.frontend_url - - -def test_runtime_rejects_multinode_engine_group_before_spawning() -> None: - with pytest.raises(ValueError, match="fit on one node"): - ManagedDynamoRuntime(cluster=_Cluster(), config=_config(tp=8)) + with pytest.raises(RuntimeError, match="not been started"): + _ = runtime.engine_env def test_managed_service_and_frontend_environments_are_runtime_owned( monkeypatch, ) -> None: - config = _config() - config["dynamo_cfg"]["frontend_args"].update( - { - "tokenizer": "fastokens", - "tokenizer_cache": True, - "tokenizer_cache_bytes": 4096, - } + runtime = _runtime( + tokenizer="fastokens", tokenizer_cache=True, tokenizer_cache_bytes=4096 ) - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=config) runtime._manager_env = { "DYN_NAMESPACE": "managed", "DYN_DISCOVERY_BACKEND": "etcd", @@ -196,22 +119,18 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( } monkeypatch.setenv("DYN_NAMESPACE", "stale") monkeypatch.setenv("DYN_STALE_SETTING", "remove-me") - monkeypatch.setenv("ETCD_ENDPOINTS", "http://stale-etcd:2379") monkeypatch.setenv("ETCD_STALE_SETTING", "remove-me") - monkeypatch.setenv("NATS_SERVER", "nats://stale-nats:4222") monkeypatch.setenv("NATS_STALE_SETTING", "remove-me") monkeypatch.setenv("NCCL_DEBUG", "INFO") service_env = runtime._service_env() assert service_env["DYN_NAMESPACE"] == "managed" - assert service_env["DYN_DISCOVERY_BACKEND"] == "etcd" - assert "DYN_STALE_SETTING" not in service_env assert service_env["ETCD_ENDPOINTS"] == "http://managed-etcd:2379" assert service_env["NATS_SERVER"] == "nats://managed-nats:4222" + assert "DYN_STALE_SETTING" not in service_env assert "ETCD_STALE_SETTING" not in service_env assert "NATS_STALE_SETTING" not in service_env assert service_env["NCCL_DEBUG"] == "INFO" - assert "ALLOW_NONE_AUTHENTICATION" not in service_env frontend_env = runtime._frontend_env() assert frontend_env["DYN_TOKENIZER"] == "fastokens" @@ -219,99 +138,54 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( assert frontend_env["DYN_TOKENIZER_CACHE_BYTES"] == "4096" -def test_dynamo_venv_uses_explicit_env_or_repository_fallback(monkeypatch) -> None: - monkeypatch.setenv("NEMO_RL_DYNAMO_VENV_DIR", "/custom/dynamo") - assert get_dynamo_venv_dir() == Path("/custom/dynamo") - - monkeypatch.delenv("NEMO_RL_DYNAMO_VENV_DIR") - monkeypatch.setenv("NRL_CONTAINER", "1") - assert get_dynamo_venv_dir().parts[-2:] == ("venvs", "dynamo") - - -def test_vllm_node_local_port_bands_are_deterministic() -> None: - assert [_vllm_port_for_node_slot(slot) for slot in range(3)] == [7000, 7100, 7200] - - -def test_startup_failure_cleans_up_partial_worker_pool(monkeypatch, tmp_path) -> None: - calls = [] - exit_hooks = [] - pool_init_kwargs = {} - - class FailingPool: - def __init__(self, **kwargs): - calls.append("pool-init") - pool_init_kwargs.update(kwargs) - - def start(self): - calls.append("pool-start") - raise RuntimeError("worker failed") - - def shutdown(self): - calls.append("pool-shutdown") - - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) - etcd_dir = tmp_path / "etcd" - nats_dir = tmp_path / "nats" - etcd_dir.mkdir() - nats_dir.mkdir() - ports = iter([1313, 1314, 1315, 3000]) +def test_dynamo_python_resolves_the_vllm_worker_venv_as_workers_do( + monkeypatch, +) -> None: + built = [] monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime._get_node_ip_local", - lambda: "10.0.0.1", + dynamo_venv, "get_actor_python_env", lambda fqn: "uv run --locked --extra vllm" ) + # The workers' resolver: reuses an existing venv, builds a missing one per node. monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime._get_free_port_local", - lambda low, high, **kwargs: next(ports), + dynamo_venv, + "create_local_venv_on_each_node", + lambda py_executable, venv_name: built.append((py_executable, venv_name)) + or "/venvs/vllm/bin/python", ) - def start_etcd(): - calls.append("etcd") - runtime._etcd_process = object() - runtime._etcd_data_dir = str(etcd_dir) - - def start_nats(): - calls.append("nats") - runtime._nats_process = object() - runtime._nats_data_dir = str(nats_dir) + assert dynamo_venv.get_dynamo_python() == "/venvs/vllm/bin/python" + assert built == [ + ("uv run --locked --extra vllm", dynamo_venv.VLLM_ASYNC_WORKER_FQN) + ] - def stop_process(process, label, timeout_s=15): - if process is not None: - calls.append(f"stop-{label}") - monkeypatch.setattr(runtime, "_start_etcd", start_etcd) - monkeypatch.setattr(runtime, "_start_nats", start_nats) - monkeypatch.setattr(runtime, "_stop_process", stop_process) +def test_dynamo_python_passes_an_interpreter_through(monkeypatch) -> None: monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime.atexit.register", - lambda hook: exit_hooks.append(hook), + dynamo_venv, "get_actor_python_env", lambda fqn: "/usr/bin/python3" ) monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime.atexit.unregister", - lambda hook: exit_hooks.remove(hook), - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime.FixedDynamoWorkerPool", - FailingPool, + dynamo_venv, + "create_local_venv_on_each_node", + MagicMock(side_effect=AssertionError("no venv for an interpreter")), ) - with pytest.raises(RuntimeError, match="worker failed"): - runtime.start() - assert calls == [ - "etcd", - "nats", - "pool-init", - "pool-start", - "pool-shutdown", - "stop-NATS", - "stop-etcd", - ] - assert runtime._pool is None - assert runtime._started is False - assert exit_hooks == [] - assert pool_init_kwargs["manager_env"]["DYN_ENABLE_EXPERIMENTAL_PARSERS_V2"] == "1" - assert pool_init_kwargs["manager_env"]["DYN_RL_INIT_WEIGHTS_TIMEOUT_S"] == "10.0" - assert not etcd_dir.exists() - assert not nats_dir.exists() + assert dynamo_venv.get_dynamo_python() == "/usr/bin/python3" + + +def test_dynamo_services_resolve_from_bin_dir_then_path(monkeypatch, tmp_path) -> None: + etcd = tmp_path / "etcd" + etcd.write_text("#!/bin/sh\n") + etcd.chmod(0o755) + monkeypatch.setenv("NEMO_RL_DYNAMO_BIN_DIR", str(tmp_path)) + assert dynamo_venv.get_dynamo_service_executable("etcd") == str(etcd) + with pytest.raises(FileNotFoundError, match="NEMO_RL_DYNAMO_BIN_DIR"): + dynamo_venv.get_dynamo_service_executable("nats-server") + + monkeypatch.delenv("NEMO_RL_DYNAMO_BIN_DIR") + monkeypatch.setenv("PATH", str(tmp_path)) + assert dynamo_venv.get_dynamo_service_executable("etcd") == str(etcd) + with pytest.raises(FileNotFoundError, match="not on PATH"): + dynamo_venv.get_dynamo_service_executable("nats-server") def test_stop_process_escalates_its_process_group(monkeypatch) -> None: @@ -337,144 +211,17 @@ def wait(self, timeout): lambda pid, sig: signals.append((pid, sig)), ) process = EscalatingProcess() - ManagedDynamoRuntime._stop_process(process, "worker", timeout_s=0.01) + ManagedDynamoRuntime._stop_process(process, "frontend", timeout_s=0.01) assert signals == [(process.pid, signal.SIGTERM), (process.pid, signal.SIGKILL)] -def test_reservation_records_and_cleans_registered_process_group(monkeypatch) -> None: - reservation_cls = DynamoGpuReservation.__ray_metadata__.modified_class - reservation = reservation_cls() - signals = [] - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.os.killpg", - lambda pid, sig: signals.append((pid, sig)), - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.time.sleep", lambda _: None - ) - - assert reservation.register_process_group(4321) - assert reservation.cleanup_process_group() - assert reservation.cleanup_process_group() - assert signals == [(4321, signal.SIGTERM), (4321, signal.SIGKILL)] - - -def test_reservation_selects_a_free_nonexcluded_system_port(monkeypatch) -> None: - reservation_cls = DynamoGpuReservation.__ray_metadata__.modified_class - reservation = reservation_cls() - calls = [] - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker._get_free_port_local", - lambda low, high, **kwargs: calls.append((low, high, kwargs)) or 4002, - ) - - assert ( - reservation.select_free_port( - port_range_low=4000, - port_range_high=4003, - excluded_ports=[4001], - ) - == 4002 - ) - assert calls == [ - ( - 4000, - 4003, - {"max_retries": None, "excluded_ports": {4001}}, - ) - ] - - -def test_worker_argument_validation_has_startup_timeout(monkeypatch) -> None: - worker_cls = DynamoVllmWorker.__ray_metadata__.modified_class - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.subprocess.run", - lambda *args, **kwargs: (_ for _ in ()).throw( - subprocess.TimeoutExpired("validator", kwargs["timeout"]) - ), - ) - - with pytest.raises(RuntimeError, match="startup_timeout_s=7"): - worker_cls._validate_argv( - "/opt/dynamo_venv/bin/python", - ["--model", "model"], - {}, - timeout_s=7, - ) - - -def test_worker_registers_process_group_immediately_after_launch(monkeypatch) -> None: - class FakeSocket: - def __enter__(self): - return self - - def __exit__(self, *args): - return None - - def bind(self, address): - if address[1] == 7000: - raise AssertionError("VLLM_PORT must not be probed") - return None - - process = SimpleNamespace(pid=4321) - reservation = _FakeReservation() - worker_cls = DynamoVllmWorker.__ray_metadata__.modified_class - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker._get_node_ip_local", - lambda: "10.0.0.1", - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.socket.socket", - lambda *args, **kwargs: FakeSocket(), - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.get_dynamo_python", - lambda: "/opt/dynamo_venv/bin/python", - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.get_dynamo_venv_dir", - lambda: Path("/opt/dynamo_venv"), - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.subprocess.Popen", - lambda *args, **kwargs: process, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.dynamo_worker.ray.get", lambda ref: ref - ) - monkeypatch.setattr(worker_cls, "_validate_argv", MagicMock()) - monkeypatch.setattr(worker_cls, "_wait_for_system_port", MagicMock()) - - worker_cls( - _config(), - namespace="nemo-rl-test", - group_name="worker-0", - cuda_devices=[0], - system_port=4000, - vllm_port=7000, - manager_env={}, - startup_timeout_s=5, - seed=0, - cleanup_reservation=reservation, - ) - - assert reservation.register_process_group.calls == [((4321,), {})] - - def test_shutdown_guards_each_owned_resource_independently(monkeypatch) -> None: - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) + runtime = _runtime() runtime._frontend_process = object() runtime._nats_process = object() runtime._etcd_process = object() calls = [] - class FailingPool: - def shutdown(self): - calls.append("pool") - raise RuntimeError("pool failure") - - runtime._pool = FailingPool() - def stop_process(process, label, timeout_s=15): if process is None: return @@ -486,320 +233,34 @@ def stop_process(process, label, timeout_s=15): runtime.shutdown() runtime.shutdown() - assert calls[:4] == ["frontend", "pool", "NATS", "etcd"] + assert calls == ["frontend", "NATS", "etcd"] assert runtime._frontend_process is None - assert runtime._pool is None assert runtime._nats_process is None assert runtime._etcd_process is None -def test_fixed_pool_detects_worker_membership_change(monkeypatch) -> None: - expected = [{"instance_id": "worker-0", "system_url": "http://10.0.0.1:4000"}] - reservation_metadata = {"node_ip": "10.0.0.1", "gpu_id": 0} - reservation = _FakeReservation(metadata=reservation_metadata) - pool = object.__new__(FixedDynamoWorkerPool) - pool._workers = [_FakeWorker(metadata={"instance_id": "changed"})] - pool._reservations = [reservation] - pool._cleanup_reservations = [reservation] - pool._reservation_metadata = [reservation_metadata] - - def fake_get(refs, **kwargs): - if refs == [True]: - return [True] - if refs == [reservation_metadata]: - return [reservation_metadata] - return [{"instance_id": "changed"}] - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - with pytest.raises(RuntimeError, match="worker membership changed"): - pool.validate(expected) - - -def test_fixed_pool_detects_reservation_membership_change(monkeypatch) -> None: - expected_worker = { - "instance_id": "worker-0", - "system_url": "http://10.0.0.1:4000", - } - expected_reservation = {"node_ip": "10.0.0.1", "gpu_id": 0} - changed_reservation = {"node_ip": "10.0.0.2", "gpu_id": 0} - reservation = _FakeReservation(metadata=changed_reservation) - pool = object.__new__(FixedDynamoWorkerPool) - pool._workers = [_FakeWorker(metadata=expected_worker)] - pool._reservations = [reservation] - pool._cleanup_reservations = [reservation] - pool._reservation_metadata = [expected_reservation] - - def fake_get(refs, **kwargs): - if refs == [True]: - return [True] - if refs == [changed_reservation]: - return [changed_reservation] - return [expected_worker] - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - with pytest.raises(RuntimeError, match="GPU reservation membership changed"): - pool.validate([expected_worker]) - - -def test_fixed_pool_tracks_worker_before_metadata_failure(monkeypatch) -> None: - reservation = _FakeReservation(metadata={"node_ip": "10.0.0.1", "gpu_id": 0}) - worker = _FakeWorker() - - class RemoteFactory: - def __init__(self, actor): - self.actor = actor - - def remote(self, *args, **kwargs): - return self.actor - - class ReservationFactory: - @staticmethod - def options(**kwargs): - return RemoteFactory(reservation) - - class WorkerFactory: - @staticmethod - def options(**kwargs): - return RemoteFactory(worker) - - class Cluster: - @staticmethod - def get_placement_groups(): - return [SimpleNamespace(bundle_count=1)] - - pool = FixedDynamoWorkerPool( - cluster=Cluster(), - config=_config(), - namespace="nemo-rl-test", - engine_world_size=1, - manager_env={}, - startup_timeout_s=5, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.DynamoGpuReservation", - ReservationFactory, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.DynamoVllmWorker", - WorkerFactory, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.get_actor_python_env", - lambda _fqn: "python", - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.PlacementGroupSchedulingStrategy", - lambda **kwargs: kwargs, - ) - - def fake_get(refs, **kwargs): - if refs == [reservation.metadata.remote()]: - return [{"node_ip": "10.0.0.1", "gpu_id": 0}] - if refs == 4000: - return refs - raise RuntimeError("metadata failed") - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - with pytest.raises(RuntimeError, match="metadata failed"): - pool.start() - - assert pool._workers == [worker] - assert pool._cleanup_reservations == [reservation] - assert pool._metadata == [{}] - - -def test_fixed_pool_launches_all_workers_before_waiting_for_model_metadata( +def test_engines_are_ready_once_registered_and_the_model_is_served( monkeypatch, ) -> None: - reservation_objects = [ - _FakeReservation( - metadata={"node_ip": "10.0.0.1", "gpu_id": 0}, system_port=4001 - ), - _FakeReservation( - metadata={"node_ip": "10.0.0.1", "gpu_id": 1}, system_port=4002 - ), - ] - reservations = iter(reservation_objects) - workers = [ - _FakeWorker(metadata={"instance_id": "worker-0"}), - _FakeWorker(metadata={"instance_id": "worker-1"}), - ] - workers_to_launch = iter(workers) - launch_kwargs = [] - - class RemoteFactory: - def __init__(self, actor): - self.actor = actor - - def remote(self, *args, **kwargs): - return self.actor - - class ReservationFactory: - @staticmethod - def options(**kwargs): - return RemoteFactory(next(reservations)) - - class WorkerFactory: - @staticmethod - def options(**kwargs): - worker = next(workers_to_launch) - - class CapturingRemoteFactory(RemoteFactory): - def remote(self, *args, **kwargs): - launch_kwargs.append(kwargs) - return super().remote(*args, **kwargs) - - return CapturingRemoteFactory(worker) - - class Cluster: - @staticmethod - def get_placement_groups(): - return [SimpleNamespace(bundle_count=2)] - - pool = FixedDynamoWorkerPool( - cluster=Cluster(), - config=_config(), - namespace="nemo-rl-test", - engine_world_size=1, - manager_env={}, - startup_timeout_s=5, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.DynamoGpuReservation", - ReservationFactory, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.DynamoVllmWorker", - WorkerFactory, - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.get_actor_python_env", - lambda _fqn: "python", - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.PlacementGroupSchedulingStrategy", - lambda **kwargs: kwargs, - ) - - def fake_get(refs, **kwargs): - if isinstance(refs, int): - return refs - if isinstance(refs, list) and refs and "node_ip" in refs[0]: - return refs - assert pool._workers == workers - return refs - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - - pool.start() - - assert pool._metadata == [ - {"instance_id": "worker-0"}, - {"instance_id": "worker-1"}, - ] - assert [kwargs["system_port"] for kwargs in launch_kwargs] == [4001, 4002] - assert [kwargs["vllm_port"] for kwargs in launch_kwargs] == [7000, 7100] - assert [kwargs["cleanup_reservation"] for kwargs in launch_kwargs] == ( - reservation_objects - ) - assert reservation_objects[0].select_free_port.calls == [ - ( - (), - { - "port_range_low": 4000, - "port_range_high": 4100, - "excluded_ports": [], - }, - ) - ] - assert reservation_objects[1].select_free_port.calls == [ - ( - (), - { - "port_range_low": 4000, - "port_range_high": 4100, - "excluded_ports": [4001], - }, - ) - ] - - -def test_fixed_pool_shutdown_releases_workers_and_reservations(monkeypatch) -> None: - pool = object.__new__(FixedDynamoWorkerPool) - worker = _FakeWorker() - reservation = _FakeReservation() - pool._workers = [worker] - pool._reservations = [reservation] - pool._cleanup_reservations = [reservation] - pool._reservation_metadata = [] - pool._metadata = [{"instance_id": "worker-0", "process_pid": 1234}] - killed = [] - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", - lambda refs, **kwargs: [True], - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.kill", - lambda actor, **kwargs: killed.append(actor), - ) - pool.shutdown() - assert killed == [worker, reservation] - assert pool._workers == [] - assert pool._reservations == [] - - -def test_fixed_pool_shutdown_uses_registered_pid_when_worker_dies(monkeypatch) -> None: - worker = _FakeWorker() - reservation = _FakeReservation() - pool = object.__new__(FixedDynamoWorkerPool) - pool._workers = [worker] - pool._reservations = [reservation] - pool._cleanup_reservations = [reservation] - pool._reservation_metadata = [] - pool._metadata = [{}] - shutdown_ref = worker.shutdown.remote() - - def fake_get(ref, **kwargs): - if ref == shutdown_ref: - raise RuntimeError("worker died") - return True - - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.get", fake_get - ) - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.worker_pool.ray.kill", - lambda actor, **kwargs: None, - ) - - pool.shutdown() - - assert reservation.cleanup_process_group.calls == [((), {})] - - -def test_frontend_waits_for_model_after_endpoint_registration(monkeypatch) -> None: - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) - runtime._frontend_port = 3000 - runtime._frontend_process = _FakeProcess() - runtime._namespace = "nemo-rl-test" - runtime._pool = SimpleNamespace(is_alive=lambda: True) + runtime = _started(_runtime()) health = { "instances": [ { "namespace": "nemo-rl-test", "component": "backend", - "endpoint": endpoint, - "instance_id": "worker-0", + "endpoint": "generate", + "instance_id": f"engine-{idx}", + } + for idx in range(2) + ] + # Another namespace's engine does not count toward this fleet. + + [ + { + "namespace": "other", + "component": "backend", + "endpoint": "generate", + "instance_id": "foreign", } - for endpoint in ("generate", "rl") ] } responses = iter( @@ -823,44 +284,69 @@ def fake_urlopen(url, timeout): monkeypatch.setattr( "nemo_rl.models.generation.dynamo.managed_runtime.time.sleep", lambda _: None ) - runtime._wait_for_frontend(expected_workers=1) - assert urls == [ - "http://127.0.0.1:3000/health", - "http://127.0.0.1:3000/v1/models", - "http://127.0.0.1:3000/health", - "http://127.0.0.1:3000/v1/models", - ] + runtime.wait_for_engines(2, check_engines=lambda: None) + assert ( + urls + == [ + "http://127.0.0.1:3000/health", + "http://127.0.0.1:3000/v1/models", + ] + * 2 + ) -def test_frontend_wait_fails_immediately_when_worker_exits(monkeypatch) -> None: - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=_config()) - runtime._frontend_port = 3000 - runtime._frontend_process = _FakeProcess() - runtime._pool = SimpleNamespace(is_alive=lambda: False) +def test_engine_wait_fails_immediately_with_the_engines_error() -> None: + runtime = _started(_runtime()) - with pytest.raises(RuntimeError, match="worker exited while the frontend"): - runtime._wait_for_frontend(expected_workers=1) + def endpoint_stopped() -> None: + raise RuntimeError("endpoint of generation shard 0 stopped serving") + with pytest.raises(RuntimeError, match="shard 0 stopped serving"): + runtime.wait_for_engines(1, check_engines=endpoint_stopped) -def test_frontend_logs_resolved_tokenizer_environment(monkeypatch, capsys) -> None: - config = _config() - config["dynamo_cfg"]["frontend_args"].update( - {"tokenizer": "fastokens", "tokenizer_cache": True} - ) - runtime = ManagedDynamoRuntime(cluster=_Cluster(), config=config) + +def test_frontend_launches_from_the_given_interpreter(monkeypatch, capsys) -> None: + runtime = _runtime(tokenizer="fastokens", tokenizer_cache=True) runtime._frontend_port = 3000 runtime._namespace = "nemo-rl-test" - monkeypatch.setattr( - "nemo_rl.models.generation.dynamo.managed_runtime.get_dynamo_python", - lambda: "/opt/dynamo_venv/bin/python", - ) + launched = [] monkeypatch.setattr( "nemo_rl.models.generation.dynamo.managed_runtime.subprocess.Popen", - lambda *args, **kwargs: _FakeProcess(), + lambda command, **kwargs: launched.append(command) or _FakeProcess(), ) runtime._start_frontend() + assert launched[0][:3] == ["/venvs/vllm/bin/python", "-m", "dynamo.frontend"] output = capsys.readouterr().out assert "DYN_TOKENIZER': 'fastokens" in output - assert "" not in output + + +def test_shutdown_logs_the_kv_events_the_router_applied(monkeypatch, capsys) -> None: + runtime = _started(_runtime()) + metrics = ( + "# HELP dynamo_component_kv_cache_events_applied KV events applied\n" + 'dynamo_component_kv_cache_events_applied{status="ok",worker_id="1"} 40\n' + 'dynamo_component_kv_cache_events_applied{status="ok",worker_id="2"} 2\n' + 'dynamo_component_kv_cache_events_applied{status="parent_block_not_found"} 3\n' + 'dynamo_component_requests_total{status="ok"} 9\n' + ) + + class _Metrics(_FakeHttpResponse): + def read(self): + return metrics.encode() + + urls = [] + monkeypatch.setattr( + "urllib.request.urlopen", + lambda url, timeout: urls.append(url) or _Metrics(None), + ) + monkeypatch.setattr(runtime, "_stop_process", lambda *args, **kwargs: None) + + runtime.shutdown() + + assert urls == ["http://127.0.0.1:3000/metrics"] + assert ( + '[Dynamo] router applied KV events: {"ok": 42.0, "parent_block_not_found": 3.0}' + in capsys.readouterr().out + ) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py new file mode 100644 index 00000000000..f6ad9727175 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -0,0 +1,746 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NeMo RL's Dynamo engine adapter over a fake AsyncLLM. + +The host is a real ``VllmAsyncGenerationWorkerImpl`` built without its +constructor, so rollouts are sampled by the native worker's own helpers. +""" + +import asyncio +import base64 +import io +import json +from http import HTTPStatus +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from PIL import Image + +pytestmark = pytest.mark.vllm +pytest.importorskip("dynamo._core") +vllm = pytest.importorskip("vllm") + +from dynamo.llm.exceptions import HttpError, InvalidArgument # noqa: E402 +from vllm.exceptions import QueueOverflowError # noqa: E402 +from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 + +from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.dynamo.config import DynamoWorkerArgs # noqa: E402 +from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 + ROLLOUT_ANNOTATION, +) +from nemo_rl.models.generation.openai_server_utils import ( # noqa: E402 + derive_prefix_eos_count, + splice_prefix_tokens, +) +from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 + rollout_output, + rollout_result, + rollout_result_from_wire, + rollout_result_to_wire, +) +from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 + VllmAsyncGenerationWorkerImpl, +) + +MAX_MODEL_LEN = 16 + + +EOS = 2 +# The fake processor expands each image placeholder to IMAGE_TOKENS image tokens. +IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS = 90, 91, 3 + + +class _Context: + def id(self) -> str: + return "request-0" + + def is_stopped(self) -> bool: + return False + + +class _Tokenizer: + eos_token_id = EOS + + def decode(self, token_ids) -> str: + return "".join(f"<{token_id}>" for token_id in token_ids) + + +def _logprob(value: float): + return SimpleNamespace(logprob=value, rank=1, decoded_token=None) + + +def _completion(token_ids, finish_reason=None, index=0, routed_experts=None): + return SimpleNamespace( + index=index, + token_ids=token_ids, + logprobs=[{t: _logprob(-0.5)} for t in token_ids], + finish_reason=finish_reason, + routed_experts=routed_experts, + ) + + +def _output(prompt, *completions, prompt_routed_experts=None, prompt_logprobs=None): + return SimpleNamespace( + prompt_token_ids=prompt, + num_cached_tokens=0, + prompt_logprobs=prompt_logprobs, + prompt_routed_experts=prompt_routed_experts, + outputs=list(completions), + ) + + +class _FakeRenderer: + """vLLM's processing step: expands each image placeholder, as a VLM processor does.""" + + def __init__(self) -> None: + self.prompts = [] + + async def process_for_engine_async(self, prompt, arrival_time, *, skip_mm_cache): + self.prompts.append((prompt, skip_mm_cache)) + if "multi_modal_data" not in prompt: + return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} + tokens, placeholders = [], [] + for token in prompt["prompt_token_ids"]: + if token == IMAGE_PLACEHOLDER: + placeholders.append(PlaceholderRange(len(tokens), IMAGE_TOKENS)) + tokens.extend([IMAGE_TOKEN] * IMAGE_TOKENS) + else: + tokens.append(token) + return { + "type": "multimodal", + "prompt_token_ids": tokens, + "mm_kwargs": {"image": [object() for _ in placeholders]}, + "mm_placeholders": {"image": placeholders}, + } + + +class _FakeLLM: + """Yields scripted RequestOutputs and records what it was asked to generate.""" + + def __init__(self, outputs=(), *, defaults=None, admission_error=None): + self.outputs = list(outputs) + self.calls = [] + self.admission_error = admission_error + self.tokenizer = None + self.renderer = _FakeRenderer() + self.model_config = SimpleNamespace( + get_diff_sampling_param=lambda: dict(defaults or {}), + multimodal_config=None, + allowed_local_media_path="", + allowed_media_domains=None, + ) + self.vllm_config = SimpleNamespace( + model_config=SimpleNamespace(max_model_len=MAX_MODEL_LEN), + kv_events_config=None, + ) + + def check_admission(self, n, request_id): + if self.admission_error is not None: + raise self.admission_error + + async def generate(self, prompt, sampling_params, request_id): + self.calls.append((prompt, sampling_params, request_id)) + for output in self.outputs: + yield output + + +def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: + host = object.__new__(VllmAsyncGenerationWorkerImpl) + host.cfg = { + "max_new_tokens": 8, + "temperature": 0.7, + "top_p": 0.9, + "val_temperature": 0.0, + "val_top_p": 1.0, + "top_k": None, + "stop_token_ids": [2], + "stop_strings": [""], + "vllm_cfg": {"max_model_len": MAX_MODEL_LEN}, + **cfg, + } + host.SamplingParams = vllm.SamplingParams + host.model_name = "model" + host.routed_experts_dtype = torch.int16 + host.llm = llm + host._capture_media = False + return host + + +def _collect(engine, request) -> list[dict]: + async def run(): + return [chunk async for chunk in engine.generate(request, _Context())] + + return asyncio.run(run()) + + +def _rollout_request(prompt: list[int], **rollout) -> dict: + spec = {"greedy": False, "stop_strings": None, **rollout} + return {"token_ids": prompt, "annotations": [ROLLOUT_ANNOTATION + json.dumps(spec)]} + + +def _openai_request(**fields) -> dict: + return {"token_ids": [1, 2, 3, 4], **fields} + + +def test_rollouts_are_served_like_generate_async() -> None: + final = _output([1, 2, 3], _completion([10, 11, 12], "stop")) + llm = _FakeLLM([final]) + host = _host(llm) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=3) + + (chunk,) = _collect( + engine, _rollout_request([1, 2, 3], greedy=True, stop_strings=[""]) + ) + + ((_, sampling_params, _),) = llm.calls + # The request the native worker's generate_async builds... + assert sampling_params == host._rollout_sampling_params( + max_new_tokens=host._rollout_max_new_tokens(3), + greedy=True, + stop_strings=[""], + ) + assert sorted(sampling_params.stop) == ["", ""] + # ...and the result the shared pipeline builds from its final output. + expected = rollout_result( + final, + input_length=3, + return_routed_experts=False, + routed_experts_dtype=torch.int16, + device=torch.device("cpu"), + ) + assert chunk["token_ids"] == [10, 11, 12] + assert chunk["engine_data"] == { + "rollout": rollout_result_to_wire(expected), + "gen_leader_worker_idx": 3, + } + + +def test_router_replay_routes_reach_the_driver_as_generate_async_returns_them() -> None: + routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) + final = _output( + [1, 2, 3], + _completion([10, 11, 12], "stop", routed_experts=routes[3:]), + prompt_routed_experts=routes[:3], + ) + host = _host(_FakeLLM([final]), vllm_kwargs={"enable_return_routed_experts": True}) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect(engine, _rollout_request([1, 2, 3])) + + input_ids = torch.tensor([1, 2, 3]) + shipped = rollout_output( + rollout_result_from_wire(chunk["engine_data"]["rollout"]), + input_ids=input_ids, + input_length=3, + pad_token_id=0, + ) + native = rollout_output( + rollout_result( + final, + input_length=3, + return_routed_experts=True, + routed_experts_dtype=torch.int16, + device=torch.device("cpu"), + ), + input_ids=input_ids, + input_length=3, + pad_token_id=0, + ) + assert shipped.keys() == native.keys() + assert "routed_experts" in shipped + for key in native: + assert torch.equal(shipped[key], native[key]), key + + +def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect(engine, _rollout_request(list(range(MAX_MODEL_LEN)))) + + assert llm.calls == [] + rollout = chunk["engine_data"]["rollout"] + assert rollout["token_ids"] == [] and rollout["truncated"] is False + + +def test_openai_requests_map_onto_sampling_params_as_vllms_server_does() -> None: + params = native_engine.build_sampling_params( + _openai_request( + sampling_options={"top_k": 5, "seed": 7, "n": 2}, + stop_conditions={ + "max_tokens": 100, + "stop_token_ids": [9], + "stop_token_ids_hidden": [2], + }, + output_options={"logprobs": 2}, + extra_args={ + "sampling_options": { + "allowed_token_ids": [5, 6], + "bad_words_token_ids": [[7]], + } + }, + ), + # The model's generation config, for what the request leaves unset. + {"temperature": 0.6, "top_k": 20, "max_tokens": 9}, + output_room=MAX_MODEL_LEN - 4, + ) + + assert (params.temperature, params.top_k, params.seed, params.n) == (0.6, 5, 7, 2) + assert params.max_tokens == 9 + assert sorted(params.stop_token_ids) == [2, 9] + assert params.logprobs == 2 + assert params.allowed_token_ids == [5, 6] + assert params._bad_words_token_ids == [[7]] + assert params.detokenize is False + + +def test_guided_decoding_maps_onto_structured_outputs() -> None: + params = native_engine.build_sampling_params( + _openai_request( + sampling_options={"guided_decoding": {"json": {"type": "object"}}} + ), + {}, + output_room=MAX_MODEL_LEN - 4, + ) + + assert params.structured_outputs.json == {"type": "object"} + + +def test_best_of_is_rejected_as_by_vllm() -> None: + with pytest.raises(InvalidArgument, match="best_of"): + native_engine.build_sampling_params( + _openai_request(sampling_options={"n": 1, "best_of": 2}), + {}, + output_room=MAX_MODEL_LEN - 4, + ) + + +def test_openai_requests_stream_each_choice_by_index() -> None: + prompt = [1, 2, 3, 4] + llm = _FakeLLM( + [ + _output(prompt, _completion([5], index=0), _completion([7], index=1)), + _output(prompt, _completion([6], "stop", index=0)), + _output(prompt, _completion([8], "length", index=1)), + ] + ) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + chunks = _collect(engine, _openai_request(sampling_options={"n": 2})) + + assert [(c["index"], c["token_ids"]) for c in chunks] == [ + (0, [5]), + (1, [7]), + (0, [6]), + (1, [8]), + ] + assert [c.get("finish_reason") for c in chunks] == [None, None, "stop", "length"] + assert chunks[-1]["completion_usage"]["completion_tokens"] == 4 + + +def test_prompt_logprobs_are_returned_in_engine_data() -> None: + prompt = [1, 2, 3, 4] + llm = _FakeLLM( + [ + # Under DELTA, vLLM sends prompt logprobs once, at the end of prefill, + # and an empty list on every later output. + _output( + prompt, + _completion([5]), + prompt_logprobs=[None, {2: _logprob(-0.25)}, None, None], + ), + _output(prompt, _completion([6], "stop"), prompt_logprobs=[]), + ] + ) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + chunks = _collect(engine, _openai_request(output_options={"prompt_logprobs": 0})) + + assert chunks[-1]["engine_data"]["prompt_logprobs"][1]["2"]["logprob"] == -0.25 + + +def test_queue_limits_reject_with_the_engines_status() -> None: + llm = _FakeLLM(admission_error=QueueOverflowError()) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(HttpError) as raised: + _collect(engine, _openai_request()) + + assert raised.value.code == HTTPStatus.SERVICE_UNAVAILABLE + assert llm.calls == [] + + +def _start_worker_config(monkeypatch, host) -> SimpleNamespace: + """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" + configs = [] + + class _Worker: + def __init__(self, engine, config): + configs.append(config) + + async def run(self): + return None + + monkeypatch.setattr(native_engine, "Worker", _Worker) + monkeypatch.setattr(native_engine, "WorkerConfig", SimpleNamespace) + env = { + "DYN_DISCOVERY_BACKEND": "etcd", + "DYN_REQUEST_PLANE": "tcp", + "DYN_EVENT_PLANE": "nats", + } + for key, value in env.items(): + # Registered here so the variables the engine exports are restored. + monkeypatch.setenv(key, value) + + async def start(): + await native_engine.start_dynamo_endpoint( + host, namespace="ns", engine_env=env, leader_worker_idx=0 + ) + + asyncio.run(start()) + (config,) = configs + return config + + +def test_every_worker_arg_reaches_the_dynamo_worker_config( + monkeypatch, tmp_path +) -> None: + template = tmp_path / "chat.jinja" + template.write_text("{{ messages }}") + worker_args = { + "tool_call_parser": "qwen3_coder", + "reasoning_parser": "nemotron_nano", + "exclude_tools_when_tool_choice_none": True, + "enable_structural_tag": True, + "structural_tag_scope": "always", + "structural_tag_schema": "strict", + "custom_jinja_template": str(template), + "endpoint_types": ["chat"], + } + assert set(worker_args) == set(DynamoWorkerArgs.model_fields) + host = _host(_FakeLLM(), dynamo_cfg={"worker_args": worker_args}) + + config = _start_worker_config(monkeypatch, host) + + assert config.namespace == "ns" + assert config.endpoint_types == "chat" + assert config.structural_tag_mode == "on" + with open(config.custom_jinja_template) as template_file: + assert template_file.read() == "{{ messages }}" + for key in worker_args.keys() - { + "endpoint_types", + "enable_structural_tag", + "custom_jinja_template", + }: + assert getattr(config, key) == worker_args[key], key + + +def test_the_native_servers_chat_settings_reach_the_frontend(monkeypatch) -> None: + host = _host(_FakeLLM()) + host.cfg["vllm_cfg"] = { + "max_model_len": MAX_MODEL_LEN, + "reasoning_parser_plugin": "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", + "http_server_serving_chat_kwargs": { + "tool_parser": "qwen3_coder", + "reasoning_parser": "nano_v3", + # Literal Jinja, which the native server also accepts. + "chat_template": "{{ messages }}\n", + }, + } + + config = _start_worker_config(monkeypatch, host) + + assert (config.tool_call_parser, config.reasoning_parser) == ( + "qwen3_coder", + "nemotron_v3", + ) + with open(config.custom_jinja_template) as template_file: + assert template_file.read() == "{{ messages }}\n" + + +def test_a_parser_dynamo_lacks_fails_registration(monkeypatch) -> None: + host = _host( + _FakeLLM(), dynamo_cfg={"worker_args": {"tool_call_parser": "no_such_parser"}} + ) + + with pytest.raises(ValueError, match="no tool call parser named 'no_such_parser'"): + _start_worker_config(monkeypatch, host) + + +def test_an_endpoint_that_stops_serving_reports_why() -> None: + async def endpoint_errors() -> list: + async def registration_fails() -> None: + raise RuntimeError("etcd unreachable") + + host = SimpleNamespace(_dynamo_endpoint_task=None) + errors = [await VllmAsyncGenerationWorkerImpl.dynamo_endpoint_error(host)] + host._dynamo_endpoint_task = asyncio.get_running_loop().create_task( + registration_fails() + ) + await asyncio.sleep(0) + errors.append(await VllmAsyncGenerationWorkerImpl.dynamo_endpoint_error(host)) + return errors + + # Not registered yet, then the driver's registration wait gets the cause. + assert asyncio.run(endpoint_errors()) == [ + None, + "RuntimeError('etcd unreachable')", + ] + + +def test_a_prompt_that_fills_the_context_is_refused_as_natively() -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match="maximum context length"): + _collect(engine, _openai_request(token_ids=list(range(MAX_MODEL_LEN)))) + + assert llm.calls == [] + + +def _image_url() -> str: + buffer = io.BytesIO() + Image.new("RGB", (8, 8), color="red").save(buffer, format="PNG") + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + + +def test_media_is_loaded_and_processed_as_vllms_chat_parser_does() -> None: + final = _output([1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3], _completion([5], "stop")) + llm = _FakeLLM([final]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + _collect( + engine, + _openai_request( + token_ids=[1, IMAGE_PLACEHOLDER, 3], + multi_modal_data={"image_url": [{"Url": _image_url()}]}, + mm_processor_kwargs={"max_num_tiles": 2}, + ), + ) + + ((prompt, skip_mm_cache),) = llm.renderer.prompts + # The frontend's unexpanded tokens, the decoded image and the processor kwargs. + assert prompt["prompt_token_ids"] == [1, IMAGE_PLACEHOLDER, 3] + (image,) = prompt["multi_modal_data"]["image"] + assert image.size == (8, 8) and image.mode == "RGB" + assert prompt["multi_modal_uuids"] == {"image": [None]} + assert prompt["mm_processor_kwargs"] == {"max_num_tiles": 2} + assert skip_mm_cache is False + # vLLM generates from the processed prompt, and the output cap counts its tokens. + ((engine_input, params, _),) = llm.calls + assert engine_input["prompt_token_ids"] == [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + assert params.max_tokens == MAX_MODEL_LEN - 2 - IMAGE_TOKENS + + +@pytest.mark.parametrize( + ("fields", "error"), + [ + ({"prompt_embeds": "AAAA"}, "Embedding prompts"), + ({"multi_modal_data": {"image_url": [{"UuidOnly": "u"}]}}, "carry their URL"), + ( + { + "multi_modal_data": { + "image_url": [{"Url": "data:image/png;base64,eA=="}] + } + }, + "Cannot load image_url", + ), + ], +) +def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match=error): + _collect(engine, _openai_request(**fields)) + + assert llm.calls == [] + + +# --- NeMo-Gym's Dynamo dialect ------------------------------------------------- + + +def _f32(value: float) -> float: + return float(np.float32(value)) + + +def _gym_request( + *, temperature=0.7, top_p=0.9, prefix=None, sampling=None, **fields +) -> dict: + annotations = [native_engine.GYM_ROLLOUT_ANNOTATION] + if prefix is not None: + annotations.insert( + 0, native_engine.GYM_PREFIX_ANNOTATION_PREFIX + json.dumps(prefix) + ) + sampling_options = {"temperature": temperature, "top_p": top_p, **(sampling or {})} + return { + "token_ids": [1, 3, 4], + # The frontend carries sampling floats as f32. + "sampling_options": { + key: _f32(value) if isinstance(value, float) else value + for key, value in sampling_options.items() + }, + # Gym's token information: logprobs=true, top_logprobs=0. + "output_options": {"logprobs": 0}, + "annotations": annotations, + **fields, + } + + +def test_gym_calls_sample_with_nemo_rls_exact_profile() -> None: + final = _output([1, 3, 4], _completion([10, 11], "stop")) + llm = _FakeLLM([final]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect( + engine, _gym_request(stop_conditions={"max_tokens": 100, "stop": [""]}) + ) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == [1, 3, 4] + # The configured values, not the f32 values the frontend delivered. + assert (params.temperature, params.top_p) == (0.7, 0.9) + assert params.top_k == vllm.SamplingParams(top_k=-1).top_k + assert params.output_kind == vllm.sampling_params.RequestOutputKind.FINAL_ONLY + # Output fits the context, and vLLM stops on the stop strings itself. + assert params.max_tokens == MAX_MODEL_LEN - 3 + assert (params.stop, params.detokenize) == ([""], True) + assert chunk["engine_data"] == { + "prompt_token_ids": [1, 3, 4], + "completion_token_ids": [10, 11], + "completion_logprobs": [-0.5, -0.5], + } + assert (chunk["token_ids"], chunk["finish_reason"]) == ([10, 11], "stop") + assert chunk["log_probs"] == [-0.5, -0.5] + + +def test_gym_validation_calls_sample_with_the_validation_profile() -> None: + llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + _collect(engine, _gym_request(temperature=0.0, top_p=1.0)) + + ((_, params, _),) = llm.calls + assert (params.temperature, params.top_p) == (0.0, 1.0) + + +@pytest.mark.parametrize( + ("request_fields", "error"), + [ + ({"temperature": 0.5}, "matches neither"), + ({"top_p": None}, "top_p must be set explicitly"), + ({"sampling": {"top_k": 5}}, "Top k sampling parameter"), + ({"sampling": {"n": 2}}, "take n=1"), + ], +) +def test_gym_calls_that_would_sample_off_policy_are_refused( + request_fields, error +) -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match=error): + _collect(engine, _gym_request(**request_fields)) + + assert llm.calls == [] + + +def test_gym_echoed_prefix_replaces_the_rendered_history() -> None: + # The model generated [5, 6, EOS]; the frontend re-rendered that turn as [7, EOS]. + prefix, rendered = [1, 5, 6, EOS], [1, 7, EOS, 3, 4] + spliced = splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=prefix, + template_token_ids=rendered, + template_prefix_eos_count=derive_prefix_eos_count(prefix, eos_token_id=EOS), + eos_token_id=EOS, + ).token_ids + assert spliced != rendered + llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect(engine, _gym_request(prefix=prefix, token_ids=rendered)) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == spliced + assert params.max_tokens == MAX_MODEL_LEN - len(spliced) + assert chunk["engine_data"]["prompt_token_ids"] == spliced + + +def test_gym_calls_whose_splice_fills_the_context_are_refused() -> None: + prefix = [1, *range(20, 20 + MAX_MODEL_LEN), EOS] + llm = _FakeLLM() + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + with pytest.raises(InvalidArgument, match="maximum context length"): + _collect(engine, _gym_request(prefix=prefix, token_ids=[1, 7, EOS, 3])) + + assert llm.calls == [] + + +def test_gym_echoed_prefix_keeps_media_placeholders_on_their_tokens() -> None: + # Turn 1 ran [1, , 3] and generated [5, 6, EOS]; the frontend + # re-rendered that turn as [7, EOS] and appended a turn with a second image. + turn1 = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + prefix = [*turn1, 5, 6, EOS] + rendered = [1, IMAGE_PLACEHOLDER, 3, 7, EOS, IMAGE_PLACEHOLDER, 4] + spliced = [*prefix, *[IMAGE_TOKEN] * IMAGE_TOKENS, 4] + llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) + llm.tokenizer = _Tokenizer() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + (chunk,) = _collect( + engine, + _gym_request( + prefix=prefix, + token_ids=rendered, + multi_modal_data={"image_url": [{"Url": _image_url()}] * 2}, + ), + ) + + ((engine_input, params, _),) = llm.calls + assert engine_input["prompt_token_ids"] == spliced + # The first image sits in the model's own prefix; the second moved with the + # longer generated turn, as the native endpoint remaps them. + assert [ + (span.offset, span.length) for span in engine_input["mm_placeholders"]["image"] + ] == [(1, IMAGE_TOKENS), (len(prefix), IMAGE_TOKENS)] + assert params.max_tokens == MAX_MODEL_LEN - len(spliced) + assert chunk["engine_data"]["prompt_token_ids"] == spliced + + +@pytest.mark.parametrize( + ("router_mode", "prefix_caching", "publishes"), + [("kv", True, True), ("kv", False, False), ("round-robin", True, False)], +) +def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( + router_mode, prefix_caching, publishes +) -> None: + cfg = native_engine.DynamoCfg.model_validate( + {"frontend_args": {"router_mode": router_mode}} + ) + + config = native_engine.dynamo_kv_events_config(cfg, prefix_caching=prefix_caching) + + # vLLM publishes block events only for blocks its prefix cache stores. + assert (config is not None) is publishes + if publishes: + assert config["enable_kv_cache_events"] and config["publisher"] == "zmq" + # vLLM's publisher binds only a wildcard endpoint; Dynamo's source connects. + assert config["endpoint"].startswith("tcp://*:") diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py new file mode 100644 index 00000000000..bb1a00de798 --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -0,0 +1,232 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NeMo-Gym token capture through the Dynamo engine adapter. + +The worker's own capture lifecycle stages each call, as on the native chat +endpoint; only Gym's capture core is faked. +""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +pytestmark = [pytest.mark.nemo_gym, pytest.mark.vllm] +pytest.importorskip("dynamo._core") +pytest.importorskip("vllm") + +from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 +from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 + DYNAMO_CAPTURE_ANNOTATION_PREFIX, + DYNAMO_GYM_REQUEST_ANNOTATION, + DYNAMO_PREFIX_ANNOTATION_PREFIX, +) +from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 + +from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 +from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 +from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 +from nemo_rl.utils.routed_experts_codec import decode_routed_experts # noqa: E402 +from tests.unit.models.generation.test_dynamo_native_engine import ( # noqa: E402 + EOS, + IMAGE_PLACEHOLDER, + IMAGE_TOKEN, + IMAGE_TOKENS, + _image_url, + _collect, + _completion, + _FakeLLM, + _gym_request, + _host, + _output, + _Tokenizer, +) + +COORDS = {"staging_key": "rollout-1/c2"} + + +class _Capture: + """Gym's capture core: records begin/complete/fail.""" + + adapter = VLLMCaptureAdapter() + + def __init__(self) -> None: + self.begun: list[tuple] = [] + self.staged: list[dict] = [] + self.failed: list[str] = [] + + def begin_call(self, admission, *, prefix_token_ids, stream): + self.begun.append((admission, prefix_token_ids, stream)) + return SimpleNamespace(admission=admission) + + def complete_call_from_response(self, call, payload, attachments=None): + self.staged.append(payload) + return SimpleNamespace(model_dump=lambda: dict(COORDS)) + + def fail_call(self, call, *, reason) -> None: + self.failed.append(reason) + + +def _capturing_host(llm, **cfg): + host = _host(llm, **cfg) + host.token_capture = _Capture() + host._capture_calls = {} + host._chain_prefix = ChainPrefixCache() + return host + + +def _captured_request(admission: CaptureAdmission, **fields) -> dict: + request = _gym_request(**fields) + # Gym's dynamo_worker handler: no logprobs, the admission in an annotation. + request["output_options"] = {} + request["annotations"] = [ + DYNAMO_CAPTURE_ANNOTATION_PREFIX + admission.model_dump_json(), + *request["annotations"], + ] + return request + + +def test_annotations_are_the_ones_gyms_dynamo_dialect_sends() -> None: + assert native_engine.GYM_ROLLOUT_ANNOTATION == DYNAMO_GYM_REQUEST_ANNOTATION + assert native_engine.GYM_CAPTURE_ANNOTATION_PREFIX == ( + DYNAMO_CAPTURE_ANNOTATION_PREFIX + ) + assert native_engine.GYM_PREFIX_ANNOTATION_PREFIX == ( + DYNAMO_PREFIX_ANNOTATION_PREFIX + ) + + +def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: + # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. + prefix = [1, 5, 6, EOS] + admission = CaptureAdmission( + rollout_id="rollout-1", + model_call_id="c2", + parent_call_id="c1", + prev_len=len(prefix), + mode="token_in", + required_prefix_token_ids=prefix, + parent_chain_hash="0" * 64, + ) + spliced = [*prefix, 3, 4] + routes = np.arange(8 * 2 * 2, dtype=np.int16).reshape(8, 2, 2) + final = _output( + spliced, + _completion([10, 11], "stop", routed_experts=routes[6:]), + prompt_routed_experts=routes[:6], + ) + llm = _FakeLLM([final]) + llm.tokenizer = _Tokenizer() + host = _capturing_host(llm, vllm_kwargs={"enable_return_routed_experts": True}) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect( + engine, _captured_request(admission, token_ids=[1, 7, EOS, 3, 4]) + ) + + ((prompt, params, _),) = llm.calls + assert prompt["prompt_token_ids"] == spliced + # The sampled tokens' log probabilities are staged; Gym asked for none. + assert params.logprobs == 0 + ((begun_admission, begun_prefix, stream),) = host.token_capture.begun + assert (begun_admission, begun_prefix, stream) == (admission, prefix, False) + (payload,) = host.token_capture.staged + assert payload["prompt_token_ids"] == spliced + message = payload["choices"][0]["message"] + assert message["generation_token_ids"] == [10, 11] + assert message["generation_log_probs"] == [-0.5, -0.5] + # As the native endpoint stages them: the full sequence's routes (none for + # the last token) from the boundary row, the prefix's last token, on. + staged_routes = decode_routed_experts(message["routed_experts"], torch.int16) + expected = torch.full((5, 2, 2), R3_MISSING_ROUTE_SENTINEL, dtype=torch.int16) + expected[:4] = torch.from_numpy(routes[3:7]) + assert torch.equal(staged_routes, expected) + # Only the commit coordinates go back through the frontend. + assert chunk["engine_data"] == {"ng_commit_coords": COORDS} + assert "log_probs" not in chunk + assert host._capture_calls == {} + + +def test_a_call_that_fails_after_admission_fails_its_capture() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + + class _FailingLLM(_FakeLLM): + async def generate(self, prompt, sampling_params, request_id): + raise RuntimeError("engine died") + yield # pragma: no cover + + host = _capturing_host(_FailingLLM()) + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + with pytest.raises(RuntimeError, match="engine died"): + _collect(engine, _captured_request(admission)) + + assert host.token_capture.failed == ["engine_error"] + assert host._capture_calls == {} + + +def test_without_capture_installed_a_captured_call_is_served_uncaptured() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) + host = _host(llm) + host.token_capture = None + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect(engine, _captured_request(admission)) + + assert json.loads(json.dumps(chunk["engine_data"]))["completion_token_ids"] == [10] + + +def test_a_captured_media_call_stages_the_media_the_engine_ran_on() -> None: + admission = CaptureAdmission( + rollout_id="rollout-1", model_call_id="c1", mode="text" + ) + processed = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] + llm = _FakeLLM([_output(processed, _completion([10], "stop"))]) + host = _capturing_host(llm) + host._capture_media = True + captured = CapturedMedia((), None) + snapshots = [] + + def capture_request_media(engine_input, *, admission, splice): + snapshots.append((list(engine_input["prompt_token_ids"]), admission, splice)) + return captured + + host._capture_request_media = capture_request_media + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + (chunk,) = _collect( + engine, + _captured_request( + admission, + token_ids=[1, IMAGE_PLACEHOLDER, 3], + multi_modal_data={"image_url": [{"Url": _image_url()}]}, + ), + ) + + # The processor's own pixels, not cache references, as natively (#4124). + ((_, skip_mm_cache),) = llm.renderer.prompts + assert skip_mm_cache is True + # Media is snapshotted from the processed prompt the engine runs. + assert snapshots == [(processed, admission, None)] + (payload,) = host.token_capture.staged + assert payload["prompt_token_ids"] == processed + assert chunk["engine_data"] == {"ng_commit_coords": COORDS} diff --git a/tests/unit/models/generation/test_dynamo_rollout_client.py b/tests/unit/models/generation/test_dynamo_rollout_client.py new file mode 100644 index 00000000000..ac072ad72ad --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_rollout_client.py @@ -0,0 +1,67 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +import pytest + +from nemo_rl.models.generation.dynamo.rollout_client import ( + ROLLOUT_ANNOTATION, + parse_rollout_response, + rollout_payload, +) + + +def test_rollout_payload_leaves_sampling_and_stopping_to_the_engine() -> None: + payload = rollout_payload( + model_name="model", + prompt_token_ids=[5, 6, 7], + max_model_len=10, + greedy=True, + stop_strings=[""], + ) + + assert payload["prompt"] == [5, 6, 7] + # The frontend's own stop rules must never end the stream first. + assert payload["ignore_eos"] is True + assert payload["max_tokens"] == 7 + assert "stop" not in payload and "temperature" not in payload + (annotation,) = payload["nvext"]["annotations"] + assert annotation.startswith(ROLLOUT_ANNOTATION) + assert json.loads(annotation[len(ROLLOUT_ANNOTATION) :]) == { + "greedy": True, + "stop_strings": [""], + } + assert payload["nvext"]["extra_fields"] == ["engine_data"] + + +@pytest.mark.parametrize( + "response, match", + [ + ({"status": "error", "http_status": 400, "raw": "bad"}, "HTTP 400"), + ({"nvext": {}}, "no rollout engine_data"), + ({"nvext": {"engine_data": {"prompt_logprobs": []}}}, "no rollout engine_data"), + ], +) +def test_parse_rollout_response_rejects_unusable_responses(response, match) -> None: + with pytest.raises(RuntimeError, match=match): + parse_rollout_response(response, url="http://frontend/v1/completions") + + +def test_parse_rollout_response_returns_engine_data() -> None: + engine_data = {"rollout": {"token_ids": []}, "gen_leader_worker_idx": 1} + + assert ( + parse_rollout_response({"nvext": {"engine_data": engine_data}}, url="u") + == engine_data + ) diff --git a/tests/unit/models/generation/test_dynamo_token_wrapper.py b/tests/unit/models/generation/test_dynamo_token_wrapper.py deleted file mode 100644 index 53eb020bc2b..00000000000 --- a/tests/unit/models/generation/test_dynamo_token_wrapper.py +++ /dev/null @@ -1,646 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -import json - -import pytest -from transformers import AutoTokenizer - -from nemo_rl.models.generation.dynamo.token_wrapper import ( - DynamoTokenWrapperServer, - _inject_gym_token_metadata, - _validate_engine_data, - prepare_dynamo_chat_completion_request, -) - - -class _Tokenizer: - eos_token_id = 2 - eos_token = "" - - def __init__(self) -> None: - self.calls = [] - - def decode(self, token_ids): - return repr(token_ids) - - def encode(self, text, add_special_tokens=False): - assert add_special_tokens is False - token_ids = [] - while text: - if text.startswith(self.eos_token): - token_ids.append(self.eos_token_id) - text = text[len(self.eos_token) :] - elif text.startswith("next"): - token_ids.append(40) - text = text[len("next") :] - elif text.startswith("GEN"): - token_ids.append(99) - text = text[len("GEN") :] - else: - raise AssertionError(f"Unexpected text to encode: {text!r}") - return token_ids - - def apply_chat_template( - self, - conversation, - tools=None, - documents=None, - chat_template=None, - add_generation_prompt=False, - continue_final_message=False, - tokenize=True, - return_tensors=None, - return_dict=False, - **kwargs, - ): - self.calls.append( - { - "tools": tools, - "documents": documents, - "chat_template": chat_template, - "add_generation_prompt": add_generation_prompt, - "continue_final_message": continue_final_message, - "tokenize": tokenize, - "return_tensors": return_tensors, - "return_dict": return_dict, - "kwargs": kwargs, - } - ) - token_ids = [] - rendered = "" - for index, message in enumerate(conversation): - role = message["role"] - content = message.get("content") - for tool_call in message.get("tool_calls", []): - function = tool_call.get("function", tool_call) - arguments = function.get("arguments") - if arguments is not None and not isinstance(arguments, dict): - raise TypeError("Can only get item pairs from a mapping.") - if role == "user" and content == "hello": - token_ids.extend([10]) - rendered += "hello" - elif role == "assistant" and content == "first": - # Model the Nemotron template's context-dependent history - # truncation: the assistant is longer when rendered alone than - # when a later user turn is present. - has_later_user = any( - item["role"] == "user" for item in conversation[index + 1 :] - ) - token_ids.extend( - [3, self.eos_token_id] - if has_later_user - else [300, 301, 302, self.eos_token_id] - ) - rendered += f"first{self.eos_token}" - elif role == "user" and content == "next": - token_ids.extend([40]) - rendered += "next" - elif role == "assistant" and isinstance(content, str): - token_ids.extend([777, self.eos_token_id]) - rendered += f"{content}{self.eos_token}" - else: - token_ids.extend([900]) - rendered += "other" - if add_generation_prompt: - token_ids.extend([99]) - rendered += "GEN" - return token_ids if tokenize else rendered - - -TOOLS = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather for a city.", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - }, - } -] - - -def _tool_conversation() -> list[dict]: - return [ - {"role": "user", "content": "What is the weather in San Francisco?"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"city":"San Francisco"}', - }, - } - ], - }, - {"role": "tool", "content": "Sunny, 20C"}, - {"role": "user", "content": "What about tomorrow?"}, - ] - - -def test_prepare_dynamo_chat_completion_request_first_turn() -> None: - tokenizer = _Tokenizer() - body = { - "model": "dummy-model", - "messages": [{"role": "user", "content": "hello"}], - "nvext": {"extra_fields": ["timing"], "trace": "keep-me"}, - "chat_template_kwargs": { - "enable_thinking": False, - "force_nonempty_content": True, - }, - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs={"enable_thinking": True}, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["messages"] == [{"role": "user", "content": "hello"}] - assert "logprobs" not in prepared - assert "return_tokens_as_token_ids" not in prepared - assert prepared["chat_template_kwargs"] == { - "enable_thinking": False, - "force_nonempty_content": True, - } - assert prepared["nvext"] == { - "extra_fields": ["timing", "engine_data"], - "trace": "keep-me", - "token_data": [10, 99], - } - assert tokenizer.calls[0]["kwargs"] == { - "enable_thinking": False, - "force_nonempty_content": True, - } - - -def test_public_qwen3_tokenizer_first_turn_matches_chat_template( - tiny_qwen3_model_path, -) -> None: - tokenizer = AutoTokenizer.from_pretrained(tiny_qwen3_model_path) - messages = [{"role": "user", "content": "Solve 2 + 2."}] - template_kwargs = {"enable_thinking": False} - expected = tokenizer.apply_chat_template( - messages, - tools=None, - documents=None, - chat_template=None, - add_generation_prompt=True, - continue_final_message=False, - tokenize=True, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - - prepared = prepare_dynamo_chat_completion_request( - {"model": "Qwen/Qwen3-0.6B", "messages": messages}, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs=template_kwargs, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == expected - - -def test_public_qwen3_tool_conversation_matches_plain_chat_template( - tiny_qwen3_model_path, -) -> None: - tokenizer = AutoTokenizer.from_pretrained(tiny_qwen3_model_path) - messages = _tool_conversation() - template_kwargs = {"enable_thinking": False} - expected = tokenizer.apply_chat_template( - messages, - tools=TOOLS, - tokenize=True, - add_generation_prompt=True, - continue_final_message=False, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - - prepared = prepare_dynamo_chat_completion_request( - {"model": "Qwen/Qwen3-0.6B", "messages": messages, "tools": TOOLS}, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs=template_kwargs, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == expected - - -def test_public_qwen3_multiturn_prefix_splice_parity( - tiny_qwen3_model_path, -) -> None: - tokenizer = AutoTokenizer.from_pretrained(tiny_qwen3_model_path) - messages = _tool_conversation() - template_kwargs = {"enable_thinking": False} - expected = tokenizer.apply_chat_template( - messages, - tools=TOOLS, - tokenize=True, - add_generation_prompt=True, - continue_final_message=False, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - assistant_prefix = tokenizer.apply_chat_template( - messages[:2], - tools=TOOLS, - tokenize=True, - add_generation_prompt=False, - continue_final_message=False, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - first_prompt = tokenizer.apply_chat_template( - messages[:1], - tools=TOOLS, - tokenize=True, - add_generation_prompt=True, - continue_final_message=False, - return_tensors=None, - return_dict=False, - **template_kwargs, - ) - generation_token_ids = assistant_prefix[len(first_prompt) : -1] - messages[1]["prompt_token_ids"] = first_prompt - messages[1]["generation_token_ids"] = generation_token_ids - messages[1]["generation_log_probs"] = [-0.1] * len(generation_token_ids) - - assistant_eos_index = [ - index - for index, token_id in enumerate(expected) - if token_id == tokenizer.eos_token_id - ][2] - expected_splice = assistant_prefix[:-2] + expected[assistant_eos_index:] - - prepared = prepare_dynamo_chat_completion_request( - {"model": "Qwen/Qwen3-0.6B", "messages": messages, "tools": TOOLS}, - tokenizer=tokenizer, - tokenizer_chat_template_kwargs=template_kwargs, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == expected_splice - - -def test_prepare_dynamo_chat_completion_request_preserves_logprob_fields() -> None: - tokenizer = _Tokenizer() - body = { - "model": "dummy-model", - "messages": [{"role": "user", "content": "hello"}], - "logprobs": True, - "return_tokens_as_token_ids": True, - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["logprobs"] is True - assert prepared["return_tokens_as_token_ids"] is True - assert prepared["nvext"] == { - "extra_fields": ["engine_data"], - "token_data": [10, 99], - } - - -def test_prepare_dynamo_chat_completion_request_preserves_prior_prefix() -> None: - tokenizer = _Tokenizer() - body = { - "model": "dummy-model", - "required_prefix_token_ids": [999], - "messages": [ - {"role": "user", "content": "hello"}, - { - "role": "assistant", - "content": "first", - "prompt_token_ids": [10], - "generation_token_ids": [31, 32, 2], - "generation_log_probs": [-0.1, -0.2, -0.3], - }, - {"role": "user", "content": "next"}, - ], - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == [10, 31, 32, 2, 40, 99] - assert "required_prefix_token_ids" not in prepared - assert "prompt_token_ids" not in prepared["messages"][1] - assert "generation_token_ids" not in prepared["messages"][1] - assert "generation_log_probs" not in prepared["messages"][1] - assert tokenizer.calls[0]["add_generation_prompt"] is True - assert tokenizer.calls[1]["add_generation_prompt"] is False - assert tokenizer.calls[0]["tokenize"] is True - assert tokenizer.calls[1]["tokenize"] is True - - -def test_prepare_dynamo_chat_completion_request_validates_extra_fields() -> None: - body = { - "messages": [{"role": "user", "content": "hello"}], - "nvext": {"extra_fields": ["engine_data", "timing", "engine_data"]}, - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=_Tokenizer(), - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["extra_fields"] == ["engine_data", "timing"] - - body["nvext"]["extra_fields"] = "timing" - with pytest.raises(ValueError, match="extra_fields must be a JSON list"): - prepare_dynamo_chat_completion_request( - body, - tokenizer=_Tokenizer(), - exclude_tools_when_tool_choice_none=True, - ) - - -def test_tool_choice_none_honors_worker_exclusion_setting() -> None: - body = { - "messages": [{"role": "user", "content": "hello"}], - "tools": TOOLS, - "tool_choice": "none", - } - excluded_tokenizer = _Tokenizer() - included_tokenizer = _Tokenizer() - - prepare_dynamo_chat_completion_request( - body, - tokenizer=excluded_tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - prepare_dynamo_chat_completion_request( - body, - tokenizer=included_tokenizer, - exclude_tools_when_tool_choice_none=False, - ) - - assert excluded_tokenizer.calls[0]["tools"] is None - assert included_tokenizer.calls[0]["tools"] == TOOLS - - -def test_prepare_dynamo_chat_completion_request_normalizes_prior_tool_arguments() -> ( - None -): - tokenizer = _Tokenizer() - body = { - "model": "dummy-model", - "messages": [ - {"role": "user", "content": "hello"}, - { - "role": "assistant", - "content": "older tool call", - "tool_calls": [ - { - "type": "function", - "function": { - "name": "shell", - "arguments": '{"command":"pwd"}', - }, - }, - { - "type": "function", - "function": { - "name": "malformed", - "arguments": "not-json", - }, - }, - ], - }, - {"role": "user", "content": "next"}, - { - "role": "assistant", - "content": "first", - "prompt_token_ids": [10, 777, 2, 40], - "generation_token_ids": [31, 32, 2], - "generation_log_probs": [-0.1, -0.2, -0.3], - }, - {"role": "user", "content": "next"}, - ], - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == [10, 777, 2, 40, 31, 32, 2, 40, 99] - assert prepared["messages"][1]["tool_calls"][0]["function"]["arguments"] == ( - '{"command":"pwd"}' - ) - assert prepared["messages"][1]["tool_calls"][1]["function"]["arguments"] == ( - "not-json" - ) - - -def test_prepare_dynamo_chat_completion_request_normalizes_tools_without_prefix() -> ( - None -): - tokenizer = _Tokenizer() - body = { - "messages": [ - {"role": "user", "content": "hello"}, - { - "role": "assistant", - "content": "older tool call", - "tool_calls": [ - { - "type": "function", - "function": { - "name": "shell", - "arguments": '{"command":"pwd"}', - }, - } - ], - }, - {"role": "user", "content": "next"}, - ] - } - - prepared = prepare_dynamo_chat_completion_request( - body, - tokenizer=tokenizer, - exclude_tools_when_tool_choice_none=True, - ) - - assert prepared["nvext"]["token_data"] == [10, 777, 2, 40, 99] - assert prepared["messages"][1]["tool_calls"][0]["function"]["arguments"] == ( - '{"command":"pwd"}' - ) - assert len(tokenizer.calls) == 2 - - -def test_prepare_dynamo_chat_completion_request_rejects_stream() -> None: - with pytest.raises(ValueError, match="stream=True"): - prepare_dynamo_chat_completion_request( - {"messages": [{"role": "user", "content": "hello"}], "stream": True}, - tokenizer=_Tokenizer(), - exclude_tools_when_tool_choice_none=True, - ) - - -def test_prepare_dynamo_chat_completion_request_rejects_multiple_choices() -> None: - with pytest.raises(ValueError, match="only n=1"): - prepare_dynamo_chat_completion_request( - {"messages": [{"role": "user", "content": "hello"}], "n": 2}, - tokenizer=_Tokenizer(), - exclude_tools_when_tool_choice_none=True, - ) - - -def test_validate_engine_data_requires_prompt_completion_and_logprobs() -> None: - _validate_engine_data( - { - "nvext": { - "engine_data": { - "prompt_token_ids": [1, 2], - "completion_token_ids": [3], - "completion_logprobs": [-0.25], - } - } - } - ) - - with pytest.raises(ValueError, match="engine_data"): - _validate_engine_data({"nvext": {}}) - with pytest.raises(ValueError, match="prompt_token_ids"): - _validate_engine_data( - { - "nvext": { - "engine_data": { - "completion_token_ids": [], - "completion_logprobs": [], - } - } - } - ) - with pytest.raises(ValueError, match="completion_token_ids"): - _validate_engine_data( - { - "nvext": { - "engine_data": { - "prompt_token_ids": [], - "completion_logprobs": [], - } - } - } - ) - with pytest.raises(ValueError, match="completion_logprobs"): - _validate_engine_data( - { - "nvext": { - "engine_data": { - "prompt_token_ids": [], - "completion_token_ids": [], - } - } - } - ) - - -def test_inject_gym_token_metadata_validates_and_populates_message() -> None: - response = { - "choices": [ - { - "message": {"role": "assistant", "content": "answer"}, - "logprobs": None, - } - ], - "nvext": { - "engine_data": { - "prompt_token_ids": [1, 2, 3], - "completion_token_ids": [4, 5], - "completion_logprobs": [-0.25, -0.5], - } - }, - } - - _inject_gym_token_metadata(response) - - assert response["choices"][0]["message"] == { - "role": "assistant", - "content": "answer", - "prompt_token_ids": [1, 2, 3], - "generation_token_ids": [4, 5], - "generation_log_probs": [-0.25, -0.5], - } - - response["nvext"]["engine_data"]["completion_logprobs"] = [-0.25] - with pytest.raises(ValueError, match="1 generation log probabilities for 2"): - _inject_gym_token_metadata(response) - - -def test_forward_chat_completion_reuses_loop_bound_session() -> None: - class FakeResponse: - status = 200 - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - return None - - async def text(self): - return json.dumps({"choices": []}) - - class FakeSession: - def __init__(self): - self.calls = [] - - def post(self, url, *, json, headers): - self.calls.append((url, json, headers)) - return FakeResponse() - - server = DynamoTokenWrapperServer( - dynamo_frontend_base_url="http://dynamo/v1", - tokenizer=_Tokenizer(), - tokenizer_chat_template_kwargs=None, - exclude_tools_when_tool_choice_none=True, - request_timeout_s=30, - ) - session = FakeSession() - server._client_session = session - - async def forward_twice(): - await server._forward_chat_completion({}, authorization=None) - await server._forward_chat_completion({}, authorization="Bearer token") - - asyncio.run(forward_twice()) - - assert len(session.calls) == 2 - assert session.calls[1][2]["Authorization"] == "Bearer token" diff --git a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py new file mode 100644 index 00000000000..f729715eede --- /dev/null +++ b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Dynamo stays out of NeMo RL's vLLM engines. + +The engines run NeMo RL's own vLLM and use Dynamo only through its +engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only where +it also breaks native vLLM. These guards fail if NeMo RL starts importing +Dynamo's vLLM integration (or any other Dynamo module), or if the SDK itself +starts importing vLLM. +""" + +import ast +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +NEMO_RL_ROOT = Path(__file__).parents[4] / "nemo_rl" + +# Every Dynamo module NeMo RL may import. A new entry needs a reason it does not +# reach Dynamo-owned vLLM code. +DYNAMO_SDK_MODULES = { + "dynamo._core", + "dynamo.common.backend", + "dynamo.common.backend.logprobs", + "dynamo.common.backend.publisher", + "dynamo.llm.exceptions", +} + + +def _dynamo_imports() -> dict[str, set[str]]: + """The Dynamo modules each nemo_rl source file imports.""" + imports: dict[str, set[str]] = {} + for path in NEMO_RL_ROOT.rglob("*.py"): + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if isinstance(node, ast.ImportFrom) and node.level == 0: + modules = [node.module or ""] + elif isinstance(node, ast.Import): + modules = [alias.name for alias in node.names] + else: + continue + for module in modules: + if module == "dynamo" or module.startswith("dynamo."): + imports.setdefault(str(path.relative_to(NEMO_RL_ROOT)), set()).add( + module + ) + return imports + + +def test_nemo_rl_imports_only_dynamos_backend_sdk() -> None: + imports = _dynamo_imports() + + assert imports, "the Dynamo adapter's imports were not found" + outside = { + path: sorted(modules - DYNAMO_SDK_MODULES) + for path, modules in imports.items() + if modules - DYNAMO_SDK_MODULES + } + assert not outside, ( + f"nemo_rl imports Dynamo modules beyond its backend SDK: {outside}" + ) + + +@pytest.mark.vllm +def test_dynamos_backend_sdk_imports_no_vllm() -> None: + pytest.importorskip("dynamo._core") + # A fresh interpreter, so modules other tests imported do not count. + code = ( + "import json, sys\n" + + "".join(f"import {module}\n" for module in sorted(DYNAMO_SDK_MODULES)) + + "print(json.dumps(sorted(m for m in sys.modules" + " if m.split('.')[0] == 'vllm' or m.startswith('dynamo.vllm'))))" + ) + loaded = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + + assert json.loads(loaded.stdout.splitlines()[-1]) == [] diff --git a/tests/unit/models/generation/test_openai_server_utils.py b/tests/unit/models/generation/test_openai_server_utils.py index 223be2dc883..425e46442bc 100644 --- a/tests/unit/models/generation/test_openai_server_utils.py +++ b/tests/unit/models/generation/test_openai_server_utils.py @@ -17,6 +17,7 @@ from nemo_rl.models.generation.openai_server_utils import ( PrefixSplice, + derive_prefix_eos_count, replace_prefix_tokens, splice_prefix_tokens, ) @@ -160,6 +161,97 @@ def test_splice_prefix_tokens_without_tokenizer_honors_explicit_eos_ids( ) assert result == expected assert result.token_ids[: len(model_prefix)] == model_prefix + # A host without the messages (managed Dynamo's engine) passes the EOS count + # derived from the prefix instead of the template prefix render. + assert ( + splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=model_prefix, + template_token_ids=template, + eos_token_id=eos_token_id, + template_prefix_eos_count=derive_prefix_eos_count( + model_prefix, eos_token_id=eos_token_id + ), + ) + == expected + ) + with pytest.raises(ValueError, match="exactly one"): + splice_prefix_tokens( + tokenizer=None, + model_prefix_token_ids=model_prefix, + template_prefix_token_ids=template_prefix, + template_token_ids=template, + eos_token_id=eos_token_id, + template_prefix_eos_count=1, + ) + + +@pytest.mark.parametrize( + "model_name", ["Qwen/Qwen3-0.6B", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"] +) +def test_derived_prefix_eos_count_matches_real_template_renders(model_name): + """Across tool-calling turns with reasoning, the EOS count derived from the + exact prefix equals the count in the template's render of the same history.""" + transformers = pytest.importorskip("transformers") + try: + tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) + except OSError as error: + pytest.skip(f"{model_name} tokenizer unavailable: {error}") + eos = tokenizer.eos_token_id + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + + def render(messages, *, add_generation_prompt): + rendered = tokenizer.apply_chat_template( + messages, + tools=tools, + add_generation_prompt=add_generation_prompt, + tokenize=True, + ) + return list(getattr(rendered, "input_ids", rendered)) + + def generate(text): + return tokenizer.encode(text, add_special_tokens=False) + [eos] + + messages = [ + {"role": "system", "content": "You are a weather assistant."}, + {"role": "user", "content": "Weather in Paris?"}, + ] + replies = [ + "\nI should look it up.\n\n\nChecking the forecast.", + "\nIt is sunny.\n\n\nIt is sunny in Paris.", + ] + follow_ups = [ + {"role": "tool", "content": "sunny, 22C"}, + {"role": "user", "content": "And London?"}, + ] + chain = render(messages, add_generation_prompt=True) + generate(replies[0]) + for reply, follow_up, next_reply in zip(replies, follow_ups, replies[1:] + ["ok"]): + messages = [*messages, {"role": "assistant", "content": reply}, follow_up] + template_prefix = render(messages[:-1], add_generation_prompt=False) + template = render(messages, add_generation_prompt=True) + assert derive_prefix_eos_count(chain, eos_token_id=eos) == sum( + token == eos for token in template_prefix + ) + spliced = splice_prefix_tokens( + tokenizer=tokenizer, + model_prefix_token_ids=chain, + template_prefix_token_ids=template_prefix, + template_token_ids=template, + ).token_ids + chain = spliced + generate(next_reply) def test_replace_prefix_tokens_without_tokenizer_reports_missing_eos_without_decoding(): diff --git a/tests/unit/models/generation/test_swe1_dynamo_config.py b/tests/unit/models/generation/test_swe1_dynamo_config.py index c0a51db633c..8b2053e6103 100644 --- a/tests/unit/models/generation/test_swe1_dynamo_config.py +++ b/tests/unit/models/generation/test_swe1_dynamo_config.py @@ -8,7 +8,7 @@ from nemo_rl.environments.nemo_gym import setup_nemo_gym_config from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.config import DynamoConfig +from nemo_rl.models.generation.dynamo.config import dynamo_cfg from nemo_rl.utils.config import load_config, register_omegaconf_resolvers REPO_ROOT = Path(__file__).resolve().parents[4] @@ -43,9 +43,11 @@ class Tokenizer: setup_nemo_gym_config( SimpleNamespace(policy=config["policy"], env=config["env"]), tokenizer ) - validated = DynamoConfig.model_validate(configured_generation) - assert validated.vllm_cfg.async_engine is True - assert validated.vllm_cfg.expose_http_server is True + vllm_cfg = configured_generation["vllm_cfg"] + assert configured_generation["backend"] == "vllm" + assert configured_generation["dynamo"] is True + assert vllm_cfg["async_engine"] is True + assert vllm_cfg["expose_http_server"] is True assert config["policy"]["model_name"] == ( "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" @@ -63,12 +65,11 @@ class Tokenizer: "gpus_per_node": 8, "num_nodes": 1, } - assert validated.engine_world_size == 4 - assert generation["vllm_cfg"]["expert_parallel_size"] == 4 - assert validated.dynamo_cfg.frontend_args.router_mode == "kv" - assert validated.dynamo_cfg.control_timeout_s == 600 - assert validated.vllm_cfg.enable_vllm_metrics_logger is True - assert validated.vllm_cfg.load_format == "dummy" + assert vllm_cfg["tensor_parallel_size"] * vllm_cfg["pipeline_parallel_size"] == 4 + assert vllm_cfg["expert_parallel_size"] == 4 + assert dynamo_cfg(configured_generation).frontend_args.router_mode == "kv" + assert vllm_cfg["enable_vllm_metrics_logger"] is True + assert vllm_cfg["load_format"] == "dummy" assert config["env"]["nemo_gym"]["config_paths"][-1].endswith( "swe_pivot_single_step_tool_use_with_argument_comparison.yaml" ) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index dffb6b6af15..f3d4b7720c0 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -380,7 +380,7 @@ def test_vllm_generation_rejects_unregistered_worker_extension( cluster._init_placement_groups.assert_not_called() -@pytest.mark.parametrize("backend", ["sglang", "megatron", "trtllm", "dynamo"]) +@pytest.mark.parametrize("backend", ["sglang", "megatron", "trtllm"]) def test_generation_config_rejects_worker_extension_for_other_backends( backend: str, ) -> None: @@ -392,7 +392,7 @@ def test_generation_config_rejects_worker_extension_for_other_backends( configure_generation_config(config, MagicMock()) -@pytest.mark.parametrize("backend", ["vllm", "sglang", "megatron", "trtllm", "dynamo"]) +@pytest.mark.parametrize("backend", ["vllm", "sglang", "megatron", "trtllm"]) @pytest.mark.parametrize("include_null", [False, True]) def test_generation_config_allows_no_worker_extension( backend: str, @@ -2427,6 +2427,63 @@ def test_vllm_validate_settings_accepts_missing_vllm_cfg_as_refit_disabled(): VllmGeneration.validate_settings(master_config) +def _dynamo_master_config( + *, + router=False, + fleet_health=False, + nemo_gym=True, + router_replay=False, + token_capture=False, +): + vllm_config = deepcopy(basic_vllm_test_config) + vllm_config["dynamo"] = True + vllm_config["vllm_cfg"]["async_engine"] = True + vllm_config["vllm_cfg"]["expose_http_server"] = True + return types.SimpleNamespace( + policy={"generation": vllm_config, "router_replay": {"enabled": router_replay}}, + env={"should_use_nemo_gym": nemo_gym}, + async_rl=types.SimpleNamespace( + generation_router=types.SimpleNamespace(enabled=router), + generation_fleet_health=types.SimpleNamespace(enabled=fleet_health), + ), + token_capture=types.SimpleNamespace(enabled=token_capture), + ) + + +def test_vllm_validate_settings_accepts_dynamo_with_nemo_gym(): + VllmGeneration.validate_settings(_dynamo_master_config()) + VllmGeneration.validate_settings( + _dynamo_master_config(router_replay=True, token_capture=True) + ) + VllmGeneration.validate_settings( + _dynamo_master_config(router_replay=True, nemo_gym=False) + ) + + +@pytest.mark.parametrize( + ("settings", "error"), + [ + ({"router": True}, "routes the whole fleet behind one URL"), + ({"fleet_health": True}, "Disable fleet health"), + ({"router_replay": True}, "requires token_capture.enabled=true"), + ], +) +def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error): + with pytest.raises(ValueError, match=error): + VllmGeneration.validate_settings(_dynamo_master_config(**settings)) + + +@pytest.mark.parametrize("dynamo", [False, True]) +def test_vllm_client_base_urls_name_the_dynamo_frontend_when_it_serves(dynamo): + generation = object.__new__(VllmGeneration) + generation.dp_openai_server_base_urls = ["http://a/v1", "http://b/v1"] + generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None + + assert generation.client_base_urls == ( + ["http://frontend/v1"] if dynamo else ["http://a/v1", "http://b/v1"] + ) + + def test_vllm_policy_generation(policy, test_input_data, tokenizer): """Test vLLM policy generation capabilities.""" # Test generation diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py new file mode 100644 index 00000000000..99125fd1c29 --- /dev/null +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -0,0 +1,146 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The rollout result pipeline shared by generate_async and Dynamo serving.""" + +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result, + rollout_result_from_wire, + rollout_result_to_wire, +) +from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL + +LAYERS, TOPK = 2, 2 + + +def _logprob(value: float): + return SimpleNamespace(logprob=value) + + +def _final_output( + generated: list[int], + *, + finish_reason: str = "stop", + prompt_routes: int | None = None, + generated_routes: int | None = None, +): + def routes(count): + if count is None: + return None + return np.arange(count * LAYERS * TOPK, dtype=np.int16).reshape( + count, LAYERS, TOPK + ) + + return SimpleNamespace( + prompt_routed_experts=routes(prompt_routes), + num_cached_tokens=0, + outputs=[ + SimpleNamespace( + token_ids=generated, + # The second token has no reported logprob. + logprobs=[{generated[0]: _logprob(-0.5)}, {}] + + [{t: _logprob(-1.0)} for t in generated[2:]], + finish_reason=finish_reason, + routed_experts=routes(generated_routes), + ) + ], + ) + + +def _result(final_output, *, input_length=3, r3=False) -> RolloutResult: + return rollout_result( + final_output, + input_length=input_length, + return_routed_experts=r3, + routed_experts_dtype=torch.int16, + device=torch.device("cpu"), + ) + + +def test_output_layout_matches_generate_async() -> None: + result = _result(_final_output([7, 8, 9], finish_reason="length")) + output = rollout_output( + result, input_ids=torch.tensor([1, 2, 3, 0, 0]), input_length=3, pad_token_id=0 + ) + + assert output["output_ids"].tolist() == [[1, 2, 3, 7, 8, 9]] + # A position without a reported logprob stays 0.0, aligned with its token. + assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0, -0.5, 0.0, -1.0]] + assert output["generation_lengths"].tolist() == [3] + assert output["unpadded_sequence_lengths"].tolist() == [6] + assert output["truncated"].tolist() == [True] + assert "routed_experts" not in output + + +def test_nothing_generated_returns_the_prompt_untruncated() -> None: + output = rollout_output( + RolloutResult.nothing_generated(), + input_ids=torch.tensor([1, 2, 3, 0]), + input_length=3, + pad_token_id=0, + ) + + assert output["output_ids"].tolist() == [[1, 2, 3]] + assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0]] + assert output["generation_lengths"].tolist() == [0] + assert output["truncated"].tolist() == [False] + + +def test_router_replay_routes_are_aligned_and_counted() -> None: + # 3 prompt + 3 generated tokens need 5 routes; vLLM reported 3 + 1. + result = _result( + _final_output([7, 8, 9], prompt_routes=3, generated_routes=1), r3=True + ) + output = rollout_output( + result, input_ids=torch.tensor([1, 2, 3]), input_length=3, pad_token_id=0 + ) + + routed = output["routed_experts"] + assert routed.shape == (1, 6, LAYERS, TOPK) and routed.dtype == torch.int16 + assert (routed[0, 4:] == R3_MISSING_ROUTE_SENTINEL).all() + assert output["r3_routed_experts_expected_routes"].tolist() == [5] + assert output["r3_routed_experts_actual_routes"].tolist() == [4] + assert output["r3_routed_experts_missing_routes"].tolist() == [1] + + +def test_router_replay_requires_routes() -> None: + with pytest.raises(RuntimeError, match="did not include routed_experts"): + _result(_final_output([7, 8, 9]), r3=True) + + +def test_wire_round_trip_preserves_the_output() -> None: + result = _result( + _final_output([7, 8, 9], prompt_routes=3, generated_routes=3), r3=True + ) + wire = rollout_result_to_wire(result) + input_ids = torch.tensor([1, 2, 3]) + + direct = rollout_output(result, input_ids=input_ids, input_length=3, pad_token_id=0) + shipped = rollout_output( + rollout_result_from_wire(wire), + input_ids=input_ids, + input_length=3, + pad_token_id=0, + ) + + assert direct.keys() == shipped.keys() + for key in direct: + assert torch.equal(direct[key], shipped[key]), key diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index 3414010f089..25b5a2e19ee 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -230,6 +230,7 @@ def _worker_with_capture(sink: _MemorySink): for name in ( "_fetch_chain_prefix", "_capture_admission", + "_parse_capture_admission", "_resolve_admission_prefix", "_enter_request_prefix", ): diff --git a/tests/unit/models/generation/test_vllm_utils.py b/tests/unit/models/generation/test_vllm_utils.py index 7bf7546d9d0..3ce5907bc06 100644 --- a/tests/unit/models/generation/test_vllm_utils.py +++ b/tests/unit/models/generation/test_vllm_utils.py @@ -38,6 +38,7 @@ encode_counter_key, format_prompt_for_vllm_generation, model_dump_chat_response_with_dynamic_message_fields, + off_policy_sampling_error, pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) @@ -1451,3 +1452,29 @@ def test_remap_multimodal_placeholders_rebuilds_real_vllm_placeholder_range(): assert isinstance(moved, PlaceholderRange) assert (moved.offset, moved.length) == (3, 3) assert moved.is_embed is is_embed + + +@pytest.mark.parametrize( + ("temperature", "top_p", "top_k", "error"), + [ + (1.0, 1.0, None, None), # train profile + (0.0, 0.9, -1, None), # validation profile + (0.7, 1.0, None, "matches neither"), + (1.0, None, None, "top_p must be set explicitly"), + (1.0, 1.0, 5, "Top k sampling parameter"), + ], +) +def test_off_policy_sampling_error(temperature, top_p, top_k, error) -> None: + generation_config = { + "temperature": 1.0, + "top_p": 1.0, + "val_temperature": 0.0, + "val_top_p": 0.9, + } + message = off_policy_sampling_error( + temperature=temperature, + top_p=top_p, + top_k=top_k, + generation_config=generation_config, + ) + assert (message is None) if error is None else (error in message) diff --git a/tests/unit/models/policy/test_worker_refit_signatures.py b/tests/unit/models/policy/test_worker_refit_signatures.py index fc1e6200c34..d20bd2a203c 100644 --- a/tests/unit/models/policy/test_worker_refit_signatures.py +++ b/tests/unit/models/policy/test_worker_refit_signatures.py @@ -99,7 +99,6 @@ GENERATION_BACKENDS = [ "interfaces.py", "vllm/vllm_generation.py", - "dynamo/dynamo_generation.py", "trtllm/trtllm_generation.py", "megatron/megatron_generation.py", "sglang/sglang_generation.py", diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index 4f3318e9990..6acf4f133cd 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -171,13 +171,6 @@ def _make_master_config( "kv_cache_management_mode": "recompute", } policy_config["model_name"] = "test-model" - if backend == "dynamo": - # The fields DynamoGeneration.validate_settings reads, at accepted values. - generation_config["dynamo_cfg"] = { - "request_timeout_s": 900.0, - "control_timeout_s": 900.0, - } - generation_config["vllm_cfg"] = {"enable_vllm_metrics_logger": False} master_config = MasterConfig.model_construct( data_plane={ "enabled": dp_enabled, @@ -224,8 +217,6 @@ def _make_master_config( **({} if sampler_cfg is None else {"sampler": sampler_cfg}), ), ) - if backend == "dynamo": - master_config.async_rl.generation_fleet_health.refit_timeout_s = None return master_config @@ -364,51 +355,6 @@ def test_build_generation_passes_sglang_config(): generation.finish_generation.assert_called_once_with() -def test_build_generation_constructs_dynamo_from_normalized_config(): - """DynamoGeneration receives the generation config setup already validated.""" - master_config = _make_master_config(backend="dynamo") - master_config.policy["model_name"] = "Qwen/Qwen3-0.6B" - inference_cluster = MagicMock(name="inference_cluster") - - with patch.object(sc_setup_mod, "DynamoGeneration") as mock_dynamo: - generation, _ = sc_setup_mod._build_generation( - inference_cluster, - master_config, - ) - - mock_dynamo.assert_called_once_with( - cluster=inference_cluster, - config=master_config.policy["generation"], - ) - assert master_config.policy["generation"]["model_name"] == "Qwen/Qwen3-0.6B" - generation.finish_generation.assert_called_once_with() - - -def test_build_generation_rejects_deferred_dynamo_load(): - master_config = _make_master_config(backend="dynamo") - master_config.policy["model_name"] = "Qwen/Qwen3-0.6B" - - with ( - patch.object(sc_setup_mod, "DynamoGeneration") as mock_dynamo, - pytest.raises(AssertionError, match="defer_model_load"), - ): - sc_setup_mod._build_generation( - MagicMock(name="inference_cluster"), - master_config, - defer_model_load=True, - ) - mock_dynamo.assert_not_called() - - -def test_generation_max_seq_len_reads_dynamo_vllm_cfg(): - assert ( - sc_setup_mod._generation_max_seq_len( - {"backend": "dynamo", "vllm_cfg": {"max_model_len": 1024}} - ) - == 1024 - ) - - def test_build_clusters_rejects_unsupported_topology_backend(monkeypatch): """Topology planning reports the supported SC backends instead of KeyError.""" master_config = _make_master_config(colocated=False, backend="trtllm") @@ -432,7 +378,7 @@ def test_build_clusters_rejects_unsupported_topology_backend(monkeypatch): with pytest.raises( ValueError, - match="only supports vllm, sglang, megatron, or dynamo generation; got 'trtllm'", + match="only supports vllm, sglang, or megatron generation; got 'trtllm'", ): sc_setup_mod._build_clusters(master_config) @@ -1664,104 +1610,6 @@ def test_ready_first_sampler_rejects_incompatible_loss_config( patched_factories["setup_response_data"].assert_not_called() patched_factories["_build_clusters"].assert_not_called() - def test_invalid_dynamo_config_fails_before_clusters(self, patched_factories): - mc = _make_master_config(backend="dynamo") - - with ( - patch.object( - sc_setup_mod.DynamoConfig, - "model_validate", - side_effect=ValueError("bad dynamo config"), - ), - pytest.raises(ValueError, match="bad dynamo config"), - ): - setup_single_controller(mc, MagicMock(pad_token_id=0)) - - patched_factories["_build_clusters"].assert_not_called() - patched_factories["_build_generation"].assert_not_called() - - @pytest.mark.parametrize( - ("misconfigure", "match"), - [ - pytest.param( - lambda mc: mc.policy.update(router_replay={"enabled": True}), - "does not return routed experts", - id="router_replay", - ), - pytest.param( - lambda mc: mc.policy.update(is_vlm=True), - "serves text prompts only", - id="vlm", - ), - pytest.param( - lambda mc: setattr( - mc.async_rl.generation_fleet_health, "enabled", True - ), - "Dynamo frontend owns request routing", - id="fleet_health", - ), - # 300.0 is the FleetHealthConfig default, so a Dynamo config that does - # not opt out would otherwise only fail at its first weight sync. - pytest.param( - lambda mc: setattr( - mc.async_rl.generation_fleet_health, "refit_timeout_s", 300.0 - ), - "refit_timeout_s must be null", - id="refit_deadline", - ), - pytest.param( - lambda mc: mc.policy["generation"]["dynamo_cfg"].update( - control_timeout_s=600.0 - ), - "must be >= request_timeout_s", - id="drain_timeout", - ), - pytest.param( - lambda mc: mc.policy["generation"]["vllm_cfg"].update( - enable_vllm_metrics_logger=True - ), - "requires rollout_checkpointing.telemetry_interval_s", - id="unread_metrics_logger", - ), - ], - ) - def test_unsupported_dynamo_settings_fail_before_clusters( - self, patched_factories, misconfigure, match - ): - mc = _make_master_config(backend="dynamo") - misconfigure(mc) - - with ( - patch.object(sc_setup_mod.DynamoConfig, "model_validate") as validate, - pytest.raises(ValueError, match=match), - ): - validate.return_value.model_dump.return_value = mc.policy["generation"] - setup_single_controller(mc, MagicMock(pad_token_id=0)) - - patched_factories["setup_response_data"].assert_not_called() - patched_factories["_build_clusters"].assert_not_called() - patched_factories["_build_generation"].assert_not_called() - - def test_dynamo_generation_config_is_normalized(self, patched_factories): - mc = _make_master_config(backend="dynamo") - mc.policy["hf_config_overrides"] = {"rope_theta": 1.0} - normalized = { - **mc.policy["generation"], - "vllm_kwargs": {"hf_overrides": {"rope_theta": 1.0}}, - } - - with patch.object(sc_setup_mod.DynamoConfig, "model_validate") as validate: - validate.return_value.model_dump.return_value = normalized - actor_args, _ = setup_single_controller(mc, MagicMock(pad_token_id=0)) - - (raw_config,), _ = validate.call_args - assert raw_config["vllm_kwargs"]["hf_overrides"] == {"rope_theta": 1.0} - assert mc.policy["generation"] is normalized - assert actor_args.gen_handle is patched_factories["fake_gen"] - _, sync_kwargs = patched_factories["create_weight_synchronizer"].call_args - assert sync_kwargs["generation_backend"] == "dynamo" - assert sync_kwargs["colocated"] is False - def test_returns_actor_args(self, patched_factories): mc = _make_master_config() tokenizer = MagicMock(pad_token_id=0) @@ -2271,7 +2119,7 @@ def test_nemo_gym_wires_env_handle(self, patched_factories): assert data_kwargs["is_vlm"] is True mock_spinup.assert_called_once_with( mc.env, - base_urls=patched_factories["fake_gen"].dp_openai_server_base_urls, + base_urls=patched_factories["fake_gen"].client_base_urls, model_name="test-model", # Reaches the actor once, at spinup, rather than riding along with every # run_rollouts call. @@ -2384,6 +2232,44 @@ def test_token_capture_always_creates_finalizer_actor_pool( _, setup_kwargs = generation.setup_token_capture.call_args assert setup_kwargs["capture_media"] is with_processor + def test_dynamo_token_capture_tells_gym_to_speak_dynamo(self, patched_factories): + """The same vLLM workers stage the calls; Gym reaches them through the frontend.""" + mc = _make_master_config(backend="vllm") + mc.policy["generation"].update( + { + "model_name": "test-model", + "stop_strings": None, + "stop_token_ids": None, + "top_k": None, + "dynamo": True, + "vllm_cfg": {"async_engine": True, "expose_http_server": True}, + } + ) + mc.token_capture.enabled = True + mc.env = {"should_use_nemo_gym": True, "nemo_gym": {}} + patched_factories["setup_response_data"].return_value = (list(range(8)), None) + + with ( + patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), + patch.object( + sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() + ) as mock_spinup, + patch.object(sc_setup_mod, "validate_dataset_agent_coverage"), + patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), + patch( + "nemo_rl.experience.rollout_reassembler_actor.create_rollout_reassembler_actors", + return_value=[MagicMock()], + ), + ): + setup_single_controller(mc, MagicMock(pad_token_id=9)) + + assert mc.token_capture.generation_backend == "dynamo" + assert mock_spinup.call_args.kwargs["token_capture"]["generation_backend"] == ( + "dynamo" + ) + generation, _ = patched_factories["_build_generation"].return_value + generation.setup_token_capture.assert_called() + def test_nemo_gym_coverage_failure_shuts_down_shards(self, patched_factories): mc = _make_master_config(colocated=False, backend="vllm") mc.policy["generation"]["model_name"] = "test-model" diff --git a/tests/unit/test_dependency_pins.py b/tests/unit/test_dependency_pins.py index 864ee4d40ba..17c14b55aae 100644 --- a/tests/unit/test_dependency_pins.py +++ b/tests/unit/test_dependency_pins.py @@ -144,3 +144,21 @@ def test_torch_split_shims_are_still_needed() -> None: "(nemo_rl/data_plane/adapters/tq_mooncake_checkpoint.py) - drop the " "check outright; the pre-2.13 `names` comparison cannot be restored" ) + + +def test_vllm_extra_carries_base_dynamo_release() -> None: + """Dynamo serving runs NeMo RL's own vLLM workers, so the vllm extra carries the + released Dynamo runtime but never ai-dynamo[vllm], which pins Dynamo's vLLM.""" + extras = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text())["project"][ + "optional-dependencies" + ] + requirements = { + canonicalize_name(req.name): req + for req in (Requirement(dep) for dep in extras["vllm"]) + } + + dynamo, runtime = requirements["ai-dynamo"], requirements["ai-dynamo-runtime"] + assert not dynamo.extras, f"ai-dynamo must not take extras, got {dynamo.extras}" + assert dynamo.specifier == runtime.specifier, ( + "ai-dynamo and ai-dynamo-runtime must pin the same release" + ) diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index dc856d71c83..1e040d3c177 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -380,3 +380,46 @@ def test_dynamo_frontend_args_without_router_reset_states_pass(): def test_generation_without_dynamo_cfg_passes(): check_outdated_config({"policy": {"generation": {"backend": "vllm"}}}) + + +# ============================================================================ +# reject_outdated_dynamo_backend +# ============================================================================ + + +def _generation_config(**generation) -> dict: + return {"policy": {"generation": generation}} + + +def test_dynamo_backend_is_rejected_with_the_migration(): + with pytest.raises( + ValueError, match=r"backend=vllm and policy\.generation\.dynamo=true" + ): + check_outdated_config(_generation_config(backend="dynamo")) + + +@pytest.mark.parametrize( + "dynamo_cfg, removed", + [ + ({"engine": "vllm"}, r"dynamo_cfg\.engine"), + ({"control_timeout_s": 600}, r"dynamo_cfg\.control_timeout_s"), + ({"metrics_include_prefixes": None}, r"dynamo_cfg\.metrics_include_prefixes"), + ({"metrics_exclude_prefixes": None}, r"dynamo_cfg\.metrics_exclude_prefixes"), + ({"worker_args": {"extra_cli_args": []}}, r"worker_args\.extra_cli_args"), + ], +) +def test_removed_dynamo_cfg_keys_are_rejected(dynamo_cfg, removed): + with pytest.raises(ValueError, match=removed): + check_outdated_config( + _generation_config(backend="vllm", dynamo=True, dynamo_cfg=dynamo_cfg) + ) + + +def test_vllm_backend_with_dynamo_passes(): + check_outdated_config( + _generation_config( + backend="vllm", + dynamo=True, + dynamo_cfg={"frontend_args": {"router_mode": "kv"}}, + ) + ) diff --git a/tests/unit/weight_sync/test_reconcile_communicator.py b/tests/unit/weight_sync/test_reconcile_communicator.py index 39cfce24494..e0f3b22e006 100644 --- a/tests/unit/weight_sync/test_reconcile_communicator.py +++ b/tests/unit/weight_sync/test_reconcile_communicator.py @@ -643,10 +643,9 @@ class TestBackendsWithoutAWorkerGroup: """CollectiveWeightSynchronizer holds a GenerationInterface, not a VllmGeneration. Only some implementations own a Ray worker group with DP shards -- vLLM and TRT-LLM do; - Dynamo, Megatron and SGLang do not, and the interface declares nothing either way. Of - those, Dynamo reaches this synchronizer through the ordinary non-colocated branch of - the factory, so reading self._generation.worker_group broke it on the plain grpo.py - path: + Megatron and SGLang do not, and the interface declares nothing either way. Reading + self._generation.worker_group once broke the since-removed managed Dynamo backend + on the plain grpo.py path: grpo.py:1747 policy_generation.weight_synchronizer.init_communicator() AttributeError: 'DynamoGeneration' object has no attribute 'worker_group' diff --git a/tests/unit/weight_sync/test_weight_synchronizer.py b/tests/unit/weight_sync/test_weight_synchronizer.py index 785726ff446..14f4c83ac96 100644 --- a/tests/unit/weight_sync/test_weight_synchronizer.py +++ b/tests/unit/weight_sync/test_weight_synchronizer.py @@ -19,7 +19,6 @@ import pytest from nemo_rl.models.generation.constants import ( - DYNAMO_BACKEND, MEGATRON_BACKEND, SGLANG_BACKEND, VLLM_BACKEND, @@ -1556,17 +1555,6 @@ def test_non_colocated_vllm_nccl_reshard_propagates_refit_memory_release( sync.sync_weights() policy.offload_before_refit.assert_called_once_with() - def test_non_colocated_dynamo_returns_collective(self): - sync = create_weight_synchronizer( - policy=_mock_policy(), - generation=_mock_generation(), - generation_backend=DYNAMO_BACKEND, - colocated=False, - train_cluster=_mock_cluster(), - inference_cluster=_mock_cluster(), - ) - assert isinstance(sync, CollectiveWeightSynchronizer) - def test_non_colocated_sglang_returns_sglang_disaggregated(self): """SGLang owns its own weight-update group, so no clusters are needed.""" policy = _megatron_policy() diff --git a/uv.lock b/uv.lock index 80477156eaa..56bc756bbaf 100644 --- a/uv.lock +++ b/uv.lock @@ -85,12 +85,14 @@ constraints = [ { name = "wheel", specifier = ">=0.46.2" }, ] overrides = [ + { name = "aisimulate", marker = "sys_platform == 'never'" }, { name = "apache-tvm-ffi", specifier = "==0.1.11" }, { name = "av", marker = "sys_platform == 'never'" }, { name = "decord2", marker = "sys_platform == 'never'" }, { name = "deep-ep", git = "https://github.com/deepseek-ai/DeepEP.git?rev=17cfb817bccec3a9c247013360cc550c2bac441e" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.133.0,<0.137.0" }, { name = "instanttensor", marker = "platform_machine == 'x86_64'", specifier = ">=0.1.9" }, + { name = "kubernetes", marker = "sys_platform == 'never'" }, { name = "langchain", specifier = ">=0.3.28" }, { name = "langchain-core", specifier = ">=0.3.80" }, { name = "langchain-core", specifier = ">=1.2.22" }, @@ -221,6 +223,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/15/504337e302d070bd2ff2f39418bb17d2ad73c662c403a4f37cb4e5c5b237/agent_detector-2.0.0-py3-none-any.whl", hash = "sha256:22c6a1c9c23894a12f55aea625f060b8d3330fba774de72d02d5442bb580417f", size = 9352, upload-time = "2026-09-02T14:19:50.971Z" }, ] +[[package]] +name = "ai-dynamo" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ai-dynamo-runtime" }, + { name = "aiohttp" }, + { name = "msgspec" }, + { name = "prometheus-client" }, + { name = "pyzmq" }, + { name = "tqdm" }, + { name = "transformers", version = "5.12.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, + { name = "zstandard" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/19/bc00c468b1d8093abafa7f3ab88362d12cfe06f9be2e65baf0ef03d370d1/ai_dynamo-1.5.1-py3-none-any.whl", hash = "sha256:86d21cb7fbebbf1c3d0933303729b1be222ad0fb4a50b17a03316165d33377b1", size = 3512837, upload-time = "2026-10-07T05:08:09.942Z" }, +] + +[[package]] +name = "ai-dynamo-runtime" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, + { name = "uvloop" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/63/f1cde7bb2d74aebc954251fe40ee093298f0975d6f22c47f1f3c66b11949/ai_dynamo_runtime-1.5.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:88e5580daf32056dbe2da9d4189ddd2f4e4cc2472ddfebdd19412d223ac4a596", size = 65897955, upload-time = "2026-10-07T05:08:55.817Z" }, + { url = "https://files.pythonhosted.org/packages/df/05/777f6f79e8c9359d259cf500ec62fcaf6bd9ce148d3bf47910ec02d996e3/ai_dynamo_runtime-1.5.1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b5297625a4f407b3871e3039a53f49e09102d67039fc1339349802038b84ec3e", size = 67290732, upload-time = "2026-10-07T05:09:45.438Z" }, +] + [[package]] name = "aiobotocore" version = "3.9.1" @@ -345,7 +379,8 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "albucore" }, { name = "numpy" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "pyyaml" }, { name = "scipy" }, ] @@ -409,7 +444,8 @@ dependencies = [ { name = "docstring-parser", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "httpx", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "jiter", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "sniffio", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, ] @@ -431,7 +467,7 @@ dependencies = [ { name = "docstring-parser" }, { name = "httpx2" }, { name = "jiter" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "sniffio" }, { name = "typing-extensions" }, ] @@ -943,7 +979,7 @@ resolution-markers = [ ] dependencies = [ { name = "loguru" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "transformers", version = "5.12.1", source = { registry = "https://pypi.org/simple" } }, ] @@ -963,7 +999,7 @@ resolution-markers = [ dependencies = [ { name = "loguru" }, { name = "psutil" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "transformers", version = "5.12.1", source = { registry = "https://pypi.org/simple" } }, ] @@ -1585,7 +1621,8 @@ version = "0.136.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "starlette" }, { name = "typing-extensions" }, { name = "typing-inspection" }, @@ -1637,7 +1674,8 @@ dependencies = [ { name = "detect-installer" }, { name = "fastar" }, { name = "httpx" }, - { name = "pydantic", extra = ["email"] }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, extra = ["email"], marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, extra = ["email"], marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "rich-toolkit" }, { name = "rignore" }, { name = "sentry-sdk" }, @@ -3034,7 +3072,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "interegular" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/d5/41cd417ba7dfdbbcfe46cebf81fb3dfd7c591b89897560ad05bb410a465d/lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da", size = 40258, upload-time = "2025-08-24T19:37:47.527Z" } @@ -3190,7 +3228,8 @@ dependencies = [ { name = "httpx", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "httpx-sse", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "jsonschema", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "pydantic-settings", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "pyjwt", extra = ["crypto"], marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, { name = "python-multipart", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm')" }, @@ -3571,7 +3610,8 @@ dependencies = [ { name = "jsonschema" }, { name = "numpy" }, { name = "pillow" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm'" }, { name = "pydantic-extra-types", extra = ["pycountry"], marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra == 'extra-7-nemo-rl-vllm'" }, { name = "requests" }, { name = "tiktoken" }, @@ -3655,7 +3695,8 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "requests" }, @@ -3681,7 +3722,8 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/88/62/64068c0d17a7ba479f4dadfa78b3e48921cde8d20878414e7a4a929e66c0/mlflow_tracing-3.16.1.tar.gz", hash = "sha256:315ba1ace00d43498da2ad2b2bbb751f3198d6f9cd328168dfeb49911df41925", size = 1525331, upload-time = "2026-09-16T22:40:58.167Z" } wheels = [ @@ -3696,7 +3738,7 @@ dependencies = [ { name = "fastapi", extra = ["standard"], marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, { name = "httpx" }, { name = "jmespath" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, { name = "setuptools" }, { name = "starlette" }, { name = "supervisor" }, @@ -4104,8 +4146,10 @@ dependencies = [ { name = "packaging" }, { name = "psutil" }, { name = "pyasn1" }, - { name = "pydantic" }, - { name = "pydantic-core" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic-core", version = "2.46.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic-core", version = "2.46.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "pydot" }, { name = "pyjwt" }, { name = "python-multipart" }, @@ -4351,6 +4395,8 @@ trtllm = [ { name = "transformers", version = "5.5.4", source = { registry = "https://pypi.org/simple" } }, ] vllm = [ + { name = "ai-dynamo" }, + { name = "ai-dynamo-runtime" }, { name = "cuda-python" }, { name = "deep-ep" }, { name = "deep-gemm" }, @@ -4410,6 +4456,8 @@ test = [ [package.metadata] requires-dist = [ { name = "accelerate", specifier = ">=0.26" }, + { name = "ai-dynamo", marker = "extra == 'vllm'", specifier = "==1.5.1" }, + { name = "ai-dynamo-runtime", marker = "extra == 'vllm'", specifier = "==1.5.1" }, { name = "awscrt", specifier = ">=0.35.0" }, { name = "blobfile" }, { name = "causal-conv1d", marker = "extra == 'automodel'", git = "https://github.com/Dao-AILab/causal-conv1d?rev=4f6ae4e26ae5fe8af9372f8d312ab25cc4595223" }, @@ -5293,7 +5341,8 @@ dependencies = [ { name = "omegaconf", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "packaging", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "pulp", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "pyyaml", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "regex", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "rich", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -5368,7 +5417,8 @@ dependencies = [ { name = "packaging", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "protobuf", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "psutil", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-trtllm' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "pydantic-settings", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "pyyaml", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "slowapi", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-modelopt' or extra == 'extra-7-nemo-rl-nemo-gym' or extra == 'extra-7-nemo-rl-nvrx' or extra == 'extra-7-nemo-rl-sglang' or extra != 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -5533,7 +5583,7 @@ dependencies = [ { name = "distro" }, { name = "httpx" }, { name = "jiter" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "sniffio" }, { name = "tqdm" }, { name = "typing-extensions" }, @@ -5556,7 +5606,8 @@ dependencies = [ { name = "distro", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "httpx", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "jiter", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, - { name = "pydantic", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra != 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm') or (extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-vllm')" }, { name = "sniffio", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "tqdm", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-nemo-gym' or extra != 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, @@ -5577,7 +5628,8 @@ name = "openai-harmony" version = "0.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/92/94/01509d510bebf6606614e51113e5a415ced15b8f34aa98a8bf2539314650/openai_harmony-0.0.4.tar.gz", hash = "sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c", size = 279848, upload-time = "2025-08-09T01:43:24.974Z" } wheels = [ @@ -5880,7 +5932,7 @@ dependencies = [ { name = "jsonschema" }, { name = "outlines-core" }, { name = "pillow" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/0f/95627aba2d038f8ba24c31b689609b153386806a46448abf4c27029b0001/outlines-1.3.3.tar.gz", hash = "sha256:5dcd1b9d261ced1337042d6b1fd2bb4e706a09a0121957bcd9eb331d1ea6940c", size = 2898295, upload-time = "2026-08-06T12:27:14.921Z" } @@ -6430,15 +6482,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/e9/e869bcee87beb89040263c416a8a50204f7f7a83ac11897646c9e71e0daf/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55ccbe27f049743a4caf4f4221b166560d3438d0b1e5ab929e07ae1702a4d6fd", size = 2271038, upload-time = "2025-05-17T17:23:04.872Z" }, ] +[[package]] +name = "pydantic" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "annotated-types", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic-core", version = "2.46.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "typing-inspection", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/6b/69fd5c7194b21ebde0f8637e2a4ddc766ada29d472bfa6a5ca533d79549a/pydantic-2.13.0.tar.gz", hash = "sha256:b89b575b6e670ebf6e7448c01b41b244f471edd276cd0b6fe02e7e7aca320070", size = 843468, upload-time = "2026-04-13T10:51:35.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/d7/c3a52c61f5b7be648e919005820fbac33028c6149994cd64453f49951c17/pydantic-2.13.0-py3-none-any.whl", hash = "sha256:ab0078b90da5f3e2fd2e71e3d9b457ddcb35d0350854fbda93b451e28d56baaf", size = 471872, upload-time = "2026-04-13T10:51:33.343Z" }, +] + +[package.optional-dependencies] +email = [ + { name = "email-validator", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, +] + [[package]] name = "pydantic" version = "2.13.5" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, + { name = "annotated-types", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, + { name = "pydantic-core", version = "2.46.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, + { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, + { name = "typing-inspection", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } wheels = [ @@ -6447,15 +6527,38 @@ wheels = [ [package.optional-dependencies] email = [ - { name = "email-validator" }, + { name = "email-validator", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/0a/9414cddf82eda3976b14048cc0fa8f5b5d1aecb0b22e1dcd2dbfe0e139b1/pydantic_core-2.46.0.tar.gz", hash = "sha256:82d2498c96be47b47e903e1378d1d0f770097ec56ea953322f39936a7cf34977", size = 471441, upload-time = "2026-04-13T09:06:33.813Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/a0/bf9a1ba34537c2ed3872a48195291138fdec8fe26c4009776f00d63cf0c8/pydantic_core-2.46.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c525ecf8a4cdf198327b65030a7d081867ad8e60acb01a7214fff95cf9832d47", size = 1977040, upload-time = "2026-04-13T09:06:16.088Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/5885350203b72e96438eee7f94de0d8f0442f4627237ca8ef75de34db1cd/pydantic_core-2.46.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7897078fe8a13b73623c0955dfb2b3d2c9acb7177aac25144758c9e5a5265aaa", size = 2098522, upload-time = "2026-04-13T09:04:23.239Z" }, + { url = "https://files.pythonhosted.org/packages/a7/53/1958eacbfddc41aadf5ae86dd85041bf054b675f34a2fa76385935f96070/pydantic_core-2.46.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:ee1547a6b8243e73dd10f585555e5a263395e55ce6dea618a078570a1e889aef", size = 2190148, upload-time = "2026-04-13T09:06:56.151Z" }, + { url = "https://files.pythonhosted.org/packages/71/a7/abdb924620b1ac535c690b36ad5b8871f376104090f8842c08625cecf1d3/pydantic_core-2.46.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:004a2081c881abfcc6854a4623da6a09090a0d7c1398a6ae7133ca1256cee70b", size = 2383167, upload-time = "2026-04-13T09:04:52.643Z" }, ] [[package]] name = "pydantic-core" version = "2.46.5" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'linux'", + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } wheels = [ @@ -6470,7 +6573,8 @@ name = "pydantic-extra-types" version = "2.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/71/dba38ee2651f84f7842206adbd2233d8bbdb59fb85e9fa14232486a8c471/pydantic_extra_types-2.11.1.tar.gz", hash = "sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049", size = 172002, upload-time = "2026-03-16T08:08:03.92Z" } @@ -6488,7 +6592,8 @@ name = "pydantic-settings" version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] @@ -6994,7 +7099,8 @@ default = [ { name = "opentelemetry-sdk" }, { name = "prometheus-client" }, { name = "py-spy" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "requests" }, { name = "smart-open" }, { name = "virtualenv" }, @@ -7338,7 +7444,7 @@ dependencies = [ { name = "psutil" }, { name = "py-spy" }, { name = "pybase64" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "python-multipart" }, { name = "pyzmq" }, { name = "quack-kernels", version = "0.6.3", source = { registry = "https://pypi.org/simple" } }, @@ -7880,7 +7986,8 @@ dependencies = [ { name = "protobuf" }, { name = "psutil" }, { name = "pwinput" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "pydantic-settings" }, { name = "pyecharts" }, { name = "pyroaring" }, @@ -8055,7 +8162,7 @@ dependencies = [ { name = "protobuf" }, { name = "psutil" }, { name = "pulp" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "pydantic-settings", extra = ["yaml"], marker = "extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "python-multipart" }, { name = "pyzmq" }, @@ -8699,7 +8806,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/31/37/b2010e5c18b25b5016064685dbf858d880bdeb859e2d1b2d5b9cd69230fa/transformer_engine_cu13-2.18.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:d5e9049256d66e1fb1743b5e1ff78e65a580bc09c371f4977ca71cdf7899fca4", size = 278015862, upload-time = "2026-08-11T00:50:33.011Z" }, @@ -8717,7 +8824,7 @@ dependencies = [ { name = "onnx" }, { name = "onnxscript" }, { name = "packaging" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" } }, { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" } }, { name = "transformer-engine-cu13" }, ] @@ -9017,7 +9124,7 @@ dependencies = [ { name = "psutil" }, { name = "py-cpuinfo" }, { name = "pybase64" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" } }, { name = "pynvvideocodec" }, { name = "python-json-logger" }, { name = "pyyaml" }, @@ -9063,7 +9170,8 @@ dependencies = [ { name = "packaging" }, { name = "platformdirs" }, { name = "protobuf" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-automodel' or extra == 'extra-7-nemo-rl-mcore' or extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or extra != 'extra-7-nemo-rl-vllm'" }, { name = "pyyaml" }, { name = "requests" }, { name = "typing-extensions" }, @@ -9250,7 +9358,8 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, { name = "numpy" }, - { name = "pydantic" }, + { name = "pydantic", version = "2.13.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-trtllm')" }, + { name = "pydantic", version = "2.13.5", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-sglang' or extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-automodel' and extra != 'extra-7-nemo-rl-sglang' and extra != 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "torch", version = "2.11.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-7-nemo-rl-sglang' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-modelopt' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-nvrx' and extra == 'extra-7-nemo-rl-trtllm') or (extra == 'extra-7-nemo-rl-trtllm' and extra == 'extra-7-nemo-rl-vllm')" }, { name = "torch", version = "2.13.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "extra == 'extra-7-nemo-rl-trtllm' or extra == 'extra-7-nemo-rl-vllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang')" }, { name = "transformers", version = "5.5.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-7-nemo-rl-trtllm' or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-mcore') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-automodel' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-mcore' and extra == 'extra-7-nemo-rl-vllm') or (extra == 'extra-7-nemo-rl-nemo-gym' and extra == 'extra-7-nemo-rl-sglang') or (extra == 'extra-7-nemo-rl-sglang' and extra == 'extra-7-nemo-rl-vllm')" }, From 08c50d29e177ec36377554843dfaff8c92c39658 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 24 Oct 2026 17:17:05 -0700 Subject: [PATCH 02/19] fix(dynamo): address the #4905 review Run and CI breaks: - Each engine publishes KV events on its own abstract ipc:// socket, so engines on one node can no longer draw the same port (C1). - Engines get an OS-assigned DYN_SYSTEM_PORT, so a frontend on another node can read a custom chat template (C2). - The Gym capture test skips without nemo_gym (C3). - Gym calls apply the generation config's max_tokens only when the request leaves it unset, as vLLM's server does (C4). - Lint: minimized swe1 recipe (C5) and import order (C6). Behavior: - vLLM client errors reach the client as InvalidArgument with their message, not as a bare 500 (C7). - Non-capture Gym calls return routed experts with R3 on, and Gym#4817 (now a369e28) lifts them, so R3 + Gym no longer needs token capture (C8). - A disconnected capture call fails instead of being staged (C9). - EP > TP is rejected at startup with Dynamo (C10). - The KV-event block size matches vLLM's main-attention kinds (C17). Messages and docs: timeout wording and error (C11, C12), missing etcd/NATS advice (C13), dependency bounds (C19) and ray.sub's port layout (C20). Tests: a Gym-without-capture Dynamo overlay in L1 (C14), the rollout retry loop (C15), media dispatch and failure mapping (C16), hybrid block size (C17) and the Gym engine_data key contract (C18). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/design-docs/dynamo-integration.md | 15 +++- docs/guides/dynamo-generation.md | 14 ++-- ...grpo_math_1B_dynamo_single_controller.yaml | 3 +- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 1 - nemo_rl/distributed/virtual_cluster.py | 3 - nemo_rl/models/generation/__init__.py | 15 +++- nemo_rl/models/generation/dynamo/config.py | 4 +- .../generation/dynamo/managed_runtime.py | 4 +- .../models/generation/dynamo/native_engine.py | 79 ++++++++++++------- .../generation/dynamo/rollout_client.py | 8 ++ nemo_rl/models/generation/dynamo/venv.py | 9 ++- .../models/generation/vllm/vllm_generation.py | 23 +----- ray.sub | 3 +- .../functional/L1_Functional_Tests_Dynamo.sh | 2 + .../models/generation/test_dynamo_config.py | 22 +++++- .../generation/test_dynamo_managed_runtime.py | 8 ++ .../generation/test_dynamo_native_engine.py | 69 +++++++++++++++- .../test_dynamo_native_engine_capture.py | 13 ++- .../generation/test_dynamo_rollout_client.py | 56 +++++++++++-- .../models/generation/test_vllm_generation.py | 65 +++++++++------ 21 files changed, 311 insertions(+), 107 deletions(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index 5e2fb03bbab..a369e289f70 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit 5e2fb03bbabe89543291cf99c471e3d32effdd83 +Subproject commit a369e289f702c623b077e811eceeb98f2b3b26cf diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 8aa388e138d..0fea1ccc5fd 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -54,9 +54,10 @@ implements its `LLMEngine` over the worker's `AsyncLLM`: publishes no token budget. The engine owns the context check and the output cap, because a prefix splice changes the prompt after the frontend has seen it. - `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ - publisher. It is set only when the engine caches prefixes, and it binds a - wildcard endpoint, because vLLM binds only wildcard endpoints. A vLLM stat - logger publishes the router's per-engine load signal. + publisher. It is set only when the engine caches prefixes. Each engine binds + its own Linux abstract-namespace socket (`ipc://@...`), so engines on one node + need no port and cannot collide. A vLLM stat logger publishes the router's + per-engine load signal. Because the Worker runs on the actor's own loop, the worker's other methods interleave with Dynamo requests exactly as with native requests: @@ -104,3 +105,11 @@ functional overlays fail if the engines published events but none were applied (`tests/functional/check_dynamo_overlay.sh`). `ai-dynamo-runtime` caps pydantic, which pins the vLLM venvs' pydantic version. +Dependency bounds like this one can still block a vLLM bump, and the import +guard does not see them: `ai-dynamo-runtime` 1.5.1 requires +`pydantic>=2.10.6,<=2.13`, and `ai-dynamo` caps `aiohttp<4`, +`prometheus-client<1` and `zstandard<1`. A vLLM release that needs a version +outside one of these bounds does not lock with the pinned `ai-dynamo`. Until a +Dynamo release lifts the bound, override it under +`[tool.uv] override-dependencies` (as `aisimulate` and `kubernetes` are), and +let the vLLM-lane Dynamo unit tests catch a real break. diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 09915d3f276..bc554ec422f 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -27,8 +27,9 @@ independent. With `dynamo: true`: - `vllm_cfg.async_engine` is turned on, because Dynamo drives the engine's - `AsyncLLM`. Every rollout path the sync engine supports also runs on the async - one. + `AsyncLLM`. Settings the async engine does not run, such as data parallelism + inside vLLM (`expert_parallel_size > tensor_parallel_size`), are rejected at + startup. - The driver starts etcd, NATS and `dynamo.frontend`. The frontend runs from the vLLM workers' own venv, so it is the same ai-dynamo release. - Each engine turns on vLLM's KV-cache events for `router_mode: kv` and @@ -50,7 +51,7 @@ shows them all: | Key | Default | Meaning | |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | -| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timed-out rollout is not retried; NeMo-Gym's calls keep Gym's timeouts | +| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies; NeMo-Gym's calls keep Gym's timeouts | | `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | | `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | @@ -102,6 +103,8 @@ chat endpoint does: - Stop strings stop vLLM itself, so the token IDs match the native endpoint's. - The response carries `prompt_token_ids`, `completion_token_ids` and `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. + With router replay on, it also carries the call's `routed_experts`, as the + native endpoint attaches them. - Images, video and audio in the messages are processed as on the native endpoint. The frontend passes each media URL through with one unexpanded placeholder per item. The engine loads the media with vLLM's media connector @@ -145,9 +148,8 @@ Startup rejects these with `dynamo: true`: - `async_rl.generation_router`: the frontend already routes the fleet behind one URL. - `vllm_cfg.http_server_api_key_required`: the frontend checks no bearer token. -- On the SingleController, router replay with Gym but without token capture: - Gym's dialect does not return routed experts in responses. Other entrypoints - do not check this, and such rollouts replay no routes. +- `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data + parallelism inside vLLM, which the async engine Dynamo serves does not. - Parser plugins with no Dynamo parser, and the chat settings above that the frontend has no counterpart for. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 291aac186b6..9ad2b249a95 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -15,7 +15,8 @@ policy: # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: 600 # Timeout for one rollout routed through the frontend (in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS); a timed-out rollout is not retried. + # NRL_VLLM_ASYNC_TIMEOUT_SECONDS). A timeout fails the attempt as a native + # timeout does, and the rollout retry policy applies. request_timeout_s: 900 worker_args: # Dynamo's built-in parser names. null follows the native server's diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml index 010fee9cb49..d12f2536970 100644 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml @@ -82,7 +82,6 @@ policy: sequence_packing: fuse_loss: true generation: - backend: vllm dynamo: true dynamo_cfg: startup_timeout_s: 1800 diff --git a/nemo_rl/distributed/virtual_cluster.py b/nemo_rl/distributed/virtual_cluster.py index 049ddde289f..b3eda15b7ba 100644 --- a/nemo_rl/distributed/virtual_cluster.py +++ b/nemo_rl/distributed/virtual_cluster.py @@ -158,7 +158,6 @@ def uv_py_executable(extras: Sequence[str]) -> str: # 1400-1999 Master address / TCPStore (cluster.master_port_range_low/high) # [3000, 4999) Shared NeMo RL generation range (policy.generation.port_range_low/high) # [3000, 4000) Dynamo frontend HTTP endpoint -# [4000, 4100) Dynamo engine KV-event endpoints (node-local free-port selection) # 5000-5999 NeMo Gym HTTP servers (env.nemo_gym.port_range_low/high) # 6000 NeMo-Skills sandbox Nginx (NEMO_SKILLS_SANDBOX_PORT; ray.sub starts one # sidecar per allocated node, driver included) @@ -179,8 +178,6 @@ def uv_py_executable(extras: Sequence[str]) -> str: DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH = 1400 DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW = 3000 DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH = 4000 -DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW = 4000 -DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH = 4100 DEFAULT_GYM_PORT_RANGE_LOW = 5000 DEFAULT_GYM_PORT_RANGE_HIGH = 5999 # vLLM TP/DP rendezvous ports. Each engine gets PORTS_PER_ENGINE ports starting diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 888015266e3..c24aeddc52c 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -206,9 +206,20 @@ def configure_generation_config( "sleeps while the policy trains, and the Dynamo frontend would " "keep routing requests to it." ) + vllm_cfg = config["vllm_cfg"] + if vllm_cfg["expert_parallel_size"] > vllm_cfg["tensor_parallel_size"]: + # VllmGeneration refuses the async engine with DP inside vLLM. + raise ValueError( + "policy.generation.dynamo=true serves the async vLLM engine, " + "which does not run data parallelism inside vLLM " + "(vllm_cfg.expert_parallel_size > tensor_parallel_size, see " + "https://github.com/NVIDIA-NeMo/RL/issues/1101). Set " + "expert_parallel_size <= tensor_parallel_size, or remove " + "policy.generation.dynamo." + ) if not config["vllm_cfg"]["async_engine"]: - # Dynamo serves the engine's AsyncLLM. The async engine runs every - # rollout path the sync one does, so turning it on is the switch. + # Dynamo serves the engine's AsyncLLM, so turning the async engine + # on is the switch; settings it cannot run are rejected above. print( " [Dynamo] policy.generation.dynamo=true: using the async " "vLLM engine (vllm_cfg.async_engine=true)", diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index d08758ce3ed..7a6a61868d2 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -82,7 +82,9 @@ class DynamoCfg(BaseModel, extra="forbid"): # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: PositiveFloat = 600.0 # Timeout for one NeMo RL rollout routed through the frontend, in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS; a timed-out rollout is not retried. + # NRL_VLLM_ASYNC_TIMEOUT_SECONDS. A timeout fails the attempt as a native one + # does (GenerationUnavailable), and the rollout retry policy re-dispatches it + # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). # NeMo-Gym's own calls keep Gym's timeouts. request_timeout_s: PositiveFloat = 900.0 worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index df75a7d9f12..e4a0fe9507e 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -119,7 +119,9 @@ def engine_env(self) -> dict[str, str]: """The environment each engine's Dynamo worker needs to register.""" if not self._started: raise RuntimeError("Managed Dynamo runtime has not been started") - return dict(self._manager_env) + # An OS-assigned system port makes each engine serve its own model-card + # files (a custom chat template included) to a frontend on another node. + return {**self._manager_env, "DYN_SYSTEM_PORT": "0"} def start(self) -> None: """Start etcd, NATS and the frontend.""" diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index 2eb085d3e5d..39bff56a0fb 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -54,7 +54,7 @@ from dynamo.llm.exceptions import HttpError, InvalidArgument from vllm import envs as vllm_envs from vllm.entrypoints.chat_utils import load_chat_template -from vllm.exceptions import GracefulHTTPError +from vllm.exceptions import GracefulHTTPError, VLLMClientError from vllm.inputs import TokensPrompt from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY from vllm.sampling_params import ( @@ -63,14 +63,10 @@ StructuredOutputsParams, ) from vllm.transformers_utils.processor import get_video_processor_cls_name +from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase from nemo_rl.data.captured_media import MediaCaptureRejected -from nemo_rl.distributed.virtual_cluster import ( - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH, - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, - _get_free_port_local, -) from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, dynamo_cfg, @@ -150,16 +146,15 @@ def dynamo_kv_events_config( """ if cfg.frontend_args.router_mode != "kv" or not prefix_caching: return None - port = _get_free_port_local( - DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_LOW, DEFAULT_DYNAMO_SYSTEM_PORT_RANGE_HIGH - ) return { "enable_kv_cache_events": True, "publisher": "zmq", - # A wildcard host makes vLLM's publisher bind; any other TCP endpoint - # makes it connect, and Dynamo's ZMQ source connects too, so no event - # would flow. kv_event_sources subscribes on 127.0.0.1. - "endpoint": f"tcp://*:{port}", + # A Linux abstract-namespace socket (no port, no file): vLLM binds any + # ipc:// endpoint and this process's Dynamo worker connects to it, so + # engines on one node cannot collide. Random rather than the pid, since + # containers sharing the host network share these names. vLLM rejects + # ipc:// only at data_parallel_rank != 0, which Dynamo rejects at startup. + "endpoint": f"ipc://@nemo-rl-kv-events-{os.urandom(8).hex()}", } @@ -243,13 +238,14 @@ def build_sampling_params( {key: sampling[key] for key in _SAMPLING_KEYS if sampling.get(key) is not None} ) kwargs["n"] = n + # As vLLM's get_max_tokens: the generation config's max_tokens applies only + # when the request leaves max_tokens unset. + requested_max_tokens = stop.get("max_tokens") + if requested_max_tokens is None: + requested_max_tokens = defaults.get("max_tokens") kwargs["max_tokens"] = min( limit - for limit in ( - max(1, output_room), - stop.get("max_tokens"), - defaults.get("max_tokens"), - ) + for limit in (max(1, output_room), requested_max_tokens) if limit is not None ) if stop.get("min_tokens") is not None: @@ -354,7 +350,12 @@ async def start(self, worker_id: int) -> EngineConfig: for group in await self._llm.engine_core.call_utility_async( "get_kv_cache_group_metadata" ): - if group.get("kind") in ("full_attention", "mla_attention"): + # vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them. + if group.get("kind") in ( + KVCacheSpecKind.FULL_ATTENTION, + KVCacheSpecKind.MLA_ATTENTION, + KVCacheSpecKind.SINK_FULL_ATTENTION, + ): block_size = group["block_size"] break SnapshotStatLogger.num_gpu_blocks = cache.num_gpu_blocks or 0 @@ -378,6 +379,17 @@ async def start(self, worker_id: int) -> EngineConfig: async def generate( self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + try: + async for chunk in self._generate(request, context): + yield chunk + except VLLMClientError as error: + # vLLM's request errors are not ValueErrors; the native server + # answers them with a 4xx that carries the message. + raise InvalidArgument(str(error)) from error + + async def _generate( + self, request: GenerateRequest, context: Context ) -> AsyncGenerator[GenerateChunk, None]: if request.get("prompt_embeds"): raise InvalidArgument( @@ -655,6 +667,11 @@ async def _generate_gym( final_output = output if final_output is None: raise RuntimeError(f"No output received for request {context.id()}") + if admission is not None and context.is_stopped(): + # Dynamo aborts the vLLM request on a disconnect but never + # cancels this generator: fail the capture (below), as the + # native endpoint's cancellation does, instead of staging it. + return completion = final_output.outputs[0] chunk: GenerateChunk = { "token_ids": list(completion.token_ids), @@ -771,16 +788,24 @@ def _check_prompt_fits(self, prompt_len: int) -> None: "output tokens." ) - @staticmethod - def _gym_token_data(final_output: Any) -> dict[str, Any]: - """The token arrays Gym's dialect lifts onto the message, as the native endpoint reports them.""" + def _gym_token_data(self, final_output: Any) -> dict[str, Any]: + """The token fields Gym's dialect lifts onto the message, as the native endpoint reports them.""" + completion = final_output.outputs[0] token_data: dict[str, Any] = { "prompt_token_ids": list(final_output.prompt_token_ids), - "completion_token_ids": list(final_output.outputs[0].token_ids), + "completion_token_ids": list(completion.token_ids), } - if final_output.outputs[0].logprobs is not None: - _, token_data["completion_logprobs"] = chat_generation_tokens( - final_output.outputs[0] + if completion.logprobs is not None: + _, token_data["completion_logprobs"] = chat_generation_tokens(completion) + if self._host._return_routed_experts_enabled(): + # Full prompt + generation rows, as the native endpoint attaches them. + token_data["routed_experts"] = chat_routed_experts( + final_output, + completion, + prompt_token_count=len(final_output.prompt_token_ids), + device=torch.device("cpu"), + logger=LOGGER, + routed_experts_dtype=self._host.routed_experts_dtype, ) return token_data @@ -826,7 +851,7 @@ async def kv_event_sources(self) -> list[KvEventSource]: kv_events = self._llm.vllm_config.kv_events_config if kv_events is None or not kv_events.enable_kv_cache_events: return [] - return [ZmqSource(endpoint=kv_events.endpoint.replace("*", "127.0.0.1"))] + return [ZmqSource(endpoint=kv_events.endpoint)] def component_metrics_dp_ranks(self) -> list[int]: return [0] diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py index 82f5b897e77..95beb10d319 100644 --- a/nemo_rl/models/generation/dynamo/rollout_client.py +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -111,4 +111,12 @@ async def post_rollout( format_dynamo_error(response), ) await asyncio.sleep(_HTTP_RETRY_DELAY_S) + if "timeout_error" in response: + # Name the setting, as the native path's timeout message names its own. + raise RuntimeError( + f"Dynamo rollout request to {url} failed: {response['timeout_error']}. " + "For longer sequences, increase " + "policy.generation.dynamo_cfg.request_timeout_s, e.g. to " + f"{int(timeout_s * 2)}." + ) return parse_rollout_response(response, url=url) diff --git a/nemo_rl/models/generation/dynamo/venv.py b/nemo_rl/models/generation/dynamo/venv.py index 9a60b194e53..5708c8d0676 100644 --- a/nemo_rl/models/generation/dynamo/venv.py +++ b/nemo_rl/models/generation/dynamo/venv.py @@ -53,12 +53,15 @@ def get_dynamo_service_executable(name: str) -> str: if executable.is_file() and os.access(executable, os.X_OK): return str(executable) raise FileNotFoundError( - f"{name} is not an executable file in NEMO_RL_DYNAMO_BIN_DIR={bin_dir}." + f"{name} is not an executable file in NEMO_RL_DYNAMO_BIN_DIR={bin_dir}. " + "Build the image with --build-arg BUILD_DYNAMO=1, or install etcd and " + "nats-server there with docker/dynamo/install.sh." ) found = shutil.which(name) if found is None: raise FileNotFoundError( - f"{name} is not on PATH. Install it, or set NEMO_RL_DYNAMO_BIN_DIR to " - "the directory that holds etcd and nats-server." + f"{name} is not on PATH. Install etcd and nats-server with " + "docker/dynamo/install.sh and set NEMO_RL_DYNAMO_BIN_DIR to their " + "directory, or build the image with --build-arg BUILD_DYNAMO=1." ) return found diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 7dee0203c6f..1f6467032e1 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -56,15 +56,15 @@ VllmConfig, validate_vllm_quantization_config, ) -from nemo_rl.models.generation.vllm.request_pipeline import ( - rollout_output, - rollout_result_from_wire, -) from nemo_rl.models.generation.vllm.draft_utils import ( DRAFT_DISABLE_MODULE_SHARING_ENV, draft_module_sharing_disable_required, ) from nemo_rl.models.generation.vllm.metric_names import BATCH_DURATION_KEY +from nemo_rl.models.generation.vllm.request_pipeline import ( + rollout_output, + rollout_result_from_wire, +) from nemo_rl.models.generation.vllm.utils import ( aggregate_spec_decode_counters, assert_reload_refit_config_supported, @@ -188,10 +188,6 @@ def validate_settings(cls, master_config: "MasterConfig") -> None: generation_config = cast(VllmConfig, master_config.policy["generation"]) assert_reload_refit_config_supported(generation_config) if dynamo_enabled(generation_config): - # Deferred: the NeMo-Gym environment module is heavy and optional. - from nemo_rl.environments.nemo_gym import should_use_nemo_gym - from nemo_rl.models.megatron.router_replay import router_replay_enabled - if master_config.async_rl.generation_router.enabled: raise ValueError( "async_rl.generation_router.enabled=true does not apply with " @@ -206,17 +202,6 @@ def validate_settings(cls, master_config: "MasterConfig") -> None: "picks engines itself, and a restarted shard does not register " "with it. Disable fleet health." ) - if ( - should_use_nemo_gym(master_config) - and router_replay_enabled(master_config.policy) - and not master_config.token_capture.enabled - ): - raise ValueError( - "Router replay with NeMo-Gym and policy.generation.dynamo=true " - "requires token_capture.enabled=true: the engine stages routed " - "experts with the captured call, and NeMo-Gym's Dynamo dialect " - "does not return them in responses." - ) @staticmethod def init_cluster_placement_groups( diff --git a/ray.sub b/ray.sub index eb5b03a2a5f..1b9166fef64 100644 --- a/ray.sub +++ b/ray.sub @@ -189,8 +189,7 @@ fi # 1400-1999 Master address / TCPStore (cluster.master_port_range_low/high) # 2000-2999 Ray worker gRPC (min/max-worker-port) # [3000, 4999) Shared NeMo RL generation range (policy.generation.port_range_low/high) -# [3000, 4000) Dynamo frontend/token-wrapper HTTP endpoints -# [4000, 4100) Dynamo worker system endpoints (node-local free-port selection) +# [3000, 4000) Dynamo frontend HTTP endpoint # 5000-5999 NeMo Gym HTTP servers (Gym global config port_range_low/high) # 6000 Sandbox Nginx (NEMO_SKILLS_SANDBOX_PORT) # 6001-6999 Sandbox uWSGI workers (SANDBOX_BASE_PORT) diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index 225bbb65919..23d08060f17 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -37,6 +37,8 @@ run_test() { # Each keeps the native test's checks and adds check_dynamo_overlay.sh's. run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true ++policy.generation.dynamo=true +# Gym's default mode: token arrays come back through the frontend in engine_data. +run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true # Router replay with deferred routes: the engine stages each captured call's routed experts. run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index ba6edd103e9..5de0f0ef08a 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -40,7 +40,12 @@ def _vllm_config(**overrides) -> dict: "model_name": "Qwen/Qwen3-0.6B", "stop_token_ids": None, "stop_strings": None, - "vllm_cfg": {"async_engine": False, "max_model_len": 512}, + "vllm_cfg": { + "async_engine": False, + "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 1, + }, } config.update(overrides) return config @@ -76,6 +81,19 @@ def test_without_dynamo_the_vllm_config_is_untouched() -> None: (_vllm_config(dynamo=True, backend="sglang"), "backend=vllm"), # dynamo_cfg alone would be ignored. (_vllm_config(dynamo_cfg={}), "dynamo is not true"), + # The async engine Dynamo serves runs no data parallelism inside vLLM. + ( + _vllm_config( + dynamo=True, + vllm_cfg={ + "async_engine": False, + "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 2, + }, + ), + "expert_parallel_size <= tensor_parallel_size", + ), ], ) def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( @@ -133,6 +151,8 @@ def _serving(**serving_chat_kwargs) -> dict: vllm_cfg={ "async_engine": True, "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 1, "http_server_serving_chat_kwargs": serving_chat_kwargs, }, ) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 0b56635518c..d9cd8990bd8 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -132,7 +132,15 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( assert "NATS_STALE_SETTING" not in service_env assert service_env["NCCL_DEBUG"] == "INFO" + runtime._started = True + engine_env = runtime.engine_env + assert engine_env["DYN_NAMESPACE"] == "managed" + # Each engine serves its own model card, so a frontend on another node can + # read a chat template that exists only on the engine's node. + assert engine_env["DYN_SYSTEM_PORT"] == "0" + frontend_env = runtime._frontend_env() + assert "DYN_SYSTEM_PORT" not in frontend_env assert frontend_env["DYN_TOKENIZER"] == "fastokens" assert frontend_env["DYN_TOKENIZER_CACHE"] == "1" assert frontend_env["DYN_TOKENIZER_CACHE_BYTES"] == "4096" diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index f6ad9727175..63388e59ade 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -46,6 +46,7 @@ derive_prefix_eos_count, splice_prefix_tokens, ) +from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, @@ -229,7 +230,7 @@ def test_rollouts_are_served_like_generate_async() -> None: } -def test_router_replay_routes_reach_the_driver_as_generate_async_returns_them() -> None: +def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None: routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) final = _output( [1, 2, 3], @@ -265,6 +266,29 @@ def test_router_replay_routes_reach_the_driver_as_generate_async_returns_them() for key in native: assert torch.equal(shipped[key], native[key]), key + # A Gym call without capture carries the routes in engine_data, as the + # native chat endpoint attaches them to the response. + gym_final = _output( + [1, 3, 4], + _completion([10, 11], "stop", routed_experts=routes[3:]), + prompt_routed_experts=routes[:3], + ) + gym_host = _host( + _FakeLLM([gym_final]), vllm_kwargs={"enable_return_routed_experts": True} + ) + gym_engine = native_engine.NativeVllmEngine(gym_host, leader_worker_idx=0) + + (gym_chunk,) = _collect(gym_engine, _gym_request()) + + assert gym_chunk["engine_data"]["routed_experts"] == chat_routed_experts( + gym_final, + gym_final.outputs[0], + prompt_token_count=3, + device=torch.device("cpu"), + logger=native_engine.LOGGER, + routed_experts_dtype=torch.int16, + ) + def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: llm = _FakeLLM() @@ -300,7 +324,13 @@ def test_openai_requests_map_onto_sampling_params_as_vllms_server_does() -> None ) assert (params.temperature, params.top_k, params.seed, params.n) == (0.6, 5, 7, 2) - assert params.max_tokens == 9 + # The request's max_tokens, capped by the context; the generation config's + # max_tokens applies only when the request leaves it unset. + assert params.max_tokens == MAX_MODEL_LEN - 4 + unset = native_engine.build_sampling_params( + _openai_request(), {"max_tokens": 9}, output_room=MAX_MODEL_LEN - 4 + ) + assert unset.max_tokens == 9 assert sorted(params.stop_token_ids) == [2, 9] assert params.logprobs == 2 assert params.allowed_token_ids == [5, 6] @@ -502,6 +532,33 @@ async def registration_fails() -> None: ] +def test_engines_register_the_block_size_kv_events_are_hashed_with() -> None: + llm = _FakeLLM() + llm.vllm_config.cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=64) + llm.vllm_config.scheduler_config = SimpleNamespace( + max_num_seqs=8, max_num_batched_tokens=256 + ) + + async def call_utility_async(method): + assert method == "get_kv_cache_group_metadata" + # A hybrid model: the Mamba group's block size is not the attention one. + return [ + {"kind": "mamba", "block_size": 1056}, + {"kind": "full_attention", "block_size": 528}, + ] + + llm.engine_core = SimpleNamespace(call_utility_async=call_utility_async) + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + registration = asyncio.run(engine.start(worker_id=0)).llm + + # The KV router hashes events in blocks of the main-attention group; a + # wrong size would quietly turn KV routing into load-only routing. + assert registration.kv_cache_block_size == 528 + assert registration.total_kv_blocks == 64 + assert registration.context_length == MAX_MODEL_LEN + + def test_a_prompt_that_fills_the_context_is_refused_as_natively() -> None: llm = _FakeLLM() engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) @@ -559,6 +616,9 @@ def test_media_is_loaded_and_processed_as_vllms_chat_parser_does() -> None: }, "Cannot load image_url", ), + # vLLM's own request errors (not ValueErrors) keep their message, as the + # native server's 4xx does, instead of becoming a bare 500. + ({"stop_conditions": {"min_tokens": 10, "max_tokens": 5}}, "min_tokens"), ], ) def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: @@ -742,5 +802,6 @@ def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( assert (config is not None) is publishes if publishes: assert config["enable_kv_cache_events"] and config["publisher"] == "zmq" - # vLLM's publisher binds only a wildcard endpoint; Dynamo's source connects. - assert config["endpoint"].startswith("tcp://*:") + # A per-engine abstract socket, which vLLM's publisher binds and Dynamo's + # source connects to: no port for two engines on a node to collide on. + assert config["endpoint"].startswith("ipc://@") diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py index bb1a00de798..9c509a10fa0 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine_capture.py +++ b/tests/unit/models/generation/test_dynamo_native_engine_capture.py @@ -27,6 +27,8 @@ pytestmark = [pytest.mark.nemo_gym, pytest.mark.vllm] pytest.importorskip("dynamo._core") pytest.importorskip("vllm") +# The vLLM lanes import this module without the nemo_gym extra. +pytest.importorskip("nemo_gym.token_id_capture.staging") from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 @@ -35,6 +37,9 @@ DYNAMO_PREFIX_ANNOTATION_PREFIX, ) from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 +from responses_api_models.vllm_model.app import ( # noqa: E402 + _DYNAMO_ENGINE_DATA_FIELDS, +) from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 @@ -46,12 +51,12 @@ IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS, - _image_url, _collect, _completion, _FakeLLM, _gym_request, _host, + _image_url, _output, _Tokenizer, ) @@ -100,7 +105,7 @@ def _captured_request(admission: CaptureAdmission, **fields) -> dict: return request -def test_annotations_are_the_ones_gyms_dynamo_dialect_sends() -> None: +def test_annotations_and_engine_data_are_the_ones_gyms_dynamo_dialect_uses() -> None: assert native_engine.GYM_ROLLOUT_ANNOTATION == DYNAMO_GYM_REQUEST_ANNOTATION assert native_engine.GYM_CAPTURE_ANNOTATION_PREFIX == ( DYNAMO_CAPTURE_ANNOTATION_PREFIX @@ -108,6 +113,10 @@ def test_annotations_are_the_ones_gyms_dynamo_dialect_sends() -> None: assert native_engine.GYM_PREFIX_ANNOTATION_PREFIX == ( DYNAMO_PREFIX_ANNOTATION_PREFIX ) + # The engine_data keys Gym requires on a call without capture. + engine = native_engine.NativeVllmEngine(_host(_FakeLLM()), leader_worker_idx=0) + final = _output([1, 3, 4], _completion([10], "stop")) + assert engine._gym_token_data(final).keys() == _DYNAMO_ENGINE_DATA_FIELDS.keys() def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: diff --git a/tests/unit/models/generation/test_dynamo_rollout_client.py b/tests/unit/models/generation/test_dynamo_rollout_client.py index ac072ad72ad..1b2b942dc7f 100644 --- a/tests/unit/models/generation/test_dynamo_rollout_client.py +++ b/tests/unit/models/generation/test_dynamo_rollout_client.py @@ -11,13 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import json import pytest +from nemo_rl.models.generation.dynamo import rollout_client from nemo_rl.models.generation.dynamo.rollout_client import ( ROLLOUT_ANNOTATION, parse_rollout_response, + post_rollout, rollout_payload, ) @@ -58,10 +61,51 @@ def test_parse_rollout_response_rejects_unusable_responses(response, match) -> N parse_rollout_response(response, url="http://frontend/v1/completions") -def test_parse_rollout_response_returns_engine_data() -> None: - engine_data = {"rollout": {"token_ids": []}, "gen_leader_worker_idx": 1} +_ENGINE_DATA = {"rollout": {"token_ids": [8]}, "gen_leader_worker_idx": 1} - assert ( - parse_rollout_response({"nvext": {"engine_data": engine_data}}, url="u") - == engine_data - ) + +@pytest.mark.parametrize( + ("responses", "error"), + [ + # A transient failure is retried, and the rollout still succeeds. + ( + [ + {"status": "error", "http_status": 503}, + {"nvext": {"engine_data": _ENGINE_DATA}}, + ], + None, + ), + # A request the frontend refuses is final. + ([{"status": "error", "http_status": 400}], "HTTP 400"), + # A timeout is final, and names the setting that bounds it. + ( + [{"status": "error", "timeout_error": "no response within 1.0s"}], + "no response.*dynamo_cfg.request_timeout_s", + ), + # Retries are bounded. + ( + [{"status": "error", "http_status": 503}] + * rollout_client._HTTP_MAX_ATTEMPTS, + "HTTP 503", + ), + ], +) +def test_post_rollout_retries_only_transient_failures( + monkeypatch, responses, error +) -> None: + calls = [] + + async def fake_post(url, payload, timeout_s): + calls.append(payload) + return responses[len(calls) - 1] + + monkeypatch.setattr(rollout_client, "async_http_post_json", fake_post) + monkeypatch.setattr(rollout_client, "_HTTP_RETRY_DELAY_S", 0.0) + rollout = post_rollout("http://frontend/v1/completions", {}, timeout_s=1.0) + + if error is None: + assert asyncio.run(rollout) == _ENGINE_DATA + else: + with pytest.raises(RuntimeError, match=error): + asyncio.run(rollout) + assert len(calls) == len(responses) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index f3d4b7720c0..195a8bb3911 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2427,61 +2427,78 @@ def test_vllm_validate_settings_accepts_missing_vllm_cfg_as_refit_disabled(): VllmGeneration.validate_settings(master_config) -def _dynamo_master_config( - *, - router=False, - fleet_health=False, - nemo_gym=True, - router_replay=False, - token_capture=False, -): +def _dynamo_master_config(*, router=False, fleet_health=False): vllm_config = deepcopy(basic_vllm_test_config) vllm_config["dynamo"] = True vllm_config["vllm_cfg"]["async_engine"] = True vllm_config["vllm_cfg"]["expose_http_server"] = True return types.SimpleNamespace( - policy={"generation": vllm_config, "router_replay": {"enabled": router_replay}}, - env={"should_use_nemo_gym": nemo_gym}, + policy={"generation": vllm_config}, async_rl=types.SimpleNamespace( generation_router=types.SimpleNamespace(enabled=router), generation_fleet_health=types.SimpleNamespace(enabled=fleet_health), ), - token_capture=types.SimpleNamespace(enabled=token_capture), - ) - - -def test_vllm_validate_settings_accepts_dynamo_with_nemo_gym(): - VllmGeneration.validate_settings(_dynamo_master_config()) - VllmGeneration.validate_settings( - _dynamo_master_config(router_replay=True, token_capture=True) - ) - VllmGeneration.validate_settings( - _dynamo_master_config(router_replay=True, nemo_gym=False) ) @pytest.mark.parametrize( ("settings", "error"), [ + ({}, None), ({"router": True}, "routes the whole fleet behind one URL"), ({"fleet_health": True}, "Disable fleet health"), - ({"router_replay": True}, "requires token_capture.enabled=true"), ], ) def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error): + master_config = _dynamo_master_config(**settings) + if error is None: + VllmGeneration.validate_settings(master_config) + return with pytest.raises(ValueError, match=error): - VllmGeneration.validate_settings(_dynamo_master_config(**settings)) + VllmGeneration.validate_settings(master_config) @pytest.mark.parametrize("dynamo", [False, True]) -def test_vllm_client_base_urls_name_the_dynamo_frontend_when_it_serves(dynamo): +def test_vllm_token_rollouts_and_clients_use_the_dynamo_frontend_when_it_serves( + dynamo, +): generation = object.__new__(VllmGeneration) generation.dp_openai_server_base_urls = ["http://a/v1", "http://b/v1"] generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None + tokens = BatchedDataDict({"input_ids": torch.tensor([[1]])}) + # The frontend request carries token IDs only, so media prompts stay native. + media = BatchedDataDict({"input_ids": torch.tensor([[1]]), "vllm_content": [None]}) assert generation.client_base_urls == ( ["http://frontend/v1"] if dynamo else ["http://a/v1", "http://b/v1"] ) + assert generation._dynamo_serves(tokens) is dynamo + assert not generation._dynamo_serves(media) + + +def test_vllm_dynamo_frontend_failures_are_generation_unavailable(monkeypatch): + from nemo_rl.experience.failures import GenerationUnavailable + from nemo_rl.models.generation.dynamo import rollout_client + + async def frontend_down(url, payload, timeout_s): + raise RuntimeError(f"Dynamo rollout request to {url} failed: HTTP 503") + + monkeypatch.setattr(rollout_client, "post_rollout", frontend_down) + generation = object.__new__(VllmGeneration) + generation.cfg = { + "model_name": "model", + "_pad_token_id": 0, + "vllm_cfg": {"max_model_len": 16}, + } + generation.dynamo_frontend_url = "http://frontend/v1" + data = BatchedDataDict( + {"input_ids": torch.tensor([[5, 6, 0]]), "input_lengths": torch.tensor([2])} + ) + + # An infrastructure failure is re-dispatched; an unmapped RuntimeError would + # count as a DATA failure against the prompt. + with pytest.raises(GenerationUnavailable, match="HTTP 503"): + asyncio.run(anext(generation.generate_async(data))) def test_vllm_policy_generation(policy, test_input_data, tokenizer): From 2ba2caeecdd32f9bac31bb36e7426a02c773efb9 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Sat, 24 Oct 2026 17:27:00 -0700 Subject: [PATCH 03/19] style(dynamo): sort the test's chat_routed_experts import Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/models/generation/test_dynamo_native_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index 63388e59ade..836b70c5ee5 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -46,13 +46,13 @@ derive_prefix_eos_count, splice_prefix_tokens, ) -from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, rollout_result_from_wire, rollout_result_to_wire, ) +from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) From 94c0e96e2b4849edc88dd5786335920be4b0fe88 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 26 Oct 2026 14:19:32 -0700 Subject: [PATCH 04/19] build(gym): pin Gym#4817 rebased onto Gym main Gym#4817 (jthomson04/dynamo-frontend-dialect) is rebased onto Gym main and now sits on top of it; its head moves from a369e28 to a5f936c. The only conflict was Gym main's new custom-tool-strategy validator, which sits next to the Dynamo frontend validator; both are kept. Gym's dependencies are unchanged, so uv.lock needs no change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index a369e289f70..a5f936c473d 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit a369e289f702c623b077e811eceeb98f2b3b26cf +Subproject commit a5f936c473dbd477ce4632373eaba884272b7f6a From e1828110d0e2a4e91ef8cf7629924641746583e9 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 26 Oct 2026 15:50:32 -0700 Subject: [PATCH 05/19] fix(dynamo): apply the Gym Dynamo dialect per actor, after shard composition From the #4905 review: - C23: setup_nemo_gym_config wrote the dialect as a top-level policy_model overlay, which a sharded Gym plan rejects as unowned. Setup now stashes it under a NeMo RL key (dynamo_dialect, alongside tokenizer_config), and _build_gym_actor_config applies it to each actor's composed config. A shard's own template kwargs still win over the native defaults. - C24: the local install instructions export NEMO_RL_DYNAMO_BIN_DIR, so the run that follows finds etcd and nats-server. - C25: the two in-function rationale blocks in native_engine.py are one line each; the design doc carries the full reasons. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 11 +++- docs/guides/dynamo-generation.md | 6 +- nemo_rl/environments/nemo_gym.py | 65 ++++++++++++------ nemo_rl/environments/nemo_gym_shards.py | 7 +- .../models/generation/dynamo/native_engine.py | 12 +--- .../test_nemo_gym_token_capture.py | 36 +++++++--- .../unit/environments/test_nemo_gym_utils.py | 66 +++++++++++++++++++ 7 files changed, 158 insertions(+), 45 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 0fea1ccc5fd..843dad2c897 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -52,12 +52,17 @@ implements its `LLMEngine` over the worker's `AsyncLLM`: - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It publishes no token budget. The engine owns the context check and the output - cap, because a prefix splice changes the prompt after the frontend has seen it. + cap, as the native server does. With a budget, the frontend would fill an + omitted `max_tokens` from the prompt it rendered, which a prefix splice then + changes, and would refuse NeMo RL's own rollouts that fill the context. - `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ publisher. It is set only when the engine caches prefixes. Each engine binds its own Linux abstract-namespace socket (`ipc://@...`), so engines on one node - need no port and cannot collide. A vLLM stat logger publishes the router's - per-engine load signal. + need no port and cannot collide. The name is random rather than the pid, + because containers that share the host network share these names. vLLM refuses + `ipc://` only above data-parallel rank 0, and vLLM data parallelism is rejected + at startup with Dynamo. A vLLM stat logger publishes the router's per-engine + load signal. Because the Worker runs on the actor's own loop, the worker's other methods interleave with Dynamo requests exactly as with native requests: diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index bc554ec422f..f1764202897 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -173,10 +173,12 @@ docker buildx build \ . ``` -For a local checkout, install the binaries with: +For a local checkout, install the binaries and keep the variable set for the +runs that follow. The runtime looks for etcd and nats-server there, or else on `PATH`: ```bash -NEMO_RL_DYNAMO_BIN_DIR=$PWD/venvs/dynamo-bin bash docker/dynamo/install.sh +export NEMO_RL_DYNAMO_BIN_DIR="$PWD/venvs/dynamo-bin" +bash docker/dynamo/install.sh ``` ## Run diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 017f5d152dd..3a47fbeb847 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -1897,27 +1897,14 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_token_ids"] = None if dynamo_enabled(generation_config): - # Gym's policy model calls the Dynamo frontend, which takes token data in - # nvext (Gym's Dynamo dialect) instead of vLLM's request extensions. - vllm_model = ( - config.env.setdefault("nemo_gym", {}) - .setdefault("policy_model", {}) - .setdefault("responses_api_models", {}) - .setdefault("vllm_model", {}) - ) - vllm_model["dynamo_frontend"] = True - # The frontend reads `reasoning` as another name for `reasoning_content` and - # refuses an echoed assistant turn that carries both, Gym's default. - vllm_model.setdefault("reasoning_field", "reasoning_content") - # The native server merges these under every request's own template - # kwargs; the frontend has no such default, so Gym sends them. They sit - # under the kwargs Gym sends: vllm_model's own, else extra_body's. - default_kwargs = native_default_chat_template_kwargs(generation_config) - if default_kwargs: - gym_kwargs = vllm_model.get("chat_template_kwargs") or ( - (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} + # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect. + # Each actor applies it to its own composed config (_apply_dynamo_dialect), + # so it never becomes a policy_model overlay that a shard must claim. + config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = { + "default_chat_template_kwargs": native_default_chat_template_kwargs( + generation_config ) - vllm_model["chat_template_kwargs"] = {**default_kwargs, **gym_kwargs} + } # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for @@ -1987,6 +1974,39 @@ def build_nemo_gym_config( ) +def _apply_dynamo_dialect( + nemo_gym_dict: dict[str, Any], default_chat_template_kwargs: Mapping[str, Any] +) -> None: + """Point one actor's policy model at the Dynamo frontend, in Gym's Dynamo dialect. + + The frontend takes token data in nvext instead of vLLM's request extensions. + Applied to the actor's composed config (a shard's overrides included), so + settings the user wrote there win. Copies the policy_model levels it + changes, so the caller's config is not mutated. + """ + policy_model = dict(nemo_gym_dict.get("policy_model") or {}) + models = dict(policy_model.get("responses_api_models") or {}) + vllm_model = dict(models.get("vllm_model") or {}) + vllm_model["dynamo_frontend"] = True + # The frontend reads `reasoning` as another name for `reasoning_content` and + # refuses an echoed assistant turn that carries both, Gym's default. + vllm_model.setdefault("reasoning_field", "reasoning_content") + # The native server merges these under every request's own template + # kwargs; the frontend has no such default, so Gym sends them. They sit + # under the kwargs Gym sends: vllm_model's own, else extra_body's. + if default_chat_template_kwargs: + gym_kwargs = vllm_model.get("chat_template_kwargs") or ( + (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} + ) + vllm_model["chat_template_kwargs"] = { + **default_chat_template_kwargs, + **gym_kwargs, + } + models["vllm_model"] = vllm_model + policy_model["responses_api_models"] = models + nemo_gym_dict["policy_model"] = policy_model + + def _build_gym_actor_config( nemo_gym_dict: dict[str, Any], *, @@ -2015,6 +2035,11 @@ def _build_gym_actor_config( invalid_tool_call_patterns = nemo_gym_dict.pop("invalid_tool_call_patterns", None) thinking_tags = nemo_gym_dict.pop("thinking_tags", None) tokenizer_config = nemo_gym_dict.pop("tokenizer_config", None) + dynamo_dialect = nemo_gym_dict.pop("dynamo_dialect", None) + if dynamo_dialect is not None: + _apply_dynamo_dialect( + nemo_gym_dict, dynamo_dialect["default_chat_template_kwargs"] + ) external_service_readiness = extract_external_service_readiness(nemo_gym_dict) port_range = { key: value diff --git a/nemo_rl/environments/nemo_gym_shards.py b/nemo_rl/environments/nemo_gym_shards.py index d8322a8d23c..b03b9cd9eb6 100644 --- a/nemo_rl/environments/nemo_gym_shards.py +++ b/nemo_rl/environments/nemo_gym_shards.py @@ -38,7 +38,12 @@ # Dict-shaped NeMo RL integration settings that are not Gym server entries. # The actor factory or rollout code consumes these separately. NEMO_RL_DICT_CONFIG_KEYS = frozenset( - {"effort_levels", "external_service_readiness", "tokenizer_config"} + { + "dynamo_dialect", + "effort_levels", + "external_service_readiness", + "tokenizer_config", + } ) # Ray placement-group strategies a shard plan may ask for. STRICT_SPREAD is the diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index 39bff56a0fb..b3221afcf53 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -149,11 +149,7 @@ def dynamo_kv_events_config( return { "enable_kv_cache_events": True, "publisher": "zmq", - # A Linux abstract-namespace socket (no port, no file): vLLM binds any - # ipc:// endpoint and this process's Dynamo worker connects to it, so - # engines on one node cannot collide. Random rather than the pid, since - # containers sharing the host network share these names. vLLM rejects - # ipc:// only at data_parallel_rank != 0, which Dynamo rejects at startup. + # A random abstract-namespace socket per engine (dynamo-integration.md). "endpoint": f"ipc://@nemo-rl-kv-events-{os.urandom(8).hex()}", } @@ -363,11 +359,7 @@ async def start(self, worker_id: int) -> EngineConfig: return EngineConfig( model=model, served_model_name=model, - # No token budget: the engine owns prompt overflow and the output - # cap, as the native server does. A budget would make the frontend - # fill an omitted max_tokens from the prompt it rendered, which a - # NeMo-Gym prefix splice then changes, and refuse NeMo RL's own - # rollouts that fill the context. + # No token budget: the engine owns overflow (dynamo-integration.md). llm=LlmRegistration( context_length=self._max_model_len, kv_cache_block_size=block_size, diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index 3469ac88884..ac3707a315d 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -604,12 +604,11 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( assert vllm_cfg["async_engine"] is True assert vllm_cfg["expose_http_server"] is True assert should_use_nemo_gym(config) is True - policy_model = config.env["nemo_gym"].get("policy_model") if dynamo else None if dynamo: - assert policy_model["responses_api_models"]["vllm_model"] == { - "dynamo_frontend": True, - # The frontend refuses reasoning under both of Gym's keys. - "reasoning_field": "reasoning_content", + # A NeMo RL key each actor applies to its composed config, not a + # policy_model overlay a sharded plan would need an owner for. + assert config.env["nemo_gym"] == { + "dynamo_dialect": {"default_chat_template_kwargs": {}} } else: assert "nemo_gym" not in config.env @@ -633,12 +632,17 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( ), ], ) -def test_setup_nemo_gym_config_dynamo_sends_the_native_default_template_kwargs( +def test_gym_actor_config_dynamo_sends_the_native_default_template_kwargs( vllm_model: dict, expected: dict ) -> None: """The native server's default template kwargs sit under the ones Gym sends.""" from nemo_rl.algorithms.grpo import MasterConfig - from nemo_rl.environments.nemo_gym import setup_nemo_gym_config + from nemo_rl.environments.nemo_gym import ( + build_nemo_gym_config, + setup_nemo_gym_config, + ) + + user_vllm_model = copy.deepcopy(vllm_model) config = MasterConfig.model_construct( env={ @@ -665,5 +669,19 @@ def test_setup_nemo_gym_config_dynamo_sends_the_native_default_template_kwargs( ) setup_nemo_gym_config(config, tokenizer=None) - - assert vllm_model["chat_template_kwargs"] == expected + gym_config = build_nemo_gym_config( + config.env, + base_urls=["http://frontend"], + model_name="test-model", + enable_router_replay=False, + use_fastokens=False, + )["initial_global_config_dict"] + + composed = gym_config["policy_model"]["responses_api_models"]["vllm_model"] + assert composed["dynamo_frontend"] is True + # The frontend refuses reasoning under both of Gym's keys. + assert composed["reasoning_field"] == "reasoning_content" + assert composed["chat_template_kwargs"] == expected + assert "dynamo_dialect" not in gym_config + # The user's overlay is left as written. + assert vllm_model == user_vllm_model diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index ea0b267beef..bea407f3213 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -932,6 +932,72 @@ def test_build_nemo_gym_actors_hands_every_actor_the_policy_api_key( assert all(cfg["policy_api_key"] == "per-job-key" for cfg in cluster.actor_configs) +@pytest.mark.parametrize( + "env_configs", [_env_configs, _shard_env_configs], ids=["unsharded", "sharded"] +) +def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( + detected_uv_dirs, env_configs +): + """With Dynamo, each actor's composed config speaks Gym's Dynamo dialect. + + The dialect is not a policy_model overlay, which a sharded plan would reject + as unowned. A shard's own template kwargs still win over the native defaults. + """ + from nemo_rl.algorithms.grpo import MasterConfig + + env = env_configs() + sharded = "shards" in env["nemo_gym"] + if sharded: + env["nemo_gym"]["shards"][1]["overrides"] = { + "policy_model": { + "responses_api_models": { + "vllm_model": {"chat_template_kwargs": {"enable_thinking": False}} + } + } + } + config = MasterConfig.model_construct( + env=env, + policy={ + "generation": { + "backend": "vllm", + "dynamo": True, + "vllm_cfg": { + "http_server_serving_chat_kwargs": { + "default_chat_template_kwargs": {"enable_thinking": True} + } + }, + } + }, + ) + setup_nemo_gym_config(config, tokenizer=None) + cluster = _FakeGymCluster() + + with _patched_cluster(cluster): + nemo_gym_mod.build_nemo_gym_actors( + config.env, + base_urls=["http://frontend"], + model_name="test-model", + tokenizer=_TOKENIZER, + enable_router_replay=False, + use_fastokens=False, + ) + + thinking = [ + cfg["initial_global_config_dict"]["policy_model"]["responses_api_models"][ + "vllm_model" + ]["chat_template_kwargs"]["enable_thinking"] + for cfg in cluster.actor_configs + ] + # judged, then the tools shard's two replicas with their own override. + assert thinking == ([True, False, False] if sharded else [True]) + for cfg in cluster.actor_configs: + gym_config = cfg["initial_global_config_dict"] + vllm_model = gym_config["policy_model"]["responses_api_models"]["vllm_model"] + assert vllm_model["dynamo_frontend"] is True + assert vllm_model["reasoning_field"] == "reasoning_content" + assert "dynamo_dialect" not in gym_config + + def test_build_nemo_gym_actors_spreads_every_replica_onto_its_own_node( detected_uv_dirs, ): From e0d3789f2d9bce3afd0ec9a1894cd27d87a49a02 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 09:39:37 -0700 Subject: [PATCH 06/19] refactor(vllm): share the per-sample rollout pipeline and chat token helpers Split generate_async's per-sample work into building the result (request_pipeline.rollout_result: sampled logprobs, router-replay route alignment and its fallback warning) and assembling the output tensors (rollout_output). The rollout token budget and SamplingParams become worker methods (_rollout_max_new_tokens, _rollout_sampling_params). vllm/utils gains off_policy_sampling_error (the chat endpoint's on-policy checks, unchanged) and chat_routed_experts / chat_generation_tokens (the loop bodies of the attach_* response helpers). No behaviour change for native vLLM: the new generate_async wiring tests pass against both the old and the new implementation. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/vllm/request_pipeline.py | 177 +++++++++ nemo_rl/models/generation/vllm/utils.py | 243 ++++++++---- .../generation/vllm/vllm_worker_async.py | 359 +++++------------- pyrefly.toml | 1 + .../models/generation/test_vllm_generation.py | 125 ++++++ .../generation/test_vllm_request_pipeline.py | 124 ++++++ .../unit/models/generation/test_vllm_utils.py | 27 ++ 7 files changed, 717 insertions(+), 339 deletions(-) create mode 100644 nemo_rl/models/generation/vllm/request_pipeline.py create mode 100644 tests/unit/models/generation/test_vllm_request_pipeline.py diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py new file mode 100644 index 00000000000..bb77e3b0e27 --- /dev/null +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -0,0 +1,177 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Turn a finished vLLM rollout into NeMo RL's per-sample generation output. + +Building the per-sample result is kept apart from assembling the output tensors, +so a path that builds the result on the engine and assembles the output +elsewhere behaves the same as the worker's own ``generate_async``. A rollout +feature added here, such as router replay (R3), then behaves the same on every +path. +""" + +import logging +from dataclasses import dataclass +from typing import Any, Optional, cast + +import torch + +from nemo_rl.distributed.batched_data_dict import BatchedDataDict +from nemo_rl.models.generation.interfaces import GenerationOutputSpec +from nemo_rl.models.generation.vllm.utils import pad_and_align_routed_expert_indices + +LOGGER = logging.getLogger(__name__) + + +@dataclass +class RolloutResult: + """One rollout's outcome, independent of where its output is assembled.""" + + token_ids: list[int] + # The sampled token's log probability at each position; 0.0 where vLLM + # reported none. + logprobs: list[float] + # The engine stopped at max_tokens. + truncated: bool + # [prompt + generated, num_moe_layers, topk], aligned for router replay. + routed_experts: Optional[torch.Tensor] + # missing/expected/actual route counts; set when router replay is on. + r3_stats: Optional[dict[str, int]] + + @classmethod + def nothing_generated(cls) -> "RolloutResult": + """The result when no new token fits: the prompt alone, untruncated.""" + return cls([], [], False, None, None) + + +def selected_logprobs(completion: Any) -> list[float]: + """The sampled token's log probability at each position of a completion.""" + by_position = completion.logprobs or () + logprobs = [] + for idx, token_id in enumerate(completion.token_ids): + candidates = by_position[idx] if idx < len(by_position) else None + entry = candidates.get(token_id) if candidates else None + logprobs.append(0.0 if entry is None else float(entry.logprob)) + return logprobs + + +def rollout_result( + final_output: Any, + *, + input_length: int, + return_routed_experts: bool, + routed_experts_dtype: torch.dtype, + device: torch.device, + sample_idx: int = 0, +) -> RolloutResult: + """Build the result of a finished rollout from vLLM's final RequestOutput. + + Args: + final_output: The request's last (cumulative) ``RequestOutput``. + input_length: Prompt length in tokens. + return_routed_experts: Router replay is on, so routes are required. + routed_experts_dtype: The dtype router replay consumes. + device: Where to place the routed-experts tensor. + sample_idx: For log messages only. + """ + completion = final_output.outputs[0] + token_ids = list(completion.token_ids) + total_length = input_length + len(token_ids) + # return_stats=True returns the (routes, stats) pair. + routed_experts, r3_stats = cast( + tuple[Optional[torch.Tensor], dict[str, int]], + pad_and_align_routed_expert_indices( + final_output, + completion, + valid_length=total_length, + padded_length=total_length, + device=device, + require_complete_routed_experts=return_routed_experts, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ), + ) + if return_routed_experts and routed_experts is None: + raise RuntimeError( + "vLLM was asked to return routed experts but the generation output " + "did not include routed_experts." + ) + if return_routed_experts and r3_stats["missing_routes"] > 0: + LOGGER.warning( + "R3 router replay fallback: vLLM returned incomplete " + "routed_experts for sample_idx=%d, missing_token_routes=%d, " + "actual_routes=%d, expected_routes=%d. Megatron will use its " + "own router for those missing token routes.", + sample_idx, + r3_stats["missing_routes"], + r3_stats["actual_routes"], + r3_stats["expected_routes"], + ) + return RolloutResult( + token_ids=token_ids, + logprobs=selected_logprobs(completion), + truncated=completion.finish_reason == "length", + routed_experts=routed_experts, + r3_stats=r3_stats if return_routed_experts else None, + ) + + +def rollout_output( + result: RolloutResult, + *, + input_ids: torch.Tensor, + input_length: int, + pad_token_id: int, +) -> BatchedDataDict[GenerationOutputSpec]: + """The single-sample output ``generate_async`` yields for ``result``. + + Args: + result: The rollout's result. + input_ids: The sample's (right-padded) prompt row. + input_length: Prompt length in tokens. + pad_token_id: The tokenizer's pad token, as ``generate_async`` uses it. + """ + device = input_ids.device + generated = result.token_ids + output_length = input_length + len(generated) + output_ids = torch.full( + (output_length,), pad_token_id, dtype=input_ids.dtype, device=device + ) + output_ids[:input_length] = input_ids[:input_length] + output_ids[input_length:] = torch.tensor( + generated, dtype=input_ids.dtype, device=device + ) + logprobs = torch.zeros((1, output_length), dtype=torch.float32, device=device) + logprobs[0, input_length:] = torch.tensor( + result.logprobs, dtype=torch.float32, device=device + ) + output: dict[str, Any] = { + "output_ids": output_ids.unsqueeze(0), + "logprobs": logprobs, + "generation_lengths": torch.tensor( + [len(generated)], dtype=torch.long, device=device + ), + "unpadded_sequence_lengths": torch.tensor( + [output_length], dtype=torch.long, device=device + ), + "truncated": torch.tensor([result.truncated], dtype=torch.bool, device=device), + } + if result.r3_stats is not None: + for key in ("missing_routes", "expected_routes", "actual_routes"): + output[f"r3_routed_experts_{key}"] = torch.tensor( + [result.r3_stats[key]], dtype=torch.long, device=device + ) + if result.routed_experts is not None: + output["routed_experts"] = result.routed_experts.to(device).unsqueeze(0) + return BatchedDataDict[GenerationOutputSpec](output) diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index b77da9c6e6f..2d871819ae7 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -318,6 +318,57 @@ def _as_routed_experts_tensor( return tensor.to(dtype=dtype) +def off_policy_sampling_error( + *, + temperature: Optional[float], + top_p: Optional[float], + top_k: Optional[int], + generation_config: Any, +) -> Optional[str]: + """Why an HTTP rollout request would sample off-policy, or None if it would not. + + Request sampling must match NeMo RL's exactly, or inference runs off-policy and + destabilizes training. Validation rollouts are the one exception: they carry the + validation profile (``val_temperature`` / ``val_top_p``), which is metric-only. + Multi-turn agents issue their own requests, so the serving HTTP endpoint is the + one chokepoint they all pass. + + Args: + temperature: The request's temperature. + top_p: The request's top_p. It must be explicit: vLLM resolves an unset top_p + from the model's generation_config.json, not to 1.0. + top_k: The request's top_k, which must be unset or -1 (disabled), matching + ``BaseVllmGenerationWorker._build_sampling_params``. + generation_config: ``policy.generation``, with the train and validation + sampling parameters. + """ + if top_k not in (None, -1): + return f"Top k sampling parameter must be unset, empty, or -1. Got `{top_k}`" + if top_p is None: + return ( + "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " + "resolved by vLLM from the model's generation_config.json and would " + "bypass the on-policy sampling check." + ) + is_train_sampling = ( + temperature == generation_config["temperature"] + and top_p == generation_config["top_p"] + ) + is_val_sampling = ( + temperature == generation_config["val_temperature"] + and top_p == generation_config["val_top_p"] + ) + if is_train_sampling or is_val_sampling: + return None + return ( + f"request sampling (temperature={temperature}, top_p={top_p}) matches neither " + f"the train sampling params (temperature={generation_config['temperature']}, " + f"top_p={generation_config['top_p']}) nor the validation sampling params " + f"(val_temperature={generation_config['val_temperature']}, " + f"val_top_p={generation_config['val_top_p']})" + ) + + def validate_rollout_prompt(expected: list[int], actual: list[int] | None) -> None: """Reject rollouts generated from a different prompt than the learner's.""" if actual != expected: @@ -496,6 +547,58 @@ def pad_and_align_routed_expert_indices( return (full, stats) if return_stats else full +def chat_routed_experts( + final_request_output: Any, + generation_details: Any, + *, + prompt_token_count: int, + device: torch.device, + logger: Any, + routed_experts_dtype: torch.dtype, +) -> str: + """The routed-experts envelope the chat endpoint attaches to one choice.""" + generation_token_count = len(getattr(generation_details, "token_ids", []) or []) + routed_result = pad_and_align_routed_expert_indices( + final_request_output, + generation_details, + valid_length=prompt_token_count + generation_token_count, + padded_length=prompt_token_count + generation_token_count, + device=device, + require_complete_routed_experts=True, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ) + if not isinstance(routed_result, tuple): + raise RuntimeError( + "Expected routed_experts alignment to return stats for the " + "OpenAI-compatible chat endpoint." + ) + routed_experts, r3_stats = routed_result + if routed_experts is None: + raise RuntimeError( + "vLLM was asked to return routed experts for the " + "OpenAI-compatible chat endpoint but the generation " + "output did not include routed_experts." + ) + if r3_stats["missing_routes"] > 0 and logger is not None: + logger.warning( + "R3 router replay fallback: vLLM returned incomplete " + "routed_experts for chat choice_idx=%d, " + "missing_token_routes=%d, actual_routes=%d, " + "expected_routes=%d. Megatron will use its own router " + "for those missing token routes.", + generation_details.index, + r3_stats["missing_routes"], + r3_stats["actual_routes"], + r3_stats["expected_routes"], + ) + # Base64 envelope instead of .tolist(): nested JSON int lists cost + # ~1s of CPU per serialize/parse hop at long context lengths and get + # re-validated at every gym HTTP hop; a single string passes through + # the gym chain opaquely. + return encode_routed_experts(routed_experts.to(dtype=routed_experts_dtype)) + + def attach_routed_experts_to_chat_response_choices( response: Any, final_request_output: Any, @@ -519,49 +622,14 @@ def attach_routed_experts_to_chat_response_choices( if generation_details is None: continue attached_choice_indices.add(choice.index) - - generation_token_count = len(getattr(generation_details, "token_ids", []) or []) - routed_result = pad_and_align_routed_expert_indices( + choice.message.routed_experts = chat_routed_experts( final_request_output, generation_details, - valid_length=prompt_token_count + generation_token_count, - padded_length=prompt_token_count + generation_token_count, + prompt_token_count=prompt_token_count, device=device, - require_complete_routed_experts=True, - return_stats=True, + logger=logger, routed_experts_dtype=routed_experts_dtype, ) - if not isinstance(routed_result, tuple): - raise RuntimeError( - "Expected routed_experts alignment to return stats for the " - "OpenAI-compatible chat endpoint." - ) - routed_experts, r3_stats = routed_result - if routed_experts is None: - raise RuntimeError( - "vLLM was asked to return routed experts for the " - "OpenAI-compatible chat endpoint but the generation " - "output did not include routed_experts." - ) - if r3_stats["missing_routes"] > 0 and logger is not None: - logger.warning( - "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for chat choice_idx=%d, " - "missing_token_routes=%d, actual_routes=%d, " - "expected_routes=%d. Megatron will use its own router " - "for those missing token routes.", - choice.index, - r3_stats["missing_routes"], - r3_stats["actual_routes"], - r3_stats["expected_routes"], - ) - # Base64 envelope instead of .tolist(): nested JSON int lists cost - # ~1s of CPU per serialize/parse hop at long context lengths and get - # re-validated at every gym HTTP hop; a single string passes through - # the gym chain opaquely. - choice.message.routed_experts = encode_routed_experts( - routed_experts.to(dtype=routed_experts_dtype) - ) if len(attached_choice_indices) != len(choices): missing_choice_indices = sorted( @@ -623,50 +691,9 @@ def attach_token_information_to_chat_response_choices( ) for choice in choices: - generation_details = outputs_by_index[choice.index] - output_token_ids = getattr(generation_details, "token_ids", None) - if output_token_ids is None: - raise RuntimeError( - "vLLM was asked to return token information for the " - "OpenAI-compatible chat endpoint but generation output " - f"choice_idx={choice.index} did not include token_ids." - ) - generation_token_ids = list(output_token_ids) - - generation_logprob_details = getattr(generation_details, "logprobs", None) - if generation_logprob_details is None: - if generation_token_ids: - raise RuntimeError( - "vLLM was asked to return token information for the " - "OpenAI-compatible chat endpoint but generation output " - f"choice_idx={choice.index} did not include logprobs." - ) - generation_log_probs = [] - else: - if len(generation_token_ids) != len(generation_logprob_details): - raise RuntimeError( - "vLLM returned mismatched generation token IDs and log " - "probabilities for the OpenAI-compatible chat endpoint: " - f"choice_idx={choice.index}, " - f"token_count={len(generation_token_ids)}, " - f"logprob_count={len(generation_logprob_details)}." - ) - generation_log_probs = [] - for token_id, position_logprobs in zip( - generation_token_ids, generation_logprob_details - ): - selected_token_logprob = position_logprobs.get(token_id) - if selected_token_logprob is None: - raise RuntimeError( - "vLLM generation log probabilities did not include the " - "selected token while attaching token information to " - "the OpenAI-compatible chat response: " - f"choice_idx={choice.index}, token_id={token_id}." - ) - generation_log_probs.append( - max(float(selected_token_logprob.logprob), VLLM_LOGPROB_FLOOR) - ) - + generation_token_ids, generation_log_probs = chat_generation_tokens( + outputs_by_index[choice.index] + ) choice.message.prompt_token_ids = list(prompt_token_ids) choice.message.generation_token_ids = generation_token_ids choice.message.generation_log_probs = generation_log_probs @@ -674,6 +701,58 @@ def attach_token_information_to_chat_response_choices( return response +def chat_generation_tokens(generation_details: Any) -> tuple[list[int], list[float]]: + """One choice's generated token IDs and their sampled-token log probabilities. + + As the chat endpoint reports them in ``generation_token_ids`` and + ``generation_log_probs``: each log probability is floored at + ``VLLM_LOGPROB_FLOOR``. + """ + choice_index = generation_details.index + output_token_ids = getattr(generation_details, "token_ids", None) + if output_token_ids is None: + raise RuntimeError( + "vLLM was asked to return token information for the " + "OpenAI-compatible chat endpoint but generation output " + f"choice_idx={choice_index} did not include token_ids." + ) + generation_token_ids = list(output_token_ids) + + generation_logprob_details = getattr(generation_details, "logprobs", None) + if generation_logprob_details is None: + if generation_token_ids: + raise RuntimeError( + "vLLM was asked to return token information for the " + "OpenAI-compatible chat endpoint but generation output " + f"choice_idx={choice_index} did not include logprobs." + ) + return generation_token_ids, [] + if len(generation_token_ids) != len(generation_logprob_details): + raise RuntimeError( + "vLLM returned mismatched generation token IDs and log " + "probabilities for the OpenAI-compatible chat endpoint: " + f"choice_idx={choice_index}, " + f"token_count={len(generation_token_ids)}, " + f"logprob_count={len(generation_logprob_details)}." + ) + generation_log_probs = [] + for token_id, position_logprobs in zip( + generation_token_ids, generation_logprob_details + ): + selected_token_logprob = position_logprobs.get(token_id) + if selected_token_logprob is None: + raise RuntimeError( + "vLLM generation log probabilities did not include the " + "selected token while attaching token information to " + "the OpenAI-compatible chat response: " + f"choice_idx={choice_index}, token_id={token_id}." + ) + generation_log_probs.append( + max(float(selected_token_logprob.logprob), VLLM_LOGPROB_FLOOR) + ) + return generation_token_ids, generation_log_probs + + def model_dump_chat_response_with_dynamic_message_fields( response: Any, ) -> dict[str, Any]: diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 9231aa06d22..2c67cc2a1f8 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -64,13 +64,18 @@ resolve_collective_rpc_result, ) from nemo_rl.models.generation.vllm.config import parse_nvfp4_pertoken_rollout +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result, +) from nemo_rl.models.generation.vllm.utils import ( attach_routed_experts_to_chat_response_choices, attach_token_information_to_chat_response_choices, format_prompt_for_vllm_generation, + off_policy_sampling_error, validate_rollout_prompt, model_dump_chat_response_with_dynamic_message_fields, - pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) from nemo_rl.models.generation.vllm.vllm_worker import BaseVllmGenerationWorker @@ -1473,46 +1478,19 @@ class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): async def create_chat_completion( request: NeMoRLChatCompletionRequest, raw_request: Request ): - # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params - # Right now we explicitly assert set this to -1. - assert request.top_k in (None, -1), ( - f"Top k sampling parameter must be unset, empty, or -1. Got `{request.top_k}`" - ) - request.top_k = -1 - # The request sampling params need to exactly match those as are set in NeMo RL. # If they do not match, the inference will be off policy and destroy training - # stability. Validation rollouts are the one exception: they are stamped with - # the validation sampling profile (generation.val_temperature / val_top_p), - # which is metric-only and safe to serve — grpo.validate() is the only - # caller that constructs a non-train GenerationSamplingParams. Multi-turn - # agents issue their own requests, so this server-side check is the one - # chokepoint they all pass. - # vLLM resolves an unset top_p from the model's generation_config.json - # (ModelConfig.generation_config defaults to "auto"), NOT to 1.0, so a - # request omitting it would sample off-policy while passing this check. - assert request.top_p is not None, ( - "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " - "resolved by vLLM from the model's generation_config.json and would " - "bypass the on-policy sampling check." - ) - request_top_p = request.top_p - is_train_sampling = ( - request.temperature == generation_config["temperature"] - and request_top_p == generation_config["top_p"] - ) - is_val_sampling = ( - request.temperature == generation_config["val_temperature"] - and request_top_p == generation_config["val_top_p"] - ) - assert is_train_sampling or is_val_sampling, ( - f"request sampling (temperature={request.temperature}, " - f"top_p={request.top_p}) matches neither the train sampling params " - f"(temperature={generation_config['temperature']}, " - f"top_p={generation_config['top_p']}) nor the validation sampling " - f"params (val_temperature={generation_config['val_temperature']}, " - f"val_top_p={generation_config['val_top_p']})" + # stability. grpo.validate() is the only caller that constructs a non-train + # GenerationSamplingParams, and the validation profile it stamps is accepted. + sampling_error = off_policy_sampling_error( + temperature=request.temperature, + top_p=request.top_p, + top_k=request.top_k, + generation_config=generation_config, ) + assert sampling_error is None, sampling_error + # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params + request.top_k = -1 try: generator = await openai_serving_chat.create_chat_completion( @@ -1813,6 +1791,55 @@ async def init_collective_async( ), ) + def _rollout_max_new_tokens(self, input_length: int) -> int: + """Token budget for one rollout prompt; 0 means nothing fits. + + The configured cap, clipped to the context window and to speculative + decoding's lookahead. Kept apart from ``generate_async`` so every path that + serves a rollout from this engine gives it the same budget. + """ + max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) + allowed_new_tokens = max( + 0, min(self.cfg["max_new_tokens"], max_model_len - input_length) + ) + spec_cfg = self.cfg.get("vllm_kwargs", {}).get("speculative_config") or {} + spec_lookahead = int(spec_cfg.get("num_speculative_tokens", 0)) + if allowed_new_tokens > 0 and spec_lookahead > 0: + allowed_new_tokens = self._request_max_new_tokens( + configured_max_new_tokens=allowed_new_tokens, + input_length=input_length, + max_model_len=max_model_len, + cap_to_context=False, + spec_lookahead=spec_lookahead, + ) + return allowed_new_tokens + + def _rollout_sampling_params( + self, + *, + max_new_tokens: int, + greedy: bool, + stop_strings: Optional[list[str]], + ) -> Any: + """The SamplingParams for one rollout. + + Kept apart from ``generate_async`` so every path that serves a rollout + from this engine samples it the same way. + + Args: + max_new_tokens: The rollout's budget, from ``_rollout_max_new_tokens``. + greedy: Greedy decoding instead of the configured sampling. + stop_strings: The sample's own stop strings, merged with the + configured ones. + """ + return self._build_sampling_params( + greedy=greedy, + stop_strings=self._merge_stop_strings( + [stop_strings] if stop_strings else None + ), + max_new_tokens=max_new_tokens, + ) + async def generate_async( self, data: BatchedDataDict[GenerationDatumSpec], @@ -1865,231 +1892,49 @@ async def process_single_sample(sample_idx): ): per_sample_stop_strings = batch_specific_stop_strings_list[sample_idx] - final_stop_strings_for_sample = self._merge_stop_strings( - [per_sample_stop_strings] if per_sample_stop_strings else None - ) - - max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) - remaining_ctx = max_model_len - current_input_actual_length - allowed_new_tokens = max(0, min(self.cfg["max_new_tokens"], remaining_ctx)) - - spec_cfg = self.cfg.get("vllm_kwargs", {}).get("speculative_config") or {} - spec_lookahead = int(spec_cfg.get("num_speculative_tokens", 0)) - if allowed_new_tokens > 0 and spec_lookahead > 0: - allowed_new_tokens = self._request_max_new_tokens( - configured_max_new_tokens=allowed_new_tokens, - input_length=current_input_actual_length, - max_model_len=max_model_len, - cap_to_context=False, - spec_lookahead=spec_lookahead, - ) - - # Handle case where no tokens can be generated due to length constraints - if allowed_new_tokens == 0: - # Access the input data directly from the function parameters - input_ids_single_row = input_ids_batch[sample_idx] - - # Create output tensors with just the input (no generated tokens) - output_ids_single_item_batched = input_ids_single_row[ - :current_input_actual_length - ].unsqueeze(0) - - logprobs_single_item = torch.zeros( - (1, current_input_actual_length), - dtype=torch.float32, - device=input_ids_single_row.device, + max_new_tokens = self._rollout_max_new_tokens(current_input_actual_length) + input_ids_single_row = input_ids_batch[sample_idx] + if max_new_tokens == 0: + # No new token fits in the context: return the prompt alone. + result = RolloutResult.nothing_generated() + else: + sampling_params_for_request = self._rollout_sampling_params( + max_new_tokens=max_new_tokens, + greedy=greedy, + stop_strings=per_sample_stop_strings, ) + request_id = str(uuid.uuid4()) + final_request_output = None + async for req_output in self.llm.generate( + prompt=prompt, + sampling_params=sampling_params_for_request, + request_id=request_id, + ): + final_request_output = req_output + if final_request_output is None: + raise RuntimeError(f"No output received for request {request_id}") - generation_lengths_tensor = torch.tensor( - [0], dtype=torch.long, device=input_ids_single_row.device + validate_rollout_prompt( + input_ids_single_row[:current_input_actual_length].tolist(), + final_request_output.prompt_token_ids, ) - - unpadded_sequence_lengths_tensor = torch.tensor( - [current_input_actual_length], - dtype=torch.long, + result = rollout_result( + final_request_output, + input_length=current_input_actual_length, + return_routed_experts=self._return_routed_experts_enabled(), + routed_experts_dtype=self.routed_experts_dtype, device=input_ids_single_row.device, + sample_idx=sample_idx, ) - - # Not truncated since no generation was attempted (length constraint) - truncated_tensor = torch.tensor( - [False], dtype=torch.bool, device=input_ids_single_row.device - ) - - result_batch = BatchedDataDict[GenerationOutputSpec]( - { - "output_ids": output_ids_single_item_batched, - "logprobs": logprobs_single_item, - "generation_lengths": generation_lengths_tensor, - "unpadded_sequence_lengths": unpadded_sequence_lengths_tensor, - "truncated": truncated_tensor, - } - ) - - return (sample_idx, result_batch) - - sampling_params_for_request = self._build_sampling_params( - greedy=greedy, - stop_strings=final_stop_strings_for_sample, - max_new_tokens=allowed_new_tokens, - ) - - request_id = str(uuid.uuid4()) - - # Generate using vLLM async engine - vllm_request_generator = self.llm.generate( - prompt=prompt, - sampling_params=sampling_params_for_request, - request_id=request_id, - ) - - # Get the final result from the generator - final_request_output = None - async for req_output in vllm_request_generator: - final_request_output = req_output - - if final_request_output is None: - raise RuntimeError(f"No output received for request {request_id}") - - validate_rollout_prompt( - input_ids_batch[sample_idx, :current_input_actual_length].tolist(), - final_request_output.prompt_token_ids, - ) - - # Process the output - generation_details = final_request_output.outputs[0] - generated_token_ids = list(generation_details.token_ids) - num_generated_tokens = len(generated_token_ids) - return_routed_experts = self._return_routed_experts_enabled() - - original_input_ids_single_row = input_ids_batch[sample_idx] - final_output_tensor_len = current_input_actual_length + num_generated_tokens - - # Create output_ids tensor for this single item - output_ids_single_item = torch.full( - (final_output_tensor_len,), - self.cfg["_pad_token_id"], - dtype=original_input_ids_single_row.dtype, - device=original_input_ids_single_row.device, - ) - # Copy original input (up to its actual length) - output_ids_single_item[:current_input_actual_length] = ( - original_input_ids_single_row[:current_input_actual_length] - ) - # Add generated tokens after the actual input - output_ids_single_item[ - current_input_actual_length : current_input_actual_length - + num_generated_tokens - ] = torch.tensor( - generated_token_ids, - dtype=original_input_ids_single_row.dtype, - device=original_input_ids_single_row.device, - ) - - # Reshape to (1, seq_len) for BatchedDataDict - output_ids_single_item_batched = output_ids_single_item.unsqueeze(0) - - # Create logprobs tensor for this single item - logprobs_single_item = torch.zeros( - (1, final_output_tensor_len), - dtype=torch.float32, - device=original_input_ids_single_row.device, - ) - if hasattr(generation_details, "logprobs") and generation_details.logprobs: - for idx, logprob_dict_per_token in enumerate( - generation_details.logprobs - ): - if logprob_dict_per_token and idx < len(generated_token_ids): - token_id_at_idx = generated_token_ids[idx] - if token_id_at_idx in logprob_dict_per_token: - logprob_value = logprob_dict_per_token[ - token_id_at_idx - ].logprob - position_in_output_tensor = ( - current_input_actual_length + idx - ) - if position_in_output_tensor < final_output_tensor_len: - logprobs_single_item[0, position_in_output_tensor] = ( - logprob_value - ) - - # Generation lengths - generation_lengths_tensor = torch.tensor( - [num_generated_tokens], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - - # Unpadded sequence lengths (actual_input + actual_generated) - unpadded_total_length = current_input_actual_length + num_generated_tokens - unpadded_sequence_lengths_tensor = torch.tensor( - [unpadded_total_length], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - - # Check if response was truncated (hit max_tokens length limit) - is_truncated = generation_details.finish_reason == "length" - truncated_tensor = torch.tensor( - [is_truncated], - dtype=torch.bool, - device=original_input_ids_single_row.device, - ) - - result_dict = { - "output_ids": output_ids_single_item_batched, - "logprobs": logprobs_single_item, - "generation_lengths": generation_lengths_tensor, - "unpadded_sequence_lengths": unpadded_sequence_lengths_tensor, - "truncated": truncated_tensor, - } - routed_experts, r3_stats = pad_and_align_routed_expert_indices( - final_request_output, - generation_details, - valid_length=unpadded_total_length, - padded_length=final_output_tensor_len, - device=original_input_ids_single_row.device, - require_complete_routed_experts=return_routed_experts, - return_stats=True, - routed_experts_dtype=self.routed_experts_dtype, + return ( + sample_idx, + rollout_output( + result, + input_ids=input_ids_single_row, + input_length=current_input_actual_length, + pad_token_id=self.cfg["_pad_token_id"], + ), ) - if return_routed_experts and routed_experts is None: - raise RuntimeError( - "vLLM was asked to return routed experts but the generation output " - "did not include routed_experts." - ) - if return_routed_experts: - if r3_stats["missing_routes"] > 0: - LOGGER.warning( - "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for sample_idx=%d, missing_token_routes=%d, " - "actual_routes=%d, expected_routes=%d. Megatron will use its " - "own router for those missing token routes.", - sample_idx, - r3_stats["missing_routes"], - r3_stats["actual_routes"], - r3_stats["expected_routes"], - ) - result_dict["r3_routed_experts_missing_routes"] = torch.tensor( - [r3_stats["missing_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - result_dict["r3_routed_experts_expected_routes"] = torch.tensor( - [r3_stats["expected_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - result_dict["r3_routed_experts_actual_routes"] = torch.tensor( - [r3_stats["actual_routes"]], - dtype=torch.long, - device=original_input_ids_single_row.device, - ) - if routed_experts is not None: - result_dict["routed_experts"] = routed_experts.unsqueeze(0) - - result_batch = BatchedDataDict[GenerationOutputSpec](result_dict) - - return (sample_idx, result_batch) # Create tasks for all samples and yield results as they complete sample_tasks = [ diff --git a/pyrefly.toml b/pyrefly.toml index 55d8a74cf27..31acd1641e5 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -282,6 +282,7 @@ project-includes = [ "nemo_rl/models/generation/vllm/quantization/nvfp4_pertoken_config.py", "nemo_rl/models/generation/vllm/quantization/utils.py", "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", + "nemo_rl/models/generation/vllm/request_pipeline.py", "nemo_rl/models/generation/vllm/refit_layout.py", "nemo_rl/models/generation/vllm/refit_loader.py", "nemo_rl/models/generation/vllm/utils.py", diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index e25d808c663..d0533c1a734 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -474,6 +474,131 @@ async def test_async_vllm_worker_gpu_profiling_awaits_collective_rpc( worker.llm.collective_rpc.assert_awaited_once_with("stop_gpu_profiling", args=()) +class _ScriptedAsyncLLM: + """Yields one scripted final RequestOutput and records what it was asked to generate.""" + + def __init__(self, generated: list[int], finish_reason: str = "stop") -> None: + self.generated = generated + self.finish_reason = finish_reason + self.calls: list[tuple[dict[str, Any], dict[str, Any]]] = [] + + async def generate(self, prompt, sampling_params, request_id): + self.calls.append((prompt, sampling_params)) + yield types.SimpleNamespace( + prompt_token_ids=prompt["prompt_token_ids"], + prompt_routed_experts=None, + num_cached_tokens=0, + outputs=[ + types.SimpleNamespace( + token_ids=self.generated, + logprobs=[ + {t: types.SimpleNamespace(logprob=-0.5)} for t in self.generated + ], + finish_reason=self.finish_reason, + routed_experts=None, + ) + ], + ) + + +def _rollout_worker(llm: _ScriptedAsyncLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: + worker = VllmAsyncGenerationWorkerImpl.__new__(VllmAsyncGenerationWorkerImpl) + worker.cfg = { + "max_new_tokens": 8, + "temperature": 0.7, + "top_p": 0.9, + "top_k": None, + "stop_token_ids": [2], + "stop_strings": [""], + "_pad_token_id": 0, + "vllm_cfg": {"async_engine": True, "max_model_len": 16}, + **cfg, + } + # Records the sampling kwargs instead of building vLLM's SamplingParams. + worker.SamplingParams = dict + worker.routed_experts_dtype = torch.int16 + worker.llm = llm + return worker + + +async def _generate_one( + worker: VllmAsyncGenerationWorkerImpl, + prompt: list[int], + stop_strings: list[str] | None = None, +) -> BatchedDataDict: + data = BatchedDataDict( + { + "input_ids": torch.tensor([prompt]), + "input_lengths": torch.tensor([len(prompt)]), + "stop_strings": [stop_strings], + } + ) + ((sample_idx, output),) = [item async for item in worker.generate_async(data)] + assert sample_idx == 0 + return output + + +@pytest.mark.asyncio +async def test_async_vllm_generate_async_samples_and_returns_the_rollout() -> None: + llm = _ScriptedAsyncLLM([10, 11, 2], finish_reason="stop") + worker = _rollout_worker(llm) + + output = await _generate_one(worker, [1, 2, 3, 4, 5], stop_strings=[""]) + + ((prompt, sampling),) = llm.calls + assert prompt == {"prompt_token_ids": [1, 2, 3, 4, 5]} + assert sampling["max_tokens"] == 8 + assert sampling["temperature"] == 0.7 + assert sampling["top_k"] == -1 + assert sorted(sampling["stop"]) == ["", ""] + assert output["output_ids"].tolist() == [[1, 2, 3, 4, 5, 10, 11, 2]] + assert output["logprobs"].tolist() == [[0.0] * 5 + [-0.5] * 3] + assert output["generation_lengths"].tolist() == [3] + assert output["unpadded_sequence_lengths"].tolist() == [8] + assert output["truncated"].tolist() == [False] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("speculative_config", "expected_max_tokens"), + [ + # min(max_new_tokens=8, max_model_len 16 - prompt 12) + (None, 4), + # The drafter reads 2 tokens ahead: 16 - 12 - (2 + 1) + ({"num_speculative_tokens": 2}, 1), + ], +) +async def test_async_vllm_generate_async_caps_the_budget_to_the_context( + speculative_config, expected_max_tokens +) -> None: + llm = _ScriptedAsyncLLM([10], finish_reason="length") + worker = _rollout_worker( + llm, vllm_kwargs={"speculative_config": speculative_config} + ) + + output = await _generate_one(worker, list(range(1, 13))) + + ((_, sampling),) = llm.calls + assert sampling["max_tokens"] == expected_max_tokens + assert output["truncated"].tolist() == [True] + + +@pytest.mark.asyncio +async def test_async_vllm_generate_async_returns_the_prompt_when_nothing_fits() -> None: + llm = _ScriptedAsyncLLM([10]) + worker = _rollout_worker(llm) + prompt = list(range(1, 17)) + + output = await _generate_one(worker, prompt) + + assert llm.calls == [] + assert output["output_ids"].tolist() == [prompt] + assert output["logprobs"].tolist() == [[0.0] * 16] + assert output["generation_lengths"].tolist() == [0] + assert output["unpadded_sequence_lengths"].tolist() == [16] + assert output["truncated"].tolist() == [False] + + @pytest.mark.asyncio async def test_async_vllm_http_client_runs_generation_on_owner_loop() -> None: engine_loop = asyncio.get_running_loop() diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py new file mode 100644 index 00000000000..c00be9dc2c4 --- /dev/null +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -0,0 +1,124 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The rollout result pipeline behind generate_async.""" + +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result, +) +from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL + +LAYERS, TOPK = 2, 2 + + +def _logprob(value: float): + return SimpleNamespace(logprob=value) + + +def _final_output( + generated: list[int], + *, + finish_reason: str = "stop", + prompt_routes: int | None = None, + generated_routes: int | None = None, +): + def routes(count): + if count is None: + return None + return np.arange(count * LAYERS * TOPK, dtype=np.int16).reshape( + count, LAYERS, TOPK + ) + + return SimpleNamespace( + prompt_routed_experts=routes(prompt_routes), + num_cached_tokens=0, + outputs=[ + SimpleNamespace( + token_ids=generated, + # The second token has no reported logprob. + logprobs=[{generated[0]: _logprob(-0.5)}, {}] + + [{t: _logprob(-1.0)} for t in generated[2:]], + finish_reason=finish_reason, + routed_experts=routes(generated_routes), + ) + ], + ) + + +def _result(final_output, *, input_length=3, r3=False) -> RolloutResult: + return rollout_result( + final_output, + input_length=input_length, + return_routed_experts=r3, + routed_experts_dtype=torch.int16, + device=torch.device("cpu"), + ) + + +def test_output_layout_matches_generate_async() -> None: + result = _result(_final_output([7, 8, 9], finish_reason="length")) + output = rollout_output( + result, input_ids=torch.tensor([1, 2, 3, 0, 0]), input_length=3, pad_token_id=0 + ) + + assert output["output_ids"].tolist() == [[1, 2, 3, 7, 8, 9]] + # A position without a reported logprob stays 0.0, aligned with its token. + assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0, -0.5, 0.0, -1.0]] + assert output["generation_lengths"].tolist() == [3] + assert output["unpadded_sequence_lengths"].tolist() == [6] + assert output["truncated"].tolist() == [True] + assert "routed_experts" not in output + + +def test_nothing_generated_returns_the_prompt_untruncated() -> None: + output = rollout_output( + RolloutResult.nothing_generated(), + input_ids=torch.tensor([1, 2, 3, 0]), + input_length=3, + pad_token_id=0, + ) + + assert output["output_ids"].tolist() == [[1, 2, 3]] + assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0]] + assert output["generation_lengths"].tolist() == [0] + assert output["truncated"].tolist() == [False] + + +def test_router_replay_routes_are_aligned_and_counted() -> None: + # 3 prompt + 3 generated tokens need 5 routes; vLLM reported 3 + 1. + result = _result( + _final_output([7, 8, 9], prompt_routes=3, generated_routes=1), r3=True + ) + output = rollout_output( + result, input_ids=torch.tensor([1, 2, 3]), input_length=3, pad_token_id=0 + ) + + routed = output["routed_experts"] + assert routed.shape == (1, 6, LAYERS, TOPK) and routed.dtype == torch.int16 + assert (routed[0, 4:] == R3_MISSING_ROUTE_SENTINEL).all() + assert output["r3_routed_experts_expected_routes"].tolist() == [5] + assert output["r3_routed_experts_actual_routes"].tolist() == [4] + assert output["r3_routed_experts_missing_routes"].tolist() == [1] + + +def test_router_replay_requires_routes() -> None: + with pytest.raises(RuntimeError, match="did not include routed_experts"): + _result(_final_output([7, 8, 9]), r3=True) diff --git a/tests/unit/models/generation/test_vllm_utils.py b/tests/unit/models/generation/test_vllm_utils.py index 7bf7546d9d0..3ce5907bc06 100644 --- a/tests/unit/models/generation/test_vllm_utils.py +++ b/tests/unit/models/generation/test_vllm_utils.py @@ -38,6 +38,7 @@ encode_counter_key, format_prompt_for_vllm_generation, model_dump_chat_response_with_dynamic_message_fields, + off_policy_sampling_error, pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) @@ -1451,3 +1452,29 @@ def test_remap_multimodal_placeholders_rebuilds_real_vllm_placeholder_range(): assert isinstance(moved, PlaceholderRange) assert (moved.offset, moved.length) == (3, 3) assert moved.is_embed is is_embed + + +@pytest.mark.parametrize( + ("temperature", "top_p", "top_k", "error"), + [ + (1.0, 1.0, None, None), # train profile + (0.0, 0.9, -1, None), # validation profile + (0.7, 1.0, None, "matches neither"), + (1.0, None, None, "top_p must be set explicitly"), + (1.0, 1.0, 5, "Top k sampling parameter"), + ], +) +def test_off_policy_sampling_error(temperature, top_p, top_k, error) -> None: + generation_config = { + "temperature": 1.0, + "top_p": 1.0, + "val_temperature": 0.0, + "val_top_p": 0.9, + } + message = off_policy_sampling_error( + temperature=temperature, + top_p=top_p, + top_k=top_k, + generation_config=generation_config, + ) + assert (message is None) if error is None else (error in message) From d217dd89e82b5830b2a830d138f1a0f9fc4bcc09 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 09:54:29 -0700 Subject: [PATCH 07/19] refactor(dynamo): split NeMo-Gym support out to its own PR At review request, NeMo-Gym through the Dynamo frontend moves to a follow-up PR that waits on NVIDIA-NeMo/Gym#4817, so this PR no longer needs a Gym bump: - Gym's submodule is back on main's pin. - The engine's Gym path (_generate_gym, capture staging, prefix splice), the Gym dialect setup, session affinity, client_base_urls and the Gym-only startup checks move to the follow-up. - NeMo-Gym with policy.generation.dynamo=true is refused at startup in should_use_nemo_gym, before any cluster starts. - The Dynamo L1 keeps its SingleController math and reload-refit overlays; the swe1 Dynamo + Gym nightly is disabled until the follow-up. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- 3rdparty/Gym-workspace/Gym | 2 +- docs/design-docs/dynamo-integration.md | 18 +- docs/guides/dynamo-generation.md | 60 +--- ...grpo_math_1B_dynamo_single_controller.yaml | 3 - nemo_rl/algorithms/distillation.py | 2 +- nemo_rl/algorithms/grpo.py | 6 +- .../single_controller_utils/config.py | 4 +- .../single_controller_utils/setup.py | 9 +- nemo_rl/environments/nemo_gym.py | 63 +--- nemo_rl/environments/nemo_gym_shards.py | 7 +- nemo_rl/models/generation/__init__.py | 6 - nemo_rl/models/generation/dynamo/arguments.py | 6 - nemo_rl/models/generation/dynamo/config.py | 22 -- .../generation/dynamo/managed_runtime.py | 4 +- .../models/generation/dynamo/native_engine.py | 292 +----------------- .../models/generation/openai_server_utils.py | 72 +---- .../models/generation/vllm/vllm_generation.py | 37 +-- .../generation/vllm/vllm_worker_async.py | 5 +- .../functional/L1_Functional_Tests_Dynamo.sh | 5 - .../grpo_async_gym_single_controller.sh | 8 - ...lm_token_capture_single_controller_1n2g.sh | 4 - tests/test_suites/disabled.txt | 4 + tests/test_suites/nightly.txt | 1 - tests/unit/algorithms/test_distillation.py | 2 +- tests/unit/algorithms/test_grpo.py | 2 +- .../test_nemo_gym_token_capture.py | 103 +----- .../unit/environments/test_nemo_gym_utils.py | 67 ---- .../models/generation/test_dynamo_config.py | 28 +- .../generation/test_dynamo_native_engine.py | 188 +---------- .../test_dynamo_native_engine_capture.py | 241 --------------- .../generation/test_openai_server_utils.py | 92 ------ .../models/generation/test_vllm_generation.py | 12 +- .../test_vllm_token_capture_hosting.py | 1 - tests/unit/single_controller/test_setup.py | 40 +-- 34 files changed, 84 insertions(+), 1332 deletions(-) delete mode 100644 tests/unit/models/generation/test_dynamo_native_engine_capture.py diff --git a/3rdparty/Gym-workspace/Gym b/3rdparty/Gym-workspace/Gym index a5f936c473d..3f6f20593a1 160000 --- a/3rdparty/Gym-workspace/Gym +++ b/3rdparty/Gym-workspace/Gym @@ -1 +1 @@ -Subproject commit a5f936c473dbd477ce4632373eaba884272b7f6a +Subproject commit 3f6f20593a126ca9484793769a51d186aa1486c3 diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 843dad2c897..9c07ccda5c7 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -32,29 +32,17 @@ implements its `LLMEngine` over the worker's `AsyncLLM`: The result travels in `engine_data`, and the driver assembles the same output with `request_pipeline.rollout_output`. - NeMo-Gym's calls also carry an annotation (Gym's Dynamo dialect). The engine - applies the native chat endpoint's rules to them: the on-policy sampling check - (`vllm/utils.off_policy_sampling_error`), the prefix splice - (`openai_server_utils.splice_prefix_tokens`) and the context clamp. A - token-capture call stages through the worker's own capture lifecycle - (`_begin_request_capture` / `_finish_request_capture`), and the token fields - and routed experts come from the native endpoint's helpers - (`chat_generation_tokens`, `chat_routed_experts`). - A request with media carries the frontend's tokens, with one unexpanded placeholder per item, and the media URLs. The adapter loads the media with vLLM's media connector. It then builds the processed prompt with the engine's renderer (`process_for_engine_async`), the same step that follows rendering - on the native endpoint. The splice, the placeholder remap - (`remap_multimodal_placeholders`) and media capture (`_capture_request_media`) - all run on that processed prompt. + on the native endpoint. - `abort` cancels the vLLM request on client disconnect. - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It publishes no token budget. The engine owns the context check and the output - cap, as the native server does. With a budget, the frontend would fill an - omitted `max_tokens` from the prompt it rendered, which a prefix splice then - changes, and would refuse NeMo RL's own rollouts that fill the context. + cap, as the native server does. With a budget, the frontend would refuse NeMo + RL's own rollouts that fill the context. - `kv_event_sources` points Dynamo at the engine's own vLLM KV-event ZMQ publisher. It is set only when the engine caches prefixes. Each engine binds its own Linux abstract-namespace socket (`ipc://@...`), so engines on one node diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index f1764202897..b780dcda242 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -51,11 +51,10 @@ shows them all: | Key | Default | Meaning | |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | -| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies; NeMo-Gym's calls keep Gym's timeouts | -| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [NeMo-Gym](#nemo-gym) | +| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies | +| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [Chat settings](#chat-settings) | | `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | -| `frontend_args.session_affinity_ttl_secs` | 1800 | Keeps a NeMo-Gym token-capture rollout on one engine while it is idle for up to this many seconds; null disables it | | `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | @@ -69,7 +68,7 @@ is set (the image sets it to `/opt/dynamo/bin`), and on `PATH` otherwise. | `generate_async` on text token prompts, router replay (R3) included | Frontend (KV-aware); the engine samples and assembles it exactly as native `generate_async` does | | NeMo RL's own multimodal rollouts | Native shard dispatch. Those rollouts send media to the engine directly rather than as OpenAI content | | `generate_text_async`, which evaluation uses | Native shard dispatch | -| NeMo-Gym's OpenAI requests | Frontend, in Gym's Dynamo dialect; the engine serves them as the native chat endpoint does. See [NeMo-Gym](#nemo-gym) | +| NeMo-Gym's OpenAI requests | Not served yet; see [Not supported](#not-supported) | A rollout through the frontend has the same `GenerationOutputSpec` as the native path: token IDs, log probabilities, truncation, routed experts with @@ -87,67 +86,32 @@ OpenAI clients of the frontend get what vLLM's own OpenAI server applies: for output. The engine, not the frontend, checks the context, so it checks the prompt it actually runs. -## NeMo-Gym - -NeMo-Gym's policy model calls the frontend. Setup turns on Gym's Dynamo dialect -(`vllm_model.dynamo_frontend`, from -[NVIDIA-NeMo/Gym#4817](https://github.com/NVIDIA-NeMo/Gym/pull/4817)). Gym marks -every call with an annotation, and the engine serves a marked call as the native -chat endpoint does: - -- Sampling must match `temperature`/`top_p` or `val_temperature`/`val_top_p`, - with `top_k` unset. vLLM samples with the configured values. -- The token prefix Gym echoes replaces the history the frontend rendered. -- Output is clamped to the context, and a prompt that leaves no room is refused - with the native endpoint's "maximum context length" error. -- Stop strings stop vLLM itself, so the token IDs match the native endpoint's. -- The response carries `prompt_token_ids`, `completion_token_ids` and - `completion_logprobs` in `nvext.engine_data`, which Gym lifts onto the message. - With router replay on, it also carries the call's `routed_experts`, as the - native endpoint attaches them. -- Images, video and audio in the messages are processed as on the native - endpoint. The frontend passes each media URL through with one unexpanded - placeholder per item. The engine loads the media with vLLM's media connector - and runs vLLM's multimodal processor on the frontend's tokens. The splice, - placeholder remap and media capture then work on that processed prompt. - -With `token_capture.enabled`, Gym uses its `dynamo_worker` capture backend. The -same vLLM workers stage each call's tokens, log probabilities, routed experts and -processed media from their own process, exactly as the native endpoint does, and -only the commit coordinates travel back through the frontend. The frontend keeps a rollout on -one engine (`session_affinity_ttl_secs`), as the native backend pins each Gym -session to one server. - -The native server's chat settings carry over: +## Chat settings + +The native server's chat settings carry over to the frontend: | Native setting (`vllm_cfg`) | Under Dynamo | |---|---| | `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | | `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | | `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | -| `http_server_serving_chat_kwargs.default_chat_template_kwargs` | Sent by Gym under its own `chat_template_kwargs` | | `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | -Where the behavior differs from the native endpoint: - -- The engine sees the rendered prompt, not the messages. It finds where the - echoed prefix ends by counting the prefix's EOS tokens, not by rendering the - history. -- When a request sets `chat_template_kwargs.force_nonempty_content`, Dynamo - reports a reasoning-only turn as content. +Where the behavior differs from the native endpoint: when a request sets +`chat_template_kwargs.force_nonempty_content`, Dynamo reports a reasoning-only +turn as content. ## Not supported Startup rejects these with `dynamo: true`: +- NeMo-Gym (`env.should_use_nemo_gym`): the frontend does not serve Gym's + calls yet. Run Gym recipes with `dynamo: false`. - `colocated.enabled: true`: a colocated engine sleeps while the policy trains, and the frontend would keep routing to it. The SingleController never runs colocated vLLM; the other entrypoints need non-colocated generation. - `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, and a restarted shard does not register with it yet. -- `async_rl.generation_router`: the frontend already routes the fleet behind one - URL. -- `vllm_cfg.http_server_api_key_required`: the frontend checks no bearer token. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data parallelism inside vLLM, which the async engine Dynamo serves does not. - Parser plugins with no Dynamo parser, and the chat settings above that the @@ -155,8 +119,6 @@ Startup rejects these with `dynamo: true`: - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. -A Gym call with `n > 1` is refused when it arrives. - ## Build the image ai-dynamo is part of the `vllm` extra, so every vLLM worker venv carries it. diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 9ad2b249a95..1b62d39ba8d 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -37,9 +37,6 @@ policy: tokenizer_cache_bytes: 52428800 # kv: route each request to the engine holding most of its prefix. router_mode: kv - # Keep a NeMo-Gym token-capture rollout on one engine while it idles up - # to this many seconds (null disables it). - session_affinity_ttl_secs: 1800 extra_cli_args: [] logger: diff --git a/nemo_rl/algorithms/distillation.py b/nemo_rl/algorithms/distillation.py index f00109256d3..f3a5d883ae2 100644 --- a/nemo_rl/algorithms/distillation.py +++ b/nemo_rl/algorithms/distillation.py @@ -556,7 +556,7 @@ def init_vllm_deferred(): def init_nemo_gym(): shard_set = build_nemo_gym_actors( env_configs, - base_urls=cast(list[str], deferred_vllm.client_base_urls), + base_urls=cast(list[str], deferred_vllm.dp_openai_server_base_urls), model_name=generation_config["model_name"], tokenizer=tokenizer, # Distillation does not configure vLLM for router replay. diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 505b1919261..31ef02d7895 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -1771,8 +1771,8 @@ def init_megatron_generation_task(policy_future): ) vllm_reserve_time = time.perf_counter() - vllm_reserve_t0 print( - f" ✓ Reserved {len(deferred_vllm.client_base_urls)} vLLM server URLs: " - f"{deferred_vllm.client_base_urls}", + f" ✓ Reserved {len(deferred_vllm.dp_openai_server_base_urls)} vLLM server URLs: " + f"{deferred_vllm.dp_openai_server_base_urls}", flush=True, ) @@ -1786,7 +1786,7 @@ def init_vllm_deferred(): def init_nemo_gym(): """Spin up NeMo Gym servers with the pre-assigned vLLM URLs.""" return _spinup_nemo_gym( - deferred_vllm.client_base_urls, + deferred_vllm.dp_openai_server_base_urls, generation_config["model_name"], policy_api_key=deferred_vllm.http_server_api_key, ) diff --git a/nemo_rl/algorithms/single_controller_utils/config.py b/nemo_rl/algorithms/single_controller_utils/config.py index 758e836f402..a6d42371804 100644 --- a/nemo_rl/algorithms/single_controller_utils/config.py +++ b/nemo_rl/algorithms/single_controller_utils/config.py @@ -679,9 +679,7 @@ class TokenCaptureConfig(BaseModel, extra="allow"): capture_dir: Optional[str] = None # Generation backend hosting token capture. This is derived from # policy.generation.backend during setup; users should not set it separately. - # "dynamo" is the vLLM backend served through Dynamo (policy.generation.dynamo): - # the same workers host capture, and NeMo-Gym reaches them through the frontend. - generation_backend: Optional[Literal["vllm", "megatron", "dynamo"]] = None + generation_backend: Optional[Literal["vllm", "megatron"]] = None # Keep routed_experts out of canonical rows and assemble them on policy # workers from strict staged-fragment plans. defer_routed_experts_to_policy: bool = False diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index d3dc6dd53c2..e3f65134811 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -125,7 +125,6 @@ should_mask_flagged_samples, ) from nemo_rl.models.generation import resolve_generation_class -from nemo_rl.models.generation.dynamo.config import dynamo_enabled from nemo_rl.models.generation.fleet_health import ( FleetHealthPolicy, GenerationFleetHealth, @@ -1370,11 +1369,7 @@ def setup_single_controller( # Fill the derived ledger-hosting fields (see TokenCaptureConfig): a # per-run control-plane bearer token, the process-shared capture # directory used by every Gym worker, and the capture-host backend. - token_capture_cfg.generation_backend = ( - "dynamo" - if dynamo_enabled(generation_config) - else generation_config["backend"] - ) + token_capture_cfg.generation_backend = generation_config["backend"] if token_capture_cfg.control_auth_token is None: # Deferred import: only needed on the capture path. import secrets @@ -1861,7 +1856,7 @@ def _build_trainer_then_megatron_generation() -> tuple[ defer_model_load=True, ) defer_generation_model_load = True - gym_base_urls = generation.client_base_urls + gym_base_urls = generation.dp_openai_server_base_urls # Before the Gym task is built, so Gym can be handed the router's single URL. # These two statements are the only failable ones related to the port holder's creation. try: diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 5ab370dbe65..435e35198f8 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -98,10 +98,7 @@ RolloutDataFailure, http_status_is_infra, ) -from nemo_rl.models.generation.dynamo.config import ( - dynamo_enabled, - native_default_chat_template_kwargs, -) +from nemo_rl.models.generation.dynamo.config import dynamo_enabled from nemo_rl.models.generation.interfaces import ( resolve_routed_experts_dtype_name_for_model, should_use_async_rollouts, @@ -304,6 +301,12 @@ def should_use_nemo_gym(master_config: NemoGymCompatibleConfig) -> bool: return False generation_config = master_config.policy["generation"] + if dynamo_enabled(generation_config): + raise ValueError( + "policy.generation.dynamo=true does not serve NeMo-Gym rollouts yet. " + "Set policy.generation.dynamo=false to run this NeMo-Gym config on " + "NeMo RL's vLLM servers." + ) assert should_use_async_rollouts(generation_config), ( "❌ Error: In order to use NeMo-Gym, you must use a generation " "backend with `async_engine: true`!" @@ -489,11 +492,9 @@ def _external_staging_backend(token_capture: Dict[str, Any]) -> str: return "vllm_worker" if generation_backend == "megatron": return "megatron_worker" - if generation_backend == "dynamo": - return "dynamo_worker" raise ValueError( "token_capture.enabled requires setup-derived generation_backend to be " - f"'vllm', 'megatron' or 'dynamo'; got {generation_backend!r}" + f"'vllm' or 'megatron'; got {generation_backend!r}" ) @@ -2056,16 +2057,6 @@ def setup_nemo_gym_config(config, tokenizer) -> None: generation_config["stop_strings"] = None generation_config["stop_token_ids"] = None - if dynamo_enabled(generation_config): - # Gym's policy model calls the Dynamo frontend in Gym's Dynamo dialect. - # Each actor applies it to its own composed config (_apply_dynamo_dialect), - # so it never becomes a policy_model overlay that a shard must claim. - config.env.setdefault("nemo_gym", {})["dynamo_dialect"] = { - "default_chat_template_kwargs": native_default_chat_template_kwargs( - generation_config - ) - } - # For VLM runs, plumb the tokenizer config into the gym env config so the # NemoGym actor can reconstruct the processor inside itself (needed for # multi-turn multimodal postprocessing). @@ -2134,39 +2125,6 @@ def build_nemo_gym_config( ) -def _apply_dynamo_dialect( - nemo_gym_dict: dict[str, Any], default_chat_template_kwargs: Mapping[str, Any] -) -> None: - """Point one actor's policy model at the Dynamo frontend, in Gym's Dynamo dialect. - - The frontend takes token data in nvext instead of vLLM's request extensions. - Applied to the actor's composed config (a shard's overrides included), so - settings the user wrote there win. Copies the policy_model levels it - changes, so the caller's config is not mutated. - """ - policy_model = dict(nemo_gym_dict.get("policy_model") or {}) - models = dict(policy_model.get("responses_api_models") or {}) - vllm_model = dict(models.get("vllm_model") or {}) - vllm_model["dynamo_frontend"] = True - # The frontend reads `reasoning` as another name for `reasoning_content` and - # refuses an echoed assistant turn that carries both, Gym's default. - vllm_model.setdefault("reasoning_field", "reasoning_content") - # The native server merges these under every request's own template - # kwargs; the frontend has no such default, so Gym sends them. They sit - # under the kwargs Gym sends: vllm_model's own, else extra_body's. - if default_chat_template_kwargs: - gym_kwargs = vllm_model.get("chat_template_kwargs") or ( - (vllm_model.get("extra_body") or {}).get("chat_template_kwargs") or {} - ) - vllm_model["chat_template_kwargs"] = { - **default_chat_template_kwargs, - **gym_kwargs, - } - models["vllm_model"] = vllm_model - policy_model["responses_api_models"] = models - nemo_gym_dict["policy_model"] = policy_model - - def _build_gym_actor_config( nemo_gym_dict: dict[str, Any], *, @@ -2195,11 +2153,6 @@ def _build_gym_actor_config( invalid_tool_call_patterns = nemo_gym_dict.pop("invalid_tool_call_patterns", None) thinking_tags = nemo_gym_dict.pop("thinking_tags", None) tokenizer_config = nemo_gym_dict.pop("tokenizer_config", None) - dynamo_dialect = nemo_gym_dict.pop("dynamo_dialect", None) - if dynamo_dialect is not None: - _apply_dynamo_dialect( - nemo_gym_dict, dynamo_dialect["default_chat_template_kwargs"] - ) external_service_readiness = extract_external_service_readiness(nemo_gym_dict) port_range = { key: value diff --git a/nemo_rl/environments/nemo_gym_shards.py b/nemo_rl/environments/nemo_gym_shards.py index b03b9cd9eb6..d8322a8d23c 100644 --- a/nemo_rl/environments/nemo_gym_shards.py +++ b/nemo_rl/environments/nemo_gym_shards.py @@ -38,12 +38,7 @@ # Dict-shaped NeMo RL integration settings that are not Gym server entries. # The actor factory or rollout code consumes these separately. NEMO_RL_DICT_CONFIG_KEYS = frozenset( - { - "dynamo_dialect", - "effort_levels", - "external_service_readiness", - "tokenizer_config", - } + {"effort_levels", "external_service_readiness", "tokenizer_config"} ) # Ray placement-group strategies a shard plan may ask for. STRICT_SPREAD is the diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 8f4ea47ce9f..c8547bbce41 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -191,12 +191,6 @@ def configure_generation_config( dynamo_cfg(config) dynamo_parsers(config) check_dynamo_serving_chat_kwargs(config) - if config["vllm_cfg"].get("http_server_api_key_required"): - raise ValueError( - "policy.generation.vllm_cfg.http_server_api_key_required is not " - "supported with policy.generation.dynamo=true: NeMo-Gym calls the " - "Dynamo frontend, which checks no bearer token." - ) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( diff --git a/nemo_rl/models/generation/dynamo/arguments.py b/nemo_rl/models/generation/dynamo/arguments.py index c08d9873200..e5a6abebee5 100644 --- a/nemo_rl/models/generation/dynamo/arguments.py +++ b/nemo_rl/models/generation/dynamo/arguments.py @@ -111,12 +111,6 @@ def build_dynamo_frontend_argv( dynamo_cfg.frontend_args.router_mode, source="dynamo_cfg.frontend_args.router_mode", ) - if dynamo_cfg.frontend_args.session_affinity_ttl_secs is not None: - builder.add( - "--router-session-affinity-ttl-secs", - dynamo_cfg.frontend_args.session_affinity_ttl_secs, - source="dynamo_cfg.frontend_args.session_affinity_ttl_secs", - ) builder.add_raw( dynamo_cfg.frontend_args.extra_cli_args, source="dynamo_cfg.frontend_args.extra_cli_args", diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index 7a6a61868d2..b6ae598b1a1 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -67,12 +67,6 @@ class DynamoFrontendArgs(BaseModel, extra="forbid"): "least-loaded", "device-aware-weighted", ] = "kv" - # Idle TTL in seconds for keeping one session on one engine, or None to - # disable it. NeMo-Gym names the session on token-capture calls only (its - # rollout id), as the native backend pins each Gym session to one server; the - # engine splices a call's staged prefix after routing, so the KV router alone - # cannot see where that prefix is cached. - session_affinity_ttl_secs: PositiveInt | None = 1800 extra_cli_args: list[str] = Field(default_factory=list) @@ -85,7 +79,6 @@ class DynamoCfg(BaseModel, extra="forbid"): # NRL_VLLM_ASYNC_TIMEOUT_SECONDS. A timeout fails the attempt as a native one # does (GenerationUnavailable), and the rollout retry policy re-dispatches it # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). - # NeMo-Gym's own calls keep Gym's timeouts. request_timeout_s: PositiveFloat = 900.0 worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) @@ -196,21 +189,6 @@ def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: return serving.get("chat_template") -def native_default_chat_template_kwargs( - generation_config: Mapping[str, Any], -) -> dict[str, Any]: - """The template kwargs the native server merges under every request's own. - - ``default_chat_template_kwargs``, or the older ``chat_template_kwargs`` - spelling, of ``vllm_cfg.http_server_serving_chat_kwargs``. - """ - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - kwargs = serving.get("default_chat_template_kwargs") - if kwargs is None: - kwargs = serving.get("chat_template_kwargs") - return dict(kwargs or {}) - - def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" return bool(generation_config.get("dynamo")) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index e4a0fe9507e..ad9ad9b7659 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -167,8 +167,8 @@ def allocate_port(*, low: int, high: int) -> int: "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. - # Its v2 parser path preserves engine_data, which NeMo-Gym needs for - # exact token IDs and log probabilities. + # Its v2 parser path preserves engine_data, which carries every NeMo + # RL rollout's result. "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index b3221afcf53..ff31f98dfca 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -33,7 +33,6 @@ from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional -import numpy as np import torch from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names from dynamo.common.backend import ( @@ -66,7 +65,6 @@ from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase -from nemo_rl.data.captured_media import MediaCaptureRejected from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, dynamo_cfg, @@ -78,22 +76,12 @@ DYNAMO_ENDPOINT, ) from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION -from nemo_rl.models.generation.openai_server_utils import ( - derive_prefix_eos_count, - splice_prefix_tokens, -) from nemo_rl.models.generation.vllm.request_pipeline import ( RolloutResult, rollout_result, rollout_result_to_wire, ) -from nemo_rl.models.generation.vllm.utils import ( - chat_generation_tokens, - chat_routed_experts, - off_policy_sampling_error, - remap_multimodal_placeholders, - validate_rollout_prompt, -) +from nemo_rl.models.generation.vllm.utils import validate_rollout_prompt if TYPE_CHECKING: from nemo_rl.models.generation.vllm.vllm_worker_async import ( @@ -102,16 +90,6 @@ LOGGER = logging.getLogger(__name__) -# Annotations NeMo-Gym's Dynamo dialect (vllm_model with dynamo_frontend, Gym#4817) -# sets on its requests; Dynamo forwards nvext.annotations to the engine verbatim. -GYM_ROLLOUT_ANNOTATION = "nemo_gym:rollout" -# A token-capture call's admission JSON. -GYM_CAPTURE_ANNOTATION_PREFIX = "ng_capture:" -# Without capture: the last echoed assistant turn's prompt + generation token -# IDs, which the native server reads from the messages instead. -GYM_PREFIX_ANNOTATION_PREFIX = "ng_prefix:" -_GYM_SAMPLING_KEYS = ("temperature", "top_p", "val_temperature", "val_top_p") - # The vLLM modality each key of Dynamo's multi_modal_data holds. _DYNAMO_MEDIA_MODALITIES = { "image_url": "image", @@ -172,11 +150,6 @@ def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: return None if rollout is None else json.loads(rollout) -def _as_f32(value: Optional[float]) -> Optional[float]: - """Dynamo's frontend carries sampling floats as f32 (0.7 arrives as 0.69999998).""" - return None if value is None else float(np.float32(value)) - - def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. @@ -325,11 +298,6 @@ def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int self._llm = host.llm self._max_model_len = self._llm.vllm_config.model_config.max_model_len self._defaults = self._llm.model_config.get_diff_sampling_param() - # The train and validation sampling NeMo-Gym rollouts must match, at the - # f32 precision the frontend delivers. - self._gym_sampling_profile = { - key: _as_f32(host.cfg[key]) for key in _GYM_SAMPLING_KEYS - } @classmethod async def from_args(cls, argv=None): @@ -393,11 +361,6 @@ async def _generate( async for chunk in self._generate_rollout(request, rollout, context): yield chunk return - annotations = request.get("annotations") or () - if GYM_ROLLOUT_ANNOTATION in annotations: - async for chunk in self._generate_gym(request, context): - yield chunk - return engine_input = await self._engine_input(request) prompt_len = len(engine_input["prompt_token_ids"]) self._check_prompt_fits(prompt_len) @@ -511,191 +474,6 @@ async def _generate_rollout( chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) yield chunk - async def _generate_gym( - self, request: GenerateRequest, context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - """One NeMo-Gym model call, served as the native chat endpoint serves it. - - The request was rendered by the frontend. As on the native server, it must - sample on-policy, its echoed or staged prefix replaces the rendered - history, output is clamped to the context, and a token-capture call stages - its delta from this process. Tokens, log probabilities and routed experts - come from vLLM's final output through the native endpoint's helpers. One - chunk carries the whole completion and, in ``engine_data``, the token - arrays Gym's dialect reads, or a captured call's commit coordinates. - """ - host = self._host - sampling: dict[str, Any] = request.get("sampling_options") or {} - sampling_error = off_policy_sampling_error( - temperature=_as_f32(sampling.get("temperature")), - top_p=_as_f32(sampling.get("top_p")), - top_k=sampling.get("top_k"), - generation_config=self._gym_sampling_profile, - ) - if sampling_error is not None: - raise InvalidArgument(sampling_error) - if (sampling.get("n") or 1) != 1: - raise InvalidArgument( - "NeMo-Gym calls through Dynamo take n=1: engine_data carries one " - "choice's tokens." - ) - # The profile's own values, so vLLM samples exactly as natively. - validation = ( - _as_f32(sampling["temperature"]) - != self._gym_sampling_profile["temperature"] - or _as_f32(sampling["top_p"]) != self._gym_sampling_profile["top_p"] - ) - temperature, top_p = ( - (host.cfg["val_temperature"], host.cfg["val_top_p"]) - if validation - else (host.cfg["temperature"], host.cfg["top_p"]) - ) - - capture_context = _annotation_value(request, GYM_CAPTURE_ANNOTATION_PREFIX) - admission = ( - None - if capture_context is None - else host._parse_capture_admission(json.loads(capture_context)) - ) - prefix_token_ids = None - if admission is not None: - if admission.mode == "token_in": - # A blocking data-plane read; keep it off the engine loop. - prefix_token_ids = await asyncio.to_thread( - host._resolve_admission_prefix, admission - ) - elif ( - echoed := _annotation_value(request, GYM_PREFIX_ANNOTATION_PREFIX) - ) is not None: - prefix_token_ids = json.loads(echoed) - # As the native endpoint does: a captured call reads the processor's own - # pixels, not cache references (#4124). - engine_input = await self._engine_input( - request, skip_mm_cache=admission is not None and host._capture_media - ) - template_token_ids = engine_input["prompt_token_ids"] - self._check_prompt_fits(len(template_token_ids)) - token_ids, splice = template_token_ids, None - if prefix_token_ids is not None: - # The frontend rendered the whole conversation; replace its history - # with the exact tokens the model generated for it. The engine sees no - # messages, so the prefix's EOS count stands in for the native - # endpoint's render of the history. - tokenizer = self._llm.tokenizer - splice = splice_prefix_tokens( - tokenizer=tokenizer, - model_prefix_token_ids=prefix_token_ids, - template_token_ids=template_token_ids, - template_prefix_eos_count=derive_prefix_eos_count( - prefix_token_ids, eos_token_id=tokenizer.eos_token_id - ), - ) - token_ids = splice.token_ids - self._check_prompt_fits(len(token_ids)) - try: - # Off the loop: a retained-media check reads staged calls. - media = await asyncio.to_thread( - host._capture_request_media, - engine_input, - admission=admission, - splice=splice, - ) - except MediaCaptureRejected as error: - raise InvalidArgument(f"{error} ({error.code})") from error - if ( - media is None - and splice is not None - and (placeholders := engine_input.get("mm_placeholders")) - ): - # Media capture already moved them to spliced coordinates. - engine_input["mm_placeholders"] = remap_multimodal_placeholders( - template_token_ids=template_token_ids, - final_token_ids=token_ids, - mm_placeholders=placeholders, - splice=splice, - ) - engine_input["prompt_token_ids"] = token_ids - - stop = (request.get("stop_conditions") or {}).get("stop") - overrides: dict[str, Any] = { - "temperature": temperature, - "top_p": top_p, - # As the native endpoint does: Gym requests leave top_k to NeMo RL. - "top_k": -1, - } - if stop: - # vLLM stops on them as natively, so the token arrays match; the - # frontend then cuts the text at the same place. - overrides.update(stop=stop, detokenize=True) - if admission is not None: - # A captured call stages the sampled tokens' log probabilities. - overrides["logprobs"] = 0 - sampling_params = build_sampling_params( - request, - self._defaults, - output_room=self._max_model_len - len(token_ids), - overrides=overrides, - output_kind=RequestOutputKind.FINAL_ONLY, - ) - try: - self._llm.check_admission(1, context.id()) - except GracefulHTTPError as error: - raise HttpError(int(error.http_status), error.message) from error - - if admission is not None: - host._begin_request_capture( - request, - token_ids, - admission=admission, - prefix_token_ids=prefix_token_ids, - media=media, - ) - staged = False - try: - final_output = None - async for output in self._llm.generate( - engine_input, sampling_params, context.id() - ): - final_output = output - if final_output is None: - raise RuntimeError(f"No output received for request {context.id()}") - if admission is not None and context.is_stopped(): - # Dynamo aborts the vLLM request on a disconnect but never - # cancels this generator: fail the capture (below), as the - # native endpoint's cancellation does, instead of staging it. - return - completion = final_output.outputs[0] - chunk: GenerateChunk = { - "token_ids": list(completion.token_ids), - "index": 0, - "finish_reason": completion.finish_reason, - "completion_usage": _usage(final_output, len(completion.token_ids)), - } - if admission is not None: - chunk["engine_data"] = { - "ng_commit_coords": await self._stage_gym_call( - request, final_output - ) - } - staged = True - else: - chunk["engine_data"] = self._gym_token_data(final_output) - log_probs, top_logprobs = extract_from_completion_output( - completion, 0, tokenizer=self._llm.tokenizer, include_bytes=True - ) - if log_probs is not None: - chunk["log_probs"] = log_probs - chunk["top_logprobs"] = top_logprobs - yield chunk - finally: - if admission is not None and not staged: - host._abort_request_capture( - request, - reason=( - "client_disconnect" if context.is_stopped() else "engine_error" - ), - ) - async def _load_media( self, request: GenerateRequest ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: @@ -748,9 +526,7 @@ async def _load_media( mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) return mm_data, mm_uuids - async def _engine_input( - self, request: GenerateRequest, *, skip_mm_cache: bool = False - ) -> dict[str, Any]: + async def _engine_input(self, request: GenerateRequest) -> dict[str, Any]: """The processed vLLM prompt for the request, as the native chat endpoint builds it. The frontend's tokens carry one unexpanded placeholder per media item; @@ -763,15 +539,13 @@ async def _engine_input( prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media if request.get("mm_processor_kwargs"): prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] - return await self._llm.renderer.process_for_engine_async( - prompt, time.time(), skip_mm_cache=skip_mm_cache - ) + return await self._llm.renderer.process_for_engine_async(prompt, time.time()) def _check_prompt_fits(self, prompt_len: int) -> None: """Refuse a prompt that leaves no room for output, as the native endpoint does. - NeMo-Gym reads "context length" in a 400 as a length refusal rather than an - engine failure. + OpenAI clients read "context length" in a 400 as a length refusal rather + than an engine failure. """ if prompt_len >= self._max_model_len: raise InvalidArgument( @@ -780,62 +554,6 @@ def _check_prompt_fits(self, prompt_len: int) -> None: "output tokens." ) - def _gym_token_data(self, final_output: Any) -> dict[str, Any]: - """The token fields Gym's dialect lifts onto the message, as the native endpoint reports them.""" - completion = final_output.outputs[0] - token_data: dict[str, Any] = { - "prompt_token_ids": list(final_output.prompt_token_ids), - "completion_token_ids": list(completion.token_ids), - } - if completion.logprobs is not None: - _, token_data["completion_logprobs"] = chat_generation_tokens(completion) - if self._host._return_routed_experts_enabled(): - # Full prompt + generation rows, as the native endpoint attaches them. - token_data["routed_experts"] = chat_routed_experts( - final_output, - completion, - prompt_token_count=len(final_output.prompt_token_ids), - device=torch.device("cpu"), - logger=LOGGER, - routed_experts_dtype=self._host.routed_experts_dtype, - ) - return token_data - - async def _stage_gym_call(self, request: GenerateRequest, final_output: Any) -> Any: - """Stage a captured call from this process; return its commit coordinates.""" - host = self._host - completion = final_output.outputs[0] - generation_token_ids, generation_log_probs = chat_generation_tokens(completion) - message: dict[str, Any] = { - "generation_token_ids": generation_token_ids, - "generation_log_probs": generation_log_probs, - } - if host._return_routed_experts_enabled(): - message["routed_experts"] = chat_routed_experts( - final_output, - completion, - prompt_token_count=len(final_output.prompt_token_ids), - device=torch.device("cpu"), - logger=LOGGER, - routed_experts_dtype=host.routed_experts_dtype, - ) - # The native endpoint's response shape; the prompt is the one recorded at - # begin_call. - content = { - "choices": [ - { - "index": 0, - "finish_reason": completion.finish_reason, - "message": message, - } - ] - } - # The sink write is a blocking data-plane round trip. - content = await asyncio.to_thread( - host._finish_request_capture, request, content - ) - return content["ng_commit_coords"] - async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) diff --git a/nemo_rl/models/generation/openai_server_utils.py b/nemo_rl/models/generation/openai_server_utils.py index 53a757dc04f..f01cd1f6091 100644 --- a/nemo_rl/models/generation/openai_server_utils.py +++ b/nemo_rl/models/generation/openai_server_utils.py @@ -58,48 +58,13 @@ def replace_prefix_tokens( ).token_ids -def _eos_token_id_set(eos_token_id: int | Collection[int]) -> frozenset[int]: - eos_token_ids = ( - frozenset([eos_token_id]) - if isinstance(eos_token_id, int) - else frozenset(eos_token_id) - ) - assert eos_token_ids, "EOS token id collection must not be empty" - return eos_token_ids - - -def derive_prefix_eos_count( - model_prefix_token_ids: list[int], *, eos_token_id: int | Collection[int] -) -> int: - """The EOS count a template render of the prefix's history would hold. - - For hosts that see only the rendered prompt, not the messages (the managed - Dynamo engine receives token ids from its frontend), this stands in for - counting EOS in ``template_prefix_token_ids``. Every EOS in the prefix - closed one message, and a template closes the final assistant turn with one - EOS even when the model was truncated before emitting it. The two counts - agree when the history up to the prefix's last assistant turn is the - history the prefix was generated from (Gym's lineage admission checks this - for captured calls; without capture it is assumed, as on the native server) - and the model emitted no EOS text as ordinary tokens. A render with fewer - EOS than this count fails the splice; an extra EOS before the boundary cuts - early instead. - """ - eos_token_ids = _eos_token_id_set(eos_token_id) - count = sum(tid in eos_token_ids for tid in model_prefix_token_ids) - if not model_prefix_token_ids or model_prefix_token_ids[-1] not in eos_token_ids: - count += 1 - return count - - def splice_prefix_tokens( *, tokenizer: Any, model_prefix_token_ids: list[int], - template_prefix_token_ids: list[int] | None = None, + template_prefix_token_ids: list[int], template_token_ids: list[int], eos_token_id: int | Collection[int] | None = None, - template_prefix_eos_count: int | None = None, ) -> PrefixSplice: """This is a subroutine used inside the OpenAI-compatible Chat Completion server. @@ -167,24 +132,19 @@ def splice_prefix_tokens( toward the boundary, and when the model's prefix ends on one of them that exact id is kept in place of the template's, so the spliced prompt still starts with the model's tokens verbatim. - - Pass exactly one of ``template_prefix_token_ids`` and - ``template_prefix_eos_count``. The prefix render is only ever used for its - EOS count, so a host without the messages passes the count directly (see - :func:`derive_prefix_eos_count`). """ - if (template_prefix_token_ids is None) == (template_prefix_eos_count is None): - raise ValueError( - "Pass exactly one of template_prefix_token_ids and " - "template_prefix_eos_count" - ) if not model_prefix_token_ids: return PrefixSplice(template_token_ids, 0, 0) if eos_token_id is None: eos_token_id = tokenizer.eos_token_id assert eos_token_id is not None, "Tokenizer must have an EOS token ID" - eos_token_ids = _eos_token_id_set(eos_token_id) + eos_token_ids = ( + frozenset([eos_token_id]) + if isinstance(eos_token_id, int) + else frozenset(eos_token_id) + ) + assert eos_token_ids, "EOS token id collection must not be empty" # The model isn't guaranteed to end on EOS (e.g. it hit max_tokens); chat # templates always add one, so cut the model input to just before its EOS. @@ -197,11 +157,7 @@ def splice_prefix_tokens( # templates may strip prior reasoning blocks when re-rendering history; # EOS counting preserves the original generated reasoning tokens without # requiring a customized chat template. - count_needed = ( - template_prefix_eos_count - if template_prefix_token_ids is None - else sum(tid in eos_token_ids for tid in template_prefix_token_ids) - ) + count_needed = sum(tid in eos_token_ids for tid in template_prefix_token_ids) count_seen = 0 template_cut_start = -1 for pos, tid in enumerate(template_token_ids): @@ -215,14 +171,14 @@ def splice_prefix_tokens( message = ( f"EOS token #{count_needed} not found in template_token_ids " f"(only found {count_seen} EOS tokens total)!\n" + f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" + f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" ) - if template_prefix_token_ids is not None: - message += f"Template prefix token IDs (everything before the final assistant message): {template_prefix_token_ids}\n\n" - message += f"Template token IDs (everything that was sent to the model endpoint): {template_token_ids}" if tokenizer is not None: - if template_prefix_token_ids is not None: - message += f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}" - message += f"\n\nTemplate repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" + message += ( + f"\n\nTemplate prefix repr (detokenized): {repr(tokenizer.decode(template_prefix_token_ids))}\n\n" + f"Template repr (detokenized): {repr(tokenizer.decode(template_token_ids))}" + ) raise AssertionError(message) boundary = ( diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 18fc37208e6..42eb7ccceed 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -191,21 +191,16 @@ def validate_settings(cls, master_config: "MasterConfig") -> None: """Reject pure-config vLLM settings the SC entrypoint cannot honor.""" generation_config = cast(VllmConfig, master_config.policy["generation"]) assert_reload_refit_config_supported(generation_config) - if dynamo_enabled(generation_config): - if master_config.async_rl.generation_router.enabled: - raise ValueError( - "async_rl.generation_router.enabled=true does not apply with " - "policy.generation.dynamo=true: NeMo-Gym calls the Dynamo " - "frontend, which routes the whole fleet behind one URL. " - "Disable the generation router." - ) - if master_config.async_rl.generation_fleet_health.enabled: - raise ValueError( - "async_rl.generation_fleet_health.enabled=true is not supported " - "with policy.generation.dynamo=true yet: the Dynamo frontend " - "picks engines itself, and a restarted shard does not register " - "with it. Disable fleet health." - ) + if ( + dynamo_enabled(generation_config) + and master_config.async_rl.generation_fleet_health.enabled + ): + raise ValueError( + "async_rl.generation_fleet_health.enabled=true is not supported " + "with policy.generation.dynamo=true yet: the Dynamo frontend " + "picks engines itself, and a restarted shard does not register " + "with it. Disable fleet health." + ) @staticmethod def init_cluster_placement_groups( @@ -722,18 +717,6 @@ def load_and_start(self) -> None: self.device_uuids = self._report_device_id() self._register_dynamo_engines() - @property - def client_base_urls(self) -> list[Optional[str]]: - """Where OpenAI clients such as NeMo-Gym send requests. - - The Dynamo frontend when it serves this fleet (``policy.generation.dynamo``); - otherwise every data-parallel shard's own server, as - ``dp_openai_server_base_urls``. Known before a deferred model load. - """ - if self.dynamo_frontend_url is not None: - return [self.dynamo_frontend_url] - return self.dp_openai_server_base_urls - def _start_dynamo_services(self) -> None: """Start the etcd, NATS and frontend this fleet's engines register with.""" from nemo_rl.models.generation.dynamo.managed_runtime import ( diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 1434af1c1d7..4b26762926a 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -757,10 +757,7 @@ def _capture_admission(self, request: Any) -> Any | None: context. The dict itself is never mutated: the admission is the typed, read-only contract that the prefix resolution and ``begin_call`` share. """ - return self._parse_capture_admission(getattr(request, "ng_capture", None)) - - def _parse_capture_admission(self, context: Any) -> Any | None: - """``_capture_admission`` for a raw ``ng_capture`` context (Dynamo's annotation).""" + context = getattr(request, "ng_capture", None) if self.token_capture is None or not context: return None # Deferred: nemo_gym is an optional extra absent in non-gym runs. diff --git a/tests/functional/L1_Functional_Tests_Dynamo.sh b/tests/functional/L1_Functional_Tests_Dynamo.sh index 23d08060f17..8f7e689228b 100644 --- a/tests/functional/L1_Functional_Tests_Dynamo.sh +++ b/tests/functional/L1_Functional_Tests_Dynamo.sh @@ -36,12 +36,7 @@ run_test() { # Dynamo overlays: native SingleController tests rerun with policy.generation.dynamo=true. # Each keeps the native test's checks and adds check_dynamo_overlay.sh's. run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller.sh ++policy.generation.dynamo=true -run_test fast uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true ++policy.generation.dynamo=true -# Gym's default mode: token arrays come back through the frontend in engine_data. -run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++policy.generation.dynamo=true run_test fast uv run --no-sync bash ./tests/functional/grpo_dp_single_controller_reload_refit.sh ++policy.generation.dynamo=true -# Router replay with deferred routes: the engine stages each captured call's routed experts. -run_test uv run --no-sync bash ./tests/functional/grpo_async_gym_single_controller.sh ++token_capture.enabled=true policy.model_name=silence09/DeepSeek-R1-Small-2layers ++policy.router_replay.enabled=true ++token_capture.defer_routed_experts_to_policy=true ++policy.generation.dynamo=true cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/grpo_async_gym_single_controller.sh b/tests/functional/grpo_async_gym_single_controller.sh index a69e17c847f..6acd46388cb 100755 --- a/tests/functional/grpo_async_gym_single_controller.sh +++ b/tests/functional/grpo_async_gym_single_controller.sh @@ -113,10 +113,6 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then if [[ "$*" == *token_capture.enabled=true* ]]; then # Nonzero only when a finalizer actor ran, i.e. capture really was on. EXTRA_CHECKS+=('max(data["train/finalize/total_ms"]) > 0') - if [[ "$*" == *router_replay.enabled=true* ]]; then - # Every captured row carries routed experts for the policy to replay. - EXTRA_CHECKS+=('min(data["train/finalize/routed_experts_row_coverage"]) == 1') - fi fi # Observed to be between 0.8-1.3 @@ -125,7 +121,3 @@ if [[ "${RUN_CONVERGENCE_CHECKS:-1}" == "1" ]]; then 'max(data["train/reward"]) > 0' \ "${EXTRA_CHECKS[@]}" fi - -if [[ "$*" == *generation.dynamo=true* ]]; then - bash $SCRIPT_DIR/check_dynamo_overlay.sh $RUN_LOG -fi diff --git a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh index ee3da965de3..2d265e6153b 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh @@ -157,7 +157,3 @@ uv run --no-sync tests/check_metrics.py "${JSON_METRICS}" \ 'min(data["train/finalize/media_row_rate"]) == 1' \ 'max(data["train/gen_kl_error"]) < 0.05' \ 'all_finite(data["train/reward"])' - -if [[ "$*" == *generation.dynamo=true* ]]; then - bash "${SCRIPT_DIR}/check_dynamo_overlay.sh" "${RUN_LOG}" -fi diff --git a/tests/test_suites/disabled.txt b/tests/test_suites/disabled.txt index 2d43c913cc5..83b5213a584 100644 --- a/tests/test_suites/disabled.txt +++ b/tests/test_suites/disabled.txt @@ -102,3 +102,7 @@ research/template_project/tests/test_suites/llm/single_update_1n8g.sh # Full Flow-GRPO recipe requires an externally served GenRM OCR judge. research/flow_grpo/tests/test_suites/flow_grpo_qwen_image_ocr.sh + +# A Dynamo + NeMo-Gym recipe: policy.generation.dynamo does not serve NeMo-Gym +# rollouts yet, and setup refuses the pairing. +tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index b1e4b880346..fad27b6913d 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -275,7 +275,6 @@ tests/test_suites/llm/grpo-qwen3-30ba3b-2n8g-megatron_fused_linear_logprobs.sh tests/test_suites/llm/sft-gpt-oss-20b-1n8g-automodel-ep8.sh # Nemotron 3 Nano 30B A3B Base BF16 tests -tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-automodel-ep8.sh tests/test_suites/llm/sft-nanov3-30BA3B-2n8g-automodel-lora.sh diff --git a/tests/unit/algorithms/test_distillation.py b/tests/unit/algorithms/test_distillation.py index f1c87daa2b4..8a34019eac8 100644 --- a/tests/unit/algorithms/test_distillation.py +++ b/tests/unit/algorithms/test_distillation.py @@ -1332,7 +1332,7 @@ class DummyVllmGeneration: def __init__(self, cluster, config, defer_model_load=False): self.cfg = config self.defer_model_load = defer_model_load - self.client_base_urls = ["http://reserved-vllm"] + self.dp_openai_server_base_urls = ["http://reserved-vllm"] # The per-job bearer token the real class generates when the # recipe requires one; Gym's model server must receive it. self.http_server_api_key = "k" diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index 1578e27c9a7..c161e7289d2 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -4408,7 +4408,7 @@ def offload_after_refit(self): class DummyVllmGeneration: weight_synchronizer = None - client_base_urls = ["http://vllm-worker.example/v1"] + dp_openai_server_base_urls = ["http://vllm-worker.example/v1"] # What the real class generates when the recipe requires a key. http_server_api_key = "per-job-key" diff --git a/tests/unit/environments/test_nemo_gym_token_capture.py b/tests/unit/environments/test_nemo_gym_token_capture.py index dfa4a991168..7fc9e3c9737 100644 --- a/tests/unit/environments/test_nemo_gym_token_capture.py +++ b/tests/unit/environments/test_nemo_gym_token_capture.py @@ -39,11 +39,7 @@ def _digest(label: str) -> str: @pytest.mark.parametrize( ("generation_backend", "expected"), - [ - ("vllm", "vllm_worker"), - ("megatron", "megatron_worker"), - ("dynamo", "dynamo_worker"), - ], + [("vllm", "vllm_worker"), ("megatron", "megatron_worker")], ) def test_external_staging_backend_maps_generation_backend( generation_backend: str, expected: str @@ -577,11 +573,8 @@ def test_setup_nemo_gym_config_megatron_keeps_async_rollout_check_satisfied() -> assert should_use_nemo_gym(config) is True -@pytest.mark.parametrize("dynamo", [False, True]) -def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( - dynamo: bool, -) -> None: - """With Dynamo, Gym's policy model calls the frontend in Gym's Dynamo dialect.""" +def test_setup_nemo_gym_config_refuses_dynamo_serving() -> None: + """Dynamo does not serve NeMo-Gym's calls yet, so the pairing fails at startup.""" from nemo_rl.algorithms.grpo import MasterConfig from nemo_rl.environments.nemo_gym import ( setup_nemo_gym_config, @@ -594,7 +587,7 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( policy={ "generation": { "backend": "vllm", - "dynamo": dynamo, + "dynamo": True, "vllm_cfg": {"async_engine": False, "expose_http_server": False}, } }, @@ -602,89 +595,5 @@ def test_setup_nemo_gym_config_vllm_speaks_dynamo_dialect_only_with_dynamo( setup_nemo_gym_config(config, tokenizer=None) - vllm_cfg = config.policy["generation"]["vllm_cfg"] - assert vllm_cfg["async_engine"] is True - assert vllm_cfg["expose_http_server"] is True - assert should_use_nemo_gym(config) is True - if dynamo: - # A NeMo RL key each actor applies to its composed config, not a - # policy_model overlay a sharded plan would need an owner for. - assert config.env["nemo_gym"] == { - "dynamo_dialect": {"default_chat_template_kwargs": {}} - } - else: - assert "nemo_gym" not in config.env - - -@pytest.mark.parametrize( - ("vllm_model", "expected"), - [ - ({}, {"enable_thinking": True, "truncate_history_thinking": False}), - ( - {"chat_template_kwargs": {"enable_thinking": False}}, - {"enable_thinking": False, "truncate_history_thinking": False}, - ), - ( - {"extra_body": {"chat_template_kwargs": {"force_nonempty_content": True}}}, - { - "enable_thinking": True, - "truncate_history_thinking": False, - "force_nonempty_content": True, - }, - ), - ], -) -def test_gym_actor_config_dynamo_sends_the_native_default_template_kwargs( - vllm_model: dict, expected: dict -) -> None: - """The native server's default template kwargs sit under the ones Gym sends.""" - from nemo_rl.algorithms.grpo import MasterConfig - from nemo_rl.environments.nemo_gym import ( - build_nemo_gym_config, - setup_nemo_gym_config, - ) - - user_vllm_model = copy.deepcopy(vllm_model) - - config = MasterConfig.model_construct( - env={ - "should_use_nemo_gym": True, - "nemo_gym": { - "policy_model": {"responses_api_models": {"vllm_model": vllm_model}} - }, - }, - data={}, - policy={ - "generation": { - "backend": "vllm", - "dynamo": True, - "vllm_cfg": { - "async_engine": True, - "http_server_serving_chat_kwargs": { - "default_chat_template_kwargs": { - "enable_thinking": True, - "truncate_history_thinking": False, - } - }, - }, - } - }, - ) - - setup_nemo_gym_config(config, tokenizer=None) - gym_config = build_nemo_gym_config( - config.env, - base_urls=["http://frontend"], - model_name="test-model", - enable_router_replay=False, - use_fastokens=False, - )["initial_global_config_dict"] - - composed = gym_config["policy_model"]["responses_api_models"]["vllm_model"] - assert composed["dynamo_frontend"] is True - # The frontend refuses reasoning under both of Gym's keys. - assert composed["reasoning_field"] == "reasoning_content" - assert composed["chat_template_kwargs"] == expected - assert "dynamo_dialect" not in gym_config - # The user's overlay is left as written. - assert vllm_model == user_vllm_model + with pytest.raises(ValueError, match="does not serve NeMo-Gym rollouts yet"): + should_use_nemo_gym(config) diff --git a/tests/unit/environments/test_nemo_gym_utils.py b/tests/unit/environments/test_nemo_gym_utils.py index c13270ccdb8..990a4603c44 100644 --- a/tests/unit/environments/test_nemo_gym_utils.py +++ b/tests/unit/environments/test_nemo_gym_utils.py @@ -1016,73 +1016,6 @@ def test_build_nemo_gym_actors_hands_every_actor_the_policy_api_key( assert all(cfg["policy_api_key"] == "per-job-key" for cfg in cluster.actor_configs) -@pytest.mark.parametrize( - "env_configs", [_env_configs, _shard_env_configs], ids=["unsharded", "sharded"] -) -def test_build_nemo_gym_actors_gives_every_actor_the_dynamo_dialect( - detected_uv_dirs, env_configs -): - """With Dynamo, each actor's composed config speaks Gym's Dynamo dialect. - - The dialect is not a policy_model overlay, which a sharded plan would reject - as unowned. A shard's own template kwargs still win over the native defaults. - """ - from nemo_rl.algorithms.grpo import MasterConfig - - env = env_configs() - sharded = "shards" in env["nemo_gym"] - if sharded: - env["nemo_gym"]["shards"][1]["overrides"] = { - "policy_model": { - "responses_api_models": { - "vllm_model": {"chat_template_kwargs": {"enable_thinking": False}} - } - } - } - config = MasterConfig.model_construct( - env=env, - data={}, - policy={ - "generation": { - "backend": "vllm", - "dynamo": True, - "vllm_cfg": { - "http_server_serving_chat_kwargs": { - "default_chat_template_kwargs": {"enable_thinking": True} - } - }, - } - }, - ) - setup_nemo_gym_config(config, tokenizer=None) - cluster = _FakeGymCluster() - - with _patched_cluster(cluster): - nemo_gym_mod.build_nemo_gym_actors( - config.env, - base_urls=["http://frontend"], - model_name="test-model", - tokenizer=_TOKENIZER, - enable_router_replay=False, - use_fastokens=False, - ) - - thinking = [ - cfg["initial_global_config_dict"]["policy_model"]["responses_api_models"][ - "vllm_model" - ]["chat_template_kwargs"]["enable_thinking"] - for cfg in cluster.actor_configs - ] - # judged, then the tools shard's two replicas with their own override. - assert thinking == ([True, False, False] if sharded else [True]) - for cfg in cluster.actor_configs: - gym_config = cfg["initial_global_config_dict"] - vllm_model = gym_config["policy_model"]["responses_api_models"]["vllm_model"] - assert vllm_model["dynamo_frontend"] is True - assert vllm_model["reasoning_field"] == "reasoning_content" - assert "dynamo_dialect" not in gym_config - - def test_build_nemo_gym_actors_spreads_every_replica_onto_its_own_node( detected_uv_dirs, ): diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 5de0f0ef08a..289f8b55bf7 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -25,7 +25,6 @@ dynamo_cfg, dynamo_chat_template, dynamo_parsers, - native_default_chat_template_kwargs, ) @@ -118,15 +117,7 @@ def test_frontend_argv_is_runtime_owned() -> None: host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg ) assert argv[argv.index("--router-mode") + 1] == "kv" - # Gym names a token-capture call's session; keep the session on one engine. - assert argv[argv.index("--router-session-affinity-ttl-secs") + 1] == "1800" assert argv[-2:] == ["--foo", "1"] - no_affinity = DynamoCfg.model_validate( - {"frontend_args": {"session_affinity_ttl_secs": None}} - ) - assert "--router-session-affinity-ttl-secs" not in build_dynamo_frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=no_affinity - ) clashing = DynamoCfg.model_validate( {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} @@ -182,16 +173,9 @@ def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> Non configure_generation_config(config, _Tokenizer()) -def test_chat_template_and_default_kwargs_follow_the_native_server() -> None: - config = _serving( - chat_template="templates/chat.jinja", - default_chat_template_kwargs={"enable_thinking": False}, - ) +def test_chat_template_follows_the_native_server() -> None: + config = _serving(chat_template="templates/chat.jinja") assert dynamo_chat_template(config) == "templates/chat.jinja" - assert native_default_chat_template_kwargs(config) == {"enable_thinking": False} - # The older spelling the native server also accepts. - legacy = _serving(chat_template_kwargs={"enable_thinking": True}) - assert native_default_chat_template_kwargs(legacy) == {"enable_thinking": True} assert dynamo_chat_template(_serving()) is None @@ -212,11 +196,3 @@ def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( _serving(chat_template_content_format="auto", enable_auto_tools=True), _Tokenizer(), ) - - -def test_a_bearer_token_is_rejected_because_the_frontend_checks_none() -> None: - config = _vllm_config(dynamo=True) - config["vllm_cfg"]["http_server_api_key_required"] = True - - with pytest.raises(ValueError, match="checks no bearer token"): - configure_generation_config(config, _Tokenizer()) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index 836b70c5ee5..f12a5937bb7 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -42,17 +42,12 @@ from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) -from nemo_rl.models.generation.openai_server_utils import ( # noqa: E402 - derive_prefix_eos_count, - splice_prefix_tokens, -) from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, rollout_result_from_wire, rollout_result_to_wire, ) -from nemo_rl.models.generation.vllm.utils import chat_routed_experts # noqa: E402 from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, ) @@ -110,7 +105,9 @@ class _FakeRenderer: def __init__(self) -> None: self.prompts = [] - async def process_for_engine_async(self, prompt, arrival_time, *, skip_mm_cache): + async def process_for_engine_async( + self, prompt, arrival_time, *, skip_mm_cache=False + ): self.prompts.append((prompt, skip_mm_cache)) if "multi_modal_data" not in prompt: return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} @@ -230,7 +227,7 @@ def test_rollouts_are_served_like_generate_async() -> None: } -def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None: +def test_router_replay_routes_reach_rollouts_as_natively() -> None: routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) final = _output( [1, 2, 3], @@ -266,29 +263,6 @@ def test_router_replay_routes_reach_rollouts_and_gym_calls_as_natively() -> None for key in native: assert torch.equal(shipped[key], native[key]), key - # A Gym call without capture carries the routes in engine_data, as the - # native chat endpoint attaches them to the response. - gym_final = _output( - [1, 3, 4], - _completion([10, 11], "stop", routed_experts=routes[3:]), - prompt_routed_experts=routes[:3], - ) - gym_host = _host( - _FakeLLM([gym_final]), vllm_kwargs={"enable_return_routed_experts": True} - ) - gym_engine = native_engine.NativeVllmEngine(gym_host, leader_worker_idx=0) - - (gym_chunk,) = _collect(gym_engine, _gym_request()) - - assert gym_chunk["engine_data"]["routed_experts"] == chat_routed_experts( - gym_final, - gym_final.outputs[0], - prompt_token_count=3, - device=torch.device("cpu"), - logger=native_engine.LOGGER, - routed_experts_dtype=torch.int16, - ) - def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: llm = _FakeLLM() @@ -631,160 +605,6 @@ def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: assert llm.calls == [] -# --- NeMo-Gym's Dynamo dialect ------------------------------------------------- - - -def _f32(value: float) -> float: - return float(np.float32(value)) - - -def _gym_request( - *, temperature=0.7, top_p=0.9, prefix=None, sampling=None, **fields -) -> dict: - annotations = [native_engine.GYM_ROLLOUT_ANNOTATION] - if prefix is not None: - annotations.insert( - 0, native_engine.GYM_PREFIX_ANNOTATION_PREFIX + json.dumps(prefix) - ) - sampling_options = {"temperature": temperature, "top_p": top_p, **(sampling or {})} - return { - "token_ids": [1, 3, 4], - # The frontend carries sampling floats as f32. - "sampling_options": { - key: _f32(value) if isinstance(value, float) else value - for key, value in sampling_options.items() - }, - # Gym's token information: logprobs=true, top_logprobs=0. - "output_options": {"logprobs": 0}, - "annotations": annotations, - **fields, - } - - -def test_gym_calls_sample_with_nemo_rls_exact_profile() -> None: - final = _output([1, 3, 4], _completion([10, 11], "stop")) - llm = _FakeLLM([final]) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - (chunk,) = _collect( - engine, _gym_request(stop_conditions={"max_tokens": 100, "stop": [""]}) - ) - - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == [1, 3, 4] - # The configured values, not the f32 values the frontend delivered. - assert (params.temperature, params.top_p) == (0.7, 0.9) - assert params.top_k == vllm.SamplingParams(top_k=-1).top_k - assert params.output_kind == vllm.sampling_params.RequestOutputKind.FINAL_ONLY - # Output fits the context, and vLLM stops on the stop strings itself. - assert params.max_tokens == MAX_MODEL_LEN - 3 - assert (params.stop, params.detokenize) == ([""], True) - assert chunk["engine_data"] == { - "prompt_token_ids": [1, 3, 4], - "completion_token_ids": [10, 11], - "completion_logprobs": [-0.5, -0.5], - } - assert (chunk["token_ids"], chunk["finish_reason"]) == ([10, 11], "stop") - assert chunk["log_probs"] == [-0.5, -0.5] - - -def test_gym_validation_calls_sample_with_the_validation_profile() -> None: - llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - _collect(engine, _gym_request(temperature=0.0, top_p=1.0)) - - ((_, params, _),) = llm.calls - assert (params.temperature, params.top_p) == (0.0, 1.0) - - -@pytest.mark.parametrize( - ("request_fields", "error"), - [ - ({"temperature": 0.5}, "matches neither"), - ({"top_p": None}, "top_p must be set explicitly"), - ({"sampling": {"top_k": 5}}, "Top k sampling parameter"), - ({"sampling": {"n": 2}}, "take n=1"), - ], -) -def test_gym_calls_that_would_sample_off_policy_are_refused( - request_fields, error -) -> None: - llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match=error): - _collect(engine, _gym_request(**request_fields)) - - assert llm.calls == [] - - -def test_gym_echoed_prefix_replaces_the_rendered_history() -> None: - # The model generated [5, 6, EOS]; the frontend re-rendered that turn as [7, EOS]. - prefix, rendered = [1, 5, 6, EOS], [1, 7, EOS, 3, 4] - spliced = splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=prefix, - template_token_ids=rendered, - template_prefix_eos_count=derive_prefix_eos_count(prefix, eos_token_id=EOS), - eos_token_id=EOS, - ).token_ids - assert spliced != rendered - llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) - llm.tokenizer = _Tokenizer() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - (chunk,) = _collect(engine, _gym_request(prefix=prefix, token_ids=rendered)) - - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == spliced - assert params.max_tokens == MAX_MODEL_LEN - len(spliced) - assert chunk["engine_data"]["prompt_token_ids"] == spliced - - -def test_gym_calls_whose_splice_fills_the_context_are_refused() -> None: - prefix = [1, *range(20, 20 + MAX_MODEL_LEN), EOS] - llm = _FakeLLM() - llm.tokenizer = _Tokenizer() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match="maximum context length"): - _collect(engine, _gym_request(prefix=prefix, token_ids=[1, 7, EOS, 3])) - - assert llm.calls == [] - - -def test_gym_echoed_prefix_keeps_media_placeholders_on_their_tokens() -> None: - # Turn 1 ran [1, , 3] and generated [5, 6, EOS]; the frontend - # re-rendered that turn as [7, EOS] and appended a turn with a second image. - turn1 = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] - prefix = [*turn1, 5, 6, EOS] - rendered = [1, IMAGE_PLACEHOLDER, 3, 7, EOS, IMAGE_PLACEHOLDER, 4] - spliced = [*prefix, *[IMAGE_TOKEN] * IMAGE_TOKENS, 4] - llm = _FakeLLM([_output(spliced, _completion([10], "stop"))]) - llm.tokenizer = _Tokenizer() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - (chunk,) = _collect( - engine, - _gym_request( - prefix=prefix, - token_ids=rendered, - multi_modal_data={"image_url": [{"Url": _image_url()}] * 2}, - ), - ) - - ((engine_input, params, _),) = llm.calls - assert engine_input["prompt_token_ids"] == spliced - # The first image sits in the model's own prefix; the second moved with the - # longer generated turn, as the native endpoint remaps them. - assert [ - (span.offset, span.length) for span in engine_input["mm_placeholders"]["image"] - ] == [(1, IMAGE_TOKENS), (len(prefix), IMAGE_TOKENS)] - assert params.max_tokens == MAX_MODEL_LEN - len(spliced) - assert chunk["engine_data"]["prompt_token_ids"] == spliced - - @pytest.mark.parametrize( ("router_mode", "prefix_caching", "publishes"), [("kv", True, True), ("kv", False, False), ("round-robin", True, False)], diff --git a/tests/unit/models/generation/test_dynamo_native_engine_capture.py b/tests/unit/models/generation/test_dynamo_native_engine_capture.py deleted file mode 100644 index 9c509a10fa0..00000000000 --- a/tests/unit/models/generation/test_dynamo_native_engine_capture.py +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""NeMo-Gym token capture through the Dynamo engine adapter. - -The worker's own capture lifecycle stages each call, as on the native chat -endpoint; only Gym's capture core is faked. -""" - -import json -from types import SimpleNamespace - -import numpy as np -import pytest -import torch - -pytestmark = [pytest.mark.nemo_gym, pytest.mark.vllm] -pytest.importorskip("dynamo._core") -pytest.importorskip("vllm") -# The vLLM lanes import this module without the nemo_gym extra. -pytest.importorskip("nemo_gym.token_id_capture.staging") - -from nemo_gym.token_id_capture.adapters.vllm import VLLMCaptureAdapter # noqa: E402 -from nemo_gym.token_id_capture.external_capture import ( # noqa: E402 - DYNAMO_CAPTURE_ANNOTATION_PREFIX, - DYNAMO_GYM_REQUEST_ANNOTATION, - DYNAMO_PREFIX_ANNOTATION_PREFIX, -) -from nemo_gym.token_id_capture.staging.records import CaptureAdmission # noqa: E402 -from responses_api_models.vllm_model.app import ( # noqa: E402 - _DYNAMO_ENGINE_DATA_FIELDS, -) - -from nemo_rl.data.captured_media import CapturedMedia # noqa: E402 -from nemo_rl.data_plane.tq_token_sink import ChainPrefixCache # noqa: E402 -from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 -from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL # noqa: E402 -from nemo_rl.utils.routed_experts_codec import decode_routed_experts # noqa: E402 -from tests.unit.models.generation.test_dynamo_native_engine import ( # noqa: E402 - EOS, - IMAGE_PLACEHOLDER, - IMAGE_TOKEN, - IMAGE_TOKENS, - _collect, - _completion, - _FakeLLM, - _gym_request, - _host, - _image_url, - _output, - _Tokenizer, -) - -COORDS = {"staging_key": "rollout-1/c2"} - - -class _Capture: - """Gym's capture core: records begin/complete/fail.""" - - adapter = VLLMCaptureAdapter() - - def __init__(self) -> None: - self.begun: list[tuple] = [] - self.staged: list[dict] = [] - self.failed: list[str] = [] - - def begin_call(self, admission, *, prefix_token_ids, stream): - self.begun.append((admission, prefix_token_ids, stream)) - return SimpleNamespace(admission=admission) - - def complete_call_from_response(self, call, payload, attachments=None): - self.staged.append(payload) - return SimpleNamespace(model_dump=lambda: dict(COORDS)) - - def fail_call(self, call, *, reason) -> None: - self.failed.append(reason) - - -def _capturing_host(llm, **cfg): - host = _host(llm, **cfg) - host.token_capture = _Capture() - host._capture_calls = {} - host._chain_prefix = ChainPrefixCache() - return host - - -def _captured_request(admission: CaptureAdmission, **fields) -> dict: - request = _gym_request(**fields) - # Gym's dynamo_worker handler: no logprobs, the admission in an annotation. - request["output_options"] = {} - request["annotations"] = [ - DYNAMO_CAPTURE_ANNOTATION_PREFIX + admission.model_dump_json(), - *request["annotations"], - ] - return request - - -def test_annotations_and_engine_data_are_the_ones_gyms_dynamo_dialect_uses() -> None: - assert native_engine.GYM_ROLLOUT_ANNOTATION == DYNAMO_GYM_REQUEST_ANNOTATION - assert native_engine.GYM_CAPTURE_ANNOTATION_PREFIX == ( - DYNAMO_CAPTURE_ANNOTATION_PREFIX - ) - assert native_engine.GYM_PREFIX_ANNOTATION_PREFIX == ( - DYNAMO_PREFIX_ANNOTATION_PREFIX - ) - # The engine_data keys Gym requires on a call without capture. - engine = native_engine.NativeVllmEngine(_host(_FakeLLM()), leader_worker_idx=0) - final = _output([1, 3, 4], _completion([10], "stop")) - assert engine._gym_token_data(final).keys() == _DYNAMO_ENGINE_DATA_FIELDS.keys() - - -def test_a_captured_token_in_call_is_staged_from_the_engine() -> None: - # The staged prefix [1, 5, 6, EOS] replaces the rendered history [1, 7, EOS]. - prefix = [1, 5, 6, EOS] - admission = CaptureAdmission( - rollout_id="rollout-1", - model_call_id="c2", - parent_call_id="c1", - prev_len=len(prefix), - mode="token_in", - required_prefix_token_ids=prefix, - parent_chain_hash="0" * 64, - ) - spliced = [*prefix, 3, 4] - routes = np.arange(8 * 2 * 2, dtype=np.int16).reshape(8, 2, 2) - final = _output( - spliced, - _completion([10, 11], "stop", routed_experts=routes[6:]), - prompt_routed_experts=routes[:6], - ) - llm = _FakeLLM([final]) - llm.tokenizer = _Tokenizer() - host = _capturing_host(llm, vllm_kwargs={"enable_return_routed_experts": True}) - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - - (chunk,) = _collect( - engine, _captured_request(admission, token_ids=[1, 7, EOS, 3, 4]) - ) - - ((prompt, params, _),) = llm.calls - assert prompt["prompt_token_ids"] == spliced - # The sampled tokens' log probabilities are staged; Gym asked for none. - assert params.logprobs == 0 - ((begun_admission, begun_prefix, stream),) = host.token_capture.begun - assert (begun_admission, begun_prefix, stream) == (admission, prefix, False) - (payload,) = host.token_capture.staged - assert payload["prompt_token_ids"] == spliced - message = payload["choices"][0]["message"] - assert message["generation_token_ids"] == [10, 11] - assert message["generation_log_probs"] == [-0.5, -0.5] - # As the native endpoint stages them: the full sequence's routes (none for - # the last token) from the boundary row, the prefix's last token, on. - staged_routes = decode_routed_experts(message["routed_experts"], torch.int16) - expected = torch.full((5, 2, 2), R3_MISSING_ROUTE_SENTINEL, dtype=torch.int16) - expected[:4] = torch.from_numpy(routes[3:7]) - assert torch.equal(staged_routes, expected) - # Only the commit coordinates go back through the frontend. - assert chunk["engine_data"] == {"ng_commit_coords": COORDS} - assert "log_probs" not in chunk - assert host._capture_calls == {} - - -def test_a_call_that_fails_after_admission_fails_its_capture() -> None: - admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" - ) - - class _FailingLLM(_FakeLLM): - async def generate(self, prompt, sampling_params, request_id): - raise RuntimeError("engine died") - yield # pragma: no cover - - host = _capturing_host(_FailingLLM()) - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - - with pytest.raises(RuntimeError, match="engine died"): - _collect(engine, _captured_request(admission)) - - assert host.token_capture.failed == ["engine_error"] - assert host._capture_calls == {} - - -def test_without_capture_installed_a_captured_call_is_served_uncaptured() -> None: - admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" - ) - llm = _FakeLLM([_output([1, 3, 4], _completion([10], "stop"))]) - host = _host(llm) - host.token_capture = None - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - - (chunk,) = _collect(engine, _captured_request(admission)) - - assert json.loads(json.dumps(chunk["engine_data"]))["completion_token_ids"] == [10] - - -def test_a_captured_media_call_stages_the_media_the_engine_ran_on() -> None: - admission = CaptureAdmission( - rollout_id="rollout-1", model_call_id="c1", mode="text" - ) - processed = [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] - llm = _FakeLLM([_output(processed, _completion([10], "stop"))]) - host = _capturing_host(llm) - host._capture_media = True - captured = CapturedMedia((), None) - snapshots = [] - - def capture_request_media(engine_input, *, admission, splice): - snapshots.append((list(engine_input["prompt_token_ids"]), admission, splice)) - return captured - - host._capture_request_media = capture_request_media - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) - - (chunk,) = _collect( - engine, - _captured_request( - admission, - token_ids=[1, IMAGE_PLACEHOLDER, 3], - multi_modal_data={"image_url": [{"Url": _image_url()}]}, - ), - ) - - # The processor's own pixels, not cache references, as natively (#4124). - ((_, skip_mm_cache),) = llm.renderer.prompts - assert skip_mm_cache is True - # Media is snapshotted from the processed prompt the engine runs. - assert snapshots == [(processed, admission, None)] - (payload,) = host.token_capture.staged - assert payload["prompt_token_ids"] == processed - assert chunk["engine_data"] == {"ng_commit_coords": COORDS} diff --git a/tests/unit/models/generation/test_openai_server_utils.py b/tests/unit/models/generation/test_openai_server_utils.py index 425e46442bc..223be2dc883 100644 --- a/tests/unit/models/generation/test_openai_server_utils.py +++ b/tests/unit/models/generation/test_openai_server_utils.py @@ -17,7 +17,6 @@ from nemo_rl.models.generation.openai_server_utils import ( PrefixSplice, - derive_prefix_eos_count, replace_prefix_tokens, splice_prefix_tokens, ) @@ -161,97 +160,6 @@ def test_splice_prefix_tokens_without_tokenizer_honors_explicit_eos_ids( ) assert result == expected assert result.token_ids[: len(model_prefix)] == model_prefix - # A host without the messages (managed Dynamo's engine) passes the EOS count - # derived from the prefix instead of the template prefix render. - assert ( - splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=model_prefix, - template_token_ids=template, - eos_token_id=eos_token_id, - template_prefix_eos_count=derive_prefix_eos_count( - model_prefix, eos_token_id=eos_token_id - ), - ) - == expected - ) - with pytest.raises(ValueError, match="exactly one"): - splice_prefix_tokens( - tokenizer=None, - model_prefix_token_ids=model_prefix, - template_prefix_token_ids=template_prefix, - template_token_ids=template, - eos_token_id=eos_token_id, - template_prefix_eos_count=1, - ) - - -@pytest.mark.parametrize( - "model_name", ["Qwen/Qwen3-0.6B", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"] -) -def test_derived_prefix_eos_count_matches_real_template_renders(model_name): - """Across tool-calling turns with reasoning, the EOS count derived from the - exact prefix equals the count in the template's render of the same history.""" - transformers = pytest.importorskip("transformers") - try: - tokenizer = transformers.AutoTokenizer.from_pretrained(model_name) - except OSError as error: - pytest.skip(f"{model_name} tokenizer unavailable: {error}") - eos = tokenizer.eos_token_id - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Weather for a city.", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - }, - } - ] - - def render(messages, *, add_generation_prompt): - rendered = tokenizer.apply_chat_template( - messages, - tools=tools, - add_generation_prompt=add_generation_prompt, - tokenize=True, - ) - return list(getattr(rendered, "input_ids", rendered)) - - def generate(text): - return tokenizer.encode(text, add_special_tokens=False) + [eos] - - messages = [ - {"role": "system", "content": "You are a weather assistant."}, - {"role": "user", "content": "Weather in Paris?"}, - ] - replies = [ - "\nI should look it up.\n\n\nChecking the forecast.", - "\nIt is sunny.\n\n\nIt is sunny in Paris.", - ] - follow_ups = [ - {"role": "tool", "content": "sunny, 22C"}, - {"role": "user", "content": "And London?"}, - ] - chain = render(messages, add_generation_prompt=True) + generate(replies[0]) - for reply, follow_up, next_reply in zip(replies, follow_ups, replies[1:] + ["ok"]): - messages = [*messages, {"role": "assistant", "content": reply}, follow_up] - template_prefix = render(messages[:-1], add_generation_prompt=False) - template = render(messages, add_generation_prompt=True) - assert derive_prefix_eos_count(chain, eos_token_id=eos) == sum( - token == eos for token in template_prefix - ) - spliced = splice_prefix_tokens( - tokenizer=tokenizer, - model_prefix_token_ids=chain, - template_prefix_token_ids=template_prefix, - template_token_ids=template, - ).token_ids - chain = spliced + generate(next_reply) def test_replace_prefix_tokens_without_tokenizer_reports_missing_eos_without_decoding(): diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 8e342ae8274..7cf184b311b 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2571,7 +2571,7 @@ def test_vllm_validate_settings_accepts_missing_vllm_cfg_as_refit_disabled(): VllmGeneration.validate_settings(master_config) -def _dynamo_master_config(*, router=False, fleet_health=False): +def _dynamo_master_config(*, fleet_health=False): vllm_config = deepcopy(basic_vllm_test_config) vllm_config["dynamo"] = True vllm_config["vllm_cfg"]["async_engine"] = True @@ -2579,7 +2579,6 @@ def _dynamo_master_config(*, router=False, fleet_health=False): return types.SimpleNamespace( policy={"generation": vllm_config}, async_rl=types.SimpleNamespace( - generation_router=types.SimpleNamespace(enabled=router), generation_fleet_health=types.SimpleNamespace(enabled=fleet_health), ), ) @@ -2589,7 +2588,6 @@ def _dynamo_master_config(*, router=False, fleet_health=False): ("settings", "error"), [ ({}, None), - ({"router": True}, "routes the whole fleet behind one URL"), ({"fleet_health": True}, "Disable fleet health"), ], ) @@ -2603,19 +2601,13 @@ def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error @pytest.mark.parametrize("dynamo", [False, True]) -def test_vllm_token_rollouts_and_clients_use_the_dynamo_frontend_when_it_serves( - dynamo, -): +def test_vllm_token_rollouts_use_the_dynamo_frontend_when_it_serves(dynamo): generation = object.__new__(VllmGeneration) - generation.dp_openai_server_base_urls = ["http://a/v1", "http://b/v1"] generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None tokens = BatchedDataDict({"input_ids": torch.tensor([[1]])}) # The frontend request carries token IDs only, so media prompts stay native. media = BatchedDataDict({"input_ids": torch.tensor([[1]]), "vllm_content": [None]}) - assert generation.client_base_urls == ( - ["http://frontend/v1"] if dynamo else ["http://a/v1", "http://b/v1"] - ) assert generation._dynamo_serves(tokens) is dynamo assert not generation._dynamo_serves(media) diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index 25b5a2e19ee..3414010f089 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -230,7 +230,6 @@ def _worker_with_capture(sink: _MemorySink): for name in ( "_fetch_chain_prefix", "_capture_admission", - "_parse_capture_admission", "_resolve_admission_prefix", "_enter_request_prefix", ): diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index 4328d05736b..d81d66628d0 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -2222,7 +2222,7 @@ def test_nemo_gym_wires_env_handle(self, patched_factories): assert data_kwargs["is_vlm"] is True mock_spinup.assert_called_once_with( mc.env, - base_urls=patched_factories["fake_gen"].client_base_urls, + base_urls=patched_factories["fake_gen"].dp_openai_server_base_urls, model_name="test-model", # Reaches the actor once, at spinup, rather than riding along with every # run_rollouts call. @@ -2375,44 +2375,6 @@ def test_token_capture_always_creates_finalizer_actor_pool( _, setup_kwargs = generation.setup_token_capture.call_args assert setup_kwargs["capture_media"] is with_processor - def test_dynamo_token_capture_tells_gym_to_speak_dynamo(self, patched_factories): - """The same vLLM workers stage the calls; Gym reaches them through the frontend.""" - mc = _make_master_config(backend="vllm") - mc.policy["generation"].update( - { - "model_name": "test-model", - "stop_strings": None, - "stop_token_ids": None, - "top_k": None, - "dynamo": True, - "vllm_cfg": {"async_engine": True, "expose_http_server": True}, - } - ) - mc.token_capture.enabled = True - mc.env = {"should_use_nemo_gym": True, "nemo_gym": {}} - patched_factories["setup_response_data"].return_value = (list(range(8)), None) - - with ( - patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), - patch.object( - sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() - ) as mock_spinup, - patch.object(sc_setup_mod, "validate_dataset_agent_coverage"), - patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), - patch( - "nemo_rl.experience.rollout_reassembler_actor.create_rollout_reassembler_actors", - return_value=[MagicMock()], - ), - ): - setup_single_controller(mc, MagicMock(pad_token_id=9)) - - assert mc.token_capture.generation_backend == "dynamo" - assert mock_spinup.call_args.kwargs["token_capture"]["generation_backend"] == ( - "dynamo" - ) - generation, _ = patched_factories["_build_generation"].return_value - generation.setup_token_capture.assert_called() - def test_nemo_gym_coverage_failure_shuts_down_shards(self, patched_factories): mc = _make_master_config(colocated=False, backend="vllm") mc.policy["generation"]["model_name"] = "test-model" From 39285257758a395e6955dbdee9599e447a63108f Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 13:47:47 -0700 Subject: [PATCH 08/19] refactor(vllm): serve every rollout through one worker method Address review on this PR: - _serve_rollout runs a rollout's whole sequence (budget, nothing-fits, SamplingParams, generate, prompt check, result), so every host that serves a rollout from this engine calls one method instead of copying the steps. It replaces _rollout_sampling_params. - generate_async asserts a single sample, so it no longer fans out per-sample tasks: it formats the prompt, calls _serve_rollout and yields the output. vLLM still aborts the request on cancellation. - rollout_output drops pad_token_id, whose fill never reached the output. - off_policy_sampling_error documents that vLLM fills unset top_p/top_k from generation_config.json, so callers must pin top_k=-1. - chat_routed_experts drops a dead isinstance check. - Tests: a route test for the on-policy assert and the top_k pin; the budget tests fold into one parametrization that covers a full context with speculative decoding; the wiring test asserts the exact output keys and a missing logprob; the R3 pipeline test uses vLLM 0.29's route shape. Two pipeline tests the wiring tests now cover are removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/vllm/request_pipeline.py | 36 ++--- nemo_rl/models/generation/vllm/utils.py | 37 ++--- .../generation/vllm/vllm_worker_async.py | 151 +++++++----------- .../models/generation/test_vllm_generation.py | 78 ++++++--- .../generation/test_vllm_request_pipeline.py | 40 +---- 5 files changed, 146 insertions(+), 196 deletions(-) diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index bb77e3b0e27..9e5bea46416 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -14,11 +14,11 @@ """Turn a finished vLLM rollout into NeMo RL's per-sample generation output. -Building the per-sample result is kept apart from assembling the output tensors, -so a path that builds the result on the engine and assembles the output -elsewhere behaves the same as the worker's own ``generate_async``. A rollout -feature added here, such as router replay (R3), then behaves the same on every -path. +The worker's ``_serve_rollout`` builds the result; ``rollout_output`` assembles +the output tensors from it. The two are apart so a host that builds the result +on the engine and assembles the output elsewhere returns what ``generate_async`` +returns. A rollout feature added here, such as router replay (R3), then behaves +the same on every path. """ import logging @@ -55,7 +55,7 @@ def nothing_generated(cls) -> "RolloutResult": return cls([], [], False, None, None) -def selected_logprobs(completion: Any) -> list[float]: +def _selected_logprobs(completion: Any) -> list[float]: """The sampled token's log probability at each position of a completion.""" by_position = completion.logprobs or () logprobs = [] @@ -73,7 +73,6 @@ def rollout_result( return_routed_experts: bool, routed_experts_dtype: torch.dtype, device: torch.device, - sample_idx: int = 0, ) -> RolloutResult: """Build the result of a finished rollout from vLLM's final RequestOutput. @@ -83,7 +82,6 @@ def rollout_result( return_routed_experts: Router replay is on, so routes are required. routed_experts_dtype: The dtype router replay consumes. device: Where to place the routed-experts tensor. - sample_idx: For log messages only. """ completion = final_output.outputs[0] token_ids = list(completion.token_ids) @@ -110,17 +108,16 @@ def rollout_result( if return_routed_experts and r3_stats["missing_routes"] > 0: LOGGER.warning( "R3 router replay fallback: vLLM returned incomplete " - "routed_experts for sample_idx=%d, missing_token_routes=%d, " - "actual_routes=%d, expected_routes=%d. Megatron will use its " - "own router for those missing token routes.", - sample_idx, + "routed_experts, missing_token_routes=%d, actual_routes=%d, " + "expected_routes=%d. Megatron will use its own router for those " + "missing token routes.", r3_stats["missing_routes"], r3_stats["actual_routes"], r3_stats["expected_routes"], ) return RolloutResult( token_ids=token_ids, - logprobs=selected_logprobs(completion), + logprobs=_selected_logprobs(completion), truncated=completion.finish_reason == "length", routed_experts=routed_experts, r3_stats=r3_stats if return_routed_experts else None, @@ -132,7 +129,6 @@ def rollout_output( *, input_ids: torch.Tensor, input_length: int, - pad_token_id: int, ) -> BatchedDataDict[GenerationOutputSpec]: """The single-sample output ``generate_async`` yields for ``result``. @@ -140,17 +136,15 @@ def rollout_output( result: The rollout's result. input_ids: The sample's (right-padded) prompt row. input_length: Prompt length in tokens. - pad_token_id: The tokenizer's pad token, as ``generate_async`` uses it. """ device = input_ids.device generated = result.token_ids output_length = input_length + len(generated) - output_ids = torch.full( - (output_length,), pad_token_id, dtype=input_ids.dtype, device=device - ) - output_ids[:input_length] = input_ids[:input_length] - output_ids[input_length:] = torch.tensor( - generated, dtype=input_ids.dtype, device=device + output_ids = torch.cat( + ( + input_ids[:input_length], + torch.tensor(generated, dtype=input_ids.dtype, device=device), + ) ) logprobs = torch.zeros((1, output_length), dtype=torch.float32, device=device) logprobs[0, input_length:] = torch.tensor( diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index 2d871819ae7..4a6e41e5c5f 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -15,7 +15,7 @@ from collections import defaultdict from collections.abc import Mapping from dataclasses import is_dataclass, replace -from typing import Any, Optional +from typing import Any, Optional, cast import torch @@ -336,9 +336,12 @@ def off_policy_sampling_error( Args: temperature: The request's temperature. top_p: The request's top_p. It must be explicit: vLLM resolves an unset top_p - from the model's generation_config.json, not to 1.0. + from the model's generation_config.json (``ModelConfig.generation_config`` + defaults to ``"auto"``), not necessarily 1.0. top_k: The request's top_k, which must be unset or -1 (disabled), matching - ``BaseVllmGenerationWorker._build_sampling_params``. + ``BaseVllmGenerationWorker._build_sampling_params``. vLLM resolves an + unset top_k from generation_config.json too, so None means on-policy only + once the caller sends the request with top_k=-1. generation_config: ``policy.generation``, with the train and validation sampling parameters. """ @@ -558,22 +561,20 @@ def chat_routed_experts( ) -> str: """The routed-experts envelope the chat endpoint attaches to one choice.""" generation_token_count = len(getattr(generation_details, "token_ids", []) or []) - routed_result = pad_and_align_routed_expert_indices( - final_request_output, - generation_details, - valid_length=prompt_token_count + generation_token_count, - padded_length=prompt_token_count + generation_token_count, - device=device, - require_complete_routed_experts=True, - return_stats=True, - routed_experts_dtype=routed_experts_dtype, + # return_stats=True returns the (routes, stats) pair. + routed_experts, r3_stats = cast( + tuple[Optional[torch.Tensor], dict[str, int]], + pad_and_align_routed_expert_indices( + final_request_output, + generation_details, + valid_length=prompt_token_count + generation_token_count, + padded_length=prompt_token_count + generation_token_count, + device=device, + require_complete_routed_experts=True, + return_stats=True, + routed_experts_dtype=routed_experts_dtype, + ), ) - if not isinstance(routed_result, tuple): - raise RuntimeError( - "Expected routed_experts alignment to return stats for the " - "OpenAI-compatible chat endpoint." - ) - routed_experts, r3_stats = routed_result if routed_experts is None: raise RuntimeError( "vLLM was asked to return routed experts for the " diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 2c67cc2a1f8..e47a99245d5 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -1795,8 +1795,7 @@ def _rollout_max_new_tokens(self, input_length: int) -> int: """Token budget for one rollout prompt; 0 means nothing fits. The configured cap, clipped to the context window and to speculative - decoding's lookahead. Kept apart from ``generate_async`` so every path that - serves a rollout from this engine gives it the same budget. + decoding's lookahead. """ max_model_len = int(self.cfg["vllm_cfg"]["max_model_len"]) allowed_new_tokens = max( @@ -1814,45 +1813,73 @@ def _rollout_max_new_tokens(self, input_length: int) -> int: ) return allowed_new_tokens - def _rollout_sampling_params( + async def _serve_rollout( self, + prompt: Any, *, - max_new_tokens: int, + prompt_token_ids: list[int], greedy: bool, stop_strings: Optional[list[str]], - ) -> Any: - """The SamplingParams for one rollout. + request_id: str, + device: torch.device, + ) -> tuple[RolloutResult, Optional[Any]]: + """Serve one rollout from this engine: its budget, sampling, generation and result. - Kept apart from ``generate_async`` so every path that serves a rollout - from this engine samples it the same way. + Every path that serves a rollout from this engine calls this, so a rollout + step added here runs on all of them. Args: - max_new_tokens: The rollout's budget, from ``_rollout_max_new_tokens``. + prompt: The vLLM prompt to generate from. + prompt_token_ids: The learner's prompt tokens, which vLLM must have run. greedy: Greedy decoding instead of the configured sampling. - stop_strings: The sample's own stop strings, merged with the + stop_strings: The rollout's own stop strings, merged with the configured ones. + request_id: The vLLM request ID. + device: Where to place the routed-experts tensor. + + Returns: + The result, and vLLM's final RequestOutput (None when no new token fits). """ - return self._build_sampling_params( + max_new_tokens = self._rollout_max_new_tokens(len(prompt_token_ids)) + if max_new_tokens == 0: + return RolloutResult.nothing_generated(), None + sampling_params = self._build_sampling_params( greedy=greedy, stop_strings=self._merge_stop_strings( [stop_strings] if stop_strings else None ), max_new_tokens=max_new_tokens, ) + final_output = None + async for output in self.llm.generate( + prompt=prompt, sampling_params=sampling_params, request_id=request_id + ): + final_output = output + if final_output is None: + raise RuntimeError(f"No output received for request {request_id}") + validate_rollout_prompt(prompt_token_ids, final_output.prompt_token_ids) + result = rollout_result( + final_output, + input_length=len(prompt_token_ids), + return_routed_experts=self._return_routed_experts_enabled(), + routed_experts_dtype=self.routed_experts_dtype, + device=device, + ) + return result, final_output async def generate_async( self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False, ) -> AsyncGenerator[tuple[int, BatchedDataDict[GenerationOutputSpec]], None]: - """Generate a batch of data using vLLM's AsyncLLMEngine, yielding results as they are ready. + """Generate one sample with vLLM's AsyncLLM, yielding it when it is ready. Args: - data: BatchedDataDict with input_ids and input_lengths - greedy: Whether to use greedy decoding instead of sampling + data: BatchedDataDict with input_ids and input_lengths for one sample. + greedy: Whether to use greedy decoding instead of sampling. Yields: - Tuple of (original_index, BatchedDataDict conforming to GenerationOutputSpec for the single sequence) + (0, the sample's output conforming to GenerationOutputSpec). """ if not self.cfg["vllm_cfg"]["async_engine"]: raise RuntimeError( @@ -1865,92 +1892,26 @@ async def generate_async( verify_right_padding(data, pad_value=self.cfg["_pad_token_id"]) - input_ids_batch = data["input_ids"] - input_lengths_batch = data["input_lengths"] - batch_size = input_ids_batch.shape[0] - - # Ensure generate_async only receives single samples (batch_size = 1) + batch_size = data["input_ids"].shape[0] assert batch_size == 1, ( f"generate_async is restricted to handle only single samples, " f"but received batch_size={batch_size}. Please handle batching outside this method." ) - batch_specific_stop_strings_list = data.get( - "stop_strings", [[] for _ in range(batch_size)] + input_ids = data["input_ids"][0] + input_length = int(data["input_lengths"][0].item()) + prompt = self._tokenize_prompt_with_bos( + format_prompt_for_vllm_generation(data, 0) ) - - # Create tasks for each sample in the batch - async def process_single_sample(sample_idx): - """Process a single sample and return the result.""" - current_input_actual_length = input_lengths_batch[sample_idx].item() - prompt = format_prompt_for_vllm_generation(data, sample_idx) - prompt = self._tokenize_prompt_with_bos(prompt) - - per_sample_stop_strings = None - if batch_specific_stop_strings_list and sample_idx < len( - batch_specific_stop_strings_list - ): - per_sample_stop_strings = batch_specific_stop_strings_list[sample_idx] - - max_new_tokens = self._rollout_max_new_tokens(current_input_actual_length) - input_ids_single_row = input_ids_batch[sample_idx] - if max_new_tokens == 0: - # No new token fits in the context: return the prompt alone. - result = RolloutResult.nothing_generated() - else: - sampling_params_for_request = self._rollout_sampling_params( - max_new_tokens=max_new_tokens, - greedy=greedy, - stop_strings=per_sample_stop_strings, - ) - request_id = str(uuid.uuid4()) - final_request_output = None - async for req_output in self.llm.generate( - prompt=prompt, - sampling_params=sampling_params_for_request, - request_id=request_id, - ): - final_request_output = req_output - if final_request_output is None: - raise RuntimeError(f"No output received for request {request_id}") - - validate_rollout_prompt( - input_ids_single_row[:current_input_actual_length].tolist(), - final_request_output.prompt_token_ids, - ) - result = rollout_result( - final_request_output, - input_length=current_input_actual_length, - return_routed_experts=self._return_routed_experts_enabled(), - routed_experts_dtype=self.routed_experts_dtype, - device=input_ids_single_row.device, - sample_idx=sample_idx, - ) - return ( - sample_idx, - rollout_output( - result, - input_ids=input_ids_single_row, - input_length=current_input_actual_length, - pad_token_id=self.cfg["_pad_token_id"], - ), - ) - - # Create tasks for all samples and yield results as they complete - sample_tasks = [ - asyncio.create_task(process_single_sample(i)) for i in range(batch_size) - ] - - # Yield results as they become available - try: - for completed_task in asyncio.as_completed(sample_tasks): - result = await completed_task - yield result - finally: - for task in sample_tasks: - if not task.done(): - task.cancel() - await asyncio.gather(*sample_tasks, return_exceptions=True) + result, _ = await self._serve_rollout( + prompt, + prompt_token_ids=input_ids[:input_length].tolist(), + greedy=greedy, + stop_strings=(data.get("stop_strings") or [None])[0], + request_id=str(uuid.uuid4()), + device=input_ids.device, + ) + yield 0, rollout_output(result, input_ids=input_ids, input_length=input_length) async def generate_text_async( self, data: BatchedDataDict[GenerationDatumSpec], greedy: bool = False diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index d0533c1a734..a79543ce11e 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -491,8 +491,10 @@ async def generate(self, prompt, sampling_params, request_id): outputs=[ types.SimpleNamespace( token_ids=self.generated, + # The second token has no reported logprob. logprobs=[ - {t: types.SimpleNamespace(logprob=-0.5)} for t in self.generated + {} if i == 1 else {t: types.SimpleNamespace(logprob=-0.5)} + for i, t in enumerate(self.generated) ], finish_reason=self.finish_reason, routed_experts=None, @@ -552,51 +554,56 @@ async def test_async_vllm_generate_async_samples_and_returns_the_rollout() -> No assert sampling["top_k"] == -1 assert sorted(sampling["stop"]) == ["", ""] assert output["output_ids"].tolist() == [[1, 2, 3, 4, 5, 10, 11, 2]] - assert output["logprobs"].tolist() == [[0.0] * 5 + [-0.5] * 3] + # A position without a reported logprob stays 0.0, aligned with its token. + assert output["logprobs"].tolist() == [[0.0] * 5 + [-0.5, 0.0, -0.5]] assert output["generation_lengths"].tolist() == [3] assert output["unpadded_sequence_lengths"].tolist() == [8] assert output["truncated"].tolist() == [False] + # No router replay keys when R3 is off. + assert set(output) == { + "output_ids", + "logprobs", + "generation_lengths", + "unpadded_sequence_lengths", + "truncated", + } @pytest.mark.asyncio @pytest.mark.parametrize( - ("speculative_config", "expected_max_tokens"), + ("prompt_length", "speculative_config", "expected_max_tokens"), [ # min(max_new_tokens=8, max_model_len 16 - prompt 12) - (None, 4), + (12, None, 4), # The drafter reads 2 tokens ahead: 16 - 12 - (2 + 1) - ({"num_speculative_tokens": 2}, 1), + (12, {"num_speculative_tokens": 2}, 1), + # Nothing fits: the prompt comes back without calling vLLM. + (16, None, 0), + # The spec-decode clamp floors at 1 token; a full context must not reach it. + (16, {"num_speculative_tokens": 2}, 0), ], ) -async def test_async_vllm_generate_async_caps_the_budget_to_the_context( - speculative_config, expected_max_tokens +async def test_async_vllm_generate_async_budget( + prompt_length, speculative_config, expected_max_tokens ) -> None: llm = _ScriptedAsyncLLM([10], finish_reason="length") worker = _rollout_worker( llm, vllm_kwargs={"speculative_config": speculative_config} ) - - output = await _generate_one(worker, list(range(1, 13))) - - ((_, sampling),) = llm.calls - assert sampling["max_tokens"] == expected_max_tokens - assert output["truncated"].tolist() == [True] - - -@pytest.mark.asyncio -async def test_async_vllm_generate_async_returns_the_prompt_when_nothing_fits() -> None: - llm = _ScriptedAsyncLLM([10]) - worker = _rollout_worker(llm) - prompt = list(range(1, 17)) + prompt = list(range(1, prompt_length + 1)) output = await _generate_one(worker, prompt) - assert llm.calls == [] - assert output["output_ids"].tolist() == [prompt] - assert output["logprobs"].tolist() == [[0.0] * 16] - assert output["generation_lengths"].tolist() == [0] - assert output["unpadded_sequence_lengths"].tolist() == [16] - assert output["truncated"].tolist() == [False] + if expected_max_tokens == 0: + assert llm.calls == [] + assert output["output_ids"].tolist() == [prompt] + assert output["logprobs"].tolist() == [[0.0] * prompt_length] + assert output["generation_lengths"].tolist() == [0] + assert output["truncated"].tolist() == [False] + else: + ((_, sampling),) = llm.calls + assert sampling["max_tokens"] == expected_max_tokens + assert output["truncated"].tolist() == [True] @pytest.mark.asyncio @@ -1430,6 +1437,25 @@ async def create_chat_completion(_request, _raw_request): assert error["param"] == "messages" +@pytest.mark.asyncio +async def test_chat_handler_rejects_off_policy_sampling_and_pins_top_k(monkeypatch): + """The route asserts on-policy sampling, then hands vLLM top_k=-1.""" + _, serving_chat, chat_handler = _setup_fake_vllm_chat_handler(monkeypatch) + serving_chat.create_chat_completion = AsyncMock(return_value=iter(())) + + off_policy = _fake_chat_request() + off_policy.temperature = 0.5 + with pytest.raises(AssertionError, match="matches neither"): + await chat_handler(off_policy, MagicMock()) + serving_chat.create_chat_completion.assert_not_awaited() + + request = _fake_chat_request() + request.top_k = None + await chat_handler(request, MagicMock()) + serving_chat.create_chat_completion.assert_awaited_once() + assert request.top_k == -1 + + @pytest.mark.asyncio @pytest.mark.parametrize("logprobs", [False, True]) async def test_chat_completion_cancelled_on_client_disconnect(monkeypatch, logprobs): diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py index c00be9dc2c4..4b84739f04a 100644 --- a/tests/unit/models/generation/test_vllm_request_pipeline.py +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -73,43 +73,11 @@ def _result(final_output, *, input_length=3, r3=False) -> RolloutResult: ) -def test_output_layout_matches_generate_async() -> None: - result = _result(_final_output([7, 8, 9], finish_reason="length")) - output = rollout_output( - result, input_ids=torch.tensor([1, 2, 3, 0, 0]), input_length=3, pad_token_id=0 - ) - - assert output["output_ids"].tolist() == [[1, 2, 3, 7, 8, 9]] - # A position without a reported logprob stays 0.0, aligned with its token. - assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0, -0.5, 0.0, -1.0]] - assert output["generation_lengths"].tolist() == [3] - assert output["unpadded_sequence_lengths"].tolist() == [6] - assert output["truncated"].tolist() == [True] - assert "routed_experts" not in output - - -def test_nothing_generated_returns_the_prompt_untruncated() -> None: - output = rollout_output( - RolloutResult.nothing_generated(), - input_ids=torch.tensor([1, 2, 3, 0]), - input_length=3, - pad_token_id=0, - ) - - assert output["output_ids"].tolist() == [[1, 2, 3]] - assert output["logprobs"].tolist() == [[0.0, 0.0, 0.0]] - assert output["generation_lengths"].tolist() == [0] - assert output["truncated"].tolist() == [False] - - def test_router_replay_routes_are_aligned_and_counted() -> None: - # 3 prompt + 3 generated tokens need 5 routes; vLLM reported 3 + 1. - result = _result( - _final_output([7, 8, 9], prompt_routes=3, generated_routes=1), r3=True - ) - output = rollout_output( - result, input_ids=torch.tensor([1, 2, 3]), input_length=3, pad_token_id=0 - ) + # 3 prompt + 3 generated tokens need 5 routes; vLLM reports prompt and + # generated routes together on the completion, here only 4 of them. + result = _result(_final_output([7, 8, 9], generated_routes=4), r3=True) + output = rollout_output(result, input_ids=torch.tensor([1, 2, 3]), input_length=3) routed = output["routed_experts"] assert routed.shape == (1, 6, LAYERS, TOPK) and routed.dtype == torch.int16 From ed3ee353804675bde437e8aeb90dab131f8f88e9 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:26:12 -0700 Subject: [PATCH 09/19] refactor(dynamo): serve only NeMo RL rollouts; general OpenAI path moves to #4979 After the NeMo-Gym split nothing in this PR calls the engine's general OpenAI path, so it moves to #4979 with Gym, its only client: - NativeVllmEngine serves requests carrying ROLLOUT_ANNOTATION and refuses the rest. build_sampling_params, guided decoding, media loading, prompt logprobs, queue-limit mapping and the parser/chat-template registration go, and engines register endpoint_types="completions". - dynamo_cfg.worker_args (parsers, chat template, structural tags, endpoint types) is removed and rejected at startup with the migration. - DYN_ENABLE_EXPERIMENTAL_PARSERS_V2 goes: only Dynamo's chat path reads it, and /v1/completions returns engine_data via nvext.extra_fields. - The engine keeps skip_tokenizer_init as native chooses it, so rollouts skip detokenization as they do natively. #4979 restores all of this by reverting this commit. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 28 +- docs/guides/dynamo-generation.md | 33 +- ...grpo_math_1B_dynamo_single_controller.yaml | 13 - nemo_rl/models/generation/__init__.py | 14 +- nemo_rl/models/generation/dynamo/__init__.py | 2 - nemo_rl/models/generation/dynamo/config.py | 134 +------ .../generation/dynamo/managed_runtime.py | 6 +- .../models/generation/dynamo/native_engine.py | 370 +----------------- nemo_rl/utils/outdated_config_checks.py | 10 +- .../models/generation/test_dynamo_config.py | 81 +--- .../generation/test_dynamo_native_engine.py | 308 +-------------- .../unit/utils/test_outdated_config_checks.py | 3 +- 12 files changed, 50 insertions(+), 952 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 9c07ccda5c7..2f1acd26d4e 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -20,23 +20,12 @@ which is the loop that owns the `AsyncLLM`. vLLM. The adapter, `nemo_rl/models/generation/dynamo/native_engine.py`, implements its `LLMEngine` over the worker's `AsyncLLM`: -- `generate` maps Dynamo's token request onto vLLM `SamplingParams` the way - vLLM's OpenAI server does, and streams token deltas back. Rollouts that NeMo RL - itself sends carry an annotation. For those, the engine uses the same code as - native `generate_async`: - - the worker's `_rollout_max_new_tokens` and `_rollout_sampling_params` build - the request; - - `request_pipeline.rollout_result` builds the result, routed experts - included. - - The result travels in `engine_data`, and the driver assembles the same - output with `request_pipeline.rollout_output`. - - A request with media carries the frontend's tokens, with one unexpanded - placeholder per item, and the media URLs. The adapter loads the media with - vLLM's media connector. It then builds the processed prompt with the engine's - renderer (`process_for_engine_async`), the same step that follows rendering - on the native endpoint. +- `generate` serves only the rollouts NeMo RL itself sends, which carry an + annotation, and refuses other requests. The worker's own `_serve_rollout` runs + each rollout exactly as native `generate_async` does, and + `request_pipeline.rollout_result` builds its result, routed experts included. + The result travels in `engine_data`, and the driver assembles the same output + with `request_pipeline.rollout_output`. - `abort` cancels the vLLM request on client disconnect. - Registration advertises the context length, the KV block size (from vLLM's KV-cache group metadata), the block count and the scheduler limits. It @@ -86,10 +75,9 @@ imports `dynamo.common.backend`, `dynamo.llm.exceptions` and `dynamo._core`. What a Dynamo upgrade has to re-check: - the `dynamo.common.backend` contract (`LLMEngine`, `Worker`, `WorkerConfig`, - publisher and logprob helpers); + publisher); - the request fields of Dynamo's preprocessed request; -- the frontend's `nvext` extensions and annotation forwarding; -- the parser names the native settings map to (`dynamo/config.py`). +- the frontend's `nvext` extensions and annotation forwarding. The one vLLM↔Dynamo wire coupling is vLLM's KV-event msgpack, which Dynamo's router decodes. A change there degrades routing, not serving. The managed diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index b780dcda242..b1a543a52bf 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -52,8 +52,6 @@ shows them all: |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | | `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies | -| `worker_args.tool_call_parser`, `reasoning_parser` | null | Dynamo's built-in parser names. null follows the native server's settings; see [Chat settings](#chat-settings) | -| `worker_args.custom_jinja_template` | null | A chat template that replaces the model's own. null follows the native server's | | `frontend_args.router_mode` | `kv` | Frontend routing policy | | `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | @@ -75,31 +73,7 @@ native path: token IDs, log probabilities, truncation, routed experts with their `r3_routed_experts_*` counts, and `gen_leader_worker_idx`. Both paths build it with `nemo_rl/models/generation/vllm/request_pipeline.py`. -OpenAI clients of the frontend get what vLLM's own OpenAI server applies: -- the model's generation-config sampling defaults; -- `n > 1`; -- guided decoding (`guided_*`, `response_format`); -- `bad_words_token_ids` and `allowed_token_ids`; -- prompt log probabilities (`nvext.extra_fields: ["prompt_logprobs"]`); -- vLLM's queue limits, rejected with HTTP 503; -- a 400 naming the "maximum context length" for a prompt that leaves no room - for output. The engine, not the frontend, checks the context, so it checks - the prompt it actually runs. - -## Chat settings - -The native server's chat settings carry over to the frontend: - -| Native setting (`vllm_cfg`) | Under Dynamo | -|---|---| -| `http_server_serving_chat_kwargs.tool_parser`, `reasoning_parser` | The frontend's parsers, under Dynamo's names: NeMo RL's `nano_v3` plugin is `nemotron_v3`, vLLM's `openai` tool parser is `harmony` | -| `tool_parser_plugin`, `reasoning_parser_plugin` | Python plugins cannot run in the Rust frontend. Startup asks for `dynamo_cfg.worker_args` to name a Dynamo parser, unless the plugin has a rename above | -| `http_server_serving_chat_kwargs.chat_template` | The frontend's template, as a file, a vLLM template name or literal Jinja | -| `http_server_serving_chat_kwargs.enable_auto_tools`, `chat_template_content_format` | The frontend always parses `tool_choice: auto` calls and has no content-format setting, so only the defaults (`true`, `auto`) are accepted | - -Where the behavior differs from the native endpoint: when a request sets -`chat_template_kwargs.force_nonempty_content`, Dynamo reports a reasoning-only -turn as content. +The engines serve only these rollouts; the frontend refuses other requests. ## Not supported @@ -114,8 +88,6 @@ Startup rejects these with `dynamo: true`: and a restarted shard does not register with it yet. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data parallelism inside vLLM, which the async engine Dynamo serves does not. -- Parser plugins with no Dynamo parser, and the chat settings above that the - frontend has no counterpart for. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. @@ -168,5 +140,6 @@ rejects it with the migration: - `control_timeout_s`: refit and pause use the vLLM backend's controls; - `metrics_include_prefixes` / `metrics_exclude_prefixes`: use `vllm_cfg.enable_vllm_metrics_logger`; - - `worker_args.extra_cli_args`: engines are configured through `vllm_cfg` and + - `worker_args`: the engines serve only NeMo RL's rollouts, which need no chat + parsers or templates; engines are configured through `vllm_cfg` and `vllm_kwargs`. diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 1b62d39ba8d..1ac1a92abaf 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -18,19 +18,6 @@ policy: # NRL_VLLM_ASYNC_TIMEOUT_SECONDS). A timeout fails the attempt as a native # timeout does, and the rollout retry policy applies. request_timeout_s: 900 - worker_args: - # Dynamo's built-in parser names. null follows the native server's - # vllm_cfg.http_server_serving_chat_kwargs (tool_parser, reasoning_parser). - tool_call_parser: null - reasoning_parser: null - exclude_tools_when_tool_choice_none: false - enable_structural_tag: false - structural_tag_scope: auto - structural_tag_schema: auto - # A chat template that replaces the model's own. null follows - # vllm_cfg.http_server_serving_chat_kwargs.chat_template. - custom_jinja_template: null - endpoint_types: [chat, completions] frontend_args: tokenizer: default tokenizer_cache: false diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index c8547bbce41..7d17e9e8942 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -17,12 +17,7 @@ from transformers import PreTrainedTokenizerBase -from nemo_rl.models.generation.dynamo.config import ( - check_dynamo_serving_chat_kwargs, - dynamo_cfg, - dynamo_enabled, - dynamo_parsers, -) +from nemo_rl.models.generation.dynamo.config import dynamo_cfg, dynamo_enabled from nemo_rl.models.generation.interfaces import GenerationConfig, GenerationInterface from nemo_rl.models.generation.trtllm.config import ( TrtllmConfig, @@ -186,11 +181,8 @@ def configure_generation_config( # vision parameters; freezing alone does not exclude them from refit. config["vllm_cfg"].setdefault("load_format", "auto" if is_eval else "dummy") if dynamo_enabled(config): - # Validate before any Ray allocation; dynamo_parsers also rejects - # parser plugins that Dynamo's Rust frontend cannot run. + # Validate before any Ray allocation. dynamo_cfg(config) - dynamo_parsers(config) - check_dynamo_serving_chat_kwargs(config) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( @@ -274,8 +266,6 @@ def configure_generation_config( is_eval or config["stop_strings"] is not None or config["vllm_cfg"].get("expose_http_server", None) - # Dynamo serves OpenAI clients as the exposed server does. - or dynamo_enabled(config) ): config["vllm_cfg"]["skip_tokenizer_init"] = False else: diff --git a/nemo_rl/models/generation/dynamo/__init__.py b/nemo_rl/models/generation/dynamo/__init__.py index 40376929afa..427c1ede100 100644 --- a/nemo_rl/models/generation/dynamo/__init__.py +++ b/nemo_rl/models/generation/dynamo/__init__.py @@ -14,7 +14,6 @@ from nemo_rl.models.generation.dynamo.config import ( DynamoCfg, DynamoFrontendArgs, - DynamoWorkerArgs, dynamo_cfg, dynamo_enabled, ) @@ -22,7 +21,6 @@ __all__ = [ "DynamoCfg", "DynamoFrontendArgs", - "DynamoWorkerArgs", "dynamo_cfg", "dynamo_enabled", ] diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index b6ae598b1a1..af592bc0fd5 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -21,33 +21,7 @@ from typing import Any, Literal, Mapping -from pydantic import BaseModel, Field, PositiveFloat, PositiveInt, model_validator - - -class DynamoWorkerArgs(BaseModel, extra="forbid"): - """How each vLLM engine registers with Dynamo (``WorkerConfig`` fields).""" - - # Dynamo's built-in parser names (dynamo-parsers). None follows the native - # server's vllm_cfg.http_server_serving_chat_kwargs (see dynamo_parsers). - tool_call_parser: str | None = None - reasoning_parser: str | None = None - # False, as vLLM's OpenAIServingChat (the native server) defaults it. - exclude_tools_when_tool_choice_none: bool = False - enable_structural_tag: bool = False - structural_tag_scope: Literal["auto", "always"] = "auto" - structural_tag_schema: Literal["auto", "strict"] = "auto" - # A Jinja chat template file that replaces the model's own. None follows - # vllm_cfg.http_server_serving_chat_kwargs.chat_template. - custom_jinja_template: str | None = None - endpoint_types: list[Literal["chat", "completions"]] = ["chat", "completions"] - - @model_validator(mode="after") - def _validate_endpoint_types(self) -> "DynamoWorkerArgs": - if not self.endpoint_types: - raise ValueError("endpoint_types must contain at least one endpoint") - if len(self.endpoint_types) != len(set(self.endpoint_types)): - raise ValueError("endpoint_types must not contain duplicates") - return self +from pydantic import BaseModel, Field, PositiveFloat, PositiveInt class DynamoFrontendArgs(BaseModel, extra="forbid"): @@ -80,115 +54,9 @@ class DynamoCfg(BaseModel, extra="forbid"): # does (GenerationUnavailable), and the rollout retry policy re-dispatches it # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). request_timeout_s: PositiveFloat = 900.0 - worker_args: DynamoWorkerArgs = Field(default_factory=DynamoWorkerArgs) frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) -# vLLM parser names whose Dynamo built-in parser goes by another name. nano_v3 is -# NeMo RL's vLLM plugin: DeepSeek-R1 reasoning parsing, with the whole output as -# content when chat_template_kwargs.enable_thinking is false, as Dynamo's -# nemotron_v3 parser does. -_DYNAMO_REASONING_PARSER_NAMES = {"nano_v3": "nemotron_v3"} -_DYNAMO_TOOL_PARSER_NAMES = {"openai": "harmony"} - - -def _dynamo_parser( - configured: str | None, - native: str | None, - plugin: str | None, - renames: Mapping[str, str], - *, - kind: str, - worker_arg: str, -) -> str | None: - if configured is not None or native is None: - return configured - if native in renames: - return renames[native] - if plugin: - raise ValueError( - f"policy.generation.vllm_cfg.{kind}_parser_plugin loads a Python parser " - "into vLLM's OpenAI server, which Dynamo's Rust frontend cannot run. Set " - f"policy.generation.dynamo_cfg.worker_args.{worker_arg} to the Dynamo " - f"built-in parser for {native!r}." - ) - return native - - -def dynamo_parsers( - generation_config: Mapping[str, Any], -) -> tuple[str | None, str | None]: - """The ``(tool_call_parser, reasoning_parser)`` Dynamo's frontend applies. - - ``dynamo_cfg.worker_args`` names them explicitly. Otherwise they follow the - native server's ``vllm_cfg.http_server_serving_chat_kwargs``, renamed where - Dynamo calls the same parser differently. A parser plugin without such a - rename has no Dynamo equivalent and is rejected. - """ - worker_args = dynamo_cfg(generation_config).worker_args - vllm_cfg = generation_config["vllm_cfg"] - serving = vllm_cfg.get("http_server_serving_chat_kwargs") or {} - tool_call_parser = _dynamo_parser( - worker_args.tool_call_parser, - serving.get("tool_parser"), - vllm_cfg.get("tool_parser_plugin"), - _DYNAMO_TOOL_PARSER_NAMES, - kind="tool", - worker_arg="tool_call_parser", - ) - reasoning_parser = _dynamo_parser( - worker_args.reasoning_parser, - serving.get("reasoning_parser"), - vllm_cfg.get("reasoning_parser_plugin"), - _DYNAMO_REASONING_PARSER_NAMES, - kind="reasoning", - worker_arg="reasoning_parser", - ) - return tool_call_parser, reasoning_parser - - -def check_dynamo_serving_chat_kwargs(generation_config: Mapping[str, Any]) -> None: - """Reject native chat-server settings Dynamo's frontend has no counterpart for. - - The frontend renders templates and parses tool calls itself: it always - parses ``tool_choice="auto"`` calls when a tool parser is set, and has no - content-format setting. The native defaults (``enable_auto_tools: true``, - ``chat_template_content_format: auto``) match it; other values would - silently change what the policy sees. - """ - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - content_format = serving.get("chat_template_content_format") - if content_format not in (None, "auto"): - raise ValueError( - "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." - f"chat_template_content_format={content_format!r} is not supported " - "with policy.generation.dynamo=true: Dynamo's frontend renders chat " - "templates itself and has no content-format setting. Remove it or " - "set it to 'auto'." - ) - if serving.get("enable_auto_tools") is False: - raise ValueError( - "policy.generation.vllm_cfg.http_server_serving_chat_kwargs." - "enable_auto_tools=false is not supported with " - "policy.generation.dynamo=true: Dynamo's frontend always parses " - "tool_choice='auto' calls when a tool parser is set. Remove it, or " - "remove the tool parser." - ) - - -def dynamo_chat_template(generation_config: Mapping[str, Any]) -> str | None: - """The chat template the frontend renders with: a file path or literal Jinja. - - ``dynamo_cfg.worker_args.custom_jinja_template``, else the native server's - ``http_server_serving_chat_kwargs.chat_template``; None uses the model's own. - """ - configured = dynamo_cfg(generation_config).worker_args.custom_jinja_template - if configured is not None: - return configured - serving = generation_config["vllm_cfg"].get("http_server_serving_chat_kwargs") or {} - return serving.get("chat_template") - - def dynamo_enabled(generation_config: Mapping[str, Any]) -> bool: """Whether ``policy.generation.dynamo`` is on for a vLLM generation config.""" return bool(generation_config.get("dynamo")) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index ad9ad9b7659..e53c0aef8cc 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -120,7 +120,7 @@ def engine_env(self) -> dict[str, str]: if not self._started: raise RuntimeError("Managed Dynamo runtime has not been started") # An OS-assigned system port makes each engine serve its own model-card - # files (a custom chat template included) to a frontend on another node. + # files to a frontend on another node. return {**self._manager_env, "DYN_SYSTEM_PORT": "0"} def start(self) -> None: @@ -166,10 +166,6 @@ def allocate_port(*, low: int, high: int) -> int: "NATS_SERVER": f"nats://{self._host}:{self._nats_port}", "DYN_NAMESPACE": self._namespace, "DYN_DISCOVERY_BACKEND": "etcd", - # Dynamo's legacy tool jail rebuilds response chunks with nvext=None. - # Its v2 parser path preserves engine_data, which carries every NeMo - # RL rollout's result. - "DYN_ENABLE_EXPERIMENTAL_PARSERS_V2": "1", "DYN_REQUEST_PLANE": "tcp", "DYN_EVENT_PLANE": "nats", # A canary request would stall behind a refit pause; engine liveness diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index f6c877b49c1..e95f535bc25 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -23,18 +23,14 @@ from __future__ import annotations import asyncio -import functools -import hashlib import json import logging import os -import tempfile -import time from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional import torch -from dynamo._core import Context, get_reasoning_parser_names, get_tool_parser_names +from dynamo._core import Context from dynamo.common.backend import ( EngineConfig, GenerateChunk, @@ -44,33 +40,13 @@ Worker, WorkerConfig, ) -from dynamo.common.backend.logprobs import ( - extract_from_completion_output, - extract_prompt_logprobs_from_completion_output, - parse_logprob_options, -) from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource -from dynamo.llm.exceptions import HttpError, InvalidArgument -from vllm import envs as vllm_envs -from vllm.entrypoints.chat_utils import load_chat_template -from vllm.exceptions import GracefulHTTPError, VLLMClientError +from dynamo.llm.exceptions import InvalidArgument from vllm.inputs import TokensPrompt -from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY -from vllm.sampling_params import ( - RequestOutputKind, - SamplingParams, - StructuredOutputsParams, -) -from vllm.transformers_utils.processor import get_video_processor_cls_name from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - dynamo_cfg, - dynamo_chat_template, - dynamo_parsers, -) +from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.dynamo.managed_runtime import ( DYNAMO_COMPONENT, DYNAMO_ENDPOINT, @@ -85,29 +61,6 @@ LOGGER = logging.getLogger(__name__) -# The vLLM modality each key of Dynamo's multi_modal_data holds. -_DYNAMO_MEDIA_MODALITIES = { - "image_url": "image", - "video_url": "video", - "audio_url": "audio", -} - -# Filled from the model's generation config when a request leaves them unset, -# as vLLM's OpenAI server fills them. -_GENERATION_CONFIG_KEYS = ( - "repetition_penalty", - "temperature", - "top_k", - "top_p", - "min_p", -) -_SAMPLING_KEYS = ( - *_GENERATION_CONFIG_KEYS, - "seed", - "presence_penalty", - "frequency_penalty", -) - def dynamo_kv_events_config( cfg: DynamoCfg, *, prefix_caching: bool @@ -145,111 +98,6 @@ def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: return None if rollout is None else json.loads(rollout) -def structured_outputs(guided_decoding: dict[str, Any]) -> StructuredOutputsParams: - """Structured outputs for Dynamo's ``sampling_options.guided_decoding``. - - Dynamo sets at most one constraint. The decoding backend is the engine's - own (``structured_outputs_config``), as for the native server. - """ - structural_tag = guided_decoding.get("structural_tag") - if isinstance(structural_tag, dict): - structural_tag = json.dumps(structural_tag) - return StructuredOutputsParams( - json=guided_decoding.get("json"), - regex=guided_decoding.get("regex"), - choice=guided_decoding.get("choice"), - grammar=guided_decoding.get("grammar"), - whitespace_pattern=guided_decoding.get("whitespace_pattern"), - structural_tag=structural_tag, - ) - - -def build_sampling_params( - request: GenerateRequest, - defaults: dict[str, Any], - *, - output_room: int, - overrides: Optional[dict[str, Any]] = None, - output_kind: RequestOutputKind = RequestOutputKind.DELTA, -) -> SamplingParams: - """Map an OpenAI-shaped Dynamo request onto vLLM SamplingParams. - - As vLLM's OpenAI server builds them: the model's generation-config - ``defaults`` for what the request leaves unset, and ``max_tokens`` capped - by the context. Stop strings are applied by Dynamo's Rust detokenizer, so - only token-level stops reach vLLM unless ``overrides`` sets them. - - Args: - request: Dynamo's preprocessed request. - defaults: ``ModelConfig.get_diff_sampling_param()``. - output_room: The context left after the prompt the engine runs, which - media expansion or a prefix splice can make longer than the - request's ``token_ids``. - overrides: SamplingParams fields that replace the request's. - output_kind: How vLLM reports outputs. - """ - sampling = request.get("sampling_options") or {} - passthrough = (request.get("extra_args") or {}).get("sampling_options") or {} - stop = request.get("stop_conditions") or {} - n = sampling.get("n") or 1 - if sampling.get("best_of") not in (None, n): - raise InvalidArgument("best_of is not supported by vLLM's V1 engine") - - kwargs: dict[str, Any] = { - key: defaults[key] for key in _GENERATION_CONFIG_KEYS if key in defaults - } - kwargs.update( - {key: sampling[key] for key in _SAMPLING_KEYS if sampling.get(key) is not None} - ) - kwargs["n"] = n - # As vLLM's get_max_tokens: the generation config's max_tokens applies only - # when the request leaves max_tokens unset. - requested_max_tokens = stop.get("max_tokens") - if requested_max_tokens is None: - requested_max_tokens = defaults.get("max_tokens") - kwargs["max_tokens"] = min( - limit - for limit in (max(1, output_room), requested_max_tokens) - if limit is not None - ) - if stop.get("min_tokens") is not None: - kwargs["min_tokens"] = stop["min_tokens"] - if stop.get("ignore_eos") is not None: - kwargs["ignore_eos"] = stop["ignore_eos"] - stop_ids = set(stop.get("stop_token_ids") or ()) | set( - stop.get("stop_token_ids_hidden") or () - ) - if stop_ids: - kwargs["stop_token_ids"] = sorted(stop_ids) - if sampling.get("guided_decoding"): - kwargs["structured_outputs"] = structured_outputs(sampling["guided_decoding"]) - for key in ("allowed_token_ids", "logprob_token_ids"): - if passthrough.get(key) is not None: - kwargs[key] = passthrough[key] - - logprobs, prompt_logprobs = parse_logprob_options( - request.get("output_options") or {} - ) - # Explicit logprob_token_ids replace top-k selection; vLLM rejects both. - if logprobs is not None and "logprob_token_ids" not in kwargs: - kwargs["logprobs"] = logprobs - if prompt_logprobs is not None: - kwargs["prompt_logprobs"] = prompt_logprobs - kwargs["detokenize"] = False - kwargs.update(overrides or {}) - params = SamplingParams(**kwargs, output_kind=output_kind) - if passthrough.get("bad_words_token_ids") is not None: - # vLLM only tokenizes bad_words strings into this field; a token-in - # client sends the token IDs themselves. - if not hasattr(params, "_bad_words_token_ids"): - raise RuntimeError( - "vLLM SamplingParams has no _bad_words_token_ids; update the " - "bad_words_token_ids passthrough for this vLLM" - ) - params._bad_words_token_ids = passthrough["bad_words_token_ids"] - return params - - class SnapshotStatLogger(StatLoggerBase): """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" @@ -292,7 +140,6 @@ def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int self._leader_worker_idx = leader_worker_idx self._llm = host.llm self._max_model_len = self._llm.vllm_config.model_config.max_model_len - self._defaults = self._llm.model_config.get_diff_sampling_param() @classmethod async def from_args(cls, argv=None): @@ -335,84 +182,14 @@ async def start(self, worker_id: int) -> EngineConfig: async def generate( self, request: GenerateRequest, context: Context ) -> AsyncGenerator[GenerateChunk, None]: - try: - async for chunk in self._generate(request, context): - yield chunk - except VLLMClientError as error: - # vLLM's request errors are not ValueErrors; the native server - # answers them with a 4xx that carries the message. - raise InvalidArgument(str(error)) from error - - async def _generate( - self, request: GenerateRequest, context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - if request.get("prompt_embeds"): + rollout = _rollout_request(request) + if rollout is None: raise InvalidArgument( - "Embedding prompts are not served through Dynamo by NeMo RL's " - "vLLM engines." + "NeMo RL's vLLM engines serve only NeMo RL rollouts through Dynamo " + f"(requests annotated {ROLLOUT_ANNOTATION!r})." ) - rollout = _rollout_request(request) - if rollout is not None: - async for chunk in self._generate_rollout(request, rollout, context): - yield chunk - return - engine_input = await self._engine_input(request) - prompt_len = len(engine_input["prompt_token_ids"]) - self._check_prompt_fits(prompt_len) - sampling_params = build_sampling_params( - request, self._defaults, output_room=self._max_model_len - prompt_len - ) - try: - # vLLM's queue limits (max_num_queued_reqs / max_num_queued_tokens), - # checked before every request as the native server checks them. - self._llm.check_admission(sampling_params.n, context.id()) - except GracefulHTTPError as error: - raise HttpError(int(error.http_status), error.message) from error - logprobs_requested, prompt_logprobs_requested = parse_logprob_options( - request.get("output_options") or {} - ) - tokenizer = self._llm.tokenizer - completion_tokens: dict[int, int] = {} - prompt_output = None - async for output in self._llm.generate( - engine_input, sampling_params, context.id() - ): - # Under DELTA, vLLM sends every prompt logprob once, at the end of - # prefill; later outputs carry an empty list. - if getattr(output, "prompt_logprobs", None): - prompt_output = output - for completion in output.outputs: - token_ids = list(completion.token_ids) - completion_tokens[completion.index] = completion_tokens.get( - completion.index, 0 - ) + len(token_ids) - if not token_ids and completion.finish_reason is None: - continue - chunk: GenerateChunk = { - "token_ids": token_ids, - "index": completion.index, - } - log_probs, top_logprobs = extract_from_completion_output( - completion, 0, tokenizer=tokenizer, include_bytes=True - ) - if log_probs is not None: - chunk["log_probs"] = log_probs - if logprobs_requested: - chunk["top_logprobs"] = top_logprobs - if completion.finish_reason is not None: - chunk["finish_reason"] = completion.finish_reason - chunk["completion_usage"] = _usage( - output, sum(completion_tokens.values()) - ) - if prompt_logprobs_requested is not None and prompt_output: - prompt_logprobs = ( - extract_prompt_logprobs_from_completion_output( - prompt_output, tokenizer=tokenizer - ) - ) - if prompt_logprobs is not None: - chunk["engine_data"] = {"prompt_logprobs": prompt_logprobs} - yield chunk + async for chunk in self._generate_rollout(request, rollout, context): + yield chunk async def _generate_rollout( self, request: GenerateRequest, rollout: dict[str, Any], context: Context @@ -448,86 +225,6 @@ async def _generate_rollout( chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) yield chunk - async def _load_media( - self, request: GenerateRequest - ) -> Optional[tuple[dict[str, list[Any]], dict[str, list[Any]]]]: - """The request's media and their UUIDs by modality, as vLLM's chat parser loads them. - - Dynamo passes each media URL through (data: URIs included); the engine - fetches and decodes it with vLLM's own media connector and settings. - """ - media = request.get("multi_modal_data") - if not media: - return None - model_config = self._llm.model_config - mm_config = model_config.multimodal_config - connector = MEDIA_CONNECTOR_REGISTRY.load( - vllm_envs.VLLM_MEDIA_CONNECTOR, - media_io_kwargs=request.get("media_io_kwargs") - or (mm_config.media_io_kwargs if mm_config else None), - allowed_local_media_path=model_config.allowed_local_media_path, - allowed_media_domains=model_config.allowed_media_domains, - ) - fetch = { - "image": connector.fetch_image_async, - "audio": connector.fetch_audio_async, - } - if "video_url" in media: - fetch["video"] = functools.partial( - connector.fetch_video_async, - video_processor=get_video_processor_cls_name(model_config), - ) - uuids = request.get("multi_modal_uuids") or {} - mm_data: dict[str, list[Any]] = {} - mm_uuids: dict[str, list[Any]] = {} - for key, items in media.items(): - modality = _DYNAMO_MEDIA_MODALITIES.get(key) - if modality is None: - raise InvalidArgument(f"Unsupported media type {key!r}") - if any("Url" not in item for item in items): - raise InvalidArgument( - f"{key} items must carry their URL: NeMo RL's vLLM engines load " - "media themselves" - ) - try: - mm_data[modality] = list( - await asyncio.gather( - *(fetch[modality](item["Url"]) for item in items) - ) - ) - except ValueError as error: - raise InvalidArgument(f"Cannot load {key}: {error}") from error - mm_uuids[modality] = list(uuids.get(key) or [None] * len(items)) - return mm_data, mm_uuids - - async def _engine_input(self, request: GenerateRequest) -> dict[str, Any]: - """The processed vLLM prompt for the request, as the native chat endpoint builds it. - - The frontend's tokens carry one unexpanded placeholder per media item; - vLLM's processor expands them and processes the media, exactly as after - the native endpoint renders and tokenizes the messages. - """ - prompt = TokensPrompt(prompt_token_ids=list(request["token_ids"])) - media = await self._load_media(request) - if media is not None: - prompt["multi_modal_data"], prompt["multi_modal_uuids"] = media - if request.get("mm_processor_kwargs"): - prompt["mm_processor_kwargs"] = request["mm_processor_kwargs"] - return await self._llm.renderer.process_for_engine_async(prompt, time.time()) - - def _check_prompt_fits(self, prompt_len: int) -> None: - """Refuse a prompt that leaves no room for output, as the native endpoint does. - - OpenAI clients read "context length" in a 400 as a length refusal rather - than an engine failure. - """ - if prompt_len >= self._max_model_len: - raise InvalidArgument( - f"Prompt length ({prompt_len}) fills or exceeds this model's " - f"maximum context length ({self._max_model_len}). No room for " - "output tokens." - ) - async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) @@ -558,37 +255,6 @@ def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: } -def _check_parser_name(name: Optional[str], known: list[str], field: str) -> None: - if name is not None and name not in known: - raise ValueError( - f"Dynamo has no {field.replace('_', ' ')} named {name!r}; set " - f"policy.generation.dynamo_cfg.worker_args.{field} to one of " - f"{sorted(known)}." - ) - - -def _chat_template_file(chat_template: Optional[str]) -> Optional[str]: - """A file holding ``chat_template``, which Dynamo reads by path. - - Resolved as the native server resolves it (vLLM's ``load_chat_template``): - a file, a vLLM built-in template name, or literal Jinja. Named by content, - so every engine on a node, and every restart, shares one file. - """ - if chat_template is None: - return None - content = load_chat_template(chat_template) - digest = hashlib.sha256(content.encode()).hexdigest()[:16] - path = os.path.join(tempfile.gettempdir(), f"nemo_rl_chat_template_{digest}.jinja") - if not os.path.exists(path): - # Written aside and renamed, so a concurrent reader never sees a part. - with tempfile.NamedTemporaryFile( - "w", dir=os.path.dirname(path), suffix=".jinja", delete=False - ) as template_file: - template_file.write(content) - os.replace(template_file.name, path) - return path - - def start_dynamo_endpoint( host: "VllmAsyncGenerationWorkerImpl", *, @@ -608,31 +274,17 @@ def start_dynamo_endpoint( publishes = kv_events is not None and kv_events.enable_kv_cache_events # The Dynamo functional overlays check the router applied these. print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) - worker_args = dynamo_cfg(host.cfg).worker_args - tool_call_parser, reasoning_parser = dynamo_parsers(host.cfg) - _check_parser_name(tool_call_parser, get_tool_parser_names(), "tool_call_parser") - _check_parser_name( - reasoning_parser, get_reasoning_parser_names(), "reasoning_parser" - ) config = WorkerConfig( namespace=namespace, component=DYNAMO_COMPONENT, endpoint=DYNAMO_ENDPOINT, model_name=host.model_name, served_model_name=host.model_name, - endpoint_types=",".join(worker_args.endpoint_types), + # NeMo RL's rollouts go to /v1/completions only. + endpoint_types="completions", discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], request_plane=engine_env["DYN_REQUEST_PLANE"], event_plane=engine_env["DYN_EVENT_PLANE"], - custom_jinja_template=_chat_template_file(dynamo_chat_template(host.cfg)), - tool_call_parser=tool_call_parser, - reasoning_parser=reasoning_parser, - exclude_tools_when_tool_choice_none=( - worker_args.exclude_tools_when_tool_choice_none - ), - structural_tag_mode="on" if worker_args.enable_structural_tag else "off", - structural_tag_scope=worker_args.structural_tag_scope, - structural_tag_schema=worker_args.structural_tag_schema, ) return asyncio.get_running_loop().create_task( Worker(NativeVllmEngine(host, leader_worker_idx), config).run() diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 5e511555b3e..8591d1d1a17 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -208,6 +208,10 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: "control_timeout_s": "refit and pause use the vLLM backend's own controls", "metrics_include_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", "metrics_exclude_prefixes": "set vllm_cfg.enable_vllm_metrics_logger instead", + "worker_args": ( + "Dynamo serves only NeMo RL's own rollouts, which need no chat parsers or " + "templates; engines are configured through vllm_cfg and vllm_kwargs" + ), } @@ -242,12 +246,6 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: problems.append( f"policy.generation.dynamo_cfg.{key} was removed: {replacement}." ) - worker_args = dynamo_cfg.get("worker_args") - if isinstance(worker_args, dict) and "extra_cli_args" in worker_args: - problems.append( - "policy.generation.dynamo_cfg.worker_args.extra_cli_args was " - "removed: engines are configured through vllm_cfg and vllm_kwargs." - ) if problems: raise ValueError(" ".join(problems)) diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 289f8b55bf7..a14f09e95f9 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -20,12 +20,7 @@ redact_argv, redact_environment, ) -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - dynamo_cfg, - dynamo_chat_template, - dynamo_parsers, -) +from nemo_rl.models.generation.dynamo.config import DynamoCfg, dynamo_cfg class _Tokenizer: @@ -53,10 +48,10 @@ def _vllm_config(**overrides) -> dict: def test_dynamo_true_alone_is_a_complete_config() -> None: config = configure_generation_config(_vllm_config(dynamo=True), _Tokenizer()) - # Dynamo serves the AsyncLLM, so the switch selects the async engine, and it - # serves OpenAI clients, which need the engine's tokenizer. + # Dynamo serves the AsyncLLM, so the switch selects the async engine. + # Rollouts read token IDs only, so the engine skips its tokenizer as natively. assert config["vllm_cfg"]["async_engine"] is True - assert config["vllm_cfg"]["skip_tokenizer_init"] is False + assert config["vllm_cfg"]["skip_tokenizer_init"] is True assert dynamo_cfg(config) == DynamoCfg() assert dynamo_cfg(config).frontend_args.router_mode == "kv" @@ -103,10 +98,8 @@ def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( def test_dynamo_cfg_rejects_unknown_keys() -> None: - with pytest.raises(ValidationError, match="extra_cli_args"): - dynamo_cfg({"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}) - with pytest.raises(ValidationError, match="duplicates"): - dynamo_cfg({"dynamo_cfg": {"worker_args": {"endpoint_types": ["chat"] * 2}}}) + with pytest.raises(ValidationError, match="worker_args"): + dynamo_cfg({"dynamo_cfg": {"worker_args": {}}}) def test_frontend_argv_is_runtime_owned() -> None: @@ -134,65 +127,3 @@ def test_redaction_hides_credentials() -> None: "HF_TOKEN": "", "NCCL_DEBUG": "INFO", } - - -def _serving(**serving_chat_kwargs) -> dict: - return _vllm_config( - dynamo=True, - vllm_cfg={ - "async_engine": True, - "max_model_len": 512, - "tensor_parallel_size": 1, - "expert_parallel_size": 1, - "http_server_serving_chat_kwargs": serving_chat_kwargs, - }, - ) - - -def test_parsers_follow_the_native_server_under_dynamos_names() -> None: - assert dynamo_parsers(_serving(tool_parser="hermes", reasoning_parser="qwen3")) == ( - "hermes", - "qwen3", - ) - # NeMo RL's nano_v3 vLLM plugin is Dynamo's nemotron_v3 parser. - nano_v3 = _serving(reasoning_parser="nano_v3") - nano_v3["vllm_cfg"]["reasoning_parser_plugin"] = "nano_v3_reasoning_parser.py" - assert dynamo_parsers(nano_v3) == (None, "nemotron_v3") - assert dynamo_parsers(_serving(tool_parser="openai")) == ("harmony", None) - # dynamo_cfg names them explicitly. - explicit = _serving(tool_parser="hermes") - explicit["dynamo_cfg"] = {"worker_args": {"tool_call_parser": "qwen3_coder"}} - assert dynamo_parsers(explicit) == ("qwen3_coder", None) - - -def test_a_parser_plugin_without_a_dynamo_parser_is_rejected_at_startup() -> None: - config = _serving(tool_parser="my_parser") - config["vllm_cfg"]["tool_parser_plugin"] = "my_parser.py" - - with pytest.raises(ValueError, match="worker_args.tool_call_parser"): - configure_generation_config(config, _Tokenizer()) - - -def test_chat_template_follows_the_native_server() -> None: - config = _serving(chat_template="templates/chat.jinja") - assert dynamo_chat_template(config) == "templates/chat.jinja" - assert dynamo_chat_template(_serving()) is None - - -@pytest.mark.parametrize( - ("serving_chat_kwargs", "error"), - [ - ({"chat_template_content_format": "string"}, "no content-format setting"), - ({"enable_auto_tools": False}, "always parses"), - ], -) -def test_chat_settings_the_frontend_has_no_counterpart_for_are_rejected( - serving_chat_kwargs, error -) -> None: - with pytest.raises(ValueError, match=error): - configure_generation_config(_serving(**serving_chat_kwargs), _Tokenizer()) - # The native defaults are what the frontend does anyway. - configure_generation_config( - _serving(chat_template_content_format="auto", enable_auto_tools=True), - _Tokenizer(), - ) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index eca97ef67a7..ef2fbd01d83 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -18,27 +18,20 @@ """ import asyncio -import base64 -import io import json -from http import HTTPStatus from types import SimpleNamespace import numpy as np import pytest import torch -from PIL import Image pytestmark = pytest.mark.vllm pytest.importorskip("dynamo._core") vllm = pytest.importorskip("vllm") -from dynamo.llm.exceptions import HttpError, InvalidArgument # noqa: E402 -from vllm.exceptions import QueueOverflowError # noqa: E402 -from vllm.multimodal.inputs import PlaceholderRange # noqa: E402 +from dynamo.llm.exceptions import InvalidArgument # noqa: E402 from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 -from nemo_rl.models.generation.dynamo.config import DynamoWorkerArgs # noqa: E402 from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) @@ -56,8 +49,6 @@ EOS = 2 -# The fake processor expands each image placeholder to IMAGE_TOKENS image tokens. -IMAGE_PLACEHOLDER, IMAGE_TOKEN, IMAGE_TOKENS = 90, 91, 3 class _Context: @@ -99,57 +90,18 @@ def _output(prompt, *completions, prompt_routed_experts=None, prompt_logprobs=No ) -class _FakeRenderer: - """vLLM's processing step: expands each image placeholder, as a VLM processor does.""" - - def __init__(self) -> None: - self.prompts = [] - - async def process_for_engine_async( - self, prompt, arrival_time, *, skip_mm_cache=False - ): - self.prompts.append((prompt, skip_mm_cache)) - if "multi_modal_data" not in prompt: - return {"type": "token", "prompt_token_ids": prompt["prompt_token_ids"]} - tokens, placeholders = [], [] - for token in prompt["prompt_token_ids"]: - if token == IMAGE_PLACEHOLDER: - placeholders.append(PlaceholderRange(len(tokens), IMAGE_TOKENS)) - tokens.extend([IMAGE_TOKEN] * IMAGE_TOKENS) - else: - tokens.append(token) - return { - "type": "multimodal", - "prompt_token_ids": tokens, - "mm_kwargs": {"image": [object() for _ in placeholders]}, - "mm_placeholders": {"image": placeholders}, - } - - class _FakeLLM: """Yields scripted RequestOutputs and records what it was asked to generate.""" - def __init__(self, outputs=(), *, defaults=None, admission_error=None): + def __init__(self, outputs=()): self.outputs = list(outputs) self.calls = [] - self.admission_error = admission_error self.tokenizer = None - self.renderer = _FakeRenderer() - self.model_config = SimpleNamespace( - get_diff_sampling_param=lambda: dict(defaults or {}), - multimodal_config=None, - allowed_local_media_path="", - allowed_media_domains=None, - ) self.vllm_config = SimpleNamespace( model_config=SimpleNamespace(max_model_len=MAX_MODEL_LEN), kv_events_config=None, ) - def check_admission(self, n, request_id): - if self.admission_error is not None: - raise self.admission_error - async def generate(self, prompt, sampling_params, request_id): self.calls.append((prompt, sampling_params, request_id)) for output in self.outputs: @@ -190,10 +142,6 @@ def _rollout_request(prompt: list[int], **rollout) -> dict: return {"token_ids": prompt, "annotations": [ROLLOUT_ANNOTATION + json.dumps(spec)]} -def _openai_request(**fields) -> dict: - return {"token_ids": [1, 2, 3, 4], **fields} - - def test_rollouts_are_served_like_generate_async() -> None: final = _output([1, 2, 3], _completion([10, 11, 12], "stop")) llm = _FakeLLM([final]) @@ -270,119 +218,6 @@ def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: assert rollout["token_ids"] == [] and rollout["truncated"] is False -def test_openai_requests_map_onto_sampling_params_as_vllms_server_does() -> None: - params = native_engine.build_sampling_params( - _openai_request( - sampling_options={"top_k": 5, "seed": 7, "n": 2}, - stop_conditions={ - "max_tokens": 100, - "stop_token_ids": [9], - "stop_token_ids_hidden": [2], - }, - output_options={"logprobs": 2}, - extra_args={ - "sampling_options": { - "allowed_token_ids": [5, 6], - "bad_words_token_ids": [[7]], - } - }, - ), - # The model's generation config, for what the request leaves unset. - {"temperature": 0.6, "top_k": 20, "max_tokens": 9}, - output_room=MAX_MODEL_LEN - 4, - ) - - assert (params.temperature, params.top_k, params.seed, params.n) == (0.6, 5, 7, 2) - # The request's max_tokens, capped by the context; the generation config's - # max_tokens applies only when the request leaves it unset. - assert params.max_tokens == MAX_MODEL_LEN - 4 - unset = native_engine.build_sampling_params( - _openai_request(), {"max_tokens": 9}, output_room=MAX_MODEL_LEN - 4 - ) - assert unset.max_tokens == 9 - assert sorted(params.stop_token_ids) == [2, 9] - assert params.logprobs == 2 - assert params.allowed_token_ids == [5, 6] - assert params._bad_words_token_ids == [[7]] - assert params.detokenize is False - - -def test_guided_decoding_maps_onto_structured_outputs() -> None: - params = native_engine.build_sampling_params( - _openai_request( - sampling_options={"guided_decoding": {"json": {"type": "object"}}} - ), - {}, - output_room=MAX_MODEL_LEN - 4, - ) - - assert params.structured_outputs.json == {"type": "object"} - - -def test_best_of_is_rejected_as_by_vllm() -> None: - with pytest.raises(InvalidArgument, match="best_of"): - native_engine.build_sampling_params( - _openai_request(sampling_options={"n": 1, "best_of": 2}), - {}, - output_room=MAX_MODEL_LEN - 4, - ) - - -def test_openai_requests_stream_each_choice_by_index() -> None: - prompt = [1, 2, 3, 4] - llm = _FakeLLM( - [ - _output(prompt, _completion([5], index=0), _completion([7], index=1)), - _output(prompt, _completion([6], "stop", index=0)), - _output(prompt, _completion([8], "length", index=1)), - ] - ) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - chunks = _collect(engine, _openai_request(sampling_options={"n": 2})) - - assert [(c["index"], c["token_ids"]) for c in chunks] == [ - (0, [5]), - (1, [7]), - (0, [6]), - (1, [8]), - ] - assert [c.get("finish_reason") for c in chunks] == [None, None, "stop", "length"] - assert chunks[-1]["completion_usage"]["completion_tokens"] == 4 - - -def test_prompt_logprobs_are_returned_in_engine_data() -> None: - prompt = [1, 2, 3, 4] - llm = _FakeLLM( - [ - # Under DELTA, vLLM sends prompt logprobs once, at the end of prefill, - # and an empty list on every later output. - _output( - prompt, - _completion([5]), - prompt_logprobs=[None, {2: _logprob(-0.25)}, None, None], - ), - _output(prompt, _completion([6], "stop"), prompt_logprobs=[]), - ] - ) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - chunks = _collect(engine, _openai_request(output_options={"prompt_logprobs": 0})) - - assert chunks[-1]["engine_data"]["prompt_logprobs"][1]["2"]["logprob"] == -0.25 - - -def test_queue_limits_reject_with_the_engines_status() -> None: - llm = _FakeLLM(admission_error=QueueOverflowError()) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(HttpError) as raised: - _collect(engine, _openai_request()) - - assert raised.value.code == HTTPStatus.SERVICE_UNAVAILABLE - assert llm.calls == [] - - def _start_worker_config(monkeypatch, host) -> SimpleNamespace: """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" configs = [] @@ -415,69 +250,22 @@ async def start(): return config -def test_every_worker_arg_reaches_the_dynamo_worker_config( - monkeypatch, tmp_path -) -> None: - template = tmp_path / "chat.jinja" - template.write_text("{{ messages }}") - worker_args = { - "tool_call_parser": "qwen3_coder", - "reasoning_parser": "nemotron_nano", - "exclude_tools_when_tool_choice_none": True, - "enable_structural_tag": True, - "structural_tag_scope": "always", - "structural_tag_schema": "strict", - "custom_jinja_template": str(template), - "endpoint_types": ["chat"], - } - assert set(worker_args) == set(DynamoWorkerArgs.model_fields) - host = _host(_FakeLLM(), dynamo_cfg={"worker_args": worker_args}) - - config = _start_worker_config(monkeypatch, host) +def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: + config = _start_worker_config(monkeypatch, _host(_FakeLLM())) assert config.namespace == "ns" - assert config.endpoint_types == "chat" - assert config.structural_tag_mode == "on" - with open(config.custom_jinja_template) as template_file: - assert template_file.read() == "{{ messages }}" - for key in worker_args.keys() - { - "endpoint_types", - "enable_structural_tag", - "custom_jinja_template", - }: - assert getattr(config, key) == worker_args[key], key - - -def test_the_native_servers_chat_settings_reach_the_frontend(monkeypatch) -> None: - host = _host(_FakeLLM()) - host.cfg["vllm_cfg"] = { - "max_model_len": MAX_MODEL_LEN, - "reasoning_parser_plugin": "nemo_rl/models/generation/vllm/reasoning_parsers/nano_v3_reasoning_parser.py", - "http_server_serving_chat_kwargs": { - "tool_parser": "qwen3_coder", - "reasoning_parser": "nano_v3", - # Literal Jinja, which the native server also accepts. - "chat_template": "{{ messages }}\n", - }, - } - - config = _start_worker_config(monkeypatch, host) + # NeMo RL's rollouts go to /v1/completions; no chat surface is registered. + assert config.endpoint_types == "completions" - assert (config.tool_call_parser, config.reasoning_parser) == ( - "qwen3_coder", - "nemotron_v3", - ) - with open(config.custom_jinja_template) as template_file: - assert template_file.read() == "{{ messages }}\n" +def test_requests_other_than_rollouts_are_refused() -> None: + llm = _FakeLLM() + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) -def test_a_parser_dynamo_lacks_fails_registration(monkeypatch) -> None: - host = _host( - _FakeLLM(), dynamo_cfg={"worker_args": {"tool_call_parser": "no_such_parser"}} - ) + with pytest.raises(InvalidArgument, match="only NeMo RL rollouts"): + _collect(engine, {"token_ids": [1, 2, 3, 4]}) - with pytest.raises(ValueError, match="no tool call parser named 'no_such_parser'"): - _start_worker_config(monkeypatch, host) + assert llm.calls == [] def test_an_endpoint_that_stops_serving_reports_why() -> None: @@ -528,78 +316,6 @@ async def call_utility_async(method): assert registration.context_length == MAX_MODEL_LEN -def test_a_prompt_that_fills_the_context_is_refused_as_natively() -> None: - llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match="maximum context length"): - _collect(engine, _openai_request(token_ids=list(range(MAX_MODEL_LEN)))) - - assert llm.calls == [] - - -def _image_url() -> str: - buffer = io.BytesIO() - Image.new("RGB", (8, 8), color="red").save(buffer, format="PNG") - return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() - - -def test_media_is_loaded_and_processed_as_vllms_chat_parser_does() -> None: - final = _output([1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3], _completion([5], "stop")) - llm = _FakeLLM([final]) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - _collect( - engine, - _openai_request( - token_ids=[1, IMAGE_PLACEHOLDER, 3], - multi_modal_data={"image_url": [{"Url": _image_url()}]}, - mm_processor_kwargs={"max_num_tiles": 2}, - ), - ) - - ((prompt, skip_mm_cache),) = llm.renderer.prompts - # The frontend's unexpanded tokens, the decoded image and the processor kwargs. - assert prompt["prompt_token_ids"] == [1, IMAGE_PLACEHOLDER, 3] - (image,) = prompt["multi_modal_data"]["image"] - assert image.size == (8, 8) and image.mode == "RGB" - assert prompt["multi_modal_uuids"] == {"image": [None]} - assert prompt["mm_processor_kwargs"] == {"max_num_tiles": 2} - assert skip_mm_cache is False - # vLLM generates from the processed prompt, and the output cap counts its tokens. - ((engine_input, params, _),) = llm.calls - assert engine_input["prompt_token_ids"] == [1, *[IMAGE_TOKEN] * IMAGE_TOKENS, 3] - assert params.max_tokens == MAX_MODEL_LEN - 2 - IMAGE_TOKENS - - -@pytest.mark.parametrize( - ("fields", "error"), - [ - ({"prompt_embeds": "AAAA"}, "Embedding prompts"), - ({"multi_modal_data": {"image_url": [{"UuidOnly": "u"}]}}, "carry their URL"), - ( - { - "multi_modal_data": { - "image_url": [{"Url": "data:image/png;base64,eA=="}] - } - }, - "Cannot load image_url", - ), - # vLLM's own request errors (not ValueErrors) keep their message, as the - # native server's 4xx does, instead of becoming a bare 500. - ({"stop_conditions": {"min_tokens": 10, "max_tokens": 5}}, "min_tokens"), - ], -) -def test_prompts_the_engine_cannot_serve_are_refused(fields, error) -> None: - llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) - - with pytest.raises(InvalidArgument, match=error): - _collect(engine, _openai_request(**fields)) - - assert llm.calls == [] - - @pytest.mark.parametrize( ("router_mode", "prefix_caching", "publishes"), [("kv", True, True), ("kv", False, False), ("round-robin", True, False)], diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 1e040d3c177..2119afcebbb 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -405,7 +405,8 @@ def test_dynamo_backend_is_rejected_with_the_migration(): ({"control_timeout_s": 600}, r"dynamo_cfg\.control_timeout_s"), ({"metrics_include_prefixes": None}, r"dynamo_cfg\.metrics_include_prefixes"), ({"metrics_exclude_prefixes": None}, r"dynamo_cfg\.metrics_exclude_prefixes"), - ({"worker_args": {"extra_cli_args": []}}, r"worker_args\.extra_cli_args"), + ({"worker_args": {"extra_cli_args": []}}, r"dynamo_cfg\.worker_args was"), + ({"worker_args": {"reasoning_parser": None}}, r"dynamo_cfg\.worker_args was"), ], ) def test_removed_dynamo_cfg_keys_are_rejected(dynamo_cfg, removed): From 83cab88b5fdb4ba86b2c865aab5b017f7fdc77d6 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:32:35 -0700 Subject: [PATCH 10/19] fix(dynamo): address round-2 review (C26, C28-C31, C33, C34, C36, C37, C39-C45) - C26: dynamo_cfg.request_timeout_s defaults to None. It bounded the whole generation, which native never does, so long rollouts were dropped and re-sampled; on the SingleController native.generation_timeout_s bounds both paths. - C28: router_mode no longer accepts "direct", which needs a worker ID that rollouts never carry. - C29: the EP > TP rejection gives the real reason: vLLM refuses the per-engine ipc:// KV-event endpoint above DP rank 0. - C30: http_server_api_key_required is rejected again: the frontend serves the same engines on 0.0.0.0 with no bearer check. - C31: kv_event_sources connects a wildcard endpoint to 127.0.0.1. - C33: one driver test checks the request sent on success, the assembled output, and the 503 -> GenerationUnavailable mapping. - C34: check_dynamo_overlay.sh requires published KV events, and each failure prints its reason. - C36: frontend_args.tokenizer* are removed (rollouts send token IDs, so the frontend never tokenizes) and rejected at startup. - C37/C39: docs say non-colocated and non-Gym recipes, and state the adapter's own vLLM calls. - C40-C45: real WorkerConfig in the registration test, ai-dynamo pytest plugins disabled, http_status_is_infra reused, num_gpu_blocks read from vllm_config, DYN_SDK_DISABLE_ANSI_LOGGING set in the actor env, Gym-only test fakes removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 19 +++--- docs/guides/dynamo-generation.md | 23 ++++--- docs/index.md | 2 +- ...grpo_math_1B_dynamo_single_controller.yaml | 11 ++- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 3 - nemo_rl/models/generation/__init__.py | 22 +++--- nemo_rl/models/generation/dynamo/config.py | 18 ++--- .../models/generation/dynamo/http_client.py | 2 +- .../generation/dynamo/managed_runtime.py | 20 +----- .../models/generation/dynamo/native_engine.py | 15 +++-- .../generation/dynamo/rollout_client.py | 11 ++- .../models/generation/vllm/vllm_generation.py | 4 ++ nemo_rl/utils/outdated_config_checks.py | 14 ++++ pyproject.toml | 4 +- tests/functional/check_dynamo_overlay.sh | 41 ++++++------ .../models/generation/test_dynamo_config.py | 23 ++++++- .../generation/test_dynamo_http_client.py | 3 +- .../generation/test_dynamo_managed_runtime.py | 19 ++---- .../generation/test_dynamo_native_engine.py | 19 +----- .../generation/test_dynamo_rollout_client.py | 26 ------- .../models/generation/test_vllm_generation.py | 67 ++++++++++++++----- .../unit/utils/test_outdated_config_checks.py | 4 ++ 22 files changed, 190 insertions(+), 180 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 2f1acd26d4e..4b7e95277e7 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -6,8 +6,13 @@ Dynamo's frontend and KV-aware router. Two requirements shape the design: - A vLLM bump must never be blocked by Dynamo. - Dynamo CI coverage must stay consistent with native vLLM's. -Both hold only if the engine process contains no Dynamo-owned vLLM code. Then a -vLLM bump can break Dynamo serving only where it also breaks native vLLM. +Both hold only if the engine process contains no Dynamo-owned vLLM code. A vLLM +bump can then break Dynamo serving only in NeMo RL's own adapter +(`native_engine.py`), which calls a few vLLM APIs native NeMo RL does not: the +`get_kv_cache_group_metadata` engine utility and `KVCacheSpecKind` at +registration, and `StatLoggerBase` for the router's load signal. A renamed import +fails the vLLM-lane unit tests at collection, and the Dynamo functional overlays +run registration. ## Engine hosting @@ -85,12 +90,8 @@ runtime logs the router's applied-event counts at shutdown, and the Dynamo functional overlays fail if the engines published events but none were applied (`tests/functional/check_dynamo_overlay.sh`). -`ai-dynamo-runtime` caps pydantic, which pins the vLLM venvs' pydantic version. -Dependency bounds like this one can still block a vLLM bump, and the import -guard does not see them: `ai-dynamo-runtime` 1.5.1 requires -`pydantic>=2.10.6,<=2.13`, and `ai-dynamo` caps `aiohttp<4`, -`prometheus-client<1` and `zstandard<1`. A vLLM release that needs a version -outside one of these bounds does not lock with the pinned `ai-dynamo`. Until a -Dynamo release lifts the bound, override it under +ai-dynamo's own dependency bounds (`ai-dynamo-runtime` caps pydantic, for +example) can still block a vLLM bump, and the import guard does not see them. +Until a Dynamo release lifts a bound, override it under `[tool.uv] override-dependencies` (as `aisimulate` and `kubernetes` are), and let the vLLM-lane Dynamo unit tests catch a real break. diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index b1a543a52bf..7acefc88d9b 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -1,6 +1,6 @@ # Dynamo serving for vLLM generation -Any vLLM generation config can serve its rollouts through +Any non-colocated vLLM generation config can serve its rollouts through [Dynamo](https://github.com/ai-dynamo/dynamo)'s OpenAI frontend and KV-aware router. Add one line: @@ -27,9 +27,8 @@ independent. With `dynamo: true`: - `vllm_cfg.async_engine` is turned on, because Dynamo drives the engine's - `AsyncLLM`. Settings the async engine does not run, such as data parallelism - inside vLLM (`expert_parallel_size > tensor_parallel_size`), are rejected at - startup. + `AsyncLLM`. Settings Dynamo does not serve are rejected at startup; see + [Not supported](#not-supported). - The driver starts etcd, NATS and `dynamo.frontend`. The frontend runs from the vLLM workers' own venv, so it is the same ai-dynamo release. - Each engine turns on vLLM's KV-cache events for `router_mode: kv` and @@ -51,9 +50,8 @@ shows them all: | Key | Default | Meaning | |---|---|---| | `startup_timeout_s` | 600 | Bounds service start-up and engine registration | -| `request_timeout_s` | 900 | Timeout for one NeMo RL rollout through the frontend, in place of `NRL_VLLM_ASYNC_TIMEOUT_SECONDS`. A timeout fails the attempt as a native timeout does, and the rollout retry policy applies | +| `request_timeout_s` | null | Deadline for one NeMo RL rollout through the frontend. null waits as long as generation takes, as native does; on the SingleController, `async_rl.rollout_failure.native.generation_timeout_s` bounds both paths | | `frontend_args.router_mode` | `kv` | Frontend routing policy | -| `frontend_args.tokenizer`, `tokenizer_cache*` | `default`, off | Frontend tokenizer | | `frontend_args.extra_cli_args` | `[]` | Extra `dynamo.frontend` flags | The etcd and nats-server binaries are found in `NEMO_RL_DYNAMO_BIN_DIR` when it @@ -87,7 +85,12 @@ Startup rejects these with `dynamo: true`: - `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, and a restarted shard does not register with it yet. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data - parallelism inside vLLM, which the async engine Dynamo serves does not. + parallelism inside vLLM, which Dynamo does not serve yet. Above data-parallel + rank 0, vLLM rejects each engine's `ipc://` KV-event endpoint. +- `vllm_cfg.http_server_api_key_required`: the frontend serves the same engines + on 0.0.0.0 and checks no bearer token. +- `frontend_args.router_mode: direct`: it needs a worker ID on every request, and + NeMo RL's rollouts name none. - `dynamo: true` on a backend other than vLLM, and a `dynamo_cfg` block without `dynamo: true`. @@ -117,7 +120,7 @@ bash docker/dynamo/install.sh ## Run -Use any non-colocated vLLM recipe and add the switch: +Use any non-colocated vLLM recipe that does not use NeMo-Gym, and add the switch: ```bash uv run examples/run_grpo_single_controller.py \ @@ -142,4 +145,6 @@ rejects it with the migration: `vllm_cfg.enable_vllm_metrics_logger`; - `worker_args`: the engines serve only NeMo RL's rollouts, which need no chat parsers or templates; engines are configured through `vllm_cfg` and - `vllm_kwargs`. + `vllm_kwargs`; + - `frontend_args.tokenizer`, `tokenizer_cache` and `tokenizer_cache_bytes`: + rollouts send token IDs, which the frontend never tokenizes. diff --git a/docs/index.md b/docs/index.md index 9cfda03a275..5f9dba49154 100644 --- a/docs/index.md +++ b/docs/index.md @@ -83,7 +83,7 @@ Configure and launch NeMo RL on multi-node Slurm or Kubernetes clusters for dist :link: guides/dynamo-generation :link-type: doc -Serve any vLLM generation config through Dynamo's frontend and KV-aware router with `dynamo: true`. +Serve non-colocated vLLM generation through Dynamo's frontend and KV-aware router with `dynamo: true`. ::: :::: diff --git a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml index 1ac1a92abaf..a1d89235d1e 100644 --- a/examples/configs/grpo_math_1B_dynamo_single_controller.yaml +++ b/examples/configs/grpo_math_1B_dynamo_single_controller.yaml @@ -14,14 +14,11 @@ policy: dynamo_cfg: # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: 600 - # Timeout for one rollout routed through the frontend (in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS). A timeout fails the attempt as a native - # timeout does, and the rollout retry policy applies. - request_timeout_s: 900 + # Deadline for one rollout through the frontend. null waits as long as + # generation takes, as native generate_async does; on the SingleController, + # async_rl.rollout_failure.native.generation_timeout_s bounds both paths. + request_timeout_s: null frontend_args: - tokenizer: default - tokenizer_cache: false - tokenizer_cache_bytes: 52428800 # kv: route each request to the engine holding most of its prefix. router_mode: kv extra_cli_args: [] diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml index d12f2536970..72877ceb232 100644 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml @@ -87,9 +87,6 @@ policy: startup_timeout_s: 1800 request_timeout_s: 7200 frontend_args: - tokenizer: fastokens - tokenizer_cache: true - tokenizer_cache_bytes: 4294967296 router_mode: kv extra_cli_args: [] vllm_cfg: diff --git a/nemo_rl/models/generation/__init__.py b/nemo_rl/models/generation/__init__.py index 7d17e9e8942..08d28438400 100644 --- a/nemo_rl/models/generation/__init__.py +++ b/nemo_rl/models/generation/__init__.py @@ -183,6 +183,13 @@ def configure_generation_config( if dynamo_enabled(config): # Validate before any Ray allocation. dynamo_cfg(config) + if config["vllm_cfg"].get("http_server_api_key_required"): + raise ValueError( + "policy.generation.vllm_cfg.http_server_api_key_required is not " + "supported with policy.generation.dynamo=true: the Dynamo " + "frontend serves the same engines on 0.0.0.0 and checks no " + "bearer token." + ) colocated = config.get("colocated") if colocated is not None and colocated["enabled"]: raise ValueError( @@ -193,18 +200,17 @@ def configure_generation_config( ) vllm_cfg = config["vllm_cfg"] if vllm_cfg["expert_parallel_size"] > vllm_cfg["tensor_parallel_size"]: - # VllmGeneration refuses the async engine with DP inside vLLM. + # Data parallelism inside vLLM: above DP rank 0, vLLM rejects the + # per-engine ipc:// KV-event endpoint (dynamo_kv_events_config). raise ValueError( - "policy.generation.dynamo=true serves the async vLLM engine, " - "which does not run data parallelism inside vLLM " - "(vllm_cfg.expert_parallel_size > tensor_parallel_size, see " - "https://github.com/NVIDIA-NeMo/RL/issues/1101). Set " - "expert_parallel_size <= tensor_parallel_size, or remove " - "policy.generation.dynamo." + "policy.generation.dynamo=true does not serve data parallelism " + "inside vLLM yet (vllm_cfg.expert_parallel_size > " + "tensor_parallel_size). Set expert_parallel_size <= " + "tensor_parallel_size, or remove policy.generation.dynamo." ) if not config["vllm_cfg"]["async_engine"]: # Dynamo serves the engine's AsyncLLM, so turning the async engine - # on is the switch; settings it cannot run are rejected above. + # on is the switch; settings Dynamo cannot serve are rejected above. print( " [Dynamo] policy.generation.dynamo=true: using the async " "vLLM engine (vllm_cfg.async_engine=true)", diff --git a/nemo_rl/models/generation/dynamo/config.py b/nemo_rl/models/generation/dynamo/config.py index af592bc0fd5..824280c3edf 100644 --- a/nemo_rl/models/generation/dynamo/config.py +++ b/nemo_rl/models/generation/dynamo/config.py @@ -21,23 +21,20 @@ from typing import Any, Literal, Mapping -from pydantic import BaseModel, Field, PositiveFloat, PositiveInt +from pydantic import BaseModel, Field, PositiveFloat class DynamoFrontendArgs(BaseModel, extra="forbid"): """Arguments for the managed ``dynamo.frontend`` process.""" - tokenizer: Literal["default", "fastokens"] = "default" - tokenizer_cache: bool = False - tokenizer_cache_bytes: PositiveInt = 50 * 1024**2 # "kv" routes each request to the engine holding most of its prefix, fed by - # the engines' vLLM KV-cache events. + # the engines' vLLM KV-cache events. No "direct": it needs a worker ID on + # every request, and NeMo RL's rollouts name none. router_mode: Literal[ "round-robin", "random", "power-of-two", "kv", - "direct", "least-loaded", "device-aware-weighted", ] = "kv" @@ -49,11 +46,10 @@ class DynamoCfg(BaseModel, extra="forbid"): # Bounds etcd/NATS/frontend start-up and every engine's registration. startup_timeout_s: PositiveFloat = 600.0 - # Timeout for one NeMo RL rollout routed through the frontend, in place of - # NRL_VLLM_ASYNC_TIMEOUT_SECONDS. A timeout fails the attempt as a native one - # does (GenerationUnavailable), and the rollout retry policy re-dispatches it - # (SingleController: async_rl.rollout_failure.max_infra_attempts_per_prompt). - request_timeout_s: PositiveFloat = 900.0 + # Deadline for one NeMo RL rollout through the frontend. None waits as long + # as generation takes, as native generate_async does; on the SingleController, + # async_rl.rollout_failure.native.generation_timeout_s bounds both paths. + request_timeout_s: PositiveFloat | None = None frontend_args: DynamoFrontendArgs = Field(default_factory=DynamoFrontendArgs) diff --git a/nemo_rl/models/generation/dynamo/http_client.py b/nemo_rl/models/generation/dynamo/http_client.py index b7d81f3de15..77d318f84a4 100644 --- a/nemo_rl/models/generation/dynamo/http_client.py +++ b/nemo_rl/models/generation/dynamo/http_client.py @@ -36,7 +36,7 @@ def _decode_json_object(body: bytes) -> dict[str, Any]: async def async_http_post_json( - url: str, payload: dict[str, Any], timeout_s: float + url: str, payload: dict[str, Any], timeout_s: float | None ) -> dict[str, Any]: """POST JSON without blocking the rollout actor event loop.""" timeout = aiohttp.ClientTimeout(total=timeout_s) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index e53c0aef8cc..d17cf2d7763 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -216,15 +216,6 @@ def _service_env(self) -> dict[str, str]: env.update(self._manager_env) return env - def _frontend_env(self) -> dict[str, str]: - env = self._service_env() - frontend_args = self._dynamo_cfg.frontend_args - env["DYN_TOKENIZER"] = frontend_args.tokenizer - if frontend_args.tokenizer_cache: - env["DYN_TOKENIZER_CACHE"] = "1" - env["DYN_TOKENIZER_CACHE_BYTES"] = str(frontend_args.tokenizer_cache_bytes) - return env - def _start_etcd(self) -> None: self._etcd_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_etcd_") peer_url = f"http://{self._host}:{self._etcd_peer_port}" @@ -312,19 +303,10 @@ def _start_frontend(self) -> None: print( f" [Dynamo] launching frontend argv={redact_argv(command)!r}", flush=True ) - frontend_env = self._frontend_env() + frontend_env = self._service_env() dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) frontend_env["VIRTUAL_ENV"] = dynamo_venv frontend_env["UV_PROJECT_ENVIRONMENT"] = dynamo_venv - tokenizer_env = { - key: value - for key, value in frontend_env.items() - if key.startswith("DYN_TOKENIZER") - } - print( - f" [Dynamo] frontend tokenizer environment={tokenizer_env!r}", - flush=True, - ) self._frontend_process = subprocess.Popen( command, env=frontend_env, start_new_session=True ) diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index e95f535bc25..b422d21c11e 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -101,21 +101,23 @@ def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: class SnapshotStatLogger(StatLoggerBase): """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" + # Set by Dynamo after vLLM built the logger, so shared through the class. publisher: Optional[Any] = None - num_gpu_blocks: int = 0 def __init__(self, vllm_config: Any, engine_index: int = 0) -> None: self.dp_rank = engine_index + # vLLM sizes the KV cache before it builds its stat loggers. + self.num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0 def record(self, scheduler_stats, iteration_stats, *args, **kwargs) -> None: cls = type(self) if scheduler_stats is None or cls.publisher is None: return - total = max(1, cls.num_gpu_blocks) + total = max(1, self.num_gpu_blocks) usage = scheduler_stats.kv_cache_usage hit_rate = None - prefix = getattr(scheduler_stats, "prefix_cache_stats", None) - if prefix is not None and prefix.queries > 0: + prefix = scheduler_stats.prefix_cache_stats + if prefix.queries > 0: hit_rate = prefix.hits / prefix.queries cls.publisher.publish( self.dp_rank, @@ -164,7 +166,6 @@ async def start(self, worker_id: int) -> EngineConfig: ): block_size = group["block_size"] break - SnapshotStatLogger.num_gpu_blocks = cache.num_gpu_blocks or 0 model = self._host.model_name return EngineConfig( model=model, @@ -232,7 +233,9 @@ async def kv_event_sources(self) -> list[KvEventSource]: kv_events = self._llm.vllm_config.kv_events_config if kv_events is None or not kv_events.enable_kv_cache_events: return [] - return [ZmqSource(endpoint=kv_events.endpoint)] + # A user's own kv_events_config can keep vLLM's wildcard default + # (tcp://*:5557): vLLM binds it, but a subscriber cannot connect to "*". + return [ZmqSource(endpoint=kv_events.endpoint.replace("*", "127.0.0.1"))] def component_metrics_dp_ranks(self) -> list[int]: return [0] diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py index 95beb10d319..3fe35a7f7fe 100644 --- a/nemo_rl/models/generation/dynamo/rollout_client.py +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -19,6 +19,7 @@ import logging from typing import Any, Optional +from nemo_rl.experience.failures import http_status_is_infra from nemo_rl.models.generation.dynamo.http_client import ( async_http_post_json, format_dynamo_error, @@ -33,7 +34,6 @@ _HTTP_MAX_ATTEMPTS = 3 _HTTP_RETRY_DELAY_S = 1.0 -_RETRYABLE_HTTP_STATUS_CODES = {408, 429} def _is_retryable(response: dict[str, Any]) -> bool: @@ -42,9 +42,7 @@ def _is_retryable(response: dict[str, Any]) -> bool: if "transport_error" in response or response.get("json_decode_error") is True: return True status = response.get("http_status") - return isinstance(status, int) and ( - status in _RETRYABLE_HTTP_STATUS_CODES or 500 <= status < 600 - ) + return isinstance(status, int) and http_status_is_infra(status) def rollout_payload( @@ -95,7 +93,7 @@ def parse_rollout_response(response: dict[str, Any], *, url: str) -> dict[str, A async def post_rollout( - url: str, payload: dict[str, Any], timeout_s: float + url: str, payload: dict[str, Any], timeout_s: float | None ) -> dict[str, Any]: """POST one rollout, retrying transient errors; returns its ``engine_data``.""" response: dict[str, Any] = {} @@ -116,7 +114,6 @@ async def post_rollout( raise RuntimeError( f"Dynamo rollout request to {url} failed: {response['timeout_error']}. " "For longer sequences, increase " - "policy.generation.dynamo_cfg.request_timeout_s, e.g. to " - f"{int(timeout_s * 2)}." + "policy.generation.dynamo_cfg.request_timeout_s, or set it to null." ) return parse_rollout_response(response, url=url) diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 018ac34fc24..3389737cce4 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -411,6 +411,10 @@ def __init__( # RayWorkerGroup(env_vars=...) below) is inherited by them. if draft_module_sharing_disable_required(self.cfg): env_vars[DRAFT_DISABLE_MODULE_SHARING_ENV] = "1" + if dynamo_enabled(self.cfg): + # dynamo._core fixes its log format when the engine first imports it, + # before engine_env is applied, so it must be in the actor's env. + env_vars["DYN_SDK_DISABLE_ANSI_LOGGING"] = "1" # Check if we need parallelism-aware worker group creation if self.model_parallel_size > 1: diff --git a/nemo_rl/utils/outdated_config_checks.py b/nemo_rl/utils/outdated_config_checks.py index 8591d1d1a17..28c8b30f1e9 100644 --- a/nemo_rl/utils/outdated_config_checks.py +++ b/nemo_rl/utils/outdated_config_checks.py @@ -213,6 +213,11 @@ def reject_outdated_dynamo_router_reset_states(config: dict[str, Any]) -> None: "templates; engines are configured through vllm_cfg and vllm_kwargs" ), } +_REMOVED_DYNAMO_FRONTEND_KEYS = ( + "tokenizer", + "tokenizer_cache", + "tokenizer_cache_bytes", +) def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: @@ -246,6 +251,15 @@ def reject_outdated_dynamo_backend(config: dict[str, Any]) -> None: problems.append( f"policy.generation.dynamo_cfg.{key} was removed: {replacement}." ) + frontend_args = dynamo_cfg.get("frontend_args") + if isinstance(frontend_args, dict): + for key in _REMOVED_DYNAMO_FRONTEND_KEYS: + if key in frontend_args: + problems.append( + f"policy.generation.dynamo_cfg.frontend_args.{key} was " + "removed: NeMo RL's rollouts send token IDs, which the " + "frontend never tokenizes." + ) if problems: raise ValueError(" ".join(problems)) diff --git a/pyproject.toml b/pyproject.toml index 259d712d387..a514464e7d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -708,7 +708,9 @@ exclude = ''' ''' [tool.pytest.ini_options] -addopts = "--durations=100 -s -rA -x" +# ai-dynamo (vllm extra) registers pytest plugins that skip test_sglang_*, +# test_trtllm_* and test_triton_* files when those frameworks are absent. +addopts = "--durations=100 -s -rA -x -p no:vllm_tests -p no:sglang_tests -p no:trtllm_tests -p no:triton_tests" testpaths = ["tests"] python_files = "test_*.py" markers = [ diff --git a/tests/functional/check_dynamo_overlay.sh b/tests/functional/check_dynamo_overlay.sh index ee98e2671a0..42a2b381e41 100755 --- a/tests/functional/check_dynamo_overlay.sh +++ b/tests/functional/check_dynamo_overlay.sh @@ -15,10 +15,11 @@ # What a native functional test also checks when it runs as the Dynamo overlay # (++policy.generation.dynamo=true): every engine registered with the managed -# frontend, the frontend routed rollouts to them, the KV router applied the -# engines' KV-cache events (when they publish any: KV routing with prefix -# caching), and shutdown stopped its services. The test's own metric checks hold -# as they do natively. +# frontend, the frontend routed rollouts to them, the engines published KV-cache +# events and the KV router applied them, and shutdown stopped its services. The +# test's own metric checks hold as they do natively. The KV check needs the KV +# router with vLLM prefix caching (the default on sm>=80); an overlay that turns +# either off cannot use this check as is. # # bash check_dynamo_overlay.sh @@ -26,28 +27,28 @@ set -euo pipefail run_log=$1 -grep -E "\[Dynamo\] frontend .* routes to [0-9]+ engine" "${run_log}" +fail() { + echo "$1" >&2 + exit 1 +} + +grep -E "\[Dynamo\] frontend .* routes to [0-9]+ engine" "${run_log}" \ + || fail "No engine registered with the Dynamo frontend" routed=$(grep -c 'route request completed request.outcome="success"' "${run_log}" || true) echo "Requests the Dynamo frontend routed: ${routed}" -if [[ "${routed}" -eq 0 ]]; then - echo "No request went through the Dynamo frontend" >&2 - exit 1 -fi +[[ "${routed}" -gt 0 ]] || fail "No request went through the Dynamo frontend" # vLLM's KV-event wire format is the one coupling between NeMo RL's vLLM and # Dynamo's router; a format change shows up here as no applied events. -if grep -q "\[Dynamo\] engine publishes KV events: True" "${run_log}"; then - grep -E "\[Dynamo\] router applied KV events" "${run_log}" - applied=$(grep -oE '\[Dynamo\] router applied KV events: .*"ok": [0-9.]+' "${run_log}" \ - | grep -oE '[0-9.]+$' | tail -1) - if [[ -z "${applied}" || "${applied%.*}" -eq 0 ]]; then - echo "The KV router applied none of the engines' KV-cache events" >&2 - exit 1 - fi -fi +grep -q "\[Dynamo\] engine publishes KV events: True" "${run_log}" \ + || fail "No engine published KV-cache events for the KV router" +applied=$(grep -oE '\[Dynamo\] router applied KV events: .*"ok": [0-9.]+' "${run_log}" \ + | grep -oE '[0-9.]+$' | tail -1 || true) +echo "KV events the router applied: ${applied:-0}" +[[ "${applied:-0}" =~ ^[0-9]+ && "${applied%%.*}" -gt 0 ]] \ + || fail "The KV router applied none of the engines' KV-cache events" dynamo_bin_dir=${NEMO_RL_DYNAMO_BIN_DIR:-/opt/dynamo/bin} if pgrep -f "[d]ynamo.frontend|${dynamo_bin_dir}/[e]tcd|${dynamo_bin_dir}/[n]ats-server"; then - echo "Managed Dynamo services remain after shutdown" >&2 - exit 1 + fail "Managed Dynamo services remain after shutdown" fi diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index a14f09e95f9..0624535b30c 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -75,7 +75,7 @@ def test_without_dynamo_the_vllm_config_is_untouched() -> None: (_vllm_config(dynamo=True, backend="sglang"), "backend=vllm"), # dynamo_cfg alone would be ignored. (_vllm_config(dynamo_cfg={}), "dynamo is not true"), - # The async engine Dynamo serves runs no data parallelism inside vLLM. + # Dynamo does not serve data parallelism inside vLLM yet. ( _vllm_config( dynamo=True, @@ -88,6 +88,27 @@ def test_without_dynamo_the_vllm_config_is_untouched() -> None: ), "expert_parallel_size <= tensor_parallel_size", ), + # The frontend serves the same engines and checks no bearer token. + ( + _vllm_config( + dynamo=True, + vllm_cfg={ + "async_engine": True, + "max_model_len": 512, + "tensor_parallel_size": 1, + "expert_parallel_size": 1, + "http_server_api_key_required": True, + }, + ), + "checks no bearer token", + ), + # Direct routing needs a worker ID that rollouts never carry. + ( + _vllm_config( + dynamo=True, dynamo_cfg={"frontend_args": {"router_mode": "direct"}} + ), + "router_mode", + ), ], ) def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( diff --git a/tests/unit/models/generation/test_dynamo_http_client.py b/tests/unit/models/generation/test_dynamo_http_client.py index d9002bce5c1..df8eabd9344 100644 --- a/tests/unit/models/generation/test_dynamo_http_client.py +++ b/tests/unit/models/generation/test_dynamo_http_client.py @@ -83,9 +83,10 @@ def test_async_http_post_json_error_contract( lambda **kwargs: session, ) + # No deadline is the default (dynamo_cfg.request_timeout_s: null). response = asyncio.run( http_client.async_http_post_json( - "http://worker/route", {"value": 1}, timeout_s=3 + "http://worker/route", {"value": 1}, timeout_s=None ) ) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index d9cd8990bd8..524416b344b 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -108,9 +108,7 @@ def test_runtime_construction_is_inert_and_namespace_is_driver_owned( def test_managed_service_and_frontend_environments_are_runtime_owned( monkeypatch, ) -> None: - runtime = _runtime( - tokenizer="fastokens", tokenizer_cache=True, tokenizer_cache_bytes=4096 - ) + runtime = _runtime() runtime._manager_env = { "DYN_NAMESPACE": "managed", "DYN_DISCOVERY_BACKEND": "etcd", @@ -136,14 +134,9 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( engine_env = runtime.engine_env assert engine_env["DYN_NAMESPACE"] == "managed" # Each engine serves its own model card, so a frontend on another node can - # read a chat template that exists only on the engine's node. + # read model files that exist only on the engine's node. assert engine_env["DYN_SYSTEM_PORT"] == "0" - - frontend_env = runtime._frontend_env() - assert "DYN_SYSTEM_PORT" not in frontend_env - assert frontend_env["DYN_TOKENIZER"] == "fastokens" - assert frontend_env["DYN_TOKENIZER_CACHE"] == "1" - assert frontend_env["DYN_TOKENIZER_CACHE_BYTES"] == "4096" + assert "DYN_SYSTEM_PORT" not in service_env def test_dynamo_python_resolves_the_vllm_worker_venv_as_workers_do( @@ -313,8 +306,8 @@ def endpoint_stopped() -> None: runtime.wait_for_engines(1, check_engines=endpoint_stopped) -def test_frontend_launches_from_the_given_interpreter(monkeypatch, capsys) -> None: - runtime = _runtime(tokenizer="fastokens", tokenizer_cache=True) +def test_frontend_launches_from_the_given_interpreter(monkeypatch) -> None: + runtime = _runtime() runtime._frontend_port = 3000 runtime._namespace = "nemo-rl-test" launched = [] @@ -326,8 +319,6 @@ def test_frontend_launches_from_the_given_interpreter(monkeypatch, capsys) -> No runtime._start_frontend() assert launched[0][:3] == ["/venvs/vllm/bin/python", "-m", "dynamo.frontend"] - output = capsys.readouterr().out - assert "DYN_TOKENIZER': 'fastokens" in output def test_shutdown_logs_the_kv_events_the_router_applied(monkeypatch, capsys) -> None: diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index ef2fbd01d83..d1dda2bfeb4 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -48,23 +48,10 @@ MAX_MODEL_LEN = 16 -EOS = 2 - - class _Context: def id(self) -> str: return "request-0" - def is_stopped(self) -> bool: - return False - - -class _Tokenizer: - eos_token_id = EOS - - def decode(self, token_ids) -> str: - return "".join(f"<{token_id}>" for token_id in token_ids) - def _logprob(value: float): return SimpleNamespace(logprob=value, rank=1, decoded_token=None) @@ -114,8 +101,6 @@ def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: "max_new_tokens": 8, "temperature": 0.7, "top_p": 0.9, - "val_temperature": 0.0, - "val_top_p": 1.0, "top_k": None, "stop_token_ids": [2], "stop_strings": [""], @@ -126,7 +111,6 @@ def _host(llm: _FakeLLM, **cfg) -> VllmAsyncGenerationWorkerImpl: host.model_name = "model" host.routed_experts_dtype = torch.int16 host.llm = llm - host._capture_media = False return host @@ -218,7 +202,7 @@ def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: assert rollout["token_ids"] == [] and rollout["truncated"] is False -def _start_worker_config(monkeypatch, host) -> SimpleNamespace: +def _start_worker_config(monkeypatch, host) -> native_engine.WorkerConfig: """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" configs = [] @@ -230,7 +214,6 @@ async def run(self): return None monkeypatch.setattr(native_engine, "Worker", _Worker) - monkeypatch.setattr(native_engine, "WorkerConfig", SimpleNamespace) env = { "DYN_DISCOVERY_BACKEND": "etcd", "DYN_REQUEST_PLANE": "tcp", diff --git a/tests/unit/models/generation/test_dynamo_rollout_client.py b/tests/unit/models/generation/test_dynamo_rollout_client.py index 1b2b942dc7f..138b4120017 100644 --- a/tests/unit/models/generation/test_dynamo_rollout_client.py +++ b/tests/unit/models/generation/test_dynamo_rollout_client.py @@ -12,42 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import asyncio -import json import pytest from nemo_rl.models.generation.dynamo import rollout_client from nemo_rl.models.generation.dynamo.rollout_client import ( - ROLLOUT_ANNOTATION, parse_rollout_response, post_rollout, - rollout_payload, ) -def test_rollout_payload_leaves_sampling_and_stopping_to_the_engine() -> None: - payload = rollout_payload( - model_name="model", - prompt_token_ids=[5, 6, 7], - max_model_len=10, - greedy=True, - stop_strings=[""], - ) - - assert payload["prompt"] == [5, 6, 7] - # The frontend's own stop rules must never end the stream first. - assert payload["ignore_eos"] is True - assert payload["max_tokens"] == 7 - assert "stop" not in payload and "temperature" not in payload - (annotation,) = payload["nvext"]["annotations"] - assert annotation.startswith(ROLLOUT_ANNOTATION) - assert json.loads(annotation[len(ROLLOUT_ANNOTATION) :]) == { - "greedy": True, - "stop_strings": [""], - } - assert payload["nvext"]["extra_fields"] == ["engine_data"] - - @pytest.mark.parametrize( "response, match", [ diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 4693fbdac91..1de81eac1df 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2626,26 +2626,30 @@ def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error VllmGeneration.validate_settings(master_config) -@pytest.mark.parametrize("dynamo", [False, True]) -def test_vllm_token_rollouts_use_the_dynamo_frontend_when_it_serves(dynamo): - generation = object.__new__(VllmGeneration) - generation.dynamo_frontend_url = "http://frontend/v1" if dynamo else None - tokens = BatchedDataDict({"input_ids": torch.tensor([[1]])}) - # The frontend request carries token IDs only, so media prompts stay native. - media = BatchedDataDict({"input_ids": torch.tensor([[1]]), "vllm_content": [None]}) - - assert generation._dynamo_serves(tokens) is dynamo - assert not generation._dynamo_serves(media) - - -def test_vllm_dynamo_frontend_failures_are_generation_unavailable(monkeypatch): +def test_vllm_dynamo_rollouts_go_through_the_frontend(monkeypatch): from nemo_rl.experience.failures import GenerationUnavailable from nemo_rl.models.generation.dynamo import rollout_client + from nemo_rl.models.generation.vllm.request_pipeline import ( + RolloutResult, + rollout_output, + rollout_result_to_wire, + ) - async def frontend_down(url, payload, timeout_s): - raise RuntimeError(f"Dynamo rollout request to {url} failed: HTTP 503") + result = RolloutResult([7, 8], [-0.5, -0.25], False, None, None) + responses = [ + {"rollout": rollout_result_to_wire(result), "gen_leader_worker_idx": 3}, + RuntimeError("Dynamo rollout request to frontend failed: HTTP 503"), + ] + requests = [] - monkeypatch.setattr(rollout_client, "post_rollout", frontend_down) + async def frontend(url, payload, timeout_s): + requests.append((url, payload, timeout_s)) + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + monkeypatch.setattr(rollout_client, "post_rollout", frontend) generation = object.__new__(VllmGeneration) generation.cfg = { "model_name": "model", @@ -2654,9 +2658,36 @@ async def frontend_down(url, payload, timeout_s): } generation.dynamo_frontend_url = "http://frontend/v1" data = BatchedDataDict( - {"input_ids": torch.tensor([[5, 6, 0]]), "input_lengths": torch.tensor([2])} + { + "input_ids": torch.tensor([[5, 6, 0]]), + "input_lengths": torch.tensor([2]), + "stop_strings": [[""]], + } ) - + # The frontend request carries token IDs only, so media prompts stay native. + media = BatchedDataDict({**data, "vllm_content": [None]}) + assert generation._dynamo_serves(data) and not generation._dynamo_serves(media) + + _, output = asyncio.run(anext(generation.generate_async(data, greedy=True))) + + ((url, payload, timeout_s),) = requests + assert (url, timeout_s) == ("http://frontend/v1/completions", None) + # The unpadded prompt; the engine samples it from the generation config, and + # the frontend's own stop rules never end the stream before the engine does. + assert payload["prompt"] == [5, 6] + assert payload["ignore_eos"] is True and payload["max_tokens"] == 14 + assert "stop" not in payload and "temperature" not in payload + (annotation,) = payload["nvext"]["annotations"] + assert json.loads(annotation.removeprefix(rollout_client.ROLLOUT_ANNOTATION)) == { + "greedy": True, + "stop_strings": [""], + } + assert payload["nvext"]["extra_fields"] == ["engine_data"] + # The driver returns what native dispatch returns. + native = rollout_output(result, input_ids=data["input_ids"][0], input_length=2) + assert output["gen_leader_worker_idx"] == [3] + for key in native: + assert torch.equal(output[key], native[key]), key # An infrastructure failure is re-dispatched; an unmapped RuntimeError would # count as a DATA failure against the prompt. with pytest.raises(GenerationUnavailable, match="HTTP 503"): diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 2119afcebbb..4539d34b951 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -407,6 +407,10 @@ def test_dynamo_backend_is_rejected_with_the_migration(): ({"metrics_exclude_prefixes": None}, r"dynamo_cfg\.metrics_exclude_prefixes"), ({"worker_args": {"extra_cli_args": []}}, r"dynamo_cfg\.worker_args was"), ({"worker_args": {"reasoning_parser": None}}, r"dynamo_cfg\.worker_args was"), + ( + {"frontend_args": {"tokenizer_cache": False}}, + r"frontend_args\.tokenizer_cache was", + ), ], ) def test_removed_dynamo_cfg_keys_are_rejected(dynamo_cfg, removed): From 1e73994f851d9d67d3f70fea3fef9948b56ac4b6 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 29 Oct 2026 15:42:09 -0700 Subject: [PATCH 11/19] refactor(dynamo): debloat the Dynamo serving path No behaviour change: - rollout_client absorbs http_client (its only user), and the HTTP tests fold into one parametrized test. - venv.py goes: the frontend runs from worker_group.py_executable, the venv the engines already use, and etcd/nats-server are found with shutil.which in managed_runtime. - arguments.py goes: managed_runtime builds the fixed frontend argv and still rejects extra_cli_args that repeat a managed flag. Nothing logged carries credentials, so the redaction helpers go too. - decode_routed_experts(dtype=None) keeps the envelope's dtype, replacing routed_experts_envelope_dtype; the wire codec drops an impossible check. - managed_runtime: an unreachable namespace check, the atexit flag and the per-port allocation boilerplate go. - The swe1 Dynamo + NeMo-Gym recipe, its driver and its config test move to #4979 with the Gym path that can run them. - Tests: on/off config cases folded (with a check that the exemplar spells out exactly the DynamoCfg defaults), outdated-config cases parametrized, the redundant pin test and a GRPO helper's test removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/guides/dynamo-generation.md | 20 +- ...nov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml | 181 ------------------ nemo_rl/algorithms/grpo.py | 15 +- nemo_rl/models/generation/dynamo/__init__.py | 13 -- nemo_rl/models/generation/dynamo/arguments.py | 144 -------------- .../models/generation/dynamo/http_client.py | 77 -------- .../generation/dynamo/managed_runtime.py | 116 ++++++----- .../generation/dynamo/rollout_client.py | 58 +++++- nemo_rl/models/generation/dynamo/venv.py | 67 ------- .../generation/vllm/request_pipeline.py | 10 +- .../models/generation/vllm/vllm_generation.py | 5 +- .../generation/vllm/vllm_worker_async.py | 22 +-- nemo_rl/utils/routed_experts_codec.py | 18 +- pyrefly.toml | 3 - tests/test_suites/disabled.txt | 4 - ...nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh | 73 ------- tests/unit/L0_Unit_Tests_Vllm_1.sh | 2 +- tests/unit/L0_Unit_Tests_Vllm_2.sh | 2 +- tests/unit/L0_Unit_Tests_Vllm_3.sh | 2 +- tests/unit/algorithms/test_grpo.py | 49 +---- .../models/generation/test_dynamo_config.py | 54 ++---- .../generation/test_dynamo_http_client.py | 126 ------------ .../generation/test_dynamo_managed_runtime.py | 47 +---- .../generation/test_dynamo_rollout_client.py | 64 ++++++- .../generation/test_swe1_dynamo_config.py | 125 ------------ tests/unit/test_dependency_pins.py | 18 -- .../unit/utils/test_outdated_config_checks.py | 57 +++--- 27 files changed, 268 insertions(+), 1104 deletions(-) delete mode 100644 examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml delete mode 100644 nemo_rl/models/generation/dynamo/arguments.py delete mode 100644 nemo_rl/models/generation/dynamo/http_client.py delete mode 100644 nemo_rl/models/generation/dynamo/venv.py delete mode 100755 tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh delete mode 100644 tests/unit/models/generation/test_dynamo_http_client.py delete mode 100644 tests/unit/models/generation/test_swe1_dynamo_config.py diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 7acefc88d9b..52d65c6cfa3 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -133,18 +133,8 @@ is the same run with every `dynamo_cfg` default spelled out. ## Migrating from `backend: dynamo` -`backend: dynamo` launched Dynamo's own vLLM engine and was removed. Startup -rejects it with the migration: - -- Set `backend: vllm` and `dynamo: true`. `vllm_cfg` and `vllm_kwargs` keep their - meaning, and every vLLM feature is now available. -- Remove these from `dynamo_cfg`: - - `engine`; - - `control_timeout_s`: refit and pause use the vLLM backend's controls; - - `metrics_include_prefixes` / `metrics_exclude_prefixes`: use - `vllm_cfg.enable_vllm_metrics_logger`; - - `worker_args`: the engines serve only NeMo RL's rollouts, which need no chat - parsers or templates; engines are configured through `vllm_cfg` and - `vllm_kwargs`; - - `frontend_args.tokenizer`, `tokenizer_cache` and `tokenizer_cache_bytes`: - rollouts send token IDs, which the frontend never tokenizes. +`backend: dynamo` launched Dynamo's own vLLM engine and was removed. Set +`backend: vllm` and `dynamo: true`; `vllm_cfg` and `vllm_kwargs` keep their +meaning. Startup rejects the old backend and each removed `dynamo_cfg` key +(`engine`, `control_timeout_s`, `metrics_*_prefixes`, `worker_args`, +`frontend_args.tokenizer*`) with what replaces it. diff --git a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml b/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml deleted file mode 100644 index 72877ceb232..00000000000 --- a/examples/configs/recipes/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml +++ /dev/null @@ -1,181 +0,0 @@ -defaults: ../../../nemo_gym/grpo_nanov3.yaml -cluster: - num_nodes: 3 - segment_size: 1 -checkpointing: - enabled: false - checkpoint_dir: results/nemotron-3-nano-swe-dynamo - save_period: 3 - keep_top_k: 1 - checkpoint_must_save_by: 00:03:30:00 -grpo: - num_prompts_per_step: 2 - num_generations_per_prompt: 2 - val_num_generations_per_prompt: 2 - max_num_epochs: 4 - max_num_steps: 4 - advantage_clip_low: -100 - advantage_clip_high: 100 - val_period: 10000 - overlong_filtering: false - invalid_tool_call_advantage: -5.0 - malformed_thinking_advantage: -5.0 - reward_shaping: - overlong_buffer_length: 128 - overlong_buffer_penalty: 1 - max_response_length: ${policy.max_total_sequence_length} - stop_properly_penalty_coef: null - reward_scaling: - source_min: 0.0 - source_max: 1.0 - target_min: 0.0 - target_max: 1.0 - async_grpo: - enabled: true - in_flight_weight_updates: false - recompute_kv_cache_after_weight_updates: true -loss_fn: - truncated_importance_sampling_ratio: 5 - truncated_importance_sampling_ratio_min: 0.2 - truncated_importance_sampling_type: tis - force_on_policy_ratio: true - use_kl_in_reward: false -policy: - model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 - tokenizer: - name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 - chat_template_kwargs: - enable_thinking: true - hf_config_overrides: {} - train_global_batch_size: 4 - generation_batch_size: 2 - max_total_sequence_length: 196608 - logprob_chunk_size: 1024 - megatron_cfg: - empty_unused_memory_level: 2 - tensor_model_parallel_size: 4 - pipeline_model_parallel_size: 1 - context_parallel_size: 2 - moe_flex_dispatcher_backend: hybridep - moe_hybridep_num_sms: 32 - use_gloo_process_groups: false - use_fused_weighted_squared_relu: true - do_not_average_loss: true - cp_normalize: true - calculate_per_token_loss: true - scale_loss_by_dp_cp_size: false - mtp_loss_scaling_factor: 0.0 - mtp_use_repeated_layer: true - mtp_num_layers: 0 - mtp_detach_heads: true - scheduler: - override_opt_param_scheduler: true - distributed_data_parallel_config: - overlap_grad_reduce: false - fp8_cfg: - enabled: false - fp8: e4m3 - fp8_recipe: mxfp8 - fp8_param: false - checkpoint: - ckpt_assume_constant_structure: true - sequence_packing: - fuse_loss: true - generation: - dynamo: true - dynamo_cfg: - startup_timeout_s: 1800 - request_timeout_s: 7200 - frontend_args: - router_mode: kv - extra_cli_args: [] - vllm_cfg: - expert_parallel_size: 4 - precision: bfloat16 - gpu_memory_utilization: 0.85 - enable_vllm_metrics_logger: true - vllm_metrics_logger_interval: 0.5 - env_vars: null - vllm_kwargs: - attention_backend: FLASH_ATTN - moe_backend: triton - compilation_config: - cudagraph_capture_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - pass_config: - fuse_allreduce_rms: false - colocated: - enabled: false - resources: - gpus_per_node: 8 - num_nodes: 1 -data: - train: - data_path: ${oc.env:HF_HOME}/superv3_data/swe1/train-split.jsonl - validation: - data_path: ${oc.env:HF_HOME}/superv3_data/swe1/val-split.jsonl -env: - nemo_gym: - skip_venv_if_present: true - num_gpu_nodes: 0 - invalid_tool_call_patterns: - - - - - - - - - thinking_tags: - - - - - config_paths: - - responses_api_models/vllm_model/configs/vllm_model_for_training.yaml - - resources_servers/single_step_tool_use_with_argument_comparison/configs/swe_pivot_single_step_tool_use_with_argument_comparison.yaml - # Drop inherited wrappers for environments omitted from this recipe. - math_with_judge_environment_server: null - code_gen_environment_server: null - workplace_assistant_environment_server: null - mcqa_environment_server: null - instruction_following_environment_server: null - structured_outputs_environment_server: null - swe_pivot_single_step_tool_use_with_argument_comparison_environment_server: - environment_servers: - legacy_agent: - entrypoint: app.py - agent_server: - type: responses_api_agents - name: swe_pivot_single_step_tool_use_with_argument_comparison_agent - policy_model: - responses_api_models: - vllm_model: - chat_template_kwargs: - force_nonempty_content: true - single_step_tool_use_with_argument_comparison_swe: - responses_api_agents: - tool_simulation_agent: - entrypoint: app.py - resources_server: - type: resources_servers - name: swe_pivot_single_step_tool_use_with_argument_comparison_resources_server - model_server: - type: responses_api_models - name: policy_model - single_step_tool_use_with_argument_comparison_swe_environment_server: - environment_servers: - legacy_agent: - entrypoint: app.py - agent_server: - type: responses_api_agents - name: single_step_tool_use_with_argument_comparison_swe - use_absolute_ip: true -logger: - log_dir: results/nemotron-3-nano-swe-dynamo - wandb_enabled: true - tensorboard_enabled: true - wandb: - project: nemo-rl - name: grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1 diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index 31ef02d7895..6f4e7afe4b5 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -635,14 +635,6 @@ def _needs_hf_refit_handshake( return not (nccl_reshard_refit_enabled and not colocated_inference) -def _uses_managed_noncolocated_refit( - nccl_reshard_refit_enabled: bool, - offload_policy_before_refit: bool, -) -> bool: - """Whether non-colocated setup must attach a weight synchronizer.""" - return nccl_reshard_refit_enabled or offload_policy_before_refit - - def _shutdown_completed_nemo_gym_startup( future: Future[tuple[NemoGymShardSet, float]] | None, ) -> None: @@ -1935,11 +1927,8 @@ def init_trtllm(): and checkpoint_engine_config is None ): t0 = time.perf_counter() - # init collective - if _uses_managed_noncolocated_refit( - nccl_reshard_refit_enabled=nccl_reshard_refit_enabled, - offload_policy_before_refit=offload_policy_before_refit, - ): + # init collective; these refits need an attached weight synchronizer. + if nccl_reshard_refit_enabled or offload_policy_before_refit: policy_generation.weight_synchronizer = create_weight_synchronizer( policy=policy, generation=policy_generation, diff --git a/nemo_rl/models/generation/dynamo/__init__.py b/nemo_rl/models/generation/dynamo/__init__.py index 427c1ede100..4fc25d0d3c9 100644 --- a/nemo_rl/models/generation/dynamo/__init__.py +++ b/nemo_rl/models/generation/dynamo/__init__.py @@ -11,16 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from nemo_rl.models.generation.dynamo.config import ( - DynamoCfg, - DynamoFrontendArgs, - dynamo_cfg, - dynamo_enabled, -) - -__all__ = [ - "DynamoCfg", - "DynamoFrontendArgs", - "dynamo_cfg", - "dynamo_enabled", -] diff --git a/nemo_rl/models/generation/dynamo/arguments.py b/nemo_rl/models/generation/dynamo/arguments.py deleted file mode 100644 index e5a6abebee5..00000000000 --- a/nemo_rl/models/generation/dynamo/arguments.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Argument construction and log redaction for the managed Dynamo frontend.""" - -import json -from collections.abc import Mapping, Sequence -from typing import Any - -from nemo_rl.models.generation.dynamo.config import DynamoCfg - -_CREDENTIAL_FLAGS = { - "--access-token", - "--api-key", - "--apikey", - "--auth-token", - "--hf-token", - "--password", - "--secret", - "--token", -} - - -def _normalise_flag(flag: str) -> str: - flag = flag.split("=", 1)[0].replace("_", "-") - if flag.startswith("--no-"): - return "--" + flag[5:] - return flag - - -def _serialise_value(value: Any) -> str: - if isinstance(value, (dict, list, tuple)): - return json.dumps(value, separators=(",", ":"), sort_keys=True) - return str(value) - - -class _ArgvBuilder: - def __init__(self) -> None: - self.argv: list[str] = [] - self.sources: dict[str, str] = {} - - def add(self, flag: str, value: Any = None, *, source: str) -> None: - normalised = _normalise_flag(flag) - prior = self.sources.get(normalised) - if prior is not None: - raise ValueError( - f"Dynamo frontend option {normalised} is set by both {prior} and {source}." - ) - self.sources[normalised] = source - if isinstance(value, bool): - self.argv.append(flag if value else f"--no-{flag.removeprefix('--')}") - else: - self.argv.append(flag) - if value is not None: - self.argv.append(_serialise_value(value)) - - def add_raw(self, args: Sequence[str], *, source: str) -> None: - if not args: - return - current_flag: str | None = None - current_has_value = False - for token in args: - if token.startswith("--"): - current_flag = token - current_has_value = "=" in token - normalised = _normalise_flag(token) - if normalised in self.sources: - raise ValueError( - f"Dynamo frontend option {normalised} is set by both " - f"{self.sources[normalised]} and {source}." - ) - self.sources[normalised] = source - elif current_flag is None or current_has_value: - raise ValueError( - f"{source} contains invalid positional argument {token!r}; " - "values must immediately follow one --option." - ) - else: - current_has_value = True - self.argv.extend(args) - - -def build_dynamo_frontend_argv( - *, - host: str, - port: int, - namespace: str, - dynamo_cfg: DynamoCfg, -) -> list[str]: - """Build the managed ``dynamo.frontend`` argument list.""" - builder = _ArgvBuilder() - builder.add("--http-host", host, source="managed runtime") - builder.add("--http-port", port, source="managed runtime") - builder.add("--namespace-prefix", namespace, source="managed runtime") - builder.add("--discovery-backend", "etcd", source="managed runtime") - builder.add("--request-plane", "tcp", source="managed runtime") - builder.add("--event-plane", "nats", source="managed runtime") - builder.add( - "--router-mode", - dynamo_cfg.frontend_args.router_mode, - source="dynamo_cfg.frontend_args.router_mode", - ) - builder.add_raw( - dynamo_cfg.frontend_args.extra_cli_args, - source="dynamo_cfg.frontend_args.extra_cli_args", - ) - return builder.argv - - -def redact_argv(argv: Sequence[str]) -> list[str]: - """Redact values following explicit credential options for safe logging.""" - redacted = list(argv) - for idx, token in enumerate(redacted): - is_sensitive = token.startswith("--") and ( - _normalise_flag(token.lower()) in _CREDENTIAL_FLAGS - ) - if is_sensitive and "=" in token: - redacted[idx] = token.split("=", 1)[0] + "=" - elif is_sensitive and idx + 1 < len(redacted): - if not redacted[idx + 1].startswith("--"): - redacted[idx + 1] = "" - return redacted - - -def redact_environment(env: Mapping[str, str]) -> dict[str, str]: - """Redact credential-like environment values before logging.""" - sensitive_fragments = ("token", "password", "secret", "api_key", "apikey") - return { - key: "" - if any(part in key.lower() for part in sensitive_fragments) - else value - for key, value in env.items() - } diff --git a/nemo_rl/models/generation/dynamo/http_client.py b/nemo_rl/models/generation/dynamo/http_client.py deleted file mode 100644 index 77d318f84a4..00000000000 --- a/nemo_rl/models/generation/dynamo/http_client.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""HTTP helpers for requests to the managed Dynamo frontend.""" - -import json -from typing import Any - -import aiohttp - - -def _decode_json_object(body: bytes) -> dict[str, Any]: - """Decode an HTTP response body using the shared Dynamo error contract.""" - try: - decoded = json.loads(body) - except json.JSONDecodeError: - return { - "status": "error", - "json_decode_error": True, - "raw": body.decode("utf-8", "replace"), - } - if not isinstance(decoded, dict): - return {"status": "error", "raw": repr(decoded)} - return decoded - - -async def async_http_post_json( - url: str, payload: dict[str, Any], timeout_s: float | None -) -> dict[str, Any]: - """POST JSON without blocking the rollout actor event loop.""" - timeout = aiohttp.ClientTimeout(total=timeout_s) - try: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.post(url, json=payload) as response: - body = await response.read() - if response.status >= 400: - return { - "status": "error", - "http_status": response.status, - "raw": body.decode("utf-8", "replace"), - } - except TimeoutError: - # Before ClientError: aiohttp's ServerTimeoutError is both. - return { - "status": "error", - "timeout_error": f"no response within {timeout_s}s", - } - except aiohttp.ClientError as error: - return { - "status": "error", - "transport_error": f"{type(error).__name__}: {error}", - } - return _decode_json_object(body) - - -def format_dynamo_error(response: dict[str, Any]) -> str: - """Format an error object returned by :func:`async_http_post_json`.""" - if "http_status" in response: - return f"HTTP {response['http_status']}: {response.get('raw', '')}" - if "transport_error" in response: - return str(response["transport_error"]) - if "timeout_error" in response: - return str(response["timeout_error"]) - if "raw" in response: - return str(response["raw"]) - return str(response) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index d17cf2d7763..273d595c014 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -41,19 +41,61 @@ _get_free_port_local, _get_node_ip_local, ) -from nemo_rl.models.generation.dynamo.arguments import ( - build_dynamo_frontend_argv, - redact_argv, - redact_environment, -) from nemo_rl.models.generation.dynamo.config import DynamoCfg -from nemo_rl.models.generation.dynamo.venv import get_dynamo_service_executable # The engines register on this component and endpoint (see native_engine). DYNAMO_COMPONENT = "backend" DYNAMO_ENDPOINT = "generate" LOGGER = logging.getLogger(__name__) + + +def frontend_argv( + *, host: str, port: int, namespace: str, dynamo_cfg: DynamoCfg +) -> list[str]: + """The ``dynamo.frontend`` arguments; ``extra_cli_args`` may not repeat one.""" + argv = [ + "--http-host", + host, + "--http-port", + str(port), + "--namespace-prefix", + namespace, + "--discovery-backend", + "etcd", + "--request-plane", + "tcp", + "--event-plane", + "nats", + "--router-mode", + dynamo_cfg.frontend_args.router_mode, + ] + extra = dynamo_cfg.frontend_args.extra_cli_args + for token in extra: + flag = token.split("=", 1)[0].replace("_", "-") + if flag.startswith("--") and flag in argv: + raise ValueError( + f"Dynamo frontend option {flag} is set by both the managed runtime " + "and dynamo_cfg.frontend_args.extra_cli_args." + ) + return argv + list(extra) + + +def _service_executable(name: str) -> str: + """The etcd or nats-server binary: in NEMO_RL_DYNAMO_BIN_DIR when set, else on PATH.""" + bin_dir = os.environ.get("NEMO_RL_DYNAMO_BIN_DIR") + found = shutil.which(name, path=os.path.expanduser(bin_dir) if bin_dir else None) + if found is None: + where = f"NEMO_RL_DYNAMO_BIN_DIR={bin_dir}" if bin_dir else "PATH" + raise FileNotFoundError( + f"{name} is not an executable on {where}. Build the image with " + "--build-arg BUILD_DYNAMO=1, or install etcd and nats-server with " + "docker/dynamo/install.sh and set NEMO_RL_DYNAMO_BIN_DIR to their " + "directory." + ) + return found + + # The KV router's applied-event counter (Dynamo's kvrouter.KV_CACHE_EVENTS_APPLIED), # one sample per label set. _KV_EVENTS_APPLIED = re.compile( @@ -65,10 +107,7 @@ def _managed_namespace() -> str: raw = f"nemo-rl-{os.environ.get('SLURM_JOB_ID', os.getpid())}" - namespace = re.sub(r"[^a-zA-Z0-9_-]+", "-", str(raw)).strip("-_").lower() - if not namespace: - raise ValueError(f"Could not derive a valid Dynamo namespace from {raw!r}.") - return namespace + return re.sub(r"[^a-zA-Z0-9_-]+", "-", raw).strip("-_").lower() class ManagedDynamoRuntime: @@ -96,7 +135,6 @@ def __init__( self._frontend_port = 0 self._manager_env: dict[str, str] = {} self._started = False - self._atexit_registered = False self._etcd_process: subprocess.Popen | None = None self._nats_process: subprocess.Popen | None = None self._frontend_process: subprocess.Popen | None = None @@ -131,35 +169,21 @@ def start(self) -> None: # exception that escapes setup. Keep an interpreter-exit fallback in # addition to the normal explicit shutdown path. atexit.register(self.shutdown) - self._atexit_registered = True self._host = _get_node_ip_local() - used_ports: set[int] = set() - - def allocate_port(*, low: int, high: int) -> int: - port = _get_free_port_local( - low, - high, - max_retries=None, - excluded_ports=used_ports, - ) - used_ports.add(port) - return port - - self._etcd_port = allocate_port( - low=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, - high=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, - ) - self._etcd_peer_port = allocate_port( - low=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, - high=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, + control = ( + DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, + DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, ) - self._nats_port = allocate_port( - low=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_LOW, - high=DEFAULT_DYNAMO_CONTROL_PORT_RANGE_HIGH, - ) - self._frontend_port = allocate_port( - low=DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, - high=DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH, + http = (DEFAULT_DYNAMO_HTTP_PORT_RANGE_LOW, DEFAULT_DYNAMO_HTTP_PORT_RANGE_HIGH) + ports: list[int] = [] + for low, high in (control, control, control, http): + ports.append( + _get_free_port_local( + low, high, max_retries=None, excluded_ports=set(ports) + ) + ) + self._etcd_port, self._etcd_peer_port, self._nats_port, self._frontend_port = ( + ports ) self._manager_env = { "ETCD_ENDPOINTS": f"http://{self._host}:{self._etcd_port}", @@ -174,7 +198,7 @@ def allocate_port(*, low: int, high: int) -> int: "DYN_SDK_DISABLE_ANSI_LOGGING": "1", } print( - f" [Dynamo] managed environment={redact_environment(self._manager_env)!r}", + f" [Dynamo] managed environment={self._manager_env!r}", flush=True, ) try: @@ -220,7 +244,7 @@ def _start_etcd(self) -> None: self._etcd_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_etcd_") peer_url = f"http://{self._host}:{self._etcd_peer_port}" command = [ - get_dynamo_service_executable("etcd"), + _service_executable("etcd"), "--listen-client-urls", f"http://0.0.0.0:{self._etcd_port}", "--advertise-client-urls", @@ -260,7 +284,7 @@ def _start_nats(self) -> None: self._nats_data_dir = tempfile.mkdtemp(prefix="nemorl_dynamo_nats_") self._nats_process = subprocess.Popen( [ - get_dynamo_service_executable("nats-server"), + _service_executable("nats-server"), "-js", "-sd", self._nats_data_dir, @@ -288,7 +312,7 @@ def _wait_for_port( raise RuntimeError(f"{label} did not open port {port}.") def _start_frontend(self) -> None: - argv = build_dynamo_frontend_argv( + argv = frontend_argv( host="0.0.0.0", port=self._frontend_port, namespace=self._namespace, @@ -300,9 +324,7 @@ def _start_frontend(self) -> None: "dynamo.frontend", *argv, ] - print( - f" [Dynamo] launching frontend argv={redact_argv(command)!r}", flush=True - ) + print(f" [Dynamo] launching frontend argv={command!r}", flush=True) frontend_env = self._service_env() dynamo_venv = os.path.dirname(os.path.dirname(self._frontend_python)) frontend_env["VIRTUAL_ENV"] = dynamo_venv @@ -435,7 +457,5 @@ def shutdown(self) -> None: if self._nats_data_dir is not None: shutil.rmtree(self._nats_data_dir, ignore_errors=True) self._nats_data_dir = None - if self._atexit_registered: - atexit.unregister(self.shutdown) - self._atexit_registered = False + atexit.unregister(self.shutdown) self._started = False diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py index 3fe35a7f7fe..5e1346f4b2c 100644 --- a/nemo_rl/models/generation/dynamo/rollout_client.py +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -19,11 +19,9 @@ import logging from typing import Any, Optional +import aiohttp + from nemo_rl.experience.failures import http_status_is_infra -from nemo_rl.models.generation.dynamo.http_client import ( - async_http_post_json, - format_dynamo_error, -) LOGGER = logging.getLogger(__name__) @@ -36,6 +34,52 @@ _HTTP_RETRY_DELAY_S = 1.0 +async def _post_json( + url: str, payload: dict[str, Any], timeout_s: float | None +) -> dict[str, Any]: + """POST JSON; a failure comes back as a dict with ``"status": "error"``.""" + try: + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=timeout_s) + ) as session: + async with session.post(url, json=payload) as response: + body = await response.read() + if response.status >= 400: + return { + "status": "error", + "http_status": response.status, + "raw": body.decode("utf-8", "replace"), + } + except TimeoutError: + # Before ClientError: aiohttp's ServerTimeoutError is both. + return {"status": "error", "timeout_error": f"no response within {timeout_s}s"} + except aiohttp.ClientError as error: + return { + "status": "error", + "transport_error": f"{type(error).__name__}: {error}", + } + try: + decoded = json.loads(body) + except json.JSONDecodeError: + return { + "status": "error", + "json_decode_error": True, + "raw": body.decode("utf-8", "replace"), + } + if not isinstance(decoded, dict): + return {"status": "error", "raw": repr(decoded)} + return decoded + + +def _format_error(response: dict[str, Any]) -> str: + if "http_status" in response: + return f"HTTP {response['http_status']}: {response.get('raw', '')}" + for key in ("transport_error", "timeout_error", "raw"): + if key in response: + return str(response[key]) + return str(response) + + def _is_retryable(response: dict[str, Any]) -> bool: # A timeout is final: the request may still be running, and a retry would # let one hung rollout hold its slot for several request_timeout_s. @@ -81,7 +125,7 @@ def parse_rollout_response(response: dict[str, Any], *, url: str) -> dict[str, A """ if response.get("status") == "error": raise RuntimeError( - f"Dynamo rollout request to {url} failed: {format_dynamo_error(response)}" + f"Dynamo rollout request to {url} failed: {_format_error(response)}" ) engine_data = (response.get("nvext") or {}).get("engine_data") if not isinstance(engine_data, dict) or "rollout" not in engine_data: @@ -98,7 +142,7 @@ async def post_rollout( """POST one rollout, retrying transient errors; returns its ``engine_data``.""" response: dict[str, Any] = {} for attempt in range(1, _HTTP_MAX_ATTEMPTS + 1): - response = await async_http_post_json(url, payload, timeout_s) + response = await _post_json(url, payload, timeout_s) if not _is_retryable(response) or attempt == _HTTP_MAX_ATTEMPTS: break LOGGER.warning( @@ -106,7 +150,7 @@ async def post_rollout( attempt, _HTTP_MAX_ATTEMPTS, _HTTP_RETRY_DELAY_S, - format_dynamo_error(response), + _format_error(response), ) await asyncio.sleep(_HTTP_RETRY_DELAY_S) if "timeout_error" in response: diff --git a/nemo_rl/models/generation/dynamo/venv.py b/nemo_rl/models/generation/dynamo/venv.py deleted file mode 100644 index 5708c8d0676..00000000000 --- a/nemo_rl/models/generation/dynamo/venv.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Resolve the interpreter and service binaries the managed Dynamo runtime launches.""" - -import os -import shutil -from pathlib import Path - -from nemo_rl.distributed.ray_actor_environment_registry import get_actor_python_env -from nemo_rl.utils.venvs import create_local_venv_on_each_node - -# Every vLLM worker venv carries the locked ai-dynamo release, so the frontend -# runs from the async worker's venv on the same release as every engine. -VLLM_ASYNC_WORKER_FQN = ( - "nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker" -) - - -def get_dynamo_python() -> str: - """Return the vLLM async worker's Python, which carries ``ai-dynamo``. - - The driver launches ``dynamo.frontend`` with it. A uv-managed venv is - resolved as the workers' own is: an existing one is reused as-is, and a - missing one is built once on each node. Re-syncing it here would change a - venv the engines are already running from. - """ - py_executable = get_actor_python_env(VLLM_ASYNC_WORKER_FQN) - if py_executable.startswith("uv"): - return create_local_venv_on_each_node(py_executable, VLLM_ASYNC_WORKER_FQN) - return py_executable - - -def get_dynamo_service_executable(name: str) -> str: - """Return the etcd or nats-server binary the runtime starts. - - Looks in ``NEMO_RL_DYNAMO_BIN_DIR`` when it is set, otherwise on ``PATH``. - """ - bin_dir = os.environ.get("NEMO_RL_DYNAMO_BIN_DIR") - if bin_dir: - executable = Path(bin_dir).expanduser() / name - if executable.is_file() and os.access(executable, os.X_OK): - return str(executable) - raise FileNotFoundError( - f"{name} is not an executable file in NEMO_RL_DYNAMO_BIN_DIR={bin_dir}. " - "Build the image with --build-arg BUILD_DYNAMO=1, or install etcd and " - "nats-server there with docker/dynamo/install.sh." - ) - found = shutil.which(name) - if found is None: - raise FileNotFoundError( - f"{name} is not on PATH. Install etcd and nats-server with " - "docker/dynamo/install.sh and set NEMO_RL_DYNAMO_BIN_DIR to their " - "directory, or build the image with --build-arg BUILD_DYNAMO=1." - ) - return found diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index 1e7c0c418a3..66276fa5d62 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -33,7 +33,6 @@ from nemo_rl.utils.routed_experts_codec import ( decode_routed_experts, encode_routed_experts, - routed_experts_envelope_dtype, ) LOGGER = logging.getLogger(__name__) @@ -195,14 +194,7 @@ def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: """Inverse of :func:`rollout_result_to_wire`; routes keep their own dtype.""" routed_experts = wire["routed_experts"] if routed_experts is not None: - routed_experts = decode_routed_experts( - routed_experts, routed_experts_envelope_dtype(routed_experts) - ) - if len(wire["logprobs"]) != len(wire["token_ids"]): - raise ValueError( - f"rollout result has {len(wire['logprobs'])} logprobs for " - f"{len(wire['token_ids'])} tokens" - ) + routed_experts = decode_routed_experts(routed_experts, dtype=None) return RolloutResult( token_ids=list(wire["token_ids"]), logprobs=list(wire["logprobs"]), diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index 3389737cce4..12227bd21df 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -726,12 +726,13 @@ def _start_dynamo_services(self) -> None: from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, ) - from nemo_rl.models.generation.dynamo.venv import get_dynamo_python runtime = ManagedDynamoRuntime( dynamo_cfg=dynamo_cfg(self.cfg), model_name=self.cfg["model_name"], - frontend_python=get_dynamo_python(), + # The workers' venv carries ai-dynamo, so the frontend runs the same + # release as every engine. + frontend_python=self.worker_group.py_executable, ) runtime.start() self._dynamo_runtime = runtime diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 35820c8bfe9..40990504ec6 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -415,12 +415,19 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: stacklevel=1, ) llm_kwargs["compilation_config"] = CompilationConfig(**compilation_config) + self.stat_loggers = ( + [PrometheusStatLogger] + if self.cfg["vllm_cfg"].get("enable_vllm_metrics_logger", False) + else [] + ) if dynamo_enabled(self.cfg): from nemo_rl.models.generation.dynamo.native_engine import ( + SnapshotStatLogger, dynamo_kv_events_config, ) - # Dynamo's KV-aware router indexes the engine's own KV-cache events. + # Dynamo's KV-aware router indexes the engine's own KV-cache events; + # the stat logger feeds its per-engine load signal. llm_kwargs.setdefault( "kv_events_config", dynamo_kv_events_config( @@ -428,6 +435,7 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: prefix_caching=llm_kwargs["enable_prefix_caching"], ), ) + self.stat_loggers.append(SnapshotStatLogger) # AsyncEngineArgs leaves kv_events_config dicts unconverted. if isinstance(llm_kwargs.get("kv_events_config"), dict): from vllm.config import KVEventsConfig @@ -437,18 +445,6 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: ) self.llm_async_engine_args = AsyncEngineArgs(**llm_kwargs) - self.stat_loggers = ( - [PrometheusStatLogger] - if self.cfg["vllm_cfg"].get("enable_vllm_metrics_logger", False) - else [] - ) - if dynamo_enabled(self.cfg): - from nemo_rl.models.generation.dynamo.native_engine import ( - SnapshotStatLogger, - ) - - # Feeds the router's per-engine load signal. - self.stat_loggers.append(SnapshotStatLogger) self.llm = AsyncLLM.from_engine_args( self.llm_async_engine_args, stat_loggers=self.stat_loggers ) diff --git a/nemo_rl/utils/routed_experts_codec.py b/nemo_rl/utils/routed_experts_codec.py index cb05d06b814..53544004c73 100644 --- a/nemo_rl/utils/routed_experts_codec.py +++ b/nemo_rl/utils/routed_experts_codec.py @@ -29,7 +29,7 @@ """ import base64 -from typing import Any, Union +from typing import Any, Optional, Union import torch @@ -67,11 +67,13 @@ def encode_routed_experts(routed_experts: torch.Tensor) -> str: return f"{_MAGIC}:{dtype_name}:{tokens}x{num_layers}x{topk}:{data}" -def decode_routed_experts(payload: Union[str, Any], dtype: torch.dtype) -> torch.Tensor: +def decode_routed_experts( + payload: Union[str, Any], dtype: Optional[torch.dtype] +) -> torch.Tensor: """Decode routed experts into a tensor of the requested dtype. Accepts the base64 envelope produced by ``encode_routed_experts`` or the - legacy nested-list format. + legacy nested-list format. ``dtype=None`` keeps the envelope's own dtype. """ if not isinstance(payload, str): return torch.as_tensor(payload, dtype=dtype) @@ -100,20 +102,14 @@ def decode_routed_experts(payload: Union[str, Any], dtype: torch.dtype) -> torch f"routed_experts envelope has {len(raw) // wire_dtype.itemsize} " f"elements, expected {expected} for shape {shape}." ) + if dtype is None: + dtype = wire_dtype if expected == 0: return torch.empty(shape, dtype=dtype) tensor = torch.frombuffer(raw, dtype=wire_dtype).reshape(shape) return tensor if tensor.dtype == dtype else tensor.to(dtype) -def routed_experts_envelope_dtype(payload: str) -> torch.dtype: - """The dtype a base64 envelope was encoded with.""" - parts = payload.split(":", 2) - if len(parts) != 3 or parts[0] != _MAGIC or parts[1] not in _WIRE_TORCH_DTYPES: - raise ValueError("routed_experts payload is not a valid envelope.") - return _WIRE_TORCH_DTYPES[parts[1]] - - def slice_routed_experts( routed_experts: torch.Tensor, start: int, end: int ) -> torch.Tensor: diff --git a/pyrefly.toml b/pyrefly.toml index 06a05c446bd..1845d9072e8 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -233,13 +233,10 @@ project-includes = [ "nemo_rl/models/generation/__init__.py", "nemo_rl/models/generation/constants.py", "nemo_rl/models/generation/dynamo/__init__.py", - "nemo_rl/models/generation/dynamo/arguments.py", "nemo_rl/models/generation/dynamo/config.py", - "nemo_rl/models/generation/dynamo/http_client.py", "nemo_rl/models/generation/dynamo/managed_runtime.py", "nemo_rl/models/generation/dynamo/native_engine.py", "nemo_rl/models/generation/dynamo/rollout_client.py", - "nemo_rl/models/generation/dynamo/venv.py", "nemo_rl/models/generation/engine_supervisor.py", "nemo_rl/models/generation/fleet_health.py", "nemo_rl/models/generation/generation_router.py", diff --git a/tests/test_suites/disabled.txt b/tests/test_suites/disabled.txt index 83b5213a584..2d43c913cc5 100644 --- a/tests/test_suites/disabled.txt +++ b/tests/test_suites/disabled.txt @@ -102,7 +102,3 @@ research/template_project/tests/test_suites/llm/single_update_1n8g.sh # Full Flow-GRPO recipe requires an externally served GenRM OCR judge. research/flow_grpo/tests/test_suites/flow_grpo_qwen_image_ocr.sh - -# A Dynamo + NeMo-Gym recipe: policy.generation.dynamo does not serve NeMo-Gym -# rollouts yet, and setup refuses the pairing. -tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh diff --git a/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh b/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh deleted file mode 100755 index e9cf7073a7c..00000000000 --- a/tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) -source "${SCRIPT_DIR}/common.env" - -# ===== BEGIN CONFIG ===== -NUM_NODES=3 -GPUS_PER_NODE=8 -STEPS_PER_RUN=4 -MAX_STEPS=4 -NUM_RUNS=1 -NUM_MINUTES=240 -USE_GYM_CONTAINER=true -# ===== END CONFIG ===== - -exit_if_max_steps_reached - -cd "${PROJECT_ROOT}" -uv run examples/nemo_gym/run_grpo_nemo_gym.py \ - --config "${CONFIG_PATH}" \ - grpo.max_num_steps="${MAX_STEPS}" \ - logger.log_dir="${LOG_DIR}" \ - logger.wandb_enabled=true \ - logger.wandb.project=nemo-rl \ - logger.wandb.name="${EXP_NAME}" \ - logger.tensorboard_enabled=true \ - "$@" \ - 2>&1 | tee "${RUN_LOG}" - -uv run tests/json_dump_tb_logs.py "${LOG_DIR}" \ - --output_path "${JSON_METRICS}" \ - --require-tag-prefix generation_metrics/ - -last_step=$(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' "${JSON_METRICS}") -if [[ ${last_step} -lt ${MAX_STEPS} ]]; then - echo "[ERROR] Expected step ${MAX_STEPS}, but the last train/loss step is ${last_step}" - exit 1 -fi - -uv run tests/check_metrics.py "${JSON_METRICS}" \ - 'median(data["train/token_mult_prob_error"]) < 1.1' \ - "data['train/token_mult_prob_error']['${MAX_STEPS}'] < 1.1" \ - 'mean(data["train/gen_kl_error"]) < 0.02' - -refit_count=$(grep -c "✅ Ready for refit" "${RUN_LOG}" || true) -cache_invalidation_count=$(grep -c \ - "✅ Invalidated generation backend KV caches after weight update" \ - "${RUN_LOG}" || true) -cache_invalidation_failure_count=$(grep -c \ - "Failed to invalidate generation backend KV caches" \ - "${RUN_LOG}" || true) -if [[ ${cache_invalidation_failure_count} -ne 0 ]]; then - echo "[ERROR] Found ${cache_invalidation_failure_count} cache invalidation failure(s)" - exit 1 -fi -if [[ ${refit_count} -ne ${cache_invalidation_count} ]]; then - echo "[ERROR] Expected one cache invalidation per refit, but found " \ - "${cache_invalidation_count} invalidations for ${refit_count} refits" - exit 1 -fi diff --git a/tests/unit/L0_Unit_Tests_Vllm_1.sh b/tests/unit/L0_Unit_Tests_Vllm_1.sh index 163caa97126..2f2badbcd7e 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_1.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_1.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=0 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/L0_Unit_Tests_Vllm_2.sh b/tests/unit/L0_Unit_Tests_Vllm_2.sh index 211b1086e0f..e44a28f472a 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_2.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_2.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --no-sync bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=1 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/L0_Unit_Tests_Vllm_3.sh b/tests/unit/L0_Unit_Tests_Vllm_3.sh index 362044f0c49..3b5300a828b 100644 --- a/tests/unit/L0_Unit_Tests_Vllm_3.sh +++ b/tests/unit/L0_Unit_Tests_Vllm_3.sh @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/run_unit_shard_common.sh" # Base run (tests without extra markers) -uv run --extra modelopt bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_swe1_dynamo_config.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated +uv run --extra modelopt bash -x ./tests/run_unit.sh "unit/models/generation/test_vllm*.py" "unit/models/generation/test_dynamo*.py" "unit/models/generation/test_openai_server_utils.py" "unit/models/generation/test_interfaces.py" "unit/models/generation/test_fleet_health.py" "unit/models/generation/test_generation_router.py" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-report=term-missing --cov-report=json --hf-gated # vllm-only run (catch-all across all unit tests) uv run --extra vllm bash -x ./tests/run_unit.sh "unit/" "${EXCLUDED_UNIT_TESTS[@]}" --shard-id=2 --num-shards=3 --cov=nemo_rl --cov-append --cov-report=term-missing --cov-report=json --hf-gated --vllm-only diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index c161e7289d2..75be2faa00e 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -14,7 +14,6 @@ import math import os -import warnings from concurrent.futures import Future from contextlib import ExitStack, contextmanager from pathlib import Path @@ -2633,7 +2632,6 @@ def test_initial_refit_completes_before_async_collection_starts( mock_grpo_components, ) -> None: master_config = mock_grpo_components["master_config"] - master_config.policy["generation"]["backend"] = "vllm" master_config.policy["generation"]["colocated"]["enabled"] = False master_config.grpo.max_num_steps = 1 master_config.grpo.val_period = 0 @@ -2672,7 +2670,6 @@ def record_refit(*args, **kwargs): def test_async_grpo_awaits_resume_after_refit_failure(mock_grpo_components) -> None: master_config = mock_grpo_components["master_config"] - master_config.policy["generation"]["backend"] = "vllm" master_config.policy["generation"]["colocated"]["enabled"] = False master_config.grpo.max_num_steps = 1 master_config.grpo.val_period = 0 @@ -4731,10 +4728,7 @@ def init_collective(self, *_args, **_kwargs): assert master_config.grpo.skip_reference_policy_logprobs_calculation is True -@pytest.mark.parametrize("configured_gym_log_dir", [None, "logs/nemo_gym"]) -def test_setup_starts_nemo_gym_for_trtllm( - monkeypatch, mock_grpo_components, configured_gym_log_dir -): +def test_setup_starts_nemo_gym_for_trtllm(monkeypatch, mock_grpo_components): """Guard the TRT-LLM NeMo-Gym startup path in shared GRPO setup.""" from nemo_rl.algorithms import grpo as grpo_mod @@ -4839,8 +4833,6 @@ def prepare_refit_info(self, _state): }, } master_config.env = {"should_use_nemo_gym": True} - if configured_gym_log_dir is not None: - master_config.env["nemo_gym"] = {"nemo_gym_log_dir": configured_gym_log_dir} master_config.loss_fn = ClippedPGLossConfig(reference_policy_kl_penalty=0.0) master_config.grpo.val_period = 0 master_config.grpo.batch_multiplier = 1 @@ -4851,26 +4843,13 @@ def prepare_refit_info(self, _state): dataset = MagicMock() dataset.__len__ = MagicMock(return_value=1) tokenizer = MagicMock() - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = grpo_mod.setup(master_config, tokenizer, dataset, None) + result = grpo_mod.setup(master_config, tokenizer, dataset, None) assert result[2] is nemo_gym_shard_set + # Gym server logs go under the run's log directory. assert master_config.env["nemo_gym"]["nemo_gym_log_dir"] == ( "/tmp/grpo-test-results/nemo_gym" ) - log_dir_warnings = [ - str(warning.message) - for warning in caught - if "nemo_gym_log_dir" in str(warning.message) - ] - if configured_gym_log_dir is None: - assert log_dir_warnings == [] - else: - # A configured log dir is still replaced, but no longer silently. - assert len(log_dir_warnings) == 1 - assert "'logs/nemo_gym'" in log_dir_warnings[0] - assert "'/tmp/grpo-test-results/nemo_gym'" in log_dir_warnings[0] build_nemo_gym_actors.assert_called_once_with( master_config.env, base_urls=["http://trtllm.example/v1"], @@ -8382,28 +8361,6 @@ def test_needs_hf_refit_handshake(backend, nccl_reshard, colocated, expected): assert _needs_hf_refit_handshake(backend, nccl_reshard, colocated) is expected -@pytest.mark.parametrize( - ("nccl_reshard", "offload_policy", "expected"), - [ - (False, False, False), - (False, True, True), - (True, False, True), - ], -) -def test_noncolocated_refit_synchronizer_selection( - nccl_reshard, offload_policy, expected -): - from nemo_rl.algorithms import grpo as grpo_mod - - assert ( - grpo_mod._uses_managed_noncolocated_refit( - nccl_reshard_refit_enabled=nccl_reshard, - offload_policy_before_refit=offload_policy, - ) - is expected - ) - - def test_grpo_train_shuts_down_environments_after_failure(): task_to_env = {"nemo_gym": MagicMock()} val_task_to_env = task_to_env diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 0624535b30c..e552d545578 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -11,16 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path + import pytest -from pydantic import ValidationError +import yaml from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.arguments import ( - build_dynamo_frontend_argv, - redact_argv, - redact_environment, -) from nemo_rl.models.generation.dynamo.config import DynamoCfg, dynamo_cfg +from nemo_rl.models.generation.dynamo.managed_runtime import frontend_argv class _Tokenizer: @@ -45,23 +43,24 @@ def _vllm_config(**overrides) -> dict: return config -def test_dynamo_true_alone_is_a_complete_config() -> None: - config = configure_generation_config(_vllm_config(dynamo=True), _Tokenizer()) - - # Dynamo serves the AsyncLLM, so the switch selects the async engine. - # Rollouts read token IDs only, so the engine skips its tokenizer as natively. - assert config["vllm_cfg"]["async_engine"] is True - assert config["vllm_cfg"]["skip_tokenizer_init"] is True - assert dynamo_cfg(config) == DynamoCfg() - assert dynamo_cfg(config).frontend_args.router_mode == "kv" +_EXEMPLAR = ( + Path(__file__).resolve().parents[4] + / "examples/configs/grpo_math_1B_dynamo_single_controller.yaml" +) -def test_without_dynamo_the_vllm_config_is_untouched() -> None: - config = configure_generation_config(_vllm_config(), _Tokenizer()) +@pytest.mark.parametrize("dynamo", [False, True]) +def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: + config = configure_generation_config(_vllm_config(dynamo=dynamo), _Tokenizer()) - assert config["vllm_cfg"]["async_engine"] is False + # Dynamo serves the AsyncLLM, so the switch selects the async engine. + # Rollouts read token IDs only, so the engine skips its tokenizer as natively. + assert config["vllm_cfg"]["async_engine"] is dynamo assert config["vllm_cfg"]["skip_tokenizer_init"] is True assert "dynamo_cfg" not in config + # The exemplar spells out exactly the defaults. + exemplar = yaml.safe_load(_EXEMPLAR.read_text())["policy"]["generation"] + assert dynamo_cfg(config) == DynamoCfg() == dynamo_cfg(exemplar) @pytest.mark.parametrize( @@ -118,18 +117,11 @@ def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( configure_generation_config(config, _Tokenizer()) -def test_dynamo_cfg_rejects_unknown_keys() -> None: - with pytest.raises(ValidationError, match="worker_args"): - dynamo_cfg({"dynamo_cfg": {"worker_args": {}}}) - - def test_frontend_argv_is_runtime_owned() -> None: cfg = DynamoCfg.model_validate( {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} ) - argv = build_dynamo_frontend_argv( - host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg - ) + argv = frontend_argv(host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg) assert argv[argv.index("--router-mode") + 1] == "kv" assert argv[-2:] == ["--foo", "1"] @@ -137,14 +129,6 @@ def test_frontend_argv_is_runtime_owned() -> None: {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} ) with pytest.raises(ValueError, match="--http-port is set by both"): - build_dynamo_frontend_argv( + frontend_argv( host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=clashing ) - - -def test_redaction_hides_credentials() -> None: - assert redact_argv(["frontend", "--api-key", "secret"])[2] == "" - assert redact_environment({"HF_TOKEN": "secret", "NCCL_DEBUG": "INFO"}) == { - "HF_TOKEN": "", - "NCCL_DEBUG": "INFO", - } diff --git a/tests/unit/models/generation/test_dynamo_http_client.py b/tests/unit/models/generation/test_dynamo_http_client.py deleted file mode 100644 index df8eabd9344..00000000000 --- a/tests/unit/models/generation/test_dynamo_http_client.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio - -import aiohttp -import pytest - -from nemo_rl.models.generation.dynamo import http_client -from nemo_rl.models.generation.dynamo.rollout_client import _is_retryable - - -class _AsyncResponse: - def __init__(self, body: bytes, status: int = 200): - self._body = body - self.status = status - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - return None - - async def read(self) -> bytes: - return self._body - - -class _AsyncSession: - def __init__(self, response=None, error=None): - self._response = response - self._error = error - self.requests = [] - - async def __aenter__(self): - return self - - async def __aexit__(self, *args): - return None - - def post(self, url, *, json): - self.requests.append((url, json)) - if self._error is not None: - raise self._error - return self._response - - -@pytest.mark.parametrize( - ("body", "status", "expected", "retryable"), - [ - (b'{"status":"ok"}', 200, {"status": "ok"}, False), - ( - b"not-json", - 200, - {"status": "error", "json_decode_error": True, "raw": "not-json"}, - True, - ), - ( - b"busy", - 503, - {"status": "error", "http_status": 503, "raw": "busy"}, - True, - ), - ], -) -def test_async_http_post_json_error_contract( - monkeypatch, body, status, expected, retryable -) -> None: - session = _AsyncSession(_AsyncResponse(body, status)) - monkeypatch.setattr( - http_client.aiohttp, - "ClientSession", - lambda **kwargs: session, - ) - - # No deadline is the default (dynamo_cfg.request_timeout_s: null). - response = asyncio.run( - http_client.async_http_post_json( - "http://worker/route", {"value": 1}, timeout_s=None - ) - ) - - assert response == expected - assert _is_retryable(response) is retryable - if response.get("status") == "error": - assert expected["raw"] in http_client.format_dynamo_error(response) - assert session.requests == [("http://worker/route", {"value": 1})] - - -@pytest.mark.parametrize( - ("error", "key", "retryable"), - [ - (aiohttp.ClientConnectionError("refused"), "transport_error", True), - # One request_timeout_s bounds a rollout: a timeout is not retried. - (TimeoutError("slow"), "timeout_error", False), - (aiohttp.ServerTimeoutError("slow read"), "timeout_error", False), - ], -) -def test_async_http_transport_errors(monkeypatch, error, key, retryable) -> None: - session = _AsyncSession(error=error) - monkeypatch.setattr( - http_client.aiohttp, - "ClientSession", - lambda **kwargs: session, - ) - - response = asyncio.run( - http_client.async_http_post_json( - "http://worker/route", {"value": 1}, timeout_s=3 - ) - ) - - assert response["status"] == "error" - assert _is_retryable(response) is retryable - assert key in response - assert response[key] in http_client.format_dynamo_error(response) diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 524416b344b..2c5c4dc3541 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -16,15 +16,14 @@ import subprocess import sys from pathlib import Path -from unittest.mock import MagicMock import pytest -from nemo_rl.models.generation.dynamo import venv as dynamo_venv from nemo_rl.models.generation.dynamo.config import DynamoCfg from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, _managed_namespace, + _service_executable, ) @@ -139,54 +138,20 @@ def test_managed_service_and_frontend_environments_are_runtime_owned( assert "DYN_SYSTEM_PORT" not in service_env -def test_dynamo_python_resolves_the_vllm_worker_venv_as_workers_do( - monkeypatch, -) -> None: - built = [] - monkeypatch.setattr( - dynamo_venv, "get_actor_python_env", lambda fqn: "uv run --locked --extra vllm" - ) - # The workers' resolver: reuses an existing venv, builds a missing one per node. - monkeypatch.setattr( - dynamo_venv, - "create_local_venv_on_each_node", - lambda py_executable, venv_name: built.append((py_executable, venv_name)) - or "/venvs/vllm/bin/python", - ) - - assert dynamo_venv.get_dynamo_python() == "/venvs/vllm/bin/python" - assert built == [ - ("uv run --locked --extra vllm", dynamo_venv.VLLM_ASYNC_WORKER_FQN) - ] - - -def test_dynamo_python_passes_an_interpreter_through(monkeypatch) -> None: - monkeypatch.setattr( - dynamo_venv, "get_actor_python_env", lambda fqn: "/usr/bin/python3" - ) - monkeypatch.setattr( - dynamo_venv, - "create_local_venv_on_each_node", - MagicMock(side_effect=AssertionError("no venv for an interpreter")), - ) - - assert dynamo_venv.get_dynamo_python() == "/usr/bin/python3" - - def test_dynamo_services_resolve_from_bin_dir_then_path(monkeypatch, tmp_path) -> None: etcd = tmp_path / "etcd" etcd.write_text("#!/bin/sh\n") etcd.chmod(0o755) monkeypatch.setenv("NEMO_RL_DYNAMO_BIN_DIR", str(tmp_path)) - assert dynamo_venv.get_dynamo_service_executable("etcd") == str(etcd) + assert _service_executable("etcd") == str(etcd) with pytest.raises(FileNotFoundError, match="NEMO_RL_DYNAMO_BIN_DIR"): - dynamo_venv.get_dynamo_service_executable("nats-server") + _service_executable("nats-server") monkeypatch.delenv("NEMO_RL_DYNAMO_BIN_DIR") monkeypatch.setenv("PATH", str(tmp_path)) - assert dynamo_venv.get_dynamo_service_executable("etcd") == str(etcd) - with pytest.raises(FileNotFoundError, match="not on PATH"): - dynamo_venv.get_dynamo_service_executable("nats-server") + assert _service_executable("etcd") == str(etcd) + with pytest.raises(FileNotFoundError, match="on PATH"): + _service_executable("nats-server") def test_stop_process_escalates_its_process_group(monkeypatch) -> None: diff --git a/tests/unit/models/generation/test_dynamo_rollout_client.py b/tests/unit/models/generation/test_dynamo_rollout_client.py index 138b4120017..6aeab3a07a6 100644 --- a/tests/unit/models/generation/test_dynamo_rollout_client.py +++ b/tests/unit/models/generation/test_dynamo_rollout_client.py @@ -13,6 +13,7 @@ # limitations under the License. import asyncio +import aiohttp import pytest from nemo_rl.models.generation.dynamo import rollout_client @@ -22,6 +23,67 @@ ) +class _Response: + def __init__(self, status: int, body: bytes): + self.status = status + self._body = body + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + async def read(self) -> bytes: + return self._body + + +class _Session: + def __init__(self, reply): + self._reply = reply + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + def post(self, url, *, json): + if isinstance(self._reply, Exception): + raise self._reply + return _Response(*self._reply) + + +@pytest.mark.parametrize( + ("reply", "error", "retryable"), + [ + ((200, b'{"status":"ok"}'), None, False), + ((200, b"not-json"), "not-json", True), + ((503, b"busy"), "HTTP 503: busy", True), + (aiohttp.ClientConnectionError("refused"), "refused", True), + # One request_timeout_s bounds a rollout: a timeout is not retried. + (TimeoutError("slow"), "no response within", False), + (aiohttp.ServerTimeoutError("slow read"), "no response within", False), + ], +) +def test_post_json_reports_failures_as_error_dicts( + monkeypatch, reply, error, retryable +) -> None: + monkeypatch.setattr( + rollout_client.aiohttp, "ClientSession", lambda **kwargs: _Session(reply) + ) + + # No deadline is the default (dynamo_cfg.request_timeout_s: null). + response = asyncio.run( + rollout_client._post_json("http://frontend/v1/completions", {}, timeout_s=None) + ) + + assert (response.get("status") == "error") is (error is not None) + if error is not None: + assert error in rollout_client._format_error(response) + assert rollout_client._is_retryable(response) is retryable + + @pytest.mark.parametrize( "response, match", [ @@ -73,7 +135,7 @@ async def fake_post(url, payload, timeout_s): calls.append(payload) return responses[len(calls) - 1] - monkeypatch.setattr(rollout_client, "async_http_post_json", fake_post) + monkeypatch.setattr(rollout_client, "_post_json", fake_post) monkeypatch.setattr(rollout_client, "_HTTP_RETRY_DELAY_S", 0.0) rollout = post_rollout("http://frontend/v1/completions", {}, timeout_s=1.0) diff --git a/tests/unit/models/generation/test_swe1_dynamo_config.py b/tests/unit/models/generation/test_swe1_dynamo_config.py deleted file mode 100644 index 9bd87f972b1..00000000000 --- a/tests/unit/models/generation/test_swe1_dynamo_config.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. - -from pathlib import Path -from types import SimpleNamespace - -import pytest -from omegaconf import OmegaConf - -from nemo_rl.environments.nemo_gym import setup_nemo_gym_config -from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.config import dynamo_cfg -from nemo_rl.utils.config import load_config, register_omegaconf_resolvers - -REPO_ROOT = Path(__file__).resolve().parents[4] -RECIPE = ( - REPO_ROOT / "examples/configs/recipes/llm/" - "grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.yaml" -) -DRIVER = ( - REPO_ROOT / "tests/test_suites/llm/grpo-nanov3-30ba3b-3n8g-megatron-dynamo-swe1.sh" -) - - -def _load_recipe() -> dict: - register_omegaconf_resolvers() - return OmegaConf.to_container(load_config(RECIPE), resolve=True) - - -def test_public_swe_recipe_has_supported_topology_and_telemetry( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("HF_HOME", "/hf-home") - config = _load_recipe() - generation = config["policy"]["generation"] - - class Tokenizer: - pad_token_id = 0 - eos_token_id = 1 - - tokenizer = Tokenizer() - configured_generation = configure_generation_config(generation, tokenizer) - config["policy"]["generation"] = configured_generation - setup_nemo_gym_config( - SimpleNamespace( - policy=config["policy"], env=config["env"], data=config["data"] - ), - tokenizer, - ) - vllm_cfg = configured_generation["vllm_cfg"] - assert configured_generation["backend"] == "vllm" - assert configured_generation["dynamo"] is True - assert vllm_cfg["async_engine"] is True - assert vllm_cfg["expose_http_server"] is True - - assert config["policy"]["model_name"] == ( - "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" - ) - assert config["cluster"]["gpus_per_node"] == 8 - assert config["cluster"]["num_nodes"] == 3 - assert config["cluster"]["segment_size"] == 1 - assert config["grpo"]["max_num_steps"] == 4 - assert config["grpo"]["async_grpo"]["enabled"] is True - assert config["grpo"]["async_grpo"]["in_flight_weight_updates"] is False - assert ( - config["grpo"]["async_grpo"]["recompute_kv_cache_after_weight_updates"] is True - ) - assert generation["colocated"]["resources"] == { - "gpus_per_node": 8, - "num_nodes": 1, - } - assert vllm_cfg["tensor_parallel_size"] * vllm_cfg["pipeline_parallel_size"] == 4 - assert vllm_cfg["expert_parallel_size"] == 4 - assert dynamo_cfg(configured_generation).frontend_args.router_mode == "kv" - assert vllm_cfg["enable_vllm_metrics_logger"] is True - assert vllm_cfg["load_format"] == "dummy" - assert config["env"]["nemo_gym"]["config_paths"][-1].endswith( - "swe_pivot_single_step_tool_use_with_argument_comparison.yaml" - ) - assert ( - config["env"]["nemo_gym"]["policy_model"]["responses_api_models"]["vllm_model"][ - "chat_template_kwargs" - ]["force_nonempty_content"] - is True - ) - assert ( - config["env"]["nemo_gym"]["single_step_tool_use_with_argument_comparison_swe"][ - "responses_api_agents" - ]["tool_simulation_agent"]["resources_server"]["name"] - == "swe_pivot_single_step_tool_use_with_argument_comparison_resources_server" - ) - assert config["logger"]["wandb_enabled"] is True - assert config["logger"]["tensorboard_enabled"] is True - assert config["logger"]["wandb"]["project"] == "nemo-rl" - assert config["data"]["train"]["data_path"].endswith( - "/superv3_data/swe1/train-split.jsonl" - ) - assert config["data"]["validation"]["data_path"].endswith( - "/superv3_data/swe1/val-split.jsonl" - ) - - -def test_recipe_and_driver_have_no_removed_swe_modes() -> None: - recipe_text = RECIPE.read_text(encoding="utf-8") - text = recipe_text + DRIVER.read_text(encoding="utf-8") - for forbidden in ( - "/lustre/", - "/path/to", - "/home/", - "jthomson", - "openhands", - "container_formatter", - "SIF_", - "effort_levels", - "hsg_r2", - "USES_SANDBOX", - ): - assert forbidden.lower() not in text.lower() - assert "load_format:" not in recipe_text - assert "--require-tag-prefix generation_metrics/" in text - assert "project: nemo-rl" in text - assert "Expected step ${MAX_STEPS}" in text - assert 'median(data["train/token_mult_prob_error"]) < 1.1' in text - assert "data['train/token_mult_prob_error']['${MAX_STEPS}'] < 1.1" in text - assert 'mean(data["train/gen_kl_error"]) < 0.02' in text - assert "Expected one cache invalidation per refit" in text diff --git a/tests/unit/test_dependency_pins.py b/tests/unit/test_dependency_pins.py index 17c14b55aae..864ee4d40ba 100644 --- a/tests/unit/test_dependency_pins.py +++ b/tests/unit/test_dependency_pins.py @@ -144,21 +144,3 @@ def test_torch_split_shims_are_still_needed() -> None: "(nemo_rl/data_plane/adapters/tq_mooncake_checkpoint.py) - drop the " "check outright; the pre-2.13 `names` comparison cannot be restored" ) - - -def test_vllm_extra_carries_base_dynamo_release() -> None: - """Dynamo serving runs NeMo RL's own vLLM workers, so the vllm extra carries the - released Dynamo runtime but never ai-dynamo[vllm], which pins Dynamo's vLLM.""" - extras = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text())["project"][ - "optional-dependencies" - ] - requirements = { - canonicalize_name(req.name): req - for req in (Requirement(dep) for dep in extras["vllm"]) - } - - dynamo, runtime = requirements["ai-dynamo"], requirements["ai-dynamo-runtime"] - assert not dynamo.extras, f"ai-dynamo must not take extras, got {dynamo.extras}" - assert dynamo.specifier == runtime.specifier, ( - "ai-dynamo and ai-dynamo-runtime must pin the same release" - ) diff --git a/tests/unit/utils/test_outdated_config_checks.py b/tests/unit/utils/test_outdated_config_checks.py index 4539d34b951..13757622a81 100644 --- a/tests/unit/utils/test_outdated_config_checks.py +++ b/tests/unit/utils/test_outdated_config_checks.py @@ -378,8 +378,15 @@ def test_dynamo_frontend_args_without_router_reset_states_pass(): check_outdated_config(_dynamo_frontend_config({"router_mode": "kv"})) -def test_generation_without_dynamo_cfg_passes(): - check_outdated_config({"policy": {"generation": {"backend": "vllm"}}}) +@pytest.mark.parametrize( + "generation", + [ + {"backend": "vllm"}, + {"backend": "vllm", "dynamo": True, "dynamo_cfg": {"frontend_args": {}}}, + ], +) +def test_generation_without_removed_dynamo_keys_passes(generation): + check_outdated_config({"policy": {"generation": generation}}) # ============================================================================ @@ -391,40 +398,32 @@ def _generation_config(**generation) -> dict: return {"policy": {"generation": generation}} -def test_dynamo_backend_is_rejected_with_the_migration(): - with pytest.raises( - ValueError, match=r"backend=vllm and policy\.generation\.dynamo=true" - ): - check_outdated_config(_generation_config(backend="dynamo")) - - @pytest.mark.parametrize( - "dynamo_cfg, removed", + "generation, removed", [ - ({"engine": "vllm"}, r"dynamo_cfg\.engine"), - ({"control_timeout_s": 600}, r"dynamo_cfg\.control_timeout_s"), - ({"metrics_include_prefixes": None}, r"dynamo_cfg\.metrics_include_prefixes"), - ({"metrics_exclude_prefixes": None}, r"dynamo_cfg\.metrics_exclude_prefixes"), - ({"worker_args": {"extra_cli_args": []}}, r"dynamo_cfg\.worker_args was"), - ({"worker_args": {"reasoning_parser": None}}, r"dynamo_cfg\.worker_args was"), + ({"backend": "dynamo"}, r"backend=vllm and policy\.generation\.dynamo=true"), + ({"dynamo_cfg": {"engine": "vllm"}}, r"dynamo_cfg\.engine"), + ({"dynamo_cfg": {"control_timeout_s": 600}}, r"dynamo_cfg\.control_timeout_s"), + ( + {"dynamo_cfg": {"metrics_include_prefixes": None}}, + r"dynamo_cfg\.metrics_include_prefixes", + ), ( - {"frontend_args": {"tokenizer_cache": False}}, + {"dynamo_cfg": {"metrics_exclude_prefixes": None}}, + r"dynamo_cfg\.metrics_exclude_prefixes", + ), + ( + {"dynamo_cfg": {"worker_args": {"extra_cli_args": []}}}, + r"dynamo_cfg\.worker_args was", + ), + ( + {"dynamo_cfg": {"frontend_args": {"tokenizer_cache": False}}}, r"frontend_args\.tokenizer_cache was", ), ], ) -def test_removed_dynamo_cfg_keys_are_rejected(dynamo_cfg, removed): +def test_removed_dynamo_config_is_rejected(generation, removed): with pytest.raises(ValueError, match=removed): check_outdated_config( - _generation_config(backend="vllm", dynamo=True, dynamo_cfg=dynamo_cfg) + _generation_config(**{"backend": "vllm", "dynamo": True, **generation}) ) - - -def test_vllm_backend_with_dynamo_passes(): - check_outdated_config( - _generation_config( - backend="vllm", - dynamo=True, - dynamo_cfg={"frontend_args": {"router_mode": "kv"}}, - ) - ) From e798c868c4cbbf24f0e42540858f05d3aa33eca7 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 3 Nov 2026 09:48:36 -0800 Subject: [PATCH 12/19] refactor(vllm): address review nits - check_sampling_param replaces off_policy_sampling_error and raises ValueError itself, with the same messages in the same order. It raises rather than asserts so a server that answers an off-policy request with a client error can catch it; the native chat handler still fails with a 500, as its assert did. - Shorten the route-encoding and sampling-profile comments to one line. - The pipeline docstrings name their actual callers and what the final RequestOutput is for. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/vllm/request_pipeline.py | 5 +--- nemo_rl/models/generation/vllm/utils.py | 25 +++++++++++-------- .../generation/vllm/vllm_worker_async.py | 14 +++-------- .../unit/models/generation/test_vllm_utils.py | 23 ++++++++++------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index 9e5bea46416..62b7332f377 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -15,10 +15,7 @@ """Turn a finished vLLM rollout into NeMo RL's per-sample generation output. The worker's ``_serve_rollout`` builds the result; ``rollout_output`` assembles -the output tensors from it. The two are apart so a host that builds the result -on the engine and assembles the output elsewhere returns what ``generate_async`` -returns. A rollout feature added here, such as router replay (R3), then behaves -the same on every path. +its output tensors for ``generate_async``. """ import logging diff --git a/nemo_rl/models/generation/vllm/utils.py b/nemo_rl/models/generation/vllm/utils.py index 99b7fa93c05..6ca3c3cdf0d 100644 --- a/nemo_rl/models/generation/vllm/utils.py +++ b/nemo_rl/models/generation/vllm/utils.py @@ -321,14 +321,14 @@ def _as_routed_experts_tensor( return tensor.to(dtype=dtype) -def off_policy_sampling_error( +def check_sampling_param( *, temperature: Optional[float], top_p: Optional[float], top_k: Optional[int], generation_config: Any, -) -> Optional[str]: - """Why an HTTP rollout request would sample off-policy, or None if it would not. +) -> None: + """Reject an HTTP rollout request that would sample off-policy. Request sampling must match NeMo RL's exactly, or inference runs off-policy and destabilizes training. Validation rollouts are the one exception: they carry the @@ -347,11 +347,17 @@ def off_policy_sampling_error( once the caller sends the request with top_k=-1. generation_config: ``policy.generation``, with the train and validation sampling parameters. + + Raises: + ValueError: The request's top_k is set, its top_p is unset, or its + temperature and top_p match neither sampling profile. """ if top_k not in (None, -1): - return f"Top k sampling parameter must be unset, empty, or -1. Got `{top_k}`" + raise ValueError( + f"Top k sampling parameter must be unset, empty, or -1. Got `{top_k}`" + ) if top_p is None: - return ( + raise ValueError( "top_p must be set explicitly on NeMo-RL requests; an unset top_p is " "resolved by vLLM from the model's generation_config.json and would " "bypass the on-policy sampling check." @@ -365,8 +371,8 @@ def off_policy_sampling_error( and top_p == generation_config["val_top_p"] ) if is_train_sampling or is_val_sampling: - return None - return ( + return + raise ValueError( f"request sampling (temperature={temperature}, top_p={top_p}) matches neither " f"the train sampling params (temperature={generation_config['temperature']}, " f"top_p={generation_config['top_p']}) nor the validation sampling params " @@ -596,10 +602,7 @@ def chat_routed_experts( r3_stats["actual_routes"], r3_stats["expected_routes"], ) - # Base64 envelope instead of .tolist(): nested JSON int lists cost - # ~1s of CPU per serialize/parse hop at long context lengths and get - # re-validated at every gym HTTP hop; a single string passes through - # the gym chain opaquely. + # Keep routes opaque across Gym HTTP hops to avoid nested-list validation. return encode_routed_experts(routed_experts.to(dtype=routed_experts_dtype)) diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 05b5d215ae6..fb342cba95b 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -77,7 +77,7 @@ attach_token_information_to_chat_response_choices, attach_sampler_topk_logprobs_to_chat_response_choices, format_prompt_for_vllm_generation, - off_policy_sampling_error, + check_sampling_param, validate_rollout_prompt, model_dump_chat_response_with_dynamic_message_fields, remap_multimodal_placeholders, @@ -1635,17 +1635,13 @@ class NeMoRLOnlineRenderer(NeMoRLOpenAIServingMixin, OnlineRenderer): async def create_chat_completion( request: NeMoRLChatCompletionRequest, raw_request: Request ): - # The request sampling params need to exactly match those as are set in NeMo RL. - # If they do not match, the inference will be off policy and destroy training - # stability. grpo.validate() is the only caller that constructs a non-train - # GenerationSamplingParams, and the validation profile it stamps is accepted. - sampling_error = off_policy_sampling_error( + # Accept train and validation profiles; reject off-policy requests. + check_sampling_param( temperature=request.temperature, top_p=request.top_p, top_k=request.top_k, generation_config=generation_config, ) - assert sampling_error is None, sampling_error # This needs to match the behavior in nemo_rl/models/generation/vllm/vllm_worker.py::BaseVllmGenerationWorker::_build_sampling_params request.top_k = -1 @@ -2007,9 +2003,6 @@ async def _serve_rollout( ) -> tuple[RolloutResult, Optional[Any]]: """Serve one rollout from this engine: its budget, sampling, generation and result. - Every path that serves a rollout from this engine calls this, so a rollout - step added here runs on all of them. - Args: prompt: The vLLM prompt to generate from. prompt_token_ids: The learner's prompt tokens, which vLLM must have run. @@ -2021,6 +2014,7 @@ async def _serve_rollout( Returns: The result, and vLLM's final RequestOutput (None when no new token fits). + The final output supplies finish and usage metadata to Dynamo in PR #4905. """ max_new_tokens = self._rollout_max_new_tokens(len(prompt_token_ids)) if max_new_tokens == 0: diff --git a/tests/unit/models/generation/test_vllm_utils.py b/tests/unit/models/generation/test_vllm_utils.py index dcc72dc02ba..240cc592e01 100644 --- a/tests/unit/models/generation/test_vllm_utils.py +++ b/tests/unit/models/generation/test_vllm_utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import math from dataclasses import dataclass from types import SimpleNamespace @@ -39,7 +40,7 @@ encode_counter_key, format_prompt_for_vllm_generation, model_dump_chat_response_with_dynamic_message_fields, - off_policy_sampling_error, + check_sampling_param, pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) @@ -1531,17 +1532,21 @@ def test_remap_multimodal_placeholders_rebuilds_real_vllm_placeholder_range(): (1.0, 1.0, 5, "Top k sampling parameter"), ], ) -def test_off_policy_sampling_error(temperature, top_p, top_k, error) -> None: +def test_check_sampling_param(temperature, top_p, top_k, error) -> None: generation_config = { "temperature": 1.0, "top_p": 1.0, "val_temperature": 0.0, "val_top_p": 0.9, } - message = off_policy_sampling_error( - temperature=temperature, - top_p=top_p, - top_k=top_k, - generation_config=generation_config, - ) - assert (message is None) if error is None else (error in message) + with ( + contextlib.nullcontext() + if error is None + else pytest.raises(ValueError, match=error) + ): + check_sampling_param( + temperature=temperature, + top_p=top_p, + top_k=top_k, + generation_config=generation_config, + ) From d67311fc3c3e5bdea4d0c4005ca241b8164010a1 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 3 Nov 2026 11:41:16 -0800 Subject: [PATCH 13/19] style(vllm): sort check_sampling_param import Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/models/generation/test_vllm_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/models/generation/test_vllm_utils.py b/tests/unit/models/generation/test_vllm_utils.py index 240cc592e01..ad1bfdb3803 100644 --- a/tests/unit/models/generation/test_vllm_utils.py +++ b/tests/unit/models/generation/test_vllm_utils.py @@ -35,12 +35,12 @@ attach_routed_experts_to_chat_response_choices, attach_sampler_topk_logprobs_to_chat_response_choices, attach_token_information_to_chat_response_choices, + check_sampling_param, compute_engine_step_metrics, compute_spec_decode_metrics, encode_counter_key, format_prompt_for_vllm_generation, model_dump_chat_response_with_dynamic_message_fields, - check_sampling_param, pad_and_align_routed_expert_indices, remap_multimodal_placeholders, ) From d15b6f40bc8c5100932cefad4e36c15dfb3f73e1 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Tue, 3 Nov 2026 20:55:21 -0800 Subject: [PATCH 14/19] test(vllm): expect ValueError from the chat handler's sampling check check_sampling_param raises ValueError since the review rename. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- tests/unit/models/generation/test_vllm_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 6ffcbf52e14..fb9f6c6ff11 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1499,13 +1499,13 @@ async def create_chat_completion(_request, _raw_request): @pytest.mark.asyncio async def test_chat_handler_rejects_off_policy_sampling_and_pins_top_k(monkeypatch): - """The route asserts on-policy sampling, then hands vLLM top_k=-1.""" + """The route rejects off-policy sampling, then hands vLLM top_k=-1.""" _, serving_chat, chat_handler = _setup_fake_vllm_chat_handler(monkeypatch) serving_chat.create_chat_completion = AsyncMock(return_value=iter(())) off_policy = _fake_chat_request() off_policy.temperature = 0.5 - with pytest.raises(AssertionError, match="matches neither"): + with pytest.raises(ValueError, match="matches neither"): await chat_handler(off_policy, MagicMock()) serving_chat.create_chat_completion.assert_not_awaited() From ee813c3ba1381d962959031c6c9f1ed65d11542f Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 5 Nov 2026 20:13:40 -0800 Subject: [PATCH 15/19] fix(dynamo): address round-3 review - Count only stored-block KV events: every prefix-cache reset applies a payload-free "cleared" event, which hid a router that decoded no block event (C46). - Register the engine with a weight-free snapshot from Dynamo's fetch_model(ignore_weights=True): the engine loads dummy weights and gets its real ones by refit, so no node downloads the checkpoint (C47). - Send a RuntimeError from _serve_rollout back as InvalidArgument, so the driver sees the cause and does not regenerate it twice more (C48). - Ship non-finite log probabilities as strings, since Dynamo's frontend carries engine_data as strict JSON (C49). - Name Dynamo's main-attention kinds as a module constant (C55). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/dynamo/managed_runtime.py | 9 ++-- .../models/generation/dynamo/native_engine.py | 43 ++++++++++++------- .../generation/vllm/request_pipeline.py | 7 ++- .../generation/vllm/vllm_worker_async.py | 12 ++++++ 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index 273d595c014..ee4f0383836 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -97,9 +97,12 @@ def _service_executable(name: str) -> str: # The KV router's applied-event counter (Dynamo's kvrouter.KV_CACHE_EVENTS_APPLIED), -# one sample per label set. +# one sample per label set. Stored-block events only: every prefix-cache reset +# emits a payload-free "cleared" event, which applies even when the router +# decodes none of vLLM's block events. _KV_EVENTS_APPLIED = re.compile( - r'^\w*kv_cache_events_applied(?:_total)?\{(?=[^}]*status="(?P[^"]*)")' + r'^\w*kv_cache_events_applied(?:_total)?\{(?=[^}]*event_type="stored")' + r'(?=[^}]*status="(?P[^"]*)")' r"[^}]*\}\s+(?P\S+)$", re.MULTILINE, ) @@ -406,7 +409,7 @@ def _assert_services_alive(self) -> None: ) def _log_router_kv_events(self) -> None: - """Log how many engine KV-cache events the KV router applied, by status. + """Log how many stored-block KV events the KV router applied, by status. Read from the frontend's metrics before it stops. A KV router that never applied one routed on load alone. diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/native_engine.py index b422d21c11e..23937f93354 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/native_engine.py @@ -61,6 +61,15 @@ LOGGER = logging.getLogger(__name__) +# vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them +# (dynamo/vllm/cache_info.py MAIN_ATTENTION_KV_CACHE_KINDS, which NeMo RL may +# not import); test_main_attention_kinds_match_dynamos checks the copy. +_MAIN_ATTENTION_KINDS = ( + KVCacheSpecKind.FULL_ATTENTION, + KVCacheSpecKind.MLA_ATTENTION, + KVCacheSpecKind.SINK_FULL_ATTENTION, +) + def dynamo_kv_events_config( cfg: DynamoCfg, *, prefix_caching: bool @@ -158,12 +167,7 @@ async def start(self, worker_id: int) -> EngineConfig: for group in await self._llm.engine_core.call_utility_async( "get_kv_cache_group_metadata" ): - # vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them. - if group.get("kind") in ( - KVCacheSpecKind.FULL_ATTENTION, - KVCacheSpecKind.MLA_ATTENTION, - KVCacheSpecKind.SINK_FULL_ATTENTION, - ): + if group.get("kind") in _MAIN_ATTENTION_KINDS: block_size = group["block_size"] break model = self._host.model_name @@ -201,14 +205,19 @@ async def _generate_rollout( the driver assembles the same output ``generate_async`` returns. """ prompt_token_ids = list(request["token_ids"]) - result, final_output = await self._host._serve_rollout( - TokensPrompt(prompt_token_ids=prompt_token_ids), - prompt_token_ids=prompt_token_ids, - greedy=rollout["greedy"], - stop_strings=rollout["stop_strings"], - request_id=context.id(), - device=torch.device("cpu"), - ) + try: + result, final_output = await self._host._serve_rollout( + TokensPrompt(prompt_token_ids=prompt_token_ids), + prompt_token_ids=prompt_token_ids, + greedy=rollout["greedy"], + stop_strings=rollout["stop_strings"], + request_id=context.id(), + device=torch.device("cpu"), + ) + except RuntimeError as error: + # A 5xx reaches the driver as "Internal server error" and is re-sent; + # a 4xx keeps the cause, and the rollout retry policy still applies. + raise InvalidArgument(f"{type(error).__name__}: {error}") from error chunk: GenerateChunk = { "token_ids": result.token_ids, "index": 0, @@ -261,12 +270,16 @@ def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: def start_dynamo_endpoint( host: "VllmAsyncGenerationWorkerImpl", *, + model_path: str, namespace: str, engine_env: dict[str, str], leader_worker_idx: int, ) -> asyncio.Task: """Register ``host``'s engine with Dynamo and serve it on the running loop. + ``model_path`` is a local directory with the model's config and tokenizer + files; the engine is served under ``host.model_name``. + The Rust Worker serves until the process gets SIGTERM. It installs a chained handler, so Ray's actor handler still runs and the actor exits as usual; discovery drops the engine with its process. @@ -281,7 +294,7 @@ def start_dynamo_endpoint( namespace=namespace, component=DYNAMO_COMPONENT, endpoint=DYNAMO_ENDPOINT, - model_name=host.model_name, + model_name=model_path, served_model_name=host.model_name, # NeMo RL's rollouts go to /v1/completions only. endpoint_types="completions", diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index e42904e2714..107a2529a49 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -19,6 +19,7 @@ """ import logging +import math from dataclasses import dataclass from typing import Any, Optional, cast @@ -176,7 +177,9 @@ def rollout_result_to_wire(result: RolloutResult) -> dict[str, Any]: """JSON-safe form of ``result``, for engines that answer over HTTP.""" return { "token_ids": result.token_ids, - "logprobs": result.logprobs, + # Strict JSON has no NaN or infinity, and Dynamo's frontend turns them + # into null; send them as the strings float() reads back. + "logprobs": [lp if math.isfinite(lp) else str(lp) for lp in result.logprobs], "truncated": result.truncated, "routed_experts": ( None @@ -194,7 +197,7 @@ def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: routed_experts = decode_routed_experts(routed_experts, dtype=None) return RolloutResult( token_ids=list(wire["token_ids"]), - logprobs=list(wire["logprobs"]), + logprobs=[float(lp) for lp in wire["logprobs"]], truncated=bool(wire["truncated"]), routed_experts=routed_experts, r3_stats=wire["r3_stats"], diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 388f50b1b4d..9562dab8232 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -16,6 +16,7 @@ import copy import gc import logging +import os import secrets import threading import time @@ -659,12 +660,23 @@ async def start_dynamo_endpoint_async( ``leader_worker_idx`` is this worker's index in its group; rollouts the engine serves report it, as ``generate_async`` results do. """ + # ai-dynamo is optional: import it only for engines served through Dynamo. + from dynamo._core import fetch_model + from nemo_rl.models.generation.dynamo.native_engine import ( start_dynamo_endpoint, ) + # Dynamo fetches a Hub ID's whole checkpoint before it registers; the engine's + # weights come from refit, so register the config and tokenizer files only. + model_path = ( + self.model_name + if os.path.exists(self.model_name) + else await fetch_model(self.model_name, ignore_weights=True) + ) self._dynamo_endpoint_task = start_dynamo_endpoint( self, + model_path=model_path, namespace=namespace, engine_env=engine_env, leader_worker_idx=leader_worker_idx, From 614fb0a66bc32f9b83f7760b007bd9a647653e4b Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Thu, 5 Nov 2026 20:15:10 -0800 Subject: [PATCH 16/19] test(dynamo): cover the round-3 gaps; trim stale docstrings - KV-event log test uses Dynamo's real event_type label and a cleared sample (C46); registration test checks the weight-free model path (C47). - New tests: an engine error keeps its cause (C48), non-finite logprobs cross strict JSON (C49), a cancelled rollout aborts its vLLM request (C50), the main-attention kinds match Dynamo's (C55). - Two engines get different KV-event sockets (C51); the engine wait fails fast on a dead frontend too (C52). - Drop the exemplar-equals-defaults check (C53); build DynamoCfg with keyword arguments (C54). - Docstrings: drop the removed DynamoGeneration history (C56), and say a vLLM bump can break Dynamo only in NeMo RL's adapter (C57). Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- nemo_rl/models/generation/interfaces.py | 3 +- .../collective_weight_synchronizer.py | 13 +--- .../models/generation/test_dynamo_config.py | 29 ++++---- .../generation/test_dynamo_managed_runtime.py | 38 +++++++--- .../generation/test_dynamo_native_engine.py | 72 +++++++++++++++++-- .../generation/test_dynamo_vllm_decoupling.py | 8 +-- .../generation/test_vllm_request_pipeline.py | 15 +++- .../test_reconcile_communicator.py | 9 +-- 8 files changed, 131 insertions(+), 56 deletions(-) diff --git a/nemo_rl/models/generation/interfaces.py b/nemo_rl/models/generation/interfaces.py index f1ea20b88cf..c5a1f35f6a0 100644 --- a/nemo_rl/models/generation/interfaces.py +++ b/nemo_rl/models/generation/interfaces.py @@ -597,8 +597,7 @@ def shard_liveness_ref(self, shard_idx: int) -> ray.ObjectRef: Which Ray worker leads shard N depends on how shards are laid out across workers, which is the backend's business. Asking for it by shard index keeps that here rather than in the control loop, where the same arithmetic had a second copy that - also assumed every backend has a ``worker_group`` -- an assumption that has already - broken a lane once (``'DynamoGeneration' object has no attribute 'worker_group'``). + also assumed every backend has a ``worker_group``. """ raise NotImplementedError diff --git a/nemo_rl/weight_sync/collective_weight_synchronizer.py b/nemo_rl/weight_sync/collective_weight_synchronizer.py index 7ac9366d6ae..252c5ec22d4 100644 --- a/nemo_rl/weight_sync/collective_weight_synchronizer.py +++ b/nemo_rl/weight_sync/collective_weight_synchronizer.py @@ -190,16 +190,9 @@ def _desired_membership( ) -> Optional[RefitMembership]: """The membership to track, or None for a backend that owns no DP worker group. - NOT every GenerationInterface has one. vLLM and TRT-LLM do; Megatron and SGLang - do not, and the interface declares nothing either way -- so reading - ``self._generation.worker_group`` assumes a vLLM shape that this synchronizer is - not entitled to assume. - - That assumption once broke L1_Functional_Tests_Dynamo on the plain grpo.py path, - under the since-removed managed Dynamo backend: - - grpo.py:1747 policy_generation.weight_synchronizer.init_communicator() - AttributeError: 'DynamoGeneration' object has no attribute 'worker_group' + NOT every GenerationInterface has one, and the interface declares nothing either + way -- so reading ``self._generation.worker_group`` assumes a vLLM shape that + this synchronizer is not entitled to assume. None means "this backend has no shards to track", which is the same thing an unrecorded membership already meant: every reconcile falls through to "nothing to diff --git a/tests/unit/models/generation/test_dynamo_config.py b/tests/unit/models/generation/test_dynamo_config.py index 078adf6a1b2..baae91f105e 100644 --- a/tests/unit/models/generation/test_dynamo_config.py +++ b/tests/unit/models/generation/test_dynamo_config.py @@ -11,13 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path - import pytest -import yaml from nemo_rl.models.generation import configure_generation_config -from nemo_rl.models.generation.dynamo.config import DynamoCfg, dynamo_cfg +from nemo_rl.models.generation.dynamo.config import ( + DynamoCfg, + DynamoFrontendArgs, + dynamo_cfg, +) from nemo_rl.models.generation.dynamo.managed_runtime import frontend_argv @@ -45,12 +46,6 @@ def _vllm_config(**overrides) -> dict: return config -_EXEMPLAR = ( - Path(__file__).resolve().parents[4] - / "examples/configs/grpo_math_1B_dynamo_single_controller.yaml" -) - - @pytest.mark.parametrize("dynamo", [False, True]) def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: config = configure_generation_config(_vllm_config(dynamo=dynamo), _Tokenizer()) @@ -60,9 +55,7 @@ def test_dynamo_true_alone_is_a_complete_config(dynamo: bool) -> None: assert config["vllm_cfg"]["async_engine"] is dynamo assert config["vllm_cfg"]["skip_tokenizer_init"] is True assert "dynamo_cfg" not in config - # The exemplar spells out exactly the defaults. - exemplar = yaml.safe_load(_EXEMPLAR.read_text())["policy"]["generation"] - assert dynamo_cfg(config) == DynamoCfg() == dynamo_cfg(exemplar) + assert dynamo_cfg(config) == DynamoCfg() @pytest.mark.parametrize( @@ -120,15 +113,17 @@ def test_configs_dynamo_would_not_serve_as_written_fail_at_startup( def test_frontend_argv_is_runtime_owned() -> None: - cfg = DynamoCfg.model_validate( - {"frontend_args": {"router_mode": "kv", "extra_cli_args": ["--foo", "1"]}} + cfg = DynamoCfg( + frontend_args=DynamoFrontendArgs( + router_mode="kv", extra_cli_args=["--foo", "1"] + ) ) argv = frontend_argv(host="0.0.0.0", port=3001, namespace="nemo-rl", dynamo_cfg=cfg) assert argv[argv.index("--router-mode") + 1] == "kv" assert argv[-2:] == ["--foo", "1"] - clashing = DynamoCfg.model_validate( - {"frontend_args": {"extra_cli_args": ["--http-port", "9"]}} + clashing = DynamoCfg( + frontend_args=DynamoFrontendArgs(extra_cli_args=["--http-port", "9"]) ) with pytest.raises(ValueError, match="--http-port is set by both"): frontend_argv( diff --git a/tests/unit/models/generation/test_dynamo_managed_runtime.py b/tests/unit/models/generation/test_dynamo_managed_runtime.py index 2c5c4dc3541..c6dde7fcf51 100644 --- a/tests/unit/models/generation/test_dynamo_managed_runtime.py +++ b/tests/unit/models/generation/test_dynamo_managed_runtime.py @@ -19,7 +19,7 @@ import pytest -from nemo_rl.models.generation.dynamo.config import DynamoCfg +from nemo_rl.models.generation.dynamo.config import DynamoCfg, DynamoFrontendArgs from nemo_rl.models.generation.dynamo.managed_runtime import ( ManagedDynamoRuntime, _managed_namespace, @@ -29,8 +29,8 @@ def _runtime(**frontend_args) -> ManagedDynamoRuntime: return ManagedDynamoRuntime( - dynamo_cfg=DynamoCfg.model_validate( - {"startup_timeout_s": 5, "frontend_args": frontend_args} + dynamo_cfg=DynamoCfg( + startup_timeout_s=5, frontend_args=DynamoFrontendArgs(**frontend_args) ), model_name="model", frontend_python="/venvs/vllm/bin/python", @@ -261,14 +261,28 @@ def fake_urlopen(url, timeout): ) -def test_engine_wait_fails_immediately_with_the_engines_error() -> None: +@pytest.mark.parametrize( + ("stopped", "error"), + [ + ("engine", "shard 0 stopped serving"), + # A bad frontend_args.extra_cli_args flag makes the frontend exit at once. + ("frontend", r"frontend is not alive \(code=2\)"), + ], +) +def test_engine_wait_fails_immediately_when_an_engine_or_service_stops( + stopped, error +) -> None: runtime = _started(_runtime()) + if stopped == "frontend": + runtime._frontend_process.poll = lambda: 2 + runtime._frontend_process.returncode = 2 - def endpoint_stopped() -> None: - raise RuntimeError("endpoint of generation shard 0 stopped serving") + def check_engines() -> None: + if stopped == "engine": + raise RuntimeError("endpoint of generation shard 0 stopped serving") - with pytest.raises(RuntimeError, match="shard 0 stopped serving"): - runtime.wait_for_engines(1, check_engines=endpoint_stopped) + with pytest.raises(RuntimeError, match=error): + runtime.wait_for_engines(1, check_engines=check_engines) def test_frontend_launches_from_the_given_interpreter(monkeypatch) -> None: @@ -290,9 +304,11 @@ def test_shutdown_logs_the_kv_events_the_router_applied(monkeypatch, capsys) -> runtime = _started(_runtime()) metrics = ( "# HELP dynamo_component_kv_cache_events_applied KV events applied\n" - 'dynamo_component_kv_cache_events_applied{status="ok",worker_id="1"} 40\n' - 'dynamo_component_kv_cache_events_applied{status="ok",worker_id="2"} 2\n' - 'dynamo_component_kv_cache_events_applied{status="parent_block_not_found"} 3\n' + 'dynamo_component_kv_cache_events_applied{event_type="stored",status="ok",worker_id="1"} 40\n' + 'dynamo_component_kv_cache_events_applied{event_type="stored",status="ok",worker_id="2"} 2\n' + 'dynamo_component_kv_cache_events_applied{event_type="stored",status="parent_block_not_found"} 3\n' + # Every prefix-cache reset applies one; it says nothing about block events. + 'dynamo_component_kv_cache_events_applied{event_type="cleared",status="ok",worker_id="1"} 5\n' 'dynamo_component_requests_total{status="ok"} 9\n' ) diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_native_engine.py index d1dda2bfeb4..0ef89990af2 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_native_engine.py @@ -17,8 +17,11 @@ constructor, so rollouts are sampled by the native worker's own helpers. """ +import ast import asyncio +import importlib.util import json +from pathlib import Path from types import SimpleNamespace import numpy as np @@ -32,6 +35,10 @@ from dynamo.llm.exceptions import InvalidArgument # noqa: E402 from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.dynamo.config import ( # noqa: E402 + DynamoCfg, + DynamoFrontendArgs, +) from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) @@ -156,6 +163,24 @@ def test_rollouts_are_served_like_generate_async() -> None: } +def test_a_cancelled_rollout_aborts_the_vllm_request_it_runs() -> None: + llm = _FakeLLM([_output([1, 2, 3], _completion([10], "stop"))]) + aborted = [] + + async def abort(request_id): + aborted.append(request_id) + + llm.abort = abort + engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + + _collect(engine, _rollout_request([1, 2, 3])) + # Dynamo's worker calls abort() when the frontend cancels a request. + asyncio.run(engine.abort(_Context())) + + ((_, _, request_id),) = llm.calls + assert aborted == [request_id] + + def test_router_replay_routes_reach_rollouts_as_natively() -> None: routes = np.arange(5 * 2 * 2, dtype=np.int16).reshape(5, 2, 2) final = _output( @@ -225,7 +250,11 @@ async def run(self): async def start(): await native_engine.start_dynamo_endpoint( - host, namespace="ns", engine_env=env, leader_worker_idx=0 + host, + model_path="/snapshots/model", + namespace="ns", + engine_env=env, + leader_worker_idx=0, ) asyncio.run(start()) @@ -237,6 +266,11 @@ def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: config = _start_worker_config(monkeypatch, _host(_FakeLLM())) assert config.namespace == "ns" + # Registered from the weight-free snapshot, served under the model's name. + assert (config.model_name, config.served_model_name) == ( + "/snapshots/model", + "model", + ) # NeMo RL's rollouts go to /v1/completions; no chat surface is registered. assert config.endpoint_types == "completions" @@ -251,6 +285,20 @@ def test_requests_other_than_rollouts_are_refused() -> None: assert llm.calls == [] +def test_an_engine_error_reaches_the_driver_with_its_cause() -> None: + host = _host(_FakeLLM()) + + async def serve_rollout(*args, **kwargs): + raise RuntimeError("the generation output did not include routed_experts") + + host._serve_rollout = serve_rollout + engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + + # A 5xx would reach the driver as "Internal server error". + with pytest.raises(InvalidArgument, match="RuntimeError: .*routed_experts"): + _collect(engine, _rollout_request([1, 2, 3])) + + def test_an_endpoint_that_stops_serving_reports_why() -> None: async def endpoint_errors() -> list: async def registration_fails() -> None: @@ -306,9 +354,7 @@ async def call_utility_async(method): def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( router_mode, prefix_caching, publishes ) -> None: - cfg = native_engine.DynamoCfg.model_validate( - {"frontend_args": {"router_mode": router_mode}} - ) + cfg = DynamoCfg(frontend_args=DynamoFrontendArgs(router_mode=router_mode)) config = native_engine.dynamo_kv_events_config(cfg, prefix_caching=prefix_caching) @@ -319,3 +365,21 @@ def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( # A per-engine abstract socket, which vLLM's publisher binds and Dynamo's # source connects to: no port for two engines on a node to collide on. assert config["endpoint"].startswith("ipc://@") + other = native_engine.dynamo_kv_events_config(cfg, prefix_caching=True) + assert other["endpoint"] != config["endpoint"] + + +def test_main_attention_kinds_match_dynamos() -> None: + # Read from source: importing dynamo.vllm would import Dynamo's vLLM integration. + spec = importlib.util.find_spec("dynamo.vllm.cache_info") + tree = ast.parse(Path(spec.origin).read_text()) + (dynamos,) = [ + ast.literal_eval(node.value) + for node in tree.body + if isinstance(node, ast.Assign) + and any( + getattr(t, "id", None) == "MAIN_ATTENTION_KV_CACHE_KINDS" + for t in node.targets + ) + ] + assert {kind.value for kind in native_engine._MAIN_ATTENTION_KINDS} == dynamos diff --git a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py index f729715eede..62ff7e4b7c2 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py +++ b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py @@ -14,10 +14,10 @@ """Dynamo stays out of NeMo RL's vLLM engines. The engines run NeMo RL's own vLLM and use Dynamo only through its -engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only where -it also breaks native vLLM. These guards fail if NeMo RL starts importing -Dynamo's vLLM integration (or any other Dynamo module), or if the SDK itself -starts importing vLLM. +engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only in +NeMo RL's own adapter (``dynamo/native_engine.py``). These guards fail if +NeMo RL starts importing Dynamo's vLLM integration (or any other Dynamo +module), or if the SDK itself starts importing vLLM. """ import ast diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py index f26ee51a733..cc9b7cf0725 100644 --- a/tests/unit/models/generation/test_vllm_request_pipeline.py +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -13,6 +13,8 @@ # limitations under the License. """The rollout result pipeline behind generate_async.""" +import json +import math from types import SimpleNamespace import numpy as np @@ -98,7 +100,8 @@ def test_wire_round_trip_preserves_the_output() -> None: result = _result( _final_output([7, 8, 9], prompt_routes=3, generated_routes=3), r3=True ) - wire = rollout_result_to_wire(result) + # The wire crosses Dynamo's frontend as strict JSON. + wire = json.loads(json.dumps(rollout_result_to_wire(result), allow_nan=False)) input_ids = torch.tensor([1, 2, 3]) direct = rollout_output(result, input_ids=input_ids, input_length=3) @@ -111,3 +114,13 @@ def test_wire_round_trip_preserves_the_output() -> None: assert direct.keys() == shipped.keys() for key in direct: assert torch.equal(direct[key], shipped[key]), key + + +def test_wire_carries_non_finite_logprobs_as_strict_json() -> None: + result = RolloutResult( + [7, 8, 9], [-0.5, float("-inf"), float("nan")], False, None, None + ) + # Dynamo's transport is strict JSON: non-finite floats would arrive as null. + wire = json.loads(json.dumps(rollout_result_to_wire(result), allow_nan=False)) + logprobs = rollout_result_from_wire(wire).logprobs + assert logprobs[:2] == [-0.5, float("-inf")] and math.isnan(logprobs[2]) diff --git a/tests/unit/weight_sync/test_reconcile_communicator.py b/tests/unit/weight_sync/test_reconcile_communicator.py index 1253e5aab46..f87e59e8cc5 100644 --- a/tests/unit/weight_sync/test_reconcile_communicator.py +++ b/tests/unit/weight_sync/test_reconcile_communicator.py @@ -655,13 +655,8 @@ def test_an_unconfigured_deadline_still_bounds_the_wait(self): class TestBackendsWithoutAWorkerGroup: """CollectiveWeightSynchronizer holds a GenerationInterface, not a VllmGeneration. - Only some implementations own a Ray worker group with DP shards -- vLLM and TRT-LLM do; - Megatron and SGLang do not, and the interface declares nothing either way. Reading - self._generation.worker_group once broke the since-removed managed Dynamo backend - on the plain grpo.py path: - - grpo.py:1747 policy_generation.weight_synchronizer.init_communicator() - AttributeError: 'DynamoGeneration' object has no attribute 'worker_group' + Not every implementation owns a Ray worker group with DP shards, and the interface + declares nothing either way. Membership tracking is only meaningful where shards exist, so a backend without one records None and every reconcile falls through to "nothing to do" -- which is exactly From f774780817edfd57f1dd3d450ac83ceb93113e22 Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 6 Nov 2026 18:10:33 -0800 Subject: [PATCH 17/19] refactor(dynamo): split the engine adapter into a shared Dynamo side and a vLLM backend C61. `dynamo/engine_adapter.py` holds `DynamoEngineAdapter`, the Dynamo side written once for every generation backend: request kinds as handlers keyed by their annotation, the error mapping, and `start_dynamo_endpoint` (completions only). A backend implements `serve_rollout`, `start`, `abort` and `kv_event_sources`. `NativeVllmEngine` becomes `DynamoVllmAdapter` in `dynamo/vllm_adapter.py`. `RolloutResult` and its wire form move from `vllm/request_pipeline.py` to the backend-neutral `rollout_result.py`, so the shared side imports no backend; the decoupling test checks that. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../generation/dynamo/engine_adapter.py | 205 ++++++++++++++++++ .../generation/dynamo/managed_runtime.py | 4 +- .../generation/dynamo/rollout_client.py | 4 +- .../{native_engine.py => vllm_adapter.py} | 178 ++++----------- nemo_rl/models/generation/rollout_result.py | 82 +++++++ .../generation/vllm/request_pipeline.py | 59 +---- .../models/generation/vllm/vllm_generation.py | 6 +- .../generation/vllm/vllm_worker_async.py | 17 +- pyrefly.toml | 4 +- ..._engine.py => test_dynamo_vllm_adapter.py} | 43 ++-- .../generation/test_dynamo_vllm_decoupling.py | 56 +++-- .../models/generation/test_vllm_generation.py | 4 +- .../generation/test_vllm_request_pipeline.py | 8 +- 13 files changed, 418 insertions(+), 252 deletions(-) create mode 100644 nemo_rl/models/generation/dynamo/engine_adapter.py rename nemo_rl/models/generation/dynamo/{native_engine.py => vllm_adapter.py} (51%) create mode 100644 nemo_rl/models/generation/rollout_result.py rename tests/unit/models/generation/{test_dynamo_native_engine.py => test_dynamo_vllm_adapter.py} (90%) diff --git a/nemo_rl/models/generation/dynamo/engine_adapter.py b/nemo_rl/models/generation/dynamo/engine_adapter.py new file mode 100644 index 00000000000..9d21a0383fc --- /dev/null +++ b/nemo_rl/models/generation/dynamo/engine_adapter.py @@ -0,0 +1,205 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serve a NeMo RL generation worker's own engine through Dynamo's backend SDK. + +``DynamoEngineAdapter`` is the Dynamo side, written once for every backend: the +request kinds served, how errors reach the driver, and the endpoint's +registration. A backend implements four methods over its worker's engine +(``serve_rollout``, ``start``, ``abort`` and ``kv_event_sources``); +``vllm_adapter.DynamoVllmAdapter`` is the vLLM one. The engine only ever receives +token IDs, on ``/v1/completions``. This module imports no generation backend. +""" + +from __future__ import annotations + +import asyncio +import json +import os +from abc import abstractmethod +from collections.abc import AsyncGenerator, Callable +from dataclasses import dataclass +from typing import Any, Optional + +from dynamo._core import Context +from dynamo.common.backend import ( + EngineConfig, + GenerateChunk, + GenerateRequest, + LLMEngine, + Worker, + WorkerConfig, +) +from dynamo.common.backend.publisher import KvEventSource +from dynamo.llm.exceptions import InvalidArgument + +from nemo_rl.models.generation.dynamo.managed_runtime import ( + DYNAMO_COMPONENT, + DYNAMO_ENDPOINT, +) +from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION +from nemo_rl.models.generation.rollout_result import ( + RolloutResult, + rollout_result_to_wire, +) + +# Serves one request kind: the request, its annotation's value, and the context. +RequestHandler = Callable[ + [GenerateRequest, str, Context], AsyncGenerator[GenerateChunk, None] +] + + +@dataclass(frozen=True) +class ServedRollout: + """A backend's answer to one rollout.""" + + result: RolloutResult + # The OpenAI finish reason the frontend reports. + finish_reason: str + # The OpenAI completion usage; None when no new token fit. + usage: Optional[dict[str, Any]] + + +def _annotation_value(request: GenerateRequest, prefix: str) -> Optional[str]: + """The value of the first ```` annotation, or None.""" + for annotation in request.get("annotations") or (): + if annotation.startswith(prefix): + return annotation[len(prefix) :] + return None + + +class DynamoEngineAdapter(LLMEngine): + """Serves one NeMo RL generation worker through Dynamo. The worker owns the engine.""" + + def __init__(self, *, leader_worker_idx: int): + # The worker's index in its group; rollouts report it, as + # generate_async results do. + self._leader_worker_idx = leader_worker_idx + + @classmethod + async def from_args(cls, argv=None): + raise NotImplementedError("constructed in-process by the generation worker") + + def _handlers(self) -> dict[str, RequestHandler]: + """The request kinds served, keyed by the annotation prefix that marks them.""" + return {ROLLOUT_ANNOTATION: self._generate_rollout} + + async def generate( + self, request: GenerateRequest, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + handlers = self._handlers() + for prefix, handler in handlers.items(): + value = _annotation_value(request, prefix) + if value is not None: + break + else: + raise InvalidArgument( + "NeMo RL's engines serve only NeMo RL rollouts through Dynamo " + f"(requests annotated {', '.join(map(repr, handlers))})." + ) + try: + async for chunk in handler(request, value, context): + yield chunk + except RuntimeError as error: + # A 5xx reaches the driver as "Internal server error" and is re-sent; + # a 4xx keeps the cause, and the rollout retry policy still applies. + raise InvalidArgument(f"{type(error).__name__}: {error}") from error + + async def _generate_rollout( + self, request: GenerateRequest, rollout_spec: str, context: Context + ) -> AsyncGenerator[GenerateChunk, None]: + """One NeMo RL rollout, served by the backend's ``serve_rollout``. + + One chunk carries the result, routed experts included, in ``engine_data``; + the driver assembles the same output ``generate_async`` returns. + """ + rollout = json.loads(rollout_spec) + served = await self.serve_rollout( + list(request["token_ids"]), + greedy=rollout["greedy"], + stop_strings=rollout["stop_strings"], + request_id=context.id(), + ) + chunk: GenerateChunk = { + "token_ids": served.result.token_ids, + "index": 0, + "finish_reason": served.finish_reason, + "engine_data": { + "rollout": rollout_result_to_wire(served.result), + "gen_leader_worker_idx": self._leader_worker_idx, + }, + } + if served.usage is not None: + chunk["completion_usage"] = served.usage + yield chunk + + @abstractmethod + async def serve_rollout( + self, + prompt_token_ids: list[int], + *, + greedy: bool, + stop_strings: Optional[list[str]], + request_id: str, + ) -> ServedRollout: + """Serve one rollout exactly as the worker's native ``generate_async`` does.""" + + @abstractmethod + async def start(self, worker_id: int) -> EngineConfig: + """Register the engine's context length, KV block size and capacity.""" + + @abstractmethod + async def abort(self, context: Context) -> None: + """Cancel the request ``context`` names; Dynamo calls it on disconnect.""" + + @abstractmethod + async def kv_event_sources(self) -> list[KvEventSource]: + """Where the engine publishes the KV-cache events Dynamo's KV router indexes.""" + + async def cleanup(self) -> None: + """Nothing to release: the worker owns its engine and shuts it down.""" + + +def start_dynamo_endpoint( + adapter: DynamoEngineAdapter, + *, + model_path: str, + served_model_name: str, + namespace: str, + engine_env: dict[str, str], +) -> asyncio.Task: + """Register ``adapter``'s engine with Dynamo and serve it on the running loop. + + ``model_path`` is a local directory with the model's config and tokenizer + files; the engine is served under ``served_model_name``. + + The Rust Worker serves until the process gets SIGTERM. It installs a chained + handler, so Ray's actor handler still runs and the actor exits as usual; + discovery drops the engine with its process. + """ + # The Rust runtime reads these when the first Worker is constructed. + os.environ.update(engine_env) + config = WorkerConfig( + namespace=namespace, + component=DYNAMO_COMPONENT, + endpoint=DYNAMO_ENDPOINT, + model_name=model_path, + served_model_name=served_model_name, + # NeMo RL's rollouts go to /v1/completions only. + endpoint_types="completions", + discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], + request_plane=engine_env["DYN_REQUEST_PLANE"], + event_plane=engine_env["DYN_EVENT_PLANE"], + ) + return asyncio.get_running_loop().create_task(Worker(adapter, config).run()) diff --git a/nemo_rl/models/generation/dynamo/managed_runtime.py b/nemo_rl/models/generation/dynamo/managed_runtime.py index ee4f0383836..be438e817b0 100644 --- a/nemo_rl/models/generation/dynamo/managed_runtime.py +++ b/nemo_rl/models/generation/dynamo/managed_runtime.py @@ -15,7 +15,7 @@ """Driver-owned Dynamo services: etcd, NATS and the OpenAI frontend. The vLLM engines are NeMo RL's own workers; each registers itself with these -services (see ``native_engine``), so the runtime only waits to see them. +services (see ``engine_adapter``), so the runtime only waits to see them. """ import atexit @@ -43,7 +43,7 @@ ) from nemo_rl.models.generation.dynamo.config import DynamoCfg -# The engines register on this component and endpoint (see native_engine). +# The engines register on this component and endpoint (see engine_adapter). DYNAMO_COMPONENT = "backend" DYNAMO_ENDPOINT = "generate" diff --git a/nemo_rl/models/generation/dynamo/rollout_client.py b/nemo_rl/models/generation/dynamo/rollout_client.py index 5e1346f4b2c..2683e6525d2 100644 --- a/nemo_rl/models/generation/dynamo/rollout_client.py +++ b/nemo_rl/models/generation/dynamo/rollout_client.py @@ -25,7 +25,7 @@ LOGGER = logging.getLogger(__name__) -# Marks a request as a NeMo RL rollout for the engine (native_engine), which +# Marks a request as a NeMo RL rollout for the engine (engine_adapter), which # then samples it from the generation config. Defined here, not in the engine # module, so the driver need not import ai-dynamo. ROLLOUT_ANNOTATION = "nemo_rl:rollout:" @@ -120,7 +120,7 @@ def rollout_payload( def parse_rollout_response(response: dict[str, Any], *, url: str) -> dict[str, Any]: """Return the engine's ``engine_data``, or raise with the frontend's error. - It holds the rollout result (``request_pipeline.rollout_result_to_wire``) + It holds the rollout result (``rollout_result.rollout_result_to_wire``) and the serving worker's index. """ if response.get("status") == "error": diff --git a/nemo_rl/models/generation/dynamo/native_engine.py b/nemo_rl/models/generation/dynamo/vllm_adapter.py similarity index 51% rename from nemo_rl/models/generation/dynamo/native_engine.py rename to nemo_rl/models/generation/dynamo/vllm_adapter.py index 23937f93354..9c55df7d666 100644 --- a/nemo_rl/models/generation/dynamo/native_engine.py +++ b/nemo_rl/models/generation/dynamo/vllm_adapter.py @@ -14,53 +14,36 @@ """Serve a NeMo RL vLLM worker's own AsyncLLM through Dynamo's backend SDK. -The engine process holds no Dynamo-owned vLLM code. This adapter implements -``dynamo.common.backend.LLMEngine``, which imports no vLLM, over the AsyncLLM -that ``VllmAsyncGenerationWorkerImpl`` owns, so refit, pause, sleep, -quantization and speculative decoding are that worker's own. +The engine process holds no Dynamo-owned vLLM code. ``DynamoVllmAdapter`` is the +vLLM backend of ``engine_adapter.DynamoEngineAdapter``, over the AsyncLLM that +``VllmAsyncGenerationWorkerImpl`` owns, so refit, pause, sleep, quantization and +speculative decoding are that worker's own. """ from __future__ import annotations -import asyncio -import json -import logging import os -from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, Optional import torch from dynamo._core import Context -from dynamo.common.backend import ( - EngineConfig, - GenerateChunk, - GenerateRequest, - LLMEngine, - LlmRegistration, - Worker, - WorkerConfig, -) +from dynamo.common.backend import EngineConfig, LlmRegistration from dynamo.common.backend.publisher import ComponentSnapshot, KvEventSource, ZmqSource -from dynamo.llm.exceptions import InvalidArgument from vllm.inputs import TokensPrompt from vllm.v1.kv_cache_interface import KVCacheSpecKind from vllm.v1.metrics.loggers import StatLoggerBase from nemo_rl.models.generation.dynamo.config import DynamoCfg -from nemo_rl.models.generation.dynamo.managed_runtime import ( - DYNAMO_COMPONENT, - DYNAMO_ENDPOINT, +from nemo_rl.models.generation.dynamo.engine_adapter import ( + DynamoEngineAdapter, + ServedRollout, ) -from nemo_rl.models.generation.dynamo.rollout_client import ROLLOUT_ANNOTATION -from nemo_rl.models.generation.vllm.request_pipeline import rollout_result_to_wire if TYPE_CHECKING: from nemo_rl.models.generation.vllm.vllm_worker_async import ( VllmAsyncGenerationWorkerImpl, ) -LOGGER = logging.getLogger(__name__) - # vLLM's main-attention kinds, as Dynamo's own vLLM backend counts them # (dynamo/vllm/cache_info.py MAIN_ATTENTION_KV_CACHE_KINDS, which NeMo RL may # not import); test_main_attention_kinds_match_dynamos checks the copy. @@ -89,24 +72,6 @@ def dynamo_kv_events_config( } -def _annotation_value(request: GenerateRequest, prefix: str) -> Optional[str]: - """The value of the first ```` annotation, or None.""" - for annotation in request.get("annotations") or (): - if annotation.startswith(prefix): - return annotation[len(prefix) :] - return None - - -def _rollout_request(request: GenerateRequest) -> Optional[dict[str, Any]]: - """The rollout spec if ``request`` is one of NeMo RL's own rollouts. - - Those come from ``VllmGeneration.generate_async`` and are built here - exactly as the native path builds them. - """ - rollout = _annotation_value(request, ROLLOUT_ANNOTATION) - return None if rollout is None else json.loads(rollout) - - class SnapshotStatLogger(StatLoggerBase): """Push per-iteration KV usage into Dynamo's SnapshotPublisher (router load).""" @@ -143,18 +108,40 @@ def log_engine_initialized(self) -> None: pass -class NativeVllmEngine(LLMEngine): - """LLMEngine over a running NeMo RL vLLM worker. Does not own its lifecycle.""" +class DynamoVllmAdapter(DynamoEngineAdapter): + """The vLLM backend, over a running NeMo RL vLLM worker it does not own.""" - def __init__(self, host: "VllmAsyncGenerationWorkerImpl", leader_worker_idx: int): + def __init__( + self, host: "VllmAsyncGenerationWorkerImpl", *, leader_worker_idx: int + ): + super().__init__(leader_worker_idx=leader_worker_idx) self._host = host - self._leader_worker_idx = leader_worker_idx self._llm = host.llm self._max_model_len = self._llm.vllm_config.model_config.max_model_len - @classmethod - async def from_args(cls, argv=None): - raise NotImplementedError("constructed in-process by the vLLM worker") + async def serve_rollout( + self, + prompt_token_ids: list[int], + *, + greedy: bool, + stop_strings: Optional[list[str]], + request_id: str, + ) -> ServedRollout: + result, final_output = await self._host._serve_rollout( + TokensPrompt(prompt_token_ids=prompt_token_ids), + prompt_token_ids=prompt_token_ids, + greedy=greedy, + stop_strings=stop_strings, + request_id=request_id, + device=torch.device("cpu"), + ) + if final_output is None: + return ServedRollout(result, finish_reason="length", usage=None) + return ServedRollout( + result, + finish_reason=final_output.outputs[0].finish_reason, + usage=_usage(final_output, len(result.token_ids)), + ) async def start(self, worker_id: int) -> EngineConfig: del worker_id @@ -184,57 +171,6 @@ async def start(self, worker_id: int) -> EngineConfig: ), ) - async def generate( - self, request: GenerateRequest, context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - rollout = _rollout_request(request) - if rollout is None: - raise InvalidArgument( - "NeMo RL's vLLM engines serve only NeMo RL rollouts through Dynamo " - f"(requests annotated {ROLLOUT_ANNOTATION!r})." - ) - async for chunk in self._generate_rollout(request, rollout, context): - yield chunk - - async def _generate_rollout( - self, request: GenerateRequest, rollout: dict[str, Any], context: Context - ) -> AsyncGenerator[GenerateChunk, None]: - """One NeMo RL rollout, served by the worker's own ``_serve_rollout``. - - One chunk carries the result, routed experts included, in ``engine_data``; - the driver assembles the same output ``generate_async`` returns. - """ - prompt_token_ids = list(request["token_ids"]) - try: - result, final_output = await self._host._serve_rollout( - TokensPrompt(prompt_token_ids=prompt_token_ids), - prompt_token_ids=prompt_token_ids, - greedy=rollout["greedy"], - stop_strings=rollout["stop_strings"], - request_id=context.id(), - device=torch.device("cpu"), - ) - except RuntimeError as error: - # A 5xx reaches the driver as "Internal server error" and is re-sent; - # a 4xx keeps the cause, and the rollout retry policy still applies. - raise InvalidArgument(f"{type(error).__name__}: {error}") from error - chunk: GenerateChunk = { - "token_ids": result.token_ids, - "index": 0, - "finish_reason": ( - "length" - if final_output is None - else final_output.outputs[0].finish_reason - ), - "engine_data": { - "rollout": rollout_result_to_wire(result), - "gen_leader_worker_idx": self._leader_worker_idx, - }, - } - if final_output is not None: - chunk["completion_usage"] = _usage(final_output, len(result.token_ids)) - yield chunk - async def abort(self, context: Context) -> None: await self._llm.abort(context.id()) @@ -265,43 +201,3 @@ def _usage(output: Any, num_output_tokens: int) -> dict[str, Any]: "total_tokens": prompt_tokens + num_output_tokens, "prompt_tokens_details": {"cached_tokens": output.num_cached_tokens or 0}, } - - -def start_dynamo_endpoint( - host: "VllmAsyncGenerationWorkerImpl", - *, - model_path: str, - namespace: str, - engine_env: dict[str, str], - leader_worker_idx: int, -) -> asyncio.Task: - """Register ``host``'s engine with Dynamo and serve it on the running loop. - - ``model_path`` is a local directory with the model's config and tokenizer - files; the engine is served under ``host.model_name``. - - The Rust Worker serves until the process gets SIGTERM. It installs a chained - handler, so Ray's actor handler still runs and the actor exits as usual; - discovery drops the engine with its process. - """ - # The Rust runtime reads these when the first Worker is constructed. - os.environ.update(engine_env) - kv_events = host.llm.vllm_config.kv_events_config - publishes = kv_events is not None and kv_events.enable_kv_cache_events - # The Dynamo functional overlays check the router applied these. - print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) - config = WorkerConfig( - namespace=namespace, - component=DYNAMO_COMPONENT, - endpoint=DYNAMO_ENDPOINT, - model_name=model_path, - served_model_name=host.model_name, - # NeMo RL's rollouts go to /v1/completions only. - endpoint_types="completions", - discovery_backend=engine_env["DYN_DISCOVERY_BACKEND"], - request_plane=engine_env["DYN_REQUEST_PLANE"], - event_plane=engine_env["DYN_EVENT_PLANE"], - ) - return asyncio.get_running_loop().create_task( - Worker(NativeVllmEngine(host, leader_worker_idx), config).run() - ) diff --git a/nemo_rl/models/generation/rollout_result.py b/nemo_rl/models/generation/rollout_result.py new file mode 100644 index 00000000000..fe965fed8d0 --- /dev/null +++ b/nemo_rl/models/generation/rollout_result.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One rollout's result and its JSON wire form, for every generation backend. + +An engine served through Dynamo sends the result to the driver in +``engine_data``; the vLLM worker builds it with ``request_pipeline.rollout_result``. +""" + +import math +from dataclasses import dataclass +from typing import Any, Optional + +import torch + +from nemo_rl.utils.routed_experts_codec import ( + decode_routed_experts, + encode_routed_experts, +) + + +@dataclass +class RolloutResult: + """One rollout's outcome, independent of where its output is assembled.""" + + token_ids: list[int] + # The sampled token's log probability at each position; 0.0 where vLLM + # reported none. + logprobs: list[float] + # The engine stopped at max_tokens. + truncated: bool + # [prompt + generated, num_moe_layers, topk], aligned for router replay. + routed_experts: Optional[torch.Tensor] + # missing/expected/actual route counts; set when router replay is on. + r3_stats: Optional[dict[str, int]] + + @classmethod + def nothing_generated(cls) -> "RolloutResult": + """The result when no new token fits: the prompt alone, untruncated.""" + return cls([], [], False, None, None) + + +def rollout_result_to_wire(result: RolloutResult) -> dict[str, Any]: + """JSON-safe form of ``result``, for engines that answer over HTTP.""" + return { + "token_ids": result.token_ids, + # Strict JSON has no NaN or infinity, and Dynamo's frontend turns them + # into null; send them as the strings float() reads back. + "logprobs": [lp if math.isfinite(lp) else str(lp) for lp in result.logprobs], + "truncated": result.truncated, + "routed_experts": ( + None + if result.routed_experts is None + else encode_routed_experts(result.routed_experts) + ), + "r3_stats": result.r3_stats, + } + + +def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: + """Inverse of :func:`rollout_result_to_wire`; routes keep their own dtype.""" + routed_experts = wire["routed_experts"] + if routed_experts is not None: + routed_experts = decode_routed_experts(routed_experts, dtype=None) + return RolloutResult( + token_ids=list(wire["token_ids"]), + logprobs=[float(lp) for lp in wire["logprobs"]], + truncated=bool(wire["truncated"]), + routed_experts=routed_experts, + r3_stats=wire["r3_stats"], + ) diff --git a/nemo_rl/models/generation/vllm/request_pipeline.py b/nemo_rl/models/generation/vllm/request_pipeline.py index 107a2529a49..62ddf51fa6f 100644 --- a/nemo_rl/models/generation/vllm/request_pipeline.py +++ b/nemo_rl/models/generation/vllm/request_pipeline.py @@ -19,44 +19,18 @@ """ import logging -import math -from dataclasses import dataclass from typing import Any, Optional, cast import torch from nemo_rl.distributed.batched_data_dict import BatchedDataDict from nemo_rl.models.generation.interfaces import GenerationOutputSpec +from nemo_rl.models.generation.rollout_result import RolloutResult from nemo_rl.models.generation.vllm.utils import pad_and_align_routed_expert_indices -from nemo_rl.utils.routed_experts_codec import ( - decode_routed_experts, - encode_routed_experts, -) LOGGER = logging.getLogger(__name__) -@dataclass -class RolloutResult: - """One rollout's outcome, independent of where its output is assembled.""" - - token_ids: list[int] - # The sampled token's log probability at each position; 0.0 where vLLM - # reported none. - logprobs: list[float] - # The engine stopped at max_tokens. - truncated: bool - # [prompt + generated, num_moe_layers, topk], aligned for router replay. - routed_experts: Optional[torch.Tensor] - # missing/expected/actual route counts; set when router replay is on. - r3_stats: Optional[dict[str, int]] - - @classmethod - def nothing_generated(cls) -> "RolloutResult": - """The result when no new token fits: the prompt alone, untruncated.""" - return cls([], [], False, None, None) - - def _selected_logprobs(completion: Any) -> list[float]: """The sampled token's log probability at each position of a completion.""" by_position = completion.logprobs or () @@ -171,34 +145,3 @@ def rollout_output( if result.routed_experts is not None: output["routed_experts"] = result.routed_experts.to(device).unsqueeze(0) return BatchedDataDict[GenerationOutputSpec](output) - - -def rollout_result_to_wire(result: RolloutResult) -> dict[str, Any]: - """JSON-safe form of ``result``, for engines that answer over HTTP.""" - return { - "token_ids": result.token_ids, - # Strict JSON has no NaN or infinity, and Dynamo's frontend turns them - # into null; send them as the strings float() reads back. - "logprobs": [lp if math.isfinite(lp) else str(lp) for lp in result.logprobs], - "truncated": result.truncated, - "routed_experts": ( - None - if result.routed_experts is None - else encode_routed_experts(result.routed_experts) - ), - "r3_stats": result.r3_stats, - } - - -def rollout_result_from_wire(wire: dict[str, Any]) -> RolloutResult: - """Inverse of :func:`rollout_result_to_wire`; routes keep their own dtype.""" - routed_experts = wire["routed_experts"] - if routed_experts is not None: - routed_experts = decode_routed_experts(routed_experts, dtype=None) - return RolloutResult( - token_ids=list(wire["token_ids"]), - logprobs=[float(lp) for lp in wire["logprobs"]], - truncated=bool(wire["truncated"]), - routed_experts=routed_experts, - r3_stats=wire["r3_stats"], - ) diff --git a/nemo_rl/models/generation/vllm/vllm_generation.py b/nemo_rl/models/generation/vllm/vllm_generation.py index cc165461797..79ec319bc04 100644 --- a/nemo_rl/models/generation/vllm/vllm_generation.py +++ b/nemo_rl/models/generation/vllm/vllm_generation.py @@ -52,6 +52,7 @@ agreed_media_columns, verify_right_padding, ) +from nemo_rl.models.generation.rollout_result import rollout_result_from_wire from nemo_rl.models.generation.vllm.config import ( REFITTABLE_FP8_KV_CACHE_DTYPES, VllmConfig, @@ -63,10 +64,7 @@ draft_module_sharing_disable_required, ) from nemo_rl.models.generation.vllm.metric_names import BATCH_DURATION_KEY -from nemo_rl.models.generation.vllm.request_pipeline import ( - rollout_output, - rollout_result_from_wire, -) +from nemo_rl.models.generation.vllm.request_pipeline import rollout_output from nemo_rl.models.generation.vllm.utils import ( aggregate_spec_decode_counters, assert_reload_refit_config_supported, diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 9562dab8232..38ba30aaebc 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -69,8 +69,8 @@ resolve_collective_rpc_result, ) from nemo_rl.models.generation.vllm.config import parse_nvfp4_pertoken_rollout +from nemo_rl.models.generation.rollout_result import RolloutResult from nemo_rl.models.generation.vllm.request_pipeline import ( - RolloutResult, rollout_output, rollout_result, ) @@ -446,7 +446,7 @@ def _create_engine(self, llm_kwargs: dict[str, Any]) -> None: else [] ) if dynamo_enabled(self.cfg): - from nemo_rl.models.generation.dynamo.native_engine import ( + from nemo_rl.models.generation.dynamo.vllm_adapter import ( SnapshotStatLogger, dynamo_kv_events_config, ) @@ -663,9 +663,10 @@ async def start_dynamo_endpoint_async( # ai-dynamo is optional: import it only for engines served through Dynamo. from dynamo._core import fetch_model - from nemo_rl.models.generation.dynamo.native_engine import ( + from nemo_rl.models.generation.dynamo.engine_adapter import ( start_dynamo_endpoint, ) + from nemo_rl.models.generation.dynamo.vllm_adapter import DynamoVllmAdapter # Dynamo fetches a Hub ID's whole checkpoint before it registers; the engine's # weights come from refit, so register the config and tokenizer files only. @@ -674,12 +675,16 @@ async def start_dynamo_endpoint_async( if os.path.exists(self.model_name) else await fetch_model(self.model_name, ignore_weights=True) ) + kv_events = self.llm.vllm_config.kv_events_config + publishes = kv_events is not None and kv_events.enable_kv_cache_events + # The Dynamo functional overlays check the router applied these. + print(f" [Dynamo] engine publishes KV events: {publishes}", flush=True) self._dynamo_endpoint_task = start_dynamo_endpoint( - self, + DynamoVllmAdapter(self, leader_worker_idx=leader_worker_idx), model_path=model_path, + served_model_name=self.model_name, namespace=namespace, engine_env=engine_env, - leader_worker_idx=leader_worker_idx, ) async def dynamo_endpoint_error(self) -> Optional[str]: @@ -2116,7 +2121,7 @@ async def _serve_rollout( Returns: The result, and vLLM's final RequestOutput (None when no new token fits). The final output supplies finish and usage metadata to Dynamo - (``dynamo/native_engine.py``). + (``dynamo/vllm_adapter.py``). """ max_new_tokens = self._rollout_max_new_tokens(len(prompt_token_ids)) if max_new_tokens == 0: diff --git a/pyrefly.toml b/pyrefly.toml index 1361d58cad0..52beefd9ee2 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -241,7 +241,8 @@ project-includes = [ "nemo_rl/models/generation/dynamo/__init__.py", "nemo_rl/models/generation/dynamo/config.py", "nemo_rl/models/generation/dynamo/managed_runtime.py", - "nemo_rl/models/generation/dynamo/native_engine.py", + "nemo_rl/models/generation/dynamo/engine_adapter.py", + "nemo_rl/models/generation/dynamo/vllm_adapter.py", "nemo_rl/models/generation/dynamo/rollout_client.py", "nemo_rl/models/generation/engine_supervisor.py", "nemo_rl/models/generation/fleet_health.py", @@ -253,6 +254,7 @@ project-includes = [ "nemo_rl/models/generation/megatron/token_capture.py", "nemo_rl/models/generation/megatron/utils.py", "nemo_rl/models/generation/openai_server_utils.py", + "nemo_rl/models/generation/rollout_result.py", "nemo_rl/models/generation/capture_context.py", "nemo_rl/models/generation/sglang/config.py", "nemo_rl/models/generation/sglang/utils/http_utils.py", diff --git a/tests/unit/models/generation/test_dynamo_native_engine.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py similarity index 90% rename from tests/unit/models/generation/test_dynamo_native_engine.py rename to tests/unit/models/generation/test_dynamo_vllm_adapter.py index 0ef89990af2..bb52d519ff9 100644 --- a/tests/unit/models/generation/test_dynamo_native_engine.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""NeMo RL's Dynamo engine adapter over a fake AsyncLLM. +"""The vLLM Dynamo adapter over a fake AsyncLLM. The host is a real ``VllmAsyncGenerationWorkerImpl`` built without its constructor, so rollouts are sampled by the native worker's own helpers. @@ -34,7 +34,10 @@ from dynamo.llm.exceptions import InvalidArgument # noqa: E402 -from nemo_rl.models.generation.dynamo import native_engine # noqa: E402 +from nemo_rl.models.generation.dynamo import ( # noqa: E402 + engine_adapter, + vllm_adapter, +) from nemo_rl.models.generation.dynamo.config import ( # noqa: E402 DynamoCfg, DynamoFrontendArgs, @@ -42,11 +45,13 @@ from nemo_rl.models.generation.dynamo.rollout_client import ( # noqa: E402 ROLLOUT_ANNOTATION, ) +from nemo_rl.models.generation.rollout_result import ( # noqa: E402 + rollout_result_from_wire, + rollout_result_to_wire, +) from nemo_rl.models.generation.vllm.request_pipeline import ( # noqa: E402 rollout_output, rollout_result, - rollout_result_from_wire, - rollout_result_to_wire, ) from nemo_rl.models.generation.vllm.vllm_worker_async import ( # noqa: E402 VllmAsyncGenerationWorkerImpl, @@ -137,7 +142,7 @@ def test_rollouts_are_served_like_generate_async() -> None: final = _output([1, 2, 3], _completion([10, 11, 12], "stop")) llm = _FakeLLM([final]) host = _host(llm) - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=3) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=3) (chunk,) = _collect( engine, _rollout_request([1, 2, 3], greedy=True, stop_strings=[""]) @@ -171,7 +176,7 @@ async def abort(request_id): aborted.append(request_id) llm.abort = abort - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) _collect(engine, _rollout_request([1, 2, 3])) # Dynamo's worker calls abort() when the frontend cancels a request. @@ -189,7 +194,7 @@ def test_router_replay_routes_reach_rollouts_as_natively() -> None: prompt_routed_experts=routes[:3], ) host = _host(_FakeLLM([final]), vllm_kwargs={"enable_return_routed_experts": True}) - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) (chunk,) = _collect(engine, _rollout_request([1, 2, 3])) @@ -218,7 +223,7 @@ def test_router_replay_routes_reach_rollouts_as_natively() -> None: def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) (chunk,) = _collect(engine, _rollout_request(list(range(MAX_MODEL_LEN)))) @@ -227,7 +232,7 @@ def test_a_rollout_with_no_room_returns_the_prompt_untruncated() -> None: assert rollout["token_ids"] == [] and rollout["truncated"] is False -def _start_worker_config(monkeypatch, host) -> native_engine.WorkerConfig: +def _start_worker_config(monkeypatch, host) -> engine_adapter.WorkerConfig: """The WorkerConfig ``start_dynamo_endpoint`` registers ``host``'s engine with.""" configs = [] @@ -238,7 +243,7 @@ def __init__(self, engine, config): async def run(self): return None - monkeypatch.setattr(native_engine, "Worker", _Worker) + monkeypatch.setattr(engine_adapter, "Worker", _Worker) env = { "DYN_DISCOVERY_BACKEND": "etcd", "DYN_REQUEST_PLANE": "tcp", @@ -249,12 +254,12 @@ async def run(self): monkeypatch.setenv(key, value) async def start(): - await native_engine.start_dynamo_endpoint( - host, + await engine_adapter.start_dynamo_endpoint( + vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0), model_path="/snapshots/model", + served_model_name=host.model_name, namespace="ns", engine_env=env, - leader_worker_idx=0, ) asyncio.run(start()) @@ -277,7 +282,7 @@ def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: def test_requests_other_than_rollouts_are_refused() -> None: llm = _FakeLLM() - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) with pytest.raises(InvalidArgument, match="only NeMo RL rollouts"): _collect(engine, {"token_ids": [1, 2, 3, 4]}) @@ -292,7 +297,7 @@ async def serve_rollout(*args, **kwargs): raise RuntimeError("the generation output did not include routed_experts") host._serve_rollout = serve_rollout - engine = native_engine.NativeVllmEngine(host, leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(host, leader_worker_idx=0) # A 5xx would reach the driver as "Internal server error". with pytest.raises(InvalidArgument, match="RuntimeError: .*routed_experts"): @@ -336,7 +341,7 @@ async def call_utility_async(method): ] llm.engine_core = SimpleNamespace(call_utility_async=call_utility_async) - engine = native_engine.NativeVllmEngine(_host(llm), leader_worker_idx=0) + engine = vllm_adapter.DynamoVllmAdapter(_host(llm), leader_worker_idx=0) registration = asyncio.run(engine.start(worker_id=0)).llm @@ -356,7 +361,7 @@ def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( ) -> None: cfg = DynamoCfg(frontend_args=DynamoFrontendArgs(router_mode=router_mode)) - config = native_engine.dynamo_kv_events_config(cfg, prefix_caching=prefix_caching) + config = vllm_adapter.dynamo_kv_events_config(cfg, prefix_caching=prefix_caching) # vLLM publishes block events only for blocks its prefix cache stores. assert (config is not None) is publishes @@ -365,7 +370,7 @@ def test_engines_publish_kv_events_only_where_the_kv_router_can_use_them( # A per-engine abstract socket, which vLLM's publisher binds and Dynamo's # source connects to: no port for two engines on a node to collide on. assert config["endpoint"].startswith("ipc://@") - other = native_engine.dynamo_kv_events_config(cfg, prefix_caching=True) + other = vllm_adapter.dynamo_kv_events_config(cfg, prefix_caching=True) assert other["endpoint"] != config["endpoint"] @@ -382,4 +387,4 @@ def test_main_attention_kinds_match_dynamos() -> None: for t in node.targets ) ] - assert {kind.value for kind in native_engine._MAIN_ATTENTION_KINDS} == dynamos + assert {kind.value for kind in vllm_adapter._MAIN_ATTENTION_KINDS} == dynamos diff --git a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py index 62ff7e4b7c2..536f9e404f6 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_decoupling.py +++ b/tests/unit/models/generation/test_dynamo_vllm_decoupling.py @@ -15,9 +15,10 @@ The engines run NeMo RL's own vLLM and use Dynamo only through its engine-agnostic backend SDK, so a vLLM bump can break Dynamo serving only in -NeMo RL's own adapter (``dynamo/native_engine.py``). These guards fail if +NeMo RL's own adapter (``dynamo/vllm_adapter.py``). These guards fail if NeMo RL starts importing Dynamo's vLLM integration (or any other Dynamo -module), or if the SDK itself starts importing vLLM. +module), if the SDK itself starts importing vLLM, or if the Dynamo side shared +by every backend starts importing one. """ import ast @@ -41,22 +42,36 @@ } +# The Dynamo side every generation backend builds on. +BACKEND_NEUTRAL_MODULES = ( + "models/generation/dynamo/engine_adapter.py", + "models/generation/rollout_result.py", +) +GENERATION_BACKENDS = {"vllm", "sglang", "trtllm", "megatron"} + + +def _imports(path: Path) -> set[str]: + """The absolute modules ``path`` imports, at any depth.""" + modules: set[str] = set() + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if isinstance(node, ast.ImportFrom) and node.level == 0: + modules.add(node.module or "") + elif isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + return modules + + def _dynamo_imports() -> dict[str, set[str]]: """The Dynamo modules each nemo_rl source file imports.""" imports: dict[str, set[str]] = {} for path in NEMO_RL_ROOT.rglob("*.py"): - for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): - if isinstance(node, ast.ImportFrom) and node.level == 0: - modules = [node.module or ""] - elif isinstance(node, ast.Import): - modules = [alias.name for alias in node.names] - else: - continue - for module in modules: - if module == "dynamo" or module.startswith("dynamo."): - imports.setdefault(str(path.relative_to(NEMO_RL_ROOT)), set()).add( - module - ) + dynamo = { + module + for module in _imports(path) + if module == "dynamo" or module.startswith("dynamo.") + } + if dynamo: + imports[str(path.relative_to(NEMO_RL_ROOT))] = dynamo return imports @@ -74,6 +89,19 @@ def test_nemo_rl_imports_only_dynamos_backend_sdk() -> None: ) +def test_the_shared_dynamo_side_imports_no_generation_backend() -> None: + backends = { + path: sorted( + module + for module in _imports(NEMO_RL_ROOT / path) + if GENERATION_BACKENDS & set(module.split(".")) + ) + for path in BACKEND_NEUTRAL_MODULES + } + + assert backends == {path: [] for path in BACKEND_NEUTRAL_MODULES} + + @pytest.mark.vllm def test_dynamos_backend_sdk_imports_no_vllm() -> None: pytest.importorskip("dynamo._core") diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 1e8c4440a53..5c9a1a0b971 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -2867,11 +2867,11 @@ def test_vllm_validate_settings_rejects_what_dynamo_cannot_serve(settings, error def test_vllm_dynamo_rollouts_go_through_the_frontend(monkeypatch): from nemo_rl.experience.failures import GenerationUnavailable from nemo_rl.models.generation.dynamo import rollout_client - from nemo_rl.models.generation.vllm.request_pipeline import ( + from nemo_rl.models.generation.rollout_result import ( RolloutResult, - rollout_output, rollout_result_to_wire, ) + from nemo_rl.models.generation.vllm.request_pipeline import rollout_output result = RolloutResult([7, 8], [-0.5, -0.25], False, None, None) responses = [ diff --git a/tests/unit/models/generation/test_vllm_request_pipeline.py b/tests/unit/models/generation/test_vllm_request_pipeline.py index cc9b7cf0725..11244bb1af0 100644 --- a/tests/unit/models/generation/test_vllm_request_pipeline.py +++ b/tests/unit/models/generation/test_vllm_request_pipeline.py @@ -21,13 +21,15 @@ import pytest import torch -from nemo_rl.models.generation.vllm.request_pipeline import ( +from nemo_rl.models.generation.rollout_result import ( RolloutResult, - rollout_output, - rollout_result, rollout_result_from_wire, rollout_result_to_wire, ) +from nemo_rl.models.generation.vllm.request_pipeline import ( + rollout_output, + rollout_result, +) from nemo_rl.models.generation.vllm.utils import R3_MISSING_ROUTE_SENTINEL LAYERS, TOPK = 2, 2 From 096e8728b8f4c64c5581c12683848a01aa78364e Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 6 Nov 2026 18:10:33 -0800 Subject: [PATCH 18/19] docs(dynamo): scope the Dynamo guide to the SingleController C58. Every example already uses the SingleController, which is the path the Dynamo overlays test. The design doc names the adapter split. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- docs/design-docs/dynamo-integration.md | 18 ++++++++++++++---- docs/guides/dynamo-generation.md | 9 +++++---- docs/index.md | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/design-docs/dynamo-integration.md b/docs/design-docs/dynamo-integration.md index 4b7e95277e7..245e9629127 100644 --- a/docs/design-docs/dynamo-integration.md +++ b/docs/design-docs/dynamo-integration.md @@ -7,8 +7,8 @@ Dynamo's frontend and KV-aware router. Two requirements shape the design: - Dynamo CI coverage must stay consistent with native vLLM's. Both hold only if the engine process contains no Dynamo-owned vLLM code. A vLLM -bump can then break Dynamo serving only in NeMo RL's own adapter -(`native_engine.py`), which calls a few vLLM APIs native NeMo RL does not: the +bump can then break Dynamo serving only in NeMo RL's own vLLM adapter +(`vllm_adapter.py`), which calls a few vLLM APIs native NeMo RL does not: the `get_kv_cache_group_metadata` engine utility and `KVCacheSpecKind` at registration, and `StatLoggerBase` for the router's load signal. A renamed import fails the vLLM-lane unit tests at collection, and the Dynamo functional overlays @@ -22,8 +22,18 @@ also runs `dynamo.common.backend.Worker` as a task on its own actor event loop, which is the loop that owns the `AsyncLLM`. `dynamo.common.backend` is Dynamo's engine-agnostic backend SDK and imports no -vLLM. The adapter, `nemo_rl/models/generation/dynamo/native_engine.py`, -implements its `LLMEngine` over the worker's `AsyncLLM`: +vLLM. NeMo RL implements its `LLMEngine` in two parts: + +- `dynamo/engine_adapter.py`: `DynamoEngineAdapter`, the Dynamo side shared by + every generation backend. It serves request kinds as handlers keyed by their + annotation, maps engine errors to a 400 that keeps the cause, and registers the + endpoint for `/v1/completions` only. It imports no backend, and the engine only + ever receives token IDs. +- `dynamo/vllm_adapter.py`: `DynamoVllmAdapter`, the vLLM backend over the + worker's `AsyncLLM`. A backend implements `serve_rollout`, `start`, `abort` and + `kv_event_sources`. + +The two together: - `generate` serves only the rollouts NeMo RL itself sends, which carry an annotation, and refuses other requests. The worker's own `_serve_rollout` runs diff --git a/docs/guides/dynamo-generation.md b/docs/guides/dynamo-generation.md index 52d65c6cfa3..1f5652e1e5d 100644 --- a/docs/guides/dynamo-generation.md +++ b/docs/guides/dynamo-generation.md @@ -1,6 +1,7 @@ # Dynamo serving for vLLM generation -Any non-colocated vLLM generation config can serve its rollouts through +SingleController runs (`examples/run_grpo_single_controller.py`) with non-colocated +vLLM generation can serve their rollouts through [Dynamo](https://github.com/ai-dynamo/dynamo)'s OpenAI frontend and KV-aware router. Add one line: @@ -80,8 +81,7 @@ Startup rejects these with `dynamo: true`: - NeMo-Gym (`env.should_use_nemo_gym`): the frontend does not serve Gym's calls yet. Run Gym recipes with `dynamo: false`. - `colocated.enabled: true`: a colocated engine sleeps while the policy trains, - and the frontend would keep routing to it. The SingleController never runs - colocated vLLM; the other entrypoints need non-colocated generation. + and the frontend would keep routing to it. - `async_rl.generation_fleet_health.enabled`: the frontend picks engines itself, and a restarted shard does not register with it yet. - `vllm_cfg.expert_parallel_size > tensor_parallel_size`: that runs data @@ -120,7 +120,8 @@ bash docker/dynamo/install.sh ## Run -Use any non-colocated vLLM recipe that does not use NeMo-Gym, and add the switch: +Use any non-colocated SingleController vLLM recipe that does not use NeMo-Gym, +and add the switch: ```bash uv run examples/run_grpo_single_controller.py \ diff --git a/docs/index.md b/docs/index.md index 9f03e656b34..f99244492a7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -83,7 +83,7 @@ Configure and launch NeMo RL on multi-node Slurm or Kubernetes clusters for dist :link: guides/dynamo-generation :link-type: doc -Serve non-colocated vLLM generation through Dynamo's frontend and KV-aware router with `dynamo: true`. +Serve the SingleController's non-colocated vLLM generation through Dynamo's frontend and KV-aware router with `dynamo: true`. ::: :::: From c6a4c66bceb4ed9994d92d4ad9557a5d8d982d9e Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Mon, 9 Nov 2026 10:49:36 -0800 Subject: [PATCH 19/19] fix(dynamo): keep the generation worker's logging under Dynamo's Worker C59. Dynamo's Worker.run() starts with configure_dynamo_logging(), which replaces the whole actor's Python logging: it removes NeMo RL's root handler, sets the root level from DYN_LOG (so NRL_LOG_LEVEL=DEBUG records are dropped), and swaps vLLM's handlers for an always-ANSI formatter. start_dynamo_endpoint now replaces that module global with a no-op before the Worker starts. A test checks the replacement keeps the root handlers, and another fails if Worker.run() stops calling the setup by that name, so a Dynamo bump cannot silently undo it. ai-dynamo/dynamo#18447 asks for a WorkerConfig opt-out that replaces this. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- .../models/generation/dynamo/engine_adapter.py | 11 +++++++++++ .../generation/test_dynamo_vllm_adapter.py | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/nemo_rl/models/generation/dynamo/engine_adapter.py b/nemo_rl/models/generation/dynamo/engine_adapter.py index 9d21a0383fc..40ae920be17 100644 --- a/nemo_rl/models/generation/dynamo/engine_adapter.py +++ b/nemo_rl/models/generation/dynamo/engine_adapter.py @@ -41,6 +41,7 @@ Worker, WorkerConfig, ) +from dynamo.common.backend import worker as dynamo_worker from dynamo.common.backend.publisher import KvEventSource from dynamo.llm.exceptions import InvalidArgument @@ -71,6 +72,14 @@ class ServedRollout: usage: Optional[dict[str, Any]] +def _keep_host_logging(*args: object, **kwargs: object) -> None: + """Replaces Dynamo's ``configure_dynamo_logging`` (ai-dynamo/dynamo#18447). + + ``Worker.run()`` calls it first, and it would replace this process's Python + logging: NeMo RL's root handler and level, and vLLM's handlers. + """ + + def _annotation_value(request: GenerateRequest, prefix: str) -> Optional[str]: """The value of the first ```` annotation, or None.""" for annotation in request.get("annotations") or (): @@ -190,6 +199,8 @@ def start_dynamo_endpoint( """ # The Rust runtime reads these when the first Worker is constructed. os.environ.update(engine_env) + # The generation worker keeps its own logging until WorkerConfig can opt out. + dynamo_worker.configure_dynamo_logging = _keep_host_logging config = WorkerConfig( namespace=namespace, component=DYNAMO_COMPONENT, diff --git a/tests/unit/models/generation/test_dynamo_vllm_adapter.py b/tests/unit/models/generation/test_dynamo_vllm_adapter.py index bb52d519ff9..04ba37a7c77 100644 --- a/tests/unit/models/generation/test_dynamo_vllm_adapter.py +++ b/tests/unit/models/generation/test_dynamo_vllm_adapter.py @@ -21,6 +21,7 @@ import asyncio import importlib.util import json +import logging from pathlib import Path from types import SimpleNamespace @@ -244,6 +245,12 @@ async def run(self): return None monkeypatch.setattr(engine_adapter, "Worker", _Worker) + # Registered so the engine's replacement of Dynamo's logging setup is undone. + monkeypatch.setattr( + engine_adapter.dynamo_worker, + "configure_dynamo_logging", + engine_adapter.dynamo_worker.configure_dynamo_logging, + ) env = { "DYN_DISCOVERY_BACKEND": "etcd", "DYN_REQUEST_PLANE": "tcp", @@ -278,6 +285,17 @@ def test_engines_register_for_rollouts_on_completions_only(monkeypatch) -> None: ) # NeMo RL's rollouts go to /v1/completions; no chat surface is registered. assert config.endpoint_types == "completions" + # Worker.run() starts with Dynamo's logging setup, which keeps the worker's. + handlers = logging.getLogger().handlers[:] + engine_adapter.dynamo_worker.configure_dynamo_logging() + assert logging.getLogger().handlers == handlers + + +def test_worker_run_still_calls_the_logging_setup_the_engine_replaces() -> None: + # Otherwise a Dynamo bump would silently take over the worker's logging again. + run = engine_adapter.Worker.run + assert run.__globals__ is vars(engine_adapter.dynamo_worker) + assert "configure_dynamo_logging" in run.__code__.co_names def test_requests_other_than_rollouts_are_refused() -> None: