Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/design-docs/nemo-gym-integration.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ policy:
env:
should_use_nemo_gym: true # Enables NeMo Gym integration
nemo_gym:
truncate_noncontiguous_episodes: false
# NeMo Gym config paths and settings
config_paths:
- resources_servers/math/configs/math.yaml
Expand All @@ -41,6 +42,8 @@ logger:
log_nemo_gym_full_result_tables: false
```

`env.nemo_gym.truncate_noncontiguous_episodes` defaults to `false`, so a token-prefix discontinuity raises an error. When set to `true`, NeMo RL stops at the first trainable turn whose prompt does not preserve the tokens already seen, drops that turn and the remaining tail, and keeps the preceding contiguous prefix with the episode reward. This option cannot be used with `policy.router_replay.enabled: true` or `token_capture.enabled: true`. Truncation applies only to the legacy token-echo path; token-capture receipt mode bypasses that path and retains its own integrity checks. Every legacy result includes a boolean `noncontiguous_truncated` flag; its per-agent `{agent_name}/noncontiguous_truncated/mean` metric reports the fraction of episodes whose non-contiguous tail was dropped.

When `log_nemo_gym_full_result_tables` is `false`, NeMo RL does not construct the per-agent `full_result` Tables. This prevents those payloads from entering the async replay buffer and avoids uploading them to W&B. Numeric per-agent rollout metrics are unaffected. Set the flag to `true` only when the complete Gym result payloads are needed for a short debugging run.

The exposed vLLM HTTP server binds every interface of its node and, by default, checks no credential on its OpenAI routes; the NeMo Gym model server sends the placeholder key `dummy_key`. Set `policy.generation.vllm_cfg.http_server_api_key_required: true` to require a bearer token instead (the setting needs `expose_http_server: true` and is refused at setup without it): NeMo RL generates one random key per job, every vLLM worker refuses a `/v1/*` or `/tokenize` request without `Authorization: Bearer <key>` (HTTP 401), and the same key reaches the NeMo Gym model server as `policy_api_key`. The saved and logged job configuration never carries the key: the vLLM workers receive it in their own configuration copy, and the Gym servers receive it as an environment variable (`NEMO_RL_POLICY_API_KEY`) that each server resolves in its own process, so Gym's serialized global config, its head server's `/global_config_dict_yaml` endpoint and the server command lines carry only the variable name. The residual exposure is the server processes' environment and the Gym actor's constructor arguments, the same as for the token-capture secret. The setting covers the vLLM backend only: the Megatron, TRT-LLM and Dynamo exposed servers check no credential and their Gym actors keep `dummy_key` (Dynamo rejects the setting at configuration time rather than ignoring it).
Expand Down
4 changes: 4 additions & 0 deletions examples/nemo_gym/grpo_nanov3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,10 @@ env:
# RL (3000-4999) or vLLM (7000-8999). See ray.sub for the full port layout.
port_range_low: 5000
port_range_high: 5999
# Opt in to keeping the contiguous prefix when a later turn's token prefix
# diverges. The retained prefix uses the episode reward. Incompatible with
# policy.router_replay.enabled and token_capture.enabled.
truncate_noncontiguous_episodes: false
# Optional substrings/tags used to flag malformed assistant turns for advantage
# penalties (see grpo.invalid_tool_call_advantage / malformed_thinking_advantage).
# Omit to use the built-in defaults (DEFAULT_INVALID_TOOL_CALL_PATTERNS / DEFAULT_THINKING_TAGS).
Expand Down
58 changes: 49 additions & 9 deletions nemo_rl/environments/nemo_gym.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
OTEL_RESOURCE_ATTRIBUTES_ENV,
extend_otel_resource_attributes,
)
from pydantic import BaseModel, Field, field_validator, model_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from ray.util.placement_group import (
PlacementGroup,
placement_group,
Expand Down Expand Up @@ -288,6 +288,16 @@ def _require_resolved_agent_refs(nemo_gym_examples: list[dict]) -> None:
)


class NemoGymPostprocessConfig(BaseModel):
"""NeMo-RL postprocessing options nested under ``env.nemo_gym``."""

model_config = ConfigDict(extra="allow")

invalid_tool_call_patterns: list[str] | None = None
thinking_tags: list[str] | None = None
truncate_noncontiguous_episodes: bool = False


def _prepare_materialized_nemo_gym_rows(
rows: list[dict],
environment_server_routes: Mapping[str, str],
Expand Down Expand Up @@ -727,6 +737,9 @@ class NemoGymConfig(TypedDict):
pad_dynamic_image_shapes: NotRequired[
bool
] # Preserve heterogeneous shapes for native-resolution patchification
# Opt-in recovery for token-prefix discontinuities in multi-turn rollouts.
# The default is supplied by NemoGymPostprocessConfig.
truncate_noncontiguous_episodes: bool
# Ledger-authoritative token capture (token_capture.enabled): the dumped
# TokenCaptureConfig. Turns on external staging in Gym's policy model
# server, switches run_rollouts to receipt mode, and assembles receipts
Expand Down Expand Up @@ -2534,6 +2547,7 @@ def _postprocess_nemo_gym_to_nemo_rl_result(
media_geometry_failed = False
seen_token_ids: List[int] = []
batch_decode_items = []
nemo_gym_result["noncontiguous_truncated"] = False
reasoning_text = nemo_gym_result.get("reasoning_text")
answer_text = nemo_gym_result.get("answer_text")
response_token_counts = None
Expand All @@ -2555,10 +2569,19 @@ def _postprocess_nemo_gym_to_nemo_rl_result(
if not _is_trainable_output_item(output_item_dict):
continue

assert (
is_contiguous = (
seen_token_ids
== output_item_dict["prompt_token_ids"][: len(seen_token_ids)]
), f"""Non-contiguous messages found! This may be a tokenization issue where certain tokens are combined when messages are concatenated, or it may be due to part of the chat history being truncated (like if super long history is truncated or if reasoning is stripped out).
)
if not is_contiguous and self.cfg["truncate_noncontiguous_episodes"]:
print(
"[nemo_gym] WARNING: non-contiguous turn; truncating episode "
f"at {len(nemo_rl_message_log)} messages "
f"(seen={len(seen_token_ids)} tokens); dropping corrupted tail."
)
nemo_gym_result["noncontiguous_truncated"] = True
break
Comment thread
yuki-97 marked this conversation as resolved.
Resolved
assert is_contiguous, f"""Non-contiguous messages found! This may be a tokenization issue where certain tokens are combined when messages are concatenated, or it may be due to part of the chat history being truncated (like if super long history is truncated or if reasoning is stripped out).
Seen token IDs: {seen_token_ids}
Output prompt token IDs: {output_item_dict["prompt_token_ids"]}
output prompt token ids till seen: {output_item_dict["prompt_token_ids"][: len(seen_token_ids)]}
Expand Down Expand Up @@ -2971,7 +2994,7 @@ def build_nemo_gym_config(
Args:
env_configs: The master_config.env mapping; env_configs["nemo_gym"] supplies
the Gym global config plus NeMo-RL detection knobs (invalid_tool_call_patterns,
thinking_tags, num_gpu_nodes).
thinking_tags, truncate_noncontiguous_episodes, num_gpu_nodes).
base_urls: Per-DP-rank OpenAI-compatible server base URLs from the generation backend.
model_name: Served model name the Gym rollouts should target.
enable_router_replay: Sets ``require_routed_experts`` and selects the
Expand Down Expand Up @@ -3041,7 +3064,25 @@ def _build_gym_actor_config(
Shared by the unsharded path and by each shard, so every actor gets the
same treatment of NeMo-RL-side keys regardless of how it was composed.
"""
nemo_gym_dict = dict(nemo_gym_dict)
postprocess_cfg = NemoGymPostprocessConfig.model_validate(nemo_gym_dict)
nemo_gym_dict = {
key: value
for key, value in nemo_gym_dict.items()
if key not in NemoGymPostprocessConfig.model_fields
}
if postprocess_cfg.truncate_noncontiguous_episodes:
if enable_router_replay:
raise ValueError(
"truncate_noncontiguous_episodes is not compatible with router replay: "
"the next contiguous prompt is required to repair the retained prefix's "
"final route."
)
if token_capture and token_capture.get("enabled"):
raise ValueError(
"truncate_noncontiguous_episodes is not compatible with "
"token_capture.enabled: receipt mode bypasses the legacy token-prefix "
"check, so it cannot truncate non-contiguous episodes."
)

# NeMo-RL-only keys are consumed here and must never reach Gym: the merged
# config is serialized into every Gym child process, and unrecognized
Expand All @@ -3051,8 +3092,6 @@ def _build_gym_actor_config(

# NeMo-RL-side detection knobs are top-level NemoGymConfig fields
# (where the detector reads them), not part of Gym's global config.
invalid_tool_call_patterns = nemo_gym_dict.pop("invalid_tool_call_patterns", None)
thinking_tags = nemo_gym_dict.pop("thinking_tags", None)
tokenizer_config = nemo_gym_dict.pop("tokenizer_config", None)
external_service_readiness = extract_external_service_readiness(nemo_gym_dict)
port_range = {
Expand Down Expand Up @@ -3131,12 +3170,13 @@ def _build_gym_actor_config(
model_name=model_name,
base_urls=base_urls,
policy_api_key=policy_api_key,
invalid_tool_call_patterns=invalid_tool_call_patterns,
thinking_tags=thinking_tags,
invalid_tool_call_patterns=postprocess_cfg.invalid_tool_call_patterns,
thinking_tags=postprocess_cfg.thinking_tags,
tokenizer_config=tokenizer_config,
require_routed_experts=enable_router_replay,
routed_experts_dtype=routed_experts_dtype,
use_fastokens=use_fastokens,
truncate_noncontiguous_episodes=postprocess_cfg.truncate_noncontiguous_episodes,
external_service_readiness=external_service_readiness,
initial_global_config_dict=nemo_gym_dict,
token_capture=instance_token_capture,
Expand Down
1 change: 1 addition & 0 deletions tests/unit/algorithms/test_distillation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1257,6 +1257,7 @@ def test_distillation_setup_nemo_gym_uses_deferred_vllm(monkeypatch, vllm_start_
"num_gpu_nodes": 1,
"invalid_tool_call_patterns": ["bad_call"],
"thinking_tags": ["<think>"],
"truncate_noncontiguous_episodes": True,
"config_paths": ["gym.yaml"],
}

Expand Down
120 changes: 118 additions & 2 deletions tests/unit/environments/test_nemo_gym.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
extract_external_service_readiness,
extract_reward_components,
setup_nemo_gym_config,
spinup_nemo_gym_actor,
validate_reward_components_match_scalar,
)
from nemo_rl.environments.nemo_gym_multimodal import (
Expand All @@ -70,6 +71,7 @@
_expand_nemotron_video_placeholders,
_flatten_nemotron_video_frame_messages,
)
from nemo_rl.experience.metric_utils import calculate_single_metric
from nemo_rl.experience.rollouts import (
_reattach_original_multimodal_payloads,
_reattach_static_multimodal_payloads_to_result,
Expand Down Expand Up @@ -202,6 +204,19 @@ def test_multimodal_content_types_cover_responses_media_aliases():
} <= MULTIMODAL_CONTENT_TYPES


def test_spinup_nemo_gym_rejects_truncation_with_router_replay():
with pytest.raises(ValueError, match="not compatible with router replay"):
spinup_nemo_gym_actor(
{"nemo_gym": {"truncate_noncontiguous_episodes": True}},
base_urls=["http://127.0.0.1:30000/v1"],
model_name="test-model",
tokenizer=object(),
enable_router_replay=True,
use_fastokens=False,
turn_checkpointing_enabled=False,
)


def _external_health_response(
*, status: str, healthy_backends: int, total_backends: int
) -> MagicMock:
Expand Down Expand Up @@ -1432,6 +1447,7 @@ def nemo_gym(nemo_gym_vllm_generation, nemo_gym_tokenizer): # noqa: F811
model_name=nemo_gym_vllm_generation.cfg["model_name"],
base_urls=nemo_gym_vllm_generation.dp_openai_server_base_urls,
initial_global_config_dict=safe_load(yaml_str),
truncate_noncontiguous_episodes=False,
checkpoint_instance=GymCheckpointInstance("nemo_gym", 0),
turn_checkpointing_enabled=False,
)
Expand Down Expand Up @@ -1552,6 +1568,7 @@ def test_run_rollouts_takes_a_trace_carrier_without_disturbing_the_body():
asyncio.run(stream.__anext__())


@pytest.mark.parametrize("truncate_noncontiguous_episodes", [False, True])
@pytest.mark.parametrize(
("token_fields", "expected_count"),
[
Expand All @@ -1572,7 +1589,9 @@ def test_run_rollouts_takes_a_trace_carrier_without_disturbing_the_body():
),
],
)
def test_nemo_gym_postprocess_uses_batch_decode(token_fields, expected_count):
def test_nemo_gym_postprocess_uses_batch_decode(
truncate_noncontiguous_episodes, token_fields, expected_count
):
class _Tokenizer:
def __init__(self):
self.batch_decode_calls = []
Expand Down Expand Up @@ -1610,7 +1629,7 @@ def batch_decode(self, batch):
nemo_gym_result.update(token_fields)

class _MockSelf:
cfg = {}
cfg = {"truncate_noncontiguous_episodes": truncate_noncontiguous_episodes}

result = (
NemoGym.__ray_metadata__.modified_class._postprocess_nemo_gym_to_nemo_rl_result(
Expand All @@ -1626,6 +1645,7 @@ class _MockSelf:
assert result["message_log"][1]["token_ids"].tolist() == [3]
assert result["message_log"][2]["token_ids"].tolist() == [4, 5]
assert result["message_log"][3]["token_ids"].tolist() == [6, 7]
assert result["full_result"]["noncontiguous_truncated"] is False
assert result["reasoning_token_count"] == expected_count
assert result["response_token_count"] == expected_count
assert result["token_extraction_valid"] is (expected_count is not None)
Expand All @@ -1640,6 +1660,102 @@ class _MockSelf:
assert nemo_gym_result["response"]["output"][1]["generation_str"] == "6 7"


def _make_noncontiguous_nemo_gym_result():
return {
"reward": 0.75,
"response": {
"output": [
{
"prompt_token_ids": [1, 2],
"generation_token_ids": [3],
"generation_log_probs": [-0.1],
},
{
"prompt_token_ids": [1, 99, 3, 4],
"generation_token_ids": [6],
"generation_log_probs": [-0.2],
},
]
},
"responses_create_params": {"input": []},
}


class _JoinTokenizer:
def batch_decode(self, batch):
return [" ".join(map(str, token_ids)) for token_ids in batch]


def test_nemo_gym_postprocess_noncontiguous_asserts_when_truncation_disabled():
class _MockSelf:
cfg = {"truncate_noncontiguous_episodes": False}

with pytest.raises(AssertionError, match="Non-contiguous messages found"):
NemoGym.__ray_metadata__.modified_class._postprocess_nemo_gym_to_nemo_rl_result(
_MockSelf(), {}, _make_noncontiguous_nemo_gym_result(), _JoinTokenizer()
)


@pytest.mark.parametrize("divergent_prompt", [[1, 99, 3, 4], [1]])
def test_nemo_gym_postprocess_noncontiguous_truncates_when_enabled(divergent_prompt):
class _MockSelf:
cfg = {"truncate_noncontiguous_episodes": True}

nemo_gym_result = _make_noncontiguous_nemo_gym_result()
output = nemo_gym_result["response"]["output"]
output[1]["prompt_token_ids"] = divergent_prompt
# Even a later turn that matches the retained prefix belongs to the dropped tail.
output.append(
{
"prompt_token_ids": [1, 2, 3, 4],
"generation_token_ids": [7],
"generation_log_probs": [-0.3],
}
)
dropped_tail = deepcopy(output[1:])
result = (
NemoGym.__ray_metadata__.modified_class._postprocess_nemo_gym_to_nemo_rl_result(
_MockSelf(), {}, nemo_gym_result, _JoinTokenizer()
)
)

assert len(result["message_log"]) == 2
assert result["message_log"][0]["token_ids"].tolist() == [1, 2]
assert result["message_log"][1]["token_ids"].tolist() == [3]
assert result["message_log"][1]["role"] == "assistant"
torch.testing.assert_close(
result["message_log"][1]["generation_logprobs"], torch.tensor([-0.1])
)
assert result["input_message_log"] == result["message_log"][:1]
assert result["full_result"]["reward"] == 0.75
assert result["full_result"]["noncontiguous_truncated"] is True
assert output[0]["generation_str"] == "3"
assert output[1:] == dropped_tail


@pytest.mark.parametrize("first_truncated", [False, True])
def test_nemo_gym_postprocess_reports_truncation_for_every_result(first_truncated):
mock_self = SimpleNamespace(cfg={"truncate_noncontiguous_episodes": True})
results = []
for should_truncate in (first_truncated, not first_truncated):
raw_result = _make_noncontiguous_nemo_gym_result()
raw_result["noncontiguous_truncated"] = not should_truncate
if not should_truncate:
raw_result["response"]["output"][1]["prompt_token_ids"] = [1, 2, 3, 4]
result = NemoGym.__ray_metadata__.modified_class._postprocess_nemo_gym_to_nemo_rl_result(
mock_self, {}, raw_result, _JoinTokenizer()
)
assert result["full_result"]["noncontiguous_truncated"] is should_truncate
results.append(result["full_result"])

metrics = calculate_single_metric(
[float(result["noncontiguous_truncated"]) for result in results],
len(results),
"agent/noncontiguous_truncated",
)
assert metrics["agent/noncontiguous_truncated/mean"] == 0.5


@pytest.mark.parametrize("include_initial_multimodal_data", [False, True])
def test_nemo_gym_dedup_redacts_initial_images_from_actor_return(
include_initial_multimodal_data,
Expand Down
Loading
Loading