diff --git a/docs/guides/single-controller.md b/docs/guides/single-controller.md index 7bcad18f126..779a6524480 100644 --- a/docs/guides/single-controller.md +++ b/docs/guides/single-controller.md @@ -264,7 +264,7 @@ If one shard replica fails restore, every replica conservatively retires its rep If no rollout snapshot follows the newest trainer checkpoint, SC restores that checkpoint on its own: completed groups come back from TQ and unfinished turn- and prefix-level rollouts restart from their tasks. A rollout snapshot without its `gym_checkpoint.json` is refused. -The `prefix` target needs everything the `turn` target needs (listed below), and does not yet support multimodal capture, router replay, or `policy.generation.top_logprobs > 0`. Each cut must finish within `token_capture.control_timeout_s`; a call that is not cut in time restarts from its last saved turn. Some calls are never cut and also restart from their last saved turn: +The `prefix` target needs everything the `turn` target needs (listed below), and does not yet support router replay or `policy.generation.top_logprobs > 0`. Each cut must finish within `token_capture.control_timeout_s`; a call that is not cut in time restarts from its last saved turn. Some calls are never cut and also restart from their last saved turn: - Calls using stop strings, multi-token `bad_words`, or a per-request `seed` while sampling, because a resumed call could not apply them to the tokens it already decoded. - Calls with structured output (a non-text `response_format`, `guided_*` or `structured_outputs`) or a `tool_choice` other than `auto` or `none`, which NeMo Gym leaves uncut. @@ -450,6 +450,7 @@ The SC path is still under active development. Feature gaps are tracked in [issu RL hands the owned tensors (`imgs`, `imgs_sizes`, and optional `num_frames`) to Gym's `complete_call_from_response` as opaque attachments, and the TQ sink writes them in the same `put` as the token columns, so `staged` coordinates acknowledge tokens and pixels together and a failed write is `capture_failed` at call time. Tensors keep their native shapes and dtypes on the wire; two per-row flags (`media_present`, `media_has_frames`) mark which rows carry pixels and whether they are video. Media-enabled staging partitions carry these columns on every row; text-only runs register and read none of them. vLLM pixels are rearranged losslessly into packed patches; the finalizer reads the presence flags with the base columns, then issues one batched read of the tensor columns for the terminal-chain calls that carry media, and publishes `pixel_values`, `imgs_sizes`, and `num_frames` for the existing Megatron learner without resampling or normalizing the media again. Only newly introduced occurrences are staged. vLLM-specific `media_spans` extras retain placeholder positions and token hashes for multi-turn prefix replacement, including video's timestamp-separated visual-token spans. + With `rollout_recovery.target_level: prefix`, the first durable prefix row for a model call carries that call's new media metadata and a SHA-256 digest of each packed tensor, not the pixels; later token chunks carry no media. The prefix digest covers that metadata, so a restore checks it with the prefix, then reprocesses the reissued request, checks its media against those tensor digests, and stages the reprocessed tensors in the canonical terminal row. A prefix that fails its digest check makes the call regenerate from its turn boundary, like any unusable prefix; reprocessed media that does not match the digests is rejected with `retained_media_changed`. The original image or video must therefore still be resolvable by vLLM when the request is reissued: this path does not provide source-free media prefill if the original media has disappeared. Capture requests use vLLM's `skip_mm_cache=True` path to obtain concrete processor tensors. vLLM can still reuse its processor-only cache; this does not guarantee fresh preprocessing. vLLM tiles images from a dummy prompt, so the request text never changes geometry, but the images of one request share the token budget: adding an image under a tight budget can re-tile a retained one, and a warm processor-only cache can keep a geometry a fresh processor would not reproduce. If that changes retained geometry or placeholder tokens, the worker rejects the continuation before inference with HTTP 400 and error code `retained_media_changed` (other capture-time validation failures use `media_capture_rejected`). Gym's current exception middleware wraps the upstream error in HTTP 500, retaining the code in the response body. RL then classifies it as an infrastructure `GymTransportError`, eligible for the configured prompt retry policy; it is not a dedicated terminal rejection. Text-call rows carry sentinels in each column's own dtype, because TransferQueue keeps one dtype per field across live rows. Media-enabled rows also store `media_metadata_digest`, a SHA-256 checksum of the small route-less extras JSON. Sources check it before exposing media descriptors, without reading pixels or routed-expert tensors. Missing or changed checksums reject the row. This detects accidental metadata corruption; it is independent of Gym's receipt and combined `extras_digest` commitment. diff --git a/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml b/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml index 5353ff36d4c..14da19e6651 100644 --- a/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml +++ b/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml @@ -2,15 +2,35 @@ # generation, a Megatron learner, and vLLM media token capture enabled. # # Functional-test config (not a nightly recipe, so it lives beside the other -# Gym smoke configs rather than under examples/configs/recipes/): -# tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh -# supplies the data paths, the 1 trainer + 1 generator split, and the step -# budget. Token capture requires the Gym rollout path, so the native CLEVR +# Gym smoke configs rather than under examples/configs/recipes/). It pins the +# 1n2g split both Omni CLEVR vLLM tests use: one GPU trains the frozen-decoder +# policy, one hosts async vLLM. The tests +# (tests/functional/nemotron_omni_clevr_vllm_*_single_controller_1n2g.sh) +# supply the data paths, sequence lengths and step budget. Token capture +# requires the Gym rollout path, so the native CLEVR # dataset/environment of the parent recipe are swapped for Gym rows here. # Prefix caching stays off (hybrid-Mamba align cache mode asserts chunked # prefill) and the parent's mm_processor_cache_gb=0 keeps concrete pixels # available to the capture host. defaults: ../configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-clevr-8n4g-megatron-single-controller-async.v1.yaml +cluster: + num_nodes: 1 + gpus_per_node: 2 + segment_size: 1 +grpo: + val_period: 0 + val_at_start: false + val_at_end: false +async_rl: + generation_fleet_health: + refit_timeout_s: null +data_plane: + simple: + num_storage_units: 2 +logger: + tensorboard_enabled: true + wandb_enabled: false + monitor_gpus: false data: _override_: true max_input_seq_length: null @@ -52,10 +72,39 @@ env: simple_agent: max_steps: 1 policy: + train_micro_batch_size: 1 + megatron_cfg: + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + context_parallel_size: 1 + sequence_parallel: true + activation_checkpointing: true + bias_dropout_fusion: false + freeze_config: + freeze_language_model: true + optimizer: + optimizer_cpu_offload: false + optimizer_offload_fraction: 0.0 + params_dtype: bfloat16 + main_grads_dtype: bfloat16 + main_params_dtype: float16 + exp_avg_dtype: bfloat16 + exp_avg_sq_dtype: bfloat16 + store_param_remainders: false generation: backend: vllm refit_transport: null + colocated: + enabled: false + resources: + num_nodes: 1 + gpus_per_node: 1 vllm_cfg: + tensor_parallel_size: 1 + expert_parallel_size: 1 + pipeline_parallel_size: 1 + gpu_memory_utilization: 0.6 async_engine: true expose_http_server: true logprobs_mode: raw_logprobs diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index 70b1ac24288..ac8872ced81 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -1347,10 +1347,6 @@ def setup_single_controller( ) if generation_prefix_cuts_enabled: # The vLLM-only check ran above with the other turn/prefix checks. - if capture_media: - raise NotImplementedError( - "generation-prefix recovery does not yet support multimodal capture" - ) if router_replay_enabled(master_config.policy): raise NotImplementedError( "generation-prefix recovery does not yet support router replay" diff --git a/nemo_rl/data/captured_media.py b/nemo_rl/data/captured_media.py index c832e77beb0..e0ceba5a9cc 100644 --- a/nemo_rl/data/captured_media.py +++ b/nemo_rl/data/captured_media.py @@ -22,6 +22,7 @@ import hashlib import json +from collections.abc import Mapping from dataclasses import asdict, dataclass, replace from typing import TYPE_CHECKING, Any, Literal @@ -138,10 +139,67 @@ def verify_tokens(self, tokens: list[int], *, origin: int) -> None: @dataclass(frozen=True) class CapturedMedia: - """New placeholder metadata and owned tensors for the common staging sink.""" + """New placeholder metadata and owned tensors for the common staging sink. + + ``tensor_digests`` fingerprints ``tensors`` (see ``media_tensor_digests``) + when generation-prefix cuts are on. A media bundle restored from a prefix + row carries the staged digests and no tensors. + """ items: tuple[CapturedMediaItem, ...] tensors: dict[str, torch.Tensor] | None + tensor_digests: dict[str, dict[str, Any]] | None = None + + +# Generation-prefix extras key for the call's tensor digests. Prefix rows store +# these instead of the pixels: a restore reprocesses the media anyway and only +# has to prove it produced the bytes the interrupted call staged. +MEDIA_TENSOR_DIGESTS_FIELD = "media_tensor_digests" + + +def media_tensor_digests( + tensors: Mapping[str, torch.Tensor] | None, +) -> dict[str, dict[str, Any]]: + """Fingerprint each media tensor by its dtype, shape, and raw bytes.""" + digests: dict[str, dict[str, Any]] = {} + for name, value in sorted((tensors or {}).items()): + tensor = value.detach().cpu().contiguous() + raw = tensor.reshape(-1).view(torch.uint8).numpy() + digests[name] = { + "dtype": str(tensor.dtype), + "shape": list(tensor.shape), + "sha256": hashlib.sha256(raw).hexdigest(), + } + return digests + + +def verify_restored_media(captured: CapturedMedia, restored: CapturedMedia) -> None: + """Require reprocessed media to match what an interrupted call staged. + + Args: + captured: The media just reprocessed from the resumed request. + restored: The items and tensor digests read from the call's first + generation-prefix row. + + Raises: + MediaCaptureRejected: ``retained_media_changed`` when the items or + any tensor's dtype, shape, or bytes differ. + """ + current = captured.tensor_digests or {} + staged = restored.tensor_digests or {} + if captured.items != restored.items or current != staged: + # Name what differs, so a rejection can be triaged from the log. + changed = sorted( + name + for name in current.keys() | staged.keys() + if current.get(name) != staged.get(name) + ) + raise MediaCaptureRejected( + "Restored generation media differs from reprocessed input " + f"(items equal: {captured.items == restored.items}; " + f"changed tensors: {changed})", + code="retained_media_changed", + ) def verified_media_items(call: FetchedStagedCall) -> tuple[CapturedMediaItem, ...]: diff --git a/nemo_rl/models/generation/prefix_recovery/parts.py b/nemo_rl/models/generation/prefix_recovery/parts.py index 6fb2455fd5b..9f4a433594a 100644 --- a/nemo_rl/models/generation/prefix_recovery/parts.py +++ b/nemo_rl/models/generation/prefix_recovery/parts.py @@ -21,7 +21,10 @@ from typing import TYPE_CHECKING, Any from nemo_rl.models.generation.prefix_recovery.progress import ProgressReader -from nemo_rl.models.generation.prefix_recovery.state import CaptureRegistry +from nemo_rl.models.generation.prefix_recovery.state import ( + CaptureRegistry, + _RequestCaptureState, +) if TYPE_CHECKING: from nemo_rl.data_plane.tq_token_sink import TQTokenSink, TQTokenSource @@ -49,3 +52,6 @@ class PrefixRecoveryParts: # Bound on one cut, measured from its arrival; None waits indefinitely. control_timeout_s: float | None progress_reader: ProgressReader + # Extras for a call's first prefix row, from the media its capture state + # holds; None for a call without media. + prefix_row_extras: Callable[[_RequestCaptureState], dict[str, Any] | None] diff --git a/nemo_rl/models/generation/prefix_recovery/recovery.py b/nemo_rl/models/generation/prefix_recovery/recovery.py index a2caee4a862..7f568f1aa97 100644 --- a/nemo_rl/models/generation/prefix_recovery/recovery.py +++ b/nemo_rl/models/generation/prefix_recovery/recovery.py @@ -519,6 +519,7 @@ def _cut_active( prompt_token_ids=state.prompt_token_ids, generated_token_ids=generated_token_ids, generated_logprobs=frozen.generated_logprobs, + extras=self._parts.prefix_row_extras(state), ) if frozen.chunk_token_ids: chunk_record = ( @@ -670,9 +671,19 @@ def _rebuild_prefix(self, admission: Any, prefix_token_ids: list[int]) -> Any: continuation = admission.generation_cut # Gym requires at least one staging key per continuation. staging_keys = list(continuation.staging_keys) - snapshots = self._parts.staging_source.fetch(staging_keys) - if len(snapshots) != len(staging_keys): + # The finalization read also returns each row's extras, which carry a + # media call's spans and tensor digests; plain ``fetch`` does not. + chunks = self._parts.staging_source.fetch_for_finalization(staging_keys) + if len(chunks) != len(staging_keys): raise RuntimeError("generation-cut fetch did not return every staged chunk") + snapshots = [chunk.snapshot for chunk in chunks] + # A media call's spans and tensor digests ride the first chunk's extras, + # and the advertised prefix digest covers them. + if any(chunk.extras is not None for chunk in chunks[1:]): + raise RuntimeError( + "only the first generation-cut chunk may carry call extras" + ) + extras = chunks[0].extras weight_versions = [snapshot.weight_version for snapshot in snapshots] if weight_versions != sorted(weight_versions): raise RuntimeError( @@ -732,8 +743,8 @@ def _rebuild_prefix(self, admission: Any, prefix_token_ids: list[int]) -> Any: token_ids_delta=token_ids_delta, token_mask_delta=token_mask_delta, generation_log_probs_delta=generation_logprobs_delta, - extras=None, - extras_digest=compute_extras_digest(None), + extras=extras, + extras_digest=compute_extras_digest(extras), chain_hash=compute_chain_hash(admission.parent_chain_hash, token_ids_delta), cumulative_hash=hash_token_ids(prefix_token_ids + token_ids_delta), ) diff --git a/nemo_rl/models/generation/prefix_recovery/state.py b/nemo_rl/models/generation/prefix_recovery/state.py index 34589d8fbd1..f7b8181ef78 100644 --- a/nemo_rl/models/generation/prefix_recovery/state.py +++ b/nemo_rl/models/generation/prefix_recovery/state.py @@ -109,9 +109,8 @@ def __init__( self._call = call self._prompt_token_ids = list(prompt_token_ids) # Backend-owned opaque state that must survive until the terminal row - # is staged. vLLM uses this for its processed-media snapshot; prefix - # recovery is setup-guarded to text-only, but ordinary terminal capture - # still shares this lifecycle container. + # is staged. vLLM uses this for its processed-media snapshot; the first + # generation-prefix row carries its tensor digests across recovery. self._media = media self._resumed_generation_token_ids = list(resumed_generation_token_ids) self._lock = threading.Lock() diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 438bc0180f4..af531c35cf1 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -29,7 +29,7 @@ import warnings from collections.abc import Awaitable, Callable, Sequence from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from types import SimpleNamespace from typing import TYPE_CHECKING, Any, AsyncGenerator, Literal, Optional, cast @@ -41,13 +41,16 @@ from fastapi.responses import JSONResponse from nemo_rl.data.captured_media import ( + MEDIA_TENSOR_DIGESTS_FIELD, CapturedMedia, CapturedMediaItem, MediaCaptureRejected, MediaColumnSpec, capture_processed_media, + media_tensor_digests, verified_media_items, verify_media_geometry, + verify_restored_media, ) from nemo_rl.data_plane.adapters.tq_mooncake_checkpoint import run_checkpoint_command from nemo_rl.data_plane.tq_token_sink import MediaMetadataIntegrityError @@ -138,6 +141,32 @@ ) +def _media_items_from_extras( + extras: dict[str, Any] | None, + snapshot: Any, + *, + missing_message: str, +) -> tuple[CapturedMediaItem, ...]: + """Parse a staged row's media spans and verify them against its own tokens. + + Raises: + MediaCaptureRejected: ``missing_message`` when the row has no spans, or + when the spans are malformed or do not match the row's tokens. + """ + # Optional Gym dependency: only captured media calls reach here. + from nemo_gym.token_id_capture.adapters.vllm import MEDIA_SPANS_FIELD + + if not isinstance(extras, dict) or MEDIA_SPANS_FIELD not in extras: + raise MediaCaptureRejected(missing_message) + raw_items = extras[MEDIA_SPANS_FIELD] + if not isinstance(raw_items, list): + raise MediaCaptureRejected("Media spans are malformed") + items = tuple(CapturedMediaItem.from_dict(value) for value in raw_items) + for item in items: + item.verify_tokens(snapshot.token_ids_delta, origin=snapshot.prev_len) + return items + + # Paths the bearer-token check protects: the OpenAI-compatible routes this module # registers, which NeMo-Gym's model server calls. The sparse-refit routes the # receiver registers on the same app carry their own header token @@ -438,6 +467,7 @@ def _init_token_capture_state(self) -> None: # Built once per engine: it wraps vLLM's output processor. self._progress_reader: Any | None = None self._capture_media = False + self._hash_captured_media = False self._capture_patch_size: int | None = None self._capture_image_token_id: int | None = None # TQTokenSource installed by setup_token_capture. The media path reads @@ -893,6 +923,7 @@ async def setup_token_capture( self._capture_image_token_id = int(context_ids[0]) self._capture_patch_size = int(info.get_hf_config().patch_size) self._capture_media = capture_media + self._hash_captured_media = capture_media and generation_prefix_cuts_enabled self._capture_sink = sink if generation_prefix_cuts_enabled and self._http_engine_client is None: raise RuntimeError( @@ -927,6 +958,7 @@ async def setup_token_capture( control_token=generation_cut_control_token, control_timeout_s=generation_cut_control_timeout_s, progress_reader=self._progress_reader, + prefix_row_extras=self._generation_cut_record_payload, ) ) if not capture_media: @@ -1053,6 +1085,7 @@ def _capture_request_media( admission: Any | None, splice: PrefixSplice | None = None, verify_retained_media: bool = False, + restored: CapturedMedia | None = None, ) -> CapturedMedia | None: """Run off-loop: resolve retained geometry and snapshot processed pixels.""" if admission is None: @@ -1067,8 +1100,6 @@ def _capture_request_media( retained_tensors = () if verify_retained_media else None if admission.parent_call_id is not None: # Optional Gym dependency: this method only runs on captured calls. - # Gym owns the media_spans key its adapter copies into the extras. - from nemo_gym.token_id_capture.adapters.vllm import MEDIA_SPANS_FIELD from nemo_gym.token_id_capture.staging.digest import compute_chain_hash from nemo_gym.token_id_capture.staging.records import staging_key @@ -1129,19 +1160,11 @@ def _capture_request_media( if verify_retained_media: items = verified_media_items(call) else: - extras = call.extras or {} - if MEDIA_SPANS_FIELD not in extras: - raise MediaCaptureRejected( - "Retained vLLM media spans are missing" - ) - items = tuple( - CapturedMediaItem.from_dict(value) - for value in extras[MEDIA_SPANS_FIELD] + items = _media_items_from_extras( + call.extras, + call.snapshot, + missing_message="Retained vLLM media spans are missing", ) - for item in items: - item.verify_tokens( - call.snapshot.token_ids_delta, origin=call.snapshot.prev_len - ) descriptors[call.snapshot.model_call_id] = items retained = tuple(item for items in descriptors.values() for item in items) if verify_retained_media: @@ -1153,7 +1176,7 @@ def _capture_request_media( verify_media_geometry( descriptors[call.snapshot.model_call_id], tensors ) - return capture_processed_media( + captured = capture_processed_media( engine_prompt, prev_len=admission.prev_len, retained=retained, @@ -1162,6 +1185,74 @@ def _capture_request_media( patch_size=self._capture_patch_size, retained_tensors=retained_tensors, ) + if self._hash_captured_media: + # Hash once per call here, off the event loop, so a prefix cut can + # write the digests without hashing pixels under its deadline. + captured = replace( + captured, tensor_digests=media_tensor_digests(captured.tensors) + ) + if restored is not None: + verify_restored_media(captured, restored) + return captured + + def _generation_cut_record_payload( + self, state: _RequestCaptureState + ) -> dict[str, Any] | None: + """Return the media extras for a call's first generation-prefix row. + + The row stores the call's tensor digests instead of its pixels: a + restore reprocesses the media and only checks it against them, and the + terminal row still stages the pixels for training. + """ + media = state.media + if media is None: + return None + if media.tensor_digests is None: + raise RuntimeError( + "media was captured without tensor digests; generation-prefix " + "cuts must be enabled before the call is admitted" + ) + from nemo_gym.token_id_capture.adapters.vllm import MEDIA_SPANS_FIELD + + return { + MEDIA_SPANS_FIELD: [item.to_dict() for item in media.items], + MEDIA_TENSOR_DIGESTS_FIELD: media.tensor_digests, + } + + def _restore_generation_cut_media(self, snapshot: Any) -> CapturedMedia: + """Read a restored call's media items and tensor digests from its verified prefix. + + The prefix digest Gym checked covers these extras, so only a bug in our + own writer can leave them unreadable. + """ + items = _media_items_from_extras( + snapshot.extras, + snapshot, + missing_message="Generation-prefix media metadata is missing", + ) + return CapturedMedia( + items=items, + tensors=None, + tensor_digests=snapshot.extras[MEDIA_TENSOR_DIGESTS_FIELD], + ) + + def _capture_restored_media( + self, + engine_prompt: dict[str, Any], + *, + admission: Any, + splice: PrefixSplice, + snapshot: Any, + ) -> CapturedMedia | None: + """Reprocess a restored call's media and check it against the digests its cut saved.""" + if not self._capture_media: + return None + return self._capture_request_media( + engine_prompt, + admission=admission, + splice=splice, + restored=self._restore_generation_cut_media(snapshot), + ) def _fetch_chain_prefix(self, staging_chain: list[str]) -> list[int]: """Resolve a staging chain through the shared, cached TQ read.""" @@ -1822,8 +1913,6 @@ async def preprocess_chat( final_prompt_token_ids, ) - # Generation-prefix cuts are text-only (setup refuses them with - # media capture), so a restored prompt has no media to capture. media = None if generation_cut is None: media = await asyncio.to_thread( @@ -1833,6 +1922,22 @@ async def preprocess_chat( splice=splice, verify_retained_media=verify_retained_media, ) + elif worker_self._capture_media: + # A restored prompt is the durable prefix itself, so splice + # the template only to locate its media placeholders. + media_splice = splice_prefix_tokens( + tokenizer=self.renderer.tokenizer, + model_prefix_token_ids=list(capture_prefix_token_ids or ()), + template_prefix_token_ids=actual_corresponding_token_ids, + template_token_ids=engine_prompt["prompt_token_ids"], + ) + media = await asyncio.to_thread( + worker_self._capture_restored_media, + engine_prompt, + admission=admission, + splice=media_splice, + snapshot=generation_cut, + ) # Must precede the reassignment below: the remap reads the # current value of prompt_token_ids as the template-coordinate # sequence the placeholders were computed against. Reordering diff --git a/tests/functional/L1_Functional_Tests_GB200_Vllm_Omni_Single_Controller.sh b/tests/functional/L1_Functional_Tests_GB200_Vllm_Omni_Single_Controller.sh index bb0005c6def..69e07b03588 100755 --- a/tests/functional/L1_Functional_Tests_GB200_Vllm_Omni_Single_Controller.sh +++ b/tests/functional/L1_Functional_Tests_GB200_Vllm_Omni_Single_Controller.sh @@ -50,6 +50,8 @@ export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1}" run_test fast uv run --no-sync bash ./tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh run_test fast uv run --no-sync bash ./tests/functional/nemotron_omni_gym_video_vllm_token_capture_single_controller_1n2g.sh +# Kills and restores a run mid-decode on image prompts; full mode only. +run_test uv run --no-sync bash ./tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/_gym_prefix_recovery_parity.py b/tests/functional/_gym_prefix_recovery_parity.py index 1ea1fce089f..58b91562485 100644 --- a/tests/functional/_gym_prefix_recovery_parity.py +++ b/tests/functional/_gym_prefix_recovery_parity.py @@ -284,51 +284,49 @@ def _compare_training_rows( ) -def _check_cuts( +def _restored_prefix_hashes(recovery_log: str) -> dict[tuple[str, int], list[str]]: + restored: dict[tuple[str, int], list[str]] = {} + for line in restored_prefixes(recovery_log): + key = (line["source_model_call_id"], line["prefix_tokens"]) + restored.setdefault(key, []).append(line["prefix_ids_sha256"]) + return restored + + +def _calls_by_generation( + recovery: RowIndex, +) -> dict[tuple[str, int], tuple[int, int, int]]: + return { + (row["group_id"], int(row["generation_index"])): identity + for identity, row in recovery.items() + } + + +def _assert_prefix_boundary( selection: Mapping[str, Any], recovery_log: str, - baseline: RowIndex, recovery: RowIndex, -) -> tuple[dict[tuple[str, int], int], frozenset[tuple[int, int, int]]]: - """Check every restored call's trained row, and find rows to set aside. - - First, each restored call must complete once and train on its restored - prefix. A restored call closes its row, so the ``total_generation_tokens`` - it logs on completion end the row: counting back finds where the call - starts. That holds even when the trainer masked positions inside the call - (top-k/top-p ``-inf`` logprobs drop out of ``token_mask``), which a search - for the last unmasked segment would mistake for the call's start. A dropped - or duplicated token moves that start, so the restored prefix no longer sits +) -> dict[tuple[str, int], int]: + """Require each restored call to complete once and train on its restored prefix. + + A restored call closes its row, so the ``total_generation_tokens`` it logs + on completion end the row: counting back finds where the call starts. That + holds even when the trainer masked positions inside the call (top-k/top-p + ``-inf`` logprobs drop out of ``token_mask``), which a search for the last + unmasked segment would mistake for the call's start. A dropped or + duplicated token moves that start, so the restored prefix no longer sits there. - Then, vLLM sampling is not batch-invariant, so phase 1 can take the other - side of a near-tie before the cut. Recovery must then continue phase 1's - tokens, and the reference no longer predicts that row. A row is set aside - only when the reference's call opens differently at the restored prefix's - position, and everything before the closing call must still match. At - most one restored row may be set aside, and at least one must match the - reference outright: nothing checks that a set-aside row diverged rather - than restored the wrong tokens, so the cap bounds what a run can leave - unchecked. - Returns: Each cut's ``(group_id, generation_index)`` mapped to the index in its - trained row where the restored call's generated tokens begin, and the - identities of the restored rows set aside. + trained row where the restored call's generated tokens begin. """ completed: dict[tuple[str, int], list[dict[str, str]]] = {} for match in _COMPLETED_PREFIX_LINE.finditer(recovery_log): line = match.groupdict() key = (line["source_model_call_id"], int(line["prefix_tokens"])) completed.setdefault(key, []).append(line) - restored: dict[tuple[str, int], list[str]] = {} - for line in restored_prefixes(recovery_log): - key = (line["source_model_call_id"], line["prefix_tokens"]) - restored.setdefault(key, []).append(line["prefix_ids_sha256"]) - by_call = { - (row["group_id"], int(row["generation_index"])): identity - for identity, row in recovery.items() - } + restored = _restored_prefix_hashes(recovery_log) + by_call = _calls_by_generation(recovery) starts: dict[tuple[str, int], int] = {} for cut in selection["cuts"]: @@ -373,6 +371,38 @@ def _check_cuts( f"{prefix_tokens} restored prefix tokens" ) starts[key] = start + return starts + + +def _check_cuts( + selection: Mapping[str, Any], + recovery_log: str, + baseline: RowIndex, + recovery: RowIndex, +) -> tuple[dict[tuple[str, int], int], frozenset[tuple[int, int, int]]]: + """Check every restored call's trained row, and find rows to set aside. + + First, each restored call must complete once and train on its restored + prefix (``_assert_prefix_boundary``). + + Then, vLLM sampling is not batch-invariant, so phase 1 can take the other + side of a near-tie before the cut. Recovery must then continue phase 1's + tokens, and the reference no longer predicts that row. A row is set aside + only when the reference's call opens differently at the restored prefix's + position, and everything before the closing call must still match. At + most one restored row may be set aside, and at least one must match the + reference outright: nothing checks that a set-aside row diverged rather + than restored the wrong tokens, so the cap bounds what a run can leave + unchecked. + + Returns: + Each cut's ``(group_id, generation_index)`` mapped to the index in its + trained row where the restored call's generated tokens begin, and the + identities of the restored rows set aside. + """ + starts = _assert_prefix_boundary(selection, recovery_log, recovery) + restored = _restored_prefix_hashes(recovery_log) + by_call = _calls_by_generation(recovery) diverged: set[tuple[int, int, int]] = set() for cut in selection["cuts"]: diff --git a/tests/functional/_gym_prefix_recovery_phases.sh b/tests/functional/_gym_prefix_recovery_phases.sh new file mode 100644 index 00000000000..6aff50cdc49 --- /dev/null +++ b/tests/functional/_gym_prefix_recovery_phases.sh @@ -0,0 +1,78 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Phase control shared by the prefix-recovery functional tests. Source it after +# setting PROJECT_ROOT and TEST_DIR; a test starts phase one in its own process +# group and keeps its PID in PHASE1_PID. + +PHASE1_PID="" + +# Hard-kill phase one and wait until its controller can no longer write +# snapshots. +stop_phase1() { + if [[ -z "${PHASE1_PID}" ]]; then + return + fi + kill -KILL -- "-${PHASE1_PID}" 2>/dev/null || true + wait "${PHASE1_PID}" 2>/dev/null || true + PHASE1_PID="" + # Ray actors outlive the driver until their raylet notices the crash. Do + # not start phase two while the old controller can still write snapshots. + local deadline=$((SECONDS + 120)) + while pgrep -f "^ray::SingleControllerActor" >/dev/null; do + if ((SECONDS >= deadline)); then + echo "[ERROR] phase-one SingleControllerActor outlived its driver" + return 1 + fi + sleep 1 + done +} + +# Recover exactly the selected prefix: keep only the selected snapshot and no +# trainer checkpoint, either of which would represent work performed after the +# simulated crash. Sets SNAPSHOT_DIR to the restored snapshot. +restore_selected_snapshot() { + local selection_file=$1 selected_backup=$2 checkpoint_dir=$3 + local snapshot_root trainer_checkpoint + SNAPSHOT_DIR=$(uv run --directory "${PROJECT_ROOT}" --no-sync python -c \ + 'import json, sys; print(json.load(open(sys.argv[1]))["snapshot"])' \ + "${selection_file}") + snapshot_root=$(dirname "${SNAPSHOT_DIR}") + rm -rf "${snapshot_root}" + mkdir -p "${snapshot_root}" + cp -a "${selected_backup}" "${SNAPSHOT_DIR}" + for trainer_checkpoint in "${checkpoint_dir}"/step_*; do + if [[ -d "${trainer_checkpoint}" ]]; then + rm -rf "${trainer_checkpoint}" + fi + done +} + +# On exit, stop phase one and delete the given checkpoint artifacts if the test +# passed, unless SC_PREFIX_RECOVERY_KEEP_CHECKPOINTS=1. +install_phase_cleanup() { + PHASE_CLEANUP_PATHS=("$@") + trap _phase_cleanup EXIT +} + +_phase_cleanup() { + local status=$? + stop_phase1 + if [[ "${status}" -eq 0 && "${SC_PREFIX_RECOVERY_KEEP_CHECKPOINTS:-0}" != "1" ]]; then + rm -rf "${PHASE_CLEANUP_PATHS[@]}" + else + echo "Preserving prefix-recovery artifacts for inspection: ${TEST_DIR}" + fi + return "${status}" +} diff --git a/tests/functional/_gym_prefix_recovery_snapshot.py b/tests/functional/_gym_prefix_recovery_snapshot.py index 05dd658bceb..34533fad396 100644 --- a/tests/functional/_gym_prefix_recovery_snapshot.py +++ b/tests/functional/_gym_prefix_recovery_snapshot.py @@ -25,6 +25,12 @@ from pathlib import Path from typing import Any +from _gym_prefix_recovery_parity import ( + _assert_prefix_boundary, + _assert_retired_prefix_keys, + _rows_by_identity, + load_training_records, +) from _gym_recovery_snapshot_helpers import ( calendar_sentinel_count, ledger_episode_index, @@ -122,14 +128,16 @@ def _require_every_active_call_cut(snapshot: Path, phase_log: str) -> None: def inspect_snapshot( snapshot: Path, - sentinel_event: str, + sentinel_event: str | None, *, min_cut_groups: int = 1, ) -> dict[str, Any]: """Return the post-mutation nonterminal prefixes in ``snapshot``. ``cuts`` lists every one, and the snapshot qualifies only if they span at - least ``min_cut_groups`` prompt groups. + least ``min_cut_groups`` prompt groups. A ``None`` ``sentinel_event`` + selects a stateless resources server (e.g. CLEVR's string_match), whose + episodes export no resources state to check. """ required = [ @@ -164,11 +172,15 @@ def inspect_snapshot( key = _episode_key(record) recovery = ledger.get(key) cut = _recoverable_cut(record) - resource_entry = resource_index.get(key) - if recovery is None or cut is None or resource_entry is None: + if recovery is None or cut is None: continue - _, resources_state = resource_entry - sentinel_count = calendar_sentinel_count(resources_state, sentinel_event) + if sentinel_event is not None: + resource_entry = resource_index.get(key) + if resource_entry is None: + continue + _, resources_state = resource_entry + if calendar_sentinel_count(resources_state, sentinel_event) != 1: + continue instance_id = recovery["gym_instance_id"] checkpointed_episodes = { (episode["rollout_id"], episode["attempt"]) @@ -178,7 +190,6 @@ def inspect_snapshot( if ( key not in checkpointed_episodes or key not in agent_keys - or sentinel_count != 1 or recovery["restore_level"] != "prefix" or recovery["status"] != "dispatched" or not set(cut["staging_keys"]).issubset(retained_keys) @@ -222,7 +233,7 @@ def select_snapshot(args: argparse.Namespace) -> None: try: selection = inspect_snapshot( snapshot, - args.sentinel_event, + None if args.stateless_resources else args.sentinel_event, min_cut_groups=args.min_cut_groups, ) _require_every_active_call_cut( @@ -346,6 +357,27 @@ def verify_restore(args: argparse.Namespace) -> None: raise AssertionError(f"train step {step} appeared {len(matches)} times") +def verify_completion(args: argparse.Namespace) -> None: + """Without a reference run, require every restored prefix to finish and train. + + Each restored call must complete once, close its trained row, and start + with the exact prefix the worker restored (``_assert_prefix_boundary``); no + row may be trained twice, and the restored prefix rows must outlive + training only until the next snapshot. + """ + selection = read_json(Path(args.selection_file)) + records = load_training_records( + Path(args.recovery_log_dir), staging_keys=Path(args.recovery_staging_keys) + ) + starts = _assert_prefix_boundary( + selection, Path(args.phase2_log).read_text(), _rows_by_identity(records) + ) + _assert_retired_prefix_keys( + selection, records, read_jsonl(Path(args.recovery_flush_keys)) + ) + print(f"restored prefixes completed and trained: {sorted(starts)!r}", flush=True) + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() commands = parser.add_subparsers(dest="command", required=True) @@ -357,7 +389,10 @@ def _parser() -> argparse.ArgumentParser: select.add_argument("phase_log") select.add_argument("timeout_s", type=float) select.add_argument("backup_path") - select.add_argument("sentinel_event") + select.add_argument("sentinel_event", nargs="?") + # Explicit opt-out, so a Workplace caller that drops the sentinel fails + # instead of silently skipping the post-mutation resources check. + select.add_argument("--stateless-resources", action="store_true") select.add_argument("--min-cut-groups", type=int, default=1) select.set_defaults(func=select_snapshot) @@ -369,9 +404,24 @@ def _parser() -> argparse.ArgumentParser: verify.add_argument("training_info") verify.add_argument("max_steps", type=int) verify.set_defaults(func=verify_restore) + + completion = commands.add_parser("verify-completion") + completion.add_argument("selection_file") + completion.add_argument("recovery_log_dir") + completion.add_argument("recovery_staging_keys") + completion.add_argument("recovery_flush_keys") + completion.add_argument("phase2_log") + completion.set_defaults(func=verify_completion) return parser if __name__ == "__main__": - arguments = _parser().parse_args() + parser = _parser() + arguments = parser.parse_args() + if arguments.command == "select" and ( + (arguments.sentinel_event is None) != arguments.stateless_resources + ): + parser.error( + "select needs exactly one of sentinel_event or --stateless-resources" + ) arguments.func(arguments) diff --git a/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh b/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh index 474eae65b5f..f9a480a7121 100755 --- a/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh +++ b/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh @@ -33,7 +33,6 @@ SELECTION_FILE=$TEST_DIR/selected-snapshot.json SELECTED_BACKUP=$TEST_DIR/selected-snapshot TEST_DATA=$TEST_DIR/test-data.jsonl GYM_ROOT=$PROJECT_ROOT/3rdparty/Gym-workspace/Gym -PHASE1_PID="" SENTINEL_EVENT="NeMo RL prefix recovery parity sentinel" # Cuttable Workplace prompts per step, one per list length. @@ -124,36 +123,8 @@ jq -n -c \ ' > "$TEST_DATA" -stop_phase1() { - if [[ -z "$PHASE1_PID" ]]; then - return - fi - kill -KILL -- "-$PHASE1_PID" 2>/dev/null || true - wait "$PHASE1_PID" 2>/dev/null || true - PHASE1_PID="" - # Ray actors outlive the driver until their raylet notices the crash. Do - # not start phase two while the old controller can still write snapshots. - local deadline=$((SECONDS + 120)) - while pgrep -f "^ray::SingleControllerActor" >/dev/null; do - if ((SECONDS >= deadline)); then - echo "[ERROR] phase-one SingleControllerActor outlived its driver" - return 1 - fi - sleep 1 - done -} - -cleanup() { - local status=$? - stop_phase1 - if [[ "$status" -eq 0 && "${SC_PREFIX_RECOVERY_KEEP_CHECKPOINTS:-0}" != "1" ]]; then - rm -rf "$CHECKPOINT_DIR" "$BASELINE_CHECKPOINT_DIR" "$SELECTED_BACKUP" - else - echo "Preserving prefix-recovery artifacts for inspection: $TEST_DIR" - fi - return "$status" -} -trap cleanup EXIT +source "$SCRIPT_DIR/_gym_prefix_recovery_phases.sh" +install_phase_cleanup "$CHECKPOINT_DIR" "$BASELINE_CHECKPOINT_DIR" "$SELECTED_BACKUP" COMMON_OVERRIDES=( policy.model_name="$MODEL_NAME" @@ -246,21 +217,7 @@ uv run --directory "$PROJECT_ROOT" --no-sync python "$SNAPSHOT_HELPER" select \ stop_phase1 cp "$BASE_RUN_LOG" "$PHASE1_LOG" -SNAPSHOT_DIR=$(uv run --directory "$PROJECT_ROOT" --no-sync python -c \ - 'import json, sys; print(json.load(open(sys.argv[1]))["snapshot"])' \ - "$SELECTION_FILE") -SNAPSHOT_ROOT=$(dirname "$SNAPSHOT_DIR") - -# Recover exactly the prefix selected above. A newer periodic snapshot or a -# trainer checkpoint would represent work performed after the simulated crash. -rm -rf "$SNAPSHOT_ROOT" -mkdir -p "$SNAPSHOT_ROOT" -cp -a "$SELECTED_BACKUP" "$SNAPSHOT_DIR" -for trainer_checkpoint in "$CHECKPOINT_DIR"/step_*; do - if [[ -d "$trainer_checkpoint" ]]; then - rm -rf "$trainer_checkpoint" - fi -done +restore_selected_snapshot "$SELECTION_FILE" "$SELECTED_BACKUP" "$CHECKPOINT_DIR" # Avoid cutting the replacement request again while verifying that it consumes # the selected prefix and reaches normal completion. diff --git a/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh new file mode 100755 index 00000000000..eb82a8f44b0 --- /dev/null +++ b/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Multimodal smoke test for generation-prefix recovery on the Omni CLEVR vLLM +# token-capture setup. A Gym-v2 checkpoint is taken while vLLM is mid-decode on +# image prompts; the process is hard-killed and restored. Each restored call +# must reprocess its image, verify it against the durable TQ media copy, resume +# from the exact durable token prefix, and finish and train. +# +# Token-for-token parity against an uninterrupted run is the Workplace text +# test's job (grpo_async_gym_single_controller_prefix_recovery.sh). Here the +# Reasoning model's free-form thinking flips near-tied tokens between runs, so +# parity would be flaky; this test instead checks each restored prefix appears +# verbatim in its trained row. Other differences from the Workplace test: +# - Single-turn string_match rows: the cut lands in the only model call, so the +# first durable prefix row is the one that owns the image tensors. +# - string_match is stateless, so snapshot selection runs with +# --stateless-resources instead of a calendar-sentinel check. +# - Each prompt group gets a distinct image, so a restore that verified against +# the wrong group's pixels would fail (retained_media_changed) instead of +# passing on identical fixtures. + +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +PROJECT_ROOT=$(realpath "${SCRIPT_DIR}/../..") + +if [[ -z "${HF_TOKEN:-}" ]]; then + echo "SKIP: HF_TOKEN is required for the Omni checkpoint" + exit 0 +fi + +GPU_COUNT=$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l) +if (( GPU_COUNT < 2 )); then + echo "SKIP: Omni CLEVR vLLM prefix recovery requires at least two visible GPUs" + exit 0 +fi +DETECTED_CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader -i 0) +export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-${DETECTED_CUDA_ARCH}}" + +EXP_NAME=$(basename "$0" .sh) +TEST_DIR="${SCRIPT_DIR}/${EXP_NAME}" +OMNI_CONFIG="${PROJECT_ROOT}/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml" +RECOVERY_HOOK="${SCRIPT_DIR}/_single_controller_turn_recovery_hook.py" +SNAPSHOT_HELPER="${SCRIPT_DIR}/_gym_prefix_recovery_snapshot.py" +CHECKPOINT_DIR="${TEST_DIR}/recovery-checkpoints" +PHASE1_LOG="${TEST_DIR}/phase1.log" +PHASE2_LOG="${TEST_DIR}/phase2.log" +PHASE1_EVENTS="${TEST_DIR}/phase1-events.jsonl" +PHASE2_EVENTS="${TEST_DIR}/phase2-events.jsonl" +# Each phase writes into its own logger.log_dir: phase two's trained rows are +# read back from the only experiment directory under RECOVERY_LOG_DIR. +PHASE1_LOG_DIR="${TEST_DIR}/phase1-logs" +RECOVERY_LOG_DIR="${TEST_DIR}/recovery-logs" +RECOVERY_STAGING_KEYS="${TEST_DIR}/recovery-staging-keys.jsonl" +RECOVERY_FLUSH_KEYS="${TEST_DIR}/recovery-flush-keys.jsonl" +PHASE2_METRICS="${TEST_DIR}/phase2-metrics.json" +SELECTION_FILE="${TEST_DIR}/selected-snapshot.json" +SELECTED_BACKUP="${TEST_DIR}/selected-snapshot" +TEST_DATA="${TEST_DIR}/test-data.jsonl" + +# One prompt group per list length, each with its own solid-color image. The +# list keeps decode long enough to be cut mid-generation; distinct images mean +# a restore verified against another group's pixels would be rejected. +LIST_LENGTHS=(${SC_MM_PREFIX_RECOVERY_LIST_LENGTHS:-120 240}) +NUM_PROMPTS=${#LIST_LENGTHS[@]} +NUM_GENERATIONS=${SC_MM_PREFIX_RECOVERY_NUM_GENERATIONS:-2} +# Conservative per-item budget: the Omni tokenizer's digit grouping is not +# assumed, so "N\n" is budgeted like Qwen's one-token-per-digit. +LIST_TOKENS_PER_ITEM=6 +LIST_OUTPUT_SLACK_TOKENS=256 +MAX_MODEL_LEN=${SC_MM_PREFIX_RECOVERY_MAX_MODEL_LEN:-4096} +MAX_STEPS=${SC_MM_PREFIX_RECOVERY_MAX_STEPS:-2} +# Phase two trains one step past phase one, so the snapshot after step +# MAX_STEPS deletes the restored calls' retired prefix rows while the run is +# still alive to record it. +PHASE2_STEPS=$((MAX_STEPS + 1)) +SNAPSHOT_INTERVAL_S=${SC_MM_PREFIX_RECOVERY_INTERVAL_S:-0.05} +PHASE2_SNAPSHOT_INTERVAL_S=${SC_MM_PREFIX_RECOVERY_PHASE2_INTERVAL_S:-600} +SNAPSHOT_TIMEOUT_S=${SC_MM_PREFIX_RECOVERY_TIMEOUT_S:-3600} +RUN_TIMEOUT_S=${SC_MM_PREFIX_RECOVERY_RUN_TIMEOUT_S:-3600} +TRAIN_GLOBAL_BATCH_SIZE=$((NUM_PROMPTS * NUM_GENERATIONS)) +POLICY_MAX_NEW_TOKENS=0 +for list_length in "${LIST_LENGTHS[@]}"; do + list_budget=$((list_length * LIST_TOKENS_PER_ITEM + LIST_OUTPUT_SLACK_TOKENS)) + if (( list_budget > POLICY_MAX_NEW_TOKENS )); then + POLICY_MAX_NEW_TOKENS=$list_budget + fi +done + +rm -rf "${TEST_DIR}" +mkdir -p "${TEST_DIR}" +cd "${PROJECT_ROOT}" +export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}" + +# The test config trains one epoch, so the data holds a step of prompts for +# every phase-two step; phase one stops after MAX_STEPS of them. +TEST_DATA="${TEST_DATA}" DATA_STEPS="${PHASE2_STEPS}" \ +LIST_LENGTHS="${LIST_LENGTHS[*]}" LIST_TOKENS_PER_ITEM="${LIST_TOKENS_PER_ITEM}" \ +LIST_OUTPUT_SLACK_TOKENS="${LIST_OUTPUT_SLACK_TOKENS}" \ +uv run --no-sync python - <<'PY' +import base64 +import io +import json +import os + +from PIL import Image + +COLORS = ["red", "green", "blue", "yellow"] +list_lengths = [int(value) for value in os.environ["LIST_LENGTHS"].split()] +if not 1 <= len(list_lengths) <= len(COLORS): + raise SystemExit(f"need 1 to {len(COLORS)} list lengths, got {len(list_lengths)}") +per_item = int(os.environ["LIST_TOKENS_PER_ITEM"]) +slack = int(os.environ["LIST_OUTPUT_SLACK_TOKENS"]) + + +def image_url(color: str) -> str: + buffer = io.BytesIO() + Image.new("RGB", (224, 224), color=color).save(buffer, format="PNG") + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + + +with open(os.environ["TEST_DATA"], "w") as output: + for step in range(int(os.environ["DATA_STEPS"])): + for slot, list_length in enumerate(list_lengths): + color = COLORS[slot] + text = ( + f"Step {step} item {slot}: list every integer from 1 to {list_length} " + "in increasing order, one per line, with no other text. Then give " + "the color of the image as one word inside \\boxed{}." + ) + row = { + "responses_create_params": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": image_url(color), + "detail": "auto", + }, + {"type": "input_text", "text": text}, + ], + } + ], + "max_output_tokens": list_length * per_item + slack, + }, + "expected_answer": color, + "extraction_mode": "boxed", + "case_sensitive": False, + "agent_ref": { + "type": "responses_api_agents", + "name": "string_match_simple_agent", + }, + } + output.write(json.dumps(row) + "\n") +PY + +source "${SCRIPT_DIR}/_gym_prefix_recovery_phases.sh" +install_phase_cleanup "${CHECKPOINT_DIR}" "${SELECTED_BACKUP}" + +# The text prefix-recovery settings, on the YAML's Omni CLEVR topology (one +# Megatron training GPU with a frozen language model, one async vLLM GPU). +COMMON_OVERRIDES=( + # Quoted for Hydra: a bare compute capability (9.0, 10.0) parses as a + # float, which the string-typed env_vars field rejects. + policy.megatron_cfg.env_vars.TORCH_CUDA_ARCH_LIST="'${TORCH_CUDA_ARCH_LIST}'" + policy.generation.max_new_tokens="${POLICY_MAX_NEW_TOKENS}" + policy.generation.vllm_cfg.max_model_len="${MAX_MODEL_LEN}" + ++policy.generation.vllm_kwargs.max_num_batched_tokens="${MAX_MODEL_LEN}" + # Near-greedy decoding keeps each list (and so each decode) predictable. + policy.generation.temperature=1.0 + policy.generation.top_p=0.000001 + # The trainer replays the top_p filter, so it needs vLLM's logprobs after + # that filter. + ++policy.generation.vllm_cfg.logprobs_mode=processed_logprobs + policy.max_total_sequence_length="${MAX_MODEL_LEN}" + policy.train_global_batch_size="${TRAIN_GLOBAL_BATCH_SIZE}" + # Keep each step's prompt membership fixed across runs. + data.shuffle=false + data.train.data_path="${TEST_DATA}" + data.validation.data_path="${TEST_DATA}" + grpo.seed=1234 + grpo.num_prompts_per_step="${NUM_PROMPTS}" + grpo.num_generations_per_prompt="${NUM_GENERATIONS}" + grpo.max_num_steps="${MAX_STEPS}" + async_rl.sampler.name=in_order + async_rl.sampler.max_lookahead_versions=0 + async_rl.min_groups_for_streaming_train="${NUM_PROMPTS}" + async_rl.max_inflight_prompts="${NUM_PROMPTS}" + async_rl.max_buffered_rollouts="${NUM_PROMPTS}" + ++async_rl.rollout_failure.nemo_gym.rollout_timeout_s=600 + ++async_rl.stall_watchdog.interval_s=10 + ++async_rl.stall_watchdog.stall_timeout_s=900 + ++async_rl.stall_watchdog.stall_action=abort + ++token_capture.enabled=true + ++rollout_recovery.target_level=prefix + ++rollout_checkpointing.snapshot_attempt_interval_s="${SNAPSHOT_INTERVAL_S}" + ++rollout_checkpointing.keep_latest_k=16 + ++rollout_checkpointing.restore_mode=latest + # Phase two trains one step longer than phase one (PHASE2_STEPS); the step + # count only decides when the run stops, so restore must accept it. + '++rollout_checkpointing.extra_fingerprint_excluded_paths=[grpo.max_num_steps]' + checkpointing.enabled=true + checkpointing.checkpoint_dir="${CHECKPOINT_DIR}" + checkpointing.metric_name=null + # Phase two saves after step MAX_STEPS, whose snapshot deletes the retired + # prefix rows, and after its last step, whose training_info is read; every + # Omni save writes the full 30B model. + checkpointing.save_period="${MAX_STEPS}" + ++checkpointing.save_data_plane=true + # Each run dumps its trained rows (train_data_step*.jsonl in its log dir) + # for verify-completion. + ++async_rl.log_full_train_data=true +) + +OMNI_CMD=( + uv run --no-sync python "${RECOVERY_HOOK}" + --config "${OMNI_CONFIG}" + "${COMMON_OVERRIDES[@]}" +) + +echo "=== Phase 1: publish a checkpoint holding an active multimodal generation prefix ===" +command -v setsid >/dev/null +setsid env \ + SC_GYM_RECOVERY_TEST_EVENTS="${PHASE1_EVENTS}" \ + "${OMNI_CMD[@]}" \ + logger.log_dir="${PHASE1_LOG_DIR}" \ + "$@" >"${PHASE1_LOG}" 2>&1 & +PHASE1_PID=$! + +# Every prompt group must hold a recoverable prefix, so each distinct image is +# restored and verified at least once. +uv run --no-sync python "${SNAPSHOT_HELPER}" select \ + "${CHECKPOINT_DIR}/bootstrap/rollout_snapshots" \ + "${SELECTION_FILE}" \ + "${PHASE1_PID}" \ + "${PHASE1_LOG}" \ + "${SNAPSHOT_TIMEOUT_S}" \ + "${SELECTED_BACKUP}" \ + --stateless-resources \ + --min-cut-groups "${NUM_PROMPTS}" + +stop_phase1 +restore_selected_snapshot "${SELECTION_FILE}" "${SELECTED_BACKUP}" "${CHECKPOINT_DIR}" + +echo "=== Phase 2: reprocess the images, verify them, and finish each suffix ===" +# Do not cut the replacement requests again while they consume the prefix. +: > "${RECOVERY_FLUSH_KEYS}" +timeout --signal=TERM --kill-after=30s "${RUN_TIMEOUT_S}s" \ + env \ + SC_GYM_RECOVERY_TEST_EVENTS="${PHASE2_EVENTS}" \ + SC_PREFIX_RECOVERY_STAGING_KEYS="${RECOVERY_STAGING_KEYS}" \ + SC_PREFIX_RECOVERY_FLUSH_KEYS="${RECOVERY_FLUSH_KEYS}" \ + "${OMNI_CMD[@]}" \ + ++rollout_checkpointing.snapshot_attempt_interval_s="${PHASE2_SNAPSHOT_INTERVAL_S}" \ + grpo.max_num_steps="${PHASE2_STEPS}" \ + logger.log_dir="${RECOVERY_LOG_DIR}" \ + "$@" 2>&1 | tee "${PHASE2_LOG}" + +grep -Fq "Selected rollout recovery snapshot: ${SNAPSHOT_DIR}" "${PHASE2_LOG}" +grep -q "Native TQ checkpoint restored and validated" "${PHASE2_LOG}" +grep -q "Loaded .* unfinished rollout group(s)" "${PHASE2_LOG}" +grep -q "generation prefix restored:" "${PHASE2_LOG}" +grep -q "train step ${PHASE2_STEPS}/${PHASE2_STEPS}" "${PHASE2_LOG}" +# A media mismatch is rejected with retained_media_changed and the call is +# retried from the prompt, which would otherwise pass as a fresh rollout. +if grep -q "retained_media_changed" "${PHASE2_LOG}"; then + echo "[ERROR] a restored multimodal prefix failed media verification" + exit 1 +fi + +uv run --no-sync tests/json_dump_tb_logs.py "${RECOVERY_LOG_DIR}" --output_path "${PHASE2_METRICS}" +# media_row_rate == 1 proves every learner row, including the restored ones, +# still carried captured pixels after recovery. +uv run --no-sync tests/check_metrics.py "${PHASE2_METRICS}" \ + 'max(data["train/finalize/invalid_row_rate"]) == 0' \ + 'max(data["train/finalize/capture_poisoned_rollouts"]) == 0' \ + 'min(data["train/finalize/media_row_rate"]) == 1' \ + 'all_finite(data["train/reward"])' + +uv run --no-sync python "${SNAPSHOT_HELPER}" verify-restore \ + "${SELECTION_FILE}" \ + "${PHASE1_EVENTS}" \ + "${PHASE2_EVENTS}" \ + "${PHASE2_LOG}" \ + "${CHECKPOINT_DIR}/step_${PHASE2_STEPS}/training_info.json" \ + "${PHASE2_STEPS}" + +# Each restored call completed exactly once, its trained row begins that call +# with the exact restored prefix, and its restored prefix rows outlived +# training only until the next snapshot. +uv run --no-sync python "${SNAPSHOT_HELPER}" verify-completion \ + "${SELECTION_FILE}" \ + "${RECOVERY_LOG_DIR}" \ + "${RECOVERY_STAGING_KEYS}" \ + "${RECOVERY_FLUSH_KEYS}" \ + "${PHASE2_LOG}" + +echo "Omni CLEVR multimodal generation-prefix recovery test passed" diff --git a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh index 2d265e6153b..47274355ddd 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh @@ -90,60 +90,26 @@ for path, count in ((os.environ["TRAIN_PATH"], 64), (os.environ["VAL_PATH"], 2)) output.write(json.dumps(sample(index)) + "\n") PY -# SingleController requires disaggregated generation: one GPU trains the -# frozen-decoder policy, one GPU hosts async vLLM (TP1) with the capture host. +# The YAML pins the disaggregated split SingleController requires: one GPU +# trains the frozen-decoder policy, one hosts async vLLM (TP1) with the capture +# host. uv run --no-sync python examples/run_grpo_single_controller.py \ --config examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml \ - cluster.num_nodes=1 \ - cluster.gpus_per_node=2 \ - ++cluster.segment_size=1 \ policy.megatron_cfg.env_vars.TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}" \ - policy.megatron_cfg.tensor_model_parallel_size=1 \ - policy.megatron_cfg.expert_model_parallel_size=1 \ - policy.megatron_cfg.expert_tensor_parallel_size=1 \ - policy.megatron_cfg.context_parallel_size=1 \ - policy.megatron_cfg.sequence_parallel=true \ - policy.megatron_cfg.activation_checkpointing=true \ - ++policy.megatron_cfg.freeze_config.freeze_language_model=true \ - +policy.megatron_cfg.bias_dropout_fusion=false \ - policy.megatron_cfg.optimizer.optimizer_cpu_offload=false \ - policy.megatron_cfg.optimizer.optimizer_offload_fraction=0.0 \ - ++policy.megatron_cfg.optimizer.params_dtype=bfloat16 \ - ++policy.megatron_cfg.optimizer.main_grads_dtype=bfloat16 \ - ++policy.megatron_cfg.optimizer.main_params_dtype=float16 \ - ++policy.megatron_cfg.optimizer.exp_avg_dtype=bfloat16 \ - ++policy.megatron_cfg.optimizer.exp_avg_sq_dtype=bfloat16 \ - ++policy.megatron_cfg.optimizer.store_param_remainders=false \ - policy.generation.colocated.enabled=false \ - policy.generation.colocated.resources.num_nodes=1 \ - policy.generation.colocated.resources.gpus_per_node=1 \ policy.generation.max_new_tokens=128 \ - policy.generation.vllm_cfg.tensor_parallel_size=1 \ - policy.generation.vllm_cfg.expert_parallel_size=1 \ - policy.generation.vllm_cfg.pipeline_parallel_size=1 \ policy.generation.vllm_cfg.max_model_len=1024 \ - policy.generation.vllm_cfg.gpu_memory_utilization=0.6 \ ++policy.generation.vllm_kwargs.max_num_batched_tokens=1024 \ - ++async_rl.generation_fleet_health.refit_timeout_s=null \ policy.max_total_sequence_length=1024 \ data.train.data_path="${TRAIN_PATH}" \ data.validation.data_path="${VAL_PATH}" \ grpo.num_prompts_per_step=1 \ grpo.num_generations_per_prompt=2 \ grpo.max_num_steps=1 \ - grpo.val_period=0 \ - grpo.val_at_start=false \ - grpo.val_at_end=false \ policy.train_global_batch_size=2 \ - policy.train_micro_batch_size=1 \ - data_plane.simple.num_storage_units=2 \ async_rl.min_groups_for_streaming_train=1 \ async_rl.max_inflight_prompts=2 \ async_rl.max_buffered_rollouts=2 \ - logger.tensorboard_enabled=true \ logger.log_dir="${LOG_DIR}" \ - logger.wandb_enabled=false \ - logger.monitor_gpus=false \ checkpointing.enabled=false \ "$@" 2>&1 | tee "${RUN_LOG}" diff --git a/tests/unit/data/test_captured_media.py b/tests/unit/data/test_captured_media.py index 80b4e8b667f..34e77b020d5 100644 --- a/tests/unit/data/test_captured_media.py +++ b/tests/unit/data/test_captured_media.py @@ -36,9 +36,13 @@ ) from nemo_rl.data.captured_media import ( + MEDIA_TENSOR_DIGESTS_FIELD, + CapturedMedia, MediaCaptureRejected, MediaColumnSpec, + media_tensor_digests, pack_images, + verify_restored_media, ) from nemo_rl.data.captured_media import ( capture_processed_media as _capture_processed_media, @@ -64,6 +68,7 @@ from nemo_rl.experience.rollout_reassembler import RolloutReassembler from nemo_rl.experience.route_assembly import verify_route_fragment_integrity from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens +from nemo_rl.models.generation.prefix_recovery import PrefixRecovery from nemo_rl.models.generation.vllm.vllm_worker_async import ( VllmAsyncGenerationWorkerImpl, ) @@ -729,6 +734,7 @@ def test_worker_restart_validates_retained_media( ) worker = SimpleNamespace( _capture_media=True, + _hash_captured_media=False, _capture_image_token_id=18, _capture_patch_size=1, _staging_source=source, @@ -774,6 +780,214 @@ def test_worker_restart_validates_retained_media( assert excinfo.value.code == "retained_media_changed" +def _stage_media_prefix(dp, tokens, prompt): + """Capture a media call with prefix cuts on and stage its first prefix row. + + Returns the worker, the captured media, and the row's staging key; the row + is written exactly as a checkpoint cut writes it. + """ + worker = SimpleNamespace( + _capture_media=True, + _hash_captured_media=True, + _capture_image_token_id=18, + _capture_patch_size=1, + ) + admission = CaptureAdmission( + rollout_id="r0", model_call_id="c1", prev_len=0, mode="text" + ) + media = VllmAsyncGenerationWorkerImpl._capture_request_media( + worker, prompt, admission=admission + ) + sink = TQTokenSink( + dp, + staging_partition="staging", + capture_media=True, + media_pixel_dtype=torch.float32, + ) + capture = RolloutTokenCapture( + sink=sink, weight_version_fn=lambda: 3, adapter=VLLMCaptureAdapter() + ) + record = capture.build_prefix_record( + capture.begin_call(admission), + prompt_token_ids=tokens, + generated_token_ids=[31], + generated_logprobs=[-0.25], + extras=VllmAsyncGenerationWorkerImpl._generation_cut_record_payload( + worker, SimpleNamespace(media=media) + ), + ) + staged = sink.stage_generation_prefix( + record, checkpoint_id="checkpoint-1", chunk_sequence=0 + ) + assert staged.ok + return worker, media, staged.staging_key + + +@pytest.mark.parametrize( + ("tokens", "make_prompt", "pixels"), + [ + pytest.param( + [10, 18, 18, 11], + lambda tokens, pixels: engine_prompt(tokens, [(Span(1, 2), pixels)]), + torch.arange(18, dtype=torch.float32).reshape(3, 2, 3), + id="image", + ), + pytest.param( + # Two temporal tubelets separated by timestamp text. + [10, 90, 18, 18, 91, 18, 18, 11], + lambda tokens, frames: video_prompt(tokens, [(Span(1, 6), frames)]), + torch.arange(48, dtype=torch.float32).reshape(4, 3, 2, 2), + id="video", + ), + ], +) +def test_generation_prefix_restore_verifies_media_by_digest_without_pixels( + dp, tokens, make_prompt, pixels +): + worker, media, staging_key = _stage_media_prefix( + dp, tokens, make_prompt(tokens, pixels) + ) + assert media.tensor_digests == media_tensor_digests(media.tensors) + # Only a video call carries frame counts, so only its digests cover them. + is_video = media.items[0].modality == "video" + assert ("num_frames" in media.tensor_digests) is is_video + + client = RecordingClient(dp) + source = TQTokenSource(client, staging_partition="staging", capture_media=True) + [chunk] = source.fetch_for_finalization([staging_key]) + assert not chunk.media_present + restored = VllmAsyncGenerationWorkerImpl._restore_generation_cut_media( + worker, _verified_snapshot(chunk) + ) + assert restored == CapturedMedia( + items=media.items, tensors=None, tensor_digests=media.tensor_digests + ) + assert not client.tensor_reads() + + resumed = CaptureAdmission( + rollout_id="r0-a1", model_call_id="c2", prev_len=0, mode="text" + ) + verified = VllmAsyncGenerationWorkerImpl._capture_request_media( + worker, + make_prompt(tokens, pixels), + admission=resumed, + restored=restored, + ) + torch.testing.assert_close(verified.tensors["imgs"], media.tensors["imgs"]) + + with pytest.raises( + MediaCaptureRejected, match=r"changed tensors: \['imgs'\]" + ) as error: + VllmAsyncGenerationWorkerImpl._capture_request_media( + worker, + make_prompt(tokens, torch.full_like(pixels, 9)), + admission=resumed, + restored=restored, + ) + assert error.value.code == "retained_media_changed" + + +def _media_prefix_chunk(dp): + tokens = [10, 18, 18, 11] + pixels = torch.arange(18, dtype=torch.float32).reshape(3, 2, 3) + worker, _, staging_key = _stage_media_prefix( + dp, tokens, engine_prompt(tokens, [(Span(1, 2), pixels)]) + ) + source = TQTokenSource(dp, staging_partition="staging", capture_media=True) + [chunk] = source.fetch_for_finalization([staging_key]) + return worker, chunk + + +def _verified_snapshot(chunk): + """A one-chunk prefix as resolve_restore verifies it: the chunk's record with its extras.""" + return SimpleNamespace( + extras=chunk.extras, + token_ids_delta=chunk.snapshot.token_ids_delta, + prev_len=chunk.snapshot.prev_len, + ) + + +def test_generation_prefix_media_restore_fails_closed_without_extras(dp): + # The prefix digest covers the extras, so only a writer bug drops them. + worker, chunk = _media_prefix_chunk(dp) + + with pytest.raises(MediaCaptureRejected, match="media metadata is missing"): + VllmAsyncGenerationWorkerImpl._restore_generation_cut_media( + worker, _verified_snapshot(replace(chunk, extras=None)) + ) + + +def test_generation_cut_rebuild_rejects_extras_after_the_first_chunk(dp): + _, chunk = _media_prefix_chunk(dp) + later = replace(chunk, extras={MEDIA_SPANS_FIELD: []}) + recovery = SimpleNamespace( + _parts=SimpleNamespace( + staging_source=SimpleNamespace( + fetch_for_finalization=lambda keys: [chunk, later] + ) + ) + ) + admission = SimpleNamespace( + generation_cut=SimpleNamespace(staging_keys=("chunk-0", "chunk-1")) + ) + + # The check runs before the admission or recovery state is consulted. + with pytest.raises(RuntimeError, match="only the first generation-cut chunk"): + PrefixRecovery._rebuild_prefix(recovery, admission, []) + + +def _image_media() -> CapturedMedia: + return capture_processed_media( + engine_prompt([10, 18, 18, 11], [(Span(1, 2), torch.ones(3, 2, 3))]), + prev_len=0, + ) + + +@pytest.mark.parametrize( + ("change", "match"), + [ + ( + lambda media: replace(media, items=()), + r"items equal: False; changed tensors: \[\]", + ), + ( + lambda media: replace( + media, + tensors={**media.tensors, "imgs": media.tensors["imgs"] + 1}, + ), + r"items equal: True; changed tensors: \['imgs'\]", + ), + ], + ids=["items", "bytes"], +) +def test_verify_restored_media_rejects_changed_items_or_tensors(change, match): + def hashed(media: CapturedMedia) -> CapturedMedia: + # As reprocessing does whenever prefix cuts are on. + return replace(media, tensor_digests=media_tensor_digests(media.tensors)) + + media = hashed(_image_media()) + restored = replace(media, tensors=None) + verify_restored_media(media, restored) + + with pytest.raises(MediaCaptureRejected, match=match) as error: + verify_restored_media(hashed(change(media)), restored) + assert error.value.code == "retained_media_changed" + + +def test_media_tensor_digests_fingerprint_dtype_shape_and_bytes(): + pixels = torch.arange(6, dtype=torch.bfloat16).reshape(2, 3) + digests = media_tensor_digests({"imgs": pixels, "imgs_sizes": torch.ones(1, 2)}) + + assert list(digests) == ["imgs", "imgs_sizes"] + assert digests["imgs"]["dtype"] == "torch.bfloat16" + assert digests["imgs"]["shape"] == [2, 3] + assert media_tensor_digests({"imgs": pixels.clone()}) == {"imgs": digests["imgs"]} + reshaped = media_tensor_digests({"imgs": pixels.reshape(3, 2)})["imgs"] + assert reshaped["sha256"] == digests["imgs"]["sha256"] + assert reshaped["shape"] != digests["imgs"]["shape"] + assert media_tensor_digests(None) == {} + + @pytest.mark.parametrize("with_routed_experts", [False, True]) def test_worker_completion_stages_pixels_and_only_returns_capture_coordinates( dp, monkeypatch, with_routed_experts @@ -1103,6 +1317,7 @@ def retained_call(dp): source = TQTokenSource(client, staging_partition="staging", capture_media=True) worker = SimpleNamespace( _capture_media=True, + _hash_captured_media=False, _capture_image_token_id=18, _capture_patch_size=1, _staging_source=source, @@ -1437,19 +1652,26 @@ async def create_chat_completion(request, raw_request): _BUILT["chat"][0].create_chat_completion = create_chat_completion handler = dict(app.routes)["/v1/chat/completions"] - request = SimpleNamespace( - top_k=-1, - min_p=None, - repetition_penalty=None, - presence_penalty=0.0, - frequency_penalty=0.0, - logit_bias=None, - top_p=1.0, - temperature=1.0, - ng_capture=admission.model_dump(), - required_prefix_token_ids=None, - model_copy=lambda **kwargs: request, - ) + + def chat_request(ng_capture, **fields): + """A chat request with the sampling settings the server accepts.""" + request = SimpleNamespace( + top_k=-1, + min_p=None, + repetition_penalty=None, + presence_penalty=0.0, + frequency_penalty=0.0, + logit_bias=None, + top_p=1.0, + temperature=1.0, + ng_capture=ng_capture, + required_prefix_token_ids=None, + **fields, + ) + request.model_copy = lambda **kwargs: request + return request + + request = chat_request(admission.model_dump()) # The corresponding template prefix has no suffix user text. calls = 0 @@ -1505,6 +1727,110 @@ async def run(): assert isinstance(failure, GymTransportError) assert "retained_media_changed" in str(failure) + # A restored call reprocesses its image and checks it against the digests + # its cut saved, before begin_call and inference: other pixels now are + # rejected instead of resuming a prefix sampled on the old ones. + worker.token_capture.reset_mock() + inference.reset_mock() + worker._hash_captured_media = True + cut_tokens = [10, 18, 18, 11] + staged_pixels = torch.ones(3, 2, 3) + staged = capture_processed_media( + engine_prompt(cut_tokens, [(Span(1, 2), staged_pixels)]), prev_len=0 + ) + cut = SimpleNamespace( + token_ids_delta=[*cut_tokens, 31], + token_mask_delta=[0.0] * len(cut_tokens) + [1.0], + prev_len=0, + extras={ + MEDIA_SPANS_FIELD: [item.to_dict() for item in staged.items], + MEDIA_TENSOR_DIGESTS_FIELD: media_tensor_digests(staged.tensors), + }, + ) + restored_capture = { + **CaptureAdmission( + capture_key="r1", model_call_id="c9", prev_len=0, mode="text" + ).model_dump(), + "generation_cut": { + "source_capture_key": "r1", + "source_model_call_id": "c9", + "staging_keys": ["active/checkpoint-1/r1/c9/0"], + "prefix_token_count": 1, + "prefix_digest": "0" * 64, + "effective_output_limit": 8, + }, + } + restored_request = chat_request( + restored_capture, max_tokens=8, max_completion_tokens=None, min_tokens=None + ) + # resolve_restore's verified snapshot, as on main. + monkeypatch.setattr( + worker._prefix_recovery, + "resolve_restore", + lambda admission, prefix_token_ids: (admission, cut), + ) + monkeypatch.setattr( + renderer, "model_config", SimpleNamespace(max_model_len=64), raising=False + ) + renders = 0 + + async def reprocess_other_pixels(self, **kwargs): + nonlocal renders + renders += 1 + # The request's own render, then its template prefix (none: no turn + # precedes the call). + if renders == 1: + return [], [engine_prompt(cut_tokens, [(Span(1, 2), staged_pixels + 1)])] + return [], [engine_prompt([])] + + monkeypatch.setattr( + _OnlineRenderer, "preprocess_chat", reprocess_other_pixels, raising=False + ) + response = await handler(restored_request, MagicMock()) + assert response.status_code == 400 + error = json.loads(response.body)["error"] + assert error["code"] == "retained_media_changed" + assert "changed tensors: ['imgs']" in error["message"] + inference.assert_not_awaited() + worker.token_capture.begin_call.assert_not_called() + + # A recovery request whose original URL/path/blob has disappeared fails + # while vLLM renders its engine-native prompt: before any TQ read and + # before begin_call. Prefix rows hold only tensor digests, so there are no + # staged pixels it could resume from instead. + worker.token_capture.reset_mock() + inference.reset_mock() + worker._staging_source = MagicMock(spec=TQTokenSource) + worker._chain_prefix = MagicMock(spec=ChainPrefixCache) + recovery_capture = { + **admission.model_dump(), + "generation_cut": { + "source_capture_key": admission.capture_key, + "source_model_call_id": admission.model_call_id, + "staging_keys": ["active/checkpoint-1/r0/c2/0"], + "prefix_token_count": 1, + "prefix_digest": "0" * 64, + "effective_output_limit": 8, + }, + } + missing_request = chat_request(recovery_capture) + + async def missing_media(self, **kwargs): + raise FileNotFoundError("original media disappeared") + + monkeypatch.setattr( + _OnlineRenderer, "preprocess_chat", missing_media, raising=False + ) + with pytest.raises(FileNotFoundError, match="original media disappeared"): + await handler(missing_request, MagicMock()) + assert worker._capture_admission(missing_request).generation_cut is not None + assert worker._staging_source.method_calls == [] + assert worker._chain_prefix.method_calls == [] + inference.assert_not_awaited() + worker.token_capture.begin_call.assert_not_called() + worker.token_capture.complete_call_from_response.assert_not_called() + assert len(worker._capture_registry) == 0 + @pytest.mark.parametrize("routes", [False, True]) def test_rl_owned_multimodal_rows_reuse_foundation_media_and_cleanup(dp, routes): diff --git a/tests/unit/environments/test_gym_prefix_recovery_parity.py b/tests/unit/environments/test_gym_prefix_recovery_parity.py index 6bf91bd2860..c0feb95e74b 100644 --- a/tests/unit/environments/test_gym_prefix_recovery_parity.py +++ b/tests/unit/environments/test_gym_prefix_recovery_parity.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import hashlib import importlib.util import json @@ -272,6 +273,91 @@ def test_prefix_completion_retires_old_keys() -> None: ) +def _dump_restored_run( + tmp_path: Path, *, flushed_keys: list[str] +) -> argparse.Namespace: + """A phase-two trainer dump, log, and selection for ``verify-completion``.""" + from nemo_rl.algorithms.single_controller_utils.debug_writer import ( + JsonDebugWriter, + TrainChunk, + ) + + log_dir = tmp_path / "recovery-logs" + token_ids = [1, 2] + list(range(100, 112)) + dump = JsonDebugWriter(str(log_dir / "exp_001")) + receipt = dump.add_chunk( + TrainChunk( + step=0, + sample_ids=["group-a_g0"], + tags=[{"prompt_idx": 0}], + input_lengths=torch.tensor([len(token_ids)]), + sequences={ + "token_ids": torch.tensor([token_ids]), + # The trainer masked one position inside the restored call. + "token_loss_mask": torch.tensor( + [[0.0, 0.0] + [1.0] * 6 + [0.0] + [1.0] * 5] + ), + "advantages": torch.zeros(1, len(token_ids)), + "generation_logprobs": torch.zeros(1, len(token_ids)), + "prev_logprobs": torch.zeros(1, len(token_ids)), + }, + scalars={ + "sample_loss_mask": torch.tensor([1.0]), + "rewards": torch.tensor([1.0]), + }, + ) + ) + dump.finish_step(0, [receipt]) + staging_keys = tmp_path / "recovery-staging-keys.jsonl" + staging_keys.write_text( + json.dumps({"train_step": 1, "staging_sample_ids": ["prefix-key"]}) + "\n" + ) + flush_keys = tmp_path / "recovery-flush-keys.jsonl" + flush_keys.write_text( + json.dumps({"epoch": 1, "staging_sample_ids": flushed_keys}) + "\n" + ) + selection = tmp_path / "selected-snapshot.json" + selection.write_text( + json.dumps( + { + "cuts": [ + { + **_cut("call-a", 0, 5), + "group_id": "group-a", + "staging_keys": ["prefix-key"], + } + ] + } + ) + ) + phase2_log = tmp_path / "phase2.log" + phase2_log.write_text(_completion_log("call-a")) + return argparse.Namespace( + selection_file=str(selection), + recovery_log_dir=str(log_dir), + recovery_staging_keys=str(staging_keys), + recovery_flush_keys=str(flush_keys), + phase2_log=str(phase2_log), + ) + + +def test_verify_completion_checks_a_reference_free_restore(tmp_path: Path, capsys): + _SNAPSHOT_HELPER.verify_completion( + _dump_restored_run(tmp_path, flushed_keys=["new-active-key"]) + ) + + assert "('group-a', 0)" in capsys.readouterr().out + + +def test_verify_completion_rejects_a_restored_prefix_that_outlived_the_snapshot( + tmp_path: Path, +): + with pytest.raises(AssertionError, match="outlived the next snapshot"): + _SNAPSHOT_HELPER.verify_completion( + _dump_restored_run(tmp_path, flushed_keys=["prefix-key"]) + ) + + @pytest.mark.parametrize( ("live_at_training", "live_after_flush", "match"), [ diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 8f5840e6095..212010c8bf5 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1605,6 +1605,7 @@ def checkpoint_cut(inventory, *, deadline): control_token=token, control_timeout_s=timeout_s, progress_reader=None, + prefix_row_extras=lambda state: None, ) ) recovery.checkpoint_cut = checkpoint_cut diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index dba33d65d6a..476e737dbd5 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -24,12 +24,14 @@ import asyncio import hashlib +import json import logging import threading import time from collections import Counter +from concurrent.futures import ThreadPoolExecutor from copy import deepcopy -from dataclasses import asdict +from dataclasses import asdict, replace from types import SimpleNamespace from unittest.mock import MagicMock @@ -53,8 +55,15 @@ StageResult, ) -from nemo_rl.data.captured_media import MediaColumnSpec # noqa: E402 +from nemo_rl.data.captured_media import ( # noqa: E402 + MEDIA_TENSOR_DIGESTS_FIELD, + CapturedMedia, + CapturedMediaItem, + MediaColumnSpec, + media_tensor_digests, +) from nemo_rl.data_plane.tq_token_sink import ( # noqa: E402 + FetchedStagedCall, StagedPrefix, TQTokenSink, ) @@ -103,6 +112,7 @@ def __init__(self) -> None: self.records: list[StagedCallRecord] = [] self.attachments: list[dict[str, torch.Tensor] | None] = [] self.generation_prefix_records: list[tuple[str, StagedCallRecord]] = [] + self.generation_prefix_attachments: list[dict | None] = [] self.generation_prefix_keys: list[str] = [] self.cleared_generation_prefix_keys: list[str] = [] @@ -124,8 +134,8 @@ def stage_generation_prefix( chunk_sequence: int, attachments: dict | None = None, ) -> StageResult: - assert attachments is None self.generation_prefix_records.append((checkpoint_id, record)) + self.generation_prefix_attachments.append(attachments) key = ( f"__generation_cut__/{checkpoint_id}/{record.capture_key}/" f"{record.model_call_id}/{chunk_sequence}" @@ -190,6 +200,7 @@ def _cuts( control_token="secret", control_timeout_s=None, progress_reader=progress_reader or _NoCallsInVllm(), + prefix_row_extras=worker._generation_cut_record_payload, ) ) worker._prefix_recovery = recovery @@ -369,7 +380,7 @@ def test_generation_prefix_restore_counts_sum_over_workers(monkeypatch): def test_prefix_capture_setup_requires_a_control_token(monkeypatch): - # Prefix recovery with media or sampler top-k is refused at config/setup. + # Prefix recovery with sampler top-k is refused at config/setup. monkeypatch.setattr( "nemo_rl.data_plane.build_data_plane_client", lambda dp_cfg, bootstrap, segment_size=None: MagicMock(), @@ -574,15 +585,19 @@ def replay(tokenizer, **kwargs): def _captured_text_call( - worker, model_call_id: str = "c1", prompt_token_ids: tuple[int, ...] = (10, 11) + worker, + model_call_id: str = "c1", + prompt_token_ids: tuple[int, ...] = (10, 11), + *, + media: CapturedMedia | None = None, ) -> _FakeRequest: - """Begin capturing a parentless text call of rollout ``r0``.""" + """Begin capturing a parentless text-mode call of rollout ``r0``.""" request = _FakeRequest( ng_capture={"rollout_id": "r0", "model_call_id": model_call_id, "mode": "text"}, stream=False, ) VllmAsyncGenerationWorkerImpl._begin_request_capture( - worker, request, list(prompt_token_ids) + worker, request, list(prompt_token_ids), media=media ) return request @@ -594,9 +609,10 @@ def _live_call( *, model_call_id: str = "c1", prompt_token_ids: tuple[int, ...] = (10, 11), + media: CapturedMedia | None = None, ): """Begin one captured call that has decoded ``generated_token_ids``.""" - request = _captured_text_call(worker, model_call_id, prompt_token_ids) + request = _captured_text_call(worker, model_call_id, prompt_token_ids, media=media) state = worker._capture_registry.get(request) state.set_effective_output_limit(8) state.record(generated_token_ids, logprobs) @@ -1317,9 +1333,19 @@ def fetch_prefix_chains(self, staging_keys: list[str]) -> StagedPrefix: expanded=[token for key in staging_keys for token in self.deltas[key]] ) - def fetch(self, staging_keys: list[str]) -> list[StagedCallRecord]: + def fetch_for_finalization( + self, staging_keys: list[str] + ) -> list[FetchedStagedCall]: self.calls.append(list(staging_keys)) - return [self.records[key] for key in staging_keys] + return [ + FetchedStagedCall( + staging_key=key, + snapshot=self.records[key], + routed_len=0, + extras=self.records[key].extras, + ) + for key in staging_keys + ] def _served_content(gen_ids, logprobs): @@ -1734,6 +1760,7 @@ def _record_cut_constraints(state, sampling_params) -> None: control_token=None, control_timeout_s=None, progress_reader=_NoCallsInVllm(), + prefix_row_extras=lambda state: None, ) ).on_sampling_params(object(), sampling_params) @@ -1862,6 +1889,319 @@ def test_uncuttable_sampling_settings_warn_once_per_reason(monkeypatch, caplog): assert caplog.text.count("are not cut; they regenerate after a restore") == 1 +def _captured_image_media() -> CapturedMedia: + tensors = { + "imgs": torch.ones(1, 2, 3), + "imgs_sizes": torch.tensor([[1, 2]], dtype=torch.int32), + } + # As _capture_request_media returns it when prefix cuts are enabled. + return CapturedMedia( + items=( + CapturedMediaItem( + modality="image", + placeholder_offset=0, + placeholder_length=2, + placeholder_digest=hashlib.sha256( + json.dumps([18, 18], separators=(",", ":")).encode() + ).hexdigest(), + token_id=18, + embedding_spans=((0, 2),), + imgs_sizes=((1, 2),), + ), + ), + tensors=tensors, + tensor_digests=media_tensor_digests(tensors), + ) + + +def _live_image_call(worker, media: CapturedMedia | None = None): + """An active call whose prompt is one image's two placeholder tokens.""" + media = media or _captured_image_media() + request, state = _live_call( + worker, [20], [-0.1], prompt_token_ids=(18, 18), media=media + ) + return request, state, media + + +def test_generation_cut_stages_media_digests_only_with_first_prefix_row(): + sink = _MemorySink() + worker = _worker_with_capture(sink) + _, state, media = _live_image_call(worker) + + first_receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-1") + ) + state.record([21], [-0.2]) + second_receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-2") + ) + + first, second = [record for _, record in sink.generation_prefix_records] + assert first.extras == { + "media_spans": [media.items[0].to_dict()], + MEDIA_TENSOR_DIGESTS_FIELD: media.tensor_digests, + } + assert second.extras is None + # Prefix rows never hold pixels; only the terminal row stages them. + assert sink.generation_prefix_attachments == [None, None] + assert second_receipt.prefixes[0].staging_keys == ( + *first_receipt.prefixes[0].staging_keys, + sink.generation_prefix_keys[-1], + ) + assert state.snapshot().staging_keys == second_receipt.prefixes[0].staging_keys + assert not sink.cleared_generation_prefix_keys + + +def test_generation_cut_refuses_media_captured_without_digests(caplog): + sink = _MemorySink() + worker = _worker_with_capture(sink) + _, state, _ = _live_image_call( + worker, replace(_captured_image_media(), tensor_digests=None) + ) + + with caplog.at_level(logging.ERROR): + receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-1") + ) + + # Only this call is left uncut; it regenerates after a restore. + assert [ack.disposition for ack in receipt.prefixes] == ["durable_failure"] + assert "without tensor digests" in caplog.text + assert not sink.generation_prefix_records + assert not state.snapshot().cut_in_progress + assert state.snapshot().staged_token_count == 0 + + +def _restore_request(ack, *, source=("r0", "c1"), model_call_id="c2") -> _FakeRequest: + """The call Gym sends to resume ``ack``'s cut of ``source``.""" + return _FakeRequest( + ng_capture={ + "rollout_id": "r0-a1", + "model_call_id": model_call_id, + "mode": "text", + "generation_cut": { + "source_capture_key": source[0], + "source_model_call_id": source[1], + "staging_keys": list(ack.staging_keys), + "prefix_token_count": ack.prefix_token_count, + "prefix_digest": ack.prefix_digest, + "effective_output_limit": ack.effective_output_limit, + }, + }, + stream=False, + ) + + +def _cut_again_after_restore(sink, worker, ack, monkeypatch): + """Restore an image call's cut, reprocess its image, and cut the resumed call. + + The second cut's cumulative record takes its extras from the reprocessed + media, while a restore rebuilds them from the first run's row. + """ + records = { + key: record + for key, record in zip( + sink.generation_prefix_keys, + (record for _, record in sink.generation_prefix_records), + strict=True, + ) + } + resumed = _resumed_worker( + sink, records=records, weight_version=worker._rollout_weight_version + ) + resumed._capture_media = resumed._hash_captured_media = True + request = _restore_request(ack) + admission, snapshot = _cuts(resumed).resolve_restore( + resumed._capture_admission(request), [] + ) + assert snapshot is not None + # vLLM reprocesses the same image; the worker hashes it and checks it + # against the digests the first run's row saved. + monkeypatch.setattr( + "nemo_rl.models.generation.vllm.vllm_worker_async.capture_processed_media", + lambda *args, **kwargs: replace(_captured_image_media(), tensor_digests=None), + ) + media = resumed._capture_restored_media( + {"prompt_token_ids": [18, 18, 20]}, + admission=admission, + splice=None, + snapshot=snapshot, + ) + VllmAsyncGenerationWorkerImpl._begin_request_capture( + resumed, + request, + [18, 18, 20], + admission=admission, + prefix_token_ids=[], + media=media, + generation_cut=snapshot, + resumed_generation_token_ids=[20], + ) + resumed._capture_registry.get(request).record([21], [-0.2]) + (second,) = ( + _cuts(resumed) + .checkpoint_cut( + _single_call_inventory("c2", checkpoint_id="checkpoint-2", attempt=1) + ) + .prefixes + ) + # The first run's row, then the resumed call's new chunk. + assert len(second.staging_keys) == 2 + assert second.staging_keys[0] == ack.staging_keys[0] + return second + + +@pytest.mark.parametrize( + ("cuts", "resumed"), + [(1, False), (2, False), (1, True)], + ids=["image_one_cut", "image_two_cuts", "image_resumed_then_cut"], +) +def test_generation_cut_digest_written_at_the_cut_is_the_one_restore_rebuilds( + cuts: int, resumed: bool, monkeypatch +): + # test_usable_restored_prefix_is_resumed covers a text call's digest. + sink = _MemorySink() + worker = _worker_with_capture(sink) + _, state, _ = _live_image_call(worker) + for index in range(cuts): + if index: + state.record([21], [-0.2]) + receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id=f"checkpoint-{index + 1}") + ) + (ack,) = receipt.prefixes + source = ("r0", "c1") + if resumed: + ack = _cut_again_after_restore(sink, worker, ack, monkeypatch) + source = ("r0-a1", "c2") + records = dict( + zip( + sink.generation_prefix_keys, + (record for _, record in sink.generation_prefix_records), + strict=True, + ) + ) + assert list(records) == list(ack.staging_keys) + first = records[ack.staging_keys[0]] + assert MEDIA_TENSOR_DIGESTS_FIELD in first.extras + + def resolve(staged: dict[str, StagedCallRecord]): + resumed = _resumed_worker( + sink, records=staged, weight_version=worker._rollout_weight_version + ) + request = _restore_request(ack, source=source) + return _cuts(resumed)._rebuild_prefix(resumed._capture_admission(request), []) + + # Restore recomputes the cumulative digest from the staged chunks, the + # first chunk's media extras included, and must reach the cut's digest: + # Gym's begin_call rejects a restored snapshot whose digest differs. + assert resolve(records).digest == ack.prefix_digest + # The tensor digests are covered: a changed one changes the digest. + digests = dict(first.extras[MEDIA_TENSOR_DIGESTS_FIELD]) + digests["imgs"] = {**digests["imgs"], "sha256": "0" * 64} + tampered = first.model_copy( + update={"extras": {**first.extras, MEDIA_TENSOR_DIGESTS_FIELD: digests}} + ) + assert ( + resolve({**records, ack.staging_keys[0]: tampered}).digest != ack.prefix_digest + ) + + +class _BlockingPrefixSink(_MemorySink): + def __init__(self) -> None: + super().__init__() + self.prefix_write_started = threading.Event() + self.release_prefix_write = threading.Event() + + def stage_generation_prefix(self, *args, **kwargs) -> StageResult: + self.prefix_write_started.set() + assert self.release_prefix_write.wait(5) + return super().stage_generation_prefix(*args, **kwargs) + + +def _assert_stages_captured_tensors(attachments, tensors) -> None: + """The terminal row stages the captured media tensors themselves, not copies. + + The worker builds a fresh attachments dict (it may add sampler top-k), so + compare the tensors, not the dict. + """ + assert attachments.keys() == tensors.keys() + assert all(attachments[name] is tensor for name, tensor in tensors.items()) + + +def test_media_terminal_completion_waits_for_prefix_cut_then_replaces_it(): + sink = _BlockingPrefixSink() + worker = _worker_with_capture(sink) + request, _, media = _live_image_call(worker) + terminal_entered = threading.Event() + terminal_finished = threading.Event() + + def complete(): + terminal_entered.set() + try: + return VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, request, _served_content([20, 21], [-0.1, -0.2]) + ) + finally: + terminal_finished.set() + + with ThreadPoolExecutor(max_workers=2) as executor: + cut = executor.submit( + _cuts(worker).checkpoint_cut, + _single_call_inventory(checkpoint_id="checkpoint-1"), + ) + assert sink.prefix_write_started.wait(5) + terminal = executor.submit(complete) + assert terminal_entered.wait(5) + assert not terminal_finished.wait(0.05) + sink.release_prefix_write.set() + receipt = cut.result(timeout=5) + content = terminal.result(timeout=5) + + assert receipt.prefixes[0].disposition == "durable_prefix" + assert content["ng_commit_coords"]["disposition"] == "staged" + assert sink.generation_prefix_attachments == [None] + _assert_stages_captured_tensors(sink.attachments[0], media.tensors) + assert sink.cleared_generation_prefix_keys == [] + _flush_retired_rows(worker) + assert sink.cleared_generation_prefix_keys == list(receipt.prefixes[0].staging_keys) + + +def test_canonical_media_row_survives_prefix_cleanup_failure(caplog): + sink = _FailingClearSink() + worker = _worker_with_capture(sink) + request, _, media = _live_image_call(worker) + _cuts(worker).checkpoint_cut(_single_call_inventory(checkpoint_id="checkpoint-1")) + + content = VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, request, _served_content([20, 21], [-0.1, -0.2]) + ) + with caplog.at_level(logging.ERROR): + _flush_retired_rows(worker) + + assert content["ng_commit_coords"]["disposition"] == "staged" + assert sink.records[-1].staging_key == "r0/c1" + _assert_stages_captured_tensors(sink.attachments[-1], media.tensors) + assert "failed to clear 1 obsolete generation chunk rows" in caplog.text + + +def test_generation_cut_during_media_preprocessing_redispatches_call(): + sink = _MemorySink() + worker = _worker_with_capture(sink) + worker._capture_media = True + + # begin_request_capture runs only after vLLM finishes media preprocessing. + # A cut that arrives in that window sees no active state and must tell Gym + # to redispatch the model call instead of advertising an empty prefix. + receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-before-preprocess") + ) + + assert receipt.prefixes[0].disposition == "durable_failure" + assert not sink.generation_prefix_records + assert not sink.generation_prefix_attachments + + class _FailOncePrefixSink(_MemorySink): def __init__(self) -> None: super().__init__() @@ -2015,17 +2355,21 @@ def test_usable_restored_prefix_is_resumed(): worker = _resumed_worker(sink, records={cut_key: cut_record}) admission = worker._capture_admission(request) - served, snapshot = _cuts(worker).resolve_restore(admission, []) + served, resolved = _cuts(worker).resolve_restore(admission, []) assert served is admission - assert snapshot.digest == cut_record.digest + assert resolved.digest == cut_record.digest _flush_retired_rows(worker) assert sink.cleared_generation_prefix_keys == [] @pytest.mark.parametrize( "failure", - ["missing chunk", "chunk newer than the worker", "digest mismatch"], + [ + "missing chunk", + "chunk newer than the worker", + "digest mismatch", + ], ) def test_unusable_restored_prefix_regenerates_instead_of_failing(failure, caplog): sink = _MemorySink() @@ -2044,11 +2388,10 @@ def test_unusable_restored_prefix_regenerates_instead_of_failing(failure, caplog ) } ) - with caplog.at_level(logging.WARNING): - served, snapshot = _cuts(worker).resolve_restore(admission, []) + served, resolved = _cuts(worker).resolve_restore(admission, []) - assert snapshot is None + assert resolved is None assert served.generation_cut is None assert "regenerating it from its turn boundary" in caplog.text # The regenerated call is admitted as an ordinary call from its boundary. @@ -2503,10 +2846,10 @@ def failed_put(key, fields, **kwargs): assert len(worker._capture_registry) == 0 -def _cut_active_call(worker, sink, model_call_id: str = "c1"): +def _cut_active_call(worker, sink, model_call_id: str = "c1", **call): """Begin one captured call and stage its prefix at a generation cut.""" request, state = _live_call( - worker, [20, 21], [-0.1, -0.2], model_call_id=model_call_id + worker, [20, 21], [-0.1, -0.2], model_call_id=model_call_id, **call ) inventory = _single_call_inventory(model_call_id) _cuts(worker).checkpoint_cut(inventory) @@ -2533,13 +2876,20 @@ class _FailingTerminalSink(_MemorySink): def stage( self, record: StagedCallRecord, *, attachments: dict | None = None ) -> StageResult: + self.attachments.append(attachments) return StageResult(ok=False, error="tq unavailable") -def test_call_whose_terminal_staging_fails_clears_its_cut_rows(): +@pytest.mark.parametrize("media", [False, True], ids=["text", "image"]) +def test_call_whose_terminal_staging_fails_clears_its_cut_rows(media: bool): sink = _FailingTerminalSink() worker = _worker_with_capture(sink) - request = _cut_active_call(worker, sink) + image = _captured_image_media() if media else None + request = _cut_active_call( + worker, + sink, + **({"prompt_token_ids": (18, 18), "media": image} if media else {}), + ) content = VllmAsyncGenerationWorkerImpl._finish_request_capture( worker, @@ -2548,6 +2898,8 @@ def test_call_whose_terminal_staging_fails_clears_its_cut_rows(): ) assert content["ng_commit_coords"]["disposition"] == "capture_failed" + if media: + _assert_stages_captured_tensors(sink.attachments[0], image.tensors) _flush_retired_rows(worker) assert sink.cleared_generation_prefix_keys == sink.generation_prefix_keys @@ -2572,8 +2924,11 @@ def test_request_capture_abort_fails_the_call_and_drains_state(): @pytest.mark.vllm +@pytest.mark.parametrize("prefix_cuts", [False, True]) @pytest.mark.parametrize("pruning_rate", [0.0, 0.5]) -def test_omni_capture_setup_rejects_video_pruning(monkeypatch, pruning_rate): +def test_omni_capture_setup_rejects_video_pruning( + monkeypatch, pruning_rate, prefix_cuts +): # Both dependency markers select the combined vLLM + Gym lane. pytest.importorskip("vllm") from vllm.model_executor.models.nano_nemotron_vl import NanoNemotronVLProcessingInfo @@ -2597,25 +2952,34 @@ def test_omni_capture_setup_rejects_video_pruning(monkeypatch, pruning_rate): ) worker = _fake_worker() worker.llm = SimpleNamespace( - renderer=SimpleNamespace(get_mm_processor=lambda: SimpleNamespace(info=info)), + renderer=SimpleNamespace( + get_mm_processor=lambda: SimpleNamespace(info=info), + get_tokenizer=lambda: SimpleNamespace(decode=lambda ids: ""), + ), model_config=SimpleNamespace(dtype=torch.bfloat16), ) monkeypatch.setattr( "nemo_rl.data_plane.build_data_plane_client", lambda dp_cfg, bootstrap, segment_size=None: MagicMock(), ) + kwargs = {"staging_partition": "staging", "capture_media": True} + if prefix_cuts: + kwargs.update( + generation_prefix_cuts_enabled=True, + generation_cut_control_token="secret", + ) + # Prefix cuts need the vLLM HTTP server's client; the progress reader it + # would wrap is covered by the progress-read tests. + worker._http_engine_client = MagicMock(name="http_engine_client") + worker._progress_reader = MagicMock(name="progress_reader") if pruning_rate: with pytest.raises(ValueError, match="video token pruning"): asyncio.run( - VllmAsyncGenerationWorkerImpl.setup_token_capture( - worker, {}, staging_partition="staging", capture_media=True - ) + VllmAsyncGenerationWorkerImpl.setup_token_capture(worker, {}, **kwargs) ) else: spec = asyncio.run( - VllmAsyncGenerationWorkerImpl.setup_token_capture( - worker, {}, staging_partition="staging", capture_media=True - ) + VllmAsyncGenerationWorkerImpl.setup_token_capture(worker, {}, **kwargs) ) # The engine's model dtype and the HF patch size, as pinned on the sink. assert spec == MediaColumnSpec(pixel_dtype=torch.bfloat16, patch_size=2) diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index b8ae3ed9c8c..dc72e23c47c 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -829,6 +829,28 @@ def test_recovery_target_enables_required_checkpoint_capabilities( assert config.generation_prefix_cuts_enabled is generation_prefix_cuts +@contextlib.contextmanager +def _patched_gym_token_capture(finalizer_actors): + """Patch the Gym and finalizer actors a Gym token-capture setup builds. + + Yields the Gym spin-up and finalizer-creation mocks. + """ + with ( + patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), + patch.object( + sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() + ) as spinup, + patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), + patch.object(sc_setup_mod, "uses_image_placeholder", return_value=True), + patch( + "nemo_rl.experience.rollout_reassembler_actor." + "create_rollout_reassembler_actors", + return_value=finalizer_actors, + ) as create_finalizers, + ): + yield spinup, create_finalizers + + class TestSetup: """setup arg validation + actor_args assembly.""" @@ -1274,7 +1296,6 @@ def test_gym_state_recovery_rejects_a_config_that_never_saves( ("feature", "match"), [ ("router_replay", "does not yet support router replay"), - ("multimodal", "does not yet support multimodal capture"), ( "megatron", "target_level='prefix' currently supports only the vllm generation", @@ -1295,14 +1316,11 @@ def test_prefix_recovery_rejects_unsupported_features(self, feature, match): mc.rollout_checkpointing = RolloutCheckpointConfig( snapshot_attempt_interval_s=1.0 ) - processor = None if feature == "router_replay": mc.policy["router_replay"] = {"enabled": True} - elif feature == "multimodal": - processor = MagicMock() with pytest.raises(NotImplementedError, match=match): - setup_single_controller(mc, MagicMock(pad_token_id=0), processor=processor) + setup_single_controller(mc, MagicMock(pad_token_id=0)) @pytest.mark.parametrize( ("overrides", "error", "match"), @@ -1490,6 +1508,47 @@ def record_gym_log_dir(env_configs, **_kwargs): assert gym_log_dirs_at_build == [str(tmp_path / "exp_001" / "nemo_gym")] + def test_prefix_recovery_accepts_multimodal_capture( + self, + tmp_path: Path, + patched_factories, + ): + mc = _make_master_config(colocated=False, backend="vllm", megatron_enabled=True) + mc.checkpointing.update( + { + "checkpoint_dir": str(tmp_path / "checkpoints"), + "enabled": True, + "save_data_plane": True, + "save_period": 1, + } + ) + mc.policy["generation"].update( + { + "model_name": "test-model", + "stop_strings": None, + "stop_token_ids": None, + "top_k": None, + "vllm_cfg": {"async_engine": True}, + } + ) + mc.token_capture.enabled = True + mc.rollout_recovery.target_level = RecoveryTargetLevel.PREFIX + mc.rollout_checkpointing = RolloutCheckpointConfig( + snapshot_attempt_interval_s=1.0 + ) + patched_factories["setup_response_data"].return_value = (list(range(8)), None) + tokenizer = MagicMock(pad_token_id=0) + + with _patched_gym_token_capture([MagicMock(name="finalizer")]): + setup_single_controller( + mc, tokenizer, processor=MagicMock(tokenizer=tokenizer) + ) + + generation, _ = patched_factories["_build_generation"].return_value + _, setup_kwargs = generation.setup_token_capture.call_args + assert setup_kwargs["capture_media"] is True + assert setup_kwargs["generation_prefix_cuts_enabled"] is True + def test_disabled_periodic_checkpointing_ignores_existing_snapshots( self, tmp_path: Path, @@ -2782,13 +2841,11 @@ def test_token_capture_always_creates_finalizer_actor_pool( processor = MagicMock(tokenizer=tokenizer) if with_processor else None with ( - patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), - patch.object( - sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() - ) as mock_spinup, + _patched_gym_token_capture(fake_actors) as ( + mock_spinup, + mock_create_finalizer_actors, + ), patch.object(sc_setup_mod, "validate_dataset_agent_coverage"), - patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), - patch.object(sc_setup_mod, "uses_image_placeholder", return_value=True), patch.object( sc_setup_mod.opd_module, "reserve_teacher_clusters", @@ -2799,10 +2856,6 @@ def test_token_capture_always_creates_finalizer_actor_pool( "create_teacher_worker_groups", return_value=({"teacher": MagicMock()}, {"teacher": "teacher"}), ), - patch( - "nemo_rl.experience.rollout_reassembler_actor.create_rollout_reassembler_actors", - return_value=fake_actors, - ) as mock_create_finalizer_actors, ): actor_args, _ = setup_single_controller(mc, tokenizer, processor=processor)