From 3747f80cf81ef77ff9fcab939cd1a6329bc046ab Mon Sep 17 00:00:00 2001 From: Anish Mahishi Date: Mon, 2 Nov 2026 17:29:01 -0800 Subject: [PATCH 1/2] feat(checkpoint): recover multimodal generation prefixes Lift the setup refusal of generation-prefix recovery with media capture. A media call's first durable prefix row carries its media spans and a SHA-256 digest of each packed media tensor in its extras, not the pixels; later chunks carry no media. The restored prefix digest covers those extras. On restore the worker reads the items and digests from that row, reprocesses the reissued request's media, checks it against them (retained_media_changed on a mismatch), and stages the reprocessed tensors in the canonical terminal row. A row whose media metadata cannot be read makes the call regenerate from its turn boundary. Adds an Omni CLEVR multimodal prefix-recovery functional test, unit coverage of media cuts, races and restores, and shares the prefix recovery functional phases (_gym_prefix_recovery_phases.sh). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Anish Mahishi --- docs/guides/single-controller.md | 3 +- .../single_controller_utils/setup.py | 4 - nemo_rl/data/captured_media.py | 77 +++- .../generation/prefix_recovery/parts.py | 13 +- .../generation/prefix_recovery/recovery.py | 69 ++- .../generation/prefix_recovery/state.py | 5 +- .../generation/vllm/vllm_worker_async.py | 169 +++++++- ...Tests_GB200_Vllm_Omni_Single_Controller.sh | 2 + .../functional/_gym_prefix_recovery_parity.py | 94 +++-- .../functional/_gym_prefix_recovery_phases.sh | 58 +++ .../_gym_prefix_recovery_snapshot.py | 70 +++- ...c_gym_single_controller_prefix_recovery.sh | 33 +- ..._prefix_recovery_single_controller_1n2g.sh | 350 ++++++++++++++++ tests/unit/data/test_captured_media.py | 326 +++++++++++++++ .../test_gym_prefix_recovery_parity.py | 86 ++++ .../models/generation/test_vllm_generation.py | 2 + .../test_vllm_token_capture_hosting.py | 395 +++++++++++++++++- tests/unit/single_controller/test_setup.py | 65 ++- 18 files changed, 1679 insertions(+), 142 deletions(-) create mode 100644 tests/functional/_gym_prefix_recovery_phases.sh create mode 100755 tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh diff --git a/docs/guides/single-controller.md b/docs/guides/single-controller.md index 7bcad18f126..779a6524480 100644 --- a/docs/guides/single-controller.md +++ b/docs/guides/single-controller.md @@ -264,7 +264,7 @@ If one shard replica fails restore, every replica conservatively retires its rep If no rollout snapshot follows the newest trainer checkpoint, SC restores that checkpoint on its own: completed groups come back from TQ and unfinished turn- and prefix-level rollouts restart from their tasks. A rollout snapshot without its `gym_checkpoint.json` is refused. -The `prefix` target needs everything the `turn` target needs (listed below), and does not yet support multimodal capture, router replay, or `policy.generation.top_logprobs > 0`. Each cut must finish within `token_capture.control_timeout_s`; a call that is not cut in time restarts from its last saved turn. Some calls are never cut and also restart from their last saved turn: +The `prefix` target needs everything the `turn` target needs (listed below), and does not yet support router replay or `policy.generation.top_logprobs > 0`. Each cut must finish within `token_capture.control_timeout_s`; a call that is not cut in time restarts from its last saved turn. Some calls are never cut and also restart from their last saved turn: - Calls using stop strings, multi-token `bad_words`, or a per-request `seed` while sampling, because a resumed call could not apply them to the tokens it already decoded. - Calls with structured output (a non-text `response_format`, `guided_*` or `structured_outputs`) or a `tool_choice` other than `auto` or `none`, which NeMo Gym leaves uncut. @@ -450,6 +450,7 @@ The SC path is still under active development. Feature gaps are tracked in [issu RL hands the owned tensors (`imgs`, `imgs_sizes`, and optional `num_frames`) to Gym's `complete_call_from_response` as opaque attachments, and the TQ sink writes them in the same `put` as the token columns, so `staged` coordinates acknowledge tokens and pixels together and a failed write is `capture_failed` at call time. Tensors keep their native shapes and dtypes on the wire; two per-row flags (`media_present`, `media_has_frames`) mark which rows carry pixels and whether they are video. Media-enabled staging partitions carry these columns on every row; text-only runs register and read none of them. vLLM pixels are rearranged losslessly into packed patches; the finalizer reads the presence flags with the base columns, then issues one batched read of the tensor columns for the terminal-chain calls that carry media, and publishes `pixel_values`, `imgs_sizes`, and `num_frames` for the existing Megatron learner without resampling or normalizing the media again. Only newly introduced occurrences are staged. vLLM-specific `media_spans` extras retain placeholder positions and token hashes for multi-turn prefix replacement, including video's timestamp-separated visual-token spans. + With `rollout_recovery.target_level: prefix`, the first durable prefix row for a model call carries that call's new media metadata and a SHA-256 digest of each packed tensor, not the pixels; later token chunks carry no media. The prefix digest covers that metadata, so a restore checks it with the prefix, then reprocesses the reissued request, checks its media against those tensor digests, and stages the reprocessed tensors in the canonical terminal row. A prefix that fails its digest check makes the call regenerate from its turn boundary, like any unusable prefix; reprocessed media that does not match the digests is rejected with `retained_media_changed`. The original image or video must therefore still be resolvable by vLLM when the request is reissued: this path does not provide source-free media prefill if the original media has disappeared. Capture requests use vLLM's `skip_mm_cache=True` path to obtain concrete processor tensors. vLLM can still reuse its processor-only cache; this does not guarantee fresh preprocessing. vLLM tiles images from a dummy prompt, so the request text never changes geometry, but the images of one request share the token budget: adding an image under a tight budget can re-tile a retained one, and a warm processor-only cache can keep a geometry a fresh processor would not reproduce. If that changes retained geometry or placeholder tokens, the worker rejects the continuation before inference with HTTP 400 and error code `retained_media_changed` (other capture-time validation failures use `media_capture_rejected`). Gym's current exception middleware wraps the upstream error in HTTP 500, retaining the code in the response body. RL then classifies it as an infrastructure `GymTransportError`, eligible for the configured prompt retry policy; it is not a dedicated terminal rejection. Text-call rows carry sentinels in each column's own dtype, because TransferQueue keeps one dtype per field across live rows. Media-enabled rows also store `media_metadata_digest`, a SHA-256 checksum of the small route-less extras JSON. Sources check it before exposing media descriptors, without reading pixels or routed-expert tensors. Missing or changed checksums reject the row. This detects accidental metadata corruption; it is independent of Gym's receipt and combined `extras_digest` commitment. diff --git a/nemo_rl/algorithms/single_controller_utils/setup.py b/nemo_rl/algorithms/single_controller_utils/setup.py index 70b1ac24288..ac8872ced81 100644 --- a/nemo_rl/algorithms/single_controller_utils/setup.py +++ b/nemo_rl/algorithms/single_controller_utils/setup.py @@ -1347,10 +1347,6 @@ def setup_single_controller( ) if generation_prefix_cuts_enabled: # The vLLM-only check ran above with the other turn/prefix checks. - if capture_media: - raise NotImplementedError( - "generation-prefix recovery does not yet support multimodal capture" - ) if router_replay_enabled(master_config.policy): raise NotImplementedError( "generation-prefix recovery does not yet support router replay" diff --git a/nemo_rl/data/captured_media.py b/nemo_rl/data/captured_media.py index c832e77beb0..7d25054ded3 100644 --- a/nemo_rl/data/captured_media.py +++ b/nemo_rl/data/captured_media.py @@ -22,8 +22,9 @@ import hashlib import json +from collections.abc import Mapping from dataclasses import asdict, dataclass, replace -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, NoReturn import torch @@ -138,10 +139,82 @@ def verify_tokens(self, tokens: list[int], *, origin: int) -> None: @dataclass(frozen=True) class CapturedMedia: - """New placeholder metadata and owned tensors for the common staging sink.""" + """New placeholder metadata and owned tensors for the common staging sink. + + ``tensor_digests`` fingerprints ``tensors`` (see ``media_tensor_digests``) + when generation-prefix cuts are on. A media bundle restored from a prefix + row carries the staged digests and no tensors. + """ items: tuple[CapturedMediaItem, ...] tensors: dict[str, torch.Tensor] | None + tensor_digests: dict[str, dict[str, Any]] | None = None + + +# Generation-prefix extras key for the call's tensor digests. Prefix rows store +# these instead of the pixels: a restore reprocesses the media anyway and only +# has to prove it produced the bytes the interrupted call staged. +MEDIA_TENSOR_DIGESTS_FIELD = "media_tensor_digests" + + +def media_tensor_digests( + tensors: Mapping[str, torch.Tensor] | None, +) -> dict[str, dict[str, Any]]: + """Fingerprint each media tensor by its dtype, shape, and raw bytes.""" + digests: dict[str, dict[str, Any]] = {} + for name, value in sorted((tensors or {}).items()): + tensor = value.detach().cpu().contiguous() + raw = tensor.reshape(-1).view(torch.uint8).numpy() + digests[name] = { + "dtype": str(tensor.dtype), + "shape": list(tensor.shape), + "sha256": hashlib.sha256(raw).hexdigest(), + } + return digests + + +def verify_restored_media(captured: CapturedMedia, restored: CapturedMedia) -> None: + """Require reprocessed media to match what an interrupted call staged. + + Args: + captured: The media just reprocessed from the resumed request. + restored: The items and tensor digests read from the call's first + generation-prefix row. + + Raises: + MediaCaptureRejected: With ``retained_media_changed`` on the first of + an item, tensor-name, dtype/shape, or byte mismatch. + """ + + def reject(what: str) -> NoReturn: + raise MediaCaptureRejected( + f"Restored generation media {what} differs from reprocessed input", + code="retained_media_changed", + ) + + if captured.items != restored.items: + reject("metadata") + staged = restored.tensor_digests + if staged is None: + raise MediaCaptureRejected("Restored generation media has no tensor digests") + current = ( + captured.tensor_digests + if captured.tensor_digests is not None + else media_tensor_digests(captured.tensors) + ) + if set(current) != set(staged): + reject("tensor set") + for name, digest in current.items(): + expected = staged[name] + if not isinstance(expected, dict): + reject(f"tensor {name!r} digest") + if (digest["dtype"], digest["shape"]) != ( + expected.get("dtype"), + expected.get("shape"), + ): + reject(f"tensor {name!r} dtype or shape") + if digest["sha256"] != expected.get("sha256"): + reject(f"tensor {name!r} bytes") def verified_media_items(call: FetchedStagedCall) -> tuple[CapturedMediaItem, ...]: diff --git a/nemo_rl/models/generation/prefix_recovery/parts.py b/nemo_rl/models/generation/prefix_recovery/parts.py index 6fb2455fd5b..c56f42f1796 100644 --- a/nemo_rl/models/generation/prefix_recovery/parts.py +++ b/nemo_rl/models/generation/prefix_recovery/parts.py @@ -16,12 +16,15 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any from nemo_rl.models.generation.prefix_recovery.progress import ProgressReader -from nemo_rl.models.generation.prefix_recovery.state import CaptureRegistry +from nemo_rl.models.generation.prefix_recovery.state import ( + CaptureRegistry, + _RequestCaptureState, +) if TYPE_CHECKING: from nemo_rl.data_plane.tq_token_sink import TQTokenSink, TQTokenSource @@ -49,3 +52,9 @@ class PrefixRecoveryParts: # Bound on one cut, measured from its arrival; None waits indefinitely. control_timeout_s: float | None progress_reader: ProgressReader + # Extras for a call's first prefix row, from the media its capture state + # holds; None for a call without media. + prefix_row_extras: Callable[[_RequestCaptureState], dict[str, Any] | None] + # A restored call's media, read back from its fetched prefix chunks; None + # when the worker does not capture media. + restore_media: Callable[[Sequence[Any]], Any | None] diff --git a/nemo_rl/models/generation/prefix_recovery/recovery.py b/nemo_rl/models/generation/prefix_recovery/recovery.py index a2caee4a862..49a6c52ae21 100644 --- a/nemo_rl/models/generation/prefix_recovery/recovery.py +++ b/nemo_rl/models/generation/prefix_recovery/recovery.py @@ -32,6 +32,7 @@ from collections import Counter from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from typing import Any, Literal from nemo_rl.models.generation.prefix_recovery.parts import PrefixRecoveryParts @@ -91,6 +92,14 @@ def _require_inventory_identity( ) +@dataclass(frozen=True) +class _ResolvedGenerationCut: + """A verified restored prefix and the media its call carried.""" + + snapshot: Any + media: Any | None = None + + def _ends_inside_character(tokenizer: Any, generated_token_ids: list[int]) -> bool: """Whether the generated text stops partway through a multi-byte character. @@ -519,6 +528,7 @@ def _cut_active( prompt_token_ids=state.prompt_token_ids, generated_token_ids=generated_token_ids, generated_logprobs=frozen.generated_logprobs, + extras=self._parts.prefix_row_extras(state), ) if frozen.chunk_token_ids: chunk_record = ( @@ -665,14 +675,41 @@ def checkpoint_cut(self, inventory: Any, *, deadline: float | None = None) -> An self._receipts.pop(next(iter(self._receipts))) return receipt - def _rebuild_prefix(self, admission: Any, prefix_token_ids: list[int]) -> Any: - """Fetch an admission's staged prefix chunks and rebuild the cumulative prefix.""" - continuation = admission.generation_cut + def _fetch_chunks(self, continuation: Any) -> list[Any]: + """Fetch the staged chunks named by a ``GenerationCutContinuation``, in order.""" # Gym requires at least one staging key per continuation. staging_keys = list(continuation.staging_keys) - snapshots = self._parts.staging_source.fetch(staging_keys) - if len(snapshots) != len(staging_keys): + # The finalization read also returns each row's extras, which carry a + # media call's spans and tensor digests; plain ``fetch`` does not. + chunks = self._parts.staging_source.fetch_for_finalization(staging_keys) + if len(chunks) != len(staging_keys): raise RuntimeError("generation-cut fetch did not return every staged chunk") + if not chunks: + raise RuntimeError("generation-cut continuation has no staged chunks") + return chunks + + def _rebuild_prefix( + self, + admission: Any, + prefix_token_ids: list[int], + chunks: list[Any] | None = None, + ) -> Any: + """Rebuild the cumulative prefix from an admission's staged chunks. + + ``chunks`` are the already-fetched chunks, so a restore that also reads + their media fetches once; they are fetched here when omitted. + """ + continuation = admission.generation_cut + if chunks is None: + chunks = self._fetch_chunks(continuation) + snapshots = [chunk.snapshot for chunk in chunks] + # A media call's spans and tensor digests ride the first chunk's extras, + # and the advertised prefix digest covers them. + if any(chunk.extras is not None for chunk in chunks[1:]): + raise RuntimeError( + "only the first generation-cut chunk may carry call extras" + ) + extras = chunks[0].extras weight_versions = [snapshot.weight_version for snapshot in snapshots] if weight_versions != sorted(weight_versions): raise RuntimeError( @@ -732,8 +769,8 @@ def _rebuild_prefix(self, admission: Any, prefix_token_ids: list[int]) -> Any: token_ids_delta=token_ids_delta, token_mask_delta=token_mask_delta, generation_log_probs_delta=generation_logprobs_delta, - extras=None, - extras_digest=compute_extras_digest(None), + extras=extras, + extras_digest=compute_extras_digest(extras), chain_hash=compute_chain_hash(admission.parent_chain_hash, token_ids_delta), cumulative_hash=hash_token_ids(prefix_token_ids + token_ids_delta), ) @@ -764,20 +801,24 @@ def _rebuild_prefix(self, admission: Any, prefix_token_ids: list[int]) -> Any: def resolve_restore( self, admission: Any, prefix_token_ids: list[int] - ) -> tuple[Any, Any | None]: + ) -> tuple[Any, _ResolvedGenerationCut | None]: """Resolve an admission's restored prefix, or drop it so the call regenerates. Gym has already consumed the restored cut when it sends this call, and generating the whole response from the turn boundary is always valid, - so a prefix that cannot be fetched or does not match its admission - costs time, not the rollout. Returns the admission to serve and the - verified prefix snapshot, or the admission without its cut and None. + so a prefix that cannot be fetched, whose media cannot be restored, + or that does not match its admission costs time, not the rollout. + Returns the admission to serve and the verified prefix with its media, + or the admission without its cut and None. """ continuation = admission.generation_cut if continuation is None: return admission, None try: - snapshot = self._rebuild_prefix(admission, prefix_token_ids) + # One fetch serves both the media and the prefix. + chunks = self._fetch_chunks(continuation) + media = self._parts.restore_media(chunks) + snapshot = self._rebuild_prefix(admission, prefix_token_ids, chunks) # Checked here, before the prefix is spliced into the prompt; Gym's # begin_call repeats it and would otherwise fail the call. if snapshot.digest != continuation.prefix_digest: @@ -788,7 +829,7 @@ def resolve_restore( f"rebuilt prefix has {generated} generated tokens, " f"admission expects {continuation.prefix_token_count}" ) - return admission, snapshot + return admission, _ResolvedGenerationCut(snapshot=snapshot, media=media) except Exception: # noqa: BLE001 - any unusable prefix regenerates LOGGER.warning( "cannot restore the generation prefix of model call %s; " @@ -848,7 +889,7 @@ class NoPrefixRecovery: def resolve_restore( self, admission: Any, prefix_token_ids: list[int] - ) -> tuple[Any, Any | None]: + ) -> tuple[Any, _ResolvedGenerationCut | None]: if admission.generation_cut is None: return admission, None # Gym only sends a restored cut to a worker that acknowledged one, so diff --git a/nemo_rl/models/generation/prefix_recovery/state.py b/nemo_rl/models/generation/prefix_recovery/state.py index 34589d8fbd1..f7b8181ef78 100644 --- a/nemo_rl/models/generation/prefix_recovery/state.py +++ b/nemo_rl/models/generation/prefix_recovery/state.py @@ -109,9 +109,8 @@ def __init__( self._call = call self._prompt_token_ids = list(prompt_token_ids) # Backend-owned opaque state that must survive until the terminal row - # is staged. vLLM uses this for its processed-media snapshot; prefix - # recovery is setup-guarded to text-only, but ordinary terminal capture - # still shares this lifecycle container. + # is staged. vLLM uses this for its processed-media snapshot; the first + # generation-prefix row carries its tensor digests across recovery. self._media = media self._resumed_generation_token_ids = list(resumed_generation_token_ids) self._lock = threading.Lock() diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 438bc0180f4..ed08aa540f9 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -29,7 +29,7 @@ import warnings from collections.abc import Awaitable, Callable, Sequence from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from types import SimpleNamespace from typing import TYPE_CHECKING, Any, AsyncGenerator, Literal, Optional, cast @@ -41,13 +41,16 @@ from fastapi.responses import JSONResponse from nemo_rl.data.captured_media import ( + MEDIA_TENSOR_DIGESTS_FIELD, CapturedMedia, CapturedMediaItem, MediaCaptureRejected, MediaColumnSpec, capture_processed_media, + media_tensor_digests, verified_media_items, verify_media_geometry, + verify_restored_media, ) from nemo_rl.data_plane.adapters.tq_mooncake_checkpoint import run_checkpoint_command from nemo_rl.data_plane.tq_token_sink import MediaMetadataIntegrityError @@ -138,6 +141,62 @@ ) +def _media_items_from_extras( + extras: dict[str, Any] | None, + snapshot: Any, + *, + missing_message: str, +) -> tuple[CapturedMediaItem, ...]: + """Parse a staged row's media spans and verify them against its own tokens. + + Raises: + MediaCaptureRejected: ``missing_message`` when the row has no spans, or + when the spans are malformed or do not match the row's tokens. + """ + # Optional Gym dependency: only captured media calls reach here. + from nemo_gym.token_id_capture.adapters.vllm import MEDIA_SPANS_FIELD + + if not isinstance(extras, dict) or MEDIA_SPANS_FIELD not in extras: + raise MediaCaptureRejected(missing_message) + raw_items = extras[MEDIA_SPANS_FIELD] + if not isinstance(raw_items, list): + raise MediaCaptureRejected("Media spans are malformed") + items = tuple(CapturedMediaItem.from_dict(value) for value in raw_items) + for item in items: + item.verify_tokens(snapshot.token_ids_delta, origin=snapshot.prev_len) + return items + + +def _verify_resumed_media_prompt( + reprocessed_token_ids: list[int], + *, + prefix_token_ids: list[int], + generation_cut: Any, +) -> None: + """Require a resumed media call to reprocess into its durable prompt. + + The call resumes from the exact engine prompt at the cut: its carried + prefix followed by the cut's prompt tokens. Its media placeholders are + re-derived from the reprocessed prompt, so a reprocessed prompt that + differs would place them over the wrong tokens. + + Raises: + MediaCaptureRejected: With ``retained_media_changed`` on a mismatch. + """ + prompt_delta = [ + token_id + for token_id, mask in zip( + generation_cut.token_ids_delta, generation_cut.token_mask_delta, strict=True + ) + if mask == 0.0 + ] + if reprocessed_token_ids != [*prefix_token_ids, *prompt_delta]: + raise MediaCaptureRejected( + "Reprocessed multimodal prompt differs from the durable generation prefix", + code="retained_media_changed", + ) + + # Paths the bearer-token check protects: the OpenAI-compatible routes this module # registers, which NeMo-Gym's model server calls. The sparse-refit routes the # receiver registers on the same app carry their own header token @@ -438,6 +497,7 @@ def _init_token_capture_state(self) -> None: # Built once per engine: it wraps vLLM's output processor. self._progress_reader: Any | None = None self._capture_media = False + self._hash_captured_media = False self._capture_patch_size: int | None = None self._capture_image_token_id: int | None = None # TQTokenSource installed by setup_token_capture. The media path reads @@ -893,6 +953,7 @@ async def setup_token_capture( self._capture_image_token_id = int(context_ids[0]) self._capture_patch_size = int(info.get_hf_config().patch_size) self._capture_media = capture_media + self._hash_captured_media = capture_media and generation_prefix_cuts_enabled self._capture_sink = sink if generation_prefix_cuts_enabled and self._http_engine_client is None: raise RuntimeError( @@ -927,6 +988,8 @@ async def setup_token_capture( control_token=generation_cut_control_token, control_timeout_s=generation_cut_control_timeout_s, progress_reader=self._progress_reader, + prefix_row_extras=self._generation_cut_record_payload, + restore_media=self._restore_generation_cut_media, ) ) if not capture_media: @@ -1053,6 +1116,7 @@ def _capture_request_media( admission: Any | None, splice: PrefixSplice | None = None, verify_retained_media: bool = False, + restored: CapturedMedia | None = None, ) -> CapturedMedia | None: """Run off-loop: resolve retained geometry and snapshot processed pixels.""" if admission is None: @@ -1067,8 +1131,6 @@ def _capture_request_media( retained_tensors = () if verify_retained_media else None if admission.parent_call_id is not None: # Optional Gym dependency: this method only runs on captured calls. - # Gym owns the media_spans key its adapter copies into the extras. - from nemo_gym.token_id_capture.adapters.vllm import MEDIA_SPANS_FIELD from nemo_gym.token_id_capture.staging.digest import compute_chain_hash from nemo_gym.token_id_capture.staging.records import staging_key @@ -1129,19 +1191,11 @@ def _capture_request_media( if verify_retained_media: items = verified_media_items(call) else: - extras = call.extras or {} - if MEDIA_SPANS_FIELD not in extras: - raise MediaCaptureRejected( - "Retained vLLM media spans are missing" - ) - items = tuple( - CapturedMediaItem.from_dict(value) - for value in extras[MEDIA_SPANS_FIELD] + items = _media_items_from_extras( + call.extras, + call.snapshot, + missing_message="Retained vLLM media spans are missing", ) - for item in items: - item.verify_tokens( - call.snapshot.token_ids_delta, origin=call.snapshot.prev_len - ) descriptors[call.snapshot.model_call_id] = items retained = tuple(item for items in descriptors.values() for item in items) if verify_retained_media: @@ -1153,7 +1207,7 @@ def _capture_request_media( verify_media_geometry( descriptors[call.snapshot.model_call_id], tensors ) - return capture_processed_media( + captured = capture_processed_media( engine_prompt, prev_len=admission.prev_len, retained=retained, @@ -1162,6 +1216,60 @@ def _capture_request_media( patch_size=self._capture_patch_size, retained_tensors=retained_tensors, ) + if self._hash_captured_media: + # Hash once per call here, off the event loop, so a prefix cut can + # write the digests without hashing pixels under its deadline. + captured = replace( + captured, tensor_digests=media_tensor_digests(captured.tensors) + ) + if restored is not None: + verify_restored_media(captured, restored) + return captured + + def _generation_cut_record_payload( + self, state: _RequestCaptureState + ) -> dict[str, Any] | None: + """Return the media extras for a call's first generation-prefix row. + + The row stores the call's tensor digests instead of its pixels: a + restore reprocesses the media and only checks it against them, and the + terminal row still stages the pixels for training. + """ + media = state.media + if media is None: + return None + if media.tensor_digests is None: + raise RuntimeError( + "media was captured without tensor digests; generation-prefix " + "cuts must be enabled before the call is admitted" + ) + from nemo_gym.token_id_capture.adapters.vllm import MEDIA_SPANS_FIELD + + return { + MEDIA_SPANS_FIELD: [item.to_dict() for item in media.items], + MEDIA_TENSOR_DIGESTS_FIELD: media.tensor_digests, + } + + def _restore_generation_cut_media( + self, chunks: Sequence[Any] + ) -> CapturedMedia | None: + """Read one call's media items and tensor digests from its first prefix row.""" + if not self._capture_media: + return None + first = chunks[0] + items = _media_items_from_extras( + first.extras, + first.snapshot, + missing_message="Generation-prefix media metadata is missing", + ) + digests = first.extras.get(MEDIA_TENSOR_DIGESTS_FIELD) + if not isinstance(digests, dict): + raise MediaCaptureRejected("Generation-prefix media digests are missing") + if bool(items) != bool(digests): + raise MediaCaptureRejected( + "Generation-prefix media metadata and digests disagree" + ) + return CapturedMedia(items=items, tensors=None, tensor_digests=digests) def _fetch_chain_prefix(self, staging_chain: list[str]) -> list[int]: """Resolve a staging chain through the shared, cached TQ read.""" @@ -1651,13 +1759,14 @@ async def preprocess_chat( verify_retained_media = decision.verify_retained_media capture_prefix_token_ids: list[int] | None = None generation_cut = None + restored_generation_media = None resumed_generation_token_ids: list[int] = [] restored_request_output_tokens: int | None = None if serving_admission is not None: capture_prefix_token_ids = await asyncio.to_thread( worker_self._resolve_admission_prefix, serving_admission ) - resolved_admission, generation_cut = await asyncio.to_thread( + resolved_admission, resolved_cut = await asyncio.to_thread( worker_self._prefix_recovery.resolve_restore, serving_admission, capture_prefix_token_ids, @@ -1667,6 +1776,9 @@ async def preprocess_chat( if serving_admission is admission: admission = resolved_admission serving_admission = resolved_admission + if resolved_cut is not None: + generation_cut = resolved_cut.snapshot + restored_generation_media = resolved_cut.media engine_prefix_token_ids = list(capture_prefix_token_ids) if generation_cut is not None: plan = plan_restore( @@ -1822,8 +1934,6 @@ async def preprocess_chat( final_prompt_token_ids, ) - # Generation-prefix cuts are text-only (setup refuses them with - # media capture), so a restored prompt has no media to capture. media = None if generation_cut is None: media = await asyncio.to_thread( @@ -1833,6 +1943,27 @@ async def preprocess_chat( splice=splice, verify_retained_media=verify_retained_media, ) + elif worker_self._capture_media: + # A restored prompt is the durable prefix itself, so splice + # the template only to locate its media placeholders. + media_splice = splice_prefix_tokens( + tokenizer=self.renderer.tokenizer, + model_prefix_token_ids=list(capture_prefix_token_ids or ()), + template_prefix_token_ids=actual_corresponding_token_ids, + template_token_ids=engine_prompt["prompt_token_ids"], + ) + _verify_resumed_media_prompt( + media_splice.token_ids, + prefix_token_ids=list(capture_prefix_token_ids or ()), + generation_cut=generation_cut, + ) + media = await asyncio.to_thread( + worker_self._capture_request_media, + engine_prompt, + admission=admission, + splice=media_splice, + restored=restored_generation_media, + ) # Must precede the reassignment below: the remap reads the # current value of prompt_token_ids as the template-coordinate # sequence the placeholders were computed against. Reordering diff --git a/tests/functional/L1_Functional_Tests_GB200_Vllm_Omni_Single_Controller.sh b/tests/functional/L1_Functional_Tests_GB200_Vllm_Omni_Single_Controller.sh index bb0005c6def..69e07b03588 100755 --- a/tests/functional/L1_Functional_Tests_GB200_Vllm_Omni_Single_Controller.sh +++ b/tests/functional/L1_Functional_Tests_GB200_Vllm_Omni_Single_Controller.sh @@ -50,6 +50,8 @@ export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1}" run_test fast uv run --no-sync bash ./tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh run_test fast uv run --no-sync bash ./tests/functional/nemotron_omni_gym_video_vllm_token_capture_single_controller_1n2g.sh +# Kills and restores a run mid-decode on image prompts; full mode only. +run_test uv run --no-sync bash ./tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh cd "${PROJECT_ROOT}/tests" if compgen -G ".coverage*" > /dev/null; then diff --git a/tests/functional/_gym_prefix_recovery_parity.py b/tests/functional/_gym_prefix_recovery_parity.py index 1ea1fce089f..58b91562485 100644 --- a/tests/functional/_gym_prefix_recovery_parity.py +++ b/tests/functional/_gym_prefix_recovery_parity.py @@ -284,51 +284,49 @@ def _compare_training_rows( ) -def _check_cuts( +def _restored_prefix_hashes(recovery_log: str) -> dict[tuple[str, int], list[str]]: + restored: dict[tuple[str, int], list[str]] = {} + for line in restored_prefixes(recovery_log): + key = (line["source_model_call_id"], line["prefix_tokens"]) + restored.setdefault(key, []).append(line["prefix_ids_sha256"]) + return restored + + +def _calls_by_generation( + recovery: RowIndex, +) -> dict[tuple[str, int], tuple[int, int, int]]: + return { + (row["group_id"], int(row["generation_index"])): identity + for identity, row in recovery.items() + } + + +def _assert_prefix_boundary( selection: Mapping[str, Any], recovery_log: str, - baseline: RowIndex, recovery: RowIndex, -) -> tuple[dict[tuple[str, int], int], frozenset[tuple[int, int, int]]]: - """Check every restored call's trained row, and find rows to set aside. - - First, each restored call must complete once and train on its restored - prefix. A restored call closes its row, so the ``total_generation_tokens`` - it logs on completion end the row: counting back finds where the call - starts. That holds even when the trainer masked positions inside the call - (top-k/top-p ``-inf`` logprobs drop out of ``token_mask``), which a search - for the last unmasked segment would mistake for the call's start. A dropped - or duplicated token moves that start, so the restored prefix no longer sits +) -> dict[tuple[str, int], int]: + """Require each restored call to complete once and train on its restored prefix. + + A restored call closes its row, so the ``total_generation_tokens`` it logs + on completion end the row: counting back finds where the call starts. That + holds even when the trainer masked positions inside the call (top-k/top-p + ``-inf`` logprobs drop out of ``token_mask``), which a search for the last + unmasked segment would mistake for the call's start. A dropped or + duplicated token moves that start, so the restored prefix no longer sits there. - Then, vLLM sampling is not batch-invariant, so phase 1 can take the other - side of a near-tie before the cut. Recovery must then continue phase 1's - tokens, and the reference no longer predicts that row. A row is set aside - only when the reference's call opens differently at the restored prefix's - position, and everything before the closing call must still match. At - most one restored row may be set aside, and at least one must match the - reference outright: nothing checks that a set-aside row diverged rather - than restored the wrong tokens, so the cap bounds what a run can leave - unchecked. - Returns: Each cut's ``(group_id, generation_index)`` mapped to the index in its - trained row where the restored call's generated tokens begin, and the - identities of the restored rows set aside. + trained row where the restored call's generated tokens begin. """ completed: dict[tuple[str, int], list[dict[str, str]]] = {} for match in _COMPLETED_PREFIX_LINE.finditer(recovery_log): line = match.groupdict() key = (line["source_model_call_id"], int(line["prefix_tokens"])) completed.setdefault(key, []).append(line) - restored: dict[tuple[str, int], list[str]] = {} - for line in restored_prefixes(recovery_log): - key = (line["source_model_call_id"], line["prefix_tokens"]) - restored.setdefault(key, []).append(line["prefix_ids_sha256"]) - by_call = { - (row["group_id"], int(row["generation_index"])): identity - for identity, row in recovery.items() - } + restored = _restored_prefix_hashes(recovery_log) + by_call = _calls_by_generation(recovery) starts: dict[tuple[str, int], int] = {} for cut in selection["cuts"]: @@ -373,6 +371,38 @@ def _check_cuts( f"{prefix_tokens} restored prefix tokens" ) starts[key] = start + return starts + + +def _check_cuts( + selection: Mapping[str, Any], + recovery_log: str, + baseline: RowIndex, + recovery: RowIndex, +) -> tuple[dict[tuple[str, int], int], frozenset[tuple[int, int, int]]]: + """Check every restored call's trained row, and find rows to set aside. + + First, each restored call must complete once and train on its restored + prefix (``_assert_prefix_boundary``). + + Then, vLLM sampling is not batch-invariant, so phase 1 can take the other + side of a near-tie before the cut. Recovery must then continue phase 1's + tokens, and the reference no longer predicts that row. A row is set aside + only when the reference's call opens differently at the restored prefix's + position, and everything before the closing call must still match. At + most one restored row may be set aside, and at least one must match the + reference outright: nothing checks that a set-aside row diverged rather + than restored the wrong tokens, so the cap bounds what a run can leave + unchecked. + + Returns: + Each cut's ``(group_id, generation_index)`` mapped to the index in its + trained row where the restored call's generated tokens begin, and the + identities of the restored rows set aside. + """ + starts = _assert_prefix_boundary(selection, recovery_log, recovery) + restored = _restored_prefix_hashes(recovery_log) + by_call = _calls_by_generation(recovery) diverged: set[tuple[int, int, int]] = set() for cut in selection["cuts"]: diff --git a/tests/functional/_gym_prefix_recovery_phases.sh b/tests/functional/_gym_prefix_recovery_phases.sh new file mode 100644 index 00000000000..ed549378191 --- /dev/null +++ b/tests/functional/_gym_prefix_recovery_phases.sh @@ -0,0 +1,58 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Phase control shared by the prefix-recovery functional tests. Source it after +# setting TEST_DIR; a test starts phase one in its own process group and keeps +# its PID in PHASE1_PID. + +PHASE1_PID="" + +# Hard-kill phase one and wait until its controller can no longer write +# snapshots. +stop_phase1() { + if [[ -z "${PHASE1_PID}" ]]; then + return + fi + kill -KILL -- "-${PHASE1_PID}" 2>/dev/null || true + wait "${PHASE1_PID}" 2>/dev/null || true + PHASE1_PID="" + # Ray actors outlive the driver until their raylet notices the crash. Do + # not start phase two while the old controller can still write snapshots. + local deadline=$((SECONDS + 120)) + while pgrep -f "^ray::SingleControllerActor" >/dev/null; do + if ((SECONDS >= deadline)); then + echo "[ERROR] phase-one SingleControllerActor outlived its driver" + return 1 + fi + sleep 1 + done +} + +# On exit, stop phase one and delete the given checkpoint artifacts if the test +# passed, unless SC_PREFIX_RECOVERY_KEEP_CHECKPOINTS=1. +install_phase_cleanup() { + PHASE_CLEANUP_PATHS=("$@") + trap _phase_cleanup EXIT +} + +_phase_cleanup() { + local status=$? + stop_phase1 + if [[ "${status}" -eq 0 && "${SC_PREFIX_RECOVERY_KEEP_CHECKPOINTS:-0}" != "1" ]]; then + rm -rf "${PHASE_CLEANUP_PATHS[@]}" + else + echo "Preserving prefix-recovery artifacts for inspection: ${TEST_DIR}" + fi + return "${status}" +} diff --git a/tests/functional/_gym_prefix_recovery_snapshot.py b/tests/functional/_gym_prefix_recovery_snapshot.py index 05dd658bceb..34533fad396 100644 --- a/tests/functional/_gym_prefix_recovery_snapshot.py +++ b/tests/functional/_gym_prefix_recovery_snapshot.py @@ -25,6 +25,12 @@ from pathlib import Path from typing import Any +from _gym_prefix_recovery_parity import ( + _assert_prefix_boundary, + _assert_retired_prefix_keys, + _rows_by_identity, + load_training_records, +) from _gym_recovery_snapshot_helpers import ( calendar_sentinel_count, ledger_episode_index, @@ -122,14 +128,16 @@ def _require_every_active_call_cut(snapshot: Path, phase_log: str) -> None: def inspect_snapshot( snapshot: Path, - sentinel_event: str, + sentinel_event: str | None, *, min_cut_groups: int = 1, ) -> dict[str, Any]: """Return the post-mutation nonterminal prefixes in ``snapshot``. ``cuts`` lists every one, and the snapshot qualifies only if they span at - least ``min_cut_groups`` prompt groups. + least ``min_cut_groups`` prompt groups. A ``None`` ``sentinel_event`` + selects a stateless resources server (e.g. CLEVR's string_match), whose + episodes export no resources state to check. """ required = [ @@ -164,11 +172,15 @@ def inspect_snapshot( key = _episode_key(record) recovery = ledger.get(key) cut = _recoverable_cut(record) - resource_entry = resource_index.get(key) - if recovery is None or cut is None or resource_entry is None: + if recovery is None or cut is None: continue - _, resources_state = resource_entry - sentinel_count = calendar_sentinel_count(resources_state, sentinel_event) + if sentinel_event is not None: + resource_entry = resource_index.get(key) + if resource_entry is None: + continue + _, resources_state = resource_entry + if calendar_sentinel_count(resources_state, sentinel_event) != 1: + continue instance_id = recovery["gym_instance_id"] checkpointed_episodes = { (episode["rollout_id"], episode["attempt"]) @@ -178,7 +190,6 @@ def inspect_snapshot( if ( key not in checkpointed_episodes or key not in agent_keys - or sentinel_count != 1 or recovery["restore_level"] != "prefix" or recovery["status"] != "dispatched" or not set(cut["staging_keys"]).issubset(retained_keys) @@ -222,7 +233,7 @@ def select_snapshot(args: argparse.Namespace) -> None: try: selection = inspect_snapshot( snapshot, - args.sentinel_event, + None if args.stateless_resources else args.sentinel_event, min_cut_groups=args.min_cut_groups, ) _require_every_active_call_cut( @@ -346,6 +357,27 @@ def verify_restore(args: argparse.Namespace) -> None: raise AssertionError(f"train step {step} appeared {len(matches)} times") +def verify_completion(args: argparse.Namespace) -> None: + """Without a reference run, require every restored prefix to finish and train. + + Each restored call must complete once, close its trained row, and start + with the exact prefix the worker restored (``_assert_prefix_boundary``); no + row may be trained twice, and the restored prefix rows must outlive + training only until the next snapshot. + """ + selection = read_json(Path(args.selection_file)) + records = load_training_records( + Path(args.recovery_log_dir), staging_keys=Path(args.recovery_staging_keys) + ) + starts = _assert_prefix_boundary( + selection, Path(args.phase2_log).read_text(), _rows_by_identity(records) + ) + _assert_retired_prefix_keys( + selection, records, read_jsonl(Path(args.recovery_flush_keys)) + ) + print(f"restored prefixes completed and trained: {sorted(starts)!r}", flush=True) + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() commands = parser.add_subparsers(dest="command", required=True) @@ -357,7 +389,10 @@ def _parser() -> argparse.ArgumentParser: select.add_argument("phase_log") select.add_argument("timeout_s", type=float) select.add_argument("backup_path") - select.add_argument("sentinel_event") + select.add_argument("sentinel_event", nargs="?") + # Explicit opt-out, so a Workplace caller that drops the sentinel fails + # instead of silently skipping the post-mutation resources check. + select.add_argument("--stateless-resources", action="store_true") select.add_argument("--min-cut-groups", type=int, default=1) select.set_defaults(func=select_snapshot) @@ -369,9 +404,24 @@ def _parser() -> argparse.ArgumentParser: verify.add_argument("training_info") verify.add_argument("max_steps", type=int) verify.set_defaults(func=verify_restore) + + completion = commands.add_parser("verify-completion") + completion.add_argument("selection_file") + completion.add_argument("recovery_log_dir") + completion.add_argument("recovery_staging_keys") + completion.add_argument("recovery_flush_keys") + completion.add_argument("phase2_log") + completion.set_defaults(func=verify_completion) return parser if __name__ == "__main__": - arguments = _parser().parse_args() + parser = _parser() + arguments = parser.parse_args() + if arguments.command == "select" and ( + (arguments.sentinel_event is None) != arguments.stateless_resources + ): + parser.error( + "select needs exactly one of sentinel_event or --stateless-resources" + ) arguments.func(arguments) diff --git a/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh b/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh index 474eae65b5f..6f6dc77b6a2 100755 --- a/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh +++ b/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh @@ -33,7 +33,6 @@ SELECTION_FILE=$TEST_DIR/selected-snapshot.json SELECTED_BACKUP=$TEST_DIR/selected-snapshot TEST_DATA=$TEST_DIR/test-data.jsonl GYM_ROOT=$PROJECT_ROOT/3rdparty/Gym-workspace/Gym -PHASE1_PID="" SENTINEL_EVENT="NeMo RL prefix recovery parity sentinel" # Cuttable Workplace prompts per step, one per list length. @@ -124,36 +123,8 @@ jq -n -c \ ' > "$TEST_DATA" -stop_phase1() { - if [[ -z "$PHASE1_PID" ]]; then - return - fi - kill -KILL -- "-$PHASE1_PID" 2>/dev/null || true - wait "$PHASE1_PID" 2>/dev/null || true - PHASE1_PID="" - # Ray actors outlive the driver until their raylet notices the crash. Do - # not start phase two while the old controller can still write snapshots. - local deadline=$((SECONDS + 120)) - while pgrep -f "^ray::SingleControllerActor" >/dev/null; do - if ((SECONDS >= deadline)); then - echo "[ERROR] phase-one SingleControllerActor outlived its driver" - return 1 - fi - sleep 1 - done -} - -cleanup() { - local status=$? - stop_phase1 - if [[ "$status" -eq 0 && "${SC_PREFIX_RECOVERY_KEEP_CHECKPOINTS:-0}" != "1" ]]; then - rm -rf "$CHECKPOINT_DIR" "$BASELINE_CHECKPOINT_DIR" "$SELECTED_BACKUP" - else - echo "Preserving prefix-recovery artifacts for inspection: $TEST_DIR" - fi - return "$status" -} -trap cleanup EXIT +source "$SCRIPT_DIR/_gym_prefix_recovery_phases.sh" +install_phase_cleanup "$CHECKPOINT_DIR" "$BASELINE_CHECKPOINT_DIR" "$SELECTED_BACKUP" COMMON_OVERRIDES=( policy.model_name="$MODEL_NAME" diff --git a/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh new file mode 100755 index 00000000000..beb77799c85 --- /dev/null +++ b/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Multimodal smoke test for generation-prefix recovery on the Omni CLEVR vLLM +# token-capture setup. A Gym-v2 checkpoint is taken while vLLM is mid-decode on +# image prompts; the process is hard-killed and restored. Each restored call +# must reprocess its image, verify it against the durable TQ media copy, resume +# from the exact durable token prefix, and finish and train. +# +# Token-for-token parity against an uninterrupted run is the Workplace text +# test's job (grpo_async_gym_single_controller_prefix_recovery.sh). Here the +# Reasoning model's free-form thinking flips near-tied tokens between runs, so +# parity would be flaky; this test instead checks each restored prefix appears +# verbatim in its trained row. Other differences from the Workplace test: +# - Single-turn string_match rows: the cut lands in the only model call, so the +# first durable prefix row is the one that owns the image tensors. +# - string_match is stateless, so snapshot selection runs with +# --stateless-resources instead of a calendar-sentinel check. +# - Each prompt group gets a distinct image, so a restore that verified against +# the wrong group's pixels would fail (retained_media_changed) instead of +# passing on identical fixtures. + +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +PROJECT_ROOT=$(realpath "${SCRIPT_DIR}/../..") + +if [[ -z "${HF_TOKEN:-}" ]]; then + echo "SKIP: HF_TOKEN is required for the Omni checkpoint" + exit 0 +fi + +GPU_COUNT=$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l) +if (( GPU_COUNT < 2 )); then + echo "SKIP: Omni CLEVR vLLM prefix recovery requires at least two visible GPUs" + exit 0 +fi +DETECTED_CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader -i 0) +export TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST:-${DETECTED_CUDA_ARCH}}" + +EXP_NAME=$(basename "$0" .sh) +TEST_DIR="${SCRIPT_DIR}/${EXP_NAME}" +OMNI_CONFIG="${PROJECT_ROOT}/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml" +RECOVERY_HOOK="${SCRIPT_DIR}/_single_controller_turn_recovery_hook.py" +SNAPSHOT_HELPER="${SCRIPT_DIR}/_gym_prefix_recovery_snapshot.py" +CHECKPOINT_DIR="${TEST_DIR}/recovery-checkpoints" +PHASE1_LOG="${TEST_DIR}/phase1.log" +PHASE2_LOG="${TEST_DIR}/phase2.log" +PHASE1_EVENTS="${TEST_DIR}/phase1-events.jsonl" +PHASE2_EVENTS="${TEST_DIR}/phase2-events.jsonl" +# Each phase writes into its own logger.log_dir: phase two's trained rows are +# read back from the only experiment directory under RECOVERY_LOG_DIR. +PHASE1_LOG_DIR="${TEST_DIR}/phase1-logs" +RECOVERY_LOG_DIR="${TEST_DIR}/recovery-logs" +RECOVERY_STAGING_KEYS="${TEST_DIR}/recovery-staging-keys.jsonl" +RECOVERY_FLUSH_KEYS="${TEST_DIR}/recovery-flush-keys.jsonl" +PHASE2_METRICS="${TEST_DIR}/phase2-metrics.json" +SELECTION_FILE="${TEST_DIR}/selected-snapshot.json" +SELECTED_BACKUP="${TEST_DIR}/selected-snapshot" +TEST_DATA="${TEST_DIR}/test-data.jsonl" + +# One prompt group per list length, each with its own solid-color image. The +# list keeps decode long enough to be cut mid-generation; distinct images mean +# a restore verified against another group's pixels would be rejected. +LIST_LENGTHS=(${SC_MM_PREFIX_RECOVERY_LIST_LENGTHS:-120 240}) +NUM_PROMPTS=${#LIST_LENGTHS[@]} +NUM_GENERATIONS=${SC_MM_PREFIX_RECOVERY_NUM_GENERATIONS:-2} +# Conservative per-item budget: the Omni tokenizer's digit grouping is not +# assumed, so "N\n" is budgeted like Qwen's one-token-per-digit. +LIST_TOKENS_PER_ITEM=6 +LIST_OUTPUT_SLACK_TOKENS=256 +MAX_MODEL_LEN=${SC_MM_PREFIX_RECOVERY_MAX_MODEL_LEN:-4096} +MAX_STEPS=${SC_MM_PREFIX_RECOVERY_MAX_STEPS:-2} +# Phase two trains one step past phase one, so the snapshot after step +# MAX_STEPS deletes the restored calls' retired prefix rows while the run is +# still alive to record it. +PHASE2_STEPS=$((MAX_STEPS + 1)) +SNAPSHOT_INTERVAL_S=${SC_MM_PREFIX_RECOVERY_INTERVAL_S:-0.05} +PHASE2_SNAPSHOT_INTERVAL_S=${SC_MM_PREFIX_RECOVERY_PHASE2_INTERVAL_S:-600} +SNAPSHOT_TIMEOUT_S=${SC_MM_PREFIX_RECOVERY_TIMEOUT_S:-3600} +RUN_TIMEOUT_S=${SC_MM_PREFIX_RECOVERY_RUN_TIMEOUT_S:-3600} +TRAIN_GLOBAL_BATCH_SIZE=$((NUM_PROMPTS * NUM_GENERATIONS)) +POLICY_MAX_NEW_TOKENS=0 +for list_length in "${LIST_LENGTHS[@]}"; do + list_budget=$((list_length * LIST_TOKENS_PER_ITEM + LIST_OUTPUT_SLACK_TOKENS)) + if (( list_budget > POLICY_MAX_NEW_TOKENS )); then + POLICY_MAX_NEW_TOKENS=$list_budget + fi +done + +rm -rf "${TEST_DIR}" +mkdir -p "${TEST_DIR}" +cd "${PROJECT_ROOT}" +export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}" + +# The test config trains one epoch, so the data holds a step of prompts for +# every phase-two step; phase one stops after MAX_STEPS of them. +TEST_DATA="${TEST_DATA}" DATA_STEPS="${PHASE2_STEPS}" \ +LIST_LENGTHS="${LIST_LENGTHS[*]}" LIST_TOKENS_PER_ITEM="${LIST_TOKENS_PER_ITEM}" \ +LIST_OUTPUT_SLACK_TOKENS="${LIST_OUTPUT_SLACK_TOKENS}" \ +uv run --no-sync python - <<'PY' +import base64 +import io +import json +import os + +from PIL import Image + +COLORS = ["red", "green", "blue", "yellow"] +list_lengths = [int(value) for value in os.environ["LIST_LENGTHS"].split()] +if not 1 <= len(list_lengths) <= len(COLORS): + raise SystemExit(f"need 1 to {len(COLORS)} list lengths, got {len(list_lengths)}") +per_item = int(os.environ["LIST_TOKENS_PER_ITEM"]) +slack = int(os.environ["LIST_OUTPUT_SLACK_TOKENS"]) + + +def image_url(color: str) -> str: + buffer = io.BytesIO() + Image.new("RGB", (224, 224), color=color).save(buffer, format="PNG") + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + + +with open(os.environ["TEST_DATA"], "w") as output: + for step in range(int(os.environ["DATA_STEPS"])): + for slot, list_length in enumerate(list_lengths): + color = COLORS[slot] + text = ( + f"Step {step} item {slot}: list every integer from 1 to {list_length} " + "in increasing order, one per line, with no other text. Then give " + "the color of the image as one word inside \\boxed{}." + ) + row = { + "responses_create_params": { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": image_url(color), + "detail": "auto", + }, + {"type": "input_text", "text": text}, + ], + } + ], + "max_output_tokens": list_length * per_item + slack, + }, + "expected_answer": color, + "extraction_mode": "boxed", + "case_sensitive": False, + "agent_ref": { + "type": "responses_api_agents", + "name": "string_match_simple_agent", + }, + } + output.write(json.dumps(row) + "\n") +PY + +source "${SCRIPT_DIR}/_gym_prefix_recovery_phases.sh" +install_phase_cleanup "${CHECKPOINT_DIR}" "${SELECTED_BACKUP}" + +# The Omni CLEVR token-capture topology (one Megatron training GPU with a frozen +# language model, one async vLLM GPU) plus the text prefix-recovery settings. +COMMON_OVERRIDES=( + cluster.num_nodes=1 + cluster.gpus_per_node=2 + ++cluster.segment_size=1 + # Quoted for Hydra: a bare compute capability (9.0, 10.0) parses as a + # float, which the string-typed env_vars field rejects. + policy.megatron_cfg.env_vars.TORCH_CUDA_ARCH_LIST="'${TORCH_CUDA_ARCH_LIST}'" + policy.megatron_cfg.tensor_model_parallel_size=1 + policy.megatron_cfg.expert_model_parallel_size=1 + policy.megatron_cfg.expert_tensor_parallel_size=1 + policy.megatron_cfg.context_parallel_size=1 + policy.megatron_cfg.sequence_parallel=true + policy.megatron_cfg.activation_checkpointing=true + ++policy.megatron_cfg.freeze_config.freeze_language_model=true + +policy.megatron_cfg.bias_dropout_fusion=false + policy.megatron_cfg.optimizer.optimizer_cpu_offload=false + policy.megatron_cfg.optimizer.optimizer_offload_fraction=0.0 + ++policy.megatron_cfg.optimizer.params_dtype=bfloat16 + ++policy.megatron_cfg.optimizer.main_grads_dtype=bfloat16 + ++policy.megatron_cfg.optimizer.main_params_dtype=float16 + ++policy.megatron_cfg.optimizer.exp_avg_dtype=bfloat16 + ++policy.megatron_cfg.optimizer.exp_avg_sq_dtype=bfloat16 + ++policy.megatron_cfg.optimizer.store_param_remainders=false + policy.generation.colocated.enabled=false + policy.generation.colocated.resources.num_nodes=1 + policy.generation.colocated.resources.gpus_per_node=1 + policy.generation.max_new_tokens="${POLICY_MAX_NEW_TOKENS}" + policy.generation.vllm_cfg.tensor_parallel_size=1 + policy.generation.vllm_cfg.expert_parallel_size=1 + policy.generation.vllm_cfg.pipeline_parallel_size=1 + policy.generation.vllm_cfg.max_model_len="${MAX_MODEL_LEN}" + policy.generation.vllm_cfg.gpu_memory_utilization=0.6 + ++policy.generation.vllm_kwargs.max_num_batched_tokens="${MAX_MODEL_LEN}" + # Near-greedy decoding keeps each list (and so each decode) predictable. + policy.generation.temperature=1.0 + policy.generation.top_p=0.000001 + # The trainer replays the top_p filter, so it needs vLLM's logprobs after + # that filter. + ++policy.generation.vllm_cfg.logprobs_mode=processed_logprobs + ++async_rl.generation_fleet_health.refit_timeout_s=null + policy.max_total_sequence_length="${MAX_MODEL_LEN}" + policy.train_global_batch_size="${TRAIN_GLOBAL_BATCH_SIZE}" + policy.train_micro_batch_size=1 + data_plane.simple.num_storage_units=2 + # Keep each step's prompt membership fixed across runs. + data.shuffle=false + data.train.data_path="${TEST_DATA}" + data.validation.data_path="${TEST_DATA}" + grpo.seed=1234 + grpo.num_prompts_per_step="${NUM_PROMPTS}" + grpo.num_generations_per_prompt="${NUM_GENERATIONS}" + grpo.max_num_steps="${MAX_STEPS}" + grpo.val_period=0 + grpo.val_at_start=false + grpo.val_at_end=false + async_rl.sampler.name=in_order + async_rl.sampler.max_lookahead_versions=0 + async_rl.min_groups_for_streaming_train="${NUM_PROMPTS}" + async_rl.max_inflight_prompts="${NUM_PROMPTS}" + async_rl.max_buffered_rollouts="${NUM_PROMPTS}" + ++async_rl.rollout_failure.nemo_gym.rollout_timeout_s=600 + ++async_rl.stall_watchdog.interval_s=10 + ++async_rl.stall_watchdog.stall_timeout_s=900 + ++async_rl.stall_watchdog.stall_action=abort + ++token_capture.enabled=true + ++rollout_recovery.target_level=prefix + ++rollout_checkpointing.snapshot_attempt_interval_s="${SNAPSHOT_INTERVAL_S}" + ++rollout_checkpointing.keep_latest_k=16 + ++rollout_checkpointing.restore_mode=latest + # Phase two trains one step longer than phase one (PHASE2_STEPS); the step + # count only decides when the run stops, so restore must accept it. + '++rollout_checkpointing.extra_fingerprint_excluded_paths=[grpo.max_num_steps]' + checkpointing.enabled=true + checkpointing.checkpoint_dir="${CHECKPOINT_DIR}" + checkpointing.metric_name=null + # Phase two saves after step MAX_STEPS, whose snapshot deletes the retired + # prefix rows, and after its last step, whose training_info is read; every + # Omni save writes the full 30B model. + checkpointing.save_period="${MAX_STEPS}" + ++checkpointing.save_data_plane=true + # Each run dumps its trained rows (train_data_step*.jsonl in its log dir) + # for verify-completion. + ++async_rl.log_full_train_data=true + logger.tensorboard_enabled=true + logger.wandb_enabled=false + logger.monitor_gpus=false +) + +OMNI_CMD=( + uv run --no-sync python "${RECOVERY_HOOK}" + --config "${OMNI_CONFIG}" + "${COMMON_OVERRIDES[@]}" +) + +echo "=== Phase 1: publish a checkpoint holding an active multimodal generation prefix ===" +command -v setsid >/dev/null +setsid env \ + SC_GYM_RECOVERY_TEST_EVENTS="${PHASE1_EVENTS}" \ + "${OMNI_CMD[@]}" \ + logger.log_dir="${PHASE1_LOG_DIR}" \ + "$@" >"${PHASE1_LOG}" 2>&1 & +PHASE1_PID=$! + +# Every prompt group must hold a recoverable prefix, so each distinct image is +# restored and verified at least once. +uv run --no-sync python "${SNAPSHOT_HELPER}" select \ + "${CHECKPOINT_DIR}/bootstrap/rollout_snapshots" \ + "${SELECTION_FILE}" \ + "${PHASE1_PID}" \ + "${PHASE1_LOG}" \ + "${SNAPSHOT_TIMEOUT_S}" \ + "${SELECTED_BACKUP}" \ + --stateless-resources \ + --min-cut-groups "${NUM_PROMPTS}" + +stop_phase1 +SNAPSHOT_DIR=$(uv run --no-sync python -c \ + 'import json, sys; print(json.load(open(sys.argv[1]))["snapshot"])' \ + "${SELECTION_FILE}") +SNAPSHOT_ROOT=$(dirname "${SNAPSHOT_DIR}") + +# Recover exactly the prefix selected above. A newer periodic snapshot or a +# trainer checkpoint would represent work performed after the simulated crash. +rm -rf "${SNAPSHOT_ROOT}" +mkdir -p "${SNAPSHOT_ROOT}" +cp -a "${SELECTED_BACKUP}" "${SNAPSHOT_DIR}" +for trainer_checkpoint in "${CHECKPOINT_DIR}"/step_*; do + if [[ -d "${trainer_checkpoint}" ]]; then + rm -rf "${trainer_checkpoint}" + fi +done + +echo "=== Phase 2: reprocess the images, verify them, and finish each suffix ===" +# Do not cut the replacement requests again while they consume the prefix. +: > "${RECOVERY_FLUSH_KEYS}" +timeout --signal=TERM --kill-after=30s "${RUN_TIMEOUT_S}s" \ + env \ + SC_GYM_RECOVERY_TEST_EVENTS="${PHASE2_EVENTS}" \ + SC_PREFIX_RECOVERY_STAGING_KEYS="${RECOVERY_STAGING_KEYS}" \ + SC_PREFIX_RECOVERY_FLUSH_KEYS="${RECOVERY_FLUSH_KEYS}" \ + "${OMNI_CMD[@]}" \ + ++rollout_checkpointing.snapshot_attempt_interval_s="${PHASE2_SNAPSHOT_INTERVAL_S}" \ + grpo.max_num_steps="${PHASE2_STEPS}" \ + logger.log_dir="${RECOVERY_LOG_DIR}" \ + "$@" 2>&1 | tee "${PHASE2_LOG}" + +grep -Fq "Selected rollout recovery snapshot: ${SNAPSHOT_DIR}" "${PHASE2_LOG}" +grep -q "Native TQ checkpoint restored and validated" "${PHASE2_LOG}" +grep -q "Loaded .* unfinished rollout group(s)" "${PHASE2_LOG}" +grep -q "generation prefix restored:" "${PHASE2_LOG}" +grep -q "train step ${PHASE2_STEPS}/${PHASE2_STEPS}" "${PHASE2_LOG}" +# A media mismatch is rejected with retained_media_changed and the call is +# retried from the prompt, which would otherwise pass as a fresh rollout. +if grep -q "retained_media_changed" "${PHASE2_LOG}"; then + echo "[ERROR] a restored multimodal prefix failed media verification" + exit 1 +fi + +uv run --no-sync tests/json_dump_tb_logs.py "${RECOVERY_LOG_DIR}" --output_path "${PHASE2_METRICS}" +# media_row_rate == 1 proves every learner row, including the restored ones, +# still carried captured pixels after recovery. +uv run --no-sync tests/check_metrics.py "${PHASE2_METRICS}" \ + 'max(data["train/finalize/invalid_row_rate"]) == 0' \ + 'max(data["train/finalize/capture_poisoned_rollouts"]) == 0' \ + 'min(data["train/finalize/media_row_rate"]) == 1' \ + 'all_finite(data["train/reward"])' + +uv run --no-sync python "${SNAPSHOT_HELPER}" verify-restore \ + "${SELECTION_FILE}" \ + "${PHASE1_EVENTS}" \ + "${PHASE2_EVENTS}" \ + "${PHASE2_LOG}" \ + "${CHECKPOINT_DIR}/step_${PHASE2_STEPS}/training_info.json" \ + "${PHASE2_STEPS}" + +# Each restored call completed exactly once, its trained row begins that call +# with the exact restored prefix, and its restored prefix rows outlived +# training only until the next snapshot. +uv run --no-sync python "${SNAPSHOT_HELPER}" verify-completion \ + "${SELECTION_FILE}" \ + "${RECOVERY_LOG_DIR}" \ + "${RECOVERY_STAGING_KEYS}" \ + "${RECOVERY_FLUSH_KEYS}" \ + "${PHASE2_LOG}" + +echo "Omni CLEVR multimodal generation-prefix recovery test passed" diff --git a/tests/unit/data/test_captured_media.py b/tests/unit/data/test_captured_media.py index 80b4e8b667f..808c7ac89c4 100644 --- a/tests/unit/data/test_captured_media.py +++ b/tests/unit/data/test_captured_media.py @@ -36,9 +36,13 @@ ) from nemo_rl.data.captured_media import ( + MEDIA_TENSOR_DIGESTS_FIELD, + CapturedMedia, MediaCaptureRejected, MediaColumnSpec, + media_tensor_digests, pack_images, + verify_restored_media, ) from nemo_rl.data.captured_media import ( capture_processed_media as _capture_processed_media, @@ -64,8 +68,10 @@ from nemo_rl.experience.rollout_reassembler import RolloutReassembler from nemo_rl.experience.route_assembly import verify_route_fragment_integrity from nemo_rl.models.generation.openai_server_utils import splice_prefix_tokens +from nemo_rl.models.generation.prefix_recovery import PrefixRecovery from nemo_rl.models.generation.vllm.vllm_worker_async import ( VllmAsyncGenerationWorkerImpl, + _verify_resumed_media_prompt, ) pytestmark = pytest.mark.nemo_gym @@ -729,6 +735,7 @@ def test_worker_restart_validates_retained_media( ) worker = SimpleNamespace( _capture_media=True, + _hash_captured_media=False, _capture_image_token_id=18, _capture_patch_size=1, _staging_source=source, @@ -774,6 +781,280 @@ def test_worker_restart_validates_retained_media( assert excinfo.value.code == "retained_media_changed" +def _stage_media_prefix(dp, tokens, prompt): + """Capture a media call with prefix cuts on and stage its first prefix row. + + Returns the worker, the captured media, and the row's staging key; the row + is written exactly as a checkpoint cut writes it. + """ + worker = SimpleNamespace( + _capture_media=True, + _hash_captured_media=True, + _capture_image_token_id=18, + _capture_patch_size=1, + ) + admission = CaptureAdmission( + rollout_id="r0", model_call_id="c1", prev_len=0, mode="text" + ) + media = VllmAsyncGenerationWorkerImpl._capture_request_media( + worker, prompt, admission=admission + ) + sink = TQTokenSink( + dp, + staging_partition="staging", + capture_media=True, + media_pixel_dtype=torch.float32, + ) + capture = RolloutTokenCapture( + sink=sink, weight_version_fn=lambda: 3, adapter=VLLMCaptureAdapter() + ) + record = capture.build_prefix_record( + capture.begin_call(admission), + prompt_token_ids=tokens, + generated_token_ids=[31], + generated_logprobs=[-0.25], + extras=VllmAsyncGenerationWorkerImpl._generation_cut_record_payload( + worker, SimpleNamespace(media=media) + ), + ) + staged = sink.stage_generation_prefix( + record, checkpoint_id="checkpoint-1", chunk_sequence=0 + ) + assert staged.ok + return worker, media, staged.staging_key + + +@pytest.mark.parametrize( + ("tokens", "make_prompt", "pixels"), + [ + pytest.param( + [10, 18, 18, 11], + lambda tokens, pixels: engine_prompt(tokens, [(Span(1, 2), pixels)]), + torch.arange(18, dtype=torch.float32).reshape(3, 2, 3), + id="image", + ), + pytest.param( + # Two temporal tubelets separated by timestamp text. + [10, 90, 18, 18, 91, 18, 18, 11], + lambda tokens, frames: video_prompt(tokens, [(Span(1, 6), frames)]), + torch.arange(48, dtype=torch.float32).reshape(4, 3, 2, 2), + id="video", + ), + ], +) +def test_generation_prefix_restore_verifies_media_by_digest_without_pixels( + dp, tokens, make_prompt, pixels +): + worker, media, staging_key = _stage_media_prefix( + dp, tokens, make_prompt(tokens, pixels) + ) + assert media.tensor_digests == media_tensor_digests(media.tensors) + # Only a video call carries frame counts, so only its digests cover them. + is_video = media.items[0].modality == "video" + assert ("num_frames" in media.tensor_digests) is is_video + + client = RecordingClient(dp) + source = TQTokenSource(client, staging_partition="staging", capture_media=True) + [chunk] = source.fetch_for_finalization([staging_key]) + assert not chunk.media_present + restored = VllmAsyncGenerationWorkerImpl._restore_generation_cut_media( + worker, [chunk] + ) + assert restored == CapturedMedia( + items=media.items, tensors=None, tensor_digests=media.tensor_digests + ) + assert not client.tensor_reads() + + resumed = CaptureAdmission( + rollout_id="r0-a1", model_call_id="c2", prev_len=0, mode="text" + ) + verified = VllmAsyncGenerationWorkerImpl._capture_request_media( + worker, + make_prompt(tokens, pixels), + admission=resumed, + restored=restored, + ) + torch.testing.assert_close(verified.tensors["imgs"], media.tensors["imgs"]) + + with pytest.raises(MediaCaptureRejected, match="tensor 'imgs' bytes") as error: + VllmAsyncGenerationWorkerImpl._capture_request_media( + worker, + make_prompt(tokens, torch.full_like(pixels, 9)), + admission=resumed, + restored=restored, + ) + assert error.value.code == "retained_media_changed" + + +def _media_prefix_chunk(dp): + tokens = [10, 18, 18, 11] + pixels = torch.arange(18, dtype=torch.float32).reshape(3, 2, 3) + worker, _, staging_key = _stage_media_prefix( + dp, tokens, engine_prompt(tokens, [(Span(1, 2), pixels)]) + ) + source = TQTokenSource(dp, staging_partition="staging", capture_media=True) + [chunk] = source.fetch_for_finalization([staging_key]) + return worker, chunk + + +def _without(extras, key): + return {name: value for name, value in extras.items() if name != key} + + +def _off_prompt_spans(extras): + span = {**extras[MEDIA_SPANS_FIELD][0], "placeholder_digest": "0" * 64} + return {**extras, MEDIA_SPANS_FIELD: [span]} + + +@pytest.mark.parametrize( + ("corrupt", "match"), + [ + (lambda extras: None, "media metadata is missing"), + ( + lambda extras: _without(extras, MEDIA_SPANS_FIELD), + "media metadata is missing", + ), + ( + lambda extras: {**extras, MEDIA_SPANS_FIELD: {}}, + "Media spans are malformed", + ), + ( + lambda extras: _without(extras, MEDIA_TENSOR_DIGESTS_FIELD), + "media digests are missing", + ), + ( + lambda extras: {**extras, MEDIA_TENSOR_DIGESTS_FIELD: {}}, + "metadata and digests disagree", + ), + ( + lambda extras: {**extras, MEDIA_SPANS_FIELD: []}, + "metadata and digests disagree", + ), + (_off_prompt_spans, "does not match the exact carried prompt"), + ], + ids=[ + "no_extras", + "no_spans", + "spans_not_a_list", + "no_digests", + "spans_without_digests", + "digests_without_spans", + "spans_off_the_prompt", + ], +) +def test_generation_prefix_media_restore_fails_closed_on_corrupt_rows( + dp, corrupt, match +): + worker, chunk = _media_prefix_chunk(dp) + + with pytest.raises(MediaCaptureRejected, match=match): + VllmAsyncGenerationWorkerImpl._restore_generation_cut_media( + worker, [replace(chunk, extras=corrupt(chunk.extras))] + ) + + +def test_generation_cut_rebuild_rejects_extras_after_the_first_chunk(dp): + worker, chunk = _media_prefix_chunk(dp) + later = replace(chunk, extras={MEDIA_SPANS_FIELD: []}) + + # The check runs before the admission or recovery state is consulted. + with pytest.raises(RuntimeError, match="only the first generation-cut chunk"): + PrefixRecovery._rebuild_prefix( + SimpleNamespace(), SimpleNamespace(generation_cut=None), [], [chunk, later] + ) + + +def test_resumed_media_prompt_must_equal_prefix_plus_cut_prompt_tokens(): + # One prompt-delta token run, then a generated token the check ignores. + cut = SimpleNamespace( + token_ids_delta=[18, 18, 11, 31], token_mask_delta=[0.0, 0.0, 0.0, 1.0] + ) + + _verify_resumed_media_prompt( + [10, 18, 18, 11], prefix_token_ids=[10], generation_cut=cut + ) + with pytest.raises( + MediaCaptureRejected, match="differs from the durable generation prefix" + ) as error: + _verify_resumed_media_prompt( + [10, 18, 11, 11], prefix_token_ids=[10], generation_cut=cut + ) + assert error.value.code == "retained_media_changed" + + +def _image_media() -> CapturedMedia: + return capture_processed_media( + engine_prompt([10, 18, 18, 11], [(Span(1, 2), torch.ones(3, 2, 3))]), + prev_len=0, + ) + + +@pytest.mark.parametrize( + ("change", "match"), + [ + (lambda media: replace(media, items=()), "metadata"), + ( + lambda media: replace( + media, + tensors={ + **media.tensors, + "num_frames": torch.tensor([1], dtype=torch.int32), + }, + ), + "tensor set", + ), + ( + lambda media: replace( + media, + tensors={**media.tensors, "imgs": media.tensors["imgs"].double()}, + ), + "tensor 'imgs' dtype or shape", + ), + ( + lambda media: replace( + media, + tensors={**media.tensors, "imgs": media.tensors["imgs"] + 1}, + ), + "tensor 'imgs' bytes", + ), + ], + ids=["items", "tensor_set", "dtype_or_shape", "bytes"], +) +def test_verify_restored_media_rejects_each_kind_of_change(change, match): + media = _image_media() + restored = CapturedMedia( + items=media.items, + tensors=None, + tensor_digests=media_tensor_digests(media.tensors), + ) + verify_restored_media(media, restored) + + with pytest.raises(MediaCaptureRejected, match=match) as error: + verify_restored_media(change(media), restored) + assert error.value.code == "retained_media_changed" + + +def test_verify_restored_media_requires_staged_digests(): + media = _image_media() + + with pytest.raises(MediaCaptureRejected, match="no tensor digests"): + verify_restored_media(media, replace(media, tensors=None)) + + +def test_media_tensor_digests_fingerprint_dtype_shape_and_bytes(): + pixels = torch.arange(6, dtype=torch.bfloat16).reshape(2, 3) + digests = media_tensor_digests({"imgs": pixels, "imgs_sizes": torch.ones(1, 2)}) + + assert list(digests) == ["imgs", "imgs_sizes"] + assert digests["imgs"]["dtype"] == "torch.bfloat16" + assert digests["imgs"]["shape"] == [2, 3] + assert media_tensor_digests({"imgs": pixels.clone()}) == {"imgs": digests["imgs"]} + reshaped = media_tensor_digests({"imgs": pixels.reshape(3, 2)})["imgs"] + assert reshaped["sha256"] == digests["imgs"]["sha256"] + assert reshaped["shape"] != digests["imgs"]["shape"] + assert media_tensor_digests(None) == {} + + @pytest.mark.parametrize("with_routed_experts", [False, True]) def test_worker_completion_stages_pixels_and_only_returns_capture_coordinates( dp, monkeypatch, with_routed_experts @@ -1103,6 +1384,7 @@ def retained_call(dp): source = TQTokenSource(client, staging_partition="staging", capture_media=True) worker = SimpleNamespace( _capture_media=True, + _hash_captured_media=False, _capture_image_token_id=18, _capture_patch_size=1, _staging_source=source, @@ -1505,6 +1787,50 @@ async def run(): assert isinstance(failure, GymTransportError) assert "retained_media_changed" in str(failure) + # A recovery request whose original URL/path/blob has disappeared fails + # while vLLM renders its engine-native prompt: before any TQ read and + # before begin_call. Prefix rows hold only tensor digests, so there are no + # staged pixels it could resume from instead. + worker.token_capture.reset_mock() + inference.reset_mock() + worker._staging_source = MagicMock(spec=TQTokenSource) + worker._chain_prefix = MagicMock(spec=ChainPrefixCache) + recovery_capture = { + **admission.model_dump(), + "generation_cut": { + "source_capture_key": admission.capture_key, + "source_model_call_id": admission.model_call_id, + "staging_keys": ["active/checkpoint-1/r0/c2/0"], + "prefix_token_count": 1, + "prefix_digest": "0" * 64, + "effective_output_limit": 8, + }, + } + missing_request = SimpleNamespace( + top_k=-1, + top_p=1.0, + temperature=1.0, + ng_capture=recovery_capture, + required_prefix_token_ids=None, + model_copy=lambda **kwargs: missing_request, + ) + + async def missing_media(self, **kwargs): + raise FileNotFoundError("original media disappeared") + + monkeypatch.setattr( + _OnlineRenderer, "preprocess_chat", missing_media, raising=False + ) + with pytest.raises(FileNotFoundError, match="original media disappeared"): + await handler(missing_request, MagicMock()) + assert worker._capture_admission(missing_request).generation_cut is not None + assert worker._staging_source.method_calls == [] + assert worker._chain_prefix.method_calls == [] + inference.assert_not_awaited() + worker.token_capture.begin_call.assert_not_called() + worker.token_capture.complete_call_from_response.assert_not_called() + assert worker._capture_calls == {} + @pytest.mark.parametrize("routes", [False, True]) def test_rl_owned_multimodal_rows_reuse_foundation_media_and_cleanup(dp, routes): diff --git a/tests/unit/environments/test_gym_prefix_recovery_parity.py b/tests/unit/environments/test_gym_prefix_recovery_parity.py index 6bf91bd2860..c0feb95e74b 100644 --- a/tests/unit/environments/test_gym_prefix_recovery_parity.py +++ b/tests/unit/environments/test_gym_prefix_recovery_parity.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import hashlib import importlib.util import json @@ -272,6 +273,91 @@ def test_prefix_completion_retires_old_keys() -> None: ) +def _dump_restored_run( + tmp_path: Path, *, flushed_keys: list[str] +) -> argparse.Namespace: + """A phase-two trainer dump, log, and selection for ``verify-completion``.""" + from nemo_rl.algorithms.single_controller_utils.debug_writer import ( + JsonDebugWriter, + TrainChunk, + ) + + log_dir = tmp_path / "recovery-logs" + token_ids = [1, 2] + list(range(100, 112)) + dump = JsonDebugWriter(str(log_dir / "exp_001")) + receipt = dump.add_chunk( + TrainChunk( + step=0, + sample_ids=["group-a_g0"], + tags=[{"prompt_idx": 0}], + input_lengths=torch.tensor([len(token_ids)]), + sequences={ + "token_ids": torch.tensor([token_ids]), + # The trainer masked one position inside the restored call. + "token_loss_mask": torch.tensor( + [[0.0, 0.0] + [1.0] * 6 + [0.0] + [1.0] * 5] + ), + "advantages": torch.zeros(1, len(token_ids)), + "generation_logprobs": torch.zeros(1, len(token_ids)), + "prev_logprobs": torch.zeros(1, len(token_ids)), + }, + scalars={ + "sample_loss_mask": torch.tensor([1.0]), + "rewards": torch.tensor([1.0]), + }, + ) + ) + dump.finish_step(0, [receipt]) + staging_keys = tmp_path / "recovery-staging-keys.jsonl" + staging_keys.write_text( + json.dumps({"train_step": 1, "staging_sample_ids": ["prefix-key"]}) + "\n" + ) + flush_keys = tmp_path / "recovery-flush-keys.jsonl" + flush_keys.write_text( + json.dumps({"epoch": 1, "staging_sample_ids": flushed_keys}) + "\n" + ) + selection = tmp_path / "selected-snapshot.json" + selection.write_text( + json.dumps( + { + "cuts": [ + { + **_cut("call-a", 0, 5), + "group_id": "group-a", + "staging_keys": ["prefix-key"], + } + ] + } + ) + ) + phase2_log = tmp_path / "phase2.log" + phase2_log.write_text(_completion_log("call-a")) + return argparse.Namespace( + selection_file=str(selection), + recovery_log_dir=str(log_dir), + recovery_staging_keys=str(staging_keys), + recovery_flush_keys=str(flush_keys), + phase2_log=str(phase2_log), + ) + + +def test_verify_completion_checks_a_reference_free_restore(tmp_path: Path, capsys): + _SNAPSHOT_HELPER.verify_completion( + _dump_restored_run(tmp_path, flushed_keys=["new-active-key"]) + ) + + assert "('group-a', 0)" in capsys.readouterr().out + + +def test_verify_completion_rejects_a_restored_prefix_that_outlived_the_snapshot( + tmp_path: Path, +): + with pytest.raises(AssertionError, match="outlived the next snapshot"): + _SNAPSHOT_HELPER.verify_completion( + _dump_restored_run(tmp_path, flushed_keys=["prefix-key"]) + ) + + @pytest.mark.parametrize( ("live_at_training", "live_after_flush", "match"), [ diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 8f5840e6095..06cecf480ca 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1605,6 +1605,8 @@ def checkpoint_cut(inventory, *, deadline): control_token=token, control_timeout_s=timeout_s, progress_reader=None, + prefix_row_extras=lambda state: None, + restore_media=lambda chunks: None, ) ) recovery.checkpoint_cut = checkpoint_cut diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index dba33d65d6a..d616c2f169c 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -24,12 +24,14 @@ import asyncio import hashlib +import json import logging import threading import time from collections import Counter +from concurrent.futures import ThreadPoolExecutor from copy import deepcopy -from dataclasses import asdict +from dataclasses import asdict, replace from types import SimpleNamespace from unittest.mock import MagicMock @@ -53,8 +55,16 @@ StageResult, ) -from nemo_rl.data.captured_media import MediaColumnSpec # noqa: E402 +from nemo_rl.data.captured_media import ( # noqa: E402 + MEDIA_TENSOR_DIGESTS_FIELD, + CapturedMedia, + CapturedMediaItem, + MediaCaptureRejected, + MediaColumnSpec, + media_tensor_digests, +) from nemo_rl.data_plane.tq_token_sink import ( # noqa: E402 + FetchedStagedCall, StagedPrefix, TQTokenSink, ) @@ -103,6 +113,7 @@ def __init__(self) -> None: self.records: list[StagedCallRecord] = [] self.attachments: list[dict[str, torch.Tensor] | None] = [] self.generation_prefix_records: list[tuple[str, StagedCallRecord]] = [] + self.generation_prefix_attachments: list[dict | None] = [] self.generation_prefix_keys: list[str] = [] self.cleared_generation_prefix_keys: list[str] = [] @@ -124,8 +135,8 @@ def stage_generation_prefix( chunk_sequence: int, attachments: dict | None = None, ) -> StageResult: - assert attachments is None self.generation_prefix_records.append((checkpoint_id, record)) + self.generation_prefix_attachments.append(attachments) key = ( f"__generation_cut__/{checkpoint_id}/{record.capture_key}/" f"{record.model_call_id}/{chunk_sequence}" @@ -190,6 +201,8 @@ def _cuts( control_token="secret", control_timeout_s=None, progress_reader=progress_reader or _NoCallsInVllm(), + prefix_row_extras=worker._generation_cut_record_payload, + restore_media=worker._restore_generation_cut_media, ) ) worker._prefix_recovery = recovery @@ -369,7 +382,7 @@ def test_generation_prefix_restore_counts_sum_over_workers(monkeypatch): def test_prefix_capture_setup_requires_a_control_token(monkeypatch): - # Prefix recovery with media or sampler top-k is refused at config/setup. + # Prefix recovery with sampler top-k is refused at config/setup. monkeypatch.setattr( "nemo_rl.data_plane.build_data_plane_client", lambda dp_cfg, bootstrap, segment_size=None: MagicMock(), @@ -1317,9 +1330,19 @@ def fetch_prefix_chains(self, staging_keys: list[str]) -> StagedPrefix: expanded=[token for key in staging_keys for token in self.deltas[key]] ) - def fetch(self, staging_keys: list[str]) -> list[StagedCallRecord]: + def fetch_for_finalization( + self, staging_keys: list[str] + ) -> list[FetchedStagedCall]: self.calls.append(list(staging_keys)) - return [self.records[key] for key in staging_keys] + return [ + FetchedStagedCall( + staging_key=key, + snapshot=self.records[key], + routed_len=0, + extras=self.records[key].extras, + ) + for key in staging_keys + ] def _served_content(gen_ids, logprobs): @@ -1734,6 +1757,8 @@ def _record_cut_constraints(state, sampling_params) -> None: control_token=None, control_timeout_s=None, progress_reader=_NoCallsInVllm(), + prefix_row_extras=lambda state: None, + restore_media=lambda chunks: None, ) ).on_sampling_params(object(), sampling_params) @@ -1862,6 +1887,318 @@ def test_uncuttable_sampling_settings_warn_once_per_reason(monkeypatch, caplog): assert caplog.text.count("are not cut; they regenerate after a restore") == 1 +def _captured_image_media() -> CapturedMedia: + tensors = { + "imgs": torch.ones(1, 2, 3), + "imgs_sizes": torch.tensor([[1, 2]], dtype=torch.int32), + } + # As _capture_request_media returns it when prefix cuts are enabled. + return CapturedMedia( + items=( + CapturedMediaItem( + modality="image", + placeholder_offset=0, + placeholder_length=2, + placeholder_digest=hashlib.sha256( + json.dumps([18, 18], separators=(",", ":")).encode() + ).hexdigest(), + token_id=18, + embedding_spans=((0, 2),), + imgs_sizes=((1, 2),), + ), + ), + tensors=tensors, + tensor_digests=media_tensor_digests(tensors), + ) + + +def _worker_with_active_media_prefix( + sink: _MemorySink, media: CapturedMedia | None = None +): + worker = _worker_with_capture(sink) + request = _FakeRequest( + ng_capture={ + "rollout_id": "r0", + "model_call_id": "c1", + "parent_call_id": None, + "prev_len": 0, + "mode": "text", + }, + stream=False, + ) + media = media or _captured_image_media() + VllmAsyncGenerationWorkerImpl._begin_request_capture( + worker, request, [18, 18], media=media + ) + state = worker._capture_registry.get(request) + state.set_effective_output_limit(8) + state.record([20], [-0.1]) + return worker, request, state, media + + +def test_generation_cut_stages_media_digests_only_with_first_prefix_row(): + sink = _MemorySink() + worker, _, state, media = _worker_with_active_media_prefix(sink) + + first_receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-1") + ) + state.record([21], [-0.2]) + second_receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-2") + ) + + first, second = [record for _, record in sink.generation_prefix_records] + assert first.extras == { + "media_spans": [media.items[0].to_dict()], + MEDIA_TENSOR_DIGESTS_FIELD: media.tensor_digests, + } + assert second.extras is None + # Prefix rows never hold pixels; only the terminal row stages them. + assert sink.generation_prefix_attachments == [None, None] + assert second_receipt.prefixes[0].staging_keys == ( + *first_receipt.prefixes[0].staging_keys, + sink.generation_prefix_keys[-1], + ) + assert state.snapshot().staging_keys == second_receipt.prefixes[0].staging_keys + assert not sink.cleared_generation_prefix_keys + + +def test_generation_cut_refuses_media_captured_without_digests(caplog): + sink = _MemorySink() + worker, _, state, _ = _worker_with_active_media_prefix( + sink, replace(_captured_image_media(), tensor_digests=None) + ) + + with caplog.at_level(logging.ERROR): + receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-1") + ) + + # Only this call is left uncut; it regenerates after a restore. + assert [ack.disposition for ack in receipt.prefixes] == ["durable_failure"] + assert "without tensor digests" in caplog.text + assert not sink.generation_prefix_records + assert not state.snapshot().cut_in_progress + assert state.snapshot().staged_token_count == 0 + + +def _worker_with_active_text_prefix(sink: _MemorySink): + worker = _worker_with_capture(sink) + request = _FakeRequest( + ng_capture={ + "rollout_id": "r0", + "model_call_id": "c1", + "parent_call_id": None, + "prev_len": 0, + "mode": "text", + }, + stream=False, + ) + VllmAsyncGenerationWorkerImpl._begin_request_capture(worker, request, [10, 11]) + state = worker._capture_registry.get(request) + state.set_effective_output_limit(8) + state.record([20], [-0.1]) + return worker, state + + +@pytest.mark.parametrize( + ("media", "cuts"), + [(False, 1), (True, 1), (True, 2)], + ids=["text", "image_one_cut", "image_two_cuts"], +) +def test_generation_cut_digest_written_at_the_cut_is_the_one_restore_rebuilds( + media: bool, cuts: int +): + sink = _MemorySink() + if media: + worker, _, state, _ = _worker_with_active_media_prefix(sink) + else: + worker, state = _worker_with_active_text_prefix(sink) + for index in range(cuts): + if index: + state.record([21], [-0.2]) + receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id=f"checkpoint-{index + 1}") + ) + (ack,) = receipt.prefixes + records = dict( + zip( + sink.generation_prefix_keys, + (record for _, record in sink.generation_prefix_records), + strict=True, + ) + ) + assert list(records) == list(ack.staging_keys) + first = records[ack.staging_keys[0]] + assert (MEDIA_TENSOR_DIGESTS_FIELD in (first.extras or {})) is media + + def resolve(staged: dict[str, StagedCallRecord]): + resumed = _resumed_worker( + sink, records=staged, weight_version=worker._rollout_weight_version + ) + request = _FakeRequest( + ng_capture={ + "rollout_id": "r0-a1", + "model_call_id": "c2", + "mode": "text", + "generation_cut": { + "source_capture_key": "r0", + "source_model_call_id": "c1", + "staging_keys": list(ack.staging_keys), + "prefix_token_count": ack.prefix_token_count, + "prefix_digest": ack.prefix_digest, + "effective_output_limit": ack.effective_output_limit, + }, + }, + stream=False, + ) + return _cuts(resumed)._rebuild_prefix(resumed._capture_admission(request), []) + + # Restore recomputes the cumulative digest from the staged chunks, the + # first chunk's media extras included, and must reach the cut's digest: + # Gym's begin_call rejects a restored snapshot whose digest differs. + assert resolve(records).digest == ack.prefix_digest + if media: + # The tensor digests are covered: a changed one changes the digest. + digests = dict(first.extras[MEDIA_TENSOR_DIGESTS_FIELD]) + digests["imgs"] = {**digests["imgs"], "sha256": "0" * 64} + tampered = first.model_copy( + update={"extras": {**first.extras, MEDIA_TENSOR_DIGESTS_FIELD: digests}} + ) + assert ( + resolve({**records, ack.staging_keys[0]: tampered}).digest + != ack.prefix_digest + ) + + +class _BlockingPrefixSink(_MemorySink): + def __init__(self) -> None: + super().__init__() + self.prefix_write_started = threading.Event() + self.release_prefix_write = threading.Event() + + def stage_generation_prefix(self, *args, **kwargs) -> StageResult: + self.prefix_write_started.set() + assert self.release_prefix_write.wait(5) + return super().stage_generation_prefix(*args, **kwargs) + + +def _assert_stages_captured_tensors(attachments, tensors) -> None: + """The terminal row stages the captured media tensors themselves, not copies. + + The worker builds a fresh attachments dict (it may add sampler top-k), so + compare the tensors, not the dict. + """ + assert attachments.keys() == tensors.keys() + assert all(attachments[name] is tensor for name, tensor in tensors.items()) + + +def test_media_terminal_completion_waits_for_prefix_cut_then_replaces_it(): + sink = _BlockingPrefixSink() + worker, request, _, media = _worker_with_active_media_prefix(sink) + terminal_entered = threading.Event() + terminal_finished = threading.Event() + + def complete(): + terminal_entered.set() + try: + return VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, request, _served_content([20, 21], [-0.1, -0.2]) + ) + finally: + terminal_finished.set() + + with ThreadPoolExecutor(max_workers=2) as executor: + cut = executor.submit( + _cuts(worker).checkpoint_cut, + _single_call_inventory(checkpoint_id="checkpoint-1"), + ) + assert sink.prefix_write_started.wait(5) + terminal = executor.submit(complete) + assert terminal_entered.wait(5) + assert not terminal_finished.wait(0.05) + sink.release_prefix_write.set() + receipt = cut.result(timeout=5) + content = terminal.result(timeout=5) + + assert receipt.prefixes[0].disposition == "durable_prefix" + assert content["ng_commit_coords"]["disposition"] == "staged" + assert sink.generation_prefix_attachments == [None] + _assert_stages_captured_tensors(sink.attachments[0], media.tensors) + assert sink.cleared_generation_prefix_keys == [] + _flush_retired_rows(worker) + assert sink.cleared_generation_prefix_keys == list(receipt.prefixes[0].staging_keys) + + +class _FailTerminalSink(_MemorySink): + def __init__(self) -> None: + super().__init__() + self.terminal_attempts: list[tuple[StagedCallRecord, dict | None]] = [] + + def stage( + self, record: StagedCallRecord, *, attachments: dict | None = None + ) -> StageResult: + self.terminal_attempts.append((record, attachments)) + return StageResult( + ok=False, + staging_key=record.staging_key, + error="injected terminal staging failure", + ) + + +def test_media_call_whose_terminal_staging_fails_clears_its_cut_rows(): + sink = _FailTerminalSink() + worker, request, state, media = _worker_with_active_media_prefix(sink) + receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-1") + ) + + content = VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, request, _served_content([20, 21], [-0.1, -0.2]) + ) + + assert content["ng_commit_coords"]["disposition"] == "capture_failed" + _assert_stages_captured_tensors(sink.terminal_attempts[0][1], media.tensors) + # As for a text call, the failed call's cut rows are retired with it. + _flush_retired_rows(worker) + assert sink.cleared_generation_prefix_keys == list(receipt.prefixes[0].staging_keys) + + +def test_canonical_media_row_survives_prefix_cleanup_failure(caplog): + sink = _FailingClearSink() + worker, request, _, media = _worker_with_active_media_prefix(sink) + _cuts(worker).checkpoint_cut(_single_call_inventory(checkpoint_id="checkpoint-1")) + + content = VllmAsyncGenerationWorkerImpl._finish_request_capture( + worker, request, _served_content([20, 21], [-0.1, -0.2]) + ) + with caplog.at_level(logging.ERROR): + _flush_retired_rows(worker) + + assert content["ng_commit_coords"]["disposition"] == "staged" + assert sink.records[-1].staging_key == "r0/c1" + _assert_stages_captured_tensors(sink.attachments[-1], media.tensors) + assert "failed to clear 1 obsolete generation chunk rows" in caplog.text + + +def test_generation_cut_during_media_preprocessing_redispatches_call(): + sink = _MemorySink() + worker = _worker_with_capture(sink) + worker._capture_media = True + + # begin_request_capture runs only after vLLM finishes media preprocessing. + # A cut that arrives in that window sees no active state and must tell Gym + # to redispatch the model call instead of advertising an empty prefix. + receipt = _cuts(worker).checkpoint_cut( + _single_call_inventory(checkpoint_id="checkpoint-before-preprocess") + ) + + assert receipt.prefixes[0].disposition == "durable_failure" + assert not sink.generation_prefix_records + assert not sink.generation_prefix_attachments + + class _FailOncePrefixSink(_MemorySink): def __init__(self) -> None: super().__init__() @@ -2015,17 +2352,22 @@ def test_usable_restored_prefix_is_resumed(): worker = _resumed_worker(sink, records={cut_key: cut_record}) admission = worker._capture_admission(request) - served, snapshot = _cuts(worker).resolve_restore(admission, []) + served, resolved = _cuts(worker).resolve_restore(admission, []) assert served is admission - assert snapshot.digest == cut_record.digest + assert resolved.snapshot.digest == cut_record.digest _flush_retired_rows(worker) assert sink.cleared_generation_prefix_keys == [] @pytest.mark.parametrize( "failure", - ["missing chunk", "chunk newer than the worker", "digest mismatch"], + [ + "missing chunk", + "chunk newer than the worker", + "digest mismatch", + "media row unreadable", + ], ) def test_unusable_restored_prefix_regenerates_instead_of_failing(failure, caplog): sink = _MemorySink() @@ -2044,11 +2386,18 @@ def test_unusable_restored_prefix_regenerates_instead_of_failing(failure, caplog ) } ) + if failure == "media row unreadable": + # The media comes from the same fetch, inside the same fallback. + def unreadable_media(chunks): + raise MediaCaptureRejected("Generation-prefix media metadata is missing") + + worker._restore_generation_cut_media = unreadable_media + _cuts(worker, staging_source=worker._staging_source) with caplog.at_level(logging.WARNING): - served, snapshot = _cuts(worker).resolve_restore(admission, []) + served, resolved = _cuts(worker).resolve_restore(admission, []) - assert snapshot is None + assert resolved is None assert served.generation_cut is None assert "regenerating it from its turn boundary" in caplog.text # The regenerated call is admitted as an ordinary call from its boundary. @@ -2572,8 +2921,11 @@ def test_request_capture_abort_fails_the_call_and_drains_state(): @pytest.mark.vllm +@pytest.mark.parametrize("prefix_cuts", [False, True]) @pytest.mark.parametrize("pruning_rate", [0.0, 0.5]) -def test_omni_capture_setup_rejects_video_pruning(monkeypatch, pruning_rate): +def test_omni_capture_setup_rejects_video_pruning( + monkeypatch, pruning_rate, prefix_cuts +): # Both dependency markers select the combined vLLM + Gym lane. pytest.importorskip("vllm") from vllm.model_executor.models.nano_nemotron_vl import NanoNemotronVLProcessingInfo @@ -2597,25 +2949,30 @@ def test_omni_capture_setup_rejects_video_pruning(monkeypatch, pruning_rate): ) worker = _fake_worker() worker.llm = SimpleNamespace( - renderer=SimpleNamespace(get_mm_processor=lambda: SimpleNamespace(info=info)), + renderer=SimpleNamespace( + get_mm_processor=lambda: SimpleNamespace(info=info), + get_tokenizer=lambda: SimpleNamespace(decode=lambda ids: ""), + ), model_config=SimpleNamespace(dtype=torch.bfloat16), ) monkeypatch.setattr( "nemo_rl.data_plane.build_data_plane_client", lambda dp_cfg, bootstrap, segment_size=None: MagicMock(), ) + kwargs = {"staging_partition": "staging", "capture_media": True} + if prefix_cuts: + kwargs.update( + generation_prefix_cuts_enabled=True, + generation_cut_control_token="secret", + ) if pruning_rate: with pytest.raises(ValueError, match="video token pruning"): asyncio.run( - VllmAsyncGenerationWorkerImpl.setup_token_capture( - worker, {}, staging_partition="staging", capture_media=True - ) + VllmAsyncGenerationWorkerImpl.setup_token_capture(worker, {}, **kwargs) ) else: spec = asyncio.run( - VllmAsyncGenerationWorkerImpl.setup_token_capture( - worker, {}, staging_partition="staging", capture_media=True - ) + VllmAsyncGenerationWorkerImpl.setup_token_capture(worker, {}, **kwargs) ) # The engine's model dtype and the HF patch size, as pinned on the sink. assert spec == MediaColumnSpec(pixel_dtype=torch.bfloat16, patch_size=2) diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index b8ae3ed9c8c..a344835b47f 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -1274,7 +1274,6 @@ def test_gym_state_recovery_rejects_a_config_that_never_saves( ("feature", "match"), [ ("router_replay", "does not yet support router replay"), - ("multimodal", "does not yet support multimodal capture"), ( "megatron", "target_level='prefix' currently supports only the vllm generation", @@ -1295,14 +1294,11 @@ def test_prefix_recovery_rejects_unsupported_features(self, feature, match): mc.rollout_checkpointing = RolloutCheckpointConfig( snapshot_attempt_interval_s=1.0 ) - processor = None if feature == "router_replay": mc.policy["router_replay"] = {"enabled": True} - elif feature == "multimodal": - processor = MagicMock() with pytest.raises(NotImplementedError, match=match): - setup_single_controller(mc, MagicMock(pad_token_id=0), processor=processor) + setup_single_controller(mc, MagicMock(pad_token_id=0)) @pytest.mark.parametrize( ("overrides", "error", "match"), @@ -1490,6 +1486,65 @@ def record_gym_log_dir(env_configs, **_kwargs): assert gym_log_dirs_at_build == [str(tmp_path / "exp_001" / "nemo_gym")] + def test_prefix_recovery_accepts_multimodal_capture( + self, + tmp_path: Path, + patched_factories, + ): + mc = _make_master_config(colocated=False, backend="vllm", megatron_enabled=True) + mc.checkpointing.update( + { + "checkpoint_dir": str(tmp_path / "checkpoints"), + "enabled": True, + "save_data_plane": True, + "save_period": 1, + } + ) + mc.policy["generation"].update( + { + "model_name": "test-model", + "stop_strings": None, + "stop_token_ids": None, + "top_k": None, + "vllm_cfg": {"async_engine": True}, + } + ) + mc.token_capture.enabled = True + mc.rollout_recovery.target_level = RecoveryTargetLevel.PREFIX + mc.rollout_checkpointing = RolloutCheckpointConfig( + snapshot_attempt_interval_s=1.0 + ) + patched_factories["setup_response_data"].return_value = ( + list(range(8)), + None, + ) + + with ( + patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), + patch.object( + sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() + ) as mock_spinup, + patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), + patch.object(sc_setup_mod, "uses_image_placeholder", return_value=True), + patch( + "nemo_rl.experience.rollout_reassembler_actor." + "create_rollout_reassembler_actors", + return_value=[MagicMock(name="finalizer")], + ), + ): + tokenizer = MagicMock(pad_token_id=0) + setup_single_controller( + mc, + tokenizer, + processor=MagicMock(tokenizer=tokenizer), + ) + + assert mock_spinup.call_args.kwargs["turn_checkpointing_enabled"] is True + generation, _ = patched_factories["_build_generation"].return_value + _, setup_kwargs = generation.setup_token_capture.call_args + assert setup_kwargs["capture_media"] is True + assert setup_kwargs["generation_prefix_cuts_enabled"] is True + def test_disabled_periodic_checkpointing_ignores_existing_snapshots( self, tmp_path: Path, From ce47ea4bc1e02232ddc8d6cb437d25722bcdb7e5 Mon Sep 17 00:00:00 2001 From: Anish Mahishi Date: Fri, 13 Nov 2026 09:42:34 -0800 Subject: [PATCH 2/2] refactor(checkpoint): address review of multimodal prefix recovery - verify_restored_media is one comparison of items and tensor digests that names the changed tensors; the dead fallback and None branches are gone. - resolve_restore returns the verified snapshot again, as on main; the worker reads a restored call's media from that snapshot after the digest check, so _ResolvedGenerationCut, the restore_media hook, _fetch_chunks and the media row guards go away. The handler's restored-media steps sit behind _capture_restored_media. - Drop _verify_resumed_media_prompt: vLLM prefills the saved prefix, and equal media items already pin every placeholder. - Tests: cover the handler's restored-media branch and a restored image call cut again; fold duplicated helpers, sinks and setup patching; fix the missing-media handler case, which lacked the sampling fields the handler checks. - The two Omni CLEVR vLLM tests take their fixed topology from their YAML, and the prefix-recovery tests share restore_selected_snapshot. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Anish Mahishi --- ..._vllm_token_capture_single_controller.yaml | 57 +++- nemo_rl/data/captured_media.py | 45 +-- .../generation/prefix_recovery/parts.py | 5 +- .../generation/prefix_recovery/recovery.py | 50 +--- .../generation/vllm/vllm_worker_async.py | 92 +++--- .../functional/_gym_prefix_recovery_phases.sh | 24 +- ...c_gym_single_controller_prefix_recovery.sh | 16 +- ..._prefix_recovery_single_controller_1n2g.sh | 55 +--- ...lm_token_capture_single_controller_1n2g.sh | 40 +-- tests/unit/data/test_captured_media.py | 256 ++++++++--------- .../models/generation/test_vllm_generation.py | 1 - .../test_vllm_token_capture_hosting.py | 263 +++++++++--------- tests/unit/single_controller/test_setup.py | 62 ++--- 13 files changed, 434 insertions(+), 532 deletions(-) diff --git a/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml b/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml index 5353ff36d4c..14da19e6651 100644 --- a/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml +++ b/examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml @@ -2,15 +2,35 @@ # generation, a Megatron learner, and vLLM media token capture enabled. # # Functional-test config (not a nightly recipe, so it lives beside the other -# Gym smoke configs rather than under examples/configs/recipes/): -# tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh -# supplies the data paths, the 1 trainer + 1 generator split, and the step -# budget. Token capture requires the Gym rollout path, so the native CLEVR +# Gym smoke configs rather than under examples/configs/recipes/). It pins the +# 1n2g split both Omni CLEVR vLLM tests use: one GPU trains the frozen-decoder +# policy, one hosts async vLLM. The tests +# (tests/functional/nemotron_omni_clevr_vllm_*_single_controller_1n2g.sh) +# supply the data paths, sequence lengths and step budget. Token capture +# requires the Gym rollout path, so the native CLEVR # dataset/environment of the parent recipe are swapped for Gym rows here. # Prefix caching stays off (hybrid-Mamba align cache mode asserts chunked # prefill) and the parent's mm_processor_cache_gb=0 keeps concrete pixels # available to the capture host. defaults: ../configs/recipes/vlm/vlm_grpo-nemotron-omni-30ba3b-clevr-8n4g-megatron-single-controller-async.v1.yaml +cluster: + num_nodes: 1 + gpus_per_node: 2 + segment_size: 1 +grpo: + val_period: 0 + val_at_start: false + val_at_end: false +async_rl: + generation_fleet_health: + refit_timeout_s: null +data_plane: + simple: + num_storage_units: 2 +logger: + tensorboard_enabled: true + wandb_enabled: false + monitor_gpus: false data: _override_: true max_input_seq_length: null @@ -52,10 +72,39 @@ env: simple_agent: max_steps: 1 policy: + train_micro_batch_size: 1 + megatron_cfg: + tensor_model_parallel_size: 1 + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + context_parallel_size: 1 + sequence_parallel: true + activation_checkpointing: true + bias_dropout_fusion: false + freeze_config: + freeze_language_model: true + optimizer: + optimizer_cpu_offload: false + optimizer_offload_fraction: 0.0 + params_dtype: bfloat16 + main_grads_dtype: bfloat16 + main_params_dtype: float16 + exp_avg_dtype: bfloat16 + exp_avg_sq_dtype: bfloat16 + store_param_remainders: false generation: backend: vllm refit_transport: null + colocated: + enabled: false + resources: + num_nodes: 1 + gpus_per_node: 1 vllm_cfg: + tensor_parallel_size: 1 + expert_parallel_size: 1 + pipeline_parallel_size: 1 + gpu_memory_utilization: 0.6 async_engine: true expose_http_server: true logprobs_mode: raw_logprobs diff --git a/nemo_rl/data/captured_media.py b/nemo_rl/data/captured_media.py index 7d25054ded3..e0ceba5a9cc 100644 --- a/nemo_rl/data/captured_media.py +++ b/nemo_rl/data/captured_media.py @@ -24,7 +24,7 @@ import json from collections.abc import Mapping from dataclasses import asdict, dataclass, replace -from typing import TYPE_CHECKING, Any, Literal, NoReturn +from typing import TYPE_CHECKING, Any, Literal import torch @@ -182,40 +182,25 @@ def verify_restored_media(captured: CapturedMedia, restored: CapturedMedia) -> N generation-prefix row. Raises: - MediaCaptureRejected: With ``retained_media_changed`` on the first of - an item, tensor-name, dtype/shape, or byte mismatch. + MediaCaptureRejected: ``retained_media_changed`` when the items or + any tensor's dtype, shape, or bytes differ. """ - - def reject(what: str) -> NoReturn: + current = captured.tensor_digests or {} + staged = restored.tensor_digests or {} + if captured.items != restored.items or current != staged: + # Name what differs, so a rejection can be triaged from the log. + changed = sorted( + name + for name in current.keys() | staged.keys() + if current.get(name) != staged.get(name) + ) raise MediaCaptureRejected( - f"Restored generation media {what} differs from reprocessed input", + "Restored generation media differs from reprocessed input " + f"(items equal: {captured.items == restored.items}; " + f"changed tensors: {changed})", code="retained_media_changed", ) - if captured.items != restored.items: - reject("metadata") - staged = restored.tensor_digests - if staged is None: - raise MediaCaptureRejected("Restored generation media has no tensor digests") - current = ( - captured.tensor_digests - if captured.tensor_digests is not None - else media_tensor_digests(captured.tensors) - ) - if set(current) != set(staged): - reject("tensor set") - for name, digest in current.items(): - expected = staged[name] - if not isinstance(expected, dict): - reject(f"tensor {name!r} digest") - if (digest["dtype"], digest["shape"]) != ( - expected.get("dtype"), - expected.get("shape"), - ): - reject(f"tensor {name!r} dtype or shape") - if digest["sha256"] != expected.get("sha256"): - reject(f"tensor {name!r} bytes") - def verified_media_items(call: FetchedStagedCall) -> tuple[CapturedMediaItem, ...]: """Authenticate a vLLM call's new occurrences before using their layout. diff --git a/nemo_rl/models/generation/prefix_recovery/parts.py b/nemo_rl/models/generation/prefix_recovery/parts.py index c56f42f1796..9f4a433594a 100644 --- a/nemo_rl/models/generation/prefix_recovery/parts.py +++ b/nemo_rl/models/generation/prefix_recovery/parts.py @@ -16,7 +16,7 @@ from __future__ import annotations -from collections.abc import Callable, Sequence +from collections.abc import Callable from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -55,6 +55,3 @@ class PrefixRecoveryParts: # Extras for a call's first prefix row, from the media its capture state # holds; None for a call without media. prefix_row_extras: Callable[[_RequestCaptureState], dict[str, Any] | None] - # A restored call's media, read back from its fetched prefix chunks; None - # when the worker does not capture media. - restore_media: Callable[[Sequence[Any]], Any | None] diff --git a/nemo_rl/models/generation/prefix_recovery/recovery.py b/nemo_rl/models/generation/prefix_recovery/recovery.py index 49a6c52ae21..7f568f1aa97 100644 --- a/nemo_rl/models/generation/prefix_recovery/recovery.py +++ b/nemo_rl/models/generation/prefix_recovery/recovery.py @@ -32,7 +32,6 @@ from collections import Counter from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass from typing import Any, Literal from nemo_rl.models.generation.prefix_recovery.parts import PrefixRecoveryParts @@ -92,14 +91,6 @@ def _require_inventory_identity( ) -@dataclass(frozen=True) -class _ResolvedGenerationCut: - """A verified restored prefix and the media its call carried.""" - - snapshot: Any - media: Any | None = None - - def _ends_inside_character(tokenizer: Any, generated_token_ids: list[int]) -> bool: """Whether the generated text stops partway through a multi-byte character. @@ -675,8 +666,9 @@ def checkpoint_cut(self, inventory: Any, *, deadline: float | None = None) -> An self._receipts.pop(next(iter(self._receipts))) return receipt - def _fetch_chunks(self, continuation: Any) -> list[Any]: - """Fetch the staged chunks named by a ``GenerationCutContinuation``, in order.""" + def _rebuild_prefix(self, admission: Any, prefix_token_ids: list[int]) -> Any: + """Fetch an admission's staged prefix chunks and rebuild the cumulative prefix.""" + continuation = admission.generation_cut # Gym requires at least one staging key per continuation. staging_keys = list(continuation.staging_keys) # The finalization read also returns each row's extras, which carry a @@ -684,24 +676,6 @@ def _fetch_chunks(self, continuation: Any) -> list[Any]: chunks = self._parts.staging_source.fetch_for_finalization(staging_keys) if len(chunks) != len(staging_keys): raise RuntimeError("generation-cut fetch did not return every staged chunk") - if not chunks: - raise RuntimeError("generation-cut continuation has no staged chunks") - return chunks - - def _rebuild_prefix( - self, - admission: Any, - prefix_token_ids: list[int], - chunks: list[Any] | None = None, - ) -> Any: - """Rebuild the cumulative prefix from an admission's staged chunks. - - ``chunks`` are the already-fetched chunks, so a restore that also reads - their media fetches once; they are fetched here when omitted. - """ - continuation = admission.generation_cut - if chunks is None: - chunks = self._fetch_chunks(continuation) snapshots = [chunk.snapshot for chunk in chunks] # A media call's spans and tensor digests ride the first chunk's extras, # and the advertised prefix digest covers them. @@ -801,24 +775,20 @@ def _rebuild_prefix( def resolve_restore( self, admission: Any, prefix_token_ids: list[int] - ) -> tuple[Any, _ResolvedGenerationCut | None]: + ) -> tuple[Any, Any | None]: """Resolve an admission's restored prefix, or drop it so the call regenerates. Gym has already consumed the restored cut when it sends this call, and generating the whole response from the turn boundary is always valid, - so a prefix that cannot be fetched, whose media cannot be restored, - or that does not match its admission costs time, not the rollout. - Returns the admission to serve and the verified prefix with its media, - or the admission without its cut and None. + so a prefix that cannot be fetched or does not match its admission + costs time, not the rollout. Returns the admission to serve and the + verified prefix snapshot, or the admission without its cut and None. """ continuation = admission.generation_cut if continuation is None: return admission, None try: - # One fetch serves both the media and the prefix. - chunks = self._fetch_chunks(continuation) - media = self._parts.restore_media(chunks) - snapshot = self._rebuild_prefix(admission, prefix_token_ids, chunks) + snapshot = self._rebuild_prefix(admission, prefix_token_ids) # Checked here, before the prefix is spliced into the prompt; Gym's # begin_call repeats it and would otherwise fail the call. if snapshot.digest != continuation.prefix_digest: @@ -829,7 +799,7 @@ def resolve_restore( f"rebuilt prefix has {generated} generated tokens, " f"admission expects {continuation.prefix_token_count}" ) - return admission, _ResolvedGenerationCut(snapshot=snapshot, media=media) + return admission, snapshot except Exception: # noqa: BLE001 - any unusable prefix regenerates LOGGER.warning( "cannot restore the generation prefix of model call %s; " @@ -889,7 +859,7 @@ class NoPrefixRecovery: def resolve_restore( self, admission: Any, prefix_token_ids: list[int] - ) -> tuple[Any, _ResolvedGenerationCut | None]: + ) -> tuple[Any, Any | None]: if admission.generation_cut is None: return admission, None # Gym only sends a restored cut to a worker that acknowledged one, so diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index ed08aa540f9..af531c35cf1 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -167,36 +167,6 @@ def _media_items_from_extras( return items -def _verify_resumed_media_prompt( - reprocessed_token_ids: list[int], - *, - prefix_token_ids: list[int], - generation_cut: Any, -) -> None: - """Require a resumed media call to reprocess into its durable prompt. - - The call resumes from the exact engine prompt at the cut: its carried - prefix followed by the cut's prompt tokens. Its media placeholders are - re-derived from the reprocessed prompt, so a reprocessed prompt that - differs would place them over the wrong tokens. - - Raises: - MediaCaptureRejected: With ``retained_media_changed`` on a mismatch. - """ - prompt_delta = [ - token_id - for token_id, mask in zip( - generation_cut.token_ids_delta, generation_cut.token_mask_delta, strict=True - ) - if mask == 0.0 - ] - if reprocessed_token_ids != [*prefix_token_ids, *prompt_delta]: - raise MediaCaptureRejected( - "Reprocessed multimodal prompt differs from the durable generation prefix", - code="retained_media_changed", - ) - - # Paths the bearer-token check protects: the OpenAI-compatible routes this module # registers, which NeMo-Gym's model server calls. The sparse-refit routes the # receiver registers on the same app carry their own header token @@ -989,7 +959,6 @@ async def setup_token_capture( control_timeout_s=generation_cut_control_timeout_s, progress_reader=self._progress_reader, prefix_row_extras=self._generation_cut_record_payload, - restore_media=self._restore_generation_cut_media, ) ) if not capture_media: @@ -1250,26 +1219,40 @@ def _generation_cut_record_payload( MEDIA_TENSOR_DIGESTS_FIELD: media.tensor_digests, } - def _restore_generation_cut_media( - self, chunks: Sequence[Any] + def _restore_generation_cut_media(self, snapshot: Any) -> CapturedMedia: + """Read a restored call's media items and tensor digests from its verified prefix. + + The prefix digest Gym checked covers these extras, so only a bug in our + own writer can leave them unreadable. + """ + items = _media_items_from_extras( + snapshot.extras, + snapshot, + missing_message="Generation-prefix media metadata is missing", + ) + return CapturedMedia( + items=items, + tensors=None, + tensor_digests=snapshot.extras[MEDIA_TENSOR_DIGESTS_FIELD], + ) + + def _capture_restored_media( + self, + engine_prompt: dict[str, Any], + *, + admission: Any, + splice: PrefixSplice, + snapshot: Any, ) -> CapturedMedia | None: - """Read one call's media items and tensor digests from its first prefix row.""" + """Reprocess a restored call's media and check it against the digests its cut saved.""" if not self._capture_media: return None - first = chunks[0] - items = _media_items_from_extras( - first.extras, - first.snapshot, - missing_message="Generation-prefix media metadata is missing", + return self._capture_request_media( + engine_prompt, + admission=admission, + splice=splice, + restored=self._restore_generation_cut_media(snapshot), ) - digests = first.extras.get(MEDIA_TENSOR_DIGESTS_FIELD) - if not isinstance(digests, dict): - raise MediaCaptureRejected("Generation-prefix media digests are missing") - if bool(items) != bool(digests): - raise MediaCaptureRejected( - "Generation-prefix media metadata and digests disagree" - ) - return CapturedMedia(items=items, tensors=None, tensor_digests=digests) def _fetch_chain_prefix(self, staging_chain: list[str]) -> list[int]: """Resolve a staging chain through the shared, cached TQ read.""" @@ -1759,14 +1742,13 @@ async def preprocess_chat( verify_retained_media = decision.verify_retained_media capture_prefix_token_ids: list[int] | None = None generation_cut = None - restored_generation_media = None resumed_generation_token_ids: list[int] = [] restored_request_output_tokens: int | None = None if serving_admission is not None: capture_prefix_token_ids = await asyncio.to_thread( worker_self._resolve_admission_prefix, serving_admission ) - resolved_admission, resolved_cut = await asyncio.to_thread( + resolved_admission, generation_cut = await asyncio.to_thread( worker_self._prefix_recovery.resolve_restore, serving_admission, capture_prefix_token_ids, @@ -1776,9 +1758,6 @@ async def preprocess_chat( if serving_admission is admission: admission = resolved_admission serving_admission = resolved_admission - if resolved_cut is not None: - generation_cut = resolved_cut.snapshot - restored_generation_media = resolved_cut.media engine_prefix_token_ids = list(capture_prefix_token_ids) if generation_cut is not None: plan = plan_restore( @@ -1952,17 +1931,12 @@ async def preprocess_chat( template_prefix_token_ids=actual_corresponding_token_ids, template_token_ids=engine_prompt["prompt_token_ids"], ) - _verify_resumed_media_prompt( - media_splice.token_ids, - prefix_token_ids=list(capture_prefix_token_ids or ()), - generation_cut=generation_cut, - ) media = await asyncio.to_thread( - worker_self._capture_request_media, + worker_self._capture_restored_media, engine_prompt, admission=admission, splice=media_splice, - restored=restored_generation_media, + snapshot=generation_cut, ) # Must precede the reassignment below: the remap reads the # current value of prompt_token_ids as the template-coordinate diff --git a/tests/functional/_gym_prefix_recovery_phases.sh b/tests/functional/_gym_prefix_recovery_phases.sh index ed549378191..6aff50cdc49 100644 --- a/tests/functional/_gym_prefix_recovery_phases.sh +++ b/tests/functional/_gym_prefix_recovery_phases.sh @@ -13,8 +13,8 @@ # limitations under the License. # Phase control shared by the prefix-recovery functional tests. Source it after -# setting TEST_DIR; a test starts phase one in its own process group and keeps -# its PID in PHASE1_PID. +# setting PROJECT_ROOT and TEST_DIR; a test starts phase one in its own process +# group and keeps its PID in PHASE1_PID. PHASE1_PID="" @@ -39,6 +39,26 @@ stop_phase1() { done } +# Recover exactly the selected prefix: keep only the selected snapshot and no +# trainer checkpoint, either of which would represent work performed after the +# simulated crash. Sets SNAPSHOT_DIR to the restored snapshot. +restore_selected_snapshot() { + local selection_file=$1 selected_backup=$2 checkpoint_dir=$3 + local snapshot_root trainer_checkpoint + SNAPSHOT_DIR=$(uv run --directory "${PROJECT_ROOT}" --no-sync python -c \ + 'import json, sys; print(json.load(open(sys.argv[1]))["snapshot"])' \ + "${selection_file}") + snapshot_root=$(dirname "${SNAPSHOT_DIR}") + rm -rf "${snapshot_root}" + mkdir -p "${snapshot_root}" + cp -a "${selected_backup}" "${SNAPSHOT_DIR}" + for trainer_checkpoint in "${checkpoint_dir}"/step_*; do + if [[ -d "${trainer_checkpoint}" ]]; then + rm -rf "${trainer_checkpoint}" + fi + done +} + # On exit, stop phase one and delete the given checkpoint artifacts if the test # passed, unless SC_PREFIX_RECOVERY_KEEP_CHECKPOINTS=1. install_phase_cleanup() { diff --git a/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh b/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh index 6f6dc77b6a2..f9a480a7121 100755 --- a/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh +++ b/tests/functional/grpo_async_gym_single_controller_prefix_recovery.sh @@ -217,21 +217,7 @@ uv run --directory "$PROJECT_ROOT" --no-sync python "$SNAPSHOT_HELPER" select \ stop_phase1 cp "$BASE_RUN_LOG" "$PHASE1_LOG" -SNAPSHOT_DIR=$(uv run --directory "$PROJECT_ROOT" --no-sync python -c \ - 'import json, sys; print(json.load(open(sys.argv[1]))["snapshot"])' \ - "$SELECTION_FILE") -SNAPSHOT_ROOT=$(dirname "$SNAPSHOT_DIR") - -# Recover exactly the prefix selected above. A newer periodic snapshot or a -# trainer checkpoint would represent work performed after the simulated crash. -rm -rf "$SNAPSHOT_ROOT" -mkdir -p "$SNAPSHOT_ROOT" -cp -a "$SELECTED_BACKUP" "$SNAPSHOT_DIR" -for trainer_checkpoint in "$CHECKPOINT_DIR"/step_*; do - if [[ -d "$trainer_checkpoint" ]]; then - rm -rf "$trainer_checkpoint" - fi -done +restore_selected_snapshot "$SELECTION_FILE" "$SELECTED_BACKUP" "$CHECKPOINT_DIR" # Avoid cutting the replacement request again while verifying that it consumes # the selected prefix and reaches normal completion. diff --git a/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh index beb77799c85..eb82a8f44b0 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_prefix_recovery_single_controller_1n2g.sh @@ -160,40 +160,14 @@ PY source "${SCRIPT_DIR}/_gym_prefix_recovery_phases.sh" install_phase_cleanup "${CHECKPOINT_DIR}" "${SELECTED_BACKUP}" -# The Omni CLEVR token-capture topology (one Megatron training GPU with a frozen -# language model, one async vLLM GPU) plus the text prefix-recovery settings. +# The text prefix-recovery settings, on the YAML's Omni CLEVR topology (one +# Megatron training GPU with a frozen language model, one async vLLM GPU). COMMON_OVERRIDES=( - cluster.num_nodes=1 - cluster.gpus_per_node=2 - ++cluster.segment_size=1 # Quoted for Hydra: a bare compute capability (9.0, 10.0) parses as a # float, which the string-typed env_vars field rejects. policy.megatron_cfg.env_vars.TORCH_CUDA_ARCH_LIST="'${TORCH_CUDA_ARCH_LIST}'" - policy.megatron_cfg.tensor_model_parallel_size=1 - policy.megatron_cfg.expert_model_parallel_size=1 - policy.megatron_cfg.expert_tensor_parallel_size=1 - policy.megatron_cfg.context_parallel_size=1 - policy.megatron_cfg.sequence_parallel=true - policy.megatron_cfg.activation_checkpointing=true - ++policy.megatron_cfg.freeze_config.freeze_language_model=true - +policy.megatron_cfg.bias_dropout_fusion=false - policy.megatron_cfg.optimizer.optimizer_cpu_offload=false - policy.megatron_cfg.optimizer.optimizer_offload_fraction=0.0 - ++policy.megatron_cfg.optimizer.params_dtype=bfloat16 - ++policy.megatron_cfg.optimizer.main_grads_dtype=bfloat16 - ++policy.megatron_cfg.optimizer.main_params_dtype=float16 - ++policy.megatron_cfg.optimizer.exp_avg_dtype=bfloat16 - ++policy.megatron_cfg.optimizer.exp_avg_sq_dtype=bfloat16 - ++policy.megatron_cfg.optimizer.store_param_remainders=false - policy.generation.colocated.enabled=false - policy.generation.colocated.resources.num_nodes=1 - policy.generation.colocated.resources.gpus_per_node=1 policy.generation.max_new_tokens="${POLICY_MAX_NEW_TOKENS}" - policy.generation.vllm_cfg.tensor_parallel_size=1 - policy.generation.vllm_cfg.expert_parallel_size=1 - policy.generation.vllm_cfg.pipeline_parallel_size=1 policy.generation.vllm_cfg.max_model_len="${MAX_MODEL_LEN}" - policy.generation.vllm_cfg.gpu_memory_utilization=0.6 ++policy.generation.vllm_kwargs.max_num_batched_tokens="${MAX_MODEL_LEN}" # Near-greedy decoding keeps each list (and so each decode) predictable. policy.generation.temperature=1.0 @@ -201,11 +175,8 @@ COMMON_OVERRIDES=( # The trainer replays the top_p filter, so it needs vLLM's logprobs after # that filter. ++policy.generation.vllm_cfg.logprobs_mode=processed_logprobs - ++async_rl.generation_fleet_health.refit_timeout_s=null policy.max_total_sequence_length="${MAX_MODEL_LEN}" policy.train_global_batch_size="${TRAIN_GLOBAL_BATCH_SIZE}" - policy.train_micro_batch_size=1 - data_plane.simple.num_storage_units=2 # Keep each step's prompt membership fixed across runs. data.shuffle=false data.train.data_path="${TEST_DATA}" @@ -214,9 +185,6 @@ COMMON_OVERRIDES=( grpo.num_prompts_per_step="${NUM_PROMPTS}" grpo.num_generations_per_prompt="${NUM_GENERATIONS}" grpo.max_num_steps="${MAX_STEPS}" - grpo.val_period=0 - grpo.val_at_start=false - grpo.val_at_end=false async_rl.sampler.name=in_order async_rl.sampler.max_lookahead_versions=0 async_rl.min_groups_for_streaming_train="${NUM_PROMPTS}" @@ -245,9 +213,6 @@ COMMON_OVERRIDES=( # Each run dumps its trained rows (train_data_step*.jsonl in its log dir) # for verify-completion. ++async_rl.log_full_train_data=true - logger.tensorboard_enabled=true - logger.wandb_enabled=false - logger.monitor_gpus=false ) OMNI_CMD=( @@ -278,21 +243,7 @@ uv run --no-sync python "${SNAPSHOT_HELPER}" select \ --min-cut-groups "${NUM_PROMPTS}" stop_phase1 -SNAPSHOT_DIR=$(uv run --no-sync python -c \ - 'import json, sys; print(json.load(open(sys.argv[1]))["snapshot"])' \ - "${SELECTION_FILE}") -SNAPSHOT_ROOT=$(dirname "${SNAPSHOT_DIR}") - -# Recover exactly the prefix selected above. A newer periodic snapshot or a -# trainer checkpoint would represent work performed after the simulated crash. -rm -rf "${SNAPSHOT_ROOT}" -mkdir -p "${SNAPSHOT_ROOT}" -cp -a "${SELECTED_BACKUP}" "${SNAPSHOT_DIR}" -for trainer_checkpoint in "${CHECKPOINT_DIR}"/step_*; do - if [[ -d "${trainer_checkpoint}" ]]; then - rm -rf "${trainer_checkpoint}" - fi -done +restore_selected_snapshot "${SELECTION_FILE}" "${SELECTED_BACKUP}" "${CHECKPOINT_DIR}" echo "=== Phase 2: reprocess the images, verify them, and finish each suffix ===" # Do not cut the replacement requests again while they consume the prefix. diff --git a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh index 2d265e6153b..47274355ddd 100755 --- a/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh +++ b/tests/functional/nemotron_omni_clevr_vllm_token_capture_single_controller_1n2g.sh @@ -90,60 +90,26 @@ for path, count in ((os.environ["TRAIN_PATH"], 64), (os.environ["VAL_PATH"], 2)) output.write(json.dumps(sample(index)) + "\n") PY -# SingleController requires disaggregated generation: one GPU trains the -# frozen-decoder policy, one GPU hosts async vLLM (TP1) with the capture host. +# The YAML pins the disaggregated split SingleController requires: one GPU +# trains the frozen-decoder policy, one hosts async vLLM (TP1) with the capture +# host. uv run --no-sync python examples/run_grpo_single_controller.py \ --config examples/nemo_gym/vlm_grpo_nemotron_omni_clevr_vllm_token_capture_single_controller.yaml \ - cluster.num_nodes=1 \ - cluster.gpus_per_node=2 \ - ++cluster.segment_size=1 \ policy.megatron_cfg.env_vars.TORCH_CUDA_ARCH_LIST="${TORCH_CUDA_ARCH_LIST}" \ - policy.megatron_cfg.tensor_model_parallel_size=1 \ - policy.megatron_cfg.expert_model_parallel_size=1 \ - policy.megatron_cfg.expert_tensor_parallel_size=1 \ - policy.megatron_cfg.context_parallel_size=1 \ - policy.megatron_cfg.sequence_parallel=true \ - policy.megatron_cfg.activation_checkpointing=true \ - ++policy.megatron_cfg.freeze_config.freeze_language_model=true \ - +policy.megatron_cfg.bias_dropout_fusion=false \ - policy.megatron_cfg.optimizer.optimizer_cpu_offload=false \ - policy.megatron_cfg.optimizer.optimizer_offload_fraction=0.0 \ - ++policy.megatron_cfg.optimizer.params_dtype=bfloat16 \ - ++policy.megatron_cfg.optimizer.main_grads_dtype=bfloat16 \ - ++policy.megatron_cfg.optimizer.main_params_dtype=float16 \ - ++policy.megatron_cfg.optimizer.exp_avg_dtype=bfloat16 \ - ++policy.megatron_cfg.optimizer.exp_avg_sq_dtype=bfloat16 \ - ++policy.megatron_cfg.optimizer.store_param_remainders=false \ - policy.generation.colocated.enabled=false \ - policy.generation.colocated.resources.num_nodes=1 \ - policy.generation.colocated.resources.gpus_per_node=1 \ policy.generation.max_new_tokens=128 \ - policy.generation.vllm_cfg.tensor_parallel_size=1 \ - policy.generation.vllm_cfg.expert_parallel_size=1 \ - policy.generation.vllm_cfg.pipeline_parallel_size=1 \ policy.generation.vllm_cfg.max_model_len=1024 \ - policy.generation.vllm_cfg.gpu_memory_utilization=0.6 \ ++policy.generation.vllm_kwargs.max_num_batched_tokens=1024 \ - ++async_rl.generation_fleet_health.refit_timeout_s=null \ policy.max_total_sequence_length=1024 \ data.train.data_path="${TRAIN_PATH}" \ data.validation.data_path="${VAL_PATH}" \ grpo.num_prompts_per_step=1 \ grpo.num_generations_per_prompt=2 \ grpo.max_num_steps=1 \ - grpo.val_period=0 \ - grpo.val_at_start=false \ - grpo.val_at_end=false \ policy.train_global_batch_size=2 \ - policy.train_micro_batch_size=1 \ - data_plane.simple.num_storage_units=2 \ async_rl.min_groups_for_streaming_train=1 \ async_rl.max_inflight_prompts=2 \ async_rl.max_buffered_rollouts=2 \ - logger.tensorboard_enabled=true \ logger.log_dir="${LOG_DIR}" \ - logger.wandb_enabled=false \ - logger.monitor_gpus=false \ checkpointing.enabled=false \ "$@" 2>&1 | tee "${RUN_LOG}" diff --git a/tests/unit/data/test_captured_media.py b/tests/unit/data/test_captured_media.py index 808c7ac89c4..34e77b020d5 100644 --- a/tests/unit/data/test_captured_media.py +++ b/tests/unit/data/test_captured_media.py @@ -71,7 +71,6 @@ from nemo_rl.models.generation.prefix_recovery import PrefixRecovery from nemo_rl.models.generation.vllm.vllm_worker_async import ( VllmAsyncGenerationWorkerImpl, - _verify_resumed_media_prompt, ) pytestmark = pytest.mark.nemo_gym @@ -858,7 +857,7 @@ def test_generation_prefix_restore_verifies_media_by_digest_without_pixels( [chunk] = source.fetch_for_finalization([staging_key]) assert not chunk.media_present restored = VllmAsyncGenerationWorkerImpl._restore_generation_cut_media( - worker, [chunk] + worker, _verified_snapshot(chunk) ) assert restored == CapturedMedia( items=media.items, tensors=None, tensor_digests=media.tensor_digests @@ -876,7 +875,9 @@ def test_generation_prefix_restore_verifies_media_by_digest_without_pixels( ) torch.testing.assert_close(verified.tensors["imgs"], media.tensors["imgs"]) - with pytest.raises(MediaCaptureRejected, match="tensor 'imgs' bytes") as error: + with pytest.raises( + MediaCaptureRejected, match=r"changed tensors: \['imgs'\]" + ) as error: VllmAsyncGenerationWorkerImpl._capture_request_media( worker, make_prompt(tokens, torch.full_like(pixels, 9)), @@ -897,89 +898,42 @@ def _media_prefix_chunk(dp): return worker, chunk -def _without(extras, key): - return {name: value for name, value in extras.items() if name != key} - - -def _off_prompt_spans(extras): - span = {**extras[MEDIA_SPANS_FIELD][0], "placeholder_digest": "0" * 64} - return {**extras, MEDIA_SPANS_FIELD: [span]} +def _verified_snapshot(chunk): + """A one-chunk prefix as resolve_restore verifies it: the chunk's record with its extras.""" + return SimpleNamespace( + extras=chunk.extras, + token_ids_delta=chunk.snapshot.token_ids_delta, + prev_len=chunk.snapshot.prev_len, + ) -@pytest.mark.parametrize( - ("corrupt", "match"), - [ - (lambda extras: None, "media metadata is missing"), - ( - lambda extras: _without(extras, MEDIA_SPANS_FIELD), - "media metadata is missing", - ), - ( - lambda extras: {**extras, MEDIA_SPANS_FIELD: {}}, - "Media spans are malformed", - ), - ( - lambda extras: _without(extras, MEDIA_TENSOR_DIGESTS_FIELD), - "media digests are missing", - ), - ( - lambda extras: {**extras, MEDIA_TENSOR_DIGESTS_FIELD: {}}, - "metadata and digests disagree", - ), - ( - lambda extras: {**extras, MEDIA_SPANS_FIELD: []}, - "metadata and digests disagree", - ), - (_off_prompt_spans, "does not match the exact carried prompt"), - ], - ids=[ - "no_extras", - "no_spans", - "spans_not_a_list", - "no_digests", - "spans_without_digests", - "digests_without_spans", - "spans_off_the_prompt", - ], -) -def test_generation_prefix_media_restore_fails_closed_on_corrupt_rows( - dp, corrupt, match -): +def test_generation_prefix_media_restore_fails_closed_without_extras(dp): + # The prefix digest covers the extras, so only a writer bug drops them. worker, chunk = _media_prefix_chunk(dp) - with pytest.raises(MediaCaptureRejected, match=match): + with pytest.raises(MediaCaptureRejected, match="media metadata is missing"): VllmAsyncGenerationWorkerImpl._restore_generation_cut_media( - worker, [replace(chunk, extras=corrupt(chunk.extras))] + worker, _verified_snapshot(replace(chunk, extras=None)) ) def test_generation_cut_rebuild_rejects_extras_after_the_first_chunk(dp): - worker, chunk = _media_prefix_chunk(dp) + _, chunk = _media_prefix_chunk(dp) later = replace(chunk, extras={MEDIA_SPANS_FIELD: []}) - - # The check runs before the admission or recovery state is consulted. - with pytest.raises(RuntimeError, match="only the first generation-cut chunk"): - PrefixRecovery._rebuild_prefix( - SimpleNamespace(), SimpleNamespace(generation_cut=None), [], [chunk, later] + recovery = SimpleNamespace( + _parts=SimpleNamespace( + staging_source=SimpleNamespace( + fetch_for_finalization=lambda keys: [chunk, later] + ) ) - - -def test_resumed_media_prompt_must_equal_prefix_plus_cut_prompt_tokens(): - # One prompt-delta token run, then a generated token the check ignores. - cut = SimpleNamespace( - token_ids_delta=[18, 18, 11, 31], token_mask_delta=[0.0, 0.0, 0.0, 1.0] ) - - _verify_resumed_media_prompt( - [10, 18, 18, 11], prefix_token_ids=[10], generation_cut=cut + admission = SimpleNamespace( + generation_cut=SimpleNamespace(staging_keys=("chunk-0", "chunk-1")) ) - with pytest.raises( - MediaCaptureRejected, match="differs from the durable generation prefix" - ) as error: - _verify_resumed_media_prompt( - [10, 18, 11, 11], prefix_token_ids=[10], generation_cut=cut - ) - assert error.value.code == "retained_media_changed" + + # The check runs before the admission or recovery state is consulted. + with pytest.raises(RuntimeError, match="only the first generation-cut chunk"): + PrefixRecovery._rebuild_prefix(recovery, admission, []) def _image_media() -> CapturedMedia: @@ -992,55 +946,34 @@ def _image_media() -> CapturedMedia: @pytest.mark.parametrize( ("change", "match"), [ - (lambda media: replace(media, items=()), "metadata"), ( - lambda media: replace( - media, - tensors={ - **media.tensors, - "num_frames": torch.tensor([1], dtype=torch.int32), - }, - ), - "tensor set", - ), - ( - lambda media: replace( - media, - tensors={**media.tensors, "imgs": media.tensors["imgs"].double()}, - ), - "tensor 'imgs' dtype or shape", + lambda media: replace(media, items=()), + r"items equal: False; changed tensors: \[\]", ), ( lambda media: replace( media, tensors={**media.tensors, "imgs": media.tensors["imgs"] + 1}, ), - "tensor 'imgs' bytes", + r"items equal: True; changed tensors: \['imgs'\]", ), ], - ids=["items", "tensor_set", "dtype_or_shape", "bytes"], + ids=["items", "bytes"], ) -def test_verify_restored_media_rejects_each_kind_of_change(change, match): - media = _image_media() - restored = CapturedMedia( - items=media.items, - tensors=None, - tensor_digests=media_tensor_digests(media.tensors), - ) +def test_verify_restored_media_rejects_changed_items_or_tensors(change, match): + def hashed(media: CapturedMedia) -> CapturedMedia: + # As reprocessing does whenever prefix cuts are on. + return replace(media, tensor_digests=media_tensor_digests(media.tensors)) + + media = hashed(_image_media()) + restored = replace(media, tensors=None) verify_restored_media(media, restored) with pytest.raises(MediaCaptureRejected, match=match) as error: - verify_restored_media(change(media), restored) + verify_restored_media(hashed(change(media)), restored) assert error.value.code == "retained_media_changed" -def test_verify_restored_media_requires_staged_digests(): - media = _image_media() - - with pytest.raises(MediaCaptureRejected, match="no tensor digests"): - verify_restored_media(media, replace(media, tensors=None)) - - def test_media_tensor_digests_fingerprint_dtype_shape_and_bytes(): pixels = torch.arange(6, dtype=torch.bfloat16).reshape(2, 3) digests = media_tensor_digests({"imgs": pixels, "imgs_sizes": torch.ones(1, 2)}) @@ -1719,19 +1652,26 @@ async def create_chat_completion(request, raw_request): _BUILT["chat"][0].create_chat_completion = create_chat_completion handler = dict(app.routes)["/v1/chat/completions"] - request = SimpleNamespace( - top_k=-1, - min_p=None, - repetition_penalty=None, - presence_penalty=0.0, - frequency_penalty=0.0, - logit_bias=None, - top_p=1.0, - temperature=1.0, - ng_capture=admission.model_dump(), - required_prefix_token_ids=None, - model_copy=lambda **kwargs: request, - ) + + def chat_request(ng_capture, **fields): + """A chat request with the sampling settings the server accepts.""" + request = SimpleNamespace( + top_k=-1, + min_p=None, + repetition_penalty=None, + presence_penalty=0.0, + frequency_penalty=0.0, + logit_bias=None, + top_p=1.0, + temperature=1.0, + ng_capture=ng_capture, + required_prefix_token_ids=None, + **fields, + ) + request.model_copy = lambda **kwargs: request + return request + + request = chat_request(admission.model_dump()) # The corresponding template prefix has no suffix user text. calls = 0 @@ -1787,6 +1727,73 @@ async def run(): assert isinstance(failure, GymTransportError) assert "retained_media_changed" in str(failure) + # A restored call reprocesses its image and checks it against the digests + # its cut saved, before begin_call and inference: other pixels now are + # rejected instead of resuming a prefix sampled on the old ones. + worker.token_capture.reset_mock() + inference.reset_mock() + worker._hash_captured_media = True + cut_tokens = [10, 18, 18, 11] + staged_pixels = torch.ones(3, 2, 3) + staged = capture_processed_media( + engine_prompt(cut_tokens, [(Span(1, 2), staged_pixels)]), prev_len=0 + ) + cut = SimpleNamespace( + token_ids_delta=[*cut_tokens, 31], + token_mask_delta=[0.0] * len(cut_tokens) + [1.0], + prev_len=0, + extras={ + MEDIA_SPANS_FIELD: [item.to_dict() for item in staged.items], + MEDIA_TENSOR_DIGESTS_FIELD: media_tensor_digests(staged.tensors), + }, + ) + restored_capture = { + **CaptureAdmission( + capture_key="r1", model_call_id="c9", prev_len=0, mode="text" + ).model_dump(), + "generation_cut": { + "source_capture_key": "r1", + "source_model_call_id": "c9", + "staging_keys": ["active/checkpoint-1/r1/c9/0"], + "prefix_token_count": 1, + "prefix_digest": "0" * 64, + "effective_output_limit": 8, + }, + } + restored_request = chat_request( + restored_capture, max_tokens=8, max_completion_tokens=None, min_tokens=None + ) + # resolve_restore's verified snapshot, as on main. + monkeypatch.setattr( + worker._prefix_recovery, + "resolve_restore", + lambda admission, prefix_token_ids: (admission, cut), + ) + monkeypatch.setattr( + renderer, "model_config", SimpleNamespace(max_model_len=64), raising=False + ) + renders = 0 + + async def reprocess_other_pixels(self, **kwargs): + nonlocal renders + renders += 1 + # The request's own render, then its template prefix (none: no turn + # precedes the call). + if renders == 1: + return [], [engine_prompt(cut_tokens, [(Span(1, 2), staged_pixels + 1)])] + return [], [engine_prompt([])] + + monkeypatch.setattr( + _OnlineRenderer, "preprocess_chat", reprocess_other_pixels, raising=False + ) + response = await handler(restored_request, MagicMock()) + assert response.status_code == 400 + error = json.loads(response.body)["error"] + assert error["code"] == "retained_media_changed" + assert "changed tensors: ['imgs']" in error["message"] + inference.assert_not_awaited() + worker.token_capture.begin_call.assert_not_called() + # A recovery request whose original URL/path/blob has disappeared fails # while vLLM renders its engine-native prompt: before any TQ read and # before begin_call. Prefix rows hold only tensor digests, so there are no @@ -1806,14 +1813,7 @@ async def run(): "effective_output_limit": 8, }, } - missing_request = SimpleNamespace( - top_k=-1, - top_p=1.0, - temperature=1.0, - ng_capture=recovery_capture, - required_prefix_token_ids=None, - model_copy=lambda **kwargs: missing_request, - ) + missing_request = chat_request(recovery_capture) async def missing_media(self, **kwargs): raise FileNotFoundError("original media disappeared") @@ -1829,7 +1829,7 @@ async def missing_media(self, **kwargs): inference.assert_not_awaited() worker.token_capture.begin_call.assert_not_called() worker.token_capture.complete_call_from_response.assert_not_called() - assert worker._capture_calls == {} + assert len(worker._capture_registry) == 0 @pytest.mark.parametrize("routes", [False, True]) diff --git a/tests/unit/models/generation/test_vllm_generation.py b/tests/unit/models/generation/test_vllm_generation.py index 06cecf480ca..212010c8bf5 100644 --- a/tests/unit/models/generation/test_vllm_generation.py +++ b/tests/unit/models/generation/test_vllm_generation.py @@ -1606,7 +1606,6 @@ def checkpoint_cut(inventory, *, deadline): control_timeout_s=timeout_s, progress_reader=None, prefix_row_extras=lambda state: None, - restore_media=lambda chunks: None, ) ) recovery.checkpoint_cut = checkpoint_cut diff --git a/tests/unit/models/generation/test_vllm_token_capture_hosting.py b/tests/unit/models/generation/test_vllm_token_capture_hosting.py index d616c2f169c..476e737dbd5 100644 --- a/tests/unit/models/generation/test_vllm_token_capture_hosting.py +++ b/tests/unit/models/generation/test_vllm_token_capture_hosting.py @@ -59,7 +59,6 @@ MEDIA_TENSOR_DIGESTS_FIELD, CapturedMedia, CapturedMediaItem, - MediaCaptureRejected, MediaColumnSpec, media_tensor_digests, ) @@ -202,7 +201,6 @@ def _cuts( control_timeout_s=None, progress_reader=progress_reader or _NoCallsInVllm(), prefix_row_extras=worker._generation_cut_record_payload, - restore_media=worker._restore_generation_cut_media, ) ) worker._prefix_recovery = recovery @@ -587,15 +585,19 @@ def replay(tokenizer, **kwargs): def _captured_text_call( - worker, model_call_id: str = "c1", prompt_token_ids: tuple[int, ...] = (10, 11) + worker, + model_call_id: str = "c1", + prompt_token_ids: tuple[int, ...] = (10, 11), + *, + media: CapturedMedia | None = None, ) -> _FakeRequest: - """Begin capturing a parentless text call of rollout ``r0``.""" + """Begin capturing a parentless text-mode call of rollout ``r0``.""" request = _FakeRequest( ng_capture={"rollout_id": "r0", "model_call_id": model_call_id, "mode": "text"}, stream=False, ) VllmAsyncGenerationWorkerImpl._begin_request_capture( - worker, request, list(prompt_token_ids) + worker, request, list(prompt_token_ids), media=media ) return request @@ -607,9 +609,10 @@ def _live_call( *, model_call_id: str = "c1", prompt_token_ids: tuple[int, ...] = (10, 11), + media: CapturedMedia | None = None, ): """Begin one captured call that has decoded ``generated_token_ids``.""" - request = _captured_text_call(worker, model_call_id, prompt_token_ids) + request = _captured_text_call(worker, model_call_id, prompt_token_ids, media=media) state = worker._capture_registry.get(request) state.set_effective_output_limit(8) state.record(generated_token_ids, logprobs) @@ -1758,7 +1761,6 @@ def _record_cut_constraints(state, sampling_params) -> None: control_timeout_s=None, progress_reader=_NoCallsInVllm(), prefix_row_extras=lambda state: None, - restore_media=lambda chunks: None, ) ).on_sampling_params(object(), sampling_params) @@ -1912,33 +1914,19 @@ def _captured_image_media() -> CapturedMedia: ) -def _worker_with_active_media_prefix( - sink: _MemorySink, media: CapturedMedia | None = None -): - worker = _worker_with_capture(sink) - request = _FakeRequest( - ng_capture={ - "rollout_id": "r0", - "model_call_id": "c1", - "parent_call_id": None, - "prev_len": 0, - "mode": "text", - }, - stream=False, - ) +def _live_image_call(worker, media: CapturedMedia | None = None): + """An active call whose prompt is one image's two placeholder tokens.""" media = media or _captured_image_media() - VllmAsyncGenerationWorkerImpl._begin_request_capture( - worker, request, [18, 18], media=media + request, state = _live_call( + worker, [20], [-0.1], prompt_token_ids=(18, 18), media=media ) - state = worker._capture_registry.get(request) - state.set_effective_output_limit(8) - state.record([20], [-0.1]) - return worker, request, state, media + return request, state, media def test_generation_cut_stages_media_digests_only_with_first_prefix_row(): sink = _MemorySink() - worker, _, state, media = _worker_with_active_media_prefix(sink) + worker = _worker_with_capture(sink) + _, state, media = _live_image_call(worker) first_receipt = _cuts(worker).checkpoint_cut( _single_call_inventory(checkpoint_id="checkpoint-1") @@ -1966,8 +1954,9 @@ def test_generation_cut_stages_media_digests_only_with_first_prefix_row(): def test_generation_cut_refuses_media_captured_without_digests(caplog): sink = _MemorySink() - worker, _, state, _ = _worker_with_active_media_prefix( - sink, replace(_captured_image_media(), tensor_digests=None) + worker = _worker_with_capture(sink) + _, state, _ = _live_image_call( + worker, replace(_captured_image_media(), tensor_digests=None) ) with caplog.at_level(logging.ERROR): @@ -1983,38 +1972,97 @@ def test_generation_cut_refuses_media_captured_without_digests(caplog): assert state.snapshot().staged_token_count == 0 -def _worker_with_active_text_prefix(sink: _MemorySink): - worker = _worker_with_capture(sink) - request = _FakeRequest( +def _restore_request(ack, *, source=("r0", "c1"), model_call_id="c2") -> _FakeRequest: + """The call Gym sends to resume ``ack``'s cut of ``source``.""" + return _FakeRequest( ng_capture={ - "rollout_id": "r0", - "model_call_id": "c1", - "parent_call_id": None, - "prev_len": 0, + "rollout_id": "r0-a1", + "model_call_id": model_call_id, "mode": "text", + "generation_cut": { + "source_capture_key": source[0], + "source_model_call_id": source[1], + "staging_keys": list(ack.staging_keys), + "prefix_token_count": ack.prefix_token_count, + "prefix_digest": ack.prefix_digest, + "effective_output_limit": ack.effective_output_limit, + }, }, stream=False, ) - VllmAsyncGenerationWorkerImpl._begin_request_capture(worker, request, [10, 11]) - state = worker._capture_registry.get(request) - state.set_effective_output_limit(8) - state.record([20], [-0.1]) - return worker, state + + +def _cut_again_after_restore(sink, worker, ack, monkeypatch): + """Restore an image call's cut, reprocess its image, and cut the resumed call. + + The second cut's cumulative record takes its extras from the reprocessed + media, while a restore rebuilds them from the first run's row. + """ + records = { + key: record + for key, record in zip( + sink.generation_prefix_keys, + (record for _, record in sink.generation_prefix_records), + strict=True, + ) + } + resumed = _resumed_worker( + sink, records=records, weight_version=worker._rollout_weight_version + ) + resumed._capture_media = resumed._hash_captured_media = True + request = _restore_request(ack) + admission, snapshot = _cuts(resumed).resolve_restore( + resumed._capture_admission(request), [] + ) + assert snapshot is not None + # vLLM reprocesses the same image; the worker hashes it and checks it + # against the digests the first run's row saved. + monkeypatch.setattr( + "nemo_rl.models.generation.vllm.vllm_worker_async.capture_processed_media", + lambda *args, **kwargs: replace(_captured_image_media(), tensor_digests=None), + ) + media = resumed._capture_restored_media( + {"prompt_token_ids": [18, 18, 20]}, + admission=admission, + splice=None, + snapshot=snapshot, + ) + VllmAsyncGenerationWorkerImpl._begin_request_capture( + resumed, + request, + [18, 18, 20], + admission=admission, + prefix_token_ids=[], + media=media, + generation_cut=snapshot, + resumed_generation_token_ids=[20], + ) + resumed._capture_registry.get(request).record([21], [-0.2]) + (second,) = ( + _cuts(resumed) + .checkpoint_cut( + _single_call_inventory("c2", checkpoint_id="checkpoint-2", attempt=1) + ) + .prefixes + ) + # The first run's row, then the resumed call's new chunk. + assert len(second.staging_keys) == 2 + assert second.staging_keys[0] == ack.staging_keys[0] + return second @pytest.mark.parametrize( - ("media", "cuts"), - [(False, 1), (True, 1), (True, 2)], - ids=["text", "image_one_cut", "image_two_cuts"], + ("cuts", "resumed"), + [(1, False), (2, False), (1, True)], + ids=["image_one_cut", "image_two_cuts", "image_resumed_then_cut"], ) def test_generation_cut_digest_written_at_the_cut_is_the_one_restore_rebuilds( - media: bool, cuts: int + cuts: int, resumed: bool, monkeypatch ): + # test_usable_restored_prefix_is_resumed covers a text call's digest. sink = _MemorySink() - if media: - worker, _, state, _ = _worker_with_active_media_prefix(sink) - else: - worker, state = _worker_with_active_text_prefix(sink) + worker = _worker_with_capture(sink) + _, state, _ = _live_image_call(worker) for index in range(cuts): if index: state.record([21], [-0.2]) @@ -2022,6 +2070,10 @@ def test_generation_cut_digest_written_at_the_cut_is_the_one_restore_rebuilds( _single_call_inventory(checkpoint_id=f"checkpoint-{index + 1}") ) (ack,) = receipt.prefixes + source = ("r0", "c1") + if resumed: + ack = _cut_again_after_restore(sink, worker, ack, monkeypatch) + source = ("r0-a1", "c2") records = dict( zip( sink.generation_prefix_keys, @@ -2031,45 +2083,28 @@ def test_generation_cut_digest_written_at_the_cut_is_the_one_restore_rebuilds( ) assert list(records) == list(ack.staging_keys) first = records[ack.staging_keys[0]] - assert (MEDIA_TENSOR_DIGESTS_FIELD in (first.extras or {})) is media + assert MEDIA_TENSOR_DIGESTS_FIELD in first.extras def resolve(staged: dict[str, StagedCallRecord]): resumed = _resumed_worker( sink, records=staged, weight_version=worker._rollout_weight_version ) - request = _FakeRequest( - ng_capture={ - "rollout_id": "r0-a1", - "model_call_id": "c2", - "mode": "text", - "generation_cut": { - "source_capture_key": "r0", - "source_model_call_id": "c1", - "staging_keys": list(ack.staging_keys), - "prefix_token_count": ack.prefix_token_count, - "prefix_digest": ack.prefix_digest, - "effective_output_limit": ack.effective_output_limit, - }, - }, - stream=False, - ) + request = _restore_request(ack, source=source) return _cuts(resumed)._rebuild_prefix(resumed._capture_admission(request), []) # Restore recomputes the cumulative digest from the staged chunks, the # first chunk's media extras included, and must reach the cut's digest: # Gym's begin_call rejects a restored snapshot whose digest differs. assert resolve(records).digest == ack.prefix_digest - if media: - # The tensor digests are covered: a changed one changes the digest. - digests = dict(first.extras[MEDIA_TENSOR_DIGESTS_FIELD]) - digests["imgs"] = {**digests["imgs"], "sha256": "0" * 64} - tampered = first.model_copy( - update={"extras": {**first.extras, MEDIA_TENSOR_DIGESTS_FIELD: digests}} - ) - assert ( - resolve({**records, ack.staging_keys[0]: tampered}).digest - != ack.prefix_digest - ) + # The tensor digests are covered: a changed one changes the digest. + digests = dict(first.extras[MEDIA_TENSOR_DIGESTS_FIELD]) + digests["imgs"] = {**digests["imgs"], "sha256": "0" * 64} + tampered = first.model_copy( + update={"extras": {**first.extras, MEDIA_TENSOR_DIGESTS_FIELD: digests}} + ) + assert ( + resolve({**records, ack.staging_keys[0]: tampered}).digest != ack.prefix_digest + ) class _BlockingPrefixSink(_MemorySink): @@ -2096,7 +2131,8 @@ def _assert_stages_captured_tensors(attachments, tensors) -> None: def test_media_terminal_completion_waits_for_prefix_cut_then_replaces_it(): sink = _BlockingPrefixSink() - worker, request, _, media = _worker_with_active_media_prefix(sink) + worker = _worker_with_capture(sink) + request, _, media = _live_image_call(worker) terminal_entered = threading.Event() terminal_finished = threading.Event() @@ -2131,43 +2167,10 @@ def complete(): assert sink.cleared_generation_prefix_keys == list(receipt.prefixes[0].staging_keys) -class _FailTerminalSink(_MemorySink): - def __init__(self) -> None: - super().__init__() - self.terminal_attempts: list[tuple[StagedCallRecord, dict | None]] = [] - - def stage( - self, record: StagedCallRecord, *, attachments: dict | None = None - ) -> StageResult: - self.terminal_attempts.append((record, attachments)) - return StageResult( - ok=False, - staging_key=record.staging_key, - error="injected terminal staging failure", - ) - - -def test_media_call_whose_terminal_staging_fails_clears_its_cut_rows(): - sink = _FailTerminalSink() - worker, request, state, media = _worker_with_active_media_prefix(sink) - receipt = _cuts(worker).checkpoint_cut( - _single_call_inventory(checkpoint_id="checkpoint-1") - ) - - content = VllmAsyncGenerationWorkerImpl._finish_request_capture( - worker, request, _served_content([20, 21], [-0.1, -0.2]) - ) - - assert content["ng_commit_coords"]["disposition"] == "capture_failed" - _assert_stages_captured_tensors(sink.terminal_attempts[0][1], media.tensors) - # As for a text call, the failed call's cut rows are retired with it. - _flush_retired_rows(worker) - assert sink.cleared_generation_prefix_keys == list(receipt.prefixes[0].staging_keys) - - def test_canonical_media_row_survives_prefix_cleanup_failure(caplog): sink = _FailingClearSink() - worker, request, _, media = _worker_with_active_media_prefix(sink) + worker = _worker_with_capture(sink) + request, _, media = _live_image_call(worker) _cuts(worker).checkpoint_cut(_single_call_inventory(checkpoint_id="checkpoint-1")) content = VllmAsyncGenerationWorkerImpl._finish_request_capture( @@ -2355,7 +2358,7 @@ def test_usable_restored_prefix_is_resumed(): served, resolved = _cuts(worker).resolve_restore(admission, []) assert served is admission - assert resolved.snapshot.digest == cut_record.digest + assert resolved.digest == cut_record.digest _flush_retired_rows(worker) assert sink.cleared_generation_prefix_keys == [] @@ -2366,7 +2369,6 @@ def test_usable_restored_prefix_is_resumed(): "missing chunk", "chunk newer than the worker", "digest mismatch", - "media row unreadable", ], ) def test_unusable_restored_prefix_regenerates_instead_of_failing(failure, caplog): @@ -2386,14 +2388,6 @@ def test_unusable_restored_prefix_regenerates_instead_of_failing(failure, caplog ) } ) - if failure == "media row unreadable": - # The media comes from the same fetch, inside the same fallback. - def unreadable_media(chunks): - raise MediaCaptureRejected("Generation-prefix media metadata is missing") - - worker._restore_generation_cut_media = unreadable_media - _cuts(worker, staging_source=worker._staging_source) - with caplog.at_level(logging.WARNING): served, resolved = _cuts(worker).resolve_restore(admission, []) @@ -2852,10 +2846,10 @@ def failed_put(key, fields, **kwargs): assert len(worker._capture_registry) == 0 -def _cut_active_call(worker, sink, model_call_id: str = "c1"): +def _cut_active_call(worker, sink, model_call_id: str = "c1", **call): """Begin one captured call and stage its prefix at a generation cut.""" request, state = _live_call( - worker, [20, 21], [-0.1, -0.2], model_call_id=model_call_id + worker, [20, 21], [-0.1, -0.2], model_call_id=model_call_id, **call ) inventory = _single_call_inventory(model_call_id) _cuts(worker).checkpoint_cut(inventory) @@ -2882,13 +2876,20 @@ class _FailingTerminalSink(_MemorySink): def stage( self, record: StagedCallRecord, *, attachments: dict | None = None ) -> StageResult: + self.attachments.append(attachments) return StageResult(ok=False, error="tq unavailable") -def test_call_whose_terminal_staging_fails_clears_its_cut_rows(): +@pytest.mark.parametrize("media", [False, True], ids=["text", "image"]) +def test_call_whose_terminal_staging_fails_clears_its_cut_rows(media: bool): sink = _FailingTerminalSink() worker = _worker_with_capture(sink) - request = _cut_active_call(worker, sink) + image = _captured_image_media() if media else None + request = _cut_active_call( + worker, + sink, + **({"prompt_token_ids": (18, 18), "media": image} if media else {}), + ) content = VllmAsyncGenerationWorkerImpl._finish_request_capture( worker, @@ -2897,6 +2898,8 @@ def test_call_whose_terminal_staging_fails_clears_its_cut_rows(): ) assert content["ng_commit_coords"]["disposition"] == "capture_failed" + if media: + _assert_stages_captured_tensors(sink.attachments[0], image.tensors) _flush_retired_rows(worker) assert sink.cleared_generation_prefix_keys == sink.generation_prefix_keys @@ -2965,6 +2968,10 @@ def test_omni_capture_setup_rejects_video_pruning( generation_prefix_cuts_enabled=True, generation_cut_control_token="secret", ) + # Prefix cuts need the vLLM HTTP server's client; the progress reader it + # would wrap is covered by the progress-read tests. + worker._http_engine_client = MagicMock(name="http_engine_client") + worker._progress_reader = MagicMock(name="progress_reader") if pruning_rate: with pytest.raises(ValueError, match="video token pruning"): asyncio.run( diff --git a/tests/unit/single_controller/test_setup.py b/tests/unit/single_controller/test_setup.py index a344835b47f..dc72e23c47c 100644 --- a/tests/unit/single_controller/test_setup.py +++ b/tests/unit/single_controller/test_setup.py @@ -829,6 +829,28 @@ def test_recovery_target_enables_required_checkpoint_capabilities( assert config.generation_prefix_cuts_enabled is generation_prefix_cuts +@contextlib.contextmanager +def _patched_gym_token_capture(finalizer_actors): + """Patch the Gym and finalizer actors a Gym token-capture setup builds. + + Yields the Gym spin-up and finalizer-creation mocks. + """ + with ( + patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), + patch.object( + sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() + ) as spinup, + patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), + patch.object(sc_setup_mod, "uses_image_placeholder", return_value=True), + patch( + "nemo_rl.experience.rollout_reassembler_actor." + "create_rollout_reassembler_actors", + return_value=finalizer_actors, + ) as create_finalizers, + ): + yield spinup, create_finalizers + + class TestSetup: """setup arg validation + actor_args assembly.""" @@ -1514,32 +1536,14 @@ def test_prefix_recovery_accepts_multimodal_capture( mc.rollout_checkpointing = RolloutCheckpointConfig( snapshot_attempt_interval_s=1.0 ) - patched_factories["setup_response_data"].return_value = ( - list(range(8)), - None, - ) + patched_factories["setup_response_data"].return_value = (list(range(8)), None) + tokenizer = MagicMock(pad_token_id=0) - with ( - patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), - patch.object( - sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() - ) as mock_spinup, - patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), - patch.object(sc_setup_mod, "uses_image_placeholder", return_value=True), - patch( - "nemo_rl.experience.rollout_reassembler_actor." - "create_rollout_reassembler_actors", - return_value=[MagicMock(name="finalizer")], - ), - ): - tokenizer = MagicMock(pad_token_id=0) + with _patched_gym_token_capture([MagicMock(name="finalizer")]): setup_single_controller( - mc, - tokenizer, - processor=MagicMock(tokenizer=tokenizer), + mc, tokenizer, processor=MagicMock(tokenizer=tokenizer) ) - assert mock_spinup.call_args.kwargs["turn_checkpointing_enabled"] is True generation, _ = patched_factories["_build_generation"].return_value _, setup_kwargs = generation.setup_token_capture.call_args assert setup_kwargs["capture_media"] is True @@ -2837,13 +2841,11 @@ def test_token_capture_always_creates_finalizer_actor_pool( processor = MagicMock(tokenizer=tokenizer) if with_processor else None with ( - patch.object(sc_setup_mod, "should_use_nemo_gym", return_value=True), - patch.object( - sc_setup_mod, "build_nemo_gym_actors", return_value=MagicMock() - ) as mock_spinup, + _patched_gym_token_capture(fake_actors) as ( + mock_spinup, + mock_create_finalizer_actors, + ), patch.object(sc_setup_mod, "validate_dataset_agent_coverage"), - patch.object(sc_setup_mod, "router_replay_enabled", return_value=False), - patch.object(sc_setup_mod, "uses_image_placeholder", return_value=True), patch.object( sc_setup_mod.opd_module, "reserve_teacher_clusters", @@ -2854,10 +2856,6 @@ def test_token_capture_always_creates_finalizer_actor_pool( "create_teacher_worker_groups", return_value=({"teacher": MagicMock()}, {"teacher": "teacher"}), ), - patch( - "nemo_rl.experience.rollout_reassembler_actor.create_rollout_reassembler_actors", - return_value=fake_actors, - ) as mock_create_finalizer_actors, ): actor_args, _ = setup_single_controller(mc, tokenizer, processor=processor)