Skip to content

Commit da03aaa

Browse files
committed
fix(checkpoint): retain snapshot files until release is confirmed
Signed-off-by: Anish Mahishi <amahishi@nvidia.com>
1 parent 843ab09 commit da03aaa

3 files changed

Lines changed: 51 additions & 5 deletions

File tree

‎nemo_rl/algorithms/rollout_checkpointing/snapshots.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -839,10 +839,12 @@ async def save_snapshot(
839839
try:
840840
await checkpoint_io(captured.data_plane.release)
841841
except Exception as error:
842-
warnings.warn(
843-
f"Failed to release captured TQ snapshot: {error}",
844-
stacklevel=2,
845-
)
842+
# A timed-out RPC can still be writing these files. Stop
843+
# instead of deleting them or admitting another capture.
844+
raise RuntimeError(
845+
"Cannot confirm captured TQ snapshot release; "
846+
f"retaining {tmp_path} and stopping checkpointing"
847+
) from error
846848
if tmp_path.exists():
847849
await asyncio.to_thread(partial(shutil.rmtree, tmp_path))
848850
raise

‎nemo_rl/data_plane/README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -833,7 +833,11 @@ cleaned during restore; environment recovery still requires a rollout snapshot.
833833

834834
Temporary capture ownership lives in `SnapshotCapture`, separate from the
835835
serializable `CapturedRolloutState`. Failures release the retained views after
836-
outstanding writes settle and do not publish a partial snapshot.
836+
outstanding writes settle and do not publish a partial snapshot. If release
837+
cannot be confirmed within its RPC budget, checkpointing stops with an invariant
838+
error and retains the unpublished directory: a remote write may still be using
839+
it. The save lock is acquired before the next mutation barrier, so a new capture
840+
cannot race the previous snapshot's persistence/release.
837841

838842
The snapshot-save lock remains held until publication, so optimizer commits may
839843
still wait. This setting makes persistence asynchronous relative to rollout

‎tests/unit/single_controller/test_checkpointing.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1978,6 +1978,46 @@ def seed(actor) -> None:
19781978

19791979

19801980
class TestPeriodicRolloutCheckpoint:
1981+
def test_release_timeout_retains_unpublished_files_and_stops_checkpointing(
1982+
self, tmp_path: Path
1983+
) -> None:
1984+
actor = self._actor(tmp_path)
1985+
events = []
1986+
1987+
@contextlib.asynccontextmanager
1988+
async def snapshot(_path):
1989+
yield _FakeEnvironmentSnapshot()
1990+
1991+
class Capture(CheckpointCapture):
1992+
def persist(self) -> None:
1993+
events.append("persist")
1994+
raise TimeoutError("persistence is still running")
1995+
1996+
def release(self) -> None:
1997+
events.append("release")
1998+
raise TimeoutError("persistence is still running")
1999+
2000+
def capture_checkpoint(checkpoint_dir, *, metadata):
2001+
checkpoint_dir = Path(checkpoint_dir)
2002+
checkpoint_dir.mkdir(parents=True)
2003+
(checkpoint_dir / "pending-write").write_bytes(b"in progress")
2004+
return Capture()
2005+
2006+
actor._environment_checkpoint = _FakeEnvironmentCheckpoint(snapshot)
2007+
actor._dp_client.capture_checkpoint = capture_checkpoint
2008+
attach_rollout_checkpointing(actor)
2009+
try:
2010+
with pytest.raises(RuntimeError, match="stopping checkpointing") as error:
2011+
asyncio.run(actor._rollout_snapshots.save_snapshot(force=True))
2012+
finally:
2013+
actor._checkpointer.shutdown()
2014+
assert isinstance(error.value.__cause__, TimeoutError)
2015+
assert events == ["persist", "release"]
2016+
snapshots = tmp_path / "checkpoints" / BOOTSTRAP_DIRNAME / "rollout_snapshots"
2017+
assert len(list(snapshots.glob("tmp_snapshot_*"))) == 1
2018+
assert len(list(snapshots.rglob("pending-write"))) == 1
2019+
assert not list(snapshots.glob("snapshot_*"))
2020+
19812021
@pytest.mark.parametrize("failure", [None, "persist", "resume", "publish"])
19822022
def test_deferred_tq_persists_after_resume_and_releases_on_failure(
19832023
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str | None

0 commit comments

Comments
 (0)