From b6956edaa39115bb4d32b63cb9f247f902c4e449 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Sun, 4 Oct 2026 16:14:58 -0700 Subject: [PATCH 1/3] feat(sandbox): add apptainer create.private_tmp option Apptainer binds the host /tmp into every instance by default, so all sandboxes on a node share one /tmp with each other and with host-side processes (e.g. Ray's session directory under /tmp/ray). Agents commonly create and delete scratch dirs under /tmp, and graders write fixed-name files there, so concurrent sandboxes can clobber each other's files. Add an opt-in `create.private_tmp` flag. When set, `instance start` gets `--no-mount tmp` plus a bind of a per-instance `tmp` directory (mode 1777, inside the instance's staging dir) over /tmp. The directory is removed together with the staging dir. Default off; no behavior change for existing configs. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Lin Jia --- .../sandbox/providers/apptainer/README.md | 2 ++ .../sandbox/providers/apptainer/provider.py | 8 +++++ tests/unit_tests/test_apptainer_provider.py | 35 +++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/nemo_gym/sandbox/providers/apptainer/README.md b/nemo_gym/sandbox/providers/apptainer/README.md index 4ae49f2c782..5c85341ea84 100644 --- a/nemo_gym/sandbox/providers/apptainer/README.md +++ b/nemo_gym/sandbox/providers/apptainer/README.md @@ -93,6 +93,7 @@ apptainer: start_timeout_s: 600 extra_start_args: [] apply_resource_limits: true + private_tmp: false probe: command: printf apptainer-sandbox-ready expected_stdout: apptainer-sandbox-ready @@ -109,6 +110,7 @@ Settings for starting the instance (`apptainer instance start`). | `start_timeout_s` | `600` | Max seconds to wait for `instance start` (`None` = no timeout). | | `extra_start_args` | `[]` | Extra raw flags appended to `instance start`. | | `apply_resource_limits` | `true` | Add CPU/memory cgroup flags from `SandboxSpec.resources`. | +| `private_tmp` | `false` | Give each instance its own `/tmp` (`--no-mount tmp` plus a bind of a `tmp` dir inside the instance's staging dir) instead of sharing the host `/tmp` with every other sandbox on the node. | ### `exec` — `ApptainerExecConfig` diff --git a/nemo_gym/sandbox/providers/apptainer/provider.py b/nemo_gym/sandbox/providers/apptainer/provider.py index 97b55fb2a08..cd517013616 100644 --- a/nemo_gym/sandbox/providers/apptainer/provider.py +++ b/nemo_gym/sandbox/providers/apptainer/provider.py @@ -99,6 +99,9 @@ class ApptainerCreateConfig: start_timeout_s: float | None = 600 extra_start_args: list[str] = field(default_factory=list) apply_resource_limits: bool = True + # Give each instance its own /tmp (a ``tmp`` dir in its host staging dir) instead of Apptainer's + # default bind of the host /tmp, which is otherwise shared by every sandbox on the node. + private_tmp: bool = False def __post_init__(self) -> None: if self.start_timeout_s is not None and self.start_timeout_s <= 0: @@ -445,6 +448,11 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: # build the `apptainer instance start` command line. argv: list[str] = [self._binary, "instance", "start"] argv += ["--bind", f"{staging_dir}:{mount_point}"] + if self._create_config.private_tmp: + private_tmp_dir = staging_dir / "tmp" + private_tmp_dir.mkdir(exist_ok=True) + private_tmp_dir.chmod(0o1777) + argv += ["--no-mount", "tmp", "--bind", f"{private_tmp_dir}:/tmp"] for bind in self._exec_config.default_binds: argv += ["--bind", bind] for bind in extra_binds: diff --git a/tests/unit_tests/test_apptainer_provider.py b/tests/unit_tests/test_apptainer_provider.py index 6ef8b0266d1..8e73f7aec26 100644 --- a/tests/unit_tests/test_apptainer_provider.py +++ b/tests/unit_tests/test_apptainer_provider.py @@ -504,6 +504,41 @@ def responder(argv: list[str]) -> tuple[int, str, str]: assert any("stop" in call["argv"] for call in rec.calls) +async def test_create_private_tmp_binds_per_instance_tmp( + fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + staging = tmp_path / "staging" + monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) + + def responder(argv: list[str]) -> tuple[int, str, str]: + return (0, apptainer_provider.READY_PROBE_EXPECTED, "") if "exec" in argv else (0, "", "") + + provider, rec = _make_provider(monkeypatch, responder, create={"private_tmp": True}) + await provider.create(SandboxSpec(image="ubuntu:22.04")) + + start_argv = next(c["argv"] for c in rec.calls if "start" in c["argv"]) + assert _contains_seq(start_argv, ["--no-mount", "tmp", "--bind", f"{staging / 'tmp'}:/tmp"]) + assert (staging / "tmp").is_dir() + assert (staging / "tmp").stat().st_mode & 0o7777 == 0o1777 + + +async def test_create_shares_host_tmp_by_default( + fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + staging = tmp_path / "staging" + monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) + + def responder(argv: list[str]) -> tuple[int, str, str]: + return (0, apptainer_provider.READY_PROBE_EXPECTED, "") if "exec" in argv else (0, "", "") + + provider, rec = _make_provider(monkeypatch, responder) + await provider.create(SandboxSpec(image="ubuntu:22.04")) + + start_argv = next(c["argv"] for c in rec.calls if "start" in c["argv"]) + assert "--no-mount" not in start_argv + assert not (staging / "tmp").exists() + + # --------------------------------------------------------------------------- # # exec # --------------------------------------------------------------------------- # From 939b7a22230ed578776941420d9b0072a775010d Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 14 Oct 2026 22:45:48 -0700 Subject: [PATCH 2/3] docs(sandbox): clarify private tmp mount scope Signed-off-by: Lin Jia --- nemo_gym/sandbox/providers/apptainer/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nemo_gym/sandbox/providers/apptainer/README.md b/nemo_gym/sandbox/providers/apptainer/README.md index 5c85341ea84..4c7cbbaf1fc 100644 --- a/nemo_gym/sandbox/providers/apptainer/README.md +++ b/nemo_gym/sandbox/providers/apptainer/README.md @@ -110,7 +110,13 @@ Settings for starting the instance (`apptainer instance start`). | `start_timeout_s` | `600` | Max seconds to wait for `instance start` (`None` = no timeout). | | `extra_start_args` | `[]` | Extra raw flags appended to `instance start`. | | `apply_resource_limits` | `true` | Add CPU/memory cgroup flags from `SandboxSpec.resources`. | -| `private_tmp` | `false` | Give each instance its own `/tmp` (`--no-mount tmp` plus a bind of a `tmp` dir inside the instance's staging dir) instead of sharing the host `/tmp` with every other sandbox on the node. | +| `private_tmp` | `false` | Give each instance its own `/tmp` (`--no-mount tmp` plus a bind of a `tmp` dir inside the instance's staging dir). This also disables the host `/var/tmp` mount; only `/tmp` is rebound, leaving `/var/tmp` to the image. | + +Apptainer's `tmp` mount group covers both `/tmp` and `/var/tmp` +([mount implementation](https://github.com/apptainer/apptainer/blob/v1.4.0/internal/pkg/runtime/engine/apptainer/container_linux.go#L1968-L1978)). +With `private_tmp: true`, `/var/tmp` is read-only in a plain SIF image unless you +separately configure a writable bind or overlay. Workloads that write there must +provide that writable storage or use `/tmp` instead. ### `exec` — `ApptainerExecConfig` From b02091b25d244172f05c0aeea7311dbfe51d884d Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Sun, 25 Oct 2026 00:26:51 -0700 Subject: [PATCH 3/3] fix(sandbox): private_tmp also binds a per-instance /var/tmp Apptainer's tmp mount group covers both /tmp and /var/tmp, so --no-mount tmp dropped the host /var/tmp bind as well and only /tmp was re-bound; with a plain SIF image anything writing /var/tmp hit the read-only image. The instance now gets a private 1777 directory at each path (tmp and var_tmp under its staging dir), both removed with the instance. Signed-off-by: Lin Jia --- nemo_gym/sandbox/providers/apptainer/README.md | 10 +++++----- .../sandbox/providers/apptainer/provider.py | 18 ++++++++++++------ tests/unit_tests/test_apptainer_provider.py | 8 ++++++-- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/nemo_gym/sandbox/providers/apptainer/README.md b/nemo_gym/sandbox/providers/apptainer/README.md index 8f153ff0a3f..b47c80e4a60 100644 --- a/nemo_gym/sandbox/providers/apptainer/README.md +++ b/nemo_gym/sandbox/providers/apptainer/README.md @@ -110,13 +110,13 @@ Settings for starting the instance (`apptainer instance start`). | `start_timeout_s` | `600` | Max seconds to wait for `instance start` (`None` = no timeout). | | `extra_start_args` | `[]` | Extra raw flags appended to `instance start`. | | `apply_resource_limits` | `true` | Add CPU/memory cgroup flags from `SandboxSpec.resources`. | -| `private_tmp` | `false` | Give each instance its own `/tmp` (`--no-mount tmp` plus a bind of a `tmp` dir inside the instance's staging dir). This also disables the host `/var/tmp` mount; only `/tmp` is rebound, leaving `/var/tmp` to the image. | +| `private_tmp` | `false` | Give each instance its own `/tmp` and `/var/tmp` (`--no-mount tmp` plus binds of `tmp` and `var_tmp` dirs inside the instance's staging dir, mode `1777`) instead of sharing the host directories with every other sandbox on the node. Both are removed with the instance. | Apptainer's `tmp` mount group covers both `/tmp` and `/var/tmp` -([mount implementation](https://github.com/apptainer/apptainer/blob/v1.4.0/internal/pkg/runtime/engine/apptainer/container_linux.go#L1968-L1978)). -With `private_tmp: true`, `/var/tmp` is read-only in a plain SIF image unless you -separately configure a writable bind or overlay. Workloads that write there must -provide that writable storage or use `/tmp` instead. +([mount implementation](https://github.com/apptainer/apptainer/blob/v1.4.0/internal/pkg/runtime/engine/apptainer/container_linux.go#L1968-L1978)), +so `--no-mount tmp` drops both host binds. `private_tmp: true` therefore binds a +private directory at each path; nothing in the sandbox falls through to the +read-only image. ### `exec` — `ApptainerExecConfig` diff --git a/nemo_gym/sandbox/providers/apptainer/provider.py b/nemo_gym/sandbox/providers/apptainer/provider.py index b81385364d3..286e03ace6b 100644 --- a/nemo_gym/sandbox/providers/apptainer/provider.py +++ b/nemo_gym/sandbox/providers/apptainer/provider.py @@ -107,8 +107,9 @@ class ApptainerCreateConfig: start_timeout_s: float | None = 600 extra_start_args: list[str] = field(default_factory=list) apply_resource_limits: bool = True - # Give each instance its own /tmp (a ``tmp`` dir in its host staging dir) instead of Apptainer's - # default bind of the host /tmp, which is otherwise shared by every sandbox on the node. + # Give each instance its own /tmp and /var/tmp (``tmp`` and ``var_tmp`` dirs in its host staging + # dir) instead of Apptainer's default binds of the host directories, which are otherwise shared by + # every sandbox on the node. private_tmp: bool = False def __post_init__(self) -> None: @@ -466,10 +467,15 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: argv: list[str] = [self._binary, "instance", "start"] argv += ["--bind", f"{staging_dir}:{mount_point}"] if self._create_config.private_tmp: - private_tmp_dir = staging_dir / "tmp" - private_tmp_dir.mkdir(exist_ok=True) - private_tmp_dir.chmod(0o1777) - argv += ["--no-mount", "tmp", "--bind", f"{private_tmp_dir}:/tmp"] + # Apptainer's ``tmp`` mount group covers both /tmp and /var/tmp, so ``--no-mount tmp`` drops + # both host binds; give the instance a private directory for each so neither falls through + # to the read-only image. + argv += ["--no-mount", "tmp"] + for host_name, guest_path in (("tmp", "/tmp"), ("var_tmp", "/var/tmp")): + private_tmp_dir = staging_dir / host_name + private_tmp_dir.mkdir(exist_ok=True) + private_tmp_dir.chmod(0o1777) + argv += ["--bind", f"{private_tmp_dir}:{guest_path}"] for bind in self._exec_config.default_binds: argv += ["--bind", bind] for bind in extra_binds: diff --git a/tests/unit_tests/test_apptainer_provider.py b/tests/unit_tests/test_apptainer_provider.py index b63faa316f0..23d9762f32d 100644 --- a/tests/unit_tests/test_apptainer_provider.py +++ b/tests/unit_tests/test_apptainer_provider.py @@ -526,8 +526,11 @@ def responder(argv: list[str]) -> tuple[int, str, str]: start_argv = next(c["argv"] for c in rec.calls if "start" in c["argv"]) assert _contains_seq(start_argv, ["--no-mount", "tmp", "--bind", f"{staging / 'tmp'}:/tmp"]) - assert (staging / "tmp").is_dir() - assert (staging / "tmp").stat().st_mode & 0o7777 == 0o1777 + # ``--no-mount tmp`` also drops the host /var/tmp bind, so the instance gets a private one too. + assert _contains_seq(start_argv, ["--bind", f"{staging / 'var_tmp'}:/var/tmp"]) + for name in ("tmp", "var_tmp"): + assert (staging / name).is_dir() + assert (staging / name).stat().st_mode & 0o7777 == 0o1777 async def test_create_shares_host_tmp_by_default( @@ -545,6 +548,7 @@ def responder(argv: list[str]) -> tuple[int, str, str]: start_argv = next(c["argv"] for c in rec.calls if "start" in c["argv"]) assert "--no-mount" not in start_argv assert not (staging / "tmp").exists() + assert not (staging / "var_tmp").exists() # --------------------------------------------------------------------------- #