diff --git a/nemo_gym/sandbox/providers/apptainer/README.md b/nemo_gym/sandbox/providers/apptainer/README.md index cadcd983a61..b47c80e4a60 100644 --- a/nemo_gym/sandbox/providers/apptainer/README.md +++ b/nemo_gym/sandbox/providers/apptainer/README.md @@ -93,6 +93,7 @@ apptainer: start_timeout_s: 600 extra_start_args: [] apply_resource_limits: true + private_tmp: false probe: command: printf apptainer-sandbox-ready expected_stdout: apptainer-sandbox-ready @@ -109,6 +110,13 @@ Settings for starting the instance (`apptainer instance start`). | `start_timeout_s` | `600` | Max seconds to wait for `instance start` (`None` = no timeout). | | `extra_start_args` | `[]` | Extra raw flags appended to `instance start`. | | `apply_resource_limits` | `true` | Add CPU/memory cgroup flags from `SandboxSpec.resources`. | +| `private_tmp` | `false` | Give each instance its own `/tmp` and `/var/tmp` (`--no-mount tmp` plus binds of `tmp` and `var_tmp` dirs inside the instance's staging dir, mode `1777`) instead of sharing the host directories with every other sandbox on the node. Both are removed with the instance. | + +Apptainer's `tmp` mount group covers both `/tmp` and `/var/tmp` +([mount implementation](https://github.com/apptainer/apptainer/blob/v1.4.0/internal/pkg/runtime/engine/apptainer/container_linux.go#L1968-L1978)), +so `--no-mount tmp` drops both host binds. `private_tmp: true` therefore binds a +private directory at each path; nothing in the sandbox falls through to the +read-only image. ### `exec` — `ApptainerExecConfig` diff --git a/nemo_gym/sandbox/providers/apptainer/provider.py b/nemo_gym/sandbox/providers/apptainer/provider.py index 4c042d18c9d..286e03ace6b 100644 --- a/nemo_gym/sandbox/providers/apptainer/provider.py +++ b/nemo_gym/sandbox/providers/apptainer/provider.py @@ -107,6 +107,10 @@ class ApptainerCreateConfig: start_timeout_s: float | None = 600 extra_start_args: list[str] = field(default_factory=list) apply_resource_limits: bool = True + # Give each instance its own /tmp and /var/tmp (``tmp`` and ``var_tmp`` dirs in its host staging + # dir) instead of Apptainer's default binds of the host directories, which are otherwise shared by + # every sandbox on the node. + private_tmp: bool = False def __post_init__(self) -> None: if self.start_timeout_s is not None and self.start_timeout_s <= 0: @@ -462,6 +466,16 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: # build the `apptainer instance start` command line. argv: list[str] = [self._binary, "instance", "start"] argv += ["--bind", f"{staging_dir}:{mount_point}"] + if self._create_config.private_tmp: + # Apptainer's ``tmp`` mount group covers both /tmp and /var/tmp, so ``--no-mount tmp`` drops + # both host binds; give the instance a private directory for each so neither falls through + # to the read-only image. + argv += ["--no-mount", "tmp"] + for host_name, guest_path in (("tmp", "/tmp"), ("var_tmp", "/var/tmp")): + private_tmp_dir = staging_dir / host_name + private_tmp_dir.mkdir(exist_ok=True) + private_tmp_dir.chmod(0o1777) + argv += ["--bind", f"{private_tmp_dir}:{guest_path}"] for bind in self._exec_config.default_binds: argv += ["--bind", bind] for bind in extra_binds: diff --git a/tests/unit_tests/test_apptainer_provider.py b/tests/unit_tests/test_apptainer_provider.py index c61a085cb72..23d9762f32d 100644 --- a/tests/unit_tests/test_apptainer_provider.py +++ b/tests/unit_tests/test_apptainer_provider.py @@ -512,6 +512,45 @@ def responder(argv: list[str]) -> tuple[int, str, str]: assert any("stop" in call["argv"] for call in rec.calls) +async def test_create_private_tmp_binds_per_instance_tmp( + fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + staging = tmp_path / "staging" + monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) + + def responder(argv: list[str]) -> tuple[int, str, str]: + return (0, apptainer_provider.READY_PROBE_EXPECTED, "") if "exec" in argv else (0, "", "") + + provider, rec = _make_provider(monkeypatch, responder, create={"private_tmp": True}) + await provider.create(SandboxSpec(image="ubuntu:22.04")) + + start_argv = next(c["argv"] for c in rec.calls if "start" in c["argv"]) + assert _contains_seq(start_argv, ["--no-mount", "tmp", "--bind", f"{staging / 'tmp'}:/tmp"]) + # ``--no-mount tmp`` also drops the host /var/tmp bind, so the instance gets a private one too. + assert _contains_seq(start_argv, ["--bind", f"{staging / 'var_tmp'}:/var/tmp"]) + for name in ("tmp", "var_tmp"): + assert (staging / name).is_dir() + assert (staging / name).stat().st_mode & 0o7777 == 0o1777 + + +async def test_create_shares_host_tmp_by_default( + fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + staging = tmp_path / "staging" + monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) + + def responder(argv: list[str]) -> tuple[int, str, str]: + return (0, apptainer_provider.READY_PROBE_EXPECTED, "") if "exec" in argv else (0, "", "") + + provider, rec = _make_provider(monkeypatch, responder) + await provider.create(SandboxSpec(image="ubuntu:22.04")) + + start_argv = next(c["argv"] for c in rec.calls if "start" in c["argv"]) + assert "--no-mount" not in start_argv + assert not (staging / "tmp").exists() + assert not (staging / "var_tmp").exists() + + # --------------------------------------------------------------------------- # # exec # --------------------------------------------------------------------------- #