Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions benchmarks/pinchbench/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ judge_model: ${oc.env:PINCHBENCH_JUDGE_MODEL,null}
judge_base_url: ${oc.env:PINCHBENCH_JUDGE_BASE_URL,null}
judge_api_key: ${oc.env:PINCHBENCH_JUDGE_API_KEY,null}
brave_api_key: ${oc.env:PINCHBENCH_BRAVE_API_KEY,null}
tavily_api_key: ${oc.env:PINCHBENCH_TAVILY_API_KEY,null}

# `_inherit_from` keeps this benchmark isolated from the generic agent instance
# above, so overriding the dataset list here does not mutate `pinchbench_agent`.
Expand Down
85 changes: 74 additions & 11 deletions nemo_gym/sandbox/providers/apptainer/provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,13 @@
import logging
import os
import posixpath
import re
import shlex
import shutil
import signal
import tempfile
import uuid
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -54,6 +55,8 @@
# failed to run the command. Apptainer prefixes its own fatal errors with "FATAL:".
APPTAINER_RUNTIME_ERROR_MARKERS = ("fatal:", "no instance found", "instance not found", "does not exist")
APPTAINER_MISSING_INSTANCE_MARKERS = ("no instance found", "instance not found", "does not exist")
APPTAINER_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
APPTAINER_ENV_FILE_READONLY = frozenset({"EUID", "GID", "HOME", "IFS", "OPTIND", "PWD", "UID"})


class ApptainerCreateError(SandboxCreateError):
Expand Down Expand Up @@ -209,6 +212,56 @@ def _coerce_binds(value: Any) -> list[str]:
raise ApptainerCreateError(f"provider_options['binds'] must be a string or list, got {type(value).__name__}")


def _serialize_env_file(env: Mapping[str, str]) -> bytes:
"""Serialize environment variables for Apptainer ``--env-file``.

Apptainer shell-evaluates env files once while reading them. POSIX
shell-quoting preserves each value through that evaluation, while the
accompanying ``--no-eval`` flag prevents a second evaluation during
container environment injection.
"""
assignments: list[str] = []
for key, value in env.items():
if not isinstance(key, str) or APPTAINER_ENV_NAME.fullmatch(key) is None:
raise ValueError("Apptainer environment variable names must be POSIX shell identifiers")
if key in APPTAINER_ENV_FILE_READONLY:
raise ValueError("Apptainer --env-file cannot set a shell readonly environment variable")
if not isinstance(value, str):
raise TypeError("Apptainer environment variable values must be strings")
if "\x00" in value:
raise ValueError("Apptainer environment variable values cannot contain NUL")
try:
value.encode("utf-8")
except UnicodeEncodeError:
raise ValueError("Apptainer environment variable values must be UTF-8 encodable") from None
assignments.append(f"{key}={shlex.quote(value)}")
return ("\n".join(assignments) + "\n").encode("utf-8") if assignments else b""


@contextlib.contextmanager
def _private_env_file(staging_dir: Path, content: bytes) -> Iterator[Path | None]:
"""Yield a unique mode-0600 env file and remove it on every exit path."""
if not content:
yield None
return

fd = -1
path: Path | None = None
try:
fd, raw_path = tempfile.mkstemp(prefix=".apptainer-env-", dir=staging_dir)
path = Path(raw_path)
os.fchmod(fd, 0o600)
with os.fdopen(fd, "wb") as stream:
fd = -1
stream.write(content)
yield path
finally:
if fd >= 0:
os.close(fd)
if path is not None:
path.unlink(missing_ok=True)


class ApptainerProvider:
"""Sandbox provider backed by the local Apptainer CLI."""

Expand Down Expand Up @@ -319,7 +372,7 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
mount_point = self._create_config.mount_point, generate a unique
name = INSTANCE_NAME_PREFIX + uuid4().hex.
4. Build argv: [binary, "instance", "start", <--bind staging:mount_point>,
<config default_binds>, <spec.provider_options["binds"]>, <--env ...>,
<config default_binds>, <spec.provider_options["binds"]>, <--env-file ...>,
_resource_flags(spec.resources), <extra_start_args>, image, name].
5. await self._run(argv, timeout_s=self._create_config.start_timeout_s);
on non-zero return, clean up the staging dir and raise
Expand All @@ -338,6 +391,10 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
if spec.image is None:
raise ApptainerCreateError("spec.image is required for the apptainer provider")
image = _resolve_image(spec.image)
try:
env_file_content = _serialize_env_file(spec.env)
except (TypeError, ValueError) as e:
raise ApptainerCreateError(f"invalid Apptainer environment: {e}") from e

# Extra per-sandbox bind mounts (validated before we allocate anything).
extra_binds = _coerce_binds(spec.provider_options.get("binds"))
Expand All @@ -356,8 +413,6 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
argv += ["--bind", bind]
for bind in extra_binds:
argv += ["--bind", bind]
for key, value in spec.env.items():
argv += ["--env", f"{key}={value}"]
start_args = list(self._create_config.extra_start_args)
resource_limit_flags = _resource_limit_flags(spec.resources)
if resource_limit_flags and self._create_config.apply_resource_limits:
Expand All @@ -369,11 +424,18 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
argv += resource_limit_flags
argv += _resource_passthrough_flags(spec.resources)
argv += start_args
argv += [image, name]

# start the instance; clean up the staging dir on any failure.
try:
code, _out, err = await self._run(argv, timeout_s=self._create_config.start_timeout_s, daemonize=True)
with _private_env_file(staging_dir, env_file_content) as env_file:
if env_file is not None:
argv += ["--no-eval", "--env-file", str(env_file)]
argv += [image, name]
code, _out, err = await self._run(
argv,
timeout_s=self._create_config.start_timeout_s,
daemonize=True,
)
except TimeoutError as e:
shutil.rmtree(staging_dir, ignore_errors=True)
raise ApptainerCreateError(f"apptainer instance start timed out for image={image!r}: {e}") from e
Expand Down Expand Up @@ -489,9 +551,7 @@ async def exec(
merged_env = dict(getattr(inst, "env", {}))
if env:
merged_env.update(env)
if merged_env:
for key, value in merged_env.items():
flags += ["--env", f"{key}={value}"]
env_file_content = _serialize_env_file(merged_env)

effective_command = command
is_root = user == "root" or user == 0
Expand All @@ -505,11 +565,14 @@ async def exec(

flags += list(self._exec_config.extra_exec_args)

argv = [self._binary, "exec", *flags, f"instance://{inst.name}", "sh", "-c", effective_command]
effective_timeout = timeout_s if timeout_s is not None else self._exec_config.default_timeout_s

try:
code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin)
with _private_env_file(inst.staging_dir, env_file_content) as env_file:
if env_file is not None:
flags += ["--no-eval", "--env-file", str(env_file)]
argv = [self._binary, "exec", *flags, f"instance://{inst.name}", "sh", "-c", effective_command]
code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin)
except TimeoutError as e:
return SandboxExecResult(
stdout=None,
Expand Down
1 change: 1 addition & 0 deletions responses_api_agents/pinchbench/configs/pinchbench.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ pinchbench_agent:
judge_api_key: ${judge_api_key}
web_search_provider: brave
brave_api_key: ${brave_api_key}
tavily_api_key: ${tavily_api_key}
max_concurrent: 2
max_tokens: 16384
context_window: 131072
Expand Down
129 changes: 116 additions & 13 deletions tests/unit_tests/test_apptainer_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ def _contains_seq(haystack: list[str], needle: list[str]) -> bool:
return any(haystack[i : i + len(needle)] == needle for i in range(len(haystack) - len(needle) + 1))


def _env_file_path(argv: list[str]) -> Path:
return Path(argv[argv.index("--env-file") + 1])


def _make_handle(
staging: Path,
*,
Expand Down Expand Up @@ -155,6 +159,46 @@ def test_resolve_image() -> None:
assert resolve("/tmp/image.sif") == "/tmp/image.sif"


def test_serialize_env_file_quotes_special_characters() -> None:
special = "spaces '$HOME' $(date) `id` \\\n+unicode=zażółć"
assert (
apptainer_provider._serialize_env_file({"EMPTY": "", "PLAIN": "value", "SPECIAL": special})
== f"EMPTY=''\nPLAIN=value\nSPECIAL={shlex.quote(special)}\n".encode()
)


@pytest.mark.parametrize(
"env,error",
[
({"NOT-VALID": "value"}, ValueError),
({1: "value"}, ValueError),
({"HOME": "/tmp/home"}, ValueError),
({"VALID": "before\x00after"}, ValueError),
({"VALID": 1}, TypeError),
({"VALID": "\ud800"}, ValueError),
],
)
def test_serialize_env_file_rejects_invalid_input(env: dict[Any, Any], error: type[Exception]) -> None:
with pytest.raises(error):
apptainer_provider._serialize_env_file(env)


def test_private_env_file_is_mode_0600_and_always_removed(tmp_path: Path) -> None:
content = b"TOKEN='not-a-real-secret'\n"

with apptainer_provider._private_env_file(tmp_path, content) as path:
assert path is not None
assert path.read_bytes() == content
assert path.stat().st_mode & 0o777 == 0o600
assert not path.exists()

with pytest.raises(RuntimeError):
with apptainer_provider._private_env_file(tmp_path, content) as failed_path:
assert failed_path is not None
raise RuntimeError("failure")
assert not failed_path.exists()


def test_to_sandbox_status() -> None:
to_status = apptainer_provider._to_sandbox_status
assert to_status("running") is SandboxStatus.RUNNING
Expand Down Expand Up @@ -196,7 +240,15 @@ async def test_create_builds_argv_and_runs_probe(
staging = tmp_path / "staging"
monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging))

env_files: list[Path] = []

def responder(argv: list[str]) -> tuple[int, str, str]:
if "--env-file" in argv:
env_file = _env_file_path(argv)
env_files.append(env_file)
assert env_file.read_bytes() == b"FOO=bar\n"
assert env_file.stat().st_mode & 0o777 == 0o600
assert "FOO=bar" not in argv
if "start" in argv:
return (0, "", "")
if "exec" in argv:
Expand Down Expand Up @@ -232,7 +284,7 @@ def responder(argv: list[str]) -> tuple[int, str, str]:
assert start_argv[:3] == [FAKE_BINARY, "instance", "start"]
assert _contains_seq(start_argv, ["--bind", f"{staging}:/sandbox"])
assert _contains_seq(start_argv, ["--bind", "/data:/data"])
assert _contains_seq(start_argv, ["--env", "FOO=bar"])
assert _contains_seq(start_argv, ["--no-eval", "--env-file", str(env_files[0])])
assert _contains_seq(start_argv, ["--cpus", "2.0"])
assert _contains_seq(start_argv, ["--memory", "1024m"])
assert "--nv" in start_argv
Expand All @@ -244,6 +296,8 @@ def responder(argv: list[str]) -> tuple[int, str, str]:
assert f"instance://{handle.sandbox_id}" in probe_argv
assert probe_argv[-1] == apptainer_provider.READY_PROBE_COMMAND
assert rec.calls[1]["timeout_s"] == 30
assert len(env_files) == 2
assert all(not path.exists() for path in env_files)


@pytest.mark.parametrize("create_config", [{"apply_resource_limits": False}, {"extra_start_args": ["--fakeroot"]}])
Expand Down Expand Up @@ -327,10 +381,20 @@ async def test_create_start_failure_cleans_up(
) -> None:
staging = tmp_path / "staging"
monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging))
provider, _rec = _make_provider(monkeypatch, lambda argv: (1, "", "boom"))
env_file: Path | None = None

def responder(argv: list[str]) -> tuple[int, str, str]:
nonlocal env_file
env_file = _env_file_path(argv)
assert env_file.exists()
assert "not-a-real-secret" not in argv
return (1, "", "boom")

provider, _rec = _make_provider(monkeypatch, responder)

with pytest.raises(apptainer_provider.ApptainerCreateError, match="failed"):
await provider.create(SandboxSpec(image="docker://img"))
await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"}))
assert env_file is not None and not env_file.exists()
assert not staging.exists()


Expand All @@ -340,12 +404,19 @@ async def test_create_start_timeout_cleans_up(
staging = tmp_path / "staging"
monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging))

env_file: Path | None = None

def responder(argv: list[str]) -> tuple[int, str, str]:
raise TimeoutError("slow")
nonlocal env_file
env_file = _env_file_path(argv)
assert env_file.exists()
raise TimeoutError(f"slow: {argv}")

provider, _rec = _make_provider(monkeypatch, responder)
with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out"):
await provider.create(SandboxSpec(image="docker://img"))
with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out") as exc_info:
await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"}))
assert "not-a-real-secret" not in str(exc_info.value)
assert env_file is not None and not env_file.exists()
assert not staging.exists()


Expand Down Expand Up @@ -374,7 +445,17 @@ def responder(argv: list[str]) -> tuple[int, str, str]:
# exec
# --------------------------------------------------------------------------- #
async def test_exec_normal_with_cwd_and_env(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
provider, rec = _make_provider(monkeypatch, lambda argv: (0, "hello", ""))
env_file: Path | None = None

def responder(argv: list[str]) -> tuple[int, str, str]:
nonlocal env_file
env_file = _env_file_path(argv)
assert env_file.read_bytes() == b"A=b\n"
assert env_file.stat().st_mode & 0o777 == 0o600
assert "A=b" not in argv
return (0, "hello", "")

provider, rec = _make_provider(monkeypatch, responder)
handle = _make_handle(tmp_path)

result = await provider.exec(handle, "echo hi", cwd="/work", env={"A": "b"})
Expand All @@ -386,23 +467,34 @@ async def test_exec_normal_with_cwd_and_env(fake_binary: str, monkeypatch: pytes
argv = rec.calls[0]["argv"]
assert argv[:2] == [FAKE_BINARY, "exec"]
assert _contains_seq(argv, ["--pwd", "/work"])
assert _contains_seq(argv, ["--env", "A=b"])
assert env_file is not None
assert _contains_seq(argv, ["--no-eval", "--env-file", str(env_file)])
assert argv[-4:] == ["instance://nemo-gym-x", "sh", "-c", "echo hi"]
assert rec.calls[0]["timeout_s"] == 180 # default exec timeout
assert not env_file.exists()


async def test_exec_reapplies_create_env_and_overrides_call_env(
fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
provider, rec = _make_provider(monkeypatch, lambda argv: (0, "", ""))
env_file: Path | None = None

def responder(argv: list[str]) -> tuple[int, str, str]:
nonlocal env_file
env_file = _env_file_path(argv)
assert env_file.read_bytes() == b"A=from-call\nB=base\n"
return (0, "", "")

provider, rec = _make_provider(monkeypatch, responder)
handle = _make_handle(tmp_path, env={"A": "from-create", "B": "base"})

await provider.exec(handle, "env", env={"A": "from-call"})

argv = rec.calls[0]["argv"]
assert _contains_seq(argv, ["--env", "A=from-call"])
assert _contains_seq(argv, ["--env", "B=base"])
assert "A=from-call" not in argv
assert "B=base" not in argv
assert "A=from-create" not in argv
assert env_file is not None and not env_file.exists()


async def test_exec_empty_streams_are_strings(
Expand Down Expand Up @@ -458,15 +550,26 @@ async def test_exec_passes_stdin(fake_binary: str, monkeypatch: pytest.MonkeyPat


async def test_exec_timeout(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
env_file: Path | None = None

def responder(argv: list[str]) -> tuple[int, str, str]:
raise TimeoutError("too slow")
nonlocal env_file
env_file = _env_file_path(argv)
assert env_file.exists()
raise TimeoutError(f"too slow: {argv}")

provider, _rec = _make_provider(monkeypatch, responder)
result = await provider.exec(_make_handle(tmp_path), "sleep 99", timeout_s=1)
result = await provider.exec(
_make_handle(tmp_path, env={"TOKEN": "not-a-real-secret"}),
"sleep 99",
timeout_s=1,
)

assert result.return_code == apptainer_provider.SANDBOX_RUNTIME_RETURN_CODE
assert result.error_type == "timeout"
assert result.stdout is None
assert "not-a-real-secret" not in (result.stderr or "")
assert env_file is not None and not env_file.exists()


async def test_exec_runtime_failure(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
Expand Down
Loading
Loading