diff --git a/benchmarks/pinchbench/config.yaml b/benchmarks/pinchbench/config.yaml index dfe29eb4007..d87e041a330 100644 --- a/benchmarks/pinchbench/config.yaml +++ b/benchmarks/pinchbench/config.yaml @@ -25,6 +25,7 @@ judge_model: ${oc.env:PINCHBENCH_JUDGE_MODEL,null} judge_base_url: ${oc.env:PINCHBENCH_JUDGE_BASE_URL,null} judge_api_key: ${oc.env:PINCHBENCH_JUDGE_API_KEY,null} brave_api_key: ${oc.env:PINCHBENCH_BRAVE_API_KEY,null} +tavily_api_key: ${oc.env:PINCHBENCH_TAVILY_API_KEY,null} # `_inherit_from` keeps this benchmark isolated from the generic agent instance # above, so overriding the dataset list here does not mutate `pinchbench_agent`. diff --git a/nemo_gym/sandbox/providers/apptainer/provider.py b/nemo_gym/sandbox/providers/apptainer/provider.py index 54cc04923d4..33d5f5a7c76 100644 --- a/nemo_gym/sandbox/providers/apptainer/provider.py +++ b/nemo_gym/sandbox/providers/apptainer/provider.py @@ -20,12 +20,13 @@ import logging import os import posixpath +import re import shlex import shutil import signal import tempfile import uuid -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -54,6 +55,8 @@ # failed to run the command. Apptainer prefixes its own fatal errors with "FATAL:". APPTAINER_RUNTIME_ERROR_MARKERS = ("fatal:", "no instance found", "instance not found", "does not exist") APPTAINER_MISSING_INSTANCE_MARKERS = ("no instance found", "instance not found", "does not exist") +APPTAINER_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +APPTAINER_ENV_FILE_READONLY = frozenset({"EUID", "GID", "HOME", "IFS", "OPTIND", "PWD", "UID"}) class ApptainerCreateError(SandboxCreateError): @@ -209,6 +212,56 @@ def _coerce_binds(value: Any) -> list[str]: raise ApptainerCreateError(f"provider_options['binds'] must be a string or list, got {type(value).__name__}") +def _serialize_env_file(env: Mapping[str, str]) -> bytes: + """Serialize environment variables for Apptainer ``--env-file``. + + Apptainer shell-evaluates env files once while reading them. POSIX + shell-quoting preserves each value through that evaluation, while the + accompanying ``--no-eval`` flag prevents a second evaluation during + container environment injection. + """ + assignments: list[str] = [] + for key, value in env.items(): + if not isinstance(key, str) or APPTAINER_ENV_NAME.fullmatch(key) is None: + raise ValueError("Apptainer environment variable names must be POSIX shell identifiers") + if key in APPTAINER_ENV_FILE_READONLY: + raise ValueError("Apptainer --env-file cannot set a shell readonly environment variable") + if not isinstance(value, str): + raise TypeError("Apptainer environment variable values must be strings") + if "\x00" in value: + raise ValueError("Apptainer environment variable values cannot contain NUL") + try: + value.encode("utf-8") + except UnicodeEncodeError: + raise ValueError("Apptainer environment variable values must be UTF-8 encodable") from None + assignments.append(f"{key}={shlex.quote(value)}") + return ("\n".join(assignments) + "\n").encode("utf-8") if assignments else b"" + + +@contextlib.contextmanager +def _private_env_file(staging_dir: Path, content: bytes) -> Iterator[Path | None]: + """Yield a unique mode-0600 env file and remove it on every exit path.""" + if not content: + yield None + return + + fd = -1 + path: Path | None = None + try: + fd, raw_path = tempfile.mkstemp(prefix=".apptainer-env-", dir=staging_dir) + path = Path(raw_path) + os.fchmod(fd, 0o600) + with os.fdopen(fd, "wb") as stream: + fd = -1 + stream.write(content) + yield path + finally: + if fd >= 0: + os.close(fd) + if path is not None: + path.unlink(missing_ok=True) + + class ApptainerProvider: """Sandbox provider backed by the local Apptainer CLI.""" @@ -319,7 +372,7 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: mount_point = self._create_config.mount_point, generate a unique name = INSTANCE_NAME_PREFIX + uuid4().hex. 4. Build argv: [binary, "instance", "start", <--bind staging:mount_point>, - , , <--env ...>, + , , <--env-file ...>, _resource_flags(spec.resources), , image, name]. 5. await self._run(argv, timeout_s=self._create_config.start_timeout_s); on non-zero return, clean up the staging dir and raise @@ -338,6 +391,10 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: if spec.image is None: raise ApptainerCreateError("spec.image is required for the apptainer provider") image = _resolve_image(spec.image) + try: + env_file_content = _serialize_env_file(spec.env) + except (TypeError, ValueError) as e: + raise ApptainerCreateError(f"invalid Apptainer environment: {e}") from e # Extra per-sandbox bind mounts (validated before we allocate anything). extra_binds = _coerce_binds(spec.provider_options.get("binds")) @@ -356,8 +413,6 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: argv += ["--bind", bind] for bind in extra_binds: argv += ["--bind", bind] - for key, value in spec.env.items(): - argv += ["--env", f"{key}={value}"] start_args = list(self._create_config.extra_start_args) resource_limit_flags = _resource_limit_flags(spec.resources) if resource_limit_flags and self._create_config.apply_resource_limits: @@ -369,11 +424,18 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: argv += resource_limit_flags argv += _resource_passthrough_flags(spec.resources) argv += start_args - argv += [image, name] # start the instance; clean up the staging dir on any failure. try: - code, _out, err = await self._run(argv, timeout_s=self._create_config.start_timeout_s, daemonize=True) + with _private_env_file(staging_dir, env_file_content) as env_file: + if env_file is not None: + argv += ["--no-eval", "--env-file", str(env_file)] + argv += [image, name] + code, _out, err = await self._run( + argv, + timeout_s=self._create_config.start_timeout_s, + daemonize=True, + ) except TimeoutError as e: shutil.rmtree(staging_dir, ignore_errors=True) raise ApptainerCreateError(f"apptainer instance start timed out for image={image!r}: {e}") from e @@ -489,9 +551,7 @@ async def exec( merged_env = dict(getattr(inst, "env", {})) if env: merged_env.update(env) - if merged_env: - for key, value in merged_env.items(): - flags += ["--env", f"{key}={value}"] + env_file_content = _serialize_env_file(merged_env) effective_command = command is_root = user == "root" or user == 0 @@ -505,11 +565,14 @@ async def exec( flags += list(self._exec_config.extra_exec_args) - argv = [self._binary, "exec", *flags, f"instance://{inst.name}", "sh", "-c", effective_command] effective_timeout = timeout_s if timeout_s is not None else self._exec_config.default_timeout_s try: - code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin) + with _private_env_file(inst.staging_dir, env_file_content) as env_file: + if env_file is not None: + flags += ["--no-eval", "--env-file", str(env_file)] + argv = [self._binary, "exec", *flags, f"instance://{inst.name}", "sh", "-c", effective_command] + code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin) except TimeoutError as e: return SandboxExecResult( stdout=None, diff --git a/responses_api_agents/pinchbench/configs/pinchbench.yaml b/responses_api_agents/pinchbench/configs/pinchbench.yaml index cd5480ce28f..6c89d78ccd6 100644 --- a/responses_api_agents/pinchbench/configs/pinchbench.yaml +++ b/responses_api_agents/pinchbench/configs/pinchbench.yaml @@ -37,6 +37,7 @@ pinchbench_agent: judge_api_key: ${judge_api_key} web_search_provider: brave brave_api_key: ${brave_api_key} + tavily_api_key: ${tavily_api_key} max_concurrent: 2 max_tokens: 16384 context_window: 131072 diff --git a/tests/unit_tests/test_apptainer_provider.py b/tests/unit_tests/test_apptainer_provider.py index 94de183a6e1..b65e191e1b2 100644 --- a/tests/unit_tests/test_apptainer_provider.py +++ b/tests/unit_tests/test_apptainer_provider.py @@ -58,6 +58,10 @@ def _contains_seq(haystack: list[str], needle: list[str]) -> bool: return any(haystack[i : i + len(needle)] == needle for i in range(len(haystack) - len(needle) + 1)) +def _env_file_path(argv: list[str]) -> Path: + return Path(argv[argv.index("--env-file") + 1]) + + def _make_handle( staging: Path, *, @@ -155,6 +159,46 @@ def test_resolve_image() -> None: assert resolve("/tmp/image.sif") == "/tmp/image.sif" +def test_serialize_env_file_quotes_special_characters() -> None: + special = "spaces '$HOME' $(date) `id` \\\n+unicode=zażółć" + assert ( + apptainer_provider._serialize_env_file({"EMPTY": "", "PLAIN": "value", "SPECIAL": special}) + == f"EMPTY=''\nPLAIN=value\nSPECIAL={shlex.quote(special)}\n".encode() + ) + + +@pytest.mark.parametrize( + "env,error", + [ + ({"NOT-VALID": "value"}, ValueError), + ({1: "value"}, ValueError), + ({"HOME": "/tmp/home"}, ValueError), + ({"VALID": "before\x00after"}, ValueError), + ({"VALID": 1}, TypeError), + ({"VALID": "\ud800"}, ValueError), + ], +) +def test_serialize_env_file_rejects_invalid_input(env: dict[Any, Any], error: type[Exception]) -> None: + with pytest.raises(error): + apptainer_provider._serialize_env_file(env) + + +def test_private_env_file_is_mode_0600_and_always_removed(tmp_path: Path) -> None: + content = b"TOKEN='not-a-real-secret'\n" + + with apptainer_provider._private_env_file(tmp_path, content) as path: + assert path is not None + assert path.read_bytes() == content + assert path.stat().st_mode & 0o777 == 0o600 + assert not path.exists() + + with pytest.raises(RuntimeError): + with apptainer_provider._private_env_file(tmp_path, content) as failed_path: + assert failed_path is not None + raise RuntimeError("failure") + assert not failed_path.exists() + + def test_to_sandbox_status() -> None: to_status = apptainer_provider._to_sandbox_status assert to_status("running") is SandboxStatus.RUNNING @@ -196,7 +240,15 @@ async def test_create_builds_argv_and_runs_probe( staging = tmp_path / "staging" monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) + env_files: list[Path] = [] + def responder(argv: list[str]) -> tuple[int, str, str]: + if "--env-file" in argv: + env_file = _env_file_path(argv) + env_files.append(env_file) + assert env_file.read_bytes() == b"FOO=bar\n" + assert env_file.stat().st_mode & 0o777 == 0o600 + assert "FOO=bar" not in argv if "start" in argv: return (0, "", "") if "exec" in argv: @@ -232,7 +284,7 @@ def responder(argv: list[str]) -> tuple[int, str, str]: assert start_argv[:3] == [FAKE_BINARY, "instance", "start"] assert _contains_seq(start_argv, ["--bind", f"{staging}:/sandbox"]) assert _contains_seq(start_argv, ["--bind", "/data:/data"]) - assert _contains_seq(start_argv, ["--env", "FOO=bar"]) + assert _contains_seq(start_argv, ["--no-eval", "--env-file", str(env_files[0])]) assert _contains_seq(start_argv, ["--cpus", "2.0"]) assert _contains_seq(start_argv, ["--memory", "1024m"]) assert "--nv" in start_argv @@ -244,6 +296,8 @@ def responder(argv: list[str]) -> tuple[int, str, str]: assert f"instance://{handle.sandbox_id}" in probe_argv assert probe_argv[-1] == apptainer_provider.READY_PROBE_COMMAND assert rec.calls[1]["timeout_s"] == 30 + assert len(env_files) == 2 + assert all(not path.exists() for path in env_files) @pytest.mark.parametrize("create_config", [{"apply_resource_limits": False}, {"extra_start_args": ["--fakeroot"]}]) @@ -327,10 +381,20 @@ async def test_create_start_failure_cleans_up( ) -> None: staging = tmp_path / "staging" monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) - provider, _rec = _make_provider(monkeypatch, lambda argv: (1, "", "boom")) + env_file: Path | None = None + + def responder(argv: list[str]) -> tuple[int, str, str]: + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.exists() + assert "not-a-real-secret" not in argv + return (1, "", "boom") + + provider, _rec = _make_provider(monkeypatch, responder) with pytest.raises(apptainer_provider.ApptainerCreateError, match="failed"): - await provider.create(SandboxSpec(image="docker://img")) + await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"})) + assert env_file is not None and not env_file.exists() assert not staging.exists() @@ -340,12 +404,19 @@ async def test_create_start_timeout_cleans_up( staging = tmp_path / "staging" monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) + env_file: Path | None = None + def responder(argv: list[str]) -> tuple[int, str, str]: - raise TimeoutError("slow") + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.exists() + raise TimeoutError(f"slow: {argv}") provider, _rec = _make_provider(monkeypatch, responder) - with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out"): - await provider.create(SandboxSpec(image="docker://img")) + with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out") as exc_info: + await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"})) + assert "not-a-real-secret" not in str(exc_info.value) + assert env_file is not None and not env_file.exists() assert not staging.exists() @@ -374,7 +445,17 @@ def responder(argv: list[str]) -> tuple[int, str, str]: # exec # --------------------------------------------------------------------------- # async def test_exec_normal_with_cwd_and_env(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - provider, rec = _make_provider(monkeypatch, lambda argv: (0, "hello", "")) + env_file: Path | None = None + + def responder(argv: list[str]) -> tuple[int, str, str]: + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.read_bytes() == b"A=b\n" + assert env_file.stat().st_mode & 0o777 == 0o600 + assert "A=b" not in argv + return (0, "hello", "") + + provider, rec = _make_provider(monkeypatch, responder) handle = _make_handle(tmp_path) result = await provider.exec(handle, "echo hi", cwd="/work", env={"A": "b"}) @@ -386,23 +467,34 @@ async def test_exec_normal_with_cwd_and_env(fake_binary: str, monkeypatch: pytes argv = rec.calls[0]["argv"] assert argv[:2] == [FAKE_BINARY, "exec"] assert _contains_seq(argv, ["--pwd", "/work"]) - assert _contains_seq(argv, ["--env", "A=b"]) + assert env_file is not None + assert _contains_seq(argv, ["--no-eval", "--env-file", str(env_file)]) assert argv[-4:] == ["instance://nemo-gym-x", "sh", "-c", "echo hi"] assert rec.calls[0]["timeout_s"] == 180 # default exec timeout + assert not env_file.exists() async def test_exec_reapplies_create_env_and_overrides_call_env( fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - provider, rec = _make_provider(monkeypatch, lambda argv: (0, "", "")) + env_file: Path | None = None + + def responder(argv: list[str]) -> tuple[int, str, str]: + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.read_bytes() == b"A=from-call\nB=base\n" + return (0, "", "") + + provider, rec = _make_provider(monkeypatch, responder) handle = _make_handle(tmp_path, env={"A": "from-create", "B": "base"}) await provider.exec(handle, "env", env={"A": "from-call"}) argv = rec.calls[0]["argv"] - assert _contains_seq(argv, ["--env", "A=from-call"]) - assert _contains_seq(argv, ["--env", "B=base"]) + assert "A=from-call" not in argv + assert "B=base" not in argv assert "A=from-create" not in argv + assert env_file is not None and not env_file.exists() async def test_exec_empty_streams_are_strings( @@ -458,15 +550,26 @@ async def test_exec_passes_stdin(fake_binary: str, monkeypatch: pytest.MonkeyPat async def test_exec_timeout(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + env_file: Path | None = None + def responder(argv: list[str]) -> tuple[int, str, str]: - raise TimeoutError("too slow") + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.exists() + raise TimeoutError(f"too slow: {argv}") provider, _rec = _make_provider(monkeypatch, responder) - result = await provider.exec(_make_handle(tmp_path), "sleep 99", timeout_s=1) + result = await provider.exec( + _make_handle(tmp_path, env={"TOKEN": "not-a-real-secret"}), + "sleep 99", + timeout_s=1, + ) assert result.return_code == apptainer_provider.SANDBOX_RUNTIME_RETURN_CODE assert result.error_type == "timeout" assert result.stdout is None + assert "not-a-real-secret" not in (result.stderr or "") + assert env_file is not None and not env_file.exists() async def test_exec_runtime_failure(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py index 06942f3f140..fdb69311745 100644 --- a/tests/unit_tests/test_benchmarks.py +++ b/tests/unit_tests/test_benchmarks.py @@ -28,6 +28,15 @@ def _mock_global_config(config: dict = None): return OmegaConf.create(config or {}) +def test_pinchbench_tavily_key_is_derived_from_environment() -> None: + repo_root = Path(__file__).parents[2] + benchmark = safe_load((repo_root / "benchmarks/pinchbench/config.yaml").read_text()) + agent = safe_load((repo_root / "responses_api_agents/pinchbench/configs/pinchbench.yaml").read_text()) + + assert benchmark["tavily_api_key"] == "${oc.env:PINCHBENCH_TAVILY_API_KEY,null}" + assert agent["pinchbench_agent"]["responses_api_agents"]["pinchbench"]["tavily_api_key"] == ("${tavily_api_key}") + + @pytest.fixture(autouse=True) def _mock_port_allocation(monkeypatch): """Keep benchmark-discovery tests independent of socket availability."""