diff --git a/README.md b/README.md
index 11f78db7a61..9b428e38136 100644
--- a/README.md
+++ b/README.md
@@ -291,7 +291,7 @@ The Dataset column links to publicly available datasets (e.g., on HuggingFace).
| Openenv | games | Maze navigation environment via OpenEnv. Agent navigates an 8x8 grid to find the exit. | - | - | - | - | openenv_maze.yaml | - |
| Over Refusal Detection | | - | - | ✓ | - | TBD | over_refusal_detection.yaml | - |
| Physics Judge | math | Physics QA verified by NeMo Skills' physics judge LLM with math-verify symbolic fallback | Score open-ended physics benchmarks (e.g. PHYSICS) where the judge emits [Correct] / [Incorrect] verdicts | - | - | - | physics_judge.yaml | - |
-| Pinchbench | | - | - | - | - | - | pinchbench.yaml | - |
+| Pinchbench | agent | PinchBench benchmark integration | Evaluate a model as the brain of an OpenClaw agent on real-world tasks. | - | - | - | pinchbench.yaml | - |
| Polymath | math | PolyMath multilingual math benchmark with weighted (difficulty) and per-language metrics | Improve multilingual math reasoning across 18 languages and 4 difficulty tiers | - | - | - | polymath.yaml | - |
| Proof Genselect | math | Pairwise proof selection with binary correctness reward | - | - | - | - | proof_genselect.yaml | - |
| Proof Judge | math | Theorem proving with verifier + meta-verifier judge (combined env) | - | - | - | - | proof_judge.yaml | - |
diff --git a/benchmarks/pinchbench/README.md b/benchmarks/pinchbench/README.md
new file mode 100644
index 00000000000..52a9549420c
--- /dev/null
+++ b/benchmarks/pinchbench/README.md
@@ -0,0 +1,64 @@
+# PinchBench
+
+All 147 tasks of [PinchBench](https://github.com/pinchbench/skill) — calendar, email triage,
+CSV/log analysis, coding, research, writing — measuring how well a model performs as the brain
+of an [OpenClaw](https://github.com/openclaw/openclaw) agent.
+
+This benchmark config is a thin narrowing of the `pinchbench` agent server, which owns the whole
+integration (one sandbox per task, in-sandbox OpenClaw gateway, PinchBench's own grading harness).
+Read [`responses_api_agents/pinchbench/README.md`](../../responses_api_agents/pinchbench/README.md)
+first — architecture, the per-task image build, config knobs, and the parity validation against
+vanilla standalone PinchBench all live there.
+
+## Verification
+
+Grading is PinchBench's own, run inside the sandbox: deterministic checks (`automated`), an LLM
+judge (`llm_judge`), or a weighted mix of both (`hybrid`). Reward is **continuous** in `[0,1]`,
+not pass/fail — threshold it if you need a binary signal. Each rollout also returns
+`grading_type`, `grading_breakdown`, `grading_notes` and `raw_rollout`.
+
+`num_repeats` is 3: temperature 1.0 plus live web search make a single pass noisy, and 3 is what
+the agent-side parity validation used. Trust aggregates, not individual runs.
+
+## Data preparation
+
+```
+gym eval prepare --benchmark pinchbench
+```
+
+The PinchBench skill is not vendored. `prepare.py` clones it at `v2.0.0` — the same ref
+`responses_api_agents/pinchbench/Dockerfile.benchmark` bakes into the per-task image — and writes
+`data/pinchbench_benchmark.jsonl`, one row per manifest task:
+
+```json
+{"responses_create_params": {"input": [{"role": "user", "content": ""}]},
+ "verifier_metadata": {"task_id": "task_sanity"}}
+```
+
+`task_id` is the authoritative selector: at run time `run_task.sh` passes it to
+`benchmark.py --suite`, which loads the full task (prompt + assets + grading) from the skill inside
+the sandbox. `input` carries the prompt for readability only. **Subset by dropping rows** — no
+config change needed.
+
+Set `PINCHBENCH_SKILL_DIR` to an existing `v2.0.0` checkout to skip the clone. Preparation fails
+loudly if the manifest is not 147 tasks or if the regenerated rows do not byte-match the committed
+`responses_api_agents/pinchbench/data/example.jsonl`, so upstream drift can't slip through.
+
+## Quickstart
+
+Build the per-task image first (`setup_scripts/build_image.sh --apptainer`), then:
+
+```
+gym eval run --benchmark pinchbench --model-type vllm_model \
+ +sandbox_image= \
+ +model_base_url= +model_api_key= +model_name= \
+ +judge_model= +judge_base_url= +judge_api_key= \
+ +brave_api_key=
+```
+
+Those eight keys are declared in `config.yaml` with `null` defaults so `gym list` and
+`gym eval prepare` resolve without them; each also reads a `PINCHBENCH_*` env var. A `null` still
+reaching the agent fails fast — the fields are required in `PinchBenchAgentConfig`.
+
+For slow or highly concurrent endpoints, raise `openclaw_provider_timeout_seconds` **and**
+`task_timeout_s` together; see the agent README's "OpenClaw LLM idle timeout" section.
diff --git a/benchmarks/pinchbench/__init__.py b/benchmarks/pinchbench/__init__.py
new file mode 100644
index 00000000000..467079831e1
--- /dev/null
+++ b/benchmarks/pinchbench/__init__.py
@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/benchmarks/pinchbench/config.yaml b/benchmarks/pinchbench/config.yaml
new file mode 100644
index 00000000000..d87e041a330
--- /dev/null
+++ b/benchmarks/pinchbench/config.yaml
@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# PinchBench — 147 real-world OpenClaw agent tasks. Run:
+# gym eval prepare --benchmark pinchbench
+# gym eval run --benchmark pinchbench --model-type vllm_model \
+# +sandbox_image= \
+# +model_base_url= +model_api_key= +model_name= \
+# +judge_model= +judge_base_url= +judge_api_key= \
+# +brave_api_key=
+
+# Chain to existing agent config
+config_paths:
+ - responses_api_agents/pinchbench/configs/pinchbench.yaml
+
+# Run-time knobs the agent config interpolates. Declared here so `gym list` and
+# `gym eval prepare` resolve without them; a null reaching the agent fails fast in
+# PinchBenchAgentConfig (the fields are required `str`). Supply real values on the
+# command line (`+model_base_url=...`, which merges last) or via these env vars.
+sandbox_image: ${oc.env:PINCHBENCH_SANDBOX_IMAGE,null}
+model_base_url: ${oc.env:PINCHBENCH_MODEL_BASE_URL,null}
+model_api_key: ${oc.env:PINCHBENCH_MODEL_API_KEY,null}
+model_name: ${oc.env:PINCHBENCH_MODEL_NAME,null}
+judge_model: ${oc.env:PINCHBENCH_JUDGE_MODEL,null}
+judge_base_url: ${oc.env:PINCHBENCH_JUDGE_BASE_URL,null}
+judge_api_key: ${oc.env:PINCHBENCH_JUDGE_API_KEY,null}
+brave_api_key: ${oc.env:PINCHBENCH_BRAVE_API_KEY,null}
+tavily_api_key: ${oc.env:PINCHBENCH_TAVILY_API_KEY,null}
+
+# `_inherit_from` keeps this benchmark isolated from the generic agent instance
+# above, so overriding the dataset list here does not mutate `pinchbench_agent`.
+pinchbench_benchmark_agent:
+ _inherit_from: pinchbench_agent
+ responses_api_agents:
+ pinchbench:
+ datasets:
+ - name: pinchbench
+ type: benchmark
+ jsonl_fpath: benchmarks/pinchbench/data/pinchbench_benchmark.jsonl
+ prompt_config: null
+ prepare_script: benchmarks/pinchbench/prepare.py
+ # Reward is continuous and temperature 1.0 + live web search make a single
+ # pass noisy; the parity validation in the agent README used n=3.
+ num_repeats: 3
+ license: MIT
diff --git a/benchmarks/pinchbench/prepare.py b/benchmarks/pinchbench/prepare.py
new file mode 100644
index 00000000000..b3fa3fa6137
--- /dev/null
+++ b/benchmarks/pinchbench/prepare.py
@@ -0,0 +1,159 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Prepare the full 147-task PinchBench benchmark dataset.
+
+The PinchBench skill is not vendored (same convention as `harbor_agent` / `mini_swe_agent`:
+pin an upstream ref, don't copy task files). This clones it at the SAME ref the per-task
+image bakes in — `Dockerfile.benchmark`'s `PINCHBENCH_SKILL_REF` — so the prompts written
+here describe the tasks `benchmark.py` actually loads inside the sandbox.
+
+Row shape mirrors `responses_api_agents/pinchbench/dataset_preprocess.py` (which generates
+the committed 5-task `data/example.jsonl`): the task's human-readable `## Prompt` section in
+`input`, plus `verifier_metadata.task_id`. `task_id` is the authoritative selector — at run
+time `run_task.sh` passes it to `benchmark.py --suite`, which loads the full task (prompt +
+assets + grading) from the skill. `_assert_matches_example_jsonl` re-derives the 5 example
+rows and compares them byte-for-byte against that committed file, so the two generators
+cannot silently diverge and upstream prompt drift is caught here rather than mid-run.
+
+Set `PINCHBENCH_SKILL_DIR` to an existing checkout at `SKILL_REF` to skip the clone.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import yaml
+
+
+if __package__ in (None, ""):
+ sys.path.append(str(Path(__file__).resolve().parents[2]))
+
+
+BENCHMARK_DIR = Path(__file__).resolve().parent
+DATA_DIR = BENCHMARK_DIR / "data"
+OUTPUT_FPATH = DATA_DIR / "pinchbench_benchmark.jsonl"
+
+AGENT_DIR = BENCHMARK_DIR.parents[1] / "responses_api_agents" / "pinchbench"
+EXAMPLE_JSONL_FPATH = AGENT_DIR / "data" / "example.jsonl"
+
+SKILL_REPO_URL = "https://github.com/pinchbench/skill"
+# Keep in lockstep with `PINCHBENCH_SKILL_REF` in
+# responses_api_agents/pinchbench/Dockerfile.benchmark.
+SKILL_REF = "v2.0.0"
+# The v2.0.0 manifest's task count; upstream drift surfaces here rather than as a
+# silently shorter run.
+EXPECTED_TASK_COUNT = 147
+
+_PROMPT_SECTION_RE = re.compile(r"##\s*Prompt\s*\n(.*?)(?:\n##\s|\Z)", re.S)
+
+
+def _prompt_for(skill_dir: Path, task_id: str) -> str:
+ """The task's human-readable prompt = the `## Prompt` section of its skill `.md`."""
+ md = (skill_dir / "tasks" / f"{task_id}.md").read_text()
+ match = _PROMPT_SECTION_RE.search(md)
+ return (match.group(1).strip() if match else "").strip()
+
+
+def _record_line(skill_dir: Path, task_id: str) -> str:
+ record = {
+ "responses_create_params": {"input": [{"role": "user", "content": _prompt_for(skill_dir, task_id)}]},
+ "verifier_metadata": {"task_id": task_id},
+ }
+ return json.dumps(record, separators=(",", ":")) + "\n"
+
+
+def _all_task_ids(skill_dir: Path) -> list[str]:
+ """Every task in the skill's manifest, in manifest order (`run_first` first)."""
+ manifest = yaml.safe_load((skill_dir / "tasks" / "manifest.yaml").read_text())
+ task_ids: list[str] = list(manifest.get("run_first", []))
+ for category_task_ids in (manifest.get("categories") or {}).values():
+ for task_id in category_task_ids or []:
+ if task_id not in task_ids:
+ task_ids.append(task_id)
+ return task_ids
+
+
+def _assert_matches_example_jsonl(skill_dir: Path) -> None:
+ """Re-derive the committed example rows from this checkout and require an exact match.
+
+ Guards both directions: a drifted skill checkout, and this script diverging from the
+ agent-side `dataset_preprocess.py` that produced `example.jsonl`.
+ """
+ expected_lines = EXAMPLE_JSONL_FPATH.read_text().splitlines(keepends=True)
+ for line_number, expected_line in enumerate(expected_lines, 1):
+ task_id = json.loads(expected_line)["verifier_metadata"]["task_id"]
+ actual_line = _record_line(skill_dir, task_id)
+ if actual_line != expected_line:
+ raise ValueError(
+ f"Regenerated row for '{task_id}' does not match {EXAMPLE_JSONL_FPATH} line {line_number}. "
+ f"The skill checkout is not at {SKILL_REF}, or the row format has diverged from "
+ "responses_api_agents/pinchbench/dataset_preprocess.py."
+ )
+
+
+def _write_from_skill(skill_dir: Path) -> None:
+ _assert_matches_example_jsonl(skill_dir)
+
+ task_ids = _all_task_ids(skill_dir)
+ if len(task_ids) != EXPECTED_TASK_COUNT:
+ raise ValueError(
+ f"Expected {EXPECTED_TASK_COUNT} PinchBench tasks in the {SKILL_REF} manifest, "
+ f"found {len(task_ids)}; the skill checkout may not be at {SKILL_REF}."
+ )
+
+ DATA_DIR.mkdir(parents=True, exist_ok=True)
+ with OUTPUT_FPATH.open("w") as out:
+ for task_id in task_ids:
+ out.write(_record_line(skill_dir, task_id))
+ print(f"Wrote {len(task_ids)} PinchBench tasks to {OUTPUT_FPATH}")
+
+
+def _clone_skill(target_dir: Path) -> None:
+ print(f"Cloning {SKILL_REPO_URL} at {SKILL_REF}...")
+ subprocess.run(
+ ["git", "clone", "--depth", "1", "--branch", SKILL_REF, SKILL_REPO_URL, str(target_dir)],
+ check=True,
+ )
+
+
+def prepare() -> Path:
+ """Clone (or reuse) the pinned skill, write every manifest task. Returns the JSONL path."""
+ preexisting_skill_dir = os.environ.get("PINCHBENCH_SKILL_DIR")
+ if preexisting_skill_dir:
+ skill_dir = Path(preexisting_skill_dir)
+ if not (skill_dir / "tasks" / "manifest.yaml").is_file():
+ raise FileNotFoundError(
+ f"PINCHBENCH_SKILL_DIR={skill_dir} is not a PinchBench skill checkout "
+ "(tasks/manifest.yaml is missing)."
+ )
+ _write_from_skill(skill_dir)
+ return OUTPUT_FPATH
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ skill_dir = Path(tmpdir) / "pinchbench-skill"
+ _clone_skill(skill_dir)
+ _write_from_skill(skill_dir)
+
+ return OUTPUT_FPATH
+
+
+if __name__ == "__main__":
+ prepare()
diff --git a/nemo_gym/sandbox/providers/apptainer/provider.py b/nemo_gym/sandbox/providers/apptainer/provider.py
index 54cc04923d4..33d5f5a7c76 100644
--- a/nemo_gym/sandbox/providers/apptainer/provider.py
+++ b/nemo_gym/sandbox/providers/apptainer/provider.py
@@ -20,12 +20,13 @@
import logging
import os
import posixpath
+import re
import shlex
import shutil
import signal
import tempfile
import uuid
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
@@ -54,6 +55,8 @@
# failed to run the command. Apptainer prefixes its own fatal errors with "FATAL:".
APPTAINER_RUNTIME_ERROR_MARKERS = ("fatal:", "no instance found", "instance not found", "does not exist")
APPTAINER_MISSING_INSTANCE_MARKERS = ("no instance found", "instance not found", "does not exist")
+APPTAINER_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+APPTAINER_ENV_FILE_READONLY = frozenset({"EUID", "GID", "HOME", "IFS", "OPTIND", "PWD", "UID"})
class ApptainerCreateError(SandboxCreateError):
@@ -209,6 +212,56 @@ def _coerce_binds(value: Any) -> list[str]:
raise ApptainerCreateError(f"provider_options['binds'] must be a string or list, got {type(value).__name__}")
+def _serialize_env_file(env: Mapping[str, str]) -> bytes:
+ """Serialize environment variables for Apptainer ``--env-file``.
+
+ Apptainer shell-evaluates env files once while reading them. POSIX
+ shell-quoting preserves each value through that evaluation, while the
+ accompanying ``--no-eval`` flag prevents a second evaluation during
+ container environment injection.
+ """
+ assignments: list[str] = []
+ for key, value in env.items():
+ if not isinstance(key, str) or APPTAINER_ENV_NAME.fullmatch(key) is None:
+ raise ValueError("Apptainer environment variable names must be POSIX shell identifiers")
+ if key in APPTAINER_ENV_FILE_READONLY:
+ raise ValueError("Apptainer --env-file cannot set a shell readonly environment variable")
+ if not isinstance(value, str):
+ raise TypeError("Apptainer environment variable values must be strings")
+ if "\x00" in value:
+ raise ValueError("Apptainer environment variable values cannot contain NUL")
+ try:
+ value.encode("utf-8")
+ except UnicodeEncodeError:
+ raise ValueError("Apptainer environment variable values must be UTF-8 encodable") from None
+ assignments.append(f"{key}={shlex.quote(value)}")
+ return ("\n".join(assignments) + "\n").encode("utf-8") if assignments else b""
+
+
+@contextlib.contextmanager
+def _private_env_file(staging_dir: Path, content: bytes) -> Iterator[Path | None]:
+ """Yield a unique mode-0600 env file and remove it on every exit path."""
+ if not content:
+ yield None
+ return
+
+ fd = -1
+ path: Path | None = None
+ try:
+ fd, raw_path = tempfile.mkstemp(prefix=".apptainer-env-", dir=staging_dir)
+ path = Path(raw_path)
+ os.fchmod(fd, 0o600)
+ with os.fdopen(fd, "wb") as stream:
+ fd = -1
+ stream.write(content)
+ yield path
+ finally:
+ if fd >= 0:
+ os.close(fd)
+ if path is not None:
+ path.unlink(missing_ok=True)
+
+
class ApptainerProvider:
"""Sandbox provider backed by the local Apptainer CLI."""
@@ -319,7 +372,7 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
mount_point = self._create_config.mount_point, generate a unique
name = INSTANCE_NAME_PREFIX + uuid4().hex.
4. Build argv: [binary, "instance", "start", <--bind staging:mount_point>,
- , , <--env ...>,
+ , , <--env-file ...>,
_resource_flags(spec.resources), , image, name].
5. await self._run(argv, timeout_s=self._create_config.start_timeout_s);
on non-zero return, clean up the staging dir and raise
@@ -338,6 +391,10 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
if spec.image is None:
raise ApptainerCreateError("spec.image is required for the apptainer provider")
image = _resolve_image(spec.image)
+ try:
+ env_file_content = _serialize_env_file(spec.env)
+ except (TypeError, ValueError) as e:
+ raise ApptainerCreateError(f"invalid Apptainer environment: {e}") from e
# Extra per-sandbox bind mounts (validated before we allocate anything).
extra_binds = _coerce_binds(spec.provider_options.get("binds"))
@@ -356,8 +413,6 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
argv += ["--bind", bind]
for bind in extra_binds:
argv += ["--bind", bind]
- for key, value in spec.env.items():
- argv += ["--env", f"{key}={value}"]
start_args = list(self._create_config.extra_start_args)
resource_limit_flags = _resource_limit_flags(spec.resources)
if resource_limit_flags and self._create_config.apply_resource_limits:
@@ -369,11 +424,18 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle:
argv += resource_limit_flags
argv += _resource_passthrough_flags(spec.resources)
argv += start_args
- argv += [image, name]
# start the instance; clean up the staging dir on any failure.
try:
- code, _out, err = await self._run(argv, timeout_s=self._create_config.start_timeout_s, daemonize=True)
+ with _private_env_file(staging_dir, env_file_content) as env_file:
+ if env_file is not None:
+ argv += ["--no-eval", "--env-file", str(env_file)]
+ argv += [image, name]
+ code, _out, err = await self._run(
+ argv,
+ timeout_s=self._create_config.start_timeout_s,
+ daemonize=True,
+ )
except TimeoutError as e:
shutil.rmtree(staging_dir, ignore_errors=True)
raise ApptainerCreateError(f"apptainer instance start timed out for image={image!r}: {e}") from e
@@ -489,9 +551,7 @@ async def exec(
merged_env = dict(getattr(inst, "env", {}))
if env:
merged_env.update(env)
- if merged_env:
- for key, value in merged_env.items():
- flags += ["--env", f"{key}={value}"]
+ env_file_content = _serialize_env_file(merged_env)
effective_command = command
is_root = user == "root" or user == 0
@@ -505,11 +565,14 @@ async def exec(
flags += list(self._exec_config.extra_exec_args)
- argv = [self._binary, "exec", *flags, f"instance://{inst.name}", "sh", "-c", effective_command]
effective_timeout = timeout_s if timeout_s is not None else self._exec_config.default_timeout_s
try:
- code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin)
+ with _private_env_file(inst.staging_dir, env_file_content) as env_file:
+ if env_file is not None:
+ flags += ["--no-eval", "--env-file", str(env_file)]
+ argv = [self._binary, "exec", *flags, f"instance://{inst.name}", "sh", "-c", effective_command]
+ code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin)
except TimeoutError as e:
return SandboxExecResult(
stdout=None,
diff --git a/responses_api_agents/pinchbench/configs/pinchbench.yaml b/responses_api_agents/pinchbench/configs/pinchbench.yaml
index e5946591b14..6c89d78ccd6 100644
--- a/responses_api_agents/pinchbench/configs/pinchbench.yaml
+++ b/responses_api_agents/pinchbench/configs/pinchbench.yaml
@@ -2,6 +2,9 @@ pinchbench_agent:
responses_api_agents:
pinchbench:
entrypoint: app.py
+ domain: agent
+ description: PinchBench benchmark integration
+ value: Evaluate a model as the brain of an OpenClaw agent on real-world tasks.
# One sandbox per task (the isolation boundary, mirrors swe_agents/harbor). Each
# sandbox runs its OWN in-sandbox OpenClaw gateway — the only mode (at openclaw
# 2026.6.5 `openclaw agent` needs a gateway to persist transcripts).
@@ -34,6 +37,7 @@ pinchbench_agent:
judge_api_key: ${judge_api_key}
web_search_provider: brave
brave_api_key: ${brave_api_key}
+ tavily_api_key: ${tavily_api_key}
max_concurrent: 2
max_tokens: 16384
context_window: 131072
diff --git a/tests/unit_tests/test_apptainer_provider.py b/tests/unit_tests/test_apptainer_provider.py
index 94de183a6e1..b65e191e1b2 100644
--- a/tests/unit_tests/test_apptainer_provider.py
+++ b/tests/unit_tests/test_apptainer_provider.py
@@ -58,6 +58,10 @@ def _contains_seq(haystack: list[str], needle: list[str]) -> bool:
return any(haystack[i : i + len(needle)] == needle for i in range(len(haystack) - len(needle) + 1))
+def _env_file_path(argv: list[str]) -> Path:
+ return Path(argv[argv.index("--env-file") + 1])
+
+
def _make_handle(
staging: Path,
*,
@@ -155,6 +159,46 @@ def test_resolve_image() -> None:
assert resolve("/tmp/image.sif") == "/tmp/image.sif"
+def test_serialize_env_file_quotes_special_characters() -> None:
+ special = "spaces '$HOME' $(date) `id` \\\n+unicode=zażółć"
+ assert (
+ apptainer_provider._serialize_env_file({"EMPTY": "", "PLAIN": "value", "SPECIAL": special})
+ == f"EMPTY=''\nPLAIN=value\nSPECIAL={shlex.quote(special)}\n".encode()
+ )
+
+
+@pytest.mark.parametrize(
+ "env,error",
+ [
+ ({"NOT-VALID": "value"}, ValueError),
+ ({1: "value"}, ValueError),
+ ({"HOME": "/tmp/home"}, ValueError),
+ ({"VALID": "before\x00after"}, ValueError),
+ ({"VALID": 1}, TypeError),
+ ({"VALID": "\ud800"}, ValueError),
+ ],
+)
+def test_serialize_env_file_rejects_invalid_input(env: dict[Any, Any], error: type[Exception]) -> None:
+ with pytest.raises(error):
+ apptainer_provider._serialize_env_file(env)
+
+
+def test_private_env_file_is_mode_0600_and_always_removed(tmp_path: Path) -> None:
+ content = b"TOKEN='not-a-real-secret'\n"
+
+ with apptainer_provider._private_env_file(tmp_path, content) as path:
+ assert path is not None
+ assert path.read_bytes() == content
+ assert path.stat().st_mode & 0o777 == 0o600
+ assert not path.exists()
+
+ with pytest.raises(RuntimeError):
+ with apptainer_provider._private_env_file(tmp_path, content) as failed_path:
+ assert failed_path is not None
+ raise RuntimeError("failure")
+ assert not failed_path.exists()
+
+
def test_to_sandbox_status() -> None:
to_status = apptainer_provider._to_sandbox_status
assert to_status("running") is SandboxStatus.RUNNING
@@ -196,7 +240,15 @@ async def test_create_builds_argv_and_runs_probe(
staging = tmp_path / "staging"
monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging))
+ env_files: list[Path] = []
+
def responder(argv: list[str]) -> tuple[int, str, str]:
+ if "--env-file" in argv:
+ env_file = _env_file_path(argv)
+ env_files.append(env_file)
+ assert env_file.read_bytes() == b"FOO=bar\n"
+ assert env_file.stat().st_mode & 0o777 == 0o600
+ assert "FOO=bar" not in argv
if "start" in argv:
return (0, "", "")
if "exec" in argv:
@@ -232,7 +284,7 @@ def responder(argv: list[str]) -> tuple[int, str, str]:
assert start_argv[:3] == [FAKE_BINARY, "instance", "start"]
assert _contains_seq(start_argv, ["--bind", f"{staging}:/sandbox"])
assert _contains_seq(start_argv, ["--bind", "/data:/data"])
- assert _contains_seq(start_argv, ["--env", "FOO=bar"])
+ assert _contains_seq(start_argv, ["--no-eval", "--env-file", str(env_files[0])])
assert _contains_seq(start_argv, ["--cpus", "2.0"])
assert _contains_seq(start_argv, ["--memory", "1024m"])
assert "--nv" in start_argv
@@ -244,6 +296,8 @@ def responder(argv: list[str]) -> tuple[int, str, str]:
assert f"instance://{handle.sandbox_id}" in probe_argv
assert probe_argv[-1] == apptainer_provider.READY_PROBE_COMMAND
assert rec.calls[1]["timeout_s"] == 30
+ assert len(env_files) == 2
+ assert all(not path.exists() for path in env_files)
@pytest.mark.parametrize("create_config", [{"apply_resource_limits": False}, {"extra_start_args": ["--fakeroot"]}])
@@ -327,10 +381,20 @@ async def test_create_start_failure_cleans_up(
) -> None:
staging = tmp_path / "staging"
monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging))
- provider, _rec = _make_provider(monkeypatch, lambda argv: (1, "", "boom"))
+ env_file: Path | None = None
+
+ def responder(argv: list[str]) -> tuple[int, str, str]:
+ nonlocal env_file
+ env_file = _env_file_path(argv)
+ assert env_file.exists()
+ assert "not-a-real-secret" not in argv
+ return (1, "", "boom")
+
+ provider, _rec = _make_provider(monkeypatch, responder)
with pytest.raises(apptainer_provider.ApptainerCreateError, match="failed"):
- await provider.create(SandboxSpec(image="docker://img"))
+ await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"}))
+ assert env_file is not None and not env_file.exists()
assert not staging.exists()
@@ -340,12 +404,19 @@ async def test_create_start_timeout_cleans_up(
staging = tmp_path / "staging"
monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging))
+ env_file: Path | None = None
+
def responder(argv: list[str]) -> tuple[int, str, str]:
- raise TimeoutError("slow")
+ nonlocal env_file
+ env_file = _env_file_path(argv)
+ assert env_file.exists()
+ raise TimeoutError(f"slow: {argv}")
provider, _rec = _make_provider(monkeypatch, responder)
- with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out"):
- await provider.create(SandboxSpec(image="docker://img"))
+ with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out") as exc_info:
+ await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"}))
+ assert "not-a-real-secret" not in str(exc_info.value)
+ assert env_file is not None and not env_file.exists()
assert not staging.exists()
@@ -374,7 +445,17 @@ def responder(argv: list[str]) -> tuple[int, str, str]:
# exec
# --------------------------------------------------------------------------- #
async def test_exec_normal_with_cwd_and_env(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
- provider, rec = _make_provider(monkeypatch, lambda argv: (0, "hello", ""))
+ env_file: Path | None = None
+
+ def responder(argv: list[str]) -> tuple[int, str, str]:
+ nonlocal env_file
+ env_file = _env_file_path(argv)
+ assert env_file.read_bytes() == b"A=b\n"
+ assert env_file.stat().st_mode & 0o777 == 0o600
+ assert "A=b" not in argv
+ return (0, "hello", "")
+
+ provider, rec = _make_provider(monkeypatch, responder)
handle = _make_handle(tmp_path)
result = await provider.exec(handle, "echo hi", cwd="/work", env={"A": "b"})
@@ -386,23 +467,34 @@ async def test_exec_normal_with_cwd_and_env(fake_binary: str, monkeypatch: pytes
argv = rec.calls[0]["argv"]
assert argv[:2] == [FAKE_BINARY, "exec"]
assert _contains_seq(argv, ["--pwd", "/work"])
- assert _contains_seq(argv, ["--env", "A=b"])
+ assert env_file is not None
+ assert _contains_seq(argv, ["--no-eval", "--env-file", str(env_file)])
assert argv[-4:] == ["instance://nemo-gym-x", "sh", "-c", "echo hi"]
assert rec.calls[0]["timeout_s"] == 180 # default exec timeout
+ assert not env_file.exists()
async def test_exec_reapplies_create_env_and_overrides_call_env(
fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
- provider, rec = _make_provider(monkeypatch, lambda argv: (0, "", ""))
+ env_file: Path | None = None
+
+ def responder(argv: list[str]) -> tuple[int, str, str]:
+ nonlocal env_file
+ env_file = _env_file_path(argv)
+ assert env_file.read_bytes() == b"A=from-call\nB=base\n"
+ return (0, "", "")
+
+ provider, rec = _make_provider(monkeypatch, responder)
handle = _make_handle(tmp_path, env={"A": "from-create", "B": "base"})
await provider.exec(handle, "env", env={"A": "from-call"})
argv = rec.calls[0]["argv"]
- assert _contains_seq(argv, ["--env", "A=from-call"])
- assert _contains_seq(argv, ["--env", "B=base"])
+ assert "A=from-call" not in argv
+ assert "B=base" not in argv
assert "A=from-create" not in argv
+ assert env_file is not None and not env_file.exists()
async def test_exec_empty_streams_are_strings(
@@ -458,15 +550,26 @@ async def test_exec_passes_stdin(fake_binary: str, monkeypatch: pytest.MonkeyPat
async def test_exec_timeout(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ env_file: Path | None = None
+
def responder(argv: list[str]) -> tuple[int, str, str]:
- raise TimeoutError("too slow")
+ nonlocal env_file
+ env_file = _env_file_path(argv)
+ assert env_file.exists()
+ raise TimeoutError(f"too slow: {argv}")
provider, _rec = _make_provider(monkeypatch, responder)
- result = await provider.exec(_make_handle(tmp_path), "sleep 99", timeout_s=1)
+ result = await provider.exec(
+ _make_handle(tmp_path, env={"TOKEN": "not-a-real-secret"}),
+ "sleep 99",
+ timeout_s=1,
+ )
assert result.return_code == apptainer_provider.SANDBOX_RUNTIME_RETURN_CODE
assert result.error_type == "timeout"
assert result.stdout is None
+ assert "not-a-real-secret" not in (result.stderr or "")
+ assert env_file is not None and not env_file.exists()
async def test_exec_runtime_failure(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py
index 06942f3f140..fdb69311745 100644
--- a/tests/unit_tests/test_benchmarks.py
+++ b/tests/unit_tests/test_benchmarks.py
@@ -28,6 +28,15 @@ def _mock_global_config(config: dict = None):
return OmegaConf.create(config or {})
+def test_pinchbench_tavily_key_is_derived_from_environment() -> None:
+ repo_root = Path(__file__).parents[2]
+ benchmark = safe_load((repo_root / "benchmarks/pinchbench/config.yaml").read_text())
+ agent = safe_load((repo_root / "responses_api_agents/pinchbench/configs/pinchbench.yaml").read_text())
+
+ assert benchmark["tavily_api_key"] == "${oc.env:PINCHBENCH_TAVILY_API_KEY,null}"
+ assert agent["pinchbench_agent"]["responses_api_agents"]["pinchbench"]["tavily_api_key"] == ("${tavily_api_key}")
+
+
@pytest.fixture(autouse=True)
def _mock_port_allocation(monkeypatch):
"""Keep benchmark-discovery tests independent of socket availability."""