From 1e416d19bfcb27917040362de1eba76c63a79183 Mon Sep 17 00:00:00 2001 From: laszkiewiczp <85294967+laszkiewiczp@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:42:34 +0200 Subject: [PATCH 1/2] feat(pinchbench): add the benchmark config for the full 147-task suite (#2259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary PinchBench shipped as an agent server (`responses_api_agents/pinchbench`, #1810) but has no entry under `benchmarks/`, so it is neither discoverable via `gym list benchmarks` nor preparable via `gym eval prepare`. This adds the benchmark definition. - `config.yaml` chains `config_paths` to the agent config and narrows it to the benchmark dataset via `_inherit_from` β€” the `benchmarks/tau2` shape, the existing precedent for a benchmark chaining to a `responses_api_agents/` config. - `prepare.py` clones the PinchBench skill at `v2.0.0`, the same ref `Dockerfile.benchmark` bakes into the per-task image, and writes all 147 manifest tasks. - `num_repeats: 3`, matching the n=3 parity validation in the agent README (reward is continuous, and temperature 1.0 + live web search make a single pass noisy). ## Notes for reviewers **Run-time interpolations.** The agent config interpolates eight knobs (`${sandbox_image}`, `${model_base_url}`, judge/brave keys) that do not resolve on their own, so `gym eval prepare` would fail on a bare config. They are declared top-level with `oc.env:...,null` defaults. This does not break the documented `+sandbox_image=...` invocation: Hydra parses CLI args against an empty base *before* `config_paths` loads, and CLI merges last. A `null` still reaching the agent fails fast, since those fields are required `str` in `PinchBenchAgentConfig`. **Listing metadata on the base node.** `domain`/`description`/`value` are added to `responses_api_agents/pinchbench/configs/pinchbench.yaml` rather than the benchmark config, because adding them to the inheriting node raises `ConfigKeyError: Key 'domain' is not in struct`. Worth knowing: `discover_benchmarks()` uses the tolerant non-struct parse, so `ng_list_benchmarks` renders such a key fine and only the real prepare/run path trips the check. The `README.md` row is regenerated by the `update-readme-table` hook. **Drift guards.** Preparation fails loudly if the manifest is not 147 tasks, or if the regenerated rows do not byte-match the committed `responses_api_agents/pinchbench/data/example.jsonl` β€” so upstream prompt drift and divergence from `dataset_preprocess.py` cannot pass silently. ## Verification - `ng_prepare_benchmark "+config_paths=[benchmarks/pinchbench/config.yaml]"` β€” 147 rows, via both the live-clone path and `PINCHBENCH_SKILL_DIR` - Rows validated: all carry a non-empty `responses_create_params.input`, `task_id`s unique - `ng_list_benchmarks` lists `pinchbench` with domain `agent` and `num_repeats` 3 - Drift guard negative-tested (tampered prompt raises) - `pre-commit run` clean on all touched files - Generated JSONL is covered by the existing `benchmarks/.gitignore` ## Related Cherry-picked to `r0.5.0` in a companion PR (that release branch carries the agent; `r0.4.0` does not, as PinchBench landed on main after the cut). πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: plaszkiewicz Signed-off-by: NeMo Bot --- README.md | 2 +- benchmarks/pinchbench/README.md | 64 +++++++ benchmarks/pinchbench/__init__.py | 14 ++ benchmarks/pinchbench/config.yaml | 44 +++++ benchmarks/pinchbench/prepare.py | 159 ++++++++++++++++++ .../pinchbench/configs/pinchbench.yaml | 3 + 6 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 benchmarks/pinchbench/README.md create mode 100644 benchmarks/pinchbench/__init__.py create mode 100644 benchmarks/pinchbench/config.yaml create mode 100644 benchmarks/pinchbench/prepare.py diff --git a/README.md b/README.md index 11f78db7a61..9b428e38136 100644 --- a/README.md +++ b/README.md @@ -291,7 +291,7 @@ The Dataset column links to publicly available datasets (e.g., on HuggingFace). | Openenv | games | Maze navigation environment via OpenEnv. Agent navigates an 8x8 grid to find the exit. | - | - | - | - | openenv_maze.yaml | - | | Over Refusal Detection | | - | - | βœ“ | - | TBD | over_refusal_detection.yaml | - | | Physics Judge | math | Physics QA verified by NeMo Skills' physics judge LLM with math-verify symbolic fallback | Score open-ended physics benchmarks (e.g. PHYSICS) where the judge emits [Correct] / [Incorrect] verdicts | - | - | - | physics_judge.yaml | - | -| Pinchbench | | - | - | - | - | - | pinchbench.yaml | - | +| Pinchbench | agent | PinchBench benchmark integration | Evaluate a model as the brain of an OpenClaw agent on real-world tasks. | - | - | - | pinchbench.yaml | - | | Polymath | math | PolyMath multilingual math benchmark with weighted (difficulty) and per-language metrics | Improve multilingual math reasoning across 18 languages and 4 difficulty tiers | - | - | - | polymath.yaml | - | | Proof Genselect | math | Pairwise proof selection with binary correctness reward | - | - | - | - | proof_genselect.yaml | - | | Proof Judge | math | Theorem proving with verifier + meta-verifier judge (combined env) | - | - | - | - | proof_judge.yaml | - | diff --git a/benchmarks/pinchbench/README.md b/benchmarks/pinchbench/README.md new file mode 100644 index 00000000000..52a9549420c --- /dev/null +++ b/benchmarks/pinchbench/README.md @@ -0,0 +1,64 @@ +# PinchBench + +All 147 tasks of [PinchBench](https://github.com/pinchbench/skill) β€” calendar, email triage, +CSV/log analysis, coding, research, writing β€” measuring how well a model performs as the brain +of an [OpenClaw](https://github.com/openclaw/openclaw) agent. + +This benchmark config is a thin narrowing of the `pinchbench` agent server, which owns the whole +integration (one sandbox per task, in-sandbox OpenClaw gateway, PinchBench's own grading harness). +Read [`responses_api_agents/pinchbench/README.md`](../../responses_api_agents/pinchbench/README.md) +first β€” architecture, the per-task image build, config knobs, and the parity validation against +vanilla standalone PinchBench all live there. + +## Verification + +Grading is PinchBench's own, run inside the sandbox: deterministic checks (`automated`), an LLM +judge (`llm_judge`), or a weighted mix of both (`hybrid`). Reward is **continuous** in `[0,1]`, +not pass/fail β€” threshold it if you need a binary signal. Each rollout also returns +`grading_type`, `grading_breakdown`, `grading_notes` and `raw_rollout`. + +`num_repeats` is 3: temperature 1.0 plus live web search make a single pass noisy, and 3 is what +the agent-side parity validation used. Trust aggregates, not individual runs. + +## Data preparation + +``` +gym eval prepare --benchmark pinchbench +``` + +The PinchBench skill is not vendored. `prepare.py` clones it at `v2.0.0` β€” the same ref +`responses_api_agents/pinchbench/Dockerfile.benchmark` bakes into the per-task image β€” and writes +`data/pinchbench_benchmark.jsonl`, one row per manifest task: + +```json +{"responses_create_params": {"input": [{"role": "user", "content": ""}]}, + "verifier_metadata": {"task_id": "task_sanity"}} +``` + +`task_id` is the authoritative selector: at run time `run_task.sh` passes it to +`benchmark.py --suite`, which loads the full task (prompt + assets + grading) from the skill inside +the sandbox. `input` carries the prompt for readability only. **Subset by dropping rows** β€” no +config change needed. + +Set `PINCHBENCH_SKILL_DIR` to an existing `v2.0.0` checkout to skip the clone. Preparation fails +loudly if the manifest is not 147 tasks or if the regenerated rows do not byte-match the committed +`responses_api_agents/pinchbench/data/example.jsonl`, so upstream drift can't slip through. + +## Quickstart + +Build the per-task image first (`setup_scripts/build_image.sh --apptainer`), then: + +``` +gym eval run --benchmark pinchbench --model-type vllm_model \ + +sandbox_image= \ + +model_base_url= +model_api_key= +model_name= \ + +judge_model= +judge_base_url= +judge_api_key= \ + +brave_api_key= +``` + +Those eight keys are declared in `config.yaml` with `null` defaults so `gym list` and +`gym eval prepare` resolve without them; each also reads a `PINCHBENCH_*` env var. A `null` still +reaching the agent fails fast β€” the fields are required in `PinchBenchAgentConfig`. + +For slow or highly concurrent endpoints, raise `openclaw_provider_timeout_seconds` **and** +`task_timeout_s` together; see the agent README's "OpenClaw LLM idle timeout" section. diff --git a/benchmarks/pinchbench/__init__.py b/benchmarks/pinchbench/__init__.py new file mode 100644 index 00000000000..467079831e1 --- /dev/null +++ b/benchmarks/pinchbench/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/benchmarks/pinchbench/config.yaml b/benchmarks/pinchbench/config.yaml new file mode 100644 index 00000000000..dfe29eb4007 --- /dev/null +++ b/benchmarks/pinchbench/config.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# PinchBench β€” 147 real-world OpenClaw agent tasks. Run: +# gym eval prepare --benchmark pinchbench +# gym eval run --benchmark pinchbench --model-type vllm_model \ +# +sandbox_image= \ +# +model_base_url= +model_api_key= +model_name= \ +# +judge_model= +judge_base_url= +judge_api_key= \ +# +brave_api_key= + +# Chain to existing agent config +config_paths: + - responses_api_agents/pinchbench/configs/pinchbench.yaml + +# Run-time knobs the agent config interpolates. Declared here so `gym list` and +# `gym eval prepare` resolve without them; a null reaching the agent fails fast in +# PinchBenchAgentConfig (the fields are required `str`). Supply real values on the +# command line (`+model_base_url=...`, which merges last) or via these env vars. +sandbox_image: ${oc.env:PINCHBENCH_SANDBOX_IMAGE,null} +model_base_url: ${oc.env:PINCHBENCH_MODEL_BASE_URL,null} +model_api_key: ${oc.env:PINCHBENCH_MODEL_API_KEY,null} +model_name: ${oc.env:PINCHBENCH_MODEL_NAME,null} +judge_model: ${oc.env:PINCHBENCH_JUDGE_MODEL,null} +judge_base_url: ${oc.env:PINCHBENCH_JUDGE_BASE_URL,null} +judge_api_key: ${oc.env:PINCHBENCH_JUDGE_API_KEY,null} +brave_api_key: ${oc.env:PINCHBENCH_BRAVE_API_KEY,null} + +# `_inherit_from` keeps this benchmark isolated from the generic agent instance +# above, so overriding the dataset list here does not mutate `pinchbench_agent`. +pinchbench_benchmark_agent: + _inherit_from: pinchbench_agent + responses_api_agents: + pinchbench: + datasets: + - name: pinchbench + type: benchmark + jsonl_fpath: benchmarks/pinchbench/data/pinchbench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/pinchbench/prepare.py + # Reward is continuous and temperature 1.0 + live web search make a single + # pass noisy; the parity validation in the agent README used n=3. + num_repeats: 3 + license: MIT diff --git a/benchmarks/pinchbench/prepare.py b/benchmarks/pinchbench/prepare.py new file mode 100644 index 00000000000..b3fa3fa6137 --- /dev/null +++ b/benchmarks/pinchbench/prepare.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Prepare the full 147-task PinchBench benchmark dataset. + +The PinchBench skill is not vendored (same convention as `harbor_agent` / `mini_swe_agent`: +pin an upstream ref, don't copy task files). This clones it at the SAME ref the per-task +image bakes in β€” `Dockerfile.benchmark`'s `PINCHBENCH_SKILL_REF` β€” so the prompts written +here describe the tasks `benchmark.py` actually loads inside the sandbox. + +Row shape mirrors `responses_api_agents/pinchbench/dataset_preprocess.py` (which generates +the committed 5-task `data/example.jsonl`): the task's human-readable `## Prompt` section in +`input`, plus `verifier_metadata.task_id`. `task_id` is the authoritative selector β€” at run +time `run_task.sh` passes it to `benchmark.py --suite`, which loads the full task (prompt + +assets + grading) from the skill. `_assert_matches_example_jsonl` re-derives the 5 example +rows and compares them byte-for-byte against that committed file, so the two generators +cannot silently diverge and upstream prompt drift is caught here rather than mid-run. + +Set `PINCHBENCH_SKILL_DIR` to an existing checkout at `SKILL_REF` to skip the clone. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) + + +BENCHMARK_DIR = Path(__file__).resolve().parent +DATA_DIR = BENCHMARK_DIR / "data" +OUTPUT_FPATH = DATA_DIR / "pinchbench_benchmark.jsonl" + +AGENT_DIR = BENCHMARK_DIR.parents[1] / "responses_api_agents" / "pinchbench" +EXAMPLE_JSONL_FPATH = AGENT_DIR / "data" / "example.jsonl" + +SKILL_REPO_URL = "https://github.com/pinchbench/skill" +# Keep in lockstep with `PINCHBENCH_SKILL_REF` in +# responses_api_agents/pinchbench/Dockerfile.benchmark. +SKILL_REF = "v2.0.0" +# The v2.0.0 manifest's task count; upstream drift surfaces here rather than as a +# silently shorter run. +EXPECTED_TASK_COUNT = 147 + +_PROMPT_SECTION_RE = re.compile(r"##\s*Prompt\s*\n(.*?)(?:\n##\s|\Z)", re.S) + + +def _prompt_for(skill_dir: Path, task_id: str) -> str: + """The task's human-readable prompt = the `## Prompt` section of its skill `.md`.""" + md = (skill_dir / "tasks" / f"{task_id}.md").read_text() + match = _PROMPT_SECTION_RE.search(md) + return (match.group(1).strip() if match else "").strip() + + +def _record_line(skill_dir: Path, task_id: str) -> str: + record = { + "responses_create_params": {"input": [{"role": "user", "content": _prompt_for(skill_dir, task_id)}]}, + "verifier_metadata": {"task_id": task_id}, + } + return json.dumps(record, separators=(",", ":")) + "\n" + + +def _all_task_ids(skill_dir: Path) -> list[str]: + """Every task in the skill's manifest, in manifest order (`run_first` first).""" + manifest = yaml.safe_load((skill_dir / "tasks" / "manifest.yaml").read_text()) + task_ids: list[str] = list(manifest.get("run_first", [])) + for category_task_ids in (manifest.get("categories") or {}).values(): + for task_id in category_task_ids or []: + if task_id not in task_ids: + task_ids.append(task_id) + return task_ids + + +def _assert_matches_example_jsonl(skill_dir: Path) -> None: + """Re-derive the committed example rows from this checkout and require an exact match. + + Guards both directions: a drifted skill checkout, and this script diverging from the + agent-side `dataset_preprocess.py` that produced `example.jsonl`. + """ + expected_lines = EXAMPLE_JSONL_FPATH.read_text().splitlines(keepends=True) + for line_number, expected_line in enumerate(expected_lines, 1): + task_id = json.loads(expected_line)["verifier_metadata"]["task_id"] + actual_line = _record_line(skill_dir, task_id) + if actual_line != expected_line: + raise ValueError( + f"Regenerated row for '{task_id}' does not match {EXAMPLE_JSONL_FPATH} line {line_number}. " + f"The skill checkout is not at {SKILL_REF}, or the row format has diverged from " + "responses_api_agents/pinchbench/dataset_preprocess.py." + ) + + +def _write_from_skill(skill_dir: Path) -> None: + _assert_matches_example_jsonl(skill_dir) + + task_ids = _all_task_ids(skill_dir) + if len(task_ids) != EXPECTED_TASK_COUNT: + raise ValueError( + f"Expected {EXPECTED_TASK_COUNT} PinchBench tasks in the {SKILL_REF} manifest, " + f"found {len(task_ids)}; the skill checkout may not be at {SKILL_REF}." + ) + + DATA_DIR.mkdir(parents=True, exist_ok=True) + with OUTPUT_FPATH.open("w") as out: + for task_id in task_ids: + out.write(_record_line(skill_dir, task_id)) + print(f"Wrote {len(task_ids)} PinchBench tasks to {OUTPUT_FPATH}") + + +def _clone_skill(target_dir: Path) -> None: + print(f"Cloning {SKILL_REPO_URL} at {SKILL_REF}...") + subprocess.run( + ["git", "clone", "--depth", "1", "--branch", SKILL_REF, SKILL_REPO_URL, str(target_dir)], + check=True, + ) + + +def prepare() -> Path: + """Clone (or reuse) the pinned skill, write every manifest task. Returns the JSONL path.""" + preexisting_skill_dir = os.environ.get("PINCHBENCH_SKILL_DIR") + if preexisting_skill_dir: + skill_dir = Path(preexisting_skill_dir) + if not (skill_dir / "tasks" / "manifest.yaml").is_file(): + raise FileNotFoundError( + f"PINCHBENCH_SKILL_DIR={skill_dir} is not a PinchBench skill checkout " + "(tasks/manifest.yaml is missing)." + ) + _write_from_skill(skill_dir) + return OUTPUT_FPATH + + with tempfile.TemporaryDirectory() as tmpdir: + skill_dir = Path(tmpdir) / "pinchbench-skill" + _clone_skill(skill_dir) + _write_from_skill(skill_dir) + + return OUTPUT_FPATH + + +if __name__ == "__main__": + prepare() diff --git a/responses_api_agents/pinchbench/configs/pinchbench.yaml b/responses_api_agents/pinchbench/configs/pinchbench.yaml index e5946591b14..cd5480ce28f 100644 --- a/responses_api_agents/pinchbench/configs/pinchbench.yaml +++ b/responses_api_agents/pinchbench/configs/pinchbench.yaml @@ -2,6 +2,9 @@ pinchbench_agent: responses_api_agents: pinchbench: entrypoint: app.py + domain: agent + description: PinchBench benchmark integration + value: Evaluate a model as the brain of an OpenClaw agent on real-world tasks. # One sandbox per task (the isolation boundary, mirrors swe_agents/harbor). Each # sandbox runs its OWN in-sandbox OpenClaw gateway β€” the only mode (at openclaw # 2026.6.5 `openclaw agent` needs a gateway to persist transcripts). From 6d73d72311ddd6f038ce163ed9940e21fea8dedf Mon Sep 17 00:00:00 2001 From: laszkiewiczp <85294967+laszkiewiczp@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:31:50 +0200 Subject: [PATCH 2/2] fix(apptainer): keep sandbox env out of argv (#2284) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authored by @arajfer β€” I'm only carrying it to `main`. The commit keeps his authorship and `Signed-off-by`; I added mine as the submitter. ## Why this PR exists This change was written on the `cherry-pick-2259-r0.5.0` branch (commit `032ae5184`), stacked on top of the PinchBench benchmark-config cherry-pick. When #2259 merged, the bot regenerated that branch as a single squashed commit for #2283, which dropped the stacked work. The commit was never on `main` β€” it only ever existed on that branch β€” so it would have been lost. ## What it does - `nemo_gym/sandbox/providers/apptainer/provider.py` β€” pass sandbox environment through the environment rather than `argv`, so secrets and env values don't land in the process command line. - Adds `tavily_api_key` plumbing: `${tavily_api_key}` on the agent config, and the matching `${oc.env:PINCHBENCH_TAVILY_API_KEY,null}` top-level declaration on the benchmark config. Both are required together β€” the agent config interpolates the key, so the benchmark config must declare it or `gym eval prepare` fails to resolve. `PinchBenchAgentConfig` already carries `tavily_api_key: Optional[str] = None`, so nothing else is needed. - Tests: `tests/unit_tests/test_apptainer_provider.py` (+129) covering the argv/env split, and `tests/unit_tests/test_benchmarks.py` (+9) pinning the agent/benchmark tavily pairing so the two configs can't drift apart. ## Verification Cherry-picked onto current `main` (`3e210384a`) with no conflicts, then: - `pytest tests/unit_tests/test_apptainer_provider.py tests/unit_tests/test_benchmarks.py` β€” **101 passed**, including the new `test_pinchbench_tavily_key_is_derived_from_environment` - `pre-commit run` clean on all 5 touched files @arajfer please confirm this is the version you want on `main` β€” if you'd rather own the PR yourself, close this and I'll drop the branch. Once it lands it can cherry-pick to `r0.5.0` on its own, independently of #2283. πŸ€– Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: Adam Rajfer Signed-off-by: plaszkiewicz Co-authored-by: Adam Rajfer (cherry picked from commit d58f6ef0f36703abed0222e116a10fae0165d59f) --- benchmarks/pinchbench/config.yaml | 1 + .../sandbox/providers/apptainer/provider.py | 85 ++++++++++-- .../pinchbench/configs/pinchbench.yaml | 1 + tests/unit_tests/test_apptainer_provider.py | 129 ++++++++++++++++-- tests/unit_tests/test_benchmarks.py | 9 ++ 5 files changed, 201 insertions(+), 24 deletions(-) diff --git a/benchmarks/pinchbench/config.yaml b/benchmarks/pinchbench/config.yaml index dfe29eb4007..d87e041a330 100644 --- a/benchmarks/pinchbench/config.yaml +++ b/benchmarks/pinchbench/config.yaml @@ -25,6 +25,7 @@ judge_model: ${oc.env:PINCHBENCH_JUDGE_MODEL,null} judge_base_url: ${oc.env:PINCHBENCH_JUDGE_BASE_URL,null} judge_api_key: ${oc.env:PINCHBENCH_JUDGE_API_KEY,null} brave_api_key: ${oc.env:PINCHBENCH_BRAVE_API_KEY,null} +tavily_api_key: ${oc.env:PINCHBENCH_TAVILY_API_KEY,null} # `_inherit_from` keeps this benchmark isolated from the generic agent instance # above, so overriding the dataset list here does not mutate `pinchbench_agent`. diff --git a/nemo_gym/sandbox/providers/apptainer/provider.py b/nemo_gym/sandbox/providers/apptainer/provider.py index 54cc04923d4..33d5f5a7c76 100644 --- a/nemo_gym/sandbox/providers/apptainer/provider.py +++ b/nemo_gym/sandbox/providers/apptainer/provider.py @@ -20,12 +20,13 @@ import logging import os import posixpath +import re import shlex import shutil import signal import tempfile import uuid -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -54,6 +55,8 @@ # failed to run the command. Apptainer prefixes its own fatal errors with "FATAL:". APPTAINER_RUNTIME_ERROR_MARKERS = ("fatal:", "no instance found", "instance not found", "does not exist") APPTAINER_MISSING_INSTANCE_MARKERS = ("no instance found", "instance not found", "does not exist") +APPTAINER_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +APPTAINER_ENV_FILE_READONLY = frozenset({"EUID", "GID", "HOME", "IFS", "OPTIND", "PWD", "UID"}) class ApptainerCreateError(SandboxCreateError): @@ -209,6 +212,56 @@ def _coerce_binds(value: Any) -> list[str]: raise ApptainerCreateError(f"provider_options['binds'] must be a string or list, got {type(value).__name__}") +def _serialize_env_file(env: Mapping[str, str]) -> bytes: + """Serialize environment variables for Apptainer ``--env-file``. + + Apptainer shell-evaluates env files once while reading them. POSIX + shell-quoting preserves each value through that evaluation, while the + accompanying ``--no-eval`` flag prevents a second evaluation during + container environment injection. + """ + assignments: list[str] = [] + for key, value in env.items(): + if not isinstance(key, str) or APPTAINER_ENV_NAME.fullmatch(key) is None: + raise ValueError("Apptainer environment variable names must be POSIX shell identifiers") + if key in APPTAINER_ENV_FILE_READONLY: + raise ValueError("Apptainer --env-file cannot set a shell readonly environment variable") + if not isinstance(value, str): + raise TypeError("Apptainer environment variable values must be strings") + if "\x00" in value: + raise ValueError("Apptainer environment variable values cannot contain NUL") + try: + value.encode("utf-8") + except UnicodeEncodeError: + raise ValueError("Apptainer environment variable values must be UTF-8 encodable") from None + assignments.append(f"{key}={shlex.quote(value)}") + return ("\n".join(assignments) + "\n").encode("utf-8") if assignments else b"" + + +@contextlib.contextmanager +def _private_env_file(staging_dir: Path, content: bytes) -> Iterator[Path | None]: + """Yield a unique mode-0600 env file and remove it on every exit path.""" + if not content: + yield None + return + + fd = -1 + path: Path | None = None + try: + fd, raw_path = tempfile.mkstemp(prefix=".apptainer-env-", dir=staging_dir) + path = Path(raw_path) + os.fchmod(fd, 0o600) + with os.fdopen(fd, "wb") as stream: + fd = -1 + stream.write(content) + yield path + finally: + if fd >= 0: + os.close(fd) + if path is not None: + path.unlink(missing_ok=True) + + class ApptainerProvider: """Sandbox provider backed by the local Apptainer CLI.""" @@ -319,7 +372,7 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: mount_point = self._create_config.mount_point, generate a unique name = INSTANCE_NAME_PREFIX + uuid4().hex. 4. Build argv: [binary, "instance", "start", <--bind staging:mount_point>, - , , <--env ...>, + , , <--env-file ...>, _resource_flags(spec.resources), , image, name]. 5. await self._run(argv, timeout_s=self._create_config.start_timeout_s); on non-zero return, clean up the staging dir and raise @@ -338,6 +391,10 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: if spec.image is None: raise ApptainerCreateError("spec.image is required for the apptainer provider") image = _resolve_image(spec.image) + try: + env_file_content = _serialize_env_file(spec.env) + except (TypeError, ValueError) as e: + raise ApptainerCreateError(f"invalid Apptainer environment: {e}") from e # Extra per-sandbox bind mounts (validated before we allocate anything). extra_binds = _coerce_binds(spec.provider_options.get("binds")) @@ -356,8 +413,6 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: argv += ["--bind", bind] for bind in extra_binds: argv += ["--bind", bind] - for key, value in spec.env.items(): - argv += ["--env", f"{key}={value}"] start_args = list(self._create_config.extra_start_args) resource_limit_flags = _resource_limit_flags(spec.resources) if resource_limit_flags and self._create_config.apply_resource_limits: @@ -369,11 +424,18 @@ async def create(self, spec: SandboxSpec) -> SandboxHandle: argv += resource_limit_flags argv += _resource_passthrough_flags(spec.resources) argv += start_args - argv += [image, name] # start the instance; clean up the staging dir on any failure. try: - code, _out, err = await self._run(argv, timeout_s=self._create_config.start_timeout_s, daemonize=True) + with _private_env_file(staging_dir, env_file_content) as env_file: + if env_file is not None: + argv += ["--no-eval", "--env-file", str(env_file)] + argv += [image, name] + code, _out, err = await self._run( + argv, + timeout_s=self._create_config.start_timeout_s, + daemonize=True, + ) except TimeoutError as e: shutil.rmtree(staging_dir, ignore_errors=True) raise ApptainerCreateError(f"apptainer instance start timed out for image={image!r}: {e}") from e @@ -489,9 +551,7 @@ async def exec( merged_env = dict(getattr(inst, "env", {})) if env: merged_env.update(env) - if merged_env: - for key, value in merged_env.items(): - flags += ["--env", f"{key}={value}"] + env_file_content = _serialize_env_file(merged_env) effective_command = command is_root = user == "root" or user == 0 @@ -505,11 +565,14 @@ async def exec( flags += list(self._exec_config.extra_exec_args) - argv = [self._binary, "exec", *flags, f"instance://{inst.name}", "sh", "-c", effective_command] effective_timeout = timeout_s if timeout_s is not None else self._exec_config.default_timeout_s try: - code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin) + with _private_env_file(inst.staging_dir, env_file_content) as env_file: + if env_file is not None: + flags += ["--no-eval", "--env-file", str(env_file)] + argv = [self._binary, "exec", *flags, f"instance://{inst.name}", "sh", "-c", effective_command] + code, out, err = await self._run(argv, timeout_s=effective_timeout, stdin=stdin) except TimeoutError as e: return SandboxExecResult( stdout=None, diff --git a/responses_api_agents/pinchbench/configs/pinchbench.yaml b/responses_api_agents/pinchbench/configs/pinchbench.yaml index cd5480ce28f..6c89d78ccd6 100644 --- a/responses_api_agents/pinchbench/configs/pinchbench.yaml +++ b/responses_api_agents/pinchbench/configs/pinchbench.yaml @@ -37,6 +37,7 @@ pinchbench_agent: judge_api_key: ${judge_api_key} web_search_provider: brave brave_api_key: ${brave_api_key} + tavily_api_key: ${tavily_api_key} max_concurrent: 2 max_tokens: 16384 context_window: 131072 diff --git a/tests/unit_tests/test_apptainer_provider.py b/tests/unit_tests/test_apptainer_provider.py index 94de183a6e1..b65e191e1b2 100644 --- a/tests/unit_tests/test_apptainer_provider.py +++ b/tests/unit_tests/test_apptainer_provider.py @@ -58,6 +58,10 @@ def _contains_seq(haystack: list[str], needle: list[str]) -> bool: return any(haystack[i : i + len(needle)] == needle for i in range(len(haystack) - len(needle) + 1)) +def _env_file_path(argv: list[str]) -> Path: + return Path(argv[argv.index("--env-file") + 1]) + + def _make_handle( staging: Path, *, @@ -155,6 +159,46 @@ def test_resolve_image() -> None: assert resolve("/tmp/image.sif") == "/tmp/image.sif" +def test_serialize_env_file_quotes_special_characters() -> None: + special = "spaces '$HOME' $(date) `id` \\\n+unicode=zaΕΌΓ³Ε‚Δ‡" + assert ( + apptainer_provider._serialize_env_file({"EMPTY": "", "PLAIN": "value", "SPECIAL": special}) + == f"EMPTY=''\nPLAIN=value\nSPECIAL={shlex.quote(special)}\n".encode() + ) + + +@pytest.mark.parametrize( + "env,error", + [ + ({"NOT-VALID": "value"}, ValueError), + ({1: "value"}, ValueError), + ({"HOME": "/tmp/home"}, ValueError), + ({"VALID": "before\x00after"}, ValueError), + ({"VALID": 1}, TypeError), + ({"VALID": "\ud800"}, ValueError), + ], +) +def test_serialize_env_file_rejects_invalid_input(env: dict[Any, Any], error: type[Exception]) -> None: + with pytest.raises(error): + apptainer_provider._serialize_env_file(env) + + +def test_private_env_file_is_mode_0600_and_always_removed(tmp_path: Path) -> None: + content = b"TOKEN='not-a-real-secret'\n" + + with apptainer_provider._private_env_file(tmp_path, content) as path: + assert path is not None + assert path.read_bytes() == content + assert path.stat().st_mode & 0o777 == 0o600 + assert not path.exists() + + with pytest.raises(RuntimeError): + with apptainer_provider._private_env_file(tmp_path, content) as failed_path: + assert failed_path is not None + raise RuntimeError("failure") + assert not failed_path.exists() + + def test_to_sandbox_status() -> None: to_status = apptainer_provider._to_sandbox_status assert to_status("running") is SandboxStatus.RUNNING @@ -196,7 +240,15 @@ async def test_create_builds_argv_and_runs_probe( staging = tmp_path / "staging" monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) + env_files: list[Path] = [] + def responder(argv: list[str]) -> tuple[int, str, str]: + if "--env-file" in argv: + env_file = _env_file_path(argv) + env_files.append(env_file) + assert env_file.read_bytes() == b"FOO=bar\n" + assert env_file.stat().st_mode & 0o777 == 0o600 + assert "FOO=bar" not in argv if "start" in argv: return (0, "", "") if "exec" in argv: @@ -232,7 +284,7 @@ def responder(argv: list[str]) -> tuple[int, str, str]: assert start_argv[:3] == [FAKE_BINARY, "instance", "start"] assert _contains_seq(start_argv, ["--bind", f"{staging}:/sandbox"]) assert _contains_seq(start_argv, ["--bind", "/data:/data"]) - assert _contains_seq(start_argv, ["--env", "FOO=bar"]) + assert _contains_seq(start_argv, ["--no-eval", "--env-file", str(env_files[0])]) assert _contains_seq(start_argv, ["--cpus", "2.0"]) assert _contains_seq(start_argv, ["--memory", "1024m"]) assert "--nv" in start_argv @@ -244,6 +296,8 @@ def responder(argv: list[str]) -> tuple[int, str, str]: assert f"instance://{handle.sandbox_id}" in probe_argv assert probe_argv[-1] == apptainer_provider.READY_PROBE_COMMAND assert rec.calls[1]["timeout_s"] == 30 + assert len(env_files) == 2 + assert all(not path.exists() for path in env_files) @pytest.mark.parametrize("create_config", [{"apply_resource_limits": False}, {"extra_start_args": ["--fakeroot"]}]) @@ -327,10 +381,20 @@ async def test_create_start_failure_cleans_up( ) -> None: staging = tmp_path / "staging" monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) - provider, _rec = _make_provider(monkeypatch, lambda argv: (1, "", "boom")) + env_file: Path | None = None + + def responder(argv: list[str]) -> tuple[int, str, str]: + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.exists() + assert "not-a-real-secret" not in argv + return (1, "", "boom") + + provider, _rec = _make_provider(monkeypatch, responder) with pytest.raises(apptainer_provider.ApptainerCreateError, match="failed"): - await provider.create(SandboxSpec(image="docker://img")) + await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"})) + assert env_file is not None and not env_file.exists() assert not staging.exists() @@ -340,12 +404,19 @@ async def test_create_start_timeout_cleans_up( staging = tmp_path / "staging" monkeypatch.setattr(apptainer_provider.tempfile, "mkdtemp", lambda prefix: str(staging.mkdir() or staging)) + env_file: Path | None = None + def responder(argv: list[str]) -> tuple[int, str, str]: - raise TimeoutError("slow") + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.exists() + raise TimeoutError(f"slow: {argv}") provider, _rec = _make_provider(monkeypatch, responder) - with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out"): - await provider.create(SandboxSpec(image="docker://img")) + with pytest.raises(apptainer_provider.ApptainerCreateError, match="timed out") as exc_info: + await provider.create(SandboxSpec(image="docker://img", env={"TOKEN": "not-a-real-secret"})) + assert "not-a-real-secret" not in str(exc_info.value) + assert env_file is not None and not env_file.exists() assert not staging.exists() @@ -374,7 +445,17 @@ def responder(argv: list[str]) -> tuple[int, str, str]: # exec # --------------------------------------------------------------------------- # async def test_exec_normal_with_cwd_and_env(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - provider, rec = _make_provider(monkeypatch, lambda argv: (0, "hello", "")) + env_file: Path | None = None + + def responder(argv: list[str]) -> tuple[int, str, str]: + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.read_bytes() == b"A=b\n" + assert env_file.stat().st_mode & 0o777 == 0o600 + assert "A=b" not in argv + return (0, "hello", "") + + provider, rec = _make_provider(monkeypatch, responder) handle = _make_handle(tmp_path) result = await provider.exec(handle, "echo hi", cwd="/work", env={"A": "b"}) @@ -386,23 +467,34 @@ async def test_exec_normal_with_cwd_and_env(fake_binary: str, monkeypatch: pytes argv = rec.calls[0]["argv"] assert argv[:2] == [FAKE_BINARY, "exec"] assert _contains_seq(argv, ["--pwd", "/work"]) - assert _contains_seq(argv, ["--env", "A=b"]) + assert env_file is not None + assert _contains_seq(argv, ["--no-eval", "--env-file", str(env_file)]) assert argv[-4:] == ["instance://nemo-gym-x", "sh", "-c", "echo hi"] assert rec.calls[0]["timeout_s"] == 180 # default exec timeout + assert not env_file.exists() async def test_exec_reapplies_create_env_and_overrides_call_env( fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - provider, rec = _make_provider(monkeypatch, lambda argv: (0, "", "")) + env_file: Path | None = None + + def responder(argv: list[str]) -> tuple[int, str, str]: + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.read_bytes() == b"A=from-call\nB=base\n" + return (0, "", "") + + provider, rec = _make_provider(monkeypatch, responder) handle = _make_handle(tmp_path, env={"A": "from-create", "B": "base"}) await provider.exec(handle, "env", env={"A": "from-call"}) argv = rec.calls[0]["argv"] - assert _contains_seq(argv, ["--env", "A=from-call"]) - assert _contains_seq(argv, ["--env", "B=base"]) + assert "A=from-call" not in argv + assert "B=base" not in argv assert "A=from-create" not in argv + assert env_file is not None and not env_file.exists() async def test_exec_empty_streams_are_strings( @@ -458,15 +550,26 @@ async def test_exec_passes_stdin(fake_binary: str, monkeypatch: pytest.MonkeyPat async def test_exec_timeout(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + env_file: Path | None = None + def responder(argv: list[str]) -> tuple[int, str, str]: - raise TimeoutError("too slow") + nonlocal env_file + env_file = _env_file_path(argv) + assert env_file.exists() + raise TimeoutError(f"too slow: {argv}") provider, _rec = _make_provider(monkeypatch, responder) - result = await provider.exec(_make_handle(tmp_path), "sleep 99", timeout_s=1) + result = await provider.exec( + _make_handle(tmp_path, env={"TOKEN": "not-a-real-secret"}), + "sleep 99", + timeout_s=1, + ) assert result.return_code == apptainer_provider.SANDBOX_RUNTIME_RETURN_CODE assert result.error_type == "timeout" assert result.stdout is None + assert "not-a-real-secret" not in (result.stderr or "") + assert env_file is not None and not env_file.exists() async def test_exec_runtime_failure(fake_binary: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py index 06942f3f140..fdb69311745 100644 --- a/tests/unit_tests/test_benchmarks.py +++ b/tests/unit_tests/test_benchmarks.py @@ -28,6 +28,15 @@ def _mock_global_config(config: dict = None): return OmegaConf.create(config or {}) +def test_pinchbench_tavily_key_is_derived_from_environment() -> None: + repo_root = Path(__file__).parents[2] + benchmark = safe_load((repo_root / "benchmarks/pinchbench/config.yaml").read_text()) + agent = safe_load((repo_root / "responses_api_agents/pinchbench/configs/pinchbench.yaml").read_text()) + + assert benchmark["tavily_api_key"] == "${oc.env:PINCHBENCH_TAVILY_API_KEY,null}" + assert agent["pinchbench_agent"]["responses_api_agents"]["pinchbench"]["tavily_api_key"] == ("${tavily_api_key}") + + @pytest.fixture(autouse=True) def _mock_port_allocation(monkeypatch): """Keep benchmark-discovery tests independent of socket availability."""