Skip to content
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
6a2b5eb
build: lock managed environment dependencies and sync installs
svcnemo-autobot Oct 21, 2026
fe3ba43
build: reconcile locked environments with upstream changes
svcnemo-autobot Oct 21, 2026
e4269bd
fix(env): isolate wheel overlays and accept installed prereleases
svcnemo-autobot Oct 21, 2026
42968af
fix(env): align locked install consumers and retain Harbor pathspec
svcnemo-autobot Oct 21, 2026
399326f
merge(main): resolve Slurm tests alongside locked installs
svcnemo-autobot Oct 21, 2026
e914774
merge(main): preserve safe progress writes with eval resubmit
svcnemo-autobot Oct 21, 2026
df99d7a
fix(env): retain upstream WMT PyArrow compatibility cap
svcnemo-autobot Oct 21, 2026
dc605e1
merge(main): preserve locked environments with Harbor tasks
svcnemo-autobot Oct 21, 2026
52a9ce4
docs(long_transduction): fix stale prepare comment
marta-sd Oct 22, 2026
45fe61e
docs: replace requirements.txt with pyproject.toml
marta-sd Oct 22, 2026
5c7f2be
test: forbid requirements.txt in servers
marta-sd Oct 22, 2026
41fc117
build(env): merge lockfile documentation and policy followups
svcnemo-autobot Oct 22, 2026
b97c58d
fix(eval): preserve dataset prepare dependency arguments
svcnemo-autobot Oct 22, 2026
35616a5
build(env): reconcile locked environments with upstream changes
svcnemo-autobot Oct 22, 2026
9e0fa6e
build(env): merge upstream main into locked environments
svcnemo-autobot Oct 22, 2026
95e3e3b
fix(env): preserve custom component and parent compatibility
svcnemo-autobot Oct 22, 2026
f1cf954
docs(env): retain the explicit lockfile commit instruction
svcnemo-autobot Oct 22, 2026
539d189
test(sandbox): emulate root in permission transfer fixtures
svcnemo-autobot Oct 22, 2026
e9f822a
build(env): merge upstream to resolve runtime test conflict
svcnemo-autobot Oct 23, 2026
2c597d1
build(env): merge origin/main as requested
svcnemo-autobot Oct 23, 2026
079db01
test(checkpoint): wait for activation before preparing
svcnemo-autobot Oct 23, 2026
9c75ada
test(sandbox): emulate root modes in BusyBox transfer fixture
svcnemo-autobot Oct 23, 2026
f7278e6
test(terminus): isolate interrupt fixture from prompt hooks
svcnemo-autobot Oct 23, 2026
19c4b5a
fix(orchestration): acquire resubmit locks exclusively
svcnemo-autobot Oct 23, 2026
9159044
Merge origin/main to resolve checkpoint test comment conflict
svcnemo-autobot Oct 23, 2026
7777075
build(env): lock the new SWE Together environment
svcnemo-autobot Oct 23, 2026
5bebd55
build(env): reconcile upstream dependency locks
svcnemo-autobot Oct 23, 2026
6c44287
test(env): read sandbox extras from the locked project
svcnemo-autobot Oct 24, 2026
6deb120
fix(env): address locked environment review regressions
ko3n1g Nov 2, 2026
38808ba
Merge pull request #2 from ko3n1g/ko3n1g/fix/environment-lockfile-review
nemo-autobot-origin[bot] Nov 2, 2026
cbc0433
fix(env): repair locked setup regressions and coverage fixtures
svcnemo-autobot Nov 2, 2026
a6c17f5
chore(env): merge origin/main into locked environments
svcnemo-autobot Nov 2, 2026
1dd1950
chore(env): merge origin/main server runtime refactor
svcnemo-autobot Nov 2, 2026
982c264
chore(env): merge main and resolve lockfile conflicts
ko3n1g Nov 4, 2026
97d9b77
chore(env): merge latest main runtime changes
ko3n1g Nov 4, 2026
02151a6
fix(env): repair runtime pins and locked sync assertions
ko3n1g Nov 4, 2026
558eeb9
fix(pool): preserve timeout evidence when trajectory capture fails
ko3n1g Nov 4, 2026
deb1f9d
chore(env): merge main recovery and preparation changes
ko3n1g Nov 4, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
5 changes: 3 additions & 2 deletions .agents/skills/add-benchmark/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ Before starting, determine which type of benchmark you're adding:
- Agent's `/run` endpoint wraps the external library
- Pre-process from Gym schema to library input, post-process back to `BaseVerifyResponse`
- Reproduce publicly reported numbers with the original repo first, then reproduce again after Gym integration
- Add the dependency in `requirements.txt`
- Add the dependency in `pyproject.toml` and commit its `uv.lock`

## Workflow

Expand All @@ -47,7 +47,8 @@ resources_servers/my_benchmark/
├── configs/my_benchmark.yaml
├── data/.gitignore
├── tests/test_app.py
├── requirements.txt
├── pyproject.toml
├── uv.lock
└── README.md
```

Expand Down
2 changes: 1 addition & 1 deletion .agents/skills/add-benchmark/references/patterns.md
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,7 @@ class ExternalBenchmarkAgent(SimpleResponsesAPIAgent):
)
```

Add the dependency in `requirements.txt`. If needs are more complex than pip packages, use `setup.py` or `pyproject.toml`.
Add the dependency in `pyproject.toml`, then run `uv lock` and commit `uv.lock`.

Reproduction requirement: run the original repo first, reproduce published numbers, then integrate into Gym and reproduce again. This decouples Gym integration bugs from benchmark bugs.

Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/full-test-suite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -239,10 +239,14 @@ jobs:
source .venv/bin/activate
ng_init_resources_server +entrypoint=resources_servers/hello_world
test -f resources_servers/hello_world/app.py
test -f resources_servers/hello_world/requirements.txt
test -f resources_servers/hello_world/pyproject.toml
test -f resources_servers/hello_world/configs/hello_world.yaml
test -d resources_servers/hello_world/tests
grep -q "nemo-gym" resources_servers/hello_world/requirements.txt
grep -q "nemo-gym" resources_servers/hello_world/pyproject.toml
# Scaffolding emits dependency metadata; resolve and verify the lock before setup.
uv lock --project resources_servers/hello_world
uv lock --check --project resources_servers/hello_world
test -f resources_servers/hello_world/uv.lock
echo '{"responses_create_params":{"input":[{"role":"user","content":"Hello!"}]},"expected_answer":"Mock response.","verifier_metadata":{}}' \
> resources_servers/hello_world/data/example.jsonl

Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ If DCO checks fail after you have already pushed, see the [Development Setup Gui

- Declare each new import in the narrowest appropriate dependency file. Keep
environment-, agent-, and model-specific packages in that component's
`pyproject.toml` or `requirements.txt`; add a root dependency only when shared
`pyproject.toml` (with its `uv.lock`); add a root dependency only when shared
Gym runtime code requires it.
- Root dependencies in `pyproject.toml` must record why they are needed, when
they were updated, and their license. Do not introduce a license incompatible
Expand Down
9 changes: 5 additions & 4 deletions benchmarks/automationbench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,12 @@ omitted and the agent logs a warning.

## Install

The env package is not on PyPI, install it into the agent's venv for data prep.
Running environment will create its own venv.
The preparation package is not on PyPI. Its committed project lock pins the
upstream source. `gym eval prepare` syncs this project into an isolated
preparation environment; it does not install into the agent or parent venv.

```bash
uv pip install -e benchmarks/automationbench
uv sync --locked --project benchmarks/automationbench
```

This pulls `automation-bench` from the upstream repo. Note upstream declares
Expand All @@ -61,7 +62,7 @@ This pulls `automation-bench` from the upstream repo. Note upstream declares
## Data

```bash
python benchmarks/automationbench/prepare.py
uv run --locked --project benchmarks/automationbench python benchmarks/automationbench/prepare.py
```

Pulls the public taskset from the installed `automation-bench` package (600 tasks: 6 domains x 100).
2 changes: 1 addition & 1 deletion benchmarks/automationbench/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ automationbench:
prepare_script: benchmarks/automationbench/prepare.py
# prepare() imports automationbench_env from this directory's own
# package, which is not a Gym dependency.
prepare_dependencies: ['-e', benchmarks/automationbench]
prepare_project: benchmarks/automationbench
num_repeats: 1
license: MIT

Expand Down
4 changes: 2 additions & 2 deletions benchmarks/automationbench/prepare.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ def prepare(
# "not installed" sends people to reinstall something already present.
raise SystemExit(
f"Could not import automationbench_env ({type(exc).__name__}: {exc}).\n"
"If the package is missing, install it with:\n"
" uv pip install -e benchmarks/automationbench"
"If the package is missing, sync and use its isolated environment from the repo root:\n"
" uv run --locked --project benchmarks/automationbench python benchmarks/automationbench/prepare.py"
) from exc

env = load_environment(domains=domains, max_turns=max_turns, toolset=TOOLSET)
Expand Down
17 changes: 15 additions & 2 deletions benchmarks/automationbench/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
[project]
name = "automationbench-env"
version = "0.1.0"
requires-python = ">=3.13"
requires-python = ">=3.13.14"
dependencies = [
"automation-bench @ git+https://github.com/zapier/AutomationBench.git@4a8e106",
"nemo-gym",
"automation-bench @ git+https://github.com/zapier/AutomationBench.git@4a8e1061254004d9dac807054eed33fad7d1ff14",
# verifiers requires openai unbounded. Left free it pulls openai 3.x and
# openai-agents 0.22, which imports PromptCacheOptions -- a name openai
# 2.44.0 does not have. Installing into a running Gym cannot replace the
Expand All @@ -20,3 +21,15 @@ build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["automationbench_env"]

[tool.uv]
constraint-dependencies = ["grpcio>=1.81.0rc1", "msgpack>=1.2.1"]
override-dependencies = ["spacy>=3.8.13", "wcwidth; sys_platform == 'never'", "opencv-python-headless; sys_platform == 'never'", "pynvvideocodec; sys_platform == 'never'", "torchcodec; sys_platform == 'never'", "torchvision; sys_platform == 'never'", "torchaudio; sys_platform == 'never'", "pycountry; sys_platform == 'never'", "gprof2dot; sys_platform == 'never'", "pathspec; sys_platform == 'never'"]
exclude-dependencies = ["fasttext-wheel", "alembic", "blinker", "cryptography", "flask", "flask-cors", "graphene", "graphql-core", "graphql-relay", "gunicorn", "huey", "prettytable", "skops", "sqlalchemy", "werkzeug", "waitress", "diskcache"]

[tool.uv.sources]
"nemo-gym" = { path = "../..", editable = true }
"nemo-lens" = { git = "https://github.com/NVIDIA-NeMo/Lens.git", rev = "4d045c03186b90afdc815db46b671f6a21bc7e98" }

[tool.uv.workspace]
members = []
12 changes: 12 additions & 0 deletions benchmarks/automationbench/tests/test_prepare.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
# SPDX-License-Identifier: Apache-2.0

import json
import sys
import types
from pathlib import Path
from unittest.mock import patch

import pytest
from pydantic import TypeAdapter

from benchmarks.automationbench import prepare
Expand Down Expand Up @@ -75,3 +77,13 @@ def test_example_rows_carry_task_ids_and_are_clean_under_the_verifiers_agent_sch

assert [row["task_id"] for row in rows] == [f"{row['info']['task_name']}/{row['example_id']}" for row in rows]
assert _validate(rows).clean


def test_missing_dependency_guidance_uses_isolated_project(monkeypatch):
monkeypatch.setitem(sys.modules, "verifiers", None)
command = "uv run --locked --project benchmarks/automationbench python benchmarks/automationbench/prepare.py"
with pytest.raises(SystemExit, match="Could not import automationbench_env") as error:
prepare.prepare()
assert command in str(error.value)
readme = Path(prepare.__file__).with_name("README.md").read_text()
assert command in readme
Loading
Loading