Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/full-test-suite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ jobs:
echo "UV_CACHE_DIR=$UV_CACHE_DIR" >> "$GITHUB_ENV"
bash scripts/ci/install_test_dependencies.sh
curl -LsSf "$UV_INSTALL_URL" | sh
export PATH="$HOME/.local/bin:$PATH"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
uv --version
uv venv
source .venv/bin/activate
uv sync --extra dev
Expand Down
4 changes: 3 additions & 1 deletion responses_api_agents/anyswe_agent/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,9 @@ def _as_list(value: Any) -> list[str]:

def _r2e_resolved(instance: Dict[str, Any], log: str) -> bool:
statuses: Dict[str, str] = {}
for line in log.splitlines():
# pytest can color both the status and parts of the test ID in its summary.
plain_log = re.sub(r"\x1b\[[0-9;]*m", "", log)
for line in plain_log.splitlines():
fields = line.strip().replace(" - ", " ").split()
if len(fields) > 1 and fields[0] in ("PASSED", "FAILED", "ERROR", "XFAIL", "SKIPPED"):
statuses[fields[1]] = fields[0]
Expand Down
15 changes: 10 additions & 5 deletions responses_api_agents/anyswe_agent/tests/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,8 @@ def test_dataset_routes(self) -> None:
assert _dataset_family("SWE-bench_Multilingual") == "swebench_multilingual"
assert _dataset_family("R2E-Gym/R2E-Gym-Subset") == "r2e"

def test_r2e_required_tests(self) -> None:
@pytest.mark.parametrize("colored", [False, True])
def test_r2e_required_tests(self, colored: bool) -> None:
instance = {
"FAIL_TO_PASS": ["tests/test_a.py::test_fix"],
"PASS_TO_PASS": ["tests/test_b.py::test_stays_green"],
Expand All @@ -258,11 +259,15 @@ def test_r2e_required_tests(self) -> None:
"PASSED tests/test_b.py::test_stays_green",
]
)
failing = passing.replace("PASSED tests/test_a.py", "FAILED tests/test_a.py")
skipped = passing.replace("PASSED tests/test_a.py", "SKIPPED tests/test_a.py")
if colored:
passing = passing.replace("PASSED", "\x1b[32mPASSED\x1b[0m")
passing = passing.replace("::test_fix", "::\x1b[1mtest_fix\x1b[0m")
passing = passing.replace("::test_stays_green", "::\x1b[1mtest_stays_green\x1b[0m")
assert _r2e_resolved(instance, passing)
assert not _r2e_resolved(instance, failing)
assert not _r2e_resolved(instance, skipped)
for status in ("FAILED", "ERROR", "XFAIL", "SKIPPED"):
nonpassing = passing.replace("PASSED", status, 1)
assert not _r2e_resolved(instance, nonpassing)
assert not _r2e_resolved(instance, passing.splitlines()[0])

def test_safe_config_redacts_provider_key(self) -> None:
class Params:
Expand Down
12 changes: 10 additions & 2 deletions responses_api_agents/anyswe_agent/tests/test_r2e_test_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@
import asyncio
import json
import os
import re
import subprocess
import sys
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch

import pytest

from responses_api_agents.anyswe_agent.app import AnySweAgent, AnySweInstanceConfig, _r2e_resolved
from responses_api_agents.anyswe_agent.tests.test_app import _config

Expand Down Expand Up @@ -43,7 +46,11 @@ def _run(script: str, cwd: Path) -> str:


class TestR2ETestSetup:
def test_run_tests_finds_the_hidden_tests_only_after_the_link(self, tmp_path: Path) -> None:
@pytest.mark.parametrize("colored", [False, True])
def test_run_tests_finds_the_hidden_tests_only_after_the_link(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, colored: bool
) -> None:
monkeypatch.setenv("PY_COLORS", "1" if colored else "0")
repo, tests = _r2e_image_layout(tmp_path)
instance = {"FAIL_TO_PASS": ["r2e_tests/test_1.py::test_fixed"], "PASS_TO_PASS": []}

Expand All @@ -55,7 +62,8 @@ def test_run_tests_finds_the_hidden_tests_only_after_the_link(self, tmp_path: Pa
setup = AnySweAgent._r2e_test_setup_script(str(repo), str(tests))
after = _run(setup + "bash run_tests.sh", repo)
assert f">>>>> Linked {tests} into {repo}" in after
assert "PASSED r2e_tests/test_1.py::test_fixed" in after
plain_after = re.sub(r"\x1b\[[0-9;]*m", "", after)
assert "PASSED r2e_tests/test_1.py::test_fixed" in plain_after
assert _r2e_resolved(instance, after) is True
assert (repo / "r2e_tests").is_symlink()

Expand Down
65 changes: 65 additions & 0 deletions tests/unit_tests/test_ci_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,71 @@ def test_coverage_gate_compares_fractional_percentages() -> None:
assert coverage_report["fail_under"] == 95.0


def test_full_test_server_setup_exposes_uv_in_current_and_later_steps(tmp_path: Path) -> None:
workflow = yaml.safe_load(FULL_TEST_WORKFLOW.read_text())
setup = next(step["run"] for step in workflow["jobs"]["server-suite"]["steps"] if step["name"] == "Setup for test")
home = tmp_path / "home"
home.mkdir()
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
github_path = tmp_path / "github-path"
calls = tmp_path / "uv-calls"
installer = tmp_path / "install-uv.sh"
installer.write_text(
'mkdir -p "$HOME/.local/bin"\n'
"cat > \"$HOME/.local/bin/uv\" <<'UV'\n"
"#!/bin/sh\n"
'echo "$*" >> "$UV_TEST_CALLS"\n'
'case "$1" in\n'
" --version) echo 'uv 0.11.29' ;;\n"
" venv) mkdir -p .venv/bin; : > .venv/bin/activate ;;\n"
" sync) ;;\n"
" *) exit 2 ;;\n"
"esac\n"
"UV\n"
'chmod +x "$HOME/.local/bin/uv"\n'
)
# Replace network and system package setup, not the workflow's PATH handling.
for name, body in {"curl": 'cat "$UV_TEST_INSTALLER"', "bash": "exit 0"}.items():
executable = bin_dir / name
executable.write_text(f"#!/bin/sh\n{body}\n")
executable.chmod(0o755)
env = os.environ.copy()
env.update(
HOME=str(home),
PATH=f"{bin_dir}:/usr/bin:/bin",
GITHUB_ENV=str(tmp_path / "github-env"),
GITHUB_PATH=str(github_path),
UV_INSTALL_URL=workflow["env"]["UV_INSTALL_URL"],
UV_TEST_INSTALLER=str(installer),
UV_TEST_CALLS=str(calls),
)
bash = shutil.which("bash")
assert bash is not None

result = subprocess.run(
[bash, "--noprofile", "--norc", "-eo", "pipefail", "-c", setup],
cwd=tmp_path,
env=env,
capture_output=True,
text=True,
)

assert result.returncode == 0, result.stderr
assert calls.read_text().splitlines() == ["--version", "venv", "sync --extra dev"]
assert github_path.read_text().splitlines() == [str(home / ".local" / "bin")]
env["PATH"] = f"{github_path.read_text().strip()}:{env['PATH']}"
later_step = subprocess.run(
[bash, "--noprofile", "--norc", "-ec", "uv --version"],
cwd=tmp_path,
env=env,
capture_output=True,
text=True,
)
assert later_step.returncode == 0, later_step.stderr
assert later_step.stdout.strip() == "uv 0.11.29"


def test_full_test_suite_installs_telemetry_extra_for_coverage_gate() -> None:
# The full test suite measures coverage against the repo-wide gate
# (pyproject.toml fail_under). The tests/unit_tests/telemetry tests are gated on
Expand Down
Loading