Skip to content

feat(simulation): integrate BEHAVIOR with R1 Pro navigation and manipulation #10734

feat(simulation): integrate BEHAVIOR with R1 Pro navigation and manipulation

feat(simulation): integrate BEHAVIOR with R1 Pro navigation and manipulation #10734

Workflow file for this run

name: ci
on:
push:
branches: [main]
pull_request:
merge_group:
workflow_dispatch:
inputs:
force-cachix-build:
description: Publish to Cachix even if the inputs marker says nothing changed
type: boolean
default: false
concurrency:
# Cancels runs from previous pushes in a PR.
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
permissions: {}
env:
# Disable default package installs. Each job should explicitly install a group.
UV_NO_SYNC: "1"
jobs:
compute-ros-pin:
# Extracts the ros-dev image digest pinned in docker/ros-dev-pin/Dockerfile
# so downstream jobs can reference it from their `container:` field.
# That will be updated by Dependabot, and keep CI tests stable.
runs-on: ubuntu-latest
permissions:
contents: read # For checkout
outputs:
digest: ${{ steps.read.outputs.digest }}
steps:
- name: Checkout
uses: actions/checkout@v7
- id: read
run: |
DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' docker/ros-dev-pin/Dockerfile)
if [ -z "$DIGEST" ]; then
echo "::error::No sha256 digest found in docker/ros-dev-pin/Dockerfile"
exit 1
fi
echo "digest=$DIGEST" >> "$GITHUB_OUTPUT"
cachix-build-check:
# Holds no secrets, so it runs on fork PRs too: forks need its inputs-hash
# output to restore the base branch's publish marker and download the
# prebuilt binaries. Only cachix-build itself is fork-guarded.
if: ${{ !cancelled() }}
timeout-minutes: 10
runs-on: ubuntu-latest
permissions:
contents: read # For checkout
env:
CACHIX_CACHE_NAME: dimensionalos
outputs:
needs-build-linux: ${{ steps.decide.outputs.linux }}
needs-build-macos: ${{ steps.decide.outputs.macos }}
inputs-hash: ${{ steps.hash.outputs.inputs-hash }}
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Compute native-module inputs hash
id: hash
run: python3 bin/build-native-modules --inputs-hash
- name: Probe Linux publish marker
id: marker-linux
uses: actions/cache/restore@v6
with:
path: .cachix-marker
key: cachix-published-Linux-X64-${{ steps.hash.outputs.inputs-hash }}
lookup-only: true
- name: Probe macOS publish marker
id: marker-macos
uses: actions/cache/restore@v6
with:
path: .cachix-marker
key: cachix-published-macOS-ARM64-${{ steps.hash.outputs.inputs-hash }}
lookup-only: true
- name: Decide which platforms need a publish
id: decide
env:
FORCE: ${{ inputs.force-cachix-build }}
LINUX_HIT: ${{ steps.marker-linux.outputs.cache-hit }}
MACOS_HIT: ${{ steps.marker-macos.outputs.cache-hit }}
run: |
emit() { # $1=output name, $2=this platform's marker hit
if [ "$FORCE" != "true" ] && [ "$2" = "true" ]; then
echo "$1=false" >> "$GITHUB_OUTPUT"
else
echo "$1=true" >> "$GITHUB_OUTPUT"
fi
}
emit linux "$LINUX_HIT"
emit macos "$MACOS_HIT"
cachix-build:
# ⚠️ NEVER RUN ON UNTRUSTED (fork) CODE ⚠️
# This job has write access to CACHIX_AUTH_TOKEN. Any workflow change that
# reaches a `run:` step here could exfiltrate the token, letting an attacker
# push poisoned binaries to the Cachix cache — which subsequent CI runs and
# every dev with the substituter configured would unwittingly install.
# Allowed only on trusted events: push, merge_group (a maintainer must
# approve + queue the PR), and same-repo PRs — NEVER on fork `pull_request`s.
# The `if:` guard below is the only thing keeping forks out — DO NOT REMOVE.
# merge_group runs the PR's *merged* workflow with secrets, so .github/
# changes must be reviewed before a fork PR is queued.
#
# Gated on cachix-build-check so an entry into the `cachix` environment
# always corresponds to a real publish — see that job for why.
if: |
!cancelled() &&
(github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) &&
needs.cachix-build-check.outputs.needs-build-linux == 'true'
needs: cachix-build-check
timeout-minutes: 300
environment: cachix
runs-on: ubuntu-latest
permissions:
contents: read # For checkout
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Free runner disk space
uses: BRAINSia/free-disk-space@v2.1.3
- name: Install Nix (with Cachix substituter)
env:
INPUT_EXTRA_NIX_CONFIG: |
extra-substituters = https://dimensionalos.cachix.org
extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
INPUT_SET_AS_TRUSTED_USER: "true"
run: bash docker/ros/install-nix.sh
- name: Cache Nix fetcher + eval cache
# ~/.cache/nix holds the flake fetcher cache (the github tarballs of
# each `nix build github:…` ref) and the eval cache (compiled flake
# outputs). ~/.local/state/nix holds profile/gcroot bookkeeping.
# Restoring these skips the ~20s/module evaluation cost, which is
# paid every cold run otherwise. /nix/store is intentionally NOT
# cached here — that's the slow heavy part that Cachix handles.
uses: actions/cache@v6
with:
path: |
~/.cache/nix
~/.local/state/nix
key: nix-fetcher-build-${{ runner.os }}-${{ hashFiles('bin/build-native-modules', '**/flake.lock', '**/flake.nix') }}
restore-keys: nix-fetcher-build-${{ runner.os }}-
- name: Authenticate Cachix
uses: cachix/cachix-action@v17
with:
name: dimensionalos
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build native modules
env:
BUILD_WORKERS: "1"
run: python3 bin/build-native-modules
- name: Verify Cachix holds the built paths
env:
CACHIX_CACHE_NAME: dimensionalos
run: python3 bin/build-native-modules --verify-published
- name: Record the published inputs manifest
# The manifest is forensics for the marker saved below; links.txt maps
# each result symlink to its store path so consumers can materialise
# the binaries with --link-results instead of re-running nix.
env:
CACHIX_CACHE_NAME: dimensionalos
run: |
mkdir -p .cachix-marker
python3 bin/build-native-modules --inputs-hash \
> .cachix-marker/inputs-hash.txt 2> .cachix-marker/manifest.txt
python3 bin/build-native-modules --record-links > .cachix-marker/links.txt
- name: Save publish marker
uses: actions/cache/save@v6
with:
path: .cachix-marker
key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }}
cachix-build-macos:
# ⚠️ NEVER RUN ON UNTRUSTED (fork) CODE ⚠️ — holds CACHIX_AUTH_TOKEN, same
# as cachix-build; the `if:` guard below is the only thing keeping forks
# out. A separate job (not a matrix leg of cachix-build) for two reasons:
# its own job-level `if` must gate entry into the `cachix` environment on
# the macOS marker alone (matrix context isn't available in `if:`), and a
# macOS build failure must not fail cachix-build and block the Linux tests.
if: |
!cancelled() &&
(github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) &&
needs.cachix-build-check.outputs.needs-build-macos == 'true'
needs: cachix-build-check
timeout-minutes: 300
environment: cachix
runs-on: macos-14 # GitHub-hosted Apple silicon → runner.arch is ARM64
permissions:
contents: read # For checkout
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install Nix (with Cachix substituter)
env:
INPUT_EXTRA_NIX_CONFIG: |
extra-substituters = https://dimensionalos.cachix.org
extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
INPUT_SET_AS_TRUSTED_USER: "true"
run: bash docker/ros/install-nix.sh
- name: Cache Nix fetcher + eval cache
# Key includes runner.os, so this is a macOS-only entry, distinct from
# the Linux fetcher cache.
uses: actions/cache@v6
with:
path: |
~/.cache/nix
~/.local/state/nix
key: nix-fetcher-build-${{ runner.os }}-${{ hashFiles('bin/build-native-modules', '**/flake.lock', '**/flake.nix') }}
restore-keys: nix-fetcher-build-${{ runner.os }}-
- name: Authenticate Cachix
uses: cachix/cachix-action@v17
with:
name: dimensionalos
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
- name: Build native modules
env:
BUILD_WORKERS: "1"
run: python3 bin/build-native-modules
- name: Verify Cachix holds the built paths
env:
CACHIX_CACHE_NAME: dimensionalos
run: python3 bin/build-native-modules --verify-published
- name: Record the published inputs manifest
env:
CACHIX_CACHE_NAME: dimensionalos
run: |
mkdir -p .cachix-marker
python3 bin/build-native-modules --inputs-hash \
> .cachix-marker/inputs-hash.txt 2> .cachix-marker/manifest.txt
python3 bin/build-native-modules --record-links > .cachix-marker/links.txt
- name: Save publish marker
uses: actions/cache/save@v6
with:
path: .cachix-marker
key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }}
lint:
timeout-minutes: 10
runs-on: ubuntu-latest
permissions:
contents: read # For checkout
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Reject Git sources in root pyproject.toml
run: |
python3 - <<'PY'
import sys
import tomllib
with open("pyproject.toml", "rb") as file:
config = tomllib.load(file)
sources = config.get("tool", {}).get("uv", {}).get("sources", {})
rejected = []
for name, source in sources.items():
entries = source if isinstance(source, list) else [source]
if any("git" in entry for entry in entries):
rejected.append(name)
if rejected:
sys.exit(
"::error file=pyproject.toml::Git sources are not allowed in root "
"pyproject.toml. Use published packages instead: " + ", ".join(rejected)
)
PY
- name: Install uv
uses: astral-sh/setup-uv@v10.0.1
with:
enable-cache: true
prune-cache: true
- name: Install lint dependencies
run: uv sync --only-group lint --frozen
- name: Mypy
run: uv run mypy
# The deno-fmt pre-commit hook needs deno on PATH.
- name: Extract Deno version
id: deno-version
run: echo "version=$(grep -oP 'DENO_VERSION = "\K[^"]+' dimos/utils/deno.py)" >> "$GITHUB_OUTPUT"
- name: Install Deno
uses: denoland/setup-deno@v2
with:
deno-version: ${{ steps.deno-version.outputs.version }}
cache: true
- name: Run pre-commit
uses: pre-commit/action@v3.0.1
env:
# clippy and fmt are checked in the rust job, so we can skip them here
SKIP: cargo-fmt,cargo-clippy
rust:
timeout-minutes: 30
runs-on: ubuntu-latest
permissions:
contents: read # For checkout
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
with:
# Only main writes the cache, so a PR run cannot evict main's entry
# from the repo's 10GB budget. PR runs still restore from it.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: cargo fmt
run: cargo fmt --all -- --check
- name: cargo clippy
run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
- name: cargo test
run: cargo test --workspace --all-features --locked
- name: Install uv
uses: astral-sh/setup-uv@v10.0.1
with:
enable-cache: true
prune-cache: true
# Python readback and dataset export tests decode recorded JPEGs with PyTurboJPEG.
- name: Install Python audio and image dependencies
run: |
sudo apt-get update
sudo apt-get install -y portaudio19-dev libturbojpeg
- name: Build and test PyO3 bindings
# PyO3 extension modules (cdylib + `extension-module`) are the one case
# that still needs a plain build: pytest imports the compiled .so.
run: |
uv sync --group tests --frozen
source .venv/bin/activate
maturin develop -m dimos/mapping/ray_tracing/rust/py/Cargo.toml
maturin develop -m dimos/navigation/nav_3d/mls_planner/rust/py/Cargo.toml
python -c "import dimos_voxel_ray_tracing, dimos_mls_planner"
python -m pytest -c /dev/null --rootdir . --noconftest --import-mode=importlib -p no:cacheprovider -v \
dimos/mapping/ray_tracing/test_voxel_map.py \
dimos/mapping/ray_tracing/test_transformer.py \
dimos/navigation/nav_3d/mls_planner/test_mls_planner.py \
dimos/navigation/nav_3d/mls_planner/test_transformer.py
- name: Bake e2e tests
run: uv run pytest -m bake_e2e dimos/cli/bake/test_bake_e2e.py --no-cov
- name: Native module e2e tests
run: |
cargo build --release --locked -p dimos-livox -p dimos-virtual-mid360
uv run pytest -m native_e2e \
dimos/hardware/sensors/lidar/livox/test_e2e.py \
dimos/robot/manipulators/openyam/blueprints/test_learning_collection_e2e.py \
dimos/imitation/test_datacollection_e2e.py --no-cov
native:
name: Native builds (C++ and Rust)
timeout-minutes: 30
runs-on: ubuntu-latest
permissions:
contents: read # For checkout
steps:
- name: Checkout
uses: actions/checkout@v7
# The SDK headers include both transports, so liblcm and zenoh are both
# needed to build at all. Zenoh has no apt package, so it comes from its
# release
- name: Install system dependencies
env:
ZENOH_VERSION: 1.10.0
run: |
sudo apt-get update
# lcm.pc carries `Requires: glib-2.0`, so pkg-config cannot resolve
# lcm at all without glib's own .pc file
sudo apt-get install -y liblcm-dev libglib2.0-dev nlohmann-json3-dev
base=https://github.com/eclipse-zenoh
curl -fsSL -o /tmp/zenoh-c.zip \
"$base/zenoh-c/releases/download/$ZENOH_VERSION/zenoh-c-$ZENOH_VERSION-x86_64-unknown-linux-gnu-standalone.zip"
curl -fsSL -o /tmp/zenoh-cpp.zip \
"$base/zenoh-cpp/releases/download/$ZENOH_VERSION/zenohcpp-$ZENOH_VERSION-standalone.zip"
# A release asset can be replaced in place, so the bytes are pinned too
printf '%s /tmp/zenoh-c.zip\n%s /tmp/zenoh-cpp.zip\n' \
1168b3dffa7f4f48ffabfd640a3878ec0527c0a612ce825aa6f93e2cd05762d1 \
7a50a74e98fd1e1e7ed8461b8490b30d67f878649f0727dea2cf0936501780af \
| sha256sum -c -
# The zips are laid out for /usr/local, where cmake and the loader
# already look
sudo unzip -q /tmp/zenoh-c.zip -d /usr/local
sudo unzip -q /tmp/zenoh-cpp.zip -d /usr/local
sudo ldconfig
- name: Configure
run: cmake -S native/cpp -B build/native-cpp -DDIMOS_NATIVE_BUILD_TESTS=ON
- name: Build
run: cmake --build build/native-cpp -j
- name: Test
run: ctest --test-dir build/native-cpp --output-on-failure
- name: Find C++ native modules
id: find-modules
run: |
modules=$(find . -name flake.nix \
-not -path './.git/*' \
-not -path '*/build/*' \
-not -path '*/result/*' \
-printf '%h\n' \
| while read -r dir; do
[ -f "$dir/CMakeLists.txt" ] && echo "${dir#./}"
done \
| sort)
if [ -z "$modules" ]; then
echo "::error::No C++ native modules found - the discovery glob is wrong"
exit 1
fi
echo "Found C++ native modules:"
echo "$modules"
{
echo "modules<<EOF"
echo "$modules"
echo "EOF"
} >> "$GITHUB_OUTPUT"
- name: Free runner disk space
uses: BRAINSia/free-disk-space@v2.1.3
- name: Install Nix
# Substituting needs only the public key, so no auth token is used here
# and this stays safe on fork PRs.
env:
INPUT_EXTRA_NIX_CONFIG: |
extra-substituters = https://dimensionalos.cachix.org
extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
INPUT_SET_AS_TRUSTED_USER: "true"
run: bash docker/ros/install-nix.sh
- name: Build Rust memory recorder Nix package
run: |
cd dimos/experimental/memory/rust
nix build .#dimos-memory-recorder --no-write-lock-file --print-build-logs
test -x result/bin/dimos-memory-recorder
# PCL, GTSAM and the SLAM cores come from each module's flake, not apt.
- name: Build C++ native modules
env:
MODULES: ${{ steps.find-modules.outputs.modules }}
run: |
for module in $MODULES; do
echo "::group::$module"
(cd "$module" && nix build --no-write-lock-file --print-build-logs)
echo "::endgroup::"
done
- name: Test C++ native modules
env:
MODULES: ${{ steps.find-modules.outputs.modules }}
run: |
for module in $MODULES; do
if grep -qE 'enable_testing|add_test' "$module/CMakeLists.txt"; then
echo "::group::ctest $module"
(cd "$module" && ctest --test-dir build --output-on-failure)
echo "::endgroup::"
fi
done
md-babel:
timeout-minutes: 15
runs-on: ubuntu-latest
permissions:
contents: read # For checkout
steps:
- uses: actions/checkout@v7
- name: Fetch LFS data needed by doc code blocks
run: git lfs pull --include="data/.lfs/go2_bigoffice.db.tar.gz,data/.lfs/unitree_go2_bigoffice_map.pickle.tar.gz" --exclude=""
# Docs decode JPEG from SQLite via PyTurboJPEG; pyaudio needs portaudio.
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y libturbojpeg portaudio19-dev
- name: Install uv
uses: astral-sh/setup-uv@v10.0.1
with:
enable-cache: true
prune-cache: true
- uses: actions/setup-node@v7
with:
node-version: 'lts/*'
- name: Install Python dependencies
run: uv sync --group tests --frozen
- name: Restore Hugging Face model cache
id: hf-cache
uses: actions/cache/restore@v6
with:
path: ~/.cache/huggingface
# The docs determine which models get downloaded, so any docs
# change may introduce a new model: go online and re-save the
# cache then. restore-keys still seeds the new cache with the
# previously cached models, so only genuinely new models are
# downloaded.
key: hf-models-${{ hashFiles('docs/**/*.md') }}
restore-keys: |
hf-models-
- name: Prefetch Hugging Face models
if: steps.hf-cache.outputs.cache-hit != 'true'
# Anonymous downloads from shared GitHub runner IPs get rate-limited
# by huggingface.co (HTTP 429), so retry with backoff. Models already
# in the restored cache are not re-downloaded.
run: |
for i in 1 2 3 4 5; do
uv run python -c "from transformers import CLIPModel, CLIPProcessor; m = 'openai/clip-vit-base-patch32'; CLIPModel.from_pretrained(m); CLIPProcessor.from_pretrained(m)" && exit 0
echo "Hugging Face download failed (likely rate-limited); retry $i in $((i * 30))s"
sleep $((i * 30))
done
echo "::error::Could not download Hugging Face models after 5 attempts"
exit 1
- name: Save Hugging Face model cache
# Save right after the download (not in a post step) so a failure in
# the doc blocks below doesn't throw away the downloaded models.
if: steps.hf-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v6
with:
path: ~/.cache/huggingface
key: hf-models-${{ hashFiles('docs/**/*.md') }}
- name: Run Hugging Face offline on exact cache hit
# Zero huggingface.co requests when every model is already cached:
# immune to rate limiting.
if: steps.hf-cache.outputs.cache-hit == 'true'
run: echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV"
- name: Execute documentation code blocks
run: ./bin/run-doc-codeblocks --ci --no-cache
web:
timeout-minutes: 30
runs-on: ubuntu-latest
permissions:
contents: read # For checkout
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Extract Deno version
id: deno-version
run: echo "version=$(grep -oP 'DENO_VERSION = "\K[^"]+' dimos/utils/deno.py)" >> "$GITHUB_OUTPUT"
- name: Install Deno
uses: denoland/setup-deno@v2
with:
deno-version: ${{ steps.deno-version.outputs.version }}
cache: true
- name: Deno fmt
working-directory: web
run: deno fmt --check
- name: Deno lint
working-directory: web
run: deno lint
- name: Deno install (validates deno.lock)
working-directory: web
run: deno install --frozen
- name: Deno check (relay + shared)
working-directory: web
run: deno task check
- name: Deno test (relay + shared)
working-directory: web
run: deno task test
- name: SDK type-check
working-directory: web/sdk
run: deno task check
- name: SDK vitest
working-directory: web/sdk
run: deno task test
- name: Cockpit type-check
working-directory: web/cockpit
run: deno task check
- name: Cockpit vitest
working-directory: web/cockpit
run: deno task test
- name: SDK bundle build
working-directory: web/sdk
run: deno task build
- name: Cockpit build
working-directory: web/cockpit
run: deno task build
# ---- Browser e2e: dimos run --local-relay + Playwright on the dist ----
- name: Install uv
uses: astral-sh/setup-uv@v10.0.1
with:
enable-cache: true
prune-cache: true
- name: Setup Python
uses: actions/setup-python@v7.0.0
with:
python-version: '3.12'
- name: Install native dependencies (pyaudio build, JPEG encoding, audio decode)
run: |
sudo apt-get update
sudo apt-get install -y portaudio19-dev libturbojpeg ffmpeg
- name: Fetch the go2_short replay dataset
run: git lfs pull --include="data/.lfs/go2_short.db.tar.gz" --exclude=""
- name: Install dependencies
run: uv sync --group tests --group browser-tests --frozen
- name: Install Playwright browsers # the e2e runs in both engines
run: uv run playwright install --with-deps chromium firefox
- name: Cockpit + SDK browser e2e
run: >-
uv run pytest -m web_browser dimos/e2e_tests/test_cockpit_browser.py
dimos/e2e_tests/test_sdk_browser.py
dimos/e2e_tests/test_custom_channel_browser.py
dimos/e2e_tests/test_lcm_channel_browser.py
dimos/e2e_tests/test_publish_browser.py
dimos/e2e_tests/test_voice_browser.py
dimos/e2e_tests/test_stats_browser.py
dimos/e2e_tests/test_robot_picker_browser.py
dimos/e2e_tests/test_relay_auth_browser.py --no-cov
- name: Relay image build # docker/relay/Dockerfile must not rot silently
run: docker build -f docker/relay/Dockerfile .
tests:
if: |
!cancelled() &&
contains(fromJSON('["success", "skipped"]'), needs.cachix-build-check.result) &&
contains(fromJSON('["success", "skipped"]'), needs.cachix-build.result)
needs: [cachix-build-check, cachix-build]
timeout-minutes: 20
strategy:
matrix:
pyver: ['3.10', '3.11', '3.12']
os: ["ubuntu-latest"]
experimental: [false]
include:
- os: "ubuntu-24.04-arm"
pyver: "3.12"
experimental: false
fail-fast: true
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.experimental }}
env:
# Arm faulthandler from interpreter start — before conftest is even
# imported — so import-time crashes are covered too. dimos/conftest.py
# redirects it to a per-process file under $RUNNER_TEMP/pytest-crash,
# which is what makes a crash legible under xdist (one file per worker).
PYTHONFAULTHANDLER: "1"
# Zenoh's own scouting decisions, for diagnosing workers that never link.
RUST_LOG: "zenoh::net::runtime::orchestrator=debug"
permissions:
contents: read # For checkout
id-token: write # For codecov-action's OIDC upload
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Install uv
uses: astral-sh/setup-uv@v10.0.1
with:
enable-cache: true
prune-cache: true
python-version: ${{ matrix.pyver }}
- name: Extract Deno version
id: deno-version
run: echo "version=$(grep -oP 'DENO_VERSION = "\K[^"]+' dimos/utils/deno.py)" >> "$GITHUB_OUTPUT"
- name: Install Deno
uses: denoland/setup-deno@v2
with:
deno-version: ${{ steps.deno-version.outputs.version }}
cache: true
# pyaudio needs portaudio; relay/audio tests need TurboJPEG and ffmpeg.
- name: Install system dependencies (Ubuntu)
if: startsWith(matrix.os, 'ubuntu')
run: |
sudo apt-get update
sudo apt-get install -y portaudio19-dev libturbojpeg ffmpeg
- name: Install dependency for pyaudio (macOS)
if: startsWith(matrix.os, 'macos')
run: brew install portaudio
- name: Cap accidental LFS downloads at 1 MiB
run: |
sudo mv /usr/bin/git-lfs /usr/bin/git-lfs.orig
sudo install -m 0755 bin/git-lfs-guard /usr/bin/git-lfs
git config --global --unset-all filter.lfs.process || true
# The published store paths are x86_64-linux, so the arm leg (and any
# future macOS leg) skips provisioning; binary-needing tests skip there.
# On a marker miss (fork PR, eviction) the binaries are simply absent —
# hosted runners never fall back to building the heavy C++ closures.
- name: Restore publish marker
if: runner.os == 'Linux' && runner.arch == 'X64'
id: native-marker
uses: actions/cache/restore@v6
with:
path: .cachix-marker
key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }}
- name: Install Nix (with Cachix substituter)
# cache-hit implies the restore ran, which is already Linux/X64-gated.
if: steps.native-marker.outputs.cache-hit == 'true'
env:
INPUT_EXTRA_NIX_CONFIG: |
extra-substituters = https://dimensionalos.cachix.org
extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
INPUT_SET_AS_TRUSTED_USER: "true"
run: bash docker/ros/install-nix.sh
- name: Provision native modules from Cachix
if: steps.native-marker.outputs.cache-hit == 'true'
run: python3 bin/build-native-modules --link-results .cachix-marker/links.txt
- name: Install dependencies
run: uv sync --group tests --frozen
- name: Run tests
run: uv run pytest --numprocesses=logical --cov=dimos/ --junitxml=junit.xml -m 'not (self_hosted or mujoco or self_hosted_large or web_browser or bake_e2e or native_e2e)'
- name: Re-run the failing tests with maximum verbosity
if: failure()
env:
COLOR: yes
run: >- # `exit 1` makes sure that the job remains red with flaky runs
uv run pytest --no-cov -vvvvv --lf -m 'not (self_hosted or mujoco or self_hosted_large or web_browser or bake_e2e or native_e2e)' && exit 1
shell: bash
- name: Print crash diagnostics
if: failure()
shell: bash
run: |
shopt -s nullglob
for f in "$RUNNER_TEMP"/pytest-crash/*.log; do
echo "::group::$(basename "$f")"
cat "$f"
echo "::endgroup::"
done
- name: Turn coverage into xml
run: uv run python -m coverage xml
- name: Upload coverage
uses: codecov/codecov-action@v7
with:
disable_search: true
fail_ci_if_error: true
files: ./coverage.xml
flags: OS-${{ matrix.os }},Py-${{ matrix.pyver }}
use_oidc: true
# Install the CLI from PyPI (pinned) instead of cli.codecov.io:
# the GPG key for the default download path is served from a dead
# keybase.io URL, failing every upload. See
# codecov/codecov-action#1955 and #2398.
use_pypi: true
version: "11.2.8"
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v7
with:
report_type: test_results
use_oidc: true
# See the "Upload coverage" step above.
use_pypi: true
version: "11.2.8"
self-hosted-tests:
# Runs on push, merge_group (trusted — a maintainer must approve + queue the
# PR), and same-repo PRs. Skipped on fork PRs, which would expose the
# self-hosted runner to untrusted code from external contributors.
if: |
!cancelled() &&
(github.event_name == 'push' || github.event_name == 'merge_group' || github.event.pull_request.head.repo.full_name == github.repository) &&
needs.cachix-build-check.result == 'success' &&
contains(fromJSON('["success", "skipped"]'), needs.cachix-build.result)
needs: [compute-ros-pin, cachix-build-check, cachix-build, cachix-build-macos]
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }}
# See the `tests` job for why this is set.
PYTHONFAULTHANDLER: "1"
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- os: Linux
# GitHub Actions only honours `container:` on Linux runners.
container:
image: dimensional/ros-dev@${{ needs.compute-ros-pin.outputs.digest }}
# Hard-cap container RAM so a runaway test OOMs inside the container, not
# as a host global-OOM that takes out sshd/tmux.
options: --memory=6g --memory-swap=6g
volumes:
- /var/cache/dimos-root-cache:/root/.cache
# Persistent Nix store — without it every run reinstalls nix
# and re-downloads the modules' full runtime closure (~600 MB)
# through the runner's uplink. GC'd at the end of the job; the
# provision step roots the live closure so GC keeps it.
- /var/cache/dimos-nix:/nix
markers: "self_hosted or skipif_no_ros"
experimental: false
- os: macOS
container: null # run on host — `container:` is Linux-only
# web_browser: the cockpit browser e2e that Linux runs in the `web` job.
markers: "self_hosted or web_browser"
experimental: true
runs-on:
- self-hosted
- ${{ matrix.os }}
- base
continue-on-error: ${{ matrix.experimental }}
permissions:
contents: read # For checkout
id-token: write # For codecov-action's OIDC upload
container: ${{ matrix.container }}
steps:
- name: Checkout
uses: actions/checkout@v7
with:
clean: false
# If we ever allow external PRs on custom runner, persisting credentials
# could be abused by attackers.
persist-credentials: false
- name: Fix permissions
run: |
git config --global --add safe.directory '*'
git clean -ffdx
- name: Point uv at a persistent cache on the workspace mount
# HOME in container jobs is an ephemeral /github/home, so uv's default
# cache would be dropped after every run. The workspace mount (/__w,
# the runner's _work dir) persists on the self-hosted runner, and
# unlike the /root/.cache volume it is the same mount as .venv:
# hardlinks across two docker bind mounts fail with EXDEV even on one
# filesystem, making uv fall back to a minutes-long full copy.
if: matrix.os == 'Linux'
run: echo "UV_CACHE_DIR=$(dirname "$RUNNER_WORKSPACE")/uv-cache" >> "$GITHUB_ENV"
- name: Install uv
uses: astral-sh/setup-uv@v10.0.1
with:
# We persist on disk above, cache would clean some of this up.
enable-cache: false
- name: Install codecov CLI
# Self-hosted runners have no `pip` on PATH, so codecov-action's
# use_pypi can't work here; pre-install the pinned CLI with uv and
# point the action at it via `binary`. See codecov/codecov-action#1955
# and #2398.
run: |
uv venv .codecov-venv
uv pip install --python .codecov-venv/bin/python codecov-cli==11.2.8
- name: Install dependencies
run: uv sync --group tests-self-hosted --frozen
- name: Install web_browser e2e dependencies (macOS)
if: matrix.os == 'macOS'
run: |
brew install ffmpeg
uv sync --group tests-self-hosted --group browser-tests --frozen
uv run playwright install chromium firefox
- name: Build C++ extensions in-place
run: uv run python setup.py build_ext --inplace
- name: Source ROS environment
# The uv venv is sealed (include-system-site-packages = false), so
# `import rclpy` / `ament_index_python` would fail. Sourcing the ROS
# setup script and exporting PYTHONPATH/AMENT_PREFIX_PATH/etc into
# GITHUB_ENV makes them importable from `uv run`.
if: matrix.os == 'Linux'
shell: bash
run: |
source /opt/ros/${ROS_DISTRO}/setup.bash
{
echo "PYTHONPATH=$PYTHONPATH"
echo "AMENT_PREFIX_PATH=$AMENT_PREFIX_PATH"
echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH"
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
echo "ROS_DISTRO=$ROS_DISTRO"
echo "ROS_VERSION=$ROS_VERSION"
echo "ROS_PYTHON_VERSION=$ROS_PYTHON_VERSION"
} >> "$GITHUB_ENV"
- name: Install Nix (with Cachix substituter)
if: contains(matrix.markers, 'skipif_no_ros')
# /nix is bind-mounted from the self-hosted runner's host filesystem
# (see this job's matrix container.volumes). On warm runs Nix is
# already installed; install-nix.sh's `type -p nix` check will exit
# early, but only if the profile bin is on PATH — prepend it before
# calling. If /nix is empty (first job on a fresh runner), nix isn't
# on PATH and the install proceeds normally. After install-nix.sh,
# we unconditionally (re-)write /etc/nix/nix.conf: on warm runs
# install-nix.sh exited before it would have written the config,
# which leaves `experimental-features = nix-command flakes` absent
# and the build dying with "experimental Nix feature 'nix-command'
# is disabled". Writing it here keeps cold and warm runs identical.
run: |
export PATH="/nix/var/nix/profiles/default/bin:$PATH"
bash docker/ros/install-nix.sh
sudo mkdir -p /etc/nix
sudo tee /etc/nix/nix.conf > /dev/null <<'EOF'
experimental-features = nix-command flakes
extra-substituters = https://dimensionalos.cachix.org
extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=
always-allow-substitutes = true
build-users-group =
# Substitute-only: everything should be compiled already and stored in Cachix.
max-jobs = 0
EOF
echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH"
- name: Put Nix on PATH (macOS)
if: matrix.os == 'macOS'
run: |
bin=/nix/var/nix/profiles/default/bin
if [ ! -x "$bin/nix-store" ]; then
echo "::error::Nix is not installed on this macOS runner host — install multi-user Nix and add the dimensionalos substituter to /etc/nix/nix.conf (see this step's comment)"
exit 1
fi
echo "$bin" >> "$GITHUB_PATH"
echo "NIX_CONFIG=experimental-features = nix-command flakes" >> "$GITHUB_ENV"
- name: Restore publish marker
id: native-marker
uses: actions/cache/restore@v6
with:
path: .cachix-marker
key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }}
- name: Provision native modules from Cachix
# The marker records every result -> store path mapping, so the warm
# path is symlink recreation against the persistent /nix plus
# substitution of anything missing by exact path — no nix evaluation.
# A missed marker (evicted) falls back to the full nix build.
run: |
if [ "${{ steps.native-marker.outputs.cache-hit }}" = "true" ]; then
python3 bin/build-native-modules --link-results .cachix-marker/links.txt
else
python3 bin/build-native-modules
fi
# Linux only: root the out paths so this job's end-of-GC keeps the
# closure warm in the CI-managed bind-mounted store (the workspace
# result links die in the next run's `git clean`, so they cannot
# serve as roots). The macOS host manages its own store and GC.
if [ "${{ matrix.os }}" = "Linux" ]; then
sudo mkdir -p /nix/var/nix/gcroots/dimos-native
sudo find /nix/var/nix/gcroots/dimos-native -maxdepth 1 -type l -delete
python3 bin/build-native-modules --record-links | cut -d' ' -f2 | sort -u |
while read -r p; do
sudo ln -sfn "$p" "/nix/var/nix/gcroots/dimos-native/$(basename "$p")"
done
fi
- name: Point cargo at a persistent cache on the workspace mount
if: contains(matrix.markers, 'self_hosted')
run: |
echo "RUSTUP_HOME=$(dirname "$RUNNER_WORKSPACE")/rustup" >> "$GITHUB_ENV"
echo "CARGO_HOME=$(dirname "$RUNNER_WORKSPACE")/cargo" >> "$GITHUB_ENV"
- name: Install Rust toolchain
if: contains(matrix.markers, 'self_hosted')
uses: dtolnay/rust-toolchain@stable
- name: Build rust native modules
if: contains(matrix.markers, 'self_hosted')
run: cargo build --release --locked -p dimos-livox -p dimos-virtual-mid360
- name: Run tests
run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m '(${{ matrix.markers }}) and not mujoco'
- name: Re-run the failing tests with maximum verbosity
if: failure()
env:
COLOR: yes
run: >- # `exit 1` makes sure that the job remains red with flaky runs
uv run pytest --no-cov -vvvvv --lf -m '(${{ matrix.markers }}) and not mujoco' && exit 1
shell: bash
- name: Print crash diagnostics
if: failure()
shell: bash
run: |
shopt -s nullglob
for f in "$RUNNER_TEMP"/pytest-crash/*.log; do
echo "::group::$(basename "$f")"
cat "$f"
echo "::endgroup::"
done
- name: Turn coverage into xml
run: uv run python -m coverage xml
- name: Upload coverage
uses: codecov/codecov-action@v7
with:
disable_search: true
fail_ci_if_error: true
files: ./coverage.xml
flags: SelfHosted-${{ matrix.os }}
use_oidc: true
# Use the pinned CLI from the "Install codecov CLI" step (no pip
# on self-hosted runners, and the default cli.codecov.io download
# has a broken GPG key fetch). See codecov/codecov-action#1955.
binary: .codecov-venv/bin/codecovcli
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v7
with:
report_type: test_results
use_oidc: true
# See the "Upload coverage" step above.
binary: .codecov-venv/bin/codecovcli
- name: Check disk space
if: failure()
run: |
df -h
- name: Prune the persistent uv cache
# The cache persists on the runner (see "Point uv at a persistent
# cache" above), so lock bumps accumulate superseded wheels
# forever. Drop unreachable objects each run; past 25 GB, `--ci` also
# evicts pre-built wheels (re-downloaded on the next run) while keeping
# the expensive wheels built from source. uv has nothing age-based.
# Not always(): on a cancelled run, lingering test processes still
# hold the shared cache lock and prune would block on it until the
# job timeout.
if: ${{ !cancelled() && matrix.os == 'Linux' }}
run: |
uv cache prune
size_mb=$(du -sm "$UV_CACHE_DIR" | cut -f1)
echo "uv cache size: ${size_mb} MB"
if [ "$size_mb" -gt 25600 ]; then uv cache prune --ci; fi
- name: GC the persistent Nix store
# /nix persists on the runner (container volume above), so superseded
# module closures accumulate as inputs change. Unrooted paths go; the
# provision step's gcroots keep the live closure across runs. Not
# always(): a cancelled run can leave nix processes holding locks.
if: ${{ !cancelled() && matrix.os == 'Linux' }}
run: nix-collect-garbage --delete-older-than 3d
self-hosted-large-tests:
# Skip on PRs from forks which would expose the self-hosted runner to untrusted code from external contributors.
if: |
!cancelled() &&
(github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) &&
needs.cachix-build-check.result == 'success' &&
contains(fromJSON('["success", "skipped"]'), needs.cachix-build.result)
needs: [cachix-build-check, cachix-build]
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }}
DIMSIM_RENDER: gpu
# See the `tests` job for why this is set.
PYTHONFAULTHANDLER: "1"
timeout-minutes: 30
runs-on: [self-hosted, Linux, large]
permissions:
contents: read # For checkout
id-token: write # For codecov-action's OIDC upload
steps:
- name: Checkout
uses: actions/checkout@v7
with:
clean: false
# If we ever allow external PRs on custom runner, persisting credentials
# could be abused by attackers.
persist-credentials: false
- name: Fix permissions
run: |
sudo git config --global --add safe.directory '*'
sudo git clean -ffdx
- name: Fetch DimSim LFS assets only
# The checkout above doesn't fetch LFS, so the dimsim scene/embodiment
# glbs are pointer stubs and the browser sim fails to load them. Pull
# ONLY misc/DimSim assets — a full `lfs: true` would also drag in main's
# multi-GB data/.lfs/* objects (some missing on the server → 404).
# persist-credentials is false above, so supply the token for this pull.
run: |
git config --local http.https://github.com/.extraheader \
"AUTHORIZATION: basic $(printf 'x-access-token:%s' '${{ github.token }}' | base64 -w0)"
git lfs pull --include="misc/DimSim/**"
git config --local --unset-all http.https://github.com/.extraheader
- name: Install uv
uses: astral-sh/setup-uv@v10.0.1
with:
# We persist on disk, cache would clean some of this up.
enable-cache: false
- name: Install codecov CLI
# Self-hosted runners have no `pip` on PATH, so codecov-action's
# use_pypi can't work here; pre-install the pinned CLI with uv and
# point the action at it via `binary`. See codecov/codecov-action#1955
# and #2398.
run: |
uv venv .codecov-venv
uv pip install --python .codecov-venv/bin/python codecov-cli==11.2.8
- name: Install apt dependencies
run: |
sudo apt-get update
sudo apt-get install -y \
`# Compiler toolchain for the in-tree pybind11 C++ extension.` \
build-essential \
`# Runtime lib for opencv-python (cv2 loads libGL.so.1 on import).` \
libgl1 \
`# Runtime lib for PyTurboJPEG (fast JPEG encode/decode).` \
libturbojpeg \
`# Dev package for PyAudio (built from source against PortAudio).` \
portaudio19-dev \
`# Chromium runtime libs required by Playwright (used by dimsim browser tests).` \
libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libgbm1 \
libpango-1.0-0 libxcomposite1 libxdamage1 libxkbcommon0 libxrandr2 \
`# Rerun viewer pulls in X11 keyboard bindings when DISPLAY is set.` \
libxkbcommon-x11-0
- name: Install dependencies
run: uv sync --group tests-self-hosted --frozen
- name: Build C++ extensions in-place
run: uv run python setup.py build_ext --inplace
- name: Run tests
# The runner must have an Xorg server with NVIDIA GPU access running
# on DISPLAY=:0 — dimsim's headless Chromium needs real WebGL for the
# heavier scenes (`apt`). Software X via xvfb is not enough.
env:
DISPLAY: ":0"
run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m self_hosted_large
- name: Re-run the failing tests with maximum verbosity
if: failure()
env:
COLOR: yes
DISPLAY: ":0"
run: >- # `exit 1` makes sure that the job remains red with flaky runs
uv run pytest --no-cov -vvvvv --lf -m self_hosted_large && exit 1
shell: bash
- name: Print crash diagnostics
if: failure()
shell: bash
run: |
shopt -s nullglob
for f in "$RUNNER_TEMP"/pytest-crash/*.log; do
echo "::group::$(basename "$f")"
cat "$f"
echo "::endgroup::"
done
- name: Turn coverage into xml
run: uv run python -m coverage xml
- name: Upload coverage
uses: codecov/codecov-action@v7
with:
disable_search: true
fail_ci_if_error: true
files: ./coverage.xml
flags: SelfHosted-Large
use_oidc: true
# Use the pinned CLI from the "Install codecov CLI" step (no pip
# on self-hosted runners, and the default cli.codecov.io download
# has a broken GPG key fetch). See codecov/codecov-action#1955.
binary: .codecov-venv/bin/codecovcli
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v7
with:
report_type: test_results
use_oidc: true
# See the "Upload coverage" step above.
binary: .codecov-venv/bin/codecovcli
- name: Check disk space
if: failure()
run: |
df -h
- name: Prune the persistent uv cache
# See self-hosted-tests; UV_CACHE_DIR isn't set here, so resolve the
# default with `uv cache dir`.
if: ${{ !cancelled() }}
run: |
uv cache prune
size_mb=$(du -sm "$(uv cache dir)" | cut -f1)
echo "uv cache size: ${size_mb} MB"
if [ "$size_mb" -gt 25600 ]; then uv cache prune --ci; fi
# Cross-job fail-fast: GitHub Actions only fail-fasts within a matrix,
# not across sibling jobs. This watcher fires the moment `tests` fails
# and cancels the whole workflow run.
fail-fast:
if: failure()
needs: [tests]
runs-on: ubuntu-latest
permissions:
actions: write # For `gh run cancel`
steps:
- name: Cancel workflow run
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: gh run cancel ${{ github.run_id }} --repo ${{ github.repository }}
ci-complete: # This is used for branch protection.
if: always()
needs:
- lint
- rust
- native
- md-babel
- web
- tests
- self-hosted-tests
#- self-hosted-large-tests
runs-on: ubuntu-latest
permissions:
id-token: write # For codecov-action's OIDC notify
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@release/v1
with:
allowed-skips: self-hosted-tests, self-hosted-large-tests
jobs: ${{ toJSON(needs) }}
- name: Trigger Codecov notifications
uses: codecov/codecov-action@v7
with:
run_command: send-notifications
use_oidc: true
fail_ci_if_error: true
# Install the CLI from PyPI (pinned) instead of cli.codecov.io:
# the GPG key for the default download path is served from a dead
# keybase.io URL, failing every upload. See
# codecov/codecov-action#1955 and #2398.
use_pypi: true
version: "11.2.8"