feat(simulation): integrate BEHAVIOR with R1 Pro navigation and manipulation #10734
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| merge_group: | |
| workflow_dispatch: | |
| inputs: | |
| force-cachix-build: | |
| description: Publish to Cachix even if the inputs marker says nothing changed | |
| type: boolean | |
| default: false | |
| concurrency: | |
| # Cancels runs from previous pushes in a PR. | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| permissions: {} | |
| env: | |
| # Disable default package installs. Each job should explicitly install a group. | |
| UV_NO_SYNC: "1" | |
| jobs: | |
| compute-ros-pin: | |
| # Extracts the ros-dev image digest pinned in docker/ros-dev-pin/Dockerfile | |
| # so downstream jobs can reference it from their `container:` field. | |
| # That will be updated by Dependabot, and keep CI tests stable. | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| outputs: | |
| digest: ${{ steps.read.outputs.digest }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - id: read | |
| run: | | |
| DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' docker/ros-dev-pin/Dockerfile) | |
| if [ -z "$DIGEST" ]; then | |
| echo "::error::No sha256 digest found in docker/ros-dev-pin/Dockerfile" | |
| exit 1 | |
| fi | |
| echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" | |
| cachix-build-check: | |
| # Holds no secrets, so it runs on fork PRs too: forks need its inputs-hash | |
| # output to restore the base branch's publish marker and download the | |
| # prebuilt binaries. Only cachix-build itself is fork-guarded. | |
| if: ${{ !cancelled() }} | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| env: | |
| CACHIX_CACHE_NAME: dimensionalos | |
| outputs: | |
| needs-build-linux: ${{ steps.decide.outputs.linux }} | |
| needs-build-macos: ${{ steps.decide.outputs.macos }} | |
| inputs-hash: ${{ steps.hash.outputs.inputs-hash }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Compute native-module inputs hash | |
| id: hash | |
| run: python3 bin/build-native-modules --inputs-hash | |
| - name: Probe Linux publish marker | |
| id: marker-linux | |
| uses: actions/cache/restore@v6 | |
| with: | |
| path: .cachix-marker | |
| key: cachix-published-Linux-X64-${{ steps.hash.outputs.inputs-hash }} | |
| lookup-only: true | |
| - name: Probe macOS publish marker | |
| id: marker-macos | |
| uses: actions/cache/restore@v6 | |
| with: | |
| path: .cachix-marker | |
| key: cachix-published-macOS-ARM64-${{ steps.hash.outputs.inputs-hash }} | |
| lookup-only: true | |
| - name: Decide which platforms need a publish | |
| id: decide | |
| env: | |
| FORCE: ${{ inputs.force-cachix-build }} | |
| LINUX_HIT: ${{ steps.marker-linux.outputs.cache-hit }} | |
| MACOS_HIT: ${{ steps.marker-macos.outputs.cache-hit }} | |
| run: | | |
| emit() { # $1=output name, $2=this platform's marker hit | |
| if [ "$FORCE" != "true" ] && [ "$2" = "true" ]; then | |
| echo "$1=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "$1=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| } | |
| emit linux "$LINUX_HIT" | |
| emit macos "$MACOS_HIT" | |
| cachix-build: | |
| # ⚠️ NEVER RUN ON UNTRUSTED (fork) CODE ⚠️ | |
| # This job has write access to CACHIX_AUTH_TOKEN. Any workflow change that | |
| # reaches a `run:` step here could exfiltrate the token, letting an attacker | |
| # push poisoned binaries to the Cachix cache — which subsequent CI runs and | |
| # every dev with the substituter configured would unwittingly install. | |
| # Allowed only on trusted events: push, merge_group (a maintainer must | |
| # approve + queue the PR), and same-repo PRs — NEVER on fork `pull_request`s. | |
| # The `if:` guard below is the only thing keeping forks out — DO NOT REMOVE. | |
| # merge_group runs the PR's *merged* workflow with secrets, so .github/ | |
| # changes must be reviewed before a fork PR is queued. | |
| # | |
| # Gated on cachix-build-check so an entry into the `cachix` environment | |
| # always corresponds to a real publish — see that job for why. | |
| if: | | |
| !cancelled() && | |
| (github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) && | |
| needs.cachix-build-check.outputs.needs-build-linux == 'true' | |
| needs: cachix-build-check | |
| timeout-minutes: 300 | |
| environment: cachix | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Free runner disk space | |
| uses: BRAINSia/free-disk-space@v2.1.3 | |
| - name: Install Nix (with Cachix substituter) | |
| env: | |
| INPUT_EXTRA_NIX_CONFIG: | | |
| extra-substituters = https://dimensionalos.cachix.org | |
| extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= | |
| INPUT_SET_AS_TRUSTED_USER: "true" | |
| run: bash docker/ros/install-nix.sh | |
| - name: Cache Nix fetcher + eval cache | |
| # ~/.cache/nix holds the flake fetcher cache (the github tarballs of | |
| # each `nix build github:…` ref) and the eval cache (compiled flake | |
| # outputs). ~/.local/state/nix holds profile/gcroot bookkeeping. | |
| # Restoring these skips the ~20s/module evaluation cost, which is | |
| # paid every cold run otherwise. /nix/store is intentionally NOT | |
| # cached here — that's the slow heavy part that Cachix handles. | |
| uses: actions/cache@v6 | |
| with: | |
| path: | | |
| ~/.cache/nix | |
| ~/.local/state/nix | |
| key: nix-fetcher-build-${{ runner.os }}-${{ hashFiles('bin/build-native-modules', '**/flake.lock', '**/flake.nix') }} | |
| restore-keys: nix-fetcher-build-${{ runner.os }}- | |
| - name: Authenticate Cachix | |
| uses: cachix/cachix-action@v17 | |
| with: | |
| name: dimensionalos | |
| authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} | |
| - name: Build native modules | |
| env: | |
| BUILD_WORKERS: "1" | |
| run: python3 bin/build-native-modules | |
| - name: Verify Cachix holds the built paths | |
| env: | |
| CACHIX_CACHE_NAME: dimensionalos | |
| run: python3 bin/build-native-modules --verify-published | |
| - name: Record the published inputs manifest | |
| # The manifest is forensics for the marker saved below; links.txt maps | |
| # each result symlink to its store path so consumers can materialise | |
| # the binaries with --link-results instead of re-running nix. | |
| env: | |
| CACHIX_CACHE_NAME: dimensionalos | |
| run: | | |
| mkdir -p .cachix-marker | |
| python3 bin/build-native-modules --inputs-hash \ | |
| > .cachix-marker/inputs-hash.txt 2> .cachix-marker/manifest.txt | |
| python3 bin/build-native-modules --record-links > .cachix-marker/links.txt | |
| - name: Save publish marker | |
| uses: actions/cache/save@v6 | |
| with: | |
| path: .cachix-marker | |
| key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }} | |
| cachix-build-macos: | |
| # ⚠️ NEVER RUN ON UNTRUSTED (fork) CODE ⚠️ — holds CACHIX_AUTH_TOKEN, same | |
| # as cachix-build; the `if:` guard below is the only thing keeping forks | |
| # out. A separate job (not a matrix leg of cachix-build) for two reasons: | |
| # its own job-level `if` must gate entry into the `cachix` environment on | |
| # the macOS marker alone (matrix context isn't available in `if:`), and a | |
| # macOS build failure must not fail cachix-build and block the Linux tests. | |
| if: | | |
| !cancelled() && | |
| (github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) && | |
| needs.cachix-build-check.outputs.needs-build-macos == 'true' | |
| needs: cachix-build-check | |
| timeout-minutes: 300 | |
| environment: cachix | |
| runs-on: macos-14 # GitHub-hosted Apple silicon → runner.arch is ARM64 | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Install Nix (with Cachix substituter) | |
| env: | |
| INPUT_EXTRA_NIX_CONFIG: | | |
| extra-substituters = https://dimensionalos.cachix.org | |
| extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= | |
| INPUT_SET_AS_TRUSTED_USER: "true" | |
| run: bash docker/ros/install-nix.sh | |
| - name: Cache Nix fetcher + eval cache | |
| # Key includes runner.os, so this is a macOS-only entry, distinct from | |
| # the Linux fetcher cache. | |
| uses: actions/cache@v6 | |
| with: | |
| path: | | |
| ~/.cache/nix | |
| ~/.local/state/nix | |
| key: nix-fetcher-build-${{ runner.os }}-${{ hashFiles('bin/build-native-modules', '**/flake.lock', '**/flake.nix') }} | |
| restore-keys: nix-fetcher-build-${{ runner.os }}- | |
| - name: Authenticate Cachix | |
| uses: cachix/cachix-action@v17 | |
| with: | |
| name: dimensionalos | |
| authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} | |
| - name: Build native modules | |
| env: | |
| BUILD_WORKERS: "1" | |
| run: python3 bin/build-native-modules | |
| - name: Verify Cachix holds the built paths | |
| env: | |
| CACHIX_CACHE_NAME: dimensionalos | |
| run: python3 bin/build-native-modules --verify-published | |
| - name: Record the published inputs manifest | |
| env: | |
| CACHIX_CACHE_NAME: dimensionalos | |
| run: | | |
| mkdir -p .cachix-marker | |
| python3 bin/build-native-modules --inputs-hash \ | |
| > .cachix-marker/inputs-hash.txt 2> .cachix-marker/manifest.txt | |
| python3 bin/build-native-modules --record-links > .cachix-marker/links.txt | |
| - name: Save publish marker | |
| uses: actions/cache/save@v6 | |
| with: | |
| path: .cachix-marker | |
| key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }} | |
| lint: | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Reject Git sources in root pyproject.toml | |
| run: | | |
| python3 - <<'PY' | |
| import sys | |
| import tomllib | |
| with open("pyproject.toml", "rb") as file: | |
| config = tomllib.load(file) | |
| sources = config.get("tool", {}).get("uv", {}).get("sources", {}) | |
| rejected = [] | |
| for name, source in sources.items(): | |
| entries = source if isinstance(source, list) else [source] | |
| if any("git" in entry for entry in entries): | |
| rejected.append(name) | |
| if rejected: | |
| sys.exit( | |
| "::error file=pyproject.toml::Git sources are not allowed in root " | |
| "pyproject.toml. Use published packages instead: " + ", ".join(rejected) | |
| ) | |
| PY | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| enable-cache: true | |
| prune-cache: true | |
| - name: Install lint dependencies | |
| run: uv sync --only-group lint --frozen | |
| - name: Mypy | |
| run: uv run mypy | |
| # The deno-fmt pre-commit hook needs deno on PATH. | |
| - name: Extract Deno version | |
| id: deno-version | |
| run: echo "version=$(grep -oP 'DENO_VERSION = "\K[^"]+' dimos/utils/deno.py)" >> "$GITHUB_OUTPUT" | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v2 | |
| with: | |
| deno-version: ${{ steps.deno-version.outputs.version }} | |
| cache: true | |
| - name: Run pre-commit | |
| uses: pre-commit/action@v3.0.1 | |
| env: | |
| # clippy and fmt are checked in the rust job, so we can skip them here | |
| SKIP: cargo-fmt,cargo-clippy | |
| rust: | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: Cache cargo build | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| # Only main writes the cache, so a PR run cannot evict main's entry | |
| # from the repo's 10GB budget. PR runs still restore from it. | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: cargo fmt | |
| run: cargo fmt --all -- --check | |
| - name: cargo clippy | |
| run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings | |
| - name: cargo test | |
| run: cargo test --workspace --all-features --locked | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| enable-cache: true | |
| prune-cache: true | |
| # Python readback and dataset export tests decode recorded JPEGs with PyTurboJPEG. | |
| - name: Install Python audio and image dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y portaudio19-dev libturbojpeg | |
| - name: Build and test PyO3 bindings | |
| # PyO3 extension modules (cdylib + `extension-module`) are the one case | |
| # that still needs a plain build: pytest imports the compiled .so. | |
| run: | | |
| uv sync --group tests --frozen | |
| source .venv/bin/activate | |
| maturin develop -m dimos/mapping/ray_tracing/rust/py/Cargo.toml | |
| maturin develop -m dimos/navigation/nav_3d/mls_planner/rust/py/Cargo.toml | |
| python -c "import dimos_voxel_ray_tracing, dimos_mls_planner" | |
| python -m pytest -c /dev/null --rootdir . --noconftest --import-mode=importlib -p no:cacheprovider -v \ | |
| dimos/mapping/ray_tracing/test_voxel_map.py \ | |
| dimos/mapping/ray_tracing/test_transformer.py \ | |
| dimos/navigation/nav_3d/mls_planner/test_mls_planner.py \ | |
| dimos/navigation/nav_3d/mls_planner/test_transformer.py | |
| - name: Bake e2e tests | |
| run: uv run pytest -m bake_e2e dimos/cli/bake/test_bake_e2e.py --no-cov | |
| - name: Native module e2e tests | |
| run: | | |
| cargo build --release --locked -p dimos-livox -p dimos-virtual-mid360 | |
| uv run pytest -m native_e2e \ | |
| dimos/hardware/sensors/lidar/livox/test_e2e.py \ | |
| dimos/robot/manipulators/openyam/blueprints/test_learning_collection_e2e.py \ | |
| dimos/imitation/test_datacollection_e2e.py --no-cov | |
| native: | |
| name: Native builds (C++ and Rust) | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| # The SDK headers include both transports, so liblcm and zenoh are both | |
| # needed to build at all. Zenoh has no apt package, so it comes from its | |
| # release | |
| - name: Install system dependencies | |
| env: | |
| ZENOH_VERSION: 1.10.0 | |
| run: | | |
| sudo apt-get update | |
| # lcm.pc carries `Requires: glib-2.0`, so pkg-config cannot resolve | |
| # lcm at all without glib's own .pc file | |
| sudo apt-get install -y liblcm-dev libglib2.0-dev nlohmann-json3-dev | |
| base=https://github.com/eclipse-zenoh | |
| curl -fsSL -o /tmp/zenoh-c.zip \ | |
| "$base/zenoh-c/releases/download/$ZENOH_VERSION/zenoh-c-$ZENOH_VERSION-x86_64-unknown-linux-gnu-standalone.zip" | |
| curl -fsSL -o /tmp/zenoh-cpp.zip \ | |
| "$base/zenoh-cpp/releases/download/$ZENOH_VERSION/zenohcpp-$ZENOH_VERSION-standalone.zip" | |
| # A release asset can be replaced in place, so the bytes are pinned too | |
| printf '%s /tmp/zenoh-c.zip\n%s /tmp/zenoh-cpp.zip\n' \ | |
| 1168b3dffa7f4f48ffabfd640a3878ec0527c0a612ce825aa6f93e2cd05762d1 \ | |
| 7a50a74e98fd1e1e7ed8461b8490b30d67f878649f0727dea2cf0936501780af \ | |
| | sha256sum -c - | |
| # The zips are laid out for /usr/local, where cmake and the loader | |
| # already look | |
| sudo unzip -q /tmp/zenoh-c.zip -d /usr/local | |
| sudo unzip -q /tmp/zenoh-cpp.zip -d /usr/local | |
| sudo ldconfig | |
| - name: Configure | |
| run: cmake -S native/cpp -B build/native-cpp -DDIMOS_NATIVE_BUILD_TESTS=ON | |
| - name: Build | |
| run: cmake --build build/native-cpp -j | |
| - name: Test | |
| run: ctest --test-dir build/native-cpp --output-on-failure | |
| - name: Find C++ native modules | |
| id: find-modules | |
| run: | | |
| modules=$(find . -name flake.nix \ | |
| -not -path './.git/*' \ | |
| -not -path '*/build/*' \ | |
| -not -path '*/result/*' \ | |
| -printf '%h\n' \ | |
| | while read -r dir; do | |
| [ -f "$dir/CMakeLists.txt" ] && echo "${dir#./}" | |
| done \ | |
| | sort) | |
| if [ -z "$modules" ]; then | |
| echo "::error::No C++ native modules found - the discovery glob is wrong" | |
| exit 1 | |
| fi | |
| echo "Found C++ native modules:" | |
| echo "$modules" | |
| { | |
| echo "modules<<EOF" | |
| echo "$modules" | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Free runner disk space | |
| uses: BRAINSia/free-disk-space@v2.1.3 | |
| - name: Install Nix | |
| # Substituting needs only the public key, so no auth token is used here | |
| # and this stays safe on fork PRs. | |
| env: | |
| INPUT_EXTRA_NIX_CONFIG: | | |
| extra-substituters = https://dimensionalos.cachix.org | |
| extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= | |
| INPUT_SET_AS_TRUSTED_USER: "true" | |
| run: bash docker/ros/install-nix.sh | |
| - name: Build Rust memory recorder Nix package | |
| run: | | |
| cd dimos/experimental/memory/rust | |
| nix build .#dimos-memory-recorder --no-write-lock-file --print-build-logs | |
| test -x result/bin/dimos-memory-recorder | |
| # PCL, GTSAM and the SLAM cores come from each module's flake, not apt. | |
| - name: Build C++ native modules | |
| env: | |
| MODULES: ${{ steps.find-modules.outputs.modules }} | |
| run: | | |
| for module in $MODULES; do | |
| echo "::group::$module" | |
| (cd "$module" && nix build --no-write-lock-file --print-build-logs) | |
| echo "::endgroup::" | |
| done | |
| - name: Test C++ native modules | |
| env: | |
| MODULES: ${{ steps.find-modules.outputs.modules }} | |
| run: | | |
| for module in $MODULES; do | |
| if grep -qE 'enable_testing|add_test' "$module/CMakeLists.txt"; then | |
| echo "::group::ctest $module" | |
| (cd "$module" && ctest --test-dir build --output-on-failure) | |
| echo "::endgroup::" | |
| fi | |
| done | |
| md-babel: | |
| timeout-minutes: 15 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Fetch LFS data needed by doc code blocks | |
| run: git lfs pull --include="data/.lfs/go2_bigoffice.db.tar.gz,data/.lfs/unitree_go2_bigoffice_map.pickle.tar.gz" --exclude="" | |
| # Docs decode JPEG from SQLite via PyTurboJPEG; pyaudio needs portaudio. | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libturbojpeg portaudio19-dev | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| enable-cache: true | |
| prune-cache: true | |
| - uses: actions/setup-node@v7 | |
| with: | |
| node-version: 'lts/*' | |
| - name: Install Python dependencies | |
| run: uv sync --group tests --frozen | |
| - name: Restore Hugging Face model cache | |
| id: hf-cache | |
| uses: actions/cache/restore@v6 | |
| with: | |
| path: ~/.cache/huggingface | |
| # The docs determine which models get downloaded, so any docs | |
| # change may introduce a new model: go online and re-save the | |
| # cache then. restore-keys still seeds the new cache with the | |
| # previously cached models, so only genuinely new models are | |
| # downloaded. | |
| key: hf-models-${{ hashFiles('docs/**/*.md') }} | |
| restore-keys: | | |
| hf-models- | |
| - name: Prefetch Hugging Face models | |
| if: steps.hf-cache.outputs.cache-hit != 'true' | |
| # Anonymous downloads from shared GitHub runner IPs get rate-limited | |
| # by huggingface.co (HTTP 429), so retry with backoff. Models already | |
| # in the restored cache are not re-downloaded. | |
| run: | | |
| for i in 1 2 3 4 5; do | |
| uv run python -c "from transformers import CLIPModel, CLIPProcessor; m = 'openai/clip-vit-base-patch32'; CLIPModel.from_pretrained(m); CLIPProcessor.from_pretrained(m)" && exit 0 | |
| echo "Hugging Face download failed (likely rate-limited); retry $i in $((i * 30))s" | |
| sleep $((i * 30)) | |
| done | |
| echo "::error::Could not download Hugging Face models after 5 attempts" | |
| exit 1 | |
| - name: Save Hugging Face model cache | |
| # Save right after the download (not in a post step) so a failure in | |
| # the doc blocks below doesn't throw away the downloaded models. | |
| if: steps.hf-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v6 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: hf-models-${{ hashFiles('docs/**/*.md') }} | |
| - name: Run Hugging Face offline on exact cache hit | |
| # Zero huggingface.co requests when every model is already cached: | |
| # immune to rate limiting. | |
| if: steps.hf-cache.outputs.cache-hit == 'true' | |
| run: echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" | |
| - name: Execute documentation code blocks | |
| run: ./bin/run-doc-codeblocks --ci --no-cache | |
| web: | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Extract Deno version | |
| id: deno-version | |
| run: echo "version=$(grep -oP 'DENO_VERSION = "\K[^"]+' dimos/utils/deno.py)" >> "$GITHUB_OUTPUT" | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v2 | |
| with: | |
| deno-version: ${{ steps.deno-version.outputs.version }} | |
| cache: true | |
| - name: Deno fmt | |
| working-directory: web | |
| run: deno fmt --check | |
| - name: Deno lint | |
| working-directory: web | |
| run: deno lint | |
| - name: Deno install (validates deno.lock) | |
| working-directory: web | |
| run: deno install --frozen | |
| - name: Deno check (relay + shared) | |
| working-directory: web | |
| run: deno task check | |
| - name: Deno test (relay + shared) | |
| working-directory: web | |
| run: deno task test | |
| - name: SDK type-check | |
| working-directory: web/sdk | |
| run: deno task check | |
| - name: SDK vitest | |
| working-directory: web/sdk | |
| run: deno task test | |
| - name: Cockpit type-check | |
| working-directory: web/cockpit | |
| run: deno task check | |
| - name: Cockpit vitest | |
| working-directory: web/cockpit | |
| run: deno task test | |
| - name: SDK bundle build | |
| working-directory: web/sdk | |
| run: deno task build | |
| - name: Cockpit build | |
| working-directory: web/cockpit | |
| run: deno task build | |
| # ---- Browser e2e: dimos run --local-relay + Playwright on the dist ---- | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| enable-cache: true | |
| prune-cache: true | |
| - name: Setup Python | |
| uses: actions/setup-python@v7.0.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Install native dependencies (pyaudio build, JPEG encoding, audio decode) | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y portaudio19-dev libturbojpeg ffmpeg | |
| - name: Fetch the go2_short replay dataset | |
| run: git lfs pull --include="data/.lfs/go2_short.db.tar.gz" --exclude="" | |
| - name: Install dependencies | |
| run: uv sync --group tests --group browser-tests --frozen | |
| - name: Install Playwright browsers # the e2e runs in both engines | |
| run: uv run playwright install --with-deps chromium firefox | |
| - name: Cockpit + SDK browser e2e | |
| run: >- | |
| uv run pytest -m web_browser dimos/e2e_tests/test_cockpit_browser.py | |
| dimos/e2e_tests/test_sdk_browser.py | |
| dimos/e2e_tests/test_custom_channel_browser.py | |
| dimos/e2e_tests/test_lcm_channel_browser.py | |
| dimos/e2e_tests/test_publish_browser.py | |
| dimos/e2e_tests/test_voice_browser.py | |
| dimos/e2e_tests/test_stats_browser.py | |
| dimos/e2e_tests/test_robot_picker_browser.py | |
| dimos/e2e_tests/test_relay_auth_browser.py --no-cov | |
| - name: Relay image build # docker/relay/Dockerfile must not rot silently | |
| run: docker build -f docker/relay/Dockerfile . | |
| tests: | |
| if: | | |
| !cancelled() && | |
| contains(fromJSON('["success", "skipped"]'), needs.cachix-build-check.result) && | |
| contains(fromJSON('["success", "skipped"]'), needs.cachix-build.result) | |
| needs: [cachix-build-check, cachix-build] | |
| timeout-minutes: 20 | |
| strategy: | |
| matrix: | |
| pyver: ['3.10', '3.11', '3.12'] | |
| os: ["ubuntu-latest"] | |
| experimental: [false] | |
| include: | |
| - os: "ubuntu-24.04-arm" | |
| pyver: "3.12" | |
| experimental: false | |
| fail-fast: true | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: ${{ matrix.experimental }} | |
| env: | |
| # Arm faulthandler from interpreter start — before conftest is even | |
| # imported — so import-time crashes are covered too. dimos/conftest.py | |
| # redirects it to a per-process file under $RUNNER_TEMP/pytest-crash, | |
| # which is what makes a crash legible under xdist (one file per worker). | |
| PYTHONFAULTHANDLER: "1" | |
| # Zenoh's own scouting decisions, for diagnosing workers that never link. | |
| RUST_LOG: "zenoh::net::runtime::orchestrator=debug" | |
| permissions: | |
| contents: read # For checkout | |
| id-token: write # For codecov-action's OIDC upload | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| enable-cache: true | |
| prune-cache: true | |
| python-version: ${{ matrix.pyver }} | |
| - name: Extract Deno version | |
| id: deno-version | |
| run: echo "version=$(grep -oP 'DENO_VERSION = "\K[^"]+' dimos/utils/deno.py)" >> "$GITHUB_OUTPUT" | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v2 | |
| with: | |
| deno-version: ${{ steps.deno-version.outputs.version }} | |
| cache: true | |
| # pyaudio needs portaudio; relay/audio tests need TurboJPEG and ffmpeg. | |
| - name: Install system dependencies (Ubuntu) | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y portaudio19-dev libturbojpeg ffmpeg | |
| - name: Install dependency for pyaudio (macOS) | |
| if: startsWith(matrix.os, 'macos') | |
| run: brew install portaudio | |
| - name: Cap accidental LFS downloads at 1 MiB | |
| run: | | |
| sudo mv /usr/bin/git-lfs /usr/bin/git-lfs.orig | |
| sudo install -m 0755 bin/git-lfs-guard /usr/bin/git-lfs | |
| git config --global --unset-all filter.lfs.process || true | |
| # The published store paths are x86_64-linux, so the arm leg (and any | |
| # future macOS leg) skips provisioning; binary-needing tests skip there. | |
| # On a marker miss (fork PR, eviction) the binaries are simply absent — | |
| # hosted runners never fall back to building the heavy C++ closures. | |
| - name: Restore publish marker | |
| if: runner.os == 'Linux' && runner.arch == 'X64' | |
| id: native-marker | |
| uses: actions/cache/restore@v6 | |
| with: | |
| path: .cachix-marker | |
| key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }} | |
| - name: Install Nix (with Cachix substituter) | |
| # cache-hit implies the restore ran, which is already Linux/X64-gated. | |
| if: steps.native-marker.outputs.cache-hit == 'true' | |
| env: | |
| INPUT_EXTRA_NIX_CONFIG: | | |
| extra-substituters = https://dimensionalos.cachix.org | |
| extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= | |
| INPUT_SET_AS_TRUSTED_USER: "true" | |
| run: bash docker/ros/install-nix.sh | |
| - name: Provision native modules from Cachix | |
| if: steps.native-marker.outputs.cache-hit == 'true' | |
| run: python3 bin/build-native-modules --link-results .cachix-marker/links.txt | |
| - name: Install dependencies | |
| run: uv sync --group tests --frozen | |
| - name: Run tests | |
| run: uv run pytest --numprocesses=logical --cov=dimos/ --junitxml=junit.xml -m 'not (self_hosted or mujoco or self_hosted_large or web_browser or bake_e2e or native_e2e)' | |
| - name: Re-run the failing tests with maximum verbosity | |
| if: failure() | |
| env: | |
| COLOR: yes | |
| run: >- # `exit 1` makes sure that the job remains red with flaky runs | |
| uv run pytest --no-cov -vvvvv --lf -m 'not (self_hosted or mujoco or self_hosted_large or web_browser or bake_e2e or native_e2e)' && exit 1 | |
| shell: bash | |
| - name: Print crash diagnostics | |
| if: failure() | |
| shell: bash | |
| run: | | |
| shopt -s nullglob | |
| for f in "$RUNNER_TEMP"/pytest-crash/*.log; do | |
| echo "::group::$(basename "$f")" | |
| cat "$f" | |
| echo "::endgroup::" | |
| done | |
| - name: Turn coverage into xml | |
| run: uv run python -m coverage xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| disable_search: true | |
| fail_ci_if_error: true | |
| files: ./coverage.xml | |
| flags: OS-${{ matrix.os }},Py-${{ matrix.pyver }} | |
| use_oidc: true | |
| # Install the CLI from PyPI (pinned) instead of cli.codecov.io: | |
| # the GPG key for the default download path is served from a dead | |
| # keybase.io URL, failing every upload. See | |
| # codecov/codecov-action#1955 and #2398. | |
| use_pypi: true | |
| version: "11.2.8" | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| report_type: test_results | |
| use_oidc: true | |
| # See the "Upload coverage" step above. | |
| use_pypi: true | |
| version: "11.2.8" | |
| self-hosted-tests: | |
| # Runs on push, merge_group (trusted — a maintainer must approve + queue the | |
| # PR), and same-repo PRs. Skipped on fork PRs, which would expose the | |
| # self-hosted runner to untrusted code from external contributors. | |
| if: | | |
| !cancelled() && | |
| (github.event_name == 'push' || github.event_name == 'merge_group' || github.event.pull_request.head.repo.full_name == github.repository) && | |
| needs.cachix-build-check.result == 'success' && | |
| contains(fromJSON('["success", "skipped"]'), needs.cachix-build.result) | |
| needs: [compute-ros-pin, cachix-build-check, cachix-build, cachix-build-macos] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }} | |
| # See the `tests` job for why this is set. | |
| PYTHONFAULTHANDLER: "1" | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: Linux | |
| # GitHub Actions only honours `container:` on Linux runners. | |
| container: | |
| image: dimensional/ros-dev@${{ needs.compute-ros-pin.outputs.digest }} | |
| # Hard-cap container RAM so a runaway test OOMs inside the container, not | |
| # as a host global-OOM that takes out sshd/tmux. | |
| options: --memory=6g --memory-swap=6g | |
| volumes: | |
| - /var/cache/dimos-root-cache:/root/.cache | |
| # Persistent Nix store — without it every run reinstalls nix | |
| # and re-downloads the modules' full runtime closure (~600 MB) | |
| # through the runner's uplink. GC'd at the end of the job; the | |
| # provision step roots the live closure so GC keeps it. | |
| - /var/cache/dimos-nix:/nix | |
| markers: "self_hosted or skipif_no_ros" | |
| experimental: false | |
| - os: macOS | |
| container: null # run on host — `container:` is Linux-only | |
| # web_browser: the cockpit browser e2e that Linux runs in the `web` job. | |
| markers: "self_hosted or web_browser" | |
| experimental: true | |
| runs-on: | |
| - self-hosted | |
| - ${{ matrix.os }} | |
| - base | |
| continue-on-error: ${{ matrix.experimental }} | |
| permissions: | |
| contents: read # For checkout | |
| id-token: write # For codecov-action's OIDC upload | |
| container: ${{ matrix.container }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| with: | |
| clean: false | |
| # If we ever allow external PRs on custom runner, persisting credentials | |
| # could be abused by attackers. | |
| persist-credentials: false | |
| - name: Fix permissions | |
| run: | | |
| git config --global --add safe.directory '*' | |
| git clean -ffdx | |
| - name: Point uv at a persistent cache on the workspace mount | |
| # HOME in container jobs is an ephemeral /github/home, so uv's default | |
| # cache would be dropped after every run. The workspace mount (/__w, | |
| # the runner's _work dir) persists on the self-hosted runner, and | |
| # unlike the /root/.cache volume it is the same mount as .venv: | |
| # hardlinks across two docker bind mounts fail with EXDEV even on one | |
| # filesystem, making uv fall back to a minutes-long full copy. | |
| if: matrix.os == 'Linux' | |
| run: echo "UV_CACHE_DIR=$(dirname "$RUNNER_WORKSPACE")/uv-cache" >> "$GITHUB_ENV" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| # We persist on disk above, cache would clean some of this up. | |
| enable-cache: false | |
| - name: Install codecov CLI | |
| # Self-hosted runners have no `pip` on PATH, so codecov-action's | |
| # use_pypi can't work here; pre-install the pinned CLI with uv and | |
| # point the action at it via `binary`. See codecov/codecov-action#1955 | |
| # and #2398. | |
| run: | | |
| uv venv .codecov-venv | |
| uv pip install --python .codecov-venv/bin/python codecov-cli==11.2.8 | |
| - name: Install dependencies | |
| run: uv sync --group tests-self-hosted --frozen | |
| - name: Install web_browser e2e dependencies (macOS) | |
| if: matrix.os == 'macOS' | |
| run: | | |
| brew install ffmpeg | |
| uv sync --group tests-self-hosted --group browser-tests --frozen | |
| uv run playwright install chromium firefox | |
| - name: Build C++ extensions in-place | |
| run: uv run python setup.py build_ext --inplace | |
| - name: Source ROS environment | |
| # The uv venv is sealed (include-system-site-packages = false), so | |
| # `import rclpy` / `ament_index_python` would fail. Sourcing the ROS | |
| # setup script and exporting PYTHONPATH/AMENT_PREFIX_PATH/etc into | |
| # GITHUB_ENV makes them importable from `uv run`. | |
| if: matrix.os == 'Linux' | |
| shell: bash | |
| run: | | |
| source /opt/ros/${ROS_DISTRO}/setup.bash | |
| { | |
| echo "PYTHONPATH=$PYTHONPATH" | |
| echo "AMENT_PREFIX_PATH=$AMENT_PREFIX_PATH" | |
| echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" | |
| echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" | |
| echo "ROS_DISTRO=$ROS_DISTRO" | |
| echo "ROS_VERSION=$ROS_VERSION" | |
| echo "ROS_PYTHON_VERSION=$ROS_PYTHON_VERSION" | |
| } >> "$GITHUB_ENV" | |
| - name: Install Nix (with Cachix substituter) | |
| if: contains(matrix.markers, 'skipif_no_ros') | |
| # /nix is bind-mounted from the self-hosted runner's host filesystem | |
| # (see this job's matrix container.volumes). On warm runs Nix is | |
| # already installed; install-nix.sh's `type -p nix` check will exit | |
| # early, but only if the profile bin is on PATH — prepend it before | |
| # calling. If /nix is empty (first job on a fresh runner), nix isn't | |
| # on PATH and the install proceeds normally. After install-nix.sh, | |
| # we unconditionally (re-)write /etc/nix/nix.conf: on warm runs | |
| # install-nix.sh exited before it would have written the config, | |
| # which leaves `experimental-features = nix-command flakes` absent | |
| # and the build dying with "experimental Nix feature 'nix-command' | |
| # is disabled". Writing it here keeps cold and warm runs identical. | |
| run: | | |
| export PATH="/nix/var/nix/profiles/default/bin:$PATH" | |
| bash docker/ros/install-nix.sh | |
| sudo mkdir -p /etc/nix | |
| sudo tee /etc/nix/nix.conf > /dev/null <<'EOF' | |
| experimental-features = nix-command flakes | |
| extra-substituters = https://dimensionalos.cachix.org | |
| extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= | |
| always-allow-substitutes = true | |
| build-users-group = | |
| # Substitute-only: everything should be compiled already and stored in Cachix. | |
| max-jobs = 0 | |
| EOF | |
| echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" | |
| - name: Put Nix on PATH (macOS) | |
| if: matrix.os == 'macOS' | |
| run: | | |
| bin=/nix/var/nix/profiles/default/bin | |
| if [ ! -x "$bin/nix-store" ]; then | |
| echo "::error::Nix is not installed on this macOS runner host — install multi-user Nix and add the dimensionalos substituter to /etc/nix/nix.conf (see this step's comment)" | |
| exit 1 | |
| fi | |
| echo "$bin" >> "$GITHUB_PATH" | |
| echo "NIX_CONFIG=experimental-features = nix-command flakes" >> "$GITHUB_ENV" | |
| - name: Restore publish marker | |
| id: native-marker | |
| uses: actions/cache/restore@v6 | |
| with: | |
| path: .cachix-marker | |
| key: cachix-published-${{ runner.os }}-${{ runner.arch }}-${{ needs.cachix-build-check.outputs.inputs-hash }} | |
| - name: Provision native modules from Cachix | |
| # The marker records every result -> store path mapping, so the warm | |
| # path is symlink recreation against the persistent /nix plus | |
| # substitution of anything missing by exact path — no nix evaluation. | |
| # A missed marker (evicted) falls back to the full nix build. | |
| run: | | |
| if [ "${{ steps.native-marker.outputs.cache-hit }}" = "true" ]; then | |
| python3 bin/build-native-modules --link-results .cachix-marker/links.txt | |
| else | |
| python3 bin/build-native-modules | |
| fi | |
| # Linux only: root the out paths so this job's end-of-GC keeps the | |
| # closure warm in the CI-managed bind-mounted store (the workspace | |
| # result links die in the next run's `git clean`, so they cannot | |
| # serve as roots). The macOS host manages its own store and GC. | |
| if [ "${{ matrix.os }}" = "Linux" ]; then | |
| sudo mkdir -p /nix/var/nix/gcroots/dimos-native | |
| sudo find /nix/var/nix/gcroots/dimos-native -maxdepth 1 -type l -delete | |
| python3 bin/build-native-modules --record-links | cut -d' ' -f2 | sort -u | | |
| while read -r p; do | |
| sudo ln -sfn "$p" "/nix/var/nix/gcroots/dimos-native/$(basename "$p")" | |
| done | |
| fi | |
| - name: Point cargo at a persistent cache on the workspace mount | |
| if: contains(matrix.markers, 'self_hosted') | |
| run: | | |
| echo "RUSTUP_HOME=$(dirname "$RUNNER_WORKSPACE")/rustup" >> "$GITHUB_ENV" | |
| echo "CARGO_HOME=$(dirname "$RUNNER_WORKSPACE")/cargo" >> "$GITHUB_ENV" | |
| - name: Install Rust toolchain | |
| if: contains(matrix.markers, 'self_hosted') | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Build rust native modules | |
| if: contains(matrix.markers, 'self_hosted') | |
| run: cargo build --release --locked -p dimos-livox -p dimos-virtual-mid360 | |
| - name: Run tests | |
| run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m '(${{ matrix.markers }}) and not mujoco' | |
| - name: Re-run the failing tests with maximum verbosity | |
| if: failure() | |
| env: | |
| COLOR: yes | |
| run: >- # `exit 1` makes sure that the job remains red with flaky runs | |
| uv run pytest --no-cov -vvvvv --lf -m '(${{ matrix.markers }}) and not mujoco' && exit 1 | |
| shell: bash | |
| - name: Print crash diagnostics | |
| if: failure() | |
| shell: bash | |
| run: | | |
| shopt -s nullglob | |
| for f in "$RUNNER_TEMP"/pytest-crash/*.log; do | |
| echo "::group::$(basename "$f")" | |
| cat "$f" | |
| echo "::endgroup::" | |
| done | |
| - name: Turn coverage into xml | |
| run: uv run python -m coverage xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| disable_search: true | |
| fail_ci_if_error: true | |
| files: ./coverage.xml | |
| flags: SelfHosted-${{ matrix.os }} | |
| use_oidc: true | |
| # Use the pinned CLI from the "Install codecov CLI" step (no pip | |
| # on self-hosted runners, and the default cli.codecov.io download | |
| # has a broken GPG key fetch). See codecov/codecov-action#1955. | |
| binary: .codecov-venv/bin/codecovcli | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| report_type: test_results | |
| use_oidc: true | |
| # See the "Upload coverage" step above. | |
| binary: .codecov-venv/bin/codecovcli | |
| - name: Check disk space | |
| if: failure() | |
| run: | | |
| df -h | |
| - name: Prune the persistent uv cache | |
| # The cache persists on the runner (see "Point uv at a persistent | |
| # cache" above), so lock bumps accumulate superseded wheels | |
| # forever. Drop unreachable objects each run; past 25 GB, `--ci` also | |
| # evicts pre-built wheels (re-downloaded on the next run) while keeping | |
| # the expensive wheels built from source. uv has nothing age-based. | |
| # Not always(): on a cancelled run, lingering test processes still | |
| # hold the shared cache lock and prune would block on it until the | |
| # job timeout. | |
| if: ${{ !cancelled() && matrix.os == 'Linux' }} | |
| run: | | |
| uv cache prune | |
| size_mb=$(du -sm "$UV_CACHE_DIR" | cut -f1) | |
| echo "uv cache size: ${size_mb} MB" | |
| if [ "$size_mb" -gt 25600 ]; then uv cache prune --ci; fi | |
| - name: GC the persistent Nix store | |
| # /nix persists on the runner (container volume above), so superseded | |
| # module closures accumulate as inputs change. Unrooted paths go; the | |
| # provision step's gcroots keep the live closure across runs. Not | |
| # always(): a cancelled run can leave nix processes holding locks. | |
| if: ${{ !cancelled() && matrix.os == 'Linux' }} | |
| run: nix-collect-garbage --delete-older-than 3d | |
| self-hosted-large-tests: | |
| # Skip on PRs from forks which would expose the self-hosted runner to untrusted code from external contributors. | |
| if: | | |
| !cancelled() && | |
| (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && | |
| needs.cachix-build-check.result == 'success' && | |
| contains(fromJSON('["success", "skipped"]'), needs.cachix-build.result) | |
| needs: [cachix-build-check, cachix-build] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }} | |
| DIMSIM_RENDER: gpu | |
| # See the `tests` job for why this is set. | |
| PYTHONFAULTHANDLER: "1" | |
| timeout-minutes: 30 | |
| runs-on: [self-hosted, Linux, large] | |
| permissions: | |
| contents: read # For checkout | |
| id-token: write # For codecov-action's OIDC upload | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| with: | |
| clean: false | |
| # If we ever allow external PRs on custom runner, persisting credentials | |
| # could be abused by attackers. | |
| persist-credentials: false | |
| - name: Fix permissions | |
| run: | | |
| sudo git config --global --add safe.directory '*' | |
| sudo git clean -ffdx | |
| - name: Fetch DimSim LFS assets only | |
| # The checkout above doesn't fetch LFS, so the dimsim scene/embodiment | |
| # glbs are pointer stubs and the browser sim fails to load them. Pull | |
| # ONLY misc/DimSim assets — a full `lfs: true` would also drag in main's | |
| # multi-GB data/.lfs/* objects (some missing on the server → 404). | |
| # persist-credentials is false above, so supply the token for this pull. | |
| run: | | |
| git config --local http.https://github.com/.extraheader \ | |
| "AUTHORIZATION: basic $(printf 'x-access-token:%s' '${{ github.token }}' | base64 -w0)" | |
| git lfs pull --include="misc/DimSim/**" | |
| git config --local --unset-all http.https://github.com/.extraheader | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| # We persist on disk, cache would clean some of this up. | |
| enable-cache: false | |
| - name: Install codecov CLI | |
| # Self-hosted runners have no `pip` on PATH, so codecov-action's | |
| # use_pypi can't work here; pre-install the pinned CLI with uv and | |
| # point the action at it via `binary`. See codecov/codecov-action#1955 | |
| # and #2398. | |
| run: | | |
| uv venv .codecov-venv | |
| uv pip install --python .codecov-venv/bin/python codecov-cli==11.2.8 | |
| - name: Install apt dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| `# Compiler toolchain for the in-tree pybind11 C++ extension.` \ | |
| build-essential \ | |
| `# Runtime lib for opencv-python (cv2 loads libGL.so.1 on import).` \ | |
| libgl1 \ | |
| `# Runtime lib for PyTurboJPEG (fast JPEG encode/decode).` \ | |
| libturbojpeg \ | |
| `# Dev package for PyAudio (built from source against PortAudio).` \ | |
| portaudio19-dev \ | |
| `# Chromium runtime libs required by Playwright (used by dimsim browser tests).` \ | |
| libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libgbm1 \ | |
| libpango-1.0-0 libxcomposite1 libxdamage1 libxkbcommon0 libxrandr2 \ | |
| `# Rerun viewer pulls in X11 keyboard bindings when DISPLAY is set.` \ | |
| libxkbcommon-x11-0 | |
| - name: Install dependencies | |
| run: uv sync --group tests-self-hosted --frozen | |
| - name: Build C++ extensions in-place | |
| run: uv run python setup.py build_ext --inplace | |
| - name: Run tests | |
| # The runner must have an Xorg server with NVIDIA GPU access running | |
| # on DISPLAY=:0 — dimsim's headless Chromium needs real WebGL for the | |
| # heavier scenes (`apt`). Software X via xvfb is not enough. | |
| env: | |
| DISPLAY: ":0" | |
| run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m self_hosted_large | |
| - name: Re-run the failing tests with maximum verbosity | |
| if: failure() | |
| env: | |
| COLOR: yes | |
| DISPLAY: ":0" | |
| run: >- # `exit 1` makes sure that the job remains red with flaky runs | |
| uv run pytest --no-cov -vvvvv --lf -m self_hosted_large && exit 1 | |
| shell: bash | |
| - name: Print crash diagnostics | |
| if: failure() | |
| shell: bash | |
| run: | | |
| shopt -s nullglob | |
| for f in "$RUNNER_TEMP"/pytest-crash/*.log; do | |
| echo "::group::$(basename "$f")" | |
| cat "$f" | |
| echo "::endgroup::" | |
| done | |
| - name: Turn coverage into xml | |
| run: uv run python -m coverage xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| disable_search: true | |
| fail_ci_if_error: true | |
| files: ./coverage.xml | |
| flags: SelfHosted-Large | |
| use_oidc: true | |
| # Use the pinned CLI from the "Install codecov CLI" step (no pip | |
| # on self-hosted runners, and the default cli.codecov.io download | |
| # has a broken GPG key fetch). See codecov/codecov-action#1955. | |
| binary: .codecov-venv/bin/codecovcli | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| report_type: test_results | |
| use_oidc: true | |
| # See the "Upload coverage" step above. | |
| binary: .codecov-venv/bin/codecovcli | |
| - name: Check disk space | |
| if: failure() | |
| run: | | |
| df -h | |
| - name: Prune the persistent uv cache | |
| # See self-hosted-tests; UV_CACHE_DIR isn't set here, so resolve the | |
| # default with `uv cache dir`. | |
| if: ${{ !cancelled() }} | |
| run: | | |
| uv cache prune | |
| size_mb=$(du -sm "$(uv cache dir)" | cut -f1) | |
| echo "uv cache size: ${size_mb} MB" | |
| if [ "$size_mb" -gt 25600 ]; then uv cache prune --ci; fi | |
| # Cross-job fail-fast: GitHub Actions only fail-fasts within a matrix, | |
| # not across sibling jobs. This watcher fires the moment `tests` fails | |
| # and cancels the whole workflow run. | |
| fail-fast: | |
| if: failure() | |
| needs: [tests] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: write # For `gh run cancel` | |
| steps: | |
| - name: Cancel workflow run | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} | |
| ci-complete: # This is used for branch protection. | |
| if: always() | |
| needs: | |
| - lint | |
| - rust | |
| - native | |
| - md-babel | |
| - web | |
| - tests | |
| - self-hosted-tests | |
| #- self-hosted-large-tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write # For codecov-action's OIDC notify | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| allowed-skips: self-hosted-tests, self-hosted-large-tests | |
| jobs: ${{ toJSON(needs) }} | |
| - name: Trigger Codecov notifications | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| run_command: send-notifications | |
| use_oidc: true | |
| fail_ci_if_error: true | |
| # Install the CLI from PyPI (pinned) instead of cli.codecov.io: | |
| # the GPG key for the default download path is served from a dead | |
| # keybase.io URL, failing every upload. See | |
| # codecov/codecov-action#1955 and #2398. | |
| use_pypi: true | |
| version: "11.2.8" |