[Batch] Fix runtime timeout setting (#2492) #5183
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Installation Tests | |
| on: | |
| workflow_dispatch: # Allows manual trigger | |
| push: | |
| branches: [ "main", "release-*" ] | |
| paths: | |
| - '.github/workflows/**' | |
| - 'build/container/**' | |
| - 'config/**' | |
| - 'development/app/**' | |
| - 'pkg/**' | |
| - 'test/**' | |
| - 'cmd/**' | |
| - 'python/**' | |
| - 'Makefile' | |
| - 'go.mod' | |
| - 'go.sum' | |
| pull_request: | |
| branches: [ "main", "release-*" ] | |
| paths: | |
| - '.github/workflows/**' | |
| - 'build/container/**' | |
| - 'config/**' | |
| - 'development/app/**' | |
| - 'pkg/**' | |
| - 'test/**' | |
| - 'cmd/**' | |
| - 'python/**' | |
| - 'Makefile' | |
| - 'go.mod' | |
| - 'go.sum' | |
| jobs: | |
| mock-app-auth-test: | |
| name: Mock app auth test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r development/app/requirements.txt pytest | |
| - name: Run mock app auth tests | |
| run: pytest development/app/test_auth_e2e.py -q | |
| build-images: | |
| name: Build image (${{ matrix.image }}) | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - image: controller-manager | |
| dockerfile: build/container/Dockerfile | |
| context: . | |
| - image: gateway-plugins | |
| dockerfile: build/container/Dockerfile.gateway | |
| context: . | |
| - image: runtime | |
| dockerfile: build/container/Dockerfile.python | |
| context: . | |
| - image: metadata-service | |
| dockerfile: build/container/Dockerfile.python | |
| context: . | |
| - image: vllm-mock | |
| dockerfile: development/app/Dockerfile | |
| context: development/app | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Build ${{ matrix.image }} | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: ${{ matrix.context }} | |
| file: ${{ matrix.dockerfile }} | |
| tags: aibrix/${{ matrix.image }}:${{ github.sha }} | |
| cache-from: type=gha,scope=${{ matrix.image }} | |
| cache-to: type=gha,mode=max,scope=${{ matrix.image }} | |
| load: true | |
| - name: Save image | |
| run: | | |
| docker save aibrix/${{ matrix.image }}:${{ github.sha }} > ${{ matrix.image }}.tar | |
| - name: Upload image artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ${{ matrix.image }}-image | |
| path: ${{ matrix.image }}.tar | |
| retention-days: 1 | |
| installation-test: | |
| needs: [build-images] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '1.22' | |
| - name: Download image artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| # Only the matrix image tarballs (e.g. controller-manager-image). Do not | |
| # download Docker buildx cache artifacts (vllm-project~aibrix~*.dockerbuild), | |
| # which are unrelated and can fail to extract after retries. | |
| pattern: '*-image' | |
| merge-multiple: false | |
| - name: Verify image artifacts | |
| run: | | |
| missing=() | |
| for image in controller-manager gateway-plugins runtime metadata-service vllm-mock; do | |
| if [ ! -f "${image}-image/${image}.tar" ]; then | |
| missing+=("${image}") | |
| fi | |
| done | |
| if [ ${#missing[@]} -gt 0 ]; then | |
| echo "::warning::Missing artifacts: ${missing[*]}. Attempting re-download..." | |
| for image in "${missing[@]}"; do | |
| gh run download ${{ github.run_id }} --name "${image}-image" --dir "${image}-image" | |
| done | |
| for image in "${missing[@]}"; do | |
| if [ ! -f "${image}-image/${image}.tar" ]; then | |
| echo "::error::Artifact ${image}-image/${image}.tar still missing after retry" | |
| exit 1 | |
| fi | |
| done | |
| fi | |
| echo "All image artifacts verified successfully" | |
| ls -lh *-image/*.tar | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Free disk space | |
| run: |- | |
| # https://github.com/actions/runner-images/issues/2840#issuecomment-2272410832 | |
| # Remove software and language runtimes we're not using | |
| sudo rm -rf \ | |
| /opt/google/chrome \ | |
| /opt/microsoft/msedge \ | |
| /opt/microsoft/powershell \ | |
| /opt/pipx \ | |
| /usr/lib/mono \ | |
| /usr/local/julia* \ | |
| /usr/local/lib/android \ | |
| /usr/local/lib/node_modules \ | |
| /usr/local/share/chromium \ | |
| /usr/local/share/powershell \ | |
| /usr/share/dotnet \ | |
| /usr/share/swift | |
| df -h / | |
| - name: Cache kind | |
| id: cache-kind | |
| uses: actions/cache@v5 | |
| with: | |
| path: /tmp/tools/kind | |
| key: kind-v0.24.0-${{ runner.os }}-${{ runner.arch }} | |
| - name: Install kind | |
| run: | | |
| if [ ! -f /tmp/tools/kind ]; then | |
| mkdir -p /tmp/tools | |
| [ $(uname -m) = x86_64 ] && curl -Lo /tmp/tools/kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64 | |
| chmod +x /tmp/tools/kind | |
| fi | |
| sudo cp /tmp/tools/kind /usr/local/bin/kind | |
| kind version | |
| - name: Cache kustomize | |
| id: cache-kustomize | |
| uses: actions/cache@v5 | |
| with: | |
| path: /tmp/tools/kustomize | |
| key: kustomize-v5.4.3-${{ runner.os }}-${{ runner.arch }} | |
| - name: Install kustomize | |
| run: | | |
| if [ ! -f /tmp/tools/kustomize ]; then | |
| mkdir -p /tmp/tools | |
| curl -L https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v5.4.3/kustomize_v5.4.3_$(go env GOOS)_$(go env GOARCH).tar.gz | tar -xz -C /tmp/tools/ | |
| chmod +x /tmp/tools/kustomize | |
| fi | |
| sudo cp /tmp/tools/kustomize /usr/local/bin/kustomize | |
| kustomize version | |
| - name: Set up testing environment | |
| run: | | |
| # Set up the cluster with kind or other Kubernetes environment needed | |
| kind create cluster --name installation-test --config=./hack/ci/kind-config.yaml | |
| kubectl cluster-info --context kind-installation-test | |
| - name: Load image into Kind | |
| run: | | |
| set -euo pipefail | |
| sha="${{ github.sha }}" | |
| # Load tarballs into the local Docker daemon in parallel (I/O-bound). | |
| for image in controller-manager gateway-plugins runtime metadata-service vllm-mock; do | |
| docker load < "${image}-image/${image}.tar" & | |
| done | |
| wait | |
| # Same digest as :sha; manifests still reference :nightly in places (e.g. job templates). | |
| for image in controller-manager gateway-plugins runtime metadata-service vllm-mock; do | |
| docker tag "aibrix/${image}:${sha}" "aibrix/${image}:nightly" | |
| done | |
| # One kind load transfers all refs to the cluster nodes (two invocations duplicated work). | |
| kind load docker-image \ | |
| "aibrix/controller-manager:${sha}" "aibrix/controller-manager:nightly" \ | |
| "aibrix/gateway-plugins:${sha}" "aibrix/gateway-plugins:nightly" \ | |
| "aibrix/metadata-service:${sha}" "aibrix/metadata-service:nightly" \ | |
| "aibrix/runtime:${sha}" "aibrix/runtime:nightly" \ | |
| "aibrix/vllm-mock:${sha}" "aibrix/vllm-mock:nightly" \ | |
| --name installation-test | |
| docker build \ | |
| --build-arg INPLACE_E2E_VERSION=v1 \ | |
| -t aibrix/inplace-e2e:v1 \ | |
| -f test/e2e/roleset-inplace-image/Dockerfile \ | |
| test/e2e/roleset-inplace-image | |
| docker build \ | |
| --build-arg INPLACE_E2E_VERSION=v2 \ | |
| -t aibrix/inplace-e2e:v2 \ | |
| -f test/e2e/roleset-inplace-image/Dockerfile \ | |
| test/e2e/roleset-inplace-image | |
| kind load docker-image \ | |
| aibrix/inplace-e2e:v1 \ | |
| aibrix/inplace-e2e:v2 \ | |
| --name installation-test | |
| - name: Deploy controller with the built image | |
| run: | | |
| kubectl apply -k config/dependency --server-side | |
| kubectl apply -k config/crd --server-side | |
| cd config/default | |
| kustomize edit set image controller=aibrix/controller-manager:${{ github.sha }} | |
| kustomize edit set image gateway-plugins=aibrix/gateway-plugins:${{ github.sha }} | |
| kustomize edit set image metadata-service=aibrix/metadata-service:${{ github.sha }} | |
| cd ${{ github.workspace }} | |
| kubectl apply -k config/test | |
| - name: Deploy Workload | |
| run: | | |
| cd development/app/config/mock | |
| kustomize edit set image aibrix/vllm-mock=aibrix/vllm-mock:${{ github.sha }} | |
| kustomize edit set image aibrix/runtime=aibrix/runtime:${{ github.sha }} | |
| kubectl apply -k . | |
| - name: Check pod status | |
| run: | | |
| sleep 60s | |
| # Verify the mock deployment status. | |
| # This pod runs two containers: `llm-engine` (app) and `aibrix-runtime` (sidecar). | |
| # We iterate on the runtime often; missing Poetry deps or startup errors | |
| # can cause CrashLoopBackOff. Make CI failures self-diagnosable by: | |
| # 1) describing the pod to capture conditions/events, and | |
| # 2) dumping the *previous* crash logs from `aibrix-runtime`. | |
| kubectl get pods --all-namespaces | |
| kubectl wait pod --all --for=condition=ready --all-namespaces --timeout=300s | |
| kubectl port-forward svc/llama2-7b 8000:8000 & | |
| kubectl -n envoy-gateway-system port-forward service/envoy-aibrix-system-aibrix-eg-903790dc 8888:80 & | |
| kubectl -n aibrix-system port-forward service/aibrix-redis-master 6379:6379 & | |
| - name: Run e2e tests | |
| run: | | |
| kind get kubeconfig --name installation-test > /tmp/admin.conf | |
| export KUBECONFIG=/tmp/admin.conf | |
| export AIBRIX_ROLESET_INPLACE_E2E=true | |
| make test-e2e | |
| export EXTERNAL_METRICS_E2E_CLUSTER=kind | |
| export EXTERNAL_METRICS_E2E_USE_EXISTING_CONTROLLER=true | |
| export KIND_CLUSTER_NAME=installation-test | |
| make test-e2e-external-metrics | |
| - name: Clean up | |
| run: kind delete cluster --name installation-test | |
| delete-artifacts: | |
| needs: [installation-test] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: vllm-mock-image |