Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/actions/kind-cluster-setup/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@ runs:
steps:
- uses: actions/checkout@v4
- name: Create k8s Kind Cluster
uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # @v1.12.0
uses: helm/kind-action@v1.12.0
with:
cluster_name: ${{ inputs.cluster_name }}
wait: 300s

- uses: "google-github-actions/auth@v2"
with:
project_id: "sunilarora-fp"
Expand Down
27 changes: 25 additions & 2 deletions .github/workflows/ci-periodic.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
run-eval:
if: github.repository == 'GoogleCloudPlatform/kubectl-ai'
runs-on: ubuntu-latest
timeout-minutes: 15
timeout-minutes: 12
# Add "id-token" with the intended permissions.
permissions:
contents: "read"
Expand All @@ -41,13 +41,36 @@ jobs:
uses: ./.github/actions/kind-cluster-setup
with:
cluster_name: periodic-eval-cluster
continue-on-error: false
timeout-minutes: 3
- name: Run an easy eval
run: |
TEST_ARGS="--llm-provider vertexai --models gemini-2.5-pro --concurrency=1 --task-pattern=scale-" ./dev/ci/periodics/run-evals.sh
for attempt in 1 2; do
echo "=== Evaluation attempt $attempt/2 ==="
if timeout 4m bash -c 'TEST_ARGS="--llm-provider vertexai --models gemini-2.5-pro --concurrency=1 --task-pattern=scale-" ./dev/ci/periodics/run-evals.sh'; then
echo "Evaluation completed successfully on attempt $attempt"
break
else
echo "Attempt $attempt failed or timed out"

# Cleanup any hanging processes
pkill -f k8s-bench || true
pkill -f kubectl-ai || true

if [ $attempt -eq 2 ]; then
echo "❌ Both attempts failed"
exit 1
else
echo "Waiting 10 seconds before retry..."
sleep 10
fi
fi
done
- name: Analyse results
run: |
./dev/ci/periodics/analyze-evals.sh
cat ${{ github.workspace }}/.build/k8s-bench.md >> ${GITHUB_STEP_SUMMARY}

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: false
Loading