Starter Smoke Tests #13
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Starter Smoke Tests | |
| on: | |
| schedule: | |
| # Every 6 hours — catches floating dep breakage | |
| - cron: "0 */6 * * *" | |
| workflow_run: | |
| workflows: ["publish / release"] | |
| types: [completed] | |
| pull_request: | |
| paths: | |
| - "examples/integrations/**" | |
| - ".github/workflows/starter-smoke.yml" | |
| workflow_dispatch: {} | |
| permissions: | |
| contents: read | |
| packages: read | |
| jobs: | |
| starter-smoke: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| strategy: | |
| matrix: | |
| starter: | |
| - langgraph-python | |
| - mastra | |
| - langgraph-js | |
| - crewai-crews | |
| - pydantic-ai | |
| - adk | |
| - agno | |
| - llamaindex | |
| - langgraph-fastapi | |
| - strands-python | |
| - ms-agent-framework-python | |
| - ms-agent-framework-dotnet | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| lfs: false | |
| - name: Run starter smoke tests | |
| working-directory: examples/integrations/${{ matrix.starter }} | |
| env: | |
| STARTER: ${{ matrix.starter }} | |
| run: | | |
| docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests | |
| - name: Capture failure cause | |
| if: failure() | |
| id: failure-cause | |
| working-directory: examples/integrations/${{ matrix.starter }} | |
| run: | | |
| { | |
| echo "summary<<CAUSE_EOF" | |
| # Capture error from container logs (containers still running at this point) | |
| build_err=$(docker compose -f docker-compose.test.yml logs app 2>&1 | grep -E "ERR_|Error:|error:|Build error|failed to" | head -3) | |
| agent_err=$(docker compose -f docker-compose.test.yml logs agent 2>&1 | grep -E "Error:|Traceback|ModuleNotFoundError|ImportError" | head -3) | |
| test_err=$(docker compose -f docker-compose.test.yml logs tests 2>&1 | grep -E "✘|FAIL|Error:|expect\(" | head -5) | |
| if [ -n "$build_err" ]; then | |
| echo "Build failure:" | |
| echo "$build_err" | |
| elif [ -n "$agent_err" ]; then | |
| echo "Agent crash:" | |
| echo "$agent_err" | |
| elif [ -n "$test_err" ]; then | |
| echo "Test failure:" | |
| echo "$test_err" | |
| else | |
| echo "Unknown failure — check run logs" | |
| fi | |
| echo "" | |
| # Identify recent changes that may have caused the failure | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| echo "Triggered by PR #${{ github.event.pull_request.number }} (${{ github.event.pull_request.user.login }}): ${{ github.event.pull_request.title }}" | |
| echo "Head: ${{ github.event.pull_request.head.sha }}" | |
| elif [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_run" ]; then | |
| # Use GitHub API instead of git log (avoids needing deep clone) | |
| echo "Recent commits touching this starter (last 12h):" | |
| gh api "repos/${{ github.repository }}/commits?path=examples/integrations/${{ matrix.starter }}&since=$(date -u -d '12 hours ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-12H +%Y-%m-%dT%H:%M:%SZ)&per_page=5" \ | |
| --jq '.[] | "\(.sha[0:7]) \(.commit.message | split("\n")[0])"' 2>/dev/null || true | |
| starter_commits=$(gh api "repos/${{ github.repository }}/commits?path=examples/integrations/${{ matrix.starter }}&since=$(date -u -d '12 hours ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-12H +%Y-%m-%dT%H:%M:%SZ)&per_page=1" --jq 'length' 2>/dev/null || echo "0") | |
| if [ "$starter_commits" = "0" ]; then | |
| echo "No starter code changes — likely a floating dependency update or upstream CopilotKit release" | |
| echo "Recent releases:" | |
| gh api "repos/${{ github.repository }}/releases?per_page=3" \ | |
| --jq '.[] | "\(.tag_name) (\(.published_at[0:10]))"' 2>/dev/null | head -3 || true | |
| fi | |
| fi | |
| echo "CAUSE_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Tear down | |
| if: always() | |
| working-directory: examples/integrations/${{ matrix.starter }} | |
| run: docker compose -f docker-compose.test.yml down -v | |
| - name: Upload test artifacts on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: starter-smoke-${{ matrix.starter }} | |
| path: showcase/tests/test-results/ | |
| retention-days: 7 | |
| - name: Build Slack payload | |
| if: failure() && (github.event_name == 'schedule' || github.event_name == 'workflow_run') | |
| id: slack-payload | |
| env: | |
| SUMMARY_RAW: ${{ steps.failure-cause.outputs.summary }} | |
| STARTER: ${{ matrix.starter }} | |
| run: | | |
| # Strip ANSI sequences (SGR, OSC, and G0/G1 charset designators), then truncate | |
| # to 200 bytes and drop any trailing partial UTF-8 bytes so we don't emit mojibake. | |
| SUMMARY=$(printf '%s' "$SUMMARY_RAW" | head -3 \ | |
| | sed -E 's/\x1b\[[0-9;?]*[A-Za-z]//g; s/\x1b\][^\x07]*\x07//g; s/\x1b[()][A-Za-z0-9]//g' \ | |
| | head -c 200 | iconv -f UTF-8 -t UTF-8//IGNORE) | |
| if [ -z "$SUMMARY" ]; then | |
| SUMMARY="(no failure detail captured — see job log)" | |
| fi | |
| URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| JOB_URL="${URL}/job/${{ github.job }}" | |
| SLACK_MSG=$(mktemp) | |
| SLACK_PAYLOAD=$(mktemp) | |
| # Build the message with REAL newlines, then hand it to jq via --rawfile so escaping is handled correctly. | |
| { | |
| printf ':x: *Starter smoke test failing: %s*\n' "$STARTER" | |
| printf '<%s|View run> · <%s|View job>\n' "$URL" "$JOB_URL" | |
| printf '```\n%s\n```\n' "$SUMMARY" | |
| } > "$SLACK_MSG" | |
| jq -n --rawfile text "$SLACK_MSG" '{text: $text}' > "$SLACK_PAYLOAD" | |
| rm -f "$SLACK_MSG" | |
| echo "payload_path=${SLACK_PAYLOAD}" >> "$GITHUB_OUTPUT" | |
| - name: Alert Slack on failure | |
| if: failure() && (github.event_name == 'schedule' || github.event_name == 'workflow_run') | |
| uses: slackapi/slack-github-action@v2.1.0 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }} | |
| webhook-type: incoming-webhook | |
| payload-file-path: ${{ steps.slack-payload.outputs.payload_path }} | |
| - name: Clean up Slack payload tmpfile | |
| if: always() && steps.slack-payload.outputs.payload_path | |
| run: rm -f "${{ steps.slack-payload.outputs.payload_path }}" |