Release 17.0.0-beta.16 #117
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| pull_request: | |
| branches: [main, dev] | |
| jobs: | |
| test: | |
| name: Test Suite | |
| runs-on: ubuntu-latest | |
| env: | |
| UMBRACO_CLIENT_ID: umbraco-back-office-mcp | |
| UMBRACO_CLIENT_SECRET: '1234567890' | |
| UMBRACO_BASE_URL: https://localhost:5201 | |
| NODE_TLS_REJECT_UNAUTHORIZED: '0' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: 'npm' | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: '10.0.x' | |
| - name: Cache NuGet packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.nuget/packages | |
| key: nuget-${{ hashFiles('**/*.csproj') }} | |
| restore-keys: nuget- | |
| - name: Install npm dependencies | |
| run: npm ci | |
| - name: Build SDK | |
| run: npm run build | |
| # Refresh workspace bin symlinks so umbraco-mcp-generate-types (shipped | |
| # by the SDK as a `bin`) is reachable from the template's postbuild | |
| # step. npm ci ran before dist/cli/generate-tool-types.js existed, so | |
| # the symlink wasn't created at install time. | |
| - name: Refresh workspace bin symlinks | |
| run: npm rebuild | |
| - name: Build template | |
| run: npm run build -w template | |
| - name: Build hosted MCP | |
| run: npm run build -w packages/hosted-mcp | |
| - name: Build CLI package | |
| run: npm run build -w packages/create-mcp-server | |
| # Smoke-test the bundled CLI: if dist/index.js can't load (e.g. a | |
| # relative-path require resolves differently after bundling), this | |
| # fails in seconds rather than surfacing only after publish. | |
| - name: CLI bundle smoke test | |
| run: | | |
| VERSION=$(node packages/create-mcp-server/dist/index.js --version) | |
| echo "CLI reports version: $VERSION" | |
| test -n "$VERSION" | |
| # ── Unit tests (no infrastructure) ── | |
| - name: SDK unit tests | |
| run: npm run test | |
| - name: Hosted MCP unit tests | |
| run: npm test -w packages/hosted-mcp | |
| - name: CLI unit tests | |
| run: npm test -w packages/create-mcp-server | |
| - name: Template tool handler tests | |
| run: npm run test:template | |
| env: | |
| USE_MOCK_API: 'true' | |
| # ── Integration tests (Wrangler, no Umbraco) ── | |
| - name: CLI integration tests | |
| run: npm run test:cli | |
| - name: Wrangler integration tests | |
| run: npm run test:integration | |
| # ── Tests requiring Umbraco ── | |
| - name: Setup .NET dev certs | |
| run: dotnet dev-certs https | |
| - name: Start Umbraco | |
| run: | | |
| cd tests/umbraco-instance | |
| ASPNETCORE_ENVIRONMENT=Development \ | |
| ASPNETCORE_URLS="http://localhost:5200;https://localhost:5201" \ | |
| dotnet run --no-launch-profile > /tmp/umbraco.log 2>&1 & | |
| echo $! > /tmp/umbraco.pid | |
| - name: Wait for Umbraco | |
| run: | | |
| echo "Waiting for Umbraco to be ready..." | |
| for i in $(seq 1 60); do | |
| if curl -sf http://localhost:5200/umbraco/management/api/v1/server/status 2>/dev/null | grep -q "Run"; then | |
| echo "Umbraco is ready!" | |
| exit 0 | |
| fi | |
| if ! kill -0 $(cat /tmp/umbraco.pid) 2>/dev/null; then | |
| echo "Umbraco process died! Last 50 lines of log:" | |
| tail -50 /tmp/umbraco.log | |
| exit 1 | |
| fi | |
| echo "Attempt $i/60 - waiting 5s..." | |
| sleep 5 | |
| done | |
| echo "Umbraco failed to start within 5 minutes. Last 50 lines of log:" | |
| tail -50 /tmp/umbraco.log | |
| exit 1 | |
| timeout-minutes: 6 | |
| # Restart Umbraco so McpOAuthComposer registers the OAuth client | |
| # (on first boot the DB doesn't exist yet, so it skips) | |
| - name: Restart Umbraco | |
| run: | | |
| kill $(cat /tmp/umbraco.pid) 2>/dev/null || true | |
| sleep 3 | |
| cd tests/umbraco-instance | |
| ASPNETCORE_ENVIRONMENT=Development \ | |
| ASPNETCORE_URLS="http://localhost:5200;https://localhost:5201" \ | |
| dotnet run --no-launch-profile > /tmp/umbraco.log 2>&1 & | |
| echo $! > /tmp/umbraco.pid | |
| for i in $(seq 1 60); do | |
| if curl -sf http://localhost:5200/umbraco/management/api/v1/server/status 2>/dev/null | grep -q "Run"; then | |
| echo "Umbraco restarted!" | |
| exit 0 | |
| fi | |
| sleep 3 | |
| done | |
| echo "Umbraco restart failed" | |
| tail -20 /tmp/umbraco.log | |
| exit 1 | |
| timeout-minutes: 4 | |
| - name: Verify OAuth client registration | |
| run: | | |
| echo "Testing on HTTP (5200)..." | |
| HTTP_RESP=$(curl -sf "http://localhost:5200/umbraco/management/api/v1/security/back-office/authorize?client_id=umbraco-back-office-mcp&redirect_uri=http://127.0.0.1:8787/callback&response_type=code&code_challenge=test&code_challenge_method=S256" -o /dev/null -w "%{http_code}" -L --max-redirs 0 2>&1 || true) | |
| echo "HTTP Response: $HTTP_RESP" | |
| echo "Testing on HTTPS (5201)..." | |
| HTTPS_RESP=$(curl -skf "https://localhost:5201/umbraco/management/api/v1/security/back-office/authorize?client_id=umbraco-back-office-mcp&redirect_uri=http://127.0.0.1:8787/callback&response_type=code&code_challenge=test&code_challenge_method=S256" -o /dev/null -w "%{http_code}" -L --max-redirs 0 2>&1 || true) | |
| echo "HTTPS Response: $HTTPS_RESP" | |
| if echo "$HTTP_RESP" | grep -qE "302|200"; then | |
| echo "OAuth client registered on HTTP" | |
| else | |
| echo "OAuth client NOT registered! Umbraco logs:" | |
| grep -i "mcp\|oauth\|openiddict\|composer" /tmp/umbraco.log | tail -20 | |
| fi | |
| - name: Chained Wrangler integration tests | |
| run: npm run test:integration:chained | |
| # Kill stale workerd processes and clean Wrangler temp dirs from integration tests | |
| - name: Clean up before E2E | |
| run: | | |
| pkill -f workerd || true | |
| rm -rf .wrangler/tmp | |
| sleep 2 | |
| - name: Install Playwright browsers | |
| run: npx playwright install --with-deps chromium | |
| - name: Verify OAuth (pre-E2E) | |
| run: | | |
| RESP=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:5200/umbraco/management/api/v1/security/back-office/authorize?client_id=umbraco-back-office-mcp&redirect_uri=http://127.0.0.1:8787/callback&response_type=code&code_challenge=test&code_challenge_method=S256" --max-redirs 0 2>&1 || true) | |
| echo "OAuth client check: HTTP $RESP" | |
| if [ "$RESP" = "302" ]; then echo "OK — client registered"; else echo "WARN — unexpected response"; fi | |
| - name: Hosted MCP Playwright E2E | |
| run: npx playwright test --config tests/hosted-mcp-e2e/e2e/playwright.config.ts --retries 1 --timeout 120000 | |
| timeout-minutes: 15 | |
| - name: Chained MCP Playwright E2E | |
| run: npx playwright test --config tests/hosted-chained-mcp-e2e/e2e/playwright.config.ts --retries 1 --timeout 120000 | |
| timeout-minutes: 10 | |
| - name: Upload Playwright traces | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-traces | |
| path: | | |
| tests/hosted-mcp-e2e/test-results/ | |
| tests/hosted-chained-mcp-e2e/test-results/ | |
| retention-days: 7 | |
| cli-e2e: | |
| name: New-instance & Existing-instance E2E | |
| runs-on: ubuntu-latest | |
| services: | |
| sqlserver: | |
| image: mcr.microsoft.com/mssql/server:2022-latest | |
| env: | |
| ACCEPT_EULA: Y | |
| MSSQL_SA_PASSWORD: Moloko99 | |
| ports: | |
| - 1433:1433 | |
| options: >- | |
| --health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Moloko99 -C -Q 'SELECT 1' || exit 1" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| NODE_TLS_REJECT_UNAUTHORIZED: '0' | |
| TEST_SQL_CONNECTION_STRING: "Server=localhost,1433;User Id=sa;Password=Moloko99;TrustServerCertificate=True" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: 'npm' | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: '10.0.x' | |
| - name: Setup .NET dev certs | |
| run: dotnet dev-certs https | |
| - name: Install PSW CLI | |
| run: dotnet tool install -g PackageScriptWriter.Cli --prerelease | |
| - name: Cache NuGet packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.nuget/packages | |
| key: nuget-${{ hashFiles('**/*.csproj') }} | |
| restore-keys: nuget- | |
| - name: Install npm dependencies | |
| run: npm ci | |
| - name: Build | |
| run: npm run build && npm run build -w packages/hosted-mcp && npm run build -w packages/create-mcp-server | |
| - name: Run new-instance E2E tests | |
| run: npm run test:e2e -w packages/create-mcp-server | |
| timeout-minutes: 15 | |
| env: | |
| KEEP_E2E_ASSETS: 'true' | |
| - name: Run existing-instance E2E tests | |
| run: npm run test:e2e:existing -w packages/create-mcp-server | |
| timeout-minutes: 10 | |
| - name: Skill E2E (build tool + integration test) | |
| if: >- | |
| github.event.pull_request.base.ref == 'main' && | |
| startsWith(github.event.pull_request.head.ref, 'release/') | |
| run: npm run test:e2e:skills -w packages/create-mcp-server | |
| timeout-minutes: 10 | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| CLAUDECODE: '' | |
| - name: Clean up E2E assets | |
| if: always() | |
| run: npm run test:e2e:cleanup -w packages/create-mcp-server || true | |
| evals: | |
| name: LLM Eval Tests | |
| if: >- | |
| github.event.pull_request.base.ref == 'main' && | |
| startsWith(github.event.pull_request.head.ref, 'release/') | |
| runs-on: ubuntu-latest | |
| env: | |
| NODE_TLS_REJECT_UNAUTHORIZED: '0' | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: 'npm' | |
| - name: Install npm dependencies | |
| run: npm ci | |
| - name: Build SDK | |
| run: npm run build | |
| # Refresh workspace bin symlinks so umbraco-mcp-generate-types (shipped | |
| # by the SDK as a `bin`) is reachable from the template's postbuild | |
| # step. npm ci ran before dist/cli/generate-tool-types.js existed, so | |
| # the symlink wasn't created at install time. | |
| - name: Refresh workspace bin symlinks | |
| run: npm rebuild | |
| - name: Build template | |
| run: npm run build -w template | |
| - name: Run CLI eval tests | |
| run: npm run test:cli:evals |