nightly #216
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nightly | |
| on: | |
| workflow_dispatch: # Allows manual triggering of the workflow | |
| schedule: | |
| - cron: '0 4 * * *' # Runs automatically at 4:00 AM UTC every day | |
| permissions: | |
| id-token: write | |
| issues: write | |
| contents: read | |
| pull-requests: read | |
| concurrency: | |
| group: single-acceptance-job-per-repo | |
| jobs: | |
| integration: | |
| environment: tool | |
| runs-on: larger | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| cache: 'pip' | |
| cache-dependency-path: '**/pyproject.toml' | |
| python-version: '3.11' | |
| - name: Install hatch | |
| run: pip install hatch==1.9.4 | |
| - name: Run unit tests and generate test coverage report | |
| run: make test | |
| # Acceptance tests are run from within tests/integration folder. | |
| # We need to make sure .coveragerc is there so that code coverage is generated for the right modules. | |
| - name: Prepare .coveragerc for integration tests | |
| run: cp .coveragerc tests/integration | |
| # Run tests from `tests/integration` as defined in .codegen.json | |
| # and generate code coverage for modules defined in .coveragerc | |
| - name: Run integration tests and generate test coverage report | |
| uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 | |
| with: | |
| vault_uri: ${{ secrets.VAULT_URI }} | |
| timeout: 2h | |
| create_issues: true | |
| codegen_path: tests/integration/.codegen.json | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} | |
| ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} | |
| # collects all coverage reports: coverage.xml from integration tests, coverage-unit.xml from unit tests | |
| - name: Publish test coverage | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| use_oidc: true | |
| integration_serverless: | |
| environment: tool | |
| runs-on: larger | |
| env: | |
| DATABRICKS_SERVERLESS_COMPUTE_ID: auto | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| cache: 'pip' | |
| cache-dependency-path: '**/pyproject.toml' | |
| python-version: '3.11' | |
| - name: Install hatch | |
| run: pip install hatch==1.9.4 | |
| - name: Run integration tests on serverless cluster | |
| uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 | |
| with: | |
| vault_uri: ${{ secrets.VAULT_URI }} | |
| timeout: 2h | |
| create_issues: true | |
| codegen_path: tests/integration/.codegen.json | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} | |
| ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} | |
| DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }} | |
| e2e: | |
| if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork | |
| environment: tool | |
| runs-on: larger | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| cache: 'pip' | |
| cache-dependency-path: '**/pyproject.toml' | |
| python-version: '3.11' | |
| - name: Install hatch | |
| run: pip install hatch==1.9.4 | |
| - name: Install Databricks CLI | |
| run: | | |
| curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh | |
| databricks --version | |
| - name: Azure login (OIDC) | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.ARM_CLIENT_ID }} | |
| tenant-id: ${{ secrets.ARM_TENANT_ID }} | |
| allow-no-subscriptions: true | |
| - name: Set env vars for Azure CLI auth | |
| run: | | |
| val=$(az keyvault secret show --id "${{ secrets.VAULT_URI }}/secrets/DATABRICKS-HOST" --query value -o tsv) | |
| echo "DATABRICKS_HOST=$val" >> $GITHUB_ENV | |
| echo "DATABRICKS_AUTH_TYPE=azure-cli" >> $GITHUB_ENV | |
| - name: Run e2e tests | |
| uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 | |
| with: | |
| vault_uri: ${{ secrets.VAULT_URI }} | |
| timeout: 2h | |
| create_issues: true | |
| codegen_path: tests/e2e/.codegen.json | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} | |
| ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} | |
| e2e_serverless: | |
| if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork | |
| environment: tool | |
| runs-on: larger | |
| env: | |
| DATABRICKS_SERVERLESS_COMPUTE_ID: auto | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| cache: 'pip' | |
| cache-dependency-path: '**/pyproject.toml' | |
| python-version: '3.10' | |
| - name: Install hatch | |
| run: pip install hatch==1.9.4 | |
| - name: Install Databricks CLI | |
| run: | | |
| curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh | |
| databricks --version | |
| - name: Azure login (OIDC) | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.ARM_CLIENT_ID }} | |
| tenant-id: ${{ secrets.ARM_TENANT_ID }} | |
| allow-no-subscriptions: true | |
| - name: Set env vars for Azure CLI auth | |
| run: | | |
| val=$(az keyvault secret show --id "${{ secrets.VAULT_URI }}/secrets/DATABRICKS-HOST" --query value -o tsv) | |
| echo "DATABRICKS_HOST=$val" >> $GITHUB_ENV | |
| echo "DATABRICKS_AUTH_TYPE=azure-cli" >> $GITHUB_ENV | |
| - name: Run e2e tests on serverless cluster | |
| uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4 | |
| with: | |
| vault_uri: ${{ secrets.VAULT_URI }} | |
| timeout: 2h | |
| create_issues: true | |
| codegen_path: tests/e2e/.codegen.json | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} | |
| ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} | |
| DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }} | |
| benchmark: | |
| environment: tool | |
| runs-on: larger | |
| env: | |
| DATABRICKS_SERVERLESS_COMPUTE_ID: auto | |
| BENCHMARKS_DIR: tests/perf/.benchmarks | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| cache: 'pip' | |
| cache-dependency-path: '**/pyproject.toml' | |
| python-version: '3.11' | |
| - name: Install hatch | |
| run: pip install hatch==1.9.4 | |
| - name: Login to Azure for azure-cli authentication | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.ARM_CLIENT_ID }} | |
| tenant-id: ${{ secrets.ARM_TENANT_ID }} | |
| allow-no-subscriptions: true | |
| - name: Run performance tests and compare with baseline | |
| timeout-minutes: 120 | |
| env: | |
| DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }} | |
| run: | | |
| export DATABRICKS_HOST=$(az keyvault secret show --id "${{ secrets.VAULT_URI }}/secrets/DATABRICKS-HOST" --query value -o tsv) | |
| export DATABRICKS_AUTH_TYPE=azure-cli | |
| # We are not using acceptance action as it does not show the comparison results. | |
| # The run fails if performance degrades by more than 25%. | |
| # Tests are run sequentially to reduce variability. | |
| # Do at least 5 rounds to get more stable results. | |
| hatch run pytest tests/perf -v -n 1 \ | |
| --benchmark-storage=$BENCHMARKS_DIR \ | |
| --benchmark-compare=baseline \ | |
| --benchmark-compare-fail=mean:25% \ | |
| --benchmark-min-rounds=5 |