Skip to content
nightly

nightly #498

Sign in to view logs

nightly

nightly #498

Workflow file for this run

.github/workflows/nightly.yml at 87a1c5e
name: nightly
on:
workflow_dispatch: # Allows manual triggering of the workflow
schedule:
- cron: '0 4 * * *' # Runs automatically at 4:00 AM UTC every day
permissions:
contents: read
concurrency:
group: single-acceptance-job-per-repo
cancel-in-progress: false # don't cancel ongoing runs to ensure fixtures are completed and resources terminated
jobs:
unit:
runs-on:
group: databrickslabs-protected-runner-group
labels: linux-ubuntu-latest
permissions:
id-token: write
steps:
- name: Checkout Code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Setup environment
uses: ./.github/actions/setup-env
- name: Run unit tests and generate test coverage report
run: make test
- name: Publish unit test coverage
uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
with:
use_oidc: true
files: coverage-unit.xml
flags: unit
integration:
environment: tool
runs-on:
group: larger-runners
labels: larger
permissions:
id-token: write
pull-requests: read
steps:
- name: Checkout Code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Setup environment
uses: ./.github/actions/setup-env
- name: Pre-build DQX wheel
uses: ./.github/actions/prebuild-wheel
# Integration tests are run from within tests/integration folder.
# Create .coveragerc with correct relative path to source code.
- name: Prepare code coverage configuration for integration tests
run: |
cat > tests/integration/.coveragerc << EOF
[run]
source = ../../src
relative_files = true
parallel = true
EOF
# Run tests from `tests/integration` as defined in .codegen.json
# and generate code coverage for modules defined in .coveragerc
- name: Run integration tests and generate test coverage report
uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
with:
vault_uri: ${{ secrets.VAULT_URI }}
timeout: 2h
create_issues: false # if enabled, it creates an issue for each test failure; disabled to reduce noise
codegen_path: tests/integration/.codegen.json
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
COVERAGE_FILE: ${{ github.workspace }}/.coverage # make sure the coverage report is preserved
- name: Merge coverage reports and convert them to XML
run: make combine-coverage
- name: Publish integration test coverage
uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
with:
use_oidc: true
files: coverage-combined.xml
flags: integration
integration_serverless:
environment: tool
runs-on:
group: larger-runners
labels: larger
permissions:
id-token: write
pull-requests: read
env:
DATABRICKS_SERVERLESS_COMPUTE_ID: auto
steps:
- name: Checkout Code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Setup environment
uses: ./.github/actions/setup-env
- name: Pre-build DQX wheel
uses: ./.github/actions/prebuild-wheel
# Integration tests are run from within tests/integration folder.
# Create .coveragerc with correct relative path to source code.
- name: Prepare code coverage configuration for integration tests
run: |
cat > tests/integration/.coveragerc << EOF
[run]
source = ../../src
relative_files = true
parallel = true
EOF
- name: Run integration tests on serverless cluster
uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
with:
vault_uri: ${{ secrets.VAULT_URI }}
timeout: 2h
create_issues: false
codegen_path: tests/integration/.codegen.json
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}
COVERAGE_FILE: ${{ github.workspace }}/.coverage # make sure the coverage report is preserved
- name: Merge coverage reports and convert them to XML
run: make combine-coverage
- name: Publish serverless integration test coverage
uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
with:
use_oidc: true
files: coverage-combined.xml
flags: integration-serverless
anomaly-tests:
environment: tool
runs-on:
group: larger-runners
labels: larger
permissions:
id-token: write
pull-requests: read
steps:
- name: Checkout Code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Setup environment
uses: ./.github/actions/setup-env
- name: Pre-build DQX wheel
uses: ./.github/actions/prebuild-wheel
# Create .coveragerc with correct relative path to source code.
- name: Prepare code coverage configuration for anomaly tests
run: |
cat > tests/integration_anomaly/.coveragerc << EOF
[run]
source = ../../src
relative_files = true
parallel = true
EOF
- name: Run anomaly integration tests
uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
with:
vault_uri: ${{ secrets.VAULT_URI }}
timeout: 3h
create_issues: false
codegen_path: tests/integration_anomaly/.codegen.json
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
COVERAGE_FILE: ${{ github.workspace }}/.coverage
MLFLOW_ENABLE_DB_SDK: "true"
MLFLOW_TRACKING_URI: databricks
MLFLOW_REGISTRY_URI: databricks-uc
MLFLOW_HTTP_REQUEST_TIMEOUT: "600"
MLFLOW_HTTP_REQUEST_MAX_RETRIES: "9"
- name: Merge coverage reports and convert them to XML
run: make combine-coverage
- name: Publish anomaly test coverage
uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
with:
use_oidc: true
files: coverage-combined.xml
flags: anomaly
anomaly-tests-serverless:
environment: tool
runs-on:
group: larger-runners
labels: larger
permissions:
id-token: write
pull-requests: read
steps:
- name: Checkout Code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Setup environment
uses: ./.github/actions/setup-env
- name: Pre-build DQX wheel
uses: ./.github/actions/prebuild-wheel
# Create .coveragerc with correct relative path to source code.
- name: Prepare code coverage configuration for anomaly tests
run: |
cat > tests/integration_anomaly/.coveragerc << EOF
[run]
source = ../../src
relative_files = true
parallel = true
EOF
- name: Run anomaly integration tests on serverless cluster
uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
with:
vault_uri: ${{ secrets.VAULT_URI }}
timeout: 3h
create_issues: false
codegen_path: tests/integration_anomaly/.codegen.json
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
COVERAGE_FILE: ${{ github.workspace }}/.coverage
DATABRICKS_SERVERLESS_COMPUTE_ID: auto
MLFLOW_ENABLE_DB_SDK: "true"
MLFLOW_TRACKING_URI: databricks
MLFLOW_REGISTRY_URI: databricks-uc
MLFLOW_HTTP_REQUEST_TIMEOUT: "600"
MLFLOW_HTTP_REQUEST_MAX_RETRIES: "9"
- name: Merge coverage reports and convert them to XML
run: make combine-coverage
- name: Publish serverless anomaly test coverage
uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
with:
use_oidc: true
files: coverage-combined.xml
flags: anomaly-serverless
e2e:
environment: tool
runs-on:
group: larger-runners
labels: larger
permissions:
id-token: write
pull-requests: read
steps:
- name: Checkout Code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Setup environment
uses: ./.github/actions/setup-env
- name: Pre-build DQX wheel
uses: ./.github/actions/prebuild-wheel
# Required for DAB (Databricks Asset Bundle) e2e tests
- name: Install Databricks CLI
uses: databricks/setup-cli@596b0a354ba14aa59921aca1b02bd67c2b0a81a5 # v0.297.2
- name: Run e2e tests
uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
with:
vault_uri: ${{ secrets.VAULT_URI }}
timeout: 2h
create_issues: false
codegen_path: tests/e2e/.codegen.json
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
e2e_serverless:
environment: tool
runs-on:
group: larger-runners
labels: larger
permissions:
id-token: write
pull-requests: read
env:
DATABRICKS_SERVERLESS_COMPUTE_ID: auto
steps:
- name: Checkout Code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Setup environment
uses: ./.github/actions/setup-env
- name: Pre-build DQX wheel
uses: ./.github/actions/prebuild-wheel
# Required for DAB (Databricks Asset Bundle) e2e tests
- name: Install Databricks CLI
uses: databricks/setup-cli@596b0a354ba14aa59921aca1b02bd67c2b0a81a5 # v0.297.2
- name: Run e2e tests on serverless cluster
uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
with:
vault_uri: ${{ secrets.VAULT_URI }}
timeout: 2h
create_issues: false
codegen_path: tests/e2e/.codegen.json
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}
# Performance benchmarks:
# 1. run tests and generate a new perf baseline
# 2. merge it with the existing one
# 3. compare against it to detect regressions (>25% mean degradation)
# 4. if the baseline changed and new benchmarks are added or removed, open a PR in the repo
# 5. PR commit is unsigned — a maintainer must pull, sign, and push before merging
benchmark:
permissions:
id-token: write
contents: write # required for creating benchmark baseline branches
pull-requests: write # required for creating benchmark baseline PRs
environment: tool
runs-on:
group: larger-runners
labels: larger
env:
DATABRICKS_SERVERLESS_COMPUTE_ID: auto
BENCHMARKS_DIR: tests/perf/.benchmarks
FINAL_BASELINE: tests/perf/.benchmarks/baseline.json
UPDATED_BASELINE: tests/perf/.benchmarks/baseline_updated.json
NEW_BASELINE: tests/perf/.benchmarks/baseline_new.json
steps:
- name: Checkout Code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Setup environment
uses: ./.github/actions/setup-env
- name: Login to Azure for azure-cli authentication
uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
with:
client-id: ${{ secrets.ARM_CLIENT_ID }}
tenant-id: ${{ secrets.ARM_TENANT_ID }}
allow-no-subscriptions: true
- name: Generate new benchmark baseline
timeout-minutes: 120
env:
DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}
run: |
export DATABRICKS_HOST=$(az keyvault secret show --id "${{ secrets.VAULT_URI }}/secrets/DATABRICKS-HOST" --query value -o tsv)
export DATABRICKS_AUTH_TYPE=azure-cli
# We don't use the acceptance action here as it may finish before all benchmark files are written.
# Tests are run sequentially to reduce variability.
# Create baseline benchmarks under: $BENCHMARKS_DIR/<machine-specific-folder>/*_baseline.json
# Do at least 5 rounds to get more stable results.
UV_FROZEN=1 uv run --all-extras pytest tests/perf -v -n 1 \
--benchmark-storage=$BENCHMARKS_DIR \
--benchmark-save=baseline \
--benchmark-min-rounds=5
- name: Update benchmark baseline
run: |
echo "----- Benchmark Files -----"
find $BENCHMARKS_DIR
# find the benchmark subfolder dynamically
BENCHMARK_SUBDIR=$(find $BENCHMARKS_DIR -mindepth 1 -maxdepth 1 -type d | head -n1)
echo "New benchmarks located in sub-folder '$BENCHMARK_SUBDIR'"
# combine individual benchmark files into a single file for easier comparison later
mkdir -p "$(dirname "$NEW_BASELINE")"
jq -s '
.[0] * {
benchmarks: (
map(.benchmarks) | add | sort_by(.fullname)
)
}
' $BENCHMARK_SUBDIR/*_baseline.json > $NEW_BASELINE
# remove the subfolder to avoid confusion
rm -rf $BENCHMARK_SUBDIR
# If baseline.json does not exist -> just use the new baseline_new.json as baseline.
# If baseline.json exists -> merge the two with this logic:
# * If a test exists only in old baseline -> remove it (since it's missing in new run).
# * If a test exists only in new -> add it.
# * If a test exists in both -> keep the old one (no overwrite).
if [ ! -f "$FINAL_BASELINE" ]; then
echo "No baseline exists, using new baseline"
cp $NEW_BASELINE $FINAL_BASELINE
else
echo "Merging new baseline into existing baseline"
# Determine added/removed benchmarks
ADDED=$(jq -s '
(.[0].benchmarks // [] | map(.fullname)) as $old
| (.[1].benchmarks // [] | map(.fullname)) as $new
| ($new - $old)[]?
' $FINAL_BASELINE $NEW_BASELINE)
REMOVED=$(jq -s '
(.[0].benchmarks // [] | map(.fullname)) as $old
| (.[1].benchmarks // [] | map(.fullname)) as $new
| ($old - $new)[]?
' $FINAL_BASELINE $NEW_BASELINE)
# Print info about added/removed
if [ -n "$ADDED" ]; then
echo "Added benchmarks:"
echo "$ADDED"
fi
if [ -n "$REMOVED" ]; then
echo "Removed benchmarks:"
echo "$REMOVED"
fi
# Merge: prefer existing benchmarks if they exist, add new, remove missing
jq -s '
(.[0].benchmarks // [] | map({(.fullname): .}) | add) as $old
| (.[1].benchmarks // [] | map({(.fullname): .}) | add) as $new
| (.[0] + {
benchmarks:
(
($new | keys_unsorted) as $keys
| [ $keys[] |
if ($old[.] != null) then $old[.] else $new[.] end
]
)
})
' $FINAL_BASELINE $NEW_BASELINE > $UPDATED_BASELINE
# Replace the old baseline with the updated one
rm -rf $FINAL_BASELINE
cp $UPDATED_BASELINE $FINAL_BASELINE
fi
echo "----- Benchmark Files -----"
find $BENCHMARKS_DIR
echo "Updating performance benchmark report"
OUT=$(UV_FROZEN=1 uv run --all-extras python tests/perf/generate_md_report.py | grep REPORT_PATH | cut -d '=' -f2)
echo "Benchmark report updated at $OUT"
echo "BENCHMARK_REPORT=$OUT" >> $GITHUB_ENV
# For troubleshooting only
- name: Upload baseline artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: baseline-artifacts
path: |
${{ env.FINAL_BASELINE }}
${{ env.NEW_BASELINE }}
${{ env.UPDATED_BASELINE }}
- name: Create PR with updated baseline if changed
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Stage baseline and report
git add $FINAL_BASELINE || true
git add $BENCHMARK_REPORT || true
# Check if there are actual changes
if git diff --cached --quiet; then
echo "No changes to benchmark baseline or report, skipping PR creation"
else
BRANCH_NAME="update-benchmark-baseline-$(date +%Y%m%d)"
git checkout -b "$BRANCH_NAME"
# Note: this commit is NOT GPG-signed.
# The reviewer should pull, sign the commit, and push to trigger CI and satisfy branch protection.
git commit -m "Update pytest-benchmark performance baseline"
git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
git push origin "$BRANCH_NAME" --force-with-lease
# Create a PR only if one doesn't already exist for this branch
EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --state open --json number --jq '.[0].number')
if [ -z "$EXISTING_PR" ]; then
gh pr create \
--title "Update performance benchmark baseline" \
--body "$(cat <<'EOF'
## Summary
- Updated `tests/perf/.benchmarks/baseline.json` with latest nightly benchmark results
- Regenerated `docs/dqx/docs/reference/benchmarks.mdx` report
## Action required
This commit is **not GPG-signed** (created by GitHub Actions).
To merge, pull this branch, sign the commit, and push:
```bash
git fetch origin <branch-name>
git checkout <branch-name>
git commit --amend --no-edit # re-signs with your GPG key
git push --force-with-lease
```
This pull request was created automatically by the nightly performance workflow.
EOF
)" \
--base main
else
echo "PR #$EXISTING_PR already exists for $BRANCH_NAME, force-pushed updated baseline"
fi
fi
- name: Run performance tests and compare with baseline
timeout-minutes: 120
env:
DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}
run: |
export DATABRICKS_HOST=$(az keyvault secret show --id "${{ secrets.VAULT_URI }}/secrets/DATABRICKS-HOST" --query value -o tsv)
export DATABRICKS_AUTH_TYPE=azure-cli
# remove the new and updated baseline files to avoid confusion, only FINAL_BASELINE is needed for comparison
rm -rf $NEW_BASELINE
rm -rf $UPDATED_BASELINE
# We are not using acceptance action as it does not show the comparison results.
# The run fails if performance degrades by more than 25%.
# Tests are run sequentially to reduce variability.
# Do at least 5 rounds to get more stable results.
UV_FROZEN=1 uv run --all-extras pytest tests/perf -v -n 1 \
--benchmark-storage=$BENCHMARKS_DIR \
--benchmark-compare=baseline \
--benchmark-compare-fail=mean:25% \
--benchmark-min-rounds=5