nightly.yml

name: nightly

on:
  workflow_dispatch:  # Allows manual triggering of the workflow
  schedule:
    - cron: '0 4 * * *'  # Runs automatically at 4:00 AM UTC every day

permissions:
  contents: read

concurrency:
  group: single-acceptance-job-per-repo
  cancel-in-progress: false  # don't cancel ongoing runs to ensure fixtures are completed and resources terminated

jobs:
  unit:
    runs-on:
      group: databrickslabs-protected-runner-group
      labels: linux-ubuntu-latest
    permissions:
      id-token: write
    steps:
      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Run unit tests and generate test coverage report
        run: make test

      - name: Publish unit test coverage
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
        with:
          use_oidc: true
          files: coverage-unit.xml
          flags: unit

  integration:
    environment: tool
    runs-on:
      group: larger-runners
      labels: larger
    permissions:
      id-token: write
      pull-requests: read
    steps:
      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Pre-build DQX wheel
        uses: ./.github/actions/prebuild-wheel

      # Integration tests are run from within tests/integration folder.
      # Create .coveragerc with correct relative path to source code.
      - name: Prepare code coverage configuration for integration tests
        run: |
          cat > tests/integration/.coveragerc << EOF
          [run]
          source = ../../src
          relative_files = true
          parallel = true
          EOF

      # Run tests from `tests/integration` as defined in .codegen.json
      # and generate code coverage for modules defined in .coveragerc
      - name: Run integration tests and generate test coverage report
        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
        with:
          vault_uri: ${{ secrets.VAULT_URI }}
          timeout: 2h
          create_issues: false  # if enabled, it creates an issue for each test failure; disabled to reduce noise
          codegen_path: tests/integration/.codegen.json
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
          COVERAGE_FILE: ${{ github.workspace }}/.coverage  # make sure the coverage report is preserved
      - name: Merge coverage reports and convert them to XML
        run: make combine-coverage

      - name: Publish integration test coverage
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
        with:
          use_oidc: true
          files: coverage-combined.xml
          flags: integration

  integration_serverless:
    environment: tool
    runs-on:
      group: larger-runners
      labels: larger
    permissions:
      id-token: write
      pull-requests: read
    env:
      DATABRICKS_SERVERLESS_COMPUTE_ID: auto
    steps:
      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Pre-build DQX wheel
        uses: ./.github/actions/prebuild-wheel

      # Integration tests are run from within tests/integration folder.
      # Create .coveragerc with correct relative path to source code.
      - name: Prepare code coverage configuration for integration tests
        run: |
          cat > tests/integration/.coveragerc << EOF
          [run]
          source = ../../src
          relative_files = true
          parallel = true
          EOF

      - name: Run integration tests on serverless cluster
        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
        with:
          vault_uri: ${{ secrets.VAULT_URI }}
          timeout: 2h
          create_issues: false
          codegen_path: tests/integration/.codegen.json
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
          DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}
          COVERAGE_FILE: ${{ github.workspace }}/.coverage  # make sure the coverage report is preserved
      - name: Merge coverage reports and convert them to XML
        run: make combine-coverage

      - name: Publish serverless integration test coverage
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
        with:
          use_oidc: true
          files: coverage-combined.xml
          flags: integration-serverless

  anomaly-tests:
    environment: tool
    runs-on:
      group: larger-runners
      labels: larger
    permissions:
      id-token: write
      pull-requests: read
    steps:
      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Pre-build DQX wheel
        uses: ./.github/actions/prebuild-wheel

      # Create .coveragerc with correct relative path to source code.
      - name: Prepare code coverage configuration for anomaly tests
        run: |
          cat > tests/integration_anomaly/.coveragerc << EOF
          [run]
          source = ../../src
          relative_files = true
          parallel = true
          EOF

      - name: Run anomaly integration tests
        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
        with:
          vault_uri: ${{ secrets.VAULT_URI }}
          timeout: 3h
          create_issues: false
          codegen_path: tests/integration_anomaly/.codegen.json
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
          COVERAGE_FILE: ${{ github.workspace }}/.coverage
          MLFLOW_ENABLE_DB_SDK: "true"
          MLFLOW_TRACKING_URI: databricks
          MLFLOW_REGISTRY_URI: databricks-uc
          MLFLOW_HTTP_REQUEST_TIMEOUT: "600"
          MLFLOW_HTTP_REQUEST_MAX_RETRIES: "9"
      - name: Merge coverage reports and convert them to XML
        run: make combine-coverage

      - name: Publish anomaly test coverage
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
        with:
          use_oidc: true
          files: coverage-combined.xml
          flags: anomaly

  anomaly-tests-serverless:
    environment: tool
    runs-on:
      group: larger-runners
      labels: larger
    permissions:
      id-token: write
      pull-requests: read
    steps:
      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Pre-build DQX wheel
        uses: ./.github/actions/prebuild-wheel

      # Create .coveragerc with correct relative path to source code.
      - name: Prepare code coverage configuration for anomaly tests
        run: |
          cat > tests/integration_anomaly/.coveragerc << EOF
          [run]
          source = ../../src
          relative_files = true
          parallel = true
          EOF

      - name: Run anomaly integration tests on serverless cluster
        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
        with:
          vault_uri: ${{ secrets.VAULT_URI }}
          timeout: 3h
          create_issues: false
          codegen_path: tests/integration_anomaly/.codegen.json
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
          COVERAGE_FILE: ${{ github.workspace }}/.coverage
          DATABRICKS_SERVERLESS_COMPUTE_ID: auto
          MLFLOW_ENABLE_DB_SDK: "true"
          MLFLOW_TRACKING_URI: databricks
          MLFLOW_REGISTRY_URI: databricks-uc
          MLFLOW_HTTP_REQUEST_TIMEOUT: "600"
          MLFLOW_HTTP_REQUEST_MAX_RETRIES: "9"
      - name: Merge coverage reports and convert them to XML
        run: make combine-coverage

      - name: Publish serverless anomaly test coverage
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
        with:
          use_oidc: true
          files: coverage-combined.xml
          flags: anomaly-serverless

  e2e:
    environment: tool
    runs-on:
      group: larger-runners
      labels: larger
    permissions:
      id-token: write
      pull-requests: read
    steps:
      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Pre-build DQX wheel
        uses: ./.github/actions/prebuild-wheel

      # Required for DAB (Databricks Asset Bundle) e2e tests
      - name: Install Databricks CLI
        uses: databricks/setup-cli@596b0a354ba14aa59921aca1b02bd67c2b0a81a5 # v0.297.2

      - name: Run e2e tests
        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
        with:
          vault_uri: ${{ secrets.VAULT_URI }}
          timeout: 2h
          create_issues: false
          codegen_path: tests/e2e/.codegen.json
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}

  e2e_serverless:
    environment: tool
    runs-on:
      group: larger-runners
      labels: larger
    permissions:
      id-token: write
      pull-requests: read
    env:
      DATABRICKS_SERVERLESS_COMPUTE_ID: auto
    steps:
      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Pre-build DQX wheel
        uses: ./.github/actions/prebuild-wheel

      # Required for DAB (Databricks Asset Bundle) e2e tests
      - name: Install Databricks CLI
        uses: databricks/setup-cli@596b0a354ba14aa59921aca1b02bd67c2b0a81a5 # v0.297.2

      - name: Run e2e tests on serverless cluster
        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
        with:
          vault_uri: ${{ secrets.VAULT_URI }}
          timeout: 2h
          create_issues: false
          codegen_path: tests/e2e/.codegen.json
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
          DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}

  # Performance benchmarks:
  # 1. run tests and generate a new perf baseline
  # 2. merge it with the existing one
  # 3. compare against it to detect regressions (>25% mean degradation)
  # 4. if the baseline changed and new benchmarks are added or removed, open a PR in the repo
  # 5. PR commit is unsigned — a maintainer must pull, sign, and push before merging
  benchmark:
    permissions:
      id-token: write
      contents: write # required for creating benchmark baseline branches
      pull-requests: write # required for creating benchmark baseline PRs
    environment: tool
    runs-on:
      group: larger-runners
      labels: larger
    env:
      DATABRICKS_SERVERLESS_COMPUTE_ID: auto
      BENCHMARKS_DIR: tests/perf/.benchmarks
      FINAL_BASELINE: tests/perf/.benchmarks/baseline.json
      UPDATED_BASELINE: tests/perf/.benchmarks/baseline_updated.json
      NEW_BASELINE: tests/perf/.benchmarks/baseline_new.json
    steps:
      - name: Checkout Code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Setup environment
        uses: ./.github/actions/setup-env

      - name: Login to Azure for azure-cli authentication
        uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
        with:
          client-id: ${{ secrets.ARM_CLIENT_ID }}
          tenant-id: ${{ secrets.ARM_TENANT_ID }}
          allow-no-subscriptions: true

      - name: Generate new benchmark baseline
        timeout-minutes: 120
        env:
          DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}
        run: |
          export DATABRICKS_HOST=$(az keyvault secret show --id "${{ secrets.VAULT_URI }}/secrets/DATABRICKS-HOST" --query value -o tsv)
          export DATABRICKS_AUTH_TYPE=azure-cli
          # We don't use the acceptance action here as it may finish before all benchmark files are written.
          # Tests are run sequentially to reduce variability.
          # Create baseline benchmarks under: $BENCHMARKS_DIR/<machine-specific-folder>/*_baseline.json
          # Do at least 5 rounds to get more stable results.
          UV_FROZEN=1 uv run --all-extras pytest tests/perf -v -n 1 \
            --benchmark-storage=$BENCHMARKS_DIR \
            --benchmark-save=baseline \
            --benchmark-min-rounds=5

      - name: Update benchmark baseline
        run: |
          echo "----- Benchmark Files -----"
          find $BENCHMARKS_DIR

          # find the benchmark subfolder dynamically
          BENCHMARK_SUBDIR=$(find $BENCHMARKS_DIR -mindepth 1 -maxdepth 1 -type d | head -n1)
          echo "New benchmarks located in sub-folder '$BENCHMARK_SUBDIR'"

          # combine individual benchmark files into a single file for easier comparison later
          mkdir -p "$(dirname "$NEW_BASELINE")"
          jq -s '
            .[0] * {
              benchmarks: (
                map(.benchmarks) | add | sort_by(.fullname)
              )
            }
          ' $BENCHMARK_SUBDIR/*_baseline.json > $NEW_BASELINE

          # remove the subfolder to avoid confusion
          rm -rf $BENCHMARK_SUBDIR

          # If baseline.json does not exist -> just use the new baseline_new.json as baseline.
          # If baseline.json exists -> merge the two with this logic:
          # * If a test exists only in old baseline -> remove it (since it's missing in new run).
          # * If a test exists only in new -> add it.
          # * If a test exists in both -> keep the old one (no overwrite).
          if [ ! -f "$FINAL_BASELINE" ]; then
              echo "No baseline exists, using new baseline"
              cp $NEW_BASELINE $FINAL_BASELINE
          else
              echo "Merging new baseline into existing baseline"

              # Determine added/removed benchmarks
              ADDED=$(jq -s '
                (.[0].benchmarks // [] | map(.fullname)) as $old
                | (.[1].benchmarks // [] | map(.fullname)) as $new
                | ($new - $old)[]?
              ' $FINAL_BASELINE $NEW_BASELINE)

              REMOVED=$(jq -s '
                (.[0].benchmarks // [] | map(.fullname)) as $old
                | (.[1].benchmarks // [] | map(.fullname)) as $new
                | ($old - $new)[]?
              ' $FINAL_BASELINE $NEW_BASELINE)

              # Print info about added/removed
              if [ -n "$ADDED" ]; then
                echo "Added benchmarks:"
                echo "$ADDED"
              fi

              if [ -n "$REMOVED" ]; then
                echo "Removed benchmarks:"
                echo "$REMOVED"
              fi

              # Merge: prefer existing benchmarks if they exist, add new, remove missing
              jq -s '
                (.[0].benchmarks // [] | map({(.fullname): .}) | add) as $old
                | (.[1].benchmarks // [] | map({(.fullname): .}) | add) as $new
                | (.[0] + {
                    benchmarks:
                      (
                        ($new | keys_unsorted) as $keys
                        | [ $keys[] |
                            if ($old[.] != null) then $old[.] else $new[.] end
                          ]
                      )
                  })
              ' $FINAL_BASELINE $NEW_BASELINE > $UPDATED_BASELINE

              # Replace the old baseline with the updated one
              rm -rf $FINAL_BASELINE
              cp $UPDATED_BASELINE $FINAL_BASELINE
          fi

          echo "----- Benchmark Files -----"
          find $BENCHMARKS_DIR

          echo "Updating performance benchmark report"
          OUT=$(UV_FROZEN=1 uv run --all-extras python tests/perf/generate_md_report.py | grep REPORT_PATH | cut -d '=' -f2)
          echo "Benchmark report updated at $OUT"
          echo "BENCHMARK_REPORT=$OUT" >> $GITHUB_ENV

      # For troubleshooting only
      - name: Upload baseline artifacts
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: baseline-artifacts
          path: |
            ${{ env.FINAL_BASELINE }}
            ${{ env.NEW_BASELINE }}
            ${{ env.UPDATED_BASELINE }}

      - name: Create PR with updated baseline if changed
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          # Stage baseline and report
          git add $FINAL_BASELINE || true
          git add $BENCHMARK_REPORT || true

          # Check if there are actual changes
          if git diff --cached --quiet; then
            echo "No changes to benchmark baseline or report, skipping PR creation"
          else
            BRANCH_NAME="update-benchmark-baseline-$(date +%Y%m%d)"

            git checkout -b "$BRANCH_NAME"

            # Note: this commit is NOT GPG-signed.
            # The reviewer should pull, sign the commit, and push to trigger CI and satisfy branch protection.
            git commit -m "Update pytest-benchmark performance baseline"

            git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
            git push origin "$BRANCH_NAME" --force-with-lease

            # Create a PR only if one doesn't already exist for this branch
            EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --state open --json number --jq '.[0].number')
            if [ -z "$EXISTING_PR" ]; then
              gh pr create \
                --title "Update performance benchmark baseline" \
                --body "$(cat <<'EOF'
          ## Summary
          - Updated `tests/perf/.benchmarks/baseline.json` with latest nightly benchmark results
          - Regenerated `docs/dqx/docs/reference/benchmarks.mdx` report

          ## Action required
          This commit is **not GPG-signed** (created by GitHub Actions).
          To merge, pull this branch, sign the commit, and push:
          ```bash
          git fetch origin <branch-name>
          git checkout <branch-name>
          git commit --amend --no-edit  # re-signs with your GPG key
          git push --force-with-lease
          ```

          This pull request was created automatically by the nightly performance workflow.
          EOF
              )" \
                --base main
            else
              echo "PR #$EXISTING_PR already exists for $BRANCH_NAME, force-pushed updated baseline"
            fi
          fi

      - name: Run performance tests and compare with baseline
        timeout-minutes: 120
        env:
          DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}
        run: |
          export DATABRICKS_HOST=$(az keyvault secret show --id "${{ secrets.VAULT_URI }}/secrets/DATABRICKS-HOST" --query value -o tsv)
          export DATABRICKS_AUTH_TYPE=azure-cli

          # remove the new and updated baseline files to avoid confusion, only FINAL_BASELINE is needed for comparison
          rm -rf $NEW_BASELINE
          rm -rf $UPDATED_BASELINE

          # We are not using acceptance action as it does not show the comparison results.
          # The run fails if performance degrades by more than 25%.
          # Tests are run sequentially to reduce variability.
          # Do at least 5 rounds to get more stable results.
          UV_FROZEN=1 uv run --all-extras pytest tests/perf -v -n 1 \
            --benchmark-storage=$BENCHMARKS_DIR \
            --benchmark-compare=baseline \
            --benchmark-compare-fail=mean:25% \
            --benchmark-min-rounds=5