Merge remote-tracking branch 'upstream/main' into concat_batches_for_sort

Author: zhuqi-lucas

.github/workflows/extended.yml

@@ -101,7 +101,17 @@ jobs:
       - name: Run tests (excluding doctests)
         env:
           RUST_BACKTRACE: 1
-        run: cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,extended_tests,recursive_protection
+        run: |
+          cargo test \
+            --profile ci \
+            --exclude datafusion-examples \
+            --exclude datafusion-benchmarks \
+            --exclude datafusion-cli \
+            --workspace \
+            --lib \
+            --tests \
+            --bins \
+            --features avro,json,backtrace,extended_tests,recursive_protection
       - name: Verify Working Directory Clean
         run: git diff --exit-code
       - name: Cleanup
@@ -126,7 +136,7 @@ jobs:
       - name: Run tests
         run: |
           cd datafusion
-          cargo test  --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --features=force_hash_collisions,avro
+          cargo test  --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --exclude datafusion-cli --workspace --lib --tests --features=force_hash_collisions,avro
           cargo clean
 
   sqllogictest-sqlite:

.github/workflows/dev_pr.yml renamed to .github/workflows/labeler.yml

@@ -49,7 +49,7 @@ jobs:
         uses: actions/[email protected]
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-          configuration-path: .github/workflows/dev_pr/labeler.yml
+          configuration-path: .github/workflows/labeler/labeler-config.yml
           sync-labels: true
 
       # TODO: Enable this when eps1lon/actions-label-merge-conflict is available.

.github/workflows/dev_pr/labeler.yml renamed to .github/workflows/labeler/labeler-config.yml

@@ -41,7 +41,7 @@ physical-expr:
 
 physical-plan:
   - changed-files:
-      - any-glob-to-any-file: [datafusion/physical-plan/**/*']
+      - any-glob-to-any-file: ['datafusion/physical-plan/**/*']
 
 
 catalog:
@@ -77,6 +77,10 @@ proto:
   - changed-files:
       - any-glob-to-any-file: ['datafusion/proto/**/*', 'datafusion/proto-common/**/*']
 
+spark:
+- changed-files:
+  - any-glob-to-any-file: ['datafusion/spark/**/*']
+
 substrait:
 - changed-files:
   - any-glob-to-any-file: ['datafusion/substrait/**/*']

.github/workflows/rust.yml

@@ -39,14 +39,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  # Check license header
-  license-header-check:
-    runs-on: ubuntu-latest
-    name: Check License Header
-    steps:
-      - uses: actions/checkout@v4
-      - uses: korandoru/hawkeye@v6
-
   # Check crate compiles and base cargo check passes
   linux-build-lib:
     name: linux build test
@@ -401,8 +393,8 @@ jobs:
       - name: Run tests with headless mode
         working-directory: ./datafusion/wasmtest
         run: |
-          wasm-pack test --headless --firefox
-          wasm-pack test --headless --chrome --chromedriver $CHROMEWEBDRIVER/chromedriver
+          RUSTFLAGS='--cfg getrandom_backend="wasm_js"' wasm-pack test --headless --firefox
+          RUSTFLAGS='--cfg getrandom_backend="wasm_js"' wasm-pack test --headless --chrome --chromedriver $CHROMEWEBDRIVER/chromedriver
 
   # verify that the benchmark queries return the correct results
   verify-benchmark-results:
@@ -476,6 +468,28 @@ jobs:
           POSTGRES_HOST: postgres
           POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }}
 
+  sqllogictest-substrait:
+    name: "Run sqllogictest in Substrait round-trip mode"
+    needs: linux-build-lib
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Run sqllogictest
+        # TODO: Right now several tests are failing in Substrait round-trip mode, so this
+        #  command cannot be run for all the .slt files. Run it for just one that works (limit.slt)
+        #  until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed
+        #  and this command can be run without filters.
+        run: cargo test --test sqllogictests -- --substrait-round-trip limit.slt
+
   #  Temporarily commenting out the Windows flow, the reason is enormously slow running build
   #  Waiting for new Windows 2025 github runner
   #  Details: https://github.com/apache/datafusion/issues/13726