Merge DataFusion 55.1.0 into spiceai-55

.ai/skills/pr_review/SKILL.md

@@ -0,0 +1,35 @@
+---
+name: pr_review
+description: Review Apache DataFusion pull requests following the project's PR review guide. Use whenever asked to review a DataFusion PR or PR URL, and whenever creating a PR, to check the changes against the same criteria before submitting.
+---
+
+# DataFusion PR Review
+
+This skill describes the mechanics for doing PR reviews from the command line.
+
+When creating a PR, skip the "Collect PR context" step and instead check the
+changes against each area of the
+[PR review guide](../../../docs/source/contributor-guide/pr_review.md) before
+submitting.
+
+## Collect PR context
+
+- Check out the PR locally: `gh pr checkout <PR number>` (ask first if the
+  working tree has other work in progress).
+- Fetch the PR description, comments, and reviews:
+  `gh pr view <PR number> --json title,body,comments,reviews`
+- Fetch CI status: `gh pr checks <PR number>`.
+
+## Compute the diff
+
+```bash
+# find the remote that points at apache/datafusion (e.g. `apache`, `upstream`, or `origin`)
+UPSTREAM=$(git remote -v | grep -m1 'apache/datafusion' | cut -f1)
+git fetch $UPSTREAM main
+git diff $(git merge-base HEAD $UPSTREAM/main)
+```
+
+## Review checklist
+
+Work through each area from the
+[PR review guide](../../../docs/source/contributor-guide/pr_review.md).

.asf.yaml

@@ -78,6 +78,7 @@ github:
           - "cargo test (macos-aarch64)"
           - "Verify Vendored Code"
           - "Check cargo fmt"
+          - "Check GitHub Actions install tooling"
           - "clippy"
           - "check Cargo.toml formatting"
           - "check configs.md and ***_functions.md is up-to-date"
@@ -100,6 +101,9 @@ github:
     branch-53:
       required_pull_request_reviews:
         required_approving_review_count: 1
+    branch-54:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
   pull_requests:
     # enable updating head branches of pull requests
     allow_update_branch: true
@@ -111,4 +115,3 @@ github:
 # https://datafusion.apache.org/
 publish:
   whoami: asf-site
-

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -9,7 +9,7 @@ body:
       description: Please describe what you are trying to do.
       placeholder: >
         A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-        (This section helps Arrow developers understand the context and *why* for this feature, in addition to the *what*)
+        (This section helps DataFusion developers understand the context and *why* for this feature, in addition to the *what*)
   - type: textarea
     attributes:
       label: Describe the solution you'd like

.github/actions/setup-rust-runtime/action.yaml

@@ -31,3 +31,14 @@ runs:
       run: |
         echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
         echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV
+        # Work around intermittent "[16] Error in the HTTP2 framing layer"
+        # failures from curl when cargo fetches crates from crates.io.
+        # Disabling HTTP/2 multiplexing forces cargo to serialize requests,
+        # and raising retries makes transient network hiccups self-heal.
+        #
+        # Reference:
+        # https://doc.rust-lang.org/cargo/reference/config.html?#httpmultiplexing
+        # https://doc.rust-lang.org/cargo/reference/config.html?#netretry
+        echo "CARGO_HTTP_MULTIPLEXING=false" >> $GITHUB_ENV
+        echo "CARGO_NET_RETRY=10" >> $GITHUB_ENV
+        echo "CARGO_HTTP_RETRY=10" >> $GITHUB_ENV

.github/dependabot.yml

@@ -68,6 +68,10 @@ updates:
       interval: "weekly"
     open-pull-requests-limit: 10
     labels: [auto-dependencies]
+    groups:
+      codeql-actions:
+        patterns:
+          - "github/codeql-action/*"
   - package-ecosystem: "pip"
     directory: "/docs"
     schedule:

.github/pull_request_template.md

@@ -11,12 +11,19 @@ We generally require a GitHub issue to be filed for all bug fixes and enhancemen
 <!--
  Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed.
  Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes.  
+
+Please explain the problem you are trying to solve in terms of the user-visible
+behavior, rather than the implementation.
+
+For example, "The code in `foo.rs` doesn't handle nulls" is a symptom of the
+implementation. "COUNT(DISTINCT) returns wrong results when the column contains
+nulls" is the user-visible problem.
 -->
 
 ## What changes are included in this PR?
 
 <!--
-There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR.
+There is no need to duplicate the description in the issue here, but it is sometimes worth providing a summary of the individual changes in this PR.
 -->
 
 ## Are these changes tested?
@@ -33,8 +40,6 @@ If tests are not included in your PR, please explain why (for example, are they
 
 <!--
 If there are user-facing changes then we may require documentation to be updated before approving the PR.
--->
 
-<!--
 If there are any breaking changes to public APIs, please add the `api change` label.
 -->

.github/workflows/audit.yml

@@ -43,18 +43,14 @@ jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - name: Install cargo-audit
-        uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d  # v2.79.2
+        uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad  # v2.85.10
         with:
           tool: cargo-audit
       - name: Run audit check
         # Note: you can ignore specific RUSTSEC issues using the `--ignore` flag ,for example:
         # run: cargo audit --ignore RUSTSEC-2026-0001
-        #
-        # RUSTSEC-2026-0194 and RUSTSEC-2026-0195 (quick-xml) are ignored because
-        # the fix requires quick-xml 0.41, which is only used by object_store 0.14,
-        # a breaking upgrade that can not be backported to this maintenance branch.
-        # The same advisories are ignored on main, see
+        # TODO: remove once object_store upgrades to quick-xml >= 0.41.0
         # https://github.com/apache/datafusion/issues/23297
         run: cargo audit --ignore RUSTSEC-2026-0194 --ignore RUSTSEC-2026-0195

.github/workflows/breaking_changes_detector.yml

@@ -55,7 +55,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           fetch-depth: 0
 
@@ -89,7 +89,7 @@ jobs:
 
       - name: Install cargo-semver-checks
         if: steps.changed_crates.outputs.packages != ''
-        uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d  # v2.79.2
+        uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad  # v2.85.10
         with:
           tool: cargo-semver-checks
 

.github/workflows/breaking_changes_detector_comment.yml

@@ -104,39 +104,66 @@ jobs:
             echo "${DELIM}"
           } >> "$GITHUB_OUTPUT"
 
-      # The marker `<!-- semver-check-comment -->` is what makes the comment
-      # "sticky": maintain-one-comment uses it to find and replace (or
-      # delete) the existing comment instead of stacking new ones.
+
+      # Find any existing sticky comment by its hidden marker so we can update
+      # or delete it instead of stacking new ones.
+      - name: Find existing sticky comment
+        id: find
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ steps.read.outputs.pr_number }}
+        run: |
+          COMMENT_ID=$(gh api --paginate "repos/${REPO}/issues/${PR_NUMBER}/comments" \
+            --jq '.[] | select(.body | contains("<!-- semver-check-comment -->")) | .id' \
+            | head -n1)
+          echo "comment_id=${COMMENT_ID}" >> "$GITHUB_OUTPUT"
+
+      # update the existing comment found above, or create a new one. The hidden
+      # marker `<!-- semver-check-comment -->` stays in the body so the next run
+      # finds it again. LOGS is interpolated via a shell parameter expansion,
+      # whose result bash does not re-scan, so untrusted log content cannot
+      # inject further commands.
       - name: Upsert sticky comment
         if: steps.read.outputs.result != 'success'
-        uses: actions-cool/maintain-one-comment@909842216bc8e8658364c572ec52100f4c2cc50a  # v3.3.0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          number: ${{ steps.read.outputs.pr_number }}
-          body-include: '<!-- semver-check-comment -->'
-          body: |
-            <!-- semver-check-comment -->
-            Thank you for opening this pull request!
-
-            Reviewer note: [cargo-semver-checks](https://github.com/obi1kenobi/cargo-semver-checks) reported the current version number is not SemVer-compatible with the changes in this pull request (compared against the base branch).
-
-            <details>
-            <summary>Details</summary>
-
-            ```
-            ${{ steps.read.outputs.logs }}
-            ```
-
-            </details>
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ steps.read.outputs.pr_number }}
+          COMMENT_ID: ${{ steps.find.outputs.comment_id }}
+          LOGS: ${{ steps.read.outputs.logs }}
+        run: |
+          set -euo pipefail
+          BODY="<!-- semver-check-comment -->
+          Thank you for opening this pull request!
+
+          Reviewer note: [cargo-semver-checks](https://github.com/obi1kenobi/cargo-semver-checks) reported the current version number is not SemVer-compatible with the changes in this pull request (compared against the base branch).
+
+          <details>
+          <summary>Details</summary>
+
+          \`\`\`
+          ${LOGS}
+          \`\`\`
+
+          </details>"
+
+          # Use --raw-field (not --field): always sends the value as a literal string. while --field would treat a leading `@` as a file to read
+          # (even though the body does not start with user input we are being cautious)
+          if [ -n "$COMMENT_ID" ]; then
+            gh api "repos/${REPO}/issues/comments/${COMMENT_ID}" --method PATCH --raw-field body="$BODY"
+          else
+            gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" --method POST --raw-field body="$BODY"
+          fi
 
+      # Clear a stale comment once the breaking change is resolved.
       - name: Delete sticky comment
-        if: steps.read.outputs.result == 'success'
-        uses: actions-cool/maintain-one-comment@909842216bc8e8658364c572ec52100f4c2cc50a  # v3.3.0
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          number: ${{ steps.read.outputs.pr_number }}
-          body-include: '<!-- semver-check-comment -->'
-          delete: true
+        if: steps.read.outputs.result == 'success' && steps.find.outputs.comment_id != ''
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          COMMENT_ID: ${{ steps.find.outputs.comment_id }}
+        run: gh api -X DELETE "repos/${REPO}/issues/comments/${COMMENT_ID}"
 
       - name: Add "auto detected api change" label
         if: steps.read.outputs.result != 'success'

.github/workflows/codeql.yml

@@ -40,16 +40,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       with:
         persist-credentials: false
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
+      uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
       with:
         languages: actions
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
+      uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
       with:
         category: "/language:actions"

.github/workflows/dependencies.yml

@@ -25,6 +25,7 @@ on:
   push:
     branches-ignore:
       - 'gh-readonly-queue/**'
+      - 'dependabot/**'
   pull_request:
   merge_group:
   # manual trigger
@@ -41,7 +42,7 @@ jobs:
     container:
       image: amd64/rust
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           submodules: true
           fetch-depth: 1
@@ -50,18 +51,20 @@ jobs:
         with:
           rust-version: stable
       - name: Check dependencies
+        working-directory: dev/depcheck
         run: |
-          cd dev/depcheck
-          cargo run
+          cargo run --locked
 
   detect-unused-dependencies:
     name: Detect Unused Dependencies
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - name: Install cargo-machete
-        run: cargo install cargo-machete --version ^0.9 --locked
+        uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad  # v2.85.10
+        with:
+          tool: cargo-machete@0.9
       - name: Detect unused dependencies
         run: cargo machete --with-metadata

.github/workflows/dev.yml

@@ -20,6 +20,7 @@ on:
   push:
     branches-ignore:
       - 'gh-readonly-queue/**'
+      - 'dependabot/**'
   pull_request:
   merge_group:
 
@@ -35,19 +36,20 @@ jobs:
     runs-on: ubuntu-latest
     name: Check License Header
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - name: Install HawkEye
-      # This CI job is bound by installation time, use `--profile dev` to speed it up
-        run: cargo install hawkeye --version 6.2.0 --locked --profile dev
+        uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad  # v2.85.10
+        with:
+          tool: hawkeye@6.2.0
       - name: Run license header check
         run: ci/scripts/license_header.sh
 
   prettier:
     name: Use prettier to check formatting of documents
     runs-on: ubuntu-slim
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-      - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238  # v6.2.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
         with:
           node-version: "20"
       - name: Prettier check
@@ -58,13 +60,13 @@ jobs:
     name: Check Markdown Links
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - name: Load tool versions
         run: |
           source ci/scripts/utils/tool_versions.sh
           echo "LYCHEE_VERSION=${LYCHEE_VERSION}" >> "$GITHUB_ENV"
       - name: Install lychee
-        uses: taiki-e/install-action@213ccc1a076163c093f914550b94feb90fab916d  # v2.79.2
+        uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad  # v2.85.10
         with:
           tool: lychee@${{ env.LYCHEE_VERSION }}
       - name: Run markdown link check
@@ -74,21 +76,23 @@ jobs:
     name: Validate required_status_checks in .asf.yaml
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - run: pip install pyyaml
       - run: python3 ci/scripts/check_asf_yaml_status_checks.py
 
   typos:
     name: Spell Check with Typos
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
           persist-credentials: false
       # Version fixed on purpose. It uses heuristics to detect typos, so upgrading
       # it may cause checks to fail more often.
       # We can upgrade it manually once a while.
-      - name: Install typos-cli
-        run: cargo install typos-cli --locked --version 1.37.0
+      - name: Install typos
+        uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad  # v2.85.10
+        with:
+          tool: typos@1.37.0
       - name: Run typos check
         run: ci/scripts/typos_check.sh