yiaany · Aug 31, 2026 · Sep 1, 2026 · Sep 2, 2026 · Sep 3, 2026 · Sep 4, 2026
diff --git a/.asf.yaml b/.asf.yaml
@@ -68,6 +68,8 @@ github:
           - "cargo check datafusion-functions features"
           - "cargo check datafusion-spark features"
           - "cargo test (amd64)"
+          - "cargo test 'extended_tests' (amd64)"
+          - "cargo test hash collisions (amd64)"
           - "cargo test datafusion-cli (amd64)"
           - "cargo examples (amd64)"
           - "cargo test doc (amd64)"
@@ -76,6 +78,7 @@ github:
           - "verify benchmark results (amd64)"
           - "Run sqllogictest with Postgres runner"
           - "Run sqllogictest in Substrait round-trip mode"
+          - "Run sqllogictests with the sqlite test suite"
           - "cargo test (macos-aarch64)"
           - "Verify Vendored Code"
           - "Check cargo fmt"
@@ -84,6 +87,7 @@ github:
           - "check Cargo.toml formatting"
           - "check configs.md and ***_functions.md is up-to-date"
           - "check example README is up-to-date"
+          - "Check datafusion crates used in datafusion-examples"
           - "Verify MSRV (Min Supported Rust Version)"
     # needs to be updated as part of the release process
     # .asf.yaml doesn't support wildcard branch protection rules, only exact branch names

diff --git a/_downloads/1e336e540124589d1e27210fa4f805f6/rust_lint.sh b/_downloads/1e336e540124589d1e27210fa4f805f6/rust_lint.sh
@@ -22,7 +22,9 @@
 #
 # Note: The installed checking tools (e.g., taplo) are not guaranteed to match
 # the CI versions for simplicity, there might be some minor differences. Check
-# `.github/workflows` for the CI versions.
+# `.github/workflows` for the CI versions. When this script installs a missing
+# tool that has a pinned version in `ci/scripts/utils/tool_versions.sh`, it
+# installs that pinned version. An already installed tool is used as is.
 #
 #
 #
@@ -84,10 +86,34 @@ while [[ $# -gt 0 ]]; do
 done
 
 SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# `ci/scripts/check_asf_yaml_status_checks.py` runs with `python3` from PATH
+# and imports PyYAML. Report a missing prerequisite before any tool is
+# installed or any formatter runs, and point at `uv run`, which sets up the
+# Python dependencies from the uv workspace.
+ensure_python_with_yaml() {
+  if ! command -v python3 &> /dev/null; then
+    echo "[${SCRIPT_NAME}] python3 was not found on PATH. Please run the suite through uv, which provides Python and its packages: uv run ./dev/rust_lint.sh" >&2
+    exit 1
+  fi
+  if ! python3 -c 'import yaml' &> /dev/null; then
+    echo "[${SCRIPT_NAME}] PyYAML is not installed for $(command -v python3). Please run the suite through uv, which installs it: uv run ./dev/rust_lint.sh" >&2
+    exit 1
+  fi
+}
+
+ensure_python_with_yaml
+
+# Load the tool versions shared with CI (for example, LYCHEE_VERSION).
+source "${SCRIPT_DIR}/../ci/scripts/utils/tool_versions.sh"
 
 ensure_tool "taplo" "cargo install taplo-cli --locked"
 ensure_tool "hawkeye" "cargo install hawkeye --locked"
 ensure_tool "typos" "cargo install typos-cli --locked"
+ensure_tool "lychee" "cargo install lychee --locked --version ${LYCHEE_VERSION}"
+ensure_tool "cargo-audit" "cargo install cargo-audit --locked"
+ensure_tool "cargo-machete" "cargo install cargo-machete --locked --version ^${CARGO_MACHETE_VERSION}"
 
 run_step() {
   local name="$1"
@@ -103,11 +129,20 @@ declare -a WRITE_STEPS=(
   "ci/scripts/license_header.sh|true"
   "ci/scripts/typos_check.sh|true"
   "ci/scripts/doc_prettier_check.sh|true"
+  "ci/scripts/check_examples_docs.sh|true"
+  "ci/scripts/check_generated_docs.sh|true"
 )
 
 declare -a READONLY_STEPS=(
   "ci/scripts/check_no_cargo_install_in_workflows.sh|false"
+  "ci/scripts/check_asf_yaml_status_checks.py|false"
+  "ci/scripts/check_large_files.sh|false"
+  "ci/scripts/markdown_link_check.sh|false"
+  "ci/scripts/security_audit.sh|false"
+  "ci/scripts/check_circular_dependencies.sh|false"
+  "ci/scripts/check_unused_dependencies.sh|false"
   "ci/scripts/rust_docs.sh|false"
+  "ci/scripts/check_docs_html.sh|false"
 )
 
 for entry in "${WRITE_STEPS[@]}" "${READONLY_STEPS[@]}"; do

diff --git a/_sources/contributor-guide/api-health.md.txt b/_sources/contributor-guide/api-health.md.txt
@@ -43,6 +43,35 @@ Examples of non-breaking changes include:
 - Marking a function as deprecated (`#[deprecated]`)
 - Adding a new function to a `trait` with a default implementation
 
+### DataFusion-internal Public APIs
+
+Some internal implementation details require `pub` visibility because they are
+shared across DataFusion crates. These APIs are not intended as downstream
+extension points. They are hidden from generated documentation and are not part
+of DataFusion's supported public API, so they may be changed or removed without
+notice or a deprecation period. Examples include:
+
+1. Test helpers.
+2. Operator APIs required by the optimizer to inspect or rewrite execution plans
+   across crate boundaries.
+
+For APIs intended only for internal use, add `#[doc(hidden)]` and a doc comment
+section headed `# Public Only for Internal Use:`. Name the crate or component
+that requires access and explain why the API is not intended for downstream use.
+For example:
+
+```txt
+impl HashTableLookupExpr {
+    /// ...
+    ///
+    /// # Public Only for Internal Use:
+    /// `datafusion-proto` tests require this constructor, but it is not part of
+    /// the supported public API.
+    #[doc(hidden)]
+    pub fn new(...) {...}
+}
+```
+
 ### What is the public SQL API and what is a breaking SQL change?
 
 DataFusion is also used as a SQL engine, so changes to SQL semantics (the

diff --git a/_sources/contributor-guide/development_environment.md.txt b/_sources/contributor-guide/development_environment.md.txt
@@ -108,7 +108,7 @@ DataFusion is written in Rust and it uses a standard rust toolkit:
 
 - `rustup update stable` DataFusion generally uses the latest stable release of Rust, though it may lag when new Rust toolchains release
   - See which toolchain is currently pinned in the [`rust-toolchain.toml`](https://github.com/apache/datafusion/blob/main/rust-toolchain.toml) file
-  - This can cause issues such as not having the rust-analyzer component installed for the specified toolchain, in which case just install it manually, e.g. `rustup component add --toolchain 1.97.0 rust-analyzer`
+  - This can cause issues such as not having the rust-analyzer component installed for the specified toolchain, in which case just install it manually, e.g. `rustup component add --toolchain 1.98.1 rust-analyzer`
 - `cargo build`
 - `cargo fmt` to format the code
 - etc.
@@ -130,3 +130,17 @@ Formatting instructions:
 or run them all at once:
 
 - [dev/rust_lint.sh](../../../dev/rust_lint.sh)
+
+Debugging:
+
+The standard dev profile, used by cargo build and cargo test, disables variable-level DWARF debug information that is required by interactive debuggers.
+
+If you need to step through the code with a debugger, build with:
+
+`CARGO_PROFILE_DEV_DEBUG=2 cargo build`
+
+Alternatively, you can set `CARGO_PROFILE_DEV_DEBUG=2` as an environment variable in your IDE.
+
+This enables the additional debug information required for source-level debugging.
+
+For more details, see the DataFusion [Cargo.toml](https://github.com/apache/datafusion/blob/main/Cargo.toml#L329) configuration.
diff --git a/_sources/contributor-guide/gsoc/gsoc_project_ideas_2025.md.txt b/_sources/contributor-guide/gsoc/gsoc_project_ideas_2025.md.txt
@@ -2,7 +2,7 @@
 
 ## Introduction
 
-Welcome to the Apache DataFusion Google Summer of Code (GSoC) 2025 project ideas list. Below you can find information about the projects. Please refer to [this page](https://datafusion.apache.org/contributor-guide/gsoc_application_guidelines.html) for application guidelines.
+Welcome to the Apache DataFusion Google Summer of Code (GSoC) 2025 project ideas list. Below you can find information about the projects. Please refer to [this page](https://datafusion.apache.org/contributor-guide/gsoc/gsoc_application_guidelines_2025.html) for application guidelines.
 
 ## Projects
 

diff --git a/_sources/contributor-guide/index.md.txt b/_sources/contributor-guide/index.md.txt
@@ -122,7 +122,12 @@ before merge.
 # ./dev/rust_lint.sh --write --allow-dirty
 ```
 
-You should also run any relevant commands from the [testing quick start](testing.md#testing-quick-start).
+Please ensure your PR follows the [testing guide](testing.md). In particular:
+
+- Prefer end-to-end Public API tests such as `sqllogictest` (`.slt`) and DataFrame API,
+  over Rust unit tests where possible. See
+  [Choosing What Kind of Test to Write](testing.md#choosing-what-kind-of-test-to-write).
+- Run any relevant commands from the [testing quick start](testing.md#testing-quick-start).
 
 ## Conventional Commits & Labeling PRs
 

diff --git a/_sources/contributor-guide/pr_review.md.txt b/_sources/contributor-guide/pr_review.md.txt
@@ -133,7 +133,8 @@ Guidelines for evaluating tests:
 
 1. Prefer `sqllogictest` (`.slt`) tests or DataFrame API tests where
    possible, as they exercise **user-visible behavior** and are less coupled
-   to internal implementation details than unit tests.
+   to internal implementation details than unit tests (see
+   [Choosing What Kind of Test to Write](testing.md#choosing-what-kind-of-test-to-write)).
 2. Verify tests cover edge cases and common failure scenarios, not just the
    common successful path. However, it is NOT necessary to test every possible
    error path, especially if it is difficult to trigger or unlikely to occur in