diff --git a/.asf.yaml b/.asf.yaml index 34d036e19b409..1307c8759bc32 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -68,6 +68,8 @@ github: - "cargo check datafusion-functions features" - "cargo check datafusion-spark features" - "cargo test (amd64)" + - "cargo test 'extended_tests' (amd64)" + - "cargo test hash collisions (amd64)" - "cargo test datafusion-cli (amd64)" - "cargo examples (amd64)" - "cargo test doc (amd64)" @@ -76,6 +78,7 @@ github: - "verify benchmark results (amd64)" - "Run sqllogictest with Postgres runner" - "Run sqllogictest in Substrait round-trip mode" + - "Run sqllogictests with the sqlite test suite" - "cargo test (macos-aarch64)" - "Verify Vendored Code" - "Check cargo fmt" @@ -84,6 +87,7 @@ github: - "check Cargo.toml formatting" - "check configs.md and ***_functions.md is up-to-date" - "check example README is up-to-date" + - "Check datafusion crates used in datafusion-examples" - "Verify MSRV (Min Supported Rust Version)" # needs to be updated as part of the release process # .asf.yaml doesn't support wildcard branch protection rules, only exact branch names diff --git a/_downloads/1e336e540124589d1e27210fa4f805f6/rust_lint.sh b/_downloads/1e336e540124589d1e27210fa4f805f6/rust_lint.sh index 73cab9c7f70bd..dbb64543028de 100644 --- a/_downloads/1e336e540124589d1e27210fa4f805f6/rust_lint.sh +++ b/_downloads/1e336e540124589d1e27210fa4f805f6/rust_lint.sh @@ -22,7 +22,9 @@ # # Note: The installed checking tools (e.g., taplo) are not guaranteed to match # the CI versions for simplicity, there might be some minor differences. Check -# `.github/workflows` for the CI versions. +# `.github/workflows` for the CI versions. When this script installs a missing +# tool that has a pinned version in `ci/scripts/utils/tool_versions.sh`, it +# installs that pinned version. An already installed tool is used as is. # # # @@ -84,10 +86,34 @@ while [[ $# -gt 0 ]]; do done SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# `ci/scripts/check_asf_yaml_status_checks.py` runs with `python3` from PATH +# and imports PyYAML. Report a missing prerequisite before any tool is +# installed or any formatter runs, and point at `uv run`, which sets up the +# Python dependencies from the uv workspace. +ensure_python_with_yaml() { + if ! command -v python3 &> /dev/null; then + echo "[${SCRIPT_NAME}] python3 was not found on PATH. Please run the suite through uv, which provides Python and its packages: uv run ./dev/rust_lint.sh" >&2 + exit 1 + fi + if ! python3 -c 'import yaml' &> /dev/null; then + echo "[${SCRIPT_NAME}] PyYAML is not installed for $(command -v python3). Please run the suite through uv, which installs it: uv run ./dev/rust_lint.sh" >&2 + exit 1 + fi +} + +ensure_python_with_yaml + +# Load the tool versions shared with CI (for example, LYCHEE_VERSION). +source "${SCRIPT_DIR}/../ci/scripts/utils/tool_versions.sh" ensure_tool "taplo" "cargo install taplo-cli --locked" ensure_tool "hawkeye" "cargo install hawkeye --locked" ensure_tool "typos" "cargo install typos-cli --locked" +ensure_tool "lychee" "cargo install lychee --locked --version ${LYCHEE_VERSION}" +ensure_tool "cargo-audit" "cargo install cargo-audit --locked" +ensure_tool "cargo-machete" "cargo install cargo-machete --locked --version ^${CARGO_MACHETE_VERSION}" run_step() { local name="$1" @@ -103,11 +129,20 @@ declare -a WRITE_STEPS=( "ci/scripts/license_header.sh|true" "ci/scripts/typos_check.sh|true" "ci/scripts/doc_prettier_check.sh|true" + "ci/scripts/check_examples_docs.sh|true" + "ci/scripts/check_generated_docs.sh|true" ) declare -a READONLY_STEPS=( "ci/scripts/check_no_cargo_install_in_workflows.sh|false" + "ci/scripts/check_asf_yaml_status_checks.py|false" + "ci/scripts/check_large_files.sh|false" + "ci/scripts/markdown_link_check.sh|false" + "ci/scripts/security_audit.sh|false" + "ci/scripts/check_circular_dependencies.sh|false" + "ci/scripts/check_unused_dependencies.sh|false" "ci/scripts/rust_docs.sh|false" + "ci/scripts/check_docs_html.sh|false" ) for entry in "${WRITE_STEPS[@]}" "${READONLY_STEPS[@]}"; do diff --git a/_sources/contributor-guide/api-health.md.txt b/_sources/contributor-guide/api-health.md.txt index a20bc284cf362..8737df280f4f8 100644 --- a/_sources/contributor-guide/api-health.md.txt +++ b/_sources/contributor-guide/api-health.md.txt @@ -43,6 +43,35 @@ Examples of non-breaking changes include: - Marking a function as deprecated (`#[deprecated]`) - Adding a new function to a `trait` with a default implementation +### DataFusion-internal Public APIs + +Some internal implementation details require `pub` visibility because they are +shared across DataFusion crates. These APIs are not intended as downstream +extension points. They are hidden from generated documentation and are not part +of DataFusion's supported public API, so they may be changed or removed without +notice or a deprecation period. Examples include: + +1. Test helpers. +2. Operator APIs required by the optimizer to inspect or rewrite execution plans + across crate boundaries. + +For APIs intended only for internal use, add `#[doc(hidden)]` and a doc comment +section headed `# Public Only for Internal Use:`. Name the crate or component +that requires access and explain why the API is not intended for downstream use. +For example: + +```txt +impl HashTableLookupExpr { + /// ... + /// + /// # Public Only for Internal Use: + /// `datafusion-proto` tests require this constructor, but it is not part of + /// the supported public API. + #[doc(hidden)] + pub fn new(...) {...} +} +``` + ### What is the public SQL API and what is a breaking SQL change? DataFusion is also used as a SQL engine, so changes to SQL semantics (the diff --git a/_sources/contributor-guide/development_environment.md.txt b/_sources/contributor-guide/development_environment.md.txt index 2e4e00726580b..20cfd6edd0cec 100644 --- a/_sources/contributor-guide/development_environment.md.txt +++ b/_sources/contributor-guide/development_environment.md.txt @@ -108,7 +108,7 @@ DataFusion is written in Rust and it uses a standard rust toolkit: - `rustup update stable` DataFusion generally uses the latest stable release of Rust, though it may lag when new Rust toolchains release - See which toolchain is currently pinned in the [`rust-toolchain.toml`](https://github.com/apache/datafusion/blob/main/rust-toolchain.toml) file - - This can cause issues such as not having the rust-analyzer component installed for the specified toolchain, in which case just install it manually, e.g. `rustup component add --toolchain 1.97.0 rust-analyzer` + - This can cause issues such as not having the rust-analyzer component installed for the specified toolchain, in which case just install it manually, e.g. `rustup component add --toolchain 1.98.1 rust-analyzer` - `cargo build` - `cargo fmt` to format the code - etc. @@ -130,3 +130,17 @@ Formatting instructions: or run them all at once: - [dev/rust_lint.sh](../../../dev/rust_lint.sh) + +Debugging: + +The standard dev profile, used by cargo build and cargo test, disables variable-level DWARF debug information that is required by interactive debuggers. + +If you need to step through the code with a debugger, build with: + +`CARGO_PROFILE_DEV_DEBUG=2 cargo build` + +Alternatively, you can set `CARGO_PROFILE_DEV_DEBUG=2` as an environment variable in your IDE. + +This enables the additional debug information required for source-level debugging. + +For more details, see the DataFusion [Cargo.toml](https://github.com/apache/datafusion/blob/main/Cargo.toml#L329) configuration. diff --git a/_sources/contributor-guide/gsoc/gsoc_project_ideas_2025.md.txt b/_sources/contributor-guide/gsoc/gsoc_project_ideas_2025.md.txt index d81d9eb9adab5..5366080ffe051 100644 --- a/_sources/contributor-guide/gsoc/gsoc_project_ideas_2025.md.txt +++ b/_sources/contributor-guide/gsoc/gsoc_project_ideas_2025.md.txt @@ -2,7 +2,7 @@ ## Introduction -Welcome to the Apache DataFusion Google Summer of Code (GSoC) 2025 project ideas list. Below you can find information about the projects. Please refer to [this page](https://datafusion.apache.org/contributor-guide/gsoc_application_guidelines.html) for application guidelines. +Welcome to the Apache DataFusion Google Summer of Code (GSoC) 2025 project ideas list. Below you can find information about the projects. Please refer to [this page](https://datafusion.apache.org/contributor-guide/gsoc/gsoc_application_guidelines_2025.html) for application guidelines. ## Projects diff --git a/_sources/contributor-guide/index.md.txt b/_sources/contributor-guide/index.md.txt index 6f1a0f1c19907..d6a16680ddb79 100644 --- a/_sources/contributor-guide/index.md.txt +++ b/_sources/contributor-guide/index.md.txt @@ -122,7 +122,12 @@ before merge. # ./dev/rust_lint.sh --write --allow-dirty ``` -You should also run any relevant commands from the [testing quick start](testing.md#testing-quick-start). +Please ensure your PR follows the [testing guide](testing.md). In particular: + +- Prefer end-to-end Public API tests such as `sqllogictest` (`.slt`) and DataFrame API, + over Rust unit tests where possible. See + [Choosing What Kind of Test to Write](testing.md#choosing-what-kind-of-test-to-write). +- Run any relevant commands from the [testing quick start](testing.md#testing-quick-start). ## Conventional Commits & Labeling PRs diff --git a/_sources/contributor-guide/pr_review.md.txt b/_sources/contributor-guide/pr_review.md.txt index 1154ae0e96b04..6166a741aa0a6 100644 --- a/_sources/contributor-guide/pr_review.md.txt +++ b/_sources/contributor-guide/pr_review.md.txt @@ -133,7 +133,8 @@ Guidelines for evaluating tests: 1. Prefer `sqllogictest` (`.slt`) tests or DataFrame API tests where possible, as they exercise **user-visible behavior** and are less coupled - to internal implementation details than unit tests. + to internal implementation details than unit tests (see + [Choosing What Kind of Test to Write](testing.md#choosing-what-kind-of-test-to-write)). 2. Verify tests cover edge cases and common failure scenarios, not just the common successful path. However, it is NOT necessary to test every possible error path, especially if it is difficult to trigger or unlikely to occur in diff --git a/_sources/contributor-guide/testing.md.txt b/_sources/contributor-guide/testing.md.txt index 3b644f610b90e..3e39ef2eb1af7 100644 --- a/_sources/contributor-guide/testing.md.txt +++ b/_sources/contributor-guide/testing.md.txt @@ -74,6 +74,30 @@ cargo nextest run [the book]: https://doc.rust-lang.org/book/ [cargo-nextest]: https://nexte.st/ +## Choosing What Kind of Test to Write + +When adding tests, prefer tests that exercise **user-visible behavior** over +tests of internal implementation details. Doing so means your tests are more +likely to keep working (and keep catching regressions) while the code is +refactored. In rough order of preference: + +1. Use [sqllogictest](#sqllogictests-tests) (`.slt`) tests for any behavior that can + be expressed in SQL. + +2. End-to-end tests that use public APIs such as the DataFrame API, for behavior + that cannot be expressed in SQL. For example, DataFrame tests live in + [`datafusion/core/tests/dataframe`] and run as part of the `core_integration` + test binary: + + ```shell + cargo test -p datafusion --test core_integration -- dataframe + ``` + +3. Rust unit tests for logic that cannot practically be reached through + SQL or DataFrame public APIs. + +[`datafusion/core/tests/dataframe`]: https://github.com/apache/datafusion/tree/main/datafusion/core/tests/dataframe + ## Unit tests Tests for code in an individual module are defined in the same source file with a `test` module, following Rust convention. @@ -111,7 +135,7 @@ cargo test --profile=ci --test sqllogictests -- --complete Like similar systems such as [DuckDB](https://duckdb.org/dev/testing), DataFusion has chosen to trade off a slightly higher barrier to contribution for longer term maintainability. -DataFusion has integrated [sqlite's test suite](https://sqlite.org/sqllogictest/doc/trunk/about.wiki) as a supplemental test suite that is run whenever a PR is merged into DataFusion. To run it manually please refer to the [README](https://github.com/apache/datafusion/blob/main/datafusion/sqllogictest/README.md#running-tests-sqlite) file for instructions. +DataFusion runs [sqlite's test suite](https://sqlite.org/sqllogictest/doc/trunk/about.wiki) in the merge queue before merging PRs into `main`. For local instructions, see [Running Tests: sqlite](https://github.com/apache/datafusion/blob/main/datafusion/sqllogictest/README.md#running-tests-sqlite). ## Snapshot testing (`cargo insta`) @@ -127,15 +151,15 @@ cargo insta review ## Extended Tests -In addition to the standard CI test suite that is run on all PRs prior to merge, -DataFusion has "extended" tests (defined in [extended.yml]) that are run on each -commit to `main`. These tests rarely fail but take significantly longer to run -than the standard test suite and add important test coverage such as ensuring -correctness when there are hash collisions and running the relevant portions of -the entire [sqlite test suite]. You can run the extended tests -locally by following the [instructions in the documentation]. +DataFusion has extended tests (defined in [extended.yml]) that take significantly +longer to run than the standard suite. They provide additional correctness +coverage and must pass in the merge queue before a PR merges into `main`. + +To conserve CI resources, these tests do not run on ordinary PR updates. They +also run on pushes to release branches (`branch-*`). You can run them manually. + +For local SQLite test instructions, see the [instructions in the documentation]. -[sqlite test suite]: https://www.sqlite.org/sqllogictest/dir?ci=tip [instructions in the documentation]: https://github.com/apache/datafusion/tree/main/datafusion/sqllogictest#running-tests-sqlite [extended.yml]: https://github.com/apache/datafusion/blob/main/.github/workflows/extended.yml @@ -164,7 +188,7 @@ contributions. Kudos to [@2010YOUY01] for the initial implementation. ## Documentation Examples We use Rust [doctest] to verify examples from the documentation are correct and -up-to-date. These tests are run as part of our CI and you can run them them +up-to-date. These tests are run as part of our CI and you can run them locally with the following command: ```shell @@ -176,6 +200,20 @@ cargo test --doc As with other Rust projects, examples in doc comments in `.rs` files are automatically checked to ensure they work and evolve along with the code. +Prefix a line with `# ` (a `#` followed by a space) to keep it compiled and +run, but hidden from the rendered documentation. This is most useful for +`use` statements, since readers care about the API being demonstrated, not +its import boilerplate: + +````rust +/// ``` +/// # use std::sync::Arc; +/// let value = Arc::new(42); +/// ``` +```` + +Only the `let value = ...` line renders. + ### User Guide Documentation Rust example code from the user guide (anything marked with \`\`\`rust) is also @@ -188,7 +226,12 @@ tested in the same way using the [doc_comment] crate. See the end of ## Documentation Link Checks -Run the internal markdown link check locally: +`./dev/rust_lint.sh` runs the internal markdown link check. If `lychee` is +missing, the script installs the version pinned in +`ci/scripts/utils/tool_versions.sh`. It uses an existing installation as is, +even if the version differs from the pin. + +To run the check on its own: ```shell source ci/scripts/utils/tool_versions.sh @@ -200,6 +243,7 @@ Notes: - The script is run with `bash` and is compatible with the default Bash on macOS (no `mapfile` dependency). - The CI configuration currently checks internal markdown links only. External `http(s)` and `mailto` links are excluded to avoid flaky failures. +- The check only reports broken links. `./dev/rust_lint.sh --write` does not change them. When a link is broken, lychee prints the file and URL/path that failed. For example: @@ -214,6 +258,119 @@ Rust doc comments are validated by rustdoc in CI and can be checked locally with bash ci/scripts/rust_docs.sh ``` +## ASF Status Check Validation + +`ci/scripts/check_asf_yaml_status_checks.py` checks that every required status +check in `.asf.yaml` matches a job in `.github/workflows`, and that `rust.yml` +skips only its listed jobs on pushes to `main`. `./dev/rust_lint.sh` runs it +and needs `python3` with [PyYAML]. The [uv] workspace provides both: + +```shell +uv run ./dev/rust_lint.sh +``` + +To run the check on its own: + +```shell +uv run python3 ci/scripts/check_asf_yaml_status_checks.py +``` + +[pyyaml]: https://pypi.org/project/PyYAML/ +[uv]: https://docs.astral.sh/uv/ + +## Security Audit + +`ci/scripts/security_audit.sh` runs `cargo audit` on the root `Cargo.lock` with +the advisory exceptions that CI uses. `./dev/rust_lint.sh` runs it and installs +[cargo-audit] if it is missing. To run the audit on its own: + +```shell +./ci/scripts/security_audit.sh +``` + +The audit fetches the RustSec advisory database. A new advisory or a different +`cargo-audit` version can change the result without any change to the repository. + +[cargo-audit]: https://github.com/rustsec/rustsec/blob/main/cargo-audit/README.md + +## Large File Check + +`ci/scripts/check_large_files.sh` fails if any file committed between a base +ref and a head ref is larger than 1.5 MB, the same check the "Large files PR +check" workflow runs on pull requests. `./dev/rust_lint.sh` runs it against +the merge base of `HEAD` and `main` on the remote that points at +`apache/datafusion`, or on `origin` when there is no such remote. To run the +check on its own, or against a different range: + +```shell +./ci/scripts/check_large_files.sh +./ci/scripts/check_large_files.sh --base upstream/main --head my-branch +``` + +Only committed files are checked. Commit a change before running it. + +## Dependency Checks + +CI runs two dependency checks, and `./dev/rust_lint.sh` runs both: + +- `ci/scripts/check_circular_dependencies.sh` builds and runs [`dev/depcheck`], + which fails on dependency cycles between DataFusion crates. +- `ci/scripts/check_unused_dependencies.sh` runs `cargo machete --with-metadata` + from the repository root. The lint suite installs [cargo-machete] with the + version in `ci/scripts/utils/tool_versions.sh` if it is missing. + +To run either check on its own: + +```shell +./ci/scripts/check_circular_dependencies.sh +./ci/scripts/check_unused_dependencies.sh +``` + +[`dev/depcheck`]: https://github.com/apache/datafusion/tree/main/dev/depcheck +[cargo-machete]: https://github.com/bnjbvr/cargo-machete + +## Examples README Check + +`datafusion-examples/README.md` is generated from the documentation comments in +`datafusion-examples/examples//main.rs`. `ci/scripts/check_examples_docs.sh` +regenerates it and fails if the committed file differs. `./dev/rust_lint.sh` +runs it and needs `cargo` and `npx`. To run the check on its own, or to update +the README: + +```shell +./ci/scripts/check_examples_docs.sh +./ci/scripts/check_examples_docs.sh --write +``` + +## Config and Function Docs Check + +[`configs.md`], [`aggregate_functions.md`], [`scalar_functions.md`], and +[`window_functions.md`] are generated by [`dev/update_config_docs.sh`] and +[`dev/update_function_docs.sh`]. To check that they are up to date, run the +[`ci/scripts/check_generated_docs.sh`] script, which also runs as part of +`./dev/rust_lint.sh`. Run it with `--write` to update the pages. + +[`configs.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/configs.md +[`aggregate_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/aggregate_functions.md +[`scalar_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/scalar_functions.md +[`window_functions.md`]: https://github.com/apache/datafusion/blob/main/docs/source/user-guide/sql/window_functions.md +[`dev/update_config_docs.sh`]: https://github.com/apache/datafusion/blob/main/dev/update_config_docs.sh +[`dev/update_function_docs.sh`]: https://github.com/apache/datafusion/blob/main/dev/update_function_docs.sh +[`ci/scripts/check_generated_docs.sh`]: https://github.com/apache/datafusion/blob/main/ci/scripts/check_generated_docs.sh + +## Documentation HTML Build + +[`ci/scripts/check_docs_html.sh`] builds the documentation website, the same +build the "Test doc build" job runs. Sphinx builds with `-W`, so one warning +fails the check. `./dev/rust_lint.sh` runs it. Run the script with `--help` +for the tools it needs and the files it writes. + +```shell +./ci/scripts/check_docs_html.sh +``` + +[`ci/scripts/check_docs_html.sh`]: https://github.com/apache/datafusion/blob/main/ci/scripts/check_docs_html.sh + ## Benchmarks ### Criterion Benchmarks diff --git a/_sources/download.md.txt b/_sources/download.md.txt index 34296262071c8..2a0cc8e6e3f6e 100644 --- a/_sources/download.md.txt +++ b/_sources/download.md.txt @@ -26,7 +26,7 @@ For example: ```toml [dependencies] -datafusion = "55.0.0" +datafusion = "55.1.0" ``` While DataFusion is distributed via [crates.io] as a convenience, the diff --git a/_sources/library-user-guide/catalogs.md.txt b/_sources/library-user-guide/catalogs.md.txt index 3c8238b73edbb..dea4e413e2089 100644 --- a/_sources/library-user-guide/catalogs.md.txt +++ b/_sources/library-user-guide/catalogs.md.txt @@ -305,6 +305,74 @@ impl CatalogProviderList for MemoryCatalogProviderList { Like other traits, it also maintains the mapping of the Catalog's name to the CatalogProvider. +## Catalog Provider Factories + +The catalogs above are all registered programmatically, ahead of time, +before a query ever runs. Sometimes it is useful to let users attach a +catalog dynamically from SQL instead — for example, a catalog backed by a +remote catalog service such as an Iceberg REST catalog. This is exactly +analogous to how [`TableProviderFactory`] lets `CREATE EXTERNAL TABLE` +create a `TableProvider` "on the fly": `CatalogProviderFactory` lets +`CREATE EXTERNAL CATALOG ... STORED AS ...` create a +`CatalogProvider` on the fly. + +To support `CREATE EXTERNAL CATALOG`, implement `CatalogProviderFactory`: + +```rust +use std::sync::Arc; +use async_trait::async_trait; +use datafusion::catalog::{CatalogProvider, CatalogProviderFactory, MemoryCatalogProvider, Session}; +use datafusion::common::Result; +use datafusion::logical_expr::CreateExternalCatalog; + +#[derive(Debug)] +struct MyCatalogProviderFactory {} + +#[async_trait] +impl CatalogProviderFactory for MyCatalogProviderFactory { + async fn create( + &self, + _state: &dyn Session, + cmd: &CreateExternalCatalog, + ) -> Result> { + // `cmd.location` and `cmd.options` carry whatever was supplied in the + // `LOCATION` and `OPTIONS` clauses of the SQL statement; a real + // implementation would use them to connect to the remote catalog + // service. This example just returns an empty in-memory catalog. + Ok(Arc::new(MemoryCatalogProvider::new())) + } +} +``` + +Then register it on the `SessionState`, keyed by the `STORED AS` value that +should resolve to it: + +```rust,ignore +use datafusion::execution::session_state::SessionStateBuilder; + +let state = SessionStateBuilder::new() + .with_default_features() + .with_catalog_factory("MY_CATALOG_TYPE".to_string(), Arc::new(MyCatalogProviderFactory {})) + .build(); +``` + +With the factory registered, users can attach and detach the catalog from +SQL: + +```sql +CREATE EXTERNAL CATALOG my_catalog +STORED AS MY_CATALOG_TYPE +LOCATION 's3://bucket/warehouse' +OPTIONS ('warehouse' 'my_catalog'); + +DROP CATALOG my_catalog; +``` + +See [DDL: `CREATE EXTERNAL CATALOG`](../user-guide/sql/ddl.md#create-external-catalog) +for the full SQL syntax. + +[`tableproviderfactory`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.TableProviderFactory.html + ## Recap To recap, you need to: diff --git a/_sources/library-user-guide/custom-table-providers.md.txt b/_sources/library-user-guide/custom-table-providers.md.txt index dbbdf9d022716..30ffb2b5cf2f7 100644 --- a/_sources/library-user-guide/custom-table-providers.md.txt +++ b/_sources/library-user-guide/custom-table-providers.md.txt @@ -792,6 +792,19 @@ that a `FilterExec` is unnecessary for the `date` predicate, and the second ensures that only the relevant directories are scanned. The actual file reading happens later, in the stream produced by `execute()`. +## Row-Level DML: DELETE and UPDATE + +A custom table provider can support `DELETE` and `UPDATE` by implementing the optional [`TableProvider::delete_from()`] and [`TableProvider::update()`] methods. Their default implementations return a "not implemented" error. + +Each method builds an [ExecutionPlan] that changes the matching rows and returns the number of affected rows in a `count` column. Your provider decides how to apply and persist the changes. Perform the changes when the returned plan executes: these methods run during physical planning, including for `EXPLAIN`. + +The method documentation describes the filters, assignments, SQL semantics, and result schema that an implementation must support. [MemTable] provides an implementation of both methods over in-memory batches. + +For SQL syntax and examples, see the [DML section](../user-guide/sql/dml.md) of the user guide. + +[`tableprovider::delete_from()`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.TableProvider.html#method.delete_from +[`tableprovider::update()`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.TableProvider.html#method.update + ## Putting It All Together Here is a minimal but complete example of a custom table provider that generates diff --git a/_sources/library-user-guide/extending-sql.md.txt b/_sources/library-user-guide/extending-sql.md.txt index eea5b3b1acfc9..120bc8fab21a5 100644 --- a/_sources/library-user-guide/extending-sql.md.txt +++ b/_sources/library-user-guide/extending-sql.md.txt @@ -264,15 +264,38 @@ enables you to implement SQL constructs like: When implementing [`RelationPlanner`], you receive a [`RelationPlannerContext`] that provides utilities for planning: -| Method | Purpose | -| --------------------------- | ----------------------------------------------- | -| `plan(relation)` | Recursively plan a nested relation | -| `sql_to_expr(expr, schema)` | Convert SQL expression to DataFusion Expr | -| `context_provider()` | Access session configuration, tables, functions | +| Method | Purpose | +| --------------------------- | ------------------------------------------------ | +| `plan(relation)` | Recursively plan a nested relation | +| `get_cte(name)` | Look up a CTE visible in the current query scope | +| `sql_to_expr(expr, schema)` | Convert SQL expression to DataFusion Expr | +| `context_provider()` | Access session configuration, tables, functions | See the [RelationPlanner API documentation] for additional methods like `normalize_ident()` and `object_name_to_table_reference()`. +Relation planners are called before DataFusion's built-in CTE and catalog lookup. +If your planner recognizes ordinary relations by name, check `get_cte()` after +determining the relation is an ordinary name that your planner would otherwise +handle. Deliberately reserved names and custom syntax can keep their existing +precedence. Return `RelationPlanning::Original` when a CTE is visible so the +next planner can handle the relation. When every ordinary name-based planner in +the chain performs this check, planning reaches DataFusion's built-in CTE +resolution. Normalize the parsed name first so the lookup follows DataFusion's +existing name matching. Only do this for a table without function arguments: a +CTE named `numbers` shadows `numbers`, but it does not shadow a call such as +`numbers(10)`. + +```rust +// Run this after recognizing `relation` as an ordinary name handled here. +if let TableFactor::Table { name, args: None, .. } = &relation { + let table_ref = ctx.object_name_to_table_reference(name.clone())?; + if ctx.get_cte(&table_ref).is_some() { + return Ok(RelationPlanning::Original(Box::new(relation))); + } +} +``` + #### Implementation Strategies There are two main approaches when implementing a [`RelationPlanner`]: diff --git a/_sources/library-user-guide/functions/adding-udfs.md.txt b/_sources/library-user-guide/functions/adding-udfs.md.txt index c3a40557a006d..78e90dfa6b8d4 100644 --- a/_sources/library-user-guide/functions/adding-udfs.md.txt +++ b/_sources/library-user-guide/functions/adding-udfs.md.txt @@ -1116,6 +1116,24 @@ impl Accumulator for GeometricMean { } ``` +### Declaring how an Aggregate UDF treats `DISTINCT` + +By default DataFusion assumes an aggregate is sensitive to the `DISTINCT` modifier, which means the accumulator is expected to +read `AccumulatorArgs::is_distinct` and deduplicate its input. Override +[`AggregateUDFImpl::distinct_handling`] when that is not what your function does: + +- Return `DistinctHandling::Insensitive` when duplicates cannot change the result, that is, when merging a value the + accumulator has already seen is a no-op. `min`, `max`, `bool_and` and `bit_or` are all in this group. The optimizer + then plans `f(DISTINCT x)` as `f(x)`, which skips both the per-group hash set and the extra grouping stage that + `SingleDistinctToGroupBy` would otherwise introduce. +- Return `DistinctHandling::Unsupported` when the accumulator does not implement `DISTINCT`: it does not read + `is_distinct`, or it rejects `DISTINCT` with an error. The planner must then deduplicate the input first or reject + the query. Today this is a declaration only; rejecting such queries at planning time is a follow-up change. +- Leave the default `DistinctHandling::Sensitive` when the accumulator reads `AccumulatorArgs::is_distinct` and + deduplicates its input itself. + +Getting this wrong changes query results, so only claim `Insensitive` if your merge is genuinely idempotent. + ### Registering an Aggregate UDF To register a Aggregate UDF, you need to wrap the function implementation in a [`AggregateUDF`] struct and then register @@ -1370,6 +1388,7 @@ async fn main() -> Result<()> { [`aggregateudf`]: https://docs.rs/datafusion/latest/datafusion/logical_expr/struct.AggregateUDF.html [`create_udaf`]: https://docs.rs/datafusion/latest/datafusion/logical_expr/fn.create_udaf.html +[`aggregateudfimpl::distinct_handling`]: https://docs.rs/datafusion/latest/datafusion/logical_expr/trait.AggregateUDFImpl.html#method.distinct_handling [`advanced_udaf.rs`]: https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/udf/advanced_udaf.rs ## Adding a Table UDF diff --git a/_sources/library-user-guide/upgrading/54.0.0.md.txt b/_sources/library-user-guide/upgrading/54.0.0.md.txt index f8e7ac93c08d8..ce4d7c22a23de 100644 --- a/_sources/library-user-guide/upgrading/54.0.0.md.txt +++ b/_sources/library-user-guide/upgrading/54.0.0.md.txt @@ -335,44 +335,6 @@ This produces two user-visible changes: `ScalarSubqueryExpr` expression. Code that walks or transforms `LogicalPlan` / `ExecutionPlan` trees, as well as `EXPLAIN` output, may need updating. -### Filter predicate evaluation order may differ from query text - -The logical optimizer now reorders filters so that cheap predicates (most binary -comparisons, `IS NULL`, `Between`, `InList`, etc.) evaluate before expensive -ones (`LIKE`, regex, scalar function calls, subqueries). For example, -`WHERE col LIKE '%foo%' AND col2 = 5` may evaluate `col2 = 5` before -`col LIKE '%foo%'`. - -**Evaluation order has never been guaranteed to match the order written in the -query.** The SQL standard explicitly allows implementations to evaluate operands -in any order; major engines (PostgreSQL, SQL Server, Oracle, MySQL) document the -same. Queries should not rely on left-to-right evaluation or short-circuit -semantics for `AND` or `OR`. Previous versions of DataFusion already reordered -predicates (e.g., as part of expression simplification or predicate pushdown); -the new reordering pass just increases the scenarios where the optimizer will -change predicate evaluation order. - -**Fallible-predicate patterns are particularly affected.** For example: - -```sql -WHERE s ~ '^[0-9]+$' AND CAST(s AS INT) > 0 -``` - -The intent is likely to filter non-numeric strings before the `CAST` runs, -but this depends on evaluation-order behavior the SQL standard does not -guaranteed. The new reorder makes this kind of pattern more likely to fail -at runtime if the optimizer moves the `CAST` ahead of the regex. To force -conditional evaluation, rewrite using `CASE`, which has standardized -short-circuit semantics: - -```sql -WHERE CASE WHEN s ~ '^[0-9]+$' THEN CAST(s AS INT) > 0 ELSE false END -``` - -Volatile expressions (`random()`, `now()`, etc.) are exempt — their position -in the conjunct list is preserved so the number of times they evaluate per -query does not change. - ### `datafusion-proto`: expression deserialization now takes a `TaskContext` `Serializeable::from_bytes_with_registry` is renamed to `from_bytes_with_ctx` @@ -928,8 +890,3 @@ match register_function { RegisterFunction::Table(name, table) => {}, } ``` - -### New `Dialect::Spark` variant - -The `Dialect` enum in `datafusion_common::config` now includes a `Spark` variant. -If you match exhaustively on `Dialect`, add a `Dialect::Spark` arm. diff --git a/_sources/library-user-guide/upgrading/55.0.0.md.txt b/_sources/library-user-guide/upgrading/55.0.0.md.txt index 4483fe76059b2..abd4d0020e502 100644 --- a/_sources/library-user-guide/upgrading/55.0.0.md.txt +++ b/_sources/library-user-guide/upgrading/55.0.0.md.txt @@ -21,9 +21,43 @@ ## DataFusion 55.0.0 -**Note:** DataFusion `55.0.0` has not been released yet. The information provided -in this section pertains to features and changes that have already been merged -to the main branch and are awaiting release in this version. +### Filter predicate evaluation order may differ from query text + +The logical optimizer now reorders filters so that cheap predicates (most binary +comparisons, `IS NULL`, `Between`, `InList`, etc.) evaluate before expensive +ones (`LIKE`, regex, scalar function calls, subqueries). For example, +`WHERE col LIKE '%foo%' AND col2 = 5` may evaluate `col2 = 5` before +`col LIKE '%foo%'`. + +**Evaluation order has never been guaranteed to match the order written in the +query.** The SQL standard explicitly allows implementations to evaluate operands +in any order; major engines (PostgreSQL, SQL Server, Oracle, MySQL) document the +same. Queries should not rely on left-to-right evaluation or short-circuit +semantics for `AND` or `OR`. Previous versions of DataFusion already reordered +predicates (e.g., as part of expression simplification or predicate pushdown); +the new reordering pass just increases the scenarios where the optimizer will +change predicate evaluation order. + +**Fallible-predicate patterns are particularly affected.** For example: + +```sql +WHERE s ~ '^[0-9]+$' AND CAST(s AS INT) > 0 +``` + +The intent is likely to filter non-numeric strings before the `CAST` runs, +but this depends on evaluation-order behavior the SQL standard does not +guaranteed. The new reorder makes this kind of pattern more likely to fail +at runtime if the optimizer moves the `CAST` ahead of the regex. To force +conditional evaluation, rewrite using `CASE`, which has standardized +short-circuit semantics: + +```sql +WHERE CASE WHEN s ~ '^[0-9]+$' THEN CAST(s AS INT) > 0 ELSE false END +``` + +Volatile expressions (`random()`, `now()`, etc.) are exempt — their position +in the conjunct list is preserved so the number of times they evaluate per +query does not change. ### Map casts with nested Structs use schema-evolution rules @@ -215,6 +249,11 @@ Use `temp_file.size()` instead of [`RefCountedTempFile::current_disk_usage`]. [`refcountedtempfile::current_disk_usage`]: https://docs.rs/datafusion-execution/latest/datafusion_execution/disk_manager/struct.RefCountedTempFile.html#method.current_disk_usage [`refcountedtempfile::update_disk_usage`]: https://docs.rs/datafusion-execution/latest/datafusion_execution/disk_manager/struct.RefCountedTempFile.html#method.update_disk_usage +### New `Dialect::Spark` variant + +The `Dialect` enum in `datafusion_common::config` now includes a `Spark` variant. +If you match exhaustively on `Dialect`, add a `Dialect::Spark` arm. + ### `Dialect::AVAILABLE` replaced by `Dialect::available()` `datafusion_common::config::Dialect::AVAILABLE` has been removed. Use diff --git a/_sources/library-user-guide/upgrading/56.0.0.md.txt b/_sources/library-user-guide/upgrading/56.0.0.md.txt index 1a6b983025e39..312668c485629 100644 --- a/_sources/library-user-guide/upgrading/56.0.0.md.txt +++ b/_sources/library-user-guide/upgrading/56.0.0.md.txt @@ -25,6 +25,279 @@ in this section pertains to features and changes that have already been merged to the main branch and are awaiting release in this version. +### Upgrade arrow/parquet to 60.0.0 and object_store to 0.14.2 + +DataFusion 56.0.0 uses `arrow` and `parquet` 60.0.0, and `object_store` 0.14.2. +This may require updates to your Cargo.toml if you have direct dependencies on +these crates. + +See the [Arrow 60.0.0 release notes] and the [object_store 0.14.2 upgrade guide] for details on breaking changes in those versions. + +[arrow 60.0.0 release notes]: https://github.com/apache/arrow-rs/releases/tag/60.0.0 +[object_store 0.14.2 upgrade guide]: https://github.com/apache/arrow-rs-object-store/blob/v0.14.2/CHANGELOG.md + +#### Field and schema metadata now use arrow's `Metadata` type + +Arrow 60 introduced a dedicated [`Metadata`] type for field and schema metadata +in place of `HashMap`. Following that change, several DataFusion +APIs now use `Metadata` as well, such as `DFSchema::metadata` and +`ExprSchema::metadata` return `&Metadata`. `CastExpr::target_metadata` and +`TryCastExpr::target_metadata` return `Option<&Metadata>`. + +**Migration guide:** + +```rust,ignore +// Before +let meta: &HashMap = schema.metadata(); +let field = Field::new("a", DataType::Int64, true) + .with_metadata([("k".to_string(), "v".to_string())].into()); + +// After +let meta: &Metadata = schema.metadata(); +let field = Field::new("a", DataType::Int64, true) + .with_metadata(Metadata::new().with("k", "v")); +``` + +`Metadata` supports `.get()`, `.iter()`, `.is_empty()` and `.extend()` like +`HashMap`, and converts to and from `HashMap` and +`BTreeMap` via `From`, so most call sites need only a type +change. Use `FieldMetadata::to_hashmap` where a `HashMap` is still required. + +[`metadata`]: https://docs.rs/arrow/60.0.0/arrow/datatypes/struct.Metadata.html + +### `ParquetAccessPlan::into_overall_row_selection` is deprecated + +`ParquetAccessPlan::into_overall_row_selection` is deprecated starting in 56.0.0. +DataFusion now keeps selections local to each row group during Parquet scanning. +The method remains available with its existing conversion and selection-length +validation behavior, so direct callers continue to work but receive a deprecation +warning. + +Users who attach a `ParquetAccessPlan` to a `PartitionedFile` for DataFusion to +read need no changes. + +For custom readers, use `ParquetAccessPlan::into_inner()` to obtain the +`RowGroupAccess` entries. Their positions are the original file row-group +indexes, and each selection uses coordinates within its own row group. When +migrating away from the deprecated method, callers must handle conversion and +selection-length validation themselves. Parquet's push decoder and async reader +accept `RowGroupSelection` values through `with_row_group_selections`; the +synchronous reader still requires `with_row_groups` and an overall +`with_row_selection`. + +### Missing Parquet null counts are treated as unknown + +DataFusion now preserves an omitted Parquet `null_count` statistic as unknown. +Previously, it assumed zero nulls, which could discard matching NULL rows in +`IS NULL` filters and `ORDER BY ... NULLS FIRST LIMIT` queries, or produce an +incorrectly high `COUNT(column)` from metadata. + +Queries over affected files may read or sort more data because these optimizations +require a known null count: + +- Pruning row groups for `IS NULL` filters. +- Computing `COUNT(column)` using only file metadata. +- Pruning row groups with a dynamic `NULLS FIRST` TopK filter. +- Eliminating a sort over nullable columns in sorted, non-overlapping files. + Such queries may now retain a full `SortExec`, even when the data has no NULLs. + +parquet-rs versions before 53.1.0 omitted zero null counts. This includes files +produced with the older parquet-rs dependency used by DataFusion releases before +42.1.0. [Arrow #6490](https://github.com/apache/arrow-rs/pull/6490) changed the +writer to record known zero counts. Files with explicit counts retain their +existing behavior. + +To identify column chunks with bounds but no null count, run this query in the +DataFusion CLI: + +```sql +SELECT row_group_id, path_in_schema +FROM parquet_metadata('data.parquet') +WHERE (stats_min IS NOT NULL OR stats_max IS NOT NULL) + AND stats_null_count IS NULL; +``` + +Rewriting affected files with a current DataFusion version, for example using +`COPY ... TO` with Parquet statistics enabled, records the missing counts and +restores optimizations that depend on them. Preserve the required data ordering +and ordering metadata when rewriting sorted files. + +### `Timestamp(u, Some(tz)) - Timestamp(u, None)` reads the naive operand in `tz` + +Subtracting a timezone-naive timestamp from a timezone-aware one _at the same +time unit_ did not coerce at all in `55.0.0`: the naive operand was read as UTC, +which differed both from `=` on the same pair of values and from every other +pairing of time units. It now coerces like the rest, so the naive operand is +read in the aware operand's time zone: + +```sql +-- ts_tz is 2024-11-01T00:00:00-04:00 (America/New_York), ts is the naive +-- 2024-11-01T00:00:00; ts_tz = ts is true in both versions +-- 55.0.0: 0 days 4 hours 0 mins 0.000000000 secs +-- 56.0.0: 0 days 0 hours 0 mins 0.000000000 secs +SELECT ts_tz - ts FROM timestamps; +``` + +### `ForeignSession::create_physical_plan` is unsupported + +`ForeignSession::create_physical_plan` no longer forwards to the library that +owns the session. It now returns a `NotImplemented` error because forwarding can +re-enter an installed foreign planner, and the execution-plan handle returned by +the old callback cannot restore local Rust type identities for downcasting. +The original `FFI_SessionRef` callback slot remains in place for ABI compatibility +with DataFusion 55 consumers, but calling that callback returns the same error. + +The session-owning library should instead export its original planner as a +`datafusion_ffi::query_planner::FFI_QueryPlanner` before installing a foreign +planner. The foreign planner can retain and invoke that handle to receive a +serialized physical plan reconstructed with local type identities. See the +`datafusion_ffi::query_planner` module documentation for the complete delegation +pattern. `ForeignSession::query_planner`, `optimize`, and `physical_optimizers` +continue to forward to the owning session across the FFI boundary. + +### Range partitioning supports retained samples and scaling + +`RangePartitioning::new` is deprecated. Prefer the validated +`RangePartitioning::try_new_with_samples(ordering, samples, partition_count)`. +For exact boundaries, pass `split_points.len() + 1` as the count, or keep using +the validated `try_new` constructor. Retain additional samples when future +scaling above that count is required; scaling does not create new sample values. + +`scale` returns `Option`. It returns `None` when the target +partition count is zero or exceeds the retained sample capacity; callers can +then retain the existing layout or choose another partitioning. Equality includes +retained samples; use `has_same_layout` to compare the current key ordering and +effective boundaries. + +`FFI_RangePartitioning` has a new layout; rebuild FFI consumers against the +compatible release. The generated `PhysicalRangePartitioning` protobuf struct +also gains fields, affecting exhaustive struct literals. Older protobuf payloads +remain readable, and effective split points remain available to older readers. + +### `GroupColumn` now requires `values_preserving` + +Custom implementations of the public `GroupColumn` trait must implement +`values_preserving`. This method returns selected rows without changing the stored +values or their group indices. It preserves the requested order and supports +repeated indices. + +**Migration guide:** + +Implement `values_preserving`, call +`selection.validate_num_groups(self.len())?` before reading, and return rows in +`selection.iter()` order without changing the builder state. + +### `datafusion-proto`: common option and constraint conversions are fallible + +Protobuf conversions for `CsvOptions`, `JsonOptions`, `ParquetCdcOptions`, +`Constraint`, and `Constraints` now reject integer values that do not fit +`usize`. Their infallible `From` implementations have been replaced with +`TryFrom`. The existing `TryFrom` conversions for `ParquetOptions` and +`TableParquetOptions` now also validate all `usize`-backed fields. A constraint +without a `constraint_mode` returns an error instead of panicking. + +**Migration guide:** + +```rust,ignore +// Before +let csv = CsvOptions::from(&proto_csv); +let cdc = ParquetCdcOptions::from(proto_cdc); +let constraints: Constraints = proto_constraints.into(); + +// After +let csv = CsvOptions::try_from(&proto_csv)?; +let cdc = ParquetCdcOptions::try_from(proto_cdc)?; +let constraints = Constraints::try_from(proto_constraints)?; +``` + +See [issue #24170](https://github.com/apache/datafusion/issues/24170) for details. + +### `ExecutionOptions` has a new `enable_nlj_coordinated_fallback` field + +`ExecutionOptions` gained a public `enable_nlj_coordinated_fallback: bool` field +(default `true`). It controls whether the memory-limited `NestedLoopJoinExec` +fallback shares per-chunk build-side state across probe partitions, which is what +lets `LEFT`, `LEFT SEMI`, `LEFT ANTI`, `LEFT MARK` and `FULL` joins spill instead +of failing with `ResourcesExhausted` when the right side has several partitions. + +**Who is affected:** + +- Users constructing `ExecutionOptions` (or `ConfigOptions`) with an exhaustive + struct literal. Reading or `SET`-ing configuration is unaffected. +- Distributed engines that run each output partition as an independent task. + The coordination assumes all probe partitions run in one process; with one + coordinator per task the shared probe-thread counter never reaches zero and the + fallback would stall. Such engines should set the flag to `false`, which keeps + the previous fail-fast behaviour for the affected join types. + +**Migration guide:** + +Set the new field, or fill it from `Default`: + +```rust,ignore +// Before +ExecutionOptions { + batch_size: 8192, + // ... every other field ... +} + +// After: set it explicitly +ExecutionOptions { + batch_size: 8192, + enable_nlj_coordinated_fallback: true, + // ... every other field ... +} + +// After: or let the remaining fields come from Default, which also keeps +// future field additions from breaking the literal +ExecutionOptions { + batch_size: 8192, + ..Default::default() +} +``` + +Distributed engines opting out: + +```rust,ignore +let mut config = SessionConfig::new(); +config.options_mut().execution.enable_nlj_coordinated_fallback = false; +``` + +### `ExecutionOptions` has a new `soft_max_bytes_per_output_file` field + +`ExecutionOptions` gained a public `soft_max_bytes_per_output_file: ConfigNonZeroUsize` +field (default 4 GiB minus one byte, or 4,294,967,295 bytes). When writing +multiple output files, DataFusion starts a new file once a writer reports that +the encoded size has reached this target or the existing +`soft_max_rows_per_output_file` target has been reached. The byte limit is soft: +buffered batches, in-flight batch serialization, individual batch sizes, and +file metadata can make a completed file larger than the target. It does not +apply to single-file output. + +**Who is affected:** + +- Users constructing `ExecutionOptions` (or `ConfigOptions`) with an exhaustive + struct literal must provide the new field. +- Users writing multiple output files can set a smaller byte target to rotate + files before they reach the default size. + +**Migration guide:** + +Set the field explicitly, or fill it from `Default` as shown above. For example, +to target 128 MiB output files in Rust: + +```rust,ignore +let mut config = SessionConfig::new(); +config.options_mut().execution.soft_max_bytes_per_output_file = + ConfigNonZeroUsize::try_new(128 * 1024 * 1024)?; +``` + +The same setting can be changed with SQL: + +```sql +SET datafusion.execution.soft_max_bytes_per_output_file = 134217728; +``` + ### `datafusion.optimizer.use_statistics_registry` is deprecated and ignored The `datafusion.optimizer.use_statistics_registry` config flag is deprecated and @@ -49,6 +322,99 @@ previous default (`use_statistics_registry = false`). `StatisticsRegistry::default_with_builtin_providers()`. - To opt out, register nothing (the default). +### Generated protobuf `JsonWriterOptions` gained a `compression_level` field + +The generated public protobuf `JsonWriterOptions` struct now carries an optional +`compression_level` so JSON sink plans preserve explicitly configured +compression levels across serialization. + +**Who is affected:** + +- Users constructing generated `JsonWriterOptions` values with an exhaustive + struct literal. + +**Migration guide:** + +Set `compression_level` explicitly, or fill it from `Default`: + +```rust,ignore +// Before +JsonWriterOptions { compression } + +// After +JsonWriterOptions { + compression, + compression_level: None, +} +// or +JsonWriterOptions { + compression, + ..Default::default() +} +``` + +The protobuf wire format remains backward compatible. + +See [PR #24945](https://github.com/apache/datafusion/pull/24945) for details. + +### Generated protobuf Parquet structs gained state fields + +The generated public protobuf `ParquetScanExecNode` struct now carries an +optional `metadata_size_hint`, and `ParquetSink` now carries optional +`sorting_columns`. These fields preserve the corresponding Parquet source and +sink settings across physical-plan serialization. + +**Who is affected:** + +- Users constructing either generated struct with an exhaustive struct literal. + +**Migration guide:** + +Add `metadata_size_hint: None` to `ParquetScanExecNode` literals and +`sorting_columns: None` to `ParquetSink` literals to retain the previous +behavior, or use `..Default::default()` for unspecified fields. + +The protobuf wire format remains backward compatible. + +See [PR #25057](https://github.com/apache/datafusion/pull/25057) for details. + +### Generated protobuf `CsvWriterOptions` changed + +The generated `CsvWriterOptions` now includes `compression_level`, +`timestamp_tz_format`, and `terminator`. Its four existing format fields changed +from `String` to `Option` to distinguish unset from explicitly empty +formats. This affects exhaustive struct literals and direct format-field access. + +Wrap existing formats in `Some` or use `None`, and initialize the new fields +(explicitly or through `Default`): + +```rust,ignore +// Before (unchanged fields omitted) +CsvWriterOptions { + date_format: "%Y-%m-%d".to_string(), + datetime_format: String::new(), + timestamp_format: String::new(), + time_format: String::new(), + // ... +} + +// After +CsvWriterOptions { + date_format: Some("%Y-%m-%d".to_string()), + datetime_format: None, + timestamp_format: None, + time_format: None, + compression_level: None, + timestamp_tz_format: None, + terminator: Vec::new(), + // ... +} +``` + +The protobuf wire format remains backward compatible. + +See [PR #25058](https://github.com/apache/datafusion/pull/25058) for details. + ### `DefaultStatisticsProvider` is deprecated `datafusion_physical_plan::operator_statistics::DefaultStatisticsProvider` is @@ -70,3 +436,262 @@ Use the walk instead: StatisticsContext::new_with_registry(registry) .compute_extended(plan, &StatisticsArgs::new())?; // or .compute(...) for core Statistics ``` + +### API change for `floor` and `ceil` UDF + +The output type of the `floor` and `ceil` UDFs has been changed from the exact input type to a rescaled type with the same bit width. For example, for input `Decimal32(7,2)` `floor` now returns `Decimal32(6,0)`, where the new precision is `p - s + 1`, matching Spark's behaviour. See [#24703] for more details. + +**Who is affected:** + +- Users storing query result with these UDFs in a fixed schema +- Users relying on `arrow_typeof` for these UDFs + +**Migration guide:** + +Change the expected type or wrap the expression in `CAST`. It's recommended to avoid relying on decimal's exact precision and scale. + +### `map_extract` / `element_at` return an empty list for absent keys + +`map_extract` (and its alias `element_at`) previously returned a single-element +list containing `NULL` when the key was not present in the map. It now returns +an empty list, matching the documented behavior and DuckDB. Two related cases +changed at the same time, also matching DuckDB: + +- A `NULL` map input now yields `NULL` instead of `[NULL]`. +- A `NULL` lookup key now yields `[]` instead of `[NULL]`. + +A key that is present with a `NULL` value still returns `[NULL]`, so absent +keys and `NULL` values are now distinguishable. + +**Migration guide:** + +```sql +-- Before +SELECT map_extract(MAP {'a': 1}, 'missing'); -- [NULL] + +-- After +SELECT map_extract(MAP {'a': 1}, 'missing'); -- [] +``` + +Expressions that assumed the result always has exactly one element, for +example by checking its length, unnesting it, or comparing it to `[NULL]`, +should treat an empty list as the absent-key case instead. + +See [issue #24981](https://github.com/apache/datafusion/issues/24981) and +[issue #24983](https://github.com/apache/datafusion/issues/24983) for details. + +### `DataFrame::from_columns` accepts `IntoIterator` + +`DataFrame::from_columns` now accepts any `IntoIterator` +instead of specifically accepting a `Vec<(&str, ArrayRef)>`. + +```rust,ignore +// Existing Vec usage continues to work +let df = DataFrame::from_columns(vec![ + ("id", id), + ("name", name), +])?; + +// Arrays can now be used directly +let df = DataFrame::from_columns([ + ("id", id), + ("name", name), +])?; +``` + +Most existing call sites using `Vec` require no changes. Code that relies on +the exact non-generic function signature of `DataFrame::from_columns` may need +to be updated to account for the new generic API. + +### Aggregate ordering state field names are now namespaced + +Ordering expressions exposed as top-level aggregate state fields now use names +derived from the aggregate name and their ordering position. + +For example: + +```text +first_value(value)[ordering_0] +``` + +is now used instead of an unqualified ordering field name such as: + +```text +timestamp@0 +``` + +This guarantees unique aggregate state field names and allows +`DFSchema::check_names()` to be enforced. + +**Who is affected:** + +Users or integrations that inspect aggregate state field names directly, +including custom UDAFs and FFI integrations. + +### Physical filter pushdown resolves columns by position + +`datafusion_physical_plan::filter_pushdown::ChildFilterDescription::from_child` +and `FilterDescription::from_children` now resolve filter columns by position +instead of looking them up by name. This prevents incorrect results when a +child schema contains duplicate column names. `from_child` requires the child +field at each referenced position to have the same name as the filter column. + +`ChildFilterDescription::from_child_with_allowed_indices` is deprecated but +preserves its previous name-based mapping to the first matching child field. +Migrate to `from_child_with_column_mapping` because name resolution is ambiguous +when the child schema contains duplicate field names. + +**Migration guide:** + +Use `from_child` (or `from_children` for multiple children) when the parent and +child schemas have matching column positions and names. When a node projects, +reorders, or renames columns, use `from_child_with_column_mapping` with an +explicit map from parent output indices to child input indices. Columns absent +from the mapping cannot be pushed down. + +For example, if the parent outputs `[a, b]` and the child outputs `[b, a]`, a +filter on `a@0` must map to child column `a@1`: + +```rust,ignore +use std::collections::{HashMap, HashSet}; +use datafusion_physical_plan::filter_pushdown::ChildFilterDescription; + +// Before: allow parent column 0 and resolve "a" by name in the child. +let description = ChildFilterDescription::from_child_with_allowed_indices( + &parent_filters, + HashSet::from([0]), + &child, +)?; + +// After: explicitly map parent column 0 to child column 1. +let description = ChildFilterDescription::from_child_with_column_mapping( + &parent_filters, + HashMap::from([(0, 1)]), + &child, +)?; +``` + +### Default `datafusion.sql_parser.recursion_limit` raised from 50 to 51 + +DataFusion now uses `sqlparser` 0.63.0, which adds recursion guards to more +parse functions (for example, data type and `INTERVAL` parsing). As a result, +the same SQL statement consumes slightly more of the parser recursion budget +than it did with previous `sqlparser` versions. In particular, every expression +now needs one additional level of depth. + +To avoid rejecting queries that previously parsed successfully, the default +value of the `datafusion.sql_parser.recursion_limit` configuration option, and +of `DFParserBuilder::with_recursion_limit`, has been raised from `50` to `51`. + +If you set `datafusion.sql_parser.recursion_limit` explicitly to a value close +to the depth of your queries, you may see `RecursionLimitExceeded` errors after +upgrading and should raise the limit accordingly. + +See [PR #25278](https://github.com/apache/datafusion/pull/25278) for details. + +### `ensure_distribution` is deprecated in favour of `ensure_distribution_with_stats` + +`datafusion_physical_optimizer::enforce_distribution::ensure_distribution` now +takes its `StatisticsContext` from the caller, so one context (and its +memoization cache) can be shared across a whole bottom-up traversal. The old +two-argument form is kept as a deprecated wrapper that allocates a fresh +context per call, which recomputes each shared subtree's statistics once per +ancestor. + +```rust,ignore +// Before +let plan = DistributionContext::new_default(plan) + .transform_up(|ctx| ensure_distribution(ctx, config)) + .data()?; + +// After: one context for the whole walk. `StatsCache` is keyed by raw plan-node +// pointers, so reset it whenever a node's plan pointer actually changed. +let stats_ctx = StatisticsContext::new(); +let plan = DistributionContext::new_default(plan) + .transform_up(|ctx| { + let before = Arc::clone(&ctx.plan); + let result = ensure_distribution_with_stats(ctx, config, &stats_ctx)?; + if !Arc::ptr_eq(&before, &result.data.plan) { + stats_ctx.reset_cache(); + } + Ok(result) + }) + .data()?; +``` + +Callers that only need the previous behaviour can keep using the deprecated +form, or pass a freshly constructed `StatisticsContext` per call. + +### `MergeIntoOp` requires the SQL-visible target qualifier + +`MergeIntoOp` now stores the SQL-visible target qualifier separately from the +target provider identity. This prevents an aliased target from being confused +with a source relation that uses the target table's name. + +The struct is now non-exhaustive. Replace 55.0 struct literals with +`MergeIntoOp::new`: + +```rust,ignore +// DataFusion 55.0 +let op = MergeIntoOp { on, clauses }; + +// DataFusion 56.0 +let op = MergeIntoOp::new(target_qualifier, on, clauses); +``` + +`target_qualifier` must be the relation name referenced by the MERGE expressions: +the target alias when one is present, otherwise the target table reference. +`DmlStatement::table_name` remains the provider/catalog identity. + +### `MergeIntoOpNode` protobuf literals require `target_qualifier` + +The generated `MergeIntoOpNode` type has a new optional field. Code constructing +the generated type with a struct literal must provide it: + +```rust,ignore +// DataFusion 55.0 +let node = protobuf::MergeIntoOpNode { on, clauses }; + +// DataFusion 56.0 +let node = protobuf::MergeIntoOpNode { + on, + clauses, + target_qualifier: Some(protobuf::TableReference::from(target_qualifier)), +}; +``` + +Wire compatibility is directional: + +- A 56.0 reader accepts a 55.0 payload. When `target_qualifier` is absent, it + falls back to `DmlNode.table_name`, matching the 55.0 representation. +- A 55.0 reader must not consume an alias-preserving 56.0 MERGE payload. It + ignores the unknown field but cannot preserve the qualifier required by the + expressions, which can cause resolution failure or incorrect rebinding. + +### `FileSource::exact_filter` controls scan equivalence properties + +`FileScanConfig` no longer derives equivalence properties (constant columns +and equal columns) from `FileSource::filter`. It uses the new +`FileSource::exact_filter` method instead. This method returns the part of the +filter that every output row satisfies. A filter that the source uses only for +pruning (for example, Parquet row group statistics when +`datafusion.execution.parquet.pushdown_filters` is `false`) lets +non-matching rows through, and equivalences derived from it gave wrong query +results. + +`exact_filter` returns `None` by default. `ParquetSource` returns the pushed-down +conjuncts when `pushdown_filters` is enabled. + +**Migration guide:** + +If your custom `FileSource` applies its filter to every row, implement +`exact_filter` to keep the equivalence properties of the scan: + +```rust,ignore +impl FileSource for MySource { + fn exact_filter(&self) -> Option> { + self.filter() + } + // ... +} +``` diff --git a/_sources/user-guide/concepts-readings-events.md.txt b/_sources/user-guide/concepts-readings-events.md.txt index 366defa35eb5f..f6d4a19fbe0f6 100644 --- a/_sources/user-guide/concepts-readings-events.md.txt +++ b/_sources/user-guide/concepts-readings-events.md.txt @@ -53,13 +53,13 @@ This is a list of DataFusion related blog posts, articles, and other resources. - **2026-03-31** [Blog: Writing Custom Table Providers in Apache DataFusion](https://datafusion.apache.org/blog/2026/03/31/writing-table-providers/) -- **2026-03-20** [Blog: Turning LIMIT into an I/O Optimization: Inside DataFusion’s Multi-Layer Pruning Stack](https://datafusion.apache.org/blog/2026/03/20/multi-layer-pruning/) +- **2026-03-20** [Blog: Turning LIMIT into an I/O Optimization: Inside DataFusion’s Multi-Layer Pruning Stack](https://datafusion.apache.org/blog/2026/03/20/limit-pruning/) - **2026-02-23** [Blog: Apache DataFusion: A Data Engineer's Guide to the Query Engine Reshaping How We Build Data Systems](https://andrewmadson.substack.com/p/apache-datafusion-a-data-engineers) - **2026-02-09** [Blog: Vector search using only Parquet and DataFusion](https://blog.xiangpeng.systems/posts/vector-search-with-parquet-datafusion/) -- **2026-02-02** [Blog: Optimizing SQL CASE Expression Evaluation](https://datafusion.apache.org/blog/2026/02/02/case-expression/) +- **2026-02-02** [Blog: Optimizing SQL CASE Expression Evaluation](https://datafusion.apache.org/blog/2026/02/02/datafusion_case/) - **2026-01-12** [Blog: Extending SQL in DataFusion: from ->> to TABLESAMPLE](https://datafusion.apache.org/blog/2026/01/12/extending-sql) @@ -223,6 +223,7 @@ regular virtual event where community members share what they are building with # 🌎 Community Events +- **2026-11-12** [San Francisco Apache DataFusion Meetup](https://github.com/apache/datafusion/discussions/25665) - [RSVP](https://luma.com/k04h0om1) - **2026-09-30** [London Apache DataFusion Meetup](https://github.com/apache/datafusion/discussions/24512) - [RSVP](https://luma.com/r12f1t5o) - **2026-09-03** [Boston Apache DataFusion Meetup](https://github.com/apache/datafusion/discussions/21541) - [RSVP](https://luma.com/yexgqifv) - **2026-08-13** [San Francisco Apache DataFusion Meetup](https://github.com/apache/datafusion/discussions/24013) - [RSVP](https://luma.com/gk3s2vib) diff --git a/_sources/user-guide/configs.md.txt b/_sources/user-guide/configs.md.txt index 7c21103cd7818..ba8ea52bb0850 100644 --- a/_sources/user-guide/configs.md.txt +++ b/_sources/user-guide/configs.md.txt @@ -87,13 +87,13 @@ The following configuration settings are available: | datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | | datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | | datafusion.execution.parquet.force_filter_selections | false | (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. | -| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. | -| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. | +| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. | +| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. | | datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | | datafusion.execution.parquet.coerce_int96_tz | NULL | (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(, Some())` instead of the default `Timestamp(, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`. | | datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | | datafusion.execution.parquet.max_predicate_cache_size | NULL | (reading) The maximum predicate cache size, in bytes. When `pushdown_filters` is enabled, sets the maximum memory used to cache the results of predicate evaluation between filter evaluation and output generation. Decreasing this value will reduce memory usage, but may increase IO and CPU usage. None means use the default parquet reader setting. 0 means no caching. | -| datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger positive, non-null literal string lists on a string column use a compact sorted domain. Other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. | +| datafusion.execution.parquet.max_in_list_size | 20 | Maximum number of input values in an `IN (...)` list eligible for min/max pruning. Lists above this cap, or a cap of 0, skip this rewrite; other predicates and Bloom-filter pruning remain available. Within the cap, nonempty lists of at most 20 values use the existing per-value rewrite. Larger literal lists use a compact representation when the column type is string, variable-length binary, integer, decimal, date, time, timestamp, or duration. This applies to both `IN` and `NOT IN`, including lists with NULL members. `NOT IN` with NULL and all-NULL `IN` lists cannot match any rows. Compact lists containing NULL do not use the fully-matched-row-group optimization. Floating-point and other lists retain the existing per-value rewrite, so raising the cap can make those predicates expensive to build and evaluate. Defaults to 20. | | datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | | datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in rows | | datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | @@ -104,7 +104,7 @@ The following configuration settings are available: | datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. When `max_row_group_bytes` is also set, the writer flushes a row group when either limit is reached, whichever comes first. | | datafusion.execution.parquet.max_row_group_bytes | NULL | (writing) Target maximum size of each row group in bytes. When set, the writer flushes whenever either this limit or `max_row_group_size` is reached, whichever comes first. Useful for bounding writer memory on wide schemas where a row-count limit can map to very different byte sizes. Matches the behavior of `parquet.block.size` in parquet-mr. If `None` (the default), only the row-count limit applies. Currently only honored when `allow_single_file_parallelism` is `false`; by default the parallel file writer ignores this limit. | -| datafusion.execution.parquet.created_by | datafusion version 55.0.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.created_by | datafusion version 55.1.0 | (writing) Sets "created by" property | | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.statistics_truncate_length | 64 | (writing) Sets statistics truncate length. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | @@ -112,7 +112,7 @@ The following configuration settings are available: | datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | | datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files*n_row_groups*n_columns. | +| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files\*n_row_groups\*n_columns. | | datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | | datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | | datafusion.execution.parquet.content_defined_chunking.enabled | false | (writing) EXPERIMENTAL: Enable content-defined chunking (CDC) when writing parquet files. When enabled, parallel writing is automatically disabled since the chunker state must persist across row groups. | @@ -121,15 +121,17 @@ The following configuration settings are available: | datafusion.execution.parquet.content_defined_chunking.norm_level | 0 | Normalization level. Increasing this improves deduplication ratio but increases fragmentation. Recommended range is [-3, 3], default is 0. | | datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | | datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | -| datafusion.execution.enable_migration_aggregate | true | Temporary switch for aggregate stream implementations that are being migrated from `GroupedHashAggregateStream`. When set to true, DataFusion tries the migrated implementations when their preconditions are satisfied. When set to false, grouped aggregation falls back to `GroupedHashAggregateStream`. This option will be removed after the migration is finished. See for details. | +| datafusion.execution.enable_migration_aggregate | true | Whether aggregation uses the implementation from the major refactor completed in the 56.0.0 release. When set to `false`, aggregation falls back to the implementation used before 55.0.0. The fallback exists only as a workaround for bugs in the new implementation and will be removed, together with this option, after the 56.0.0 release. See for details. | | datafusion.execution.spill_compression | uncompressed | Sets the compression codec used when spilling data to disk. Since datafusion writes spill files using the Arrow IPC Stream format, only codecs supported by the Arrow IPC Stream Writer are allowed. Valid values are: uncompressed, lz4_frame, zstd. Note: lz4_frame offers faster (de)compression, but typically results in larger spill files. In contrast, zstd achieves higher compression ratios at the cost of slower (de)compression speed. | | datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | | datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | | datafusion.execution.sort_pushdown_buffer_capacity | 1073741824 | Maximum buffer capacity (in bytes) per partition for BufferExec inserted during sort pushdown optimization. When PushdownSort eliminates a SortExec under SortPreservingMergeExec, a BufferExec is inserted to replace SortExec's buffering role. This prevents I/O stalls by allowing the scan to run ahead of the merge. This uses strictly less memory than the SortExec it replaces (which buffers the entire partition). The buffer respects the global memory pool limit. Setting this to a large value is safe — actual memory usage is bounded by partition size and global memory limits. | | datafusion.execution.max_spill_file_size_bytes | 134217728 | Maximum size in bytes for individual spill files before rotating to a new file. When operators spill data to disk (e.g., RepartitionExec), they write multiple batches to the same file until this size limit is reached, then rotate to a new file. This reduces syscall overhead compared to one-file-per-batch while preventing files from growing too large. A larger value reduces file creation overhead but may hold more disk space. A smaller value creates more files but allows finer-grained space reclamation as files can be deleted once fully consumed. Now only `RepartitionExec` supports this spill file rotation feature, other spilling operators may create spill files larger than the limit. Default: 128 MB | +| datafusion.execution.enable_nlj_coordinated_fallback | true | Enables the memory-limited fallback for `NestedLoopJoinExec` join types that emit unmatched left rows in the final output (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) when the right side has multiple partitions. This fallback shares left-side state (the current left chunk, the visited bitmap and the probe-thread counter) across all right-side partitions, which assumes every partition runs in the same process. Distributed engines that execute each output partition as an independent task (e.g. Ballista, datafusion-distributed) give each task its own copy of this state and poll only one partition, so the cross-partition counter never reaches zero and the fallback would stall. Such engines should set this to `false`: the coordinated fallback is then disabled for left-emitting multi-partition joins, which instead fail with a resource-exhaustion error under memory pressure rather than deadlocking. Single-partition and non-left-emitting joins are unaffected and always keep the fallback. | | datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | -| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file is reached. | +| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file or soft_max_bytes_per_output_file is reached. | | datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | +| datafusion.execution.soft_max_bytes_per_output_file | 4294967295 | Target encoded size in bytes of output files when writing multiple. Writers asynchronously report the cumulative encoded size as they process RecordBatches. The final file size may exceed this limit due to batches buffered before the limit is observed, the size of a batch, and file metadata written when the file is finalized. | | datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption. This budget is split evenly between two independent points in the write pipeline (see the demuxer diagram in #7791): how many files can be in flight from the demuxer to a writer task, and how many RecordBatches are buffered for a single file's writer. Must be at least 2 so each half gets at least 1 unit of buffering - 0 or 1 would leave one side with a zero-capacity channel and panic at write time. | | datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | | datafusion.execution.listing_table_factory_infer_partitions | true | Should a `ListingTable` created through the `ListingTableFactory` infer table partitions from Hive compliant directories. Defaults to true (partition columns are inferred and will be represented in the table schema). | @@ -140,7 +142,7 @@ The following configuration settings are available: | datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | | datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | | datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | -| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. | +| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. Note: this option currently only applies to the symmetric hash join. | | datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | | datafusion.execution.enable_ansi_mode | false | Whether to enable ANSI SQL mode. The flag is experimental and relevant only for DataFusion Spark built-in functions When `enable_ansi_mode` is set to `true`, the query engine follows ANSI SQL semantics for expressions, casting, and error handling. This means: - **Strict type coercion rules:** implicit casts between incompatible types are disallowed. - **Standard SQL arithmetic behavior:** operations such as division by zero, numeric overflow, or invalid casts raise runtime errors rather than returning `NULL` or adjusted values. - **Consistent ANSI behavior** for string concatenation, comparisons, and `NULL` handling. When `enable_ansi_mode` is `false` (the default), the engine uses a more permissive, non-ANSI mode designed for user convenience and backward compatibility. In this mode: - Implicit casts between types are allowed (e.g., string to integer when possible). - Arithmetic operations are more lenient — for example, `abs()` on the minimum representable integer value returns the input value instead of raising overflow. - Division by zero or invalid casts may return `NULL` instead of failing. # Default `false` — ANSI SQL mode is disabled by default. | | datafusion.execution.hash_join_buffering_capacity | 0 | How many bytes to buffer in the probe side of hash joins while the build side is concurrently being built. Without this, hash joins will wait until the full materialization of the build side before polling the probe side. This is useful in scenarios where the query is not completely CPU bounded, allowing to do some early work concurrently and reducing the latency of the query. Note that when hash join buffering is enabled, the probe side will start eagerly polling data, not giving time for the producer side of dynamic filters to produce any meaningful predicate. Queries with dynamic filters might see performance degradation. Disabled by default, set to a number greater than 0 for enabling it. | @@ -199,7 +201,7 @@ The following configuration settings are available: | datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | | datafusion.sql_parser.map_string_types_to_utf8view | true | If true, string types (VARCHAR, CHAR, Text, and String) are mapped to `Utf8View` during SQL planning. If false, they are mapped to `Utf8`. Default is true. | | datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | -| datafusion.sql_parser.recursion_limit | 50 | Specifies the recursion depth limit when parsing complex SQL Queries | +| datafusion.sql_parser.recursion_limit | 51 | Specifies the recursion depth limit when parsing complex SQL Queries | | datafusion.sql_parser.default_null_ordering | nulls_max | Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: | | datafusion.sql_parser.enable_subquery_sort_elimination | true | When set to true, DataFusion may remove `ORDER BY` clauses from subqueries or CTEs during SQL planning when their ordering cannot affect the result, such as when no `LIMIT` or other order-sensitive operator depends on them. Disable this option to preserve explicit subquery ordering in the planned query. | | datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | diff --git a/_sources/user-guide/crate-configuration.md.txt b/_sources/user-guide/crate-configuration.md.txt index 09c65107e58c8..75ac2fbdde939 100644 --- a/_sources/user-guide/crate-configuration.md.txt +++ b/_sources/user-guide/crate-configuration.md.txt @@ -156,7 +156,7 @@ By default, Datafusion returns errors as a plain text message. You can enable mo such as backtraces by enabling the `backtrace` feature to your `Cargo.toml` file like this: ```toml -datafusion = { version = "55.0.0", features = ["backtrace"]} +datafusion = { version = "55.1.0", features = ["backtrace"]} ``` Set environment [variables](https://doc.rust-lang.org/std/backtrace/index.html#environment-variables) diff --git a/_sources/user-guide/example-usage.md.txt b/_sources/user-guide/example-usage.md.txt index 3d89405385c05..8c96b3e82dfc3 100644 --- a/_sources/user-guide/example-usage.md.txt +++ b/_sources/user-guide/example-usage.md.txt @@ -29,7 +29,7 @@ Find latest available Datafusion version on [DataFusion's crates.io] page. Add the dependency to your `Cargo.toml` file: ```toml -datafusion = "55.0.0" +datafusion = "55.1.0" tokio = { version = "1.0", features = ["rt-multi-thread"] } ``` diff --git a/_sources/user-guide/explain-usage.md.txt b/_sources/user-guide/explain-usage.md.txt index 43ea6caac3a7a..b073313c6a0a4 100644 --- a/_sources/user-guide/explain-usage.md.txt +++ b/_sources/user-guide/explain-usage.md.txt @@ -233,6 +233,7 @@ When predicate pushdown is enabled, `DataSourceExec` with `ParquetSource` gains - `row_groups_pruned_bloom_filter`: number of row groups evaluated by Bloom Filters, reporting both total checked groups and groups that matched. - `row_groups_pruned_statistics`: number of row groups evaluated by row-group statistics (min/max), reporting both total checked groups and groups that matched. - `limit_pruned_row_groups`: number of row groups pruned by the limit. +- `limit_pruned_rows`: number of rows skipped by limit pruning when fully matched page ranges contain enough rows to satisfy the limit. - `pushdown_rows_matched`: rows that were tested by any of the above filters, and passed all of them. - `pushdown_rows_pruned`: rows that were tested by any of the above filters, and did not pass at least one of them. - `predicate_evaluation_errors`: number of times evaluating the filter expression failed (expected to be zero in normal operation) diff --git a/_sources/user-guide/expressions.md.txt b/_sources/user-guide/expressions.md.txt index 3fbc11e0c92c0..55f64135dd0fb 100644 --- a/_sources/user-guide/expressions.md.txt +++ b/_sources/user-guide/expressions.md.txt @@ -191,7 +191,7 @@ select log(-1), log(0), sqrt(-1); | lpad(text, length, [, fill]) | Extends the string to length (`length`) by prepending the characters (`fill`) (a space by default). Example: `lpad('bb', 5, 'a') → aaabb` | | ltrim(text, text) | Removes all specified characters (`characters`) from the beginning of the string (`text`). Example: `ltrim('aabchelloccb', 'abc') -> helloccb` | | md5(text) | Computes the MD5 hash of the argument (`text`). | -| octet_length(text) | Returns number of bytes in the string (`text`). | +| octet_length(text) | Returns number of bytes in the string or binary (`text`). | | repeat(text, number) | Repeats the string the specified number of times. Example: `repeat('1', 4) -> 1111` | | replace(string, from, to) | Replaces a specified string (`from`) with another specified string (`to`) in the string (`string`). Example: `replace('Hello', 'replace', 'el') -> Hola` | | reverse(text) | Reverses the order of the characters in the string (`text`). Example: `reverse('hello') -> olleh` | diff --git a/_sources/user-guide/metrics.md.txt b/_sources/user-guide/metrics.md.txt index 306094c438eb7..111df66ccc08a 100644 --- a/_sources/user-guide/metrics.md.txt +++ b/_sources/user-guide/metrics.md.txt @@ -61,6 +61,146 @@ subsequent join processing time. | avg_fanout | Average number of build-side join-key matches per matched probe-side row before applying any join filter. | | array_map_created_count | Number of times `HashJoinExec` created an `ArrayMap` for perfect hash join lookup execution. | +### AggregateExec + +`AggregateExec` exposes the common `BaselineMetrics`, the operator-level +metrics below, and a timer per aggregate expression and execution phase. + +| Metric | Description | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| time_calculating_group_ids | Time spent preparing group keys; see the note below for path-specific coverage. | +| aggregate_arguments_time | Total time spent evaluating the inputs to the aggregate functions. `agg_expr_{index}_arguments_time` breaks this down per expression. | +| aggregation_time | Time spent invoking accumulator `update` and `merge` operations. | +| emitting_time | Time spent materializing group values and accumulator `state` or `evaluate` results. | +| topk_maintenance_time | Time spent maintaining Grouped TopK's priority map, including batch setup, insertion, comparison, and NULL handling. | +| skipped_aggregation_rows | Number of input rows passed through without aggregating them, when partial aggregation is skipped. | +| reduction_factor | Rows emitted per row consumed by a partial aggregation, displayed as `66.67% (2/3)`. | +| spill_count | Number of spill files written when the aggregation exceeds its memory budget. | +| spilled_bytes | Total number of bytes written to spill files. | +| spilled_rows | Total number of rows written to spill files. | +| peak_mem_used | Peak tracked memory held by the grouped aggregation, in bytes; recorded by the fallback grouped hash path only. | + +These operator-level metrics are recorded by the grouped aggregation paths +only: an `AggregateExec` without a `GROUP BY` reports just `BaselineMetrics` +and the per-aggregate timers. `reduction_factor` and `skipped_aggregation_rows` +are recorded in partial mode only. `skipped_aggregation_rows` is recorded only +when partial-aggregation skipping is enabled for a single, non-grouping-sets +`GROUP BY` whose input is not ordered by its grouping expressions; a +`datafusion.execution.skip_partial_aggregation_probe_ratio_threshold` of `>= 1.0` +disables the feature. + +`time_calculating_group_ids` covers both grouping-expression evaluation and +resolving the resulting rows to group IDs, including interning and ordering +setup. `aggregation_time` covers only accumulator `update` and `merge` calls. +On accumulator-backed grouped aggregation paths, `state` and `evaluate` are +included in `emitting_time`, including when partial aggregation materializes +state under memory pressure. For an `AggregateExec` without a `GROUP BY`, the +per-aggregate `state` and `evaluate` timers are recorded during +`elapsed_compute`; it does not report grouped operator metrics such as +`emitting_time`. Partial aggregation that skips aggregation reports +`convert_to_state` instead of `aggregation_time` for those rows. + +When the specialized Grouped TopK path is selected for a limited grouped +aggregate, it has no accumulators. Its group-key expression evaluation is +included in `time_calculating_group_ids`, and its priority-map work is reported +by `topk_maintenance_time`; it does not report accumulator phases. Planner +selection is query-shape dependent: a limited `DISTINCT` query without +ordering requirements uses the regular aggregate path instead. + +The per-aggregate timers are named `agg_expr_{index}_{phase}_time`, where +`index` is the zero-based position of an aggregate expression in the operator +and `phase` is one of the following: + +| Phase | Description | +| ------------------ | --------------------------------------------------------------------------------------------- | +| `arguments` | Evaluating the aggregate's argument expressions into input arrays. | +| `update` | Updating an accumulator from raw input values. | +| `merge` | Merging partial accumulator states. | +| `state` | Obtaining an accumulator's intermediate state for partial output or aggregate spilling. | +| `convert_to_state` | Converting raw aggregate inputs directly to partial state without normal accumulator updates. | +| `evaluate` | Evaluating an accumulator to its final result. | + +For example, when a partial stage is planned, the partial `AggregateExec` for +`SELECT SUM(a), SUM(b) FROM t` reports `agg_expr_0_arguments_time` and +`agg_expr_0_update_time` for `SUM(a)`, +and `agg_expr_1_arguments_time` and `agg_expr_1_update_time` for `SUM(b)`. The +index is positional and refers to the same position in the `aggr=[...]` list +printed on the operator's plan line, which is how an indexed timer is mapped +back to an aggregate expression. Because the index is part of the metric name, +otherwise identical functions over different columns stay distinct when +per-partition metrics are combined. + +Each per-aggregate timer additionally carries an `aggregate` label holding the +rendered aggregate expression (for example, `sum(t.a)`). Combining metrics +across partitions drops labels, so this label is only shown in the "Plan with +Full Metrics" section of `EXPLAIN ANALYZE VERBOSE`, which reports metrics per +partition. + +`arguments` is recorded in every mode. The accumulator phases that are present +depend on the aggregate mode and implementation. For non-grouped aggregation, +partial mode records `update` and `state`, partial reduce mode records `merge` +and `state`, final mode records `merge` and `evaluate`, and single mode records +`update` and `evaluate`. Hash aggregation uses `update`, `state`, and +`convert_to_state` in partial mode; `merge` and `state` in partial-reduce mode; +`merge`, `state`, and `evaluate` in final mode; and `update`, `state`, `merge`, +and `evaluate` in single mode. Its `state` timers measure intermediate-state +emission, including during spilling. The grouped TopK aggregate path records +only the per-aggregate `arguments` timer, because it maintains values directly +rather than using accumulators. + +Where an aggregate has a `FILTER` clause, its evaluation is included in +`aggregate_arguments_time`. Per-aggregate `arguments` timers include the filter +when it is evaluated per aggregate. The legacy grouped hash path evaluates +filters collectively, so its per-aggregate `arguments` timers cover argument +expressions only; their sum need not equal `aggregate_arguments_time`. + +Aggregate implementations can also expose optional internal submetrics. These +use the `agg_expr_{index}_internal_{subphase}_time` naming and `aggregate` +label. The `internal` segment keeps them separate from call-boundary timers; +`subphase` is a stable identifier owned and documented by the aggregate +implementation. They are registered lazily only when an aggregate requests +them, so aggregates without internal submetrics add no metrics. Registration +is per `(aggregate expression index, subphase, partition)`: replacement +accumulators in that partition share the same time, and normal metric display +combines that time across partitions. An aggregate may request its submetric +during accumulator construction, so it can appear even when its input is empty. +For example, `array_agg(DISTINCT ...)` records the time spent deduplicating +input values as `agg_expr_{index}_internal_distinct_time`. Grouped accumulation +records this once per input batch, rather than once per group, to avoid making +metric collection proportional to group cardinality. These submetrics complement +the `update`, `merge`, `state`, and `evaluate` timers rather than subdividing or +replacing them. An internal submetric may therefore overlap its enclosing phase +timer; it is a supplementary diagnostic and must not be added to phase timings +as a breakdown. + +Except for the `Summary` metric `reduction_factor`, these operator-level and +per-aggregate metrics are `Dev` metrics. They appear in `EXPLAIN ANALYZE` when +`datafusion.explain.analyze_level` includes `Dev` (the default), but are omitted +at the `Summary` level. The normal display combines partitions; use `EXPLAIN ANALYZE VERBOSE` to additionally show the per-partition values together with +each per-aggregate timer's `aggregate` label. For a query +such as the following, the per-expression metrics stay readable, and the +operator's `aggr=[...]` list names the aggregate behind each timer index: + +```sql +EXPLAIN ANALYZE +SELECT k, SUM(a), SUM(b), COUNT(c) +FROM t +GROUP BY k; +``` + +The abbreviated partial-aggregate plan line below shows that mapping (timings +and unrelated metrics are omitted): + +```text +AggregateExec: mode=Partial, gby=[k@0 as k], aggr=[sum(t.a), sum(t.b), count(t.c)], + metrics=[..., agg_expr_0_arguments_time=..., agg_expr_1_arguments_time=..., + agg_expr_2_arguments_time=...] +``` + +Thus `agg_expr_0_*` is `sum(t.a)`, `agg_expr_1_*` is `sum(t.b)`, and +`agg_expr_2_*` is `count(t.c)`. In verbose output, the corresponding +`aggregate` labels provide the same mapping per partition. + ## TODO Add metrics for the remaining operators diff --git a/_sources/user-guide/sql/aggregate_functions.md.txt b/_sources/user-guide/sql/aggregate_functions.md.txt index 4569a8850c076..2d59970b0067b 100644 --- a/_sources/user-guide/sql/aggregate_functions.md.txt +++ b/_sources/user-guide/sql/aggregate_functions.md.txt @@ -247,10 +247,10 @@ bool_and(expression) ### `bool_or` -Returns true if all non-null input values are true, otherwise false. +Returns true if any non-null input value is true, otherwise false. ```sql -bool_and(expression) +bool_or(expression) ``` #### Arguments @@ -260,9 +260,9 @@ bool_and(expression) #### Example ```sql -> SELECT bool_and(column_name) FROM table_name; +> SELECT bool_or(column_name) FROM table_name; +----------------------------+ -| bool_and(column_name) | +| bool_or(column_name) | +----------------------------+ | true | +----------------------------+ @@ -644,10 +644,10 @@ _Alias of [covar_samp](#covar_samp)._ ### `covar_pop` -Returns the sample covariance of a set of number pairs. +Returns the population covariance of a set of number pairs. ```sql -covar_samp(expression1, expression2) +covar_pop(expression1, expression2) ``` #### Arguments @@ -658,11 +658,11 @@ covar_samp(expression1, expression2) #### Example ```sql -> SELECT covar_samp(column1, column2) FROM table_name; +> SELECT covar_pop(column1, column2) FROM table_name; +-----------------------------------+ -| covar_samp(column1, column2) | +| covar_pop(column1, column2) | +-----------------------------------+ -| 8.25 | +| 7.63333333333 | +-----------------------------------+ ``` diff --git a/_sources/user-guide/sql/data_types.md.txt b/_sources/user-guide/sql/data_types.md.txt index 2e75fce6c7aa9..39442d38c497d 100644 --- a/_sources/user-guide/sql/data_types.md.txt +++ b/_sources/user-guide/sql/data_types.md.txt @@ -105,12 +105,42 @@ The maximum supported precision for `DECIMAL` types is 76. ## Date/Time Types -| SQL DataType | Arrow DataType | -| ------------ | :------------------------------- | -| `DATE` | `Date32` | -| `TIME` | `Time64(Nanosecond)` | -| `TIMESTAMP` | `Timestamp(Nanosecond, None)` | -| `INTERVAL` | `Interval(IntervalMonthDayNano)` | +| SQL DataType | Arrow DataType | +| ------------------------------------------------------------- | :--------------------------------------------------------- | +| `DATE` | `Date32` | +| `TIME` | `Time64(Nanosecond)` | +| `TIMESTAMP`, `TIMESTAMP(p)`, `TIMESTAMP(p) WITHOUT TIME ZONE` | `Timestamp(unit, None)` | +| `TIMESTAMPTZ(p)`, `TIMESTAMP(p) WITH TIME ZONE` | `Timestamp(unit, None)` by default — see the warning below | +| `INTERVAL` | `Interval(MonthDayNano)` | + +:::{warning} +`TIMESTAMPTZ` and `TIMESTAMP WITH TIME ZONE` do **not** give a timezone-aware +type by default. The zone comes from the +[`datafusion.execution.time_zone`] setting, and that setting is unset unless you +set it. So `'2024-01-01T12:00:00Z'::timestamptz` gives `Timestamp(unit, None)` +by default: the result type does not keep timezone metadata. + +An explicit offset in the input is still applied. DataFusion normalizes the +value to UTC. For example, with the default setting, +`'2024-01-01T12:00:00+07:00'::timestamptz` gives `2024-01-01T05:00:00`. It is +not read as the local time `12:00:00`. + +This is a known bug. See [issue #25166]. PostgreSQL and DuckDB always give an +aware type here, because their session time zone always has a value. + +Set `datafusion.execution.time_zone` to make these types aware. `'UTC'` is a +good value. +::: + +`unit` comes from the optional precision `p` on the two `TIMESTAMP` rows. Use +`0`, `3`, `6` or `9` for `Second`, `Millisecond`, `Microsecond` or `Nanosecond`. +DataFusion rejects each other value of `p`. If you omit `p`, the unit is +`Nanosecond`. The precision goes after `TIMESTAMPTZ`, but before +`WITH TIME ZONE`: use `TIMESTAMPTZ(3)` or `TIMESTAMP(3) WITH TIME ZONE`. +`DATE`, `TIME` and `INTERVAL` do not accept a precision. + +[`datafusion.execution.time_zone`]: ../configs.md +[issue #25166]: https://github.com/apache/datafusion/issues/25166 ## Boolean Types diff --git a/_sources/user-guide/sql/ddl.md.txt b/_sources/user-guide/sql/ddl.md.txt index 0d76775bcc1c6..8382795e74306 100644 --- a/_sources/user-guide/sql/ddl.md.txt +++ b/_sources/user-guide/sql/ddl.md.txt @@ -22,17 +22,69 @@ DDL stands for "Data Definition Language" and relates to creating and modifying catalog objects such as Tables. -## CREATE DATABASE +## CREATE CATALOG / CREATE DATABASE -Create catalog with specified name. +Create catalog with specified name. In DataFusion, `CREATE DATABASE` is an alias for `CREATE CATALOG`. 
-CREATE DATABASE [ IF NOT EXISTS ] catalog
+CREATE CATALOG [ IF NOT EXISTS ] catalog
```sql -- create catalog cat -CREATE DATABASE cat; +CREATE CATALOG cat; +``` + +## CREATE EXTERNAL CATALOG + +`CREATE EXTERNAL CATALOG` registers a catalog built by a registered +[`CatalogProviderFactory`], such as a catalog backed by a remote catalog +service (for example, an Iceberg REST catalog), so that it can be queried +alongside DataFusion's built-in catalogs. A `CatalogProviderFactory` must +first be registered on the `SessionState` with a key matching the +`STORED AS` clause below — see the [Catalog Provider Factories] section of +the Library User Guide for how to implement and register one. + +The supported syntax is: + +```sql +CREATE [OR REPLACE] EXTERNAL CATALOG +[ IF NOT EXISTS ] + +STORED AS +[ LOCATION ] +[ OPTIONS () ] + + := ( , , ...) +``` + +`catalog_type` identifies which registered `CatalogProviderFactory` to +invoke; it is looked up the same way `file_type` is for +[`CREATE EXTERNAL TABLE`](#create-external-table). + +```sql +CREATE EXTERNAL CATALOG my_catalog +STORED AS ICEBERG +LOCATION 's3://bucket/warehouse' +OPTIONS ('catalog.uri' 'http://rest-catalog:8181', 'warehouse' 'my_catalog'); +``` + +[`catalogproviderfactory`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.CatalogProviderFactory.html +[catalog provider factories]: ../../library-user-guide/catalogs.md#catalog-provider-factories + +## DROP CATALOG + +Removes a catalog previously registered with `CREATE EXTERNAL CATALOG` from +DataFusion's catalog list. + +
+DROP CATALOG [ IF EXISTS ] catalog_name;
+
+ +```sql +DROP CATALOG my_catalog; +-- or use 'if exists' to silently ignore if the catalog doesn't exist +DROP CATALOG IF EXISTS nonexistent_catalog; ``` ## CREATE SCHEMA diff --git a/_sources/user-guide/sql/dml.md.txt b/_sources/user-guide/sql/dml.md.txt index 4934bc2674375..cd5947d983620 100644 --- a/_sources/user-guide/sql/dml.md.txt +++ b/_sources/user-guide/sql/dml.md.txt @@ -136,3 +136,94 @@ INSERT INTO table_name { VALUES ( expression [, ...] | 2 | +-------+ ``` + +## DELETE + +Removes rows from a table. + +
+DELETE FROM table_name [ WHERE condition ]
+
+ +`DELETE` returns the number of removed rows in a column named `count`. + +If you omit the `WHERE` clause, DataFusion removes all rows. + +DataFusion removes a row only if the condition is true for that row. SQL three-valued logic applies: if the condition evaluates to `NULL`, the row remains. For example, `WHERE value > 15` keeps a row with a `NULL` value, because `NULL > 15` is `NULL`. + +Not all tables support `DELETE`. See [Table support for DELETE and UPDATE](#table-support-for-delete-and-update). + +### Examples + +Remove the rows that match a condition: + +```sql +> DELETE FROM target_table WHERE id > 1; ++-------+ +| count | ++-------+ +| 2 | ++-------+ +``` + +Remove all rows: + +```sql +> DELETE FROM target_table; ++-------+ +| count | ++-------+ +| 3 | ++-------+ +``` + +## UPDATE + +Changes the values of existing rows. + +
+UPDATE table_name SET column = expression [, ...] [ WHERE condition ]
+
+ +`UPDATE` returns the number of affected rows in a column named `count`. + +If you omit the `WHERE` clause, DataFusion changes all rows. The three-valued logic of `DELETE` also applies here. + +Each assignment expression reads the row values from before the statement. `SET a = b, b = a` therefore exchanges the two values. + +Not all tables support `UPDATE`. See [Table support for DELETE and UPDATE](#table-support-for-delete-and-update). + +### Examples + +Set one column in the rows that match a condition: + +```sql +> UPDATE target_table SET name = 'Baz' WHERE id = 2; ++-------+ +| count | ++-------+ +| 1 | ++-------+ +``` + +Set two columns, one from an expression: + +```sql +> UPDATE target_table SET value = value * 2, name = 'Doubled' WHERE id < 3; ++-------+ +| count | ++-------+ +| 2 | ++-------+ +``` + +## Table support for DELETE and UPDATE + +Not all table providers support `DELETE` and `UPDATE`. In-memory tables created with `CREATE TABLE` support both statements. File-based tables created with `CREATE EXTERNAL TABLE` and views do not. Support for custom table providers depends on the provider. + +### Known limitations + +- Subqueries in `DELETE` and `UPDATE` conditions can affect unintended rows: [#24654](https://github.com/apache/datafusion/issues/24654). +- `EXPLAIN DELETE` and `EXPLAIN UPDATE` can modify in-memory tables: [#24656](https://github.com/apache/datafusion/issues/24656). +- `DELETE` ignores `LIMIT`: [#24998](https://github.com/apache/datafusion/issues/24998). +- `UPDATE ... FROM` is not supported: [#19950](https://github.com/apache/datafusion/issues/19950). diff --git a/_sources/user-guide/sql/scalar_functions.md.txt b/_sources/user-guide/sql/scalar_functions.md.txt index bdc8efb1bd15d..1ba1e8c820005 100644 --- a/_sources/user-guide/sql/scalar_functions.md.txt +++ b/_sources/user-guide/sql/scalar_functions.md.txt @@ -1663,7 +1663,7 @@ trim(LEADING trim_str FROM str) ### `octet_length` -Returns the length of a string in bytes. +Returns the length of a string or binary in bytes. ```sql octet_length(str) @@ -1671,7 +1671,7 @@ octet_length(str) #### Arguments -- **str**: String expression to operate on. Can be a constant, column, or function, and any combination of operators. +- **str**: String or binary expression to operate on. Can be a constant, column, or function, and any combination of operators. #### Example @@ -2563,6 +2563,11 @@ date_part(part, expression) - doy (day of the year) - epoch (seconds since Unix epoch for timestamps/dates, total seconds for intervals) - isodow (ISO 8601 day of the week where Monday is 1 and Sunday is 7) + - timezone (UTC offset in seconds) + - timezone_hour (whole hours of the UTC offset) + - timezone_minute (whole minutes of the UTC offset, excluding the hours) + + The `timezone`, `timezone_hour` and `timezone_minute` parts are only defined for timestamps that carry a timezone; extracting them from a timezone-naive timestamp, a date, a time or an interval is an error. They report the offset that applies at that instant, so they follow daylight saving time: `Europe/Brussels` yields `3600` in January and `7200` in July. For a negative offset each non-zero part carries the sign, so `America/St_Johns` in January yields `-3` hours and `-30` minutes. An offset smaller than one hour has a zero hour part, which cannot show a sign: `Africa/Monrovia` before 1972 yields `0` hours and `-43` minutes. - **expression**: Time expression to operate on. Can be a constant, column, or function. @@ -2581,6 +2586,12 @@ date_part(part, expression) +----------------------------------------------------+ | 1 | +----------------------------------------------------+ +> SELECT date_part('timezone', TIMESTAMP '2024-07-01T12:00:00' AT TIME ZONE 'Europe/Brussels') AS utc_offset_seconds; ++--------------------+ +| utc_offset_seconds | ++--------------------+ +| 7200 | ++--------------------+ ``` #### Alternative Syntax @@ -2659,7 +2670,13 @@ _Alias of [date_trunc](#date_trunc)._ ### `from_unixtime` -Converts an integer to RFC3339 timestamp format (`YYYY-MM-DDT00:00:00.000000000Z`). Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`) return the corresponding timestamp. +Converts an integer to a timestamp with second precision (`Timestamp(Second)`). +The integer is interpreted as the number of seconds since the unix epoch +(`1970-01-01T00:00:00Z`). + +If the optional `timezone` argument is omitted, the timestamp is returned in the +session time zone (`datafusion.execution.time_zone`), which is unset (i.e. +timezone-naive) by default. ```sql from_unixtime(expression[, timezone]) @@ -2668,7 +2685,7 @@ from_unixtime(expression[, timezone]) #### Arguments - **expression**: The expression to operate on. Can be a constant, column, or function, and any combination of operators. -- **timezone**: Optional timezone to use when converting the integer to a timestamp. If not provided, the default timezone is UTC. +- **timezone**: Optional timezone to use when converting the integer to a timestamp. If not provided, the session time zone (`datafusion.execution.time_zone`) is used, which is unset (timezone-naive) by default. #### Example @@ -2679,6 +2696,15 @@ from_unixtime(expression[, timezone]) +-----------------------------------------------------------+ | 2020-09-08T09:42:29-04:00 | +-----------------------------------------------------------+ + +-- Without an explicit timezone the session time zone is used +> SET datafusion.execution.time_zone = 'America/New_York'; +> select from_unixtime(1599572549); ++----------------------------------+ +| from_unixtime(Int64(1599572549)) | ++----------------------------------+ +| 2020-09-08T09:42:29-04:00 | ++----------------------------------+ ``` ### `make_date` @@ -4832,7 +4858,7 @@ _Alias of [inner_product](#inner_product)._ ### `empty` -Returns 1 for an empty array or 0 for a non-empty array. +Returns true for an empty array or false for a non-empty array. ```sql empty(array) @@ -4849,7 +4875,7 @@ empty(array) +------------------+ | empty(List([1])) | +------------------+ -| 0 | +| false | +------------------+ ``` diff --git a/_sources/user-guide/sql/select.md.txt b/_sources/user-guide/sql/select.md.txt index af442de6597c1..2cbe26fee3964 100644 --- a/_sources/user-guide/sql/select.md.txt +++ b/_sources/user-guide/sql/select.md.txt @@ -319,6 +319,7 @@ SELECT a FROM table_name WHERE a > 10; ```text from_item [join_type] JOIN from_item [join_condition] +from_item ASOF JOIN from_item MATCH_CONDITION (condition) [join_condition] from_item CROSS JOIN from_item from_item NATURAL JOIN from_item from_item [join_type] JOIN LATERAL (query) AS alias [join_condition] @@ -400,6 +401,48 @@ SELECT * FROM x LEFT JOIN x AS y ON x.column_1 = y.column_2; +----------+----------+----------+----------+ ``` +### ASOF JOIN + +DataFusion follows the +[Snowflake `ASOF JOIN` syntax](https://docs.snowflake.com/en/sql-reference/constructs/asof-join). +An `ASOF JOIN` matches each left row with at most one right row according to an +ordered comparison. It preserves every left row and fills the right columns +with `NULL` when no right row matches. + +```sql +SELECT t.*, p.price +FROM trades AS t +ASOF JOIN prices AS p +MATCH_CONDITION (t.ts >= p.ts) +ON t.symbol = p.symbol; +``` + +`MATCH_CONDITION` must compare an expression from the left input with an +expression from the right input using one of the following operators. Operand +order is significant: the left input expression must appear on the left. + +| Condition | Selected right row | +| --------- | ----------------------------------------- | +| `l >= r` | Greatest `r` less than or equal to `l` | +| `l > r` | Greatest `r` strictly less than `l` | +| `l <= r` | Smallest `r` greater than or equal to `l` | +| `l < r` | Smallest `r` strictly greater than `l` | + +An optional `ON` clause containing equality conditions combined with `AND`, or +a `USING` clause, divides rows into equality groups before the ordered match. +An unqualified `USING` key appears once in wildcard output, while both qualified +input keys remain addressable. + +Without equality keys, all rows belong to one group. The initial execution +strategy collects one ordered right partition and shares it across every left +partition, so output partitioning follows the left input. The complete right +input must fit in memory and may be scanned once per left partition; spilling +and repartitioned ASOF execution are not yet supported. + +A `NULL` in either ordered expression or in any equality key does not match. +Both inputs must be bounded. If multiple right rows have the same equality keys +and ordered value, which tied row is selected is nondeterministic. + ### RIGHT OUTER JOIN The keywords `RIGHT JOIN` or `RIGHT OUTER JOIN` define a join that includes all rows from the right table even if there diff --git a/contributor-guide/api-health.html b/contributor-guide/api-health.html index d239b89b5e14f..b5c0f4a6d3d00 100644 --- a/contributor-guide/api-health.html +++ b/contributor-guide/api-health.html @@ -471,6 +471,34 @@

What is the public Rust API and what is a breaking API change?trait with a default implementation

+
+

DataFusion-internal Public APIs#

+

Some internal implementation details require pub visibility because they are +shared across DataFusion crates. These APIs are not intended as downstream +extension points. They are hidden from generated documentation and are not part +of DataFusion’s supported public API, so they may be changed or removed without +notice or a deprecation period. Examples include:

+
    +

  1. Test helpers.

    2. +

  2. Operator APIs required by the optimizer to inspect or rewrite execution plans +across crate boundaries.

    3. +
+

For APIs intended only for internal use, add #[doc(hidden)] and a doc comment +section headed # Public Only for Internal Use:. Name the crate or component +that requires access and explain why the API is not intended for downstream use. +For example:

+
impl HashTableLookupExpr {
+    /// ...
+    ///
+    /// # Public Only for Internal Use:
+    /// `datafusion-proto` tests require this constructor, but it is not part of
+    /// the supported public API.
+    #[doc(hidden)]
+    pub fn new(...) {...}
+}
+
+
+

What is the public SQL API and what is a breaking SQL change?#

DataFusion is also used as a SQL engine, so changes to SQL semantics (the @@ -591,6 +619,7 @@

Deprecation Guidelines
  • Breaking API Changes +

    Debugging:

    +

    The standard dev profile, used by cargo build and cargo test, disables variable-level DWARF debug information that is required by interactive debuggers.

    +

    If you need to step through the code with a debugger, build with:

    +

    CARGO_PROFILE_DEV_DEBUG=2 cargo build

    +

    Alternatively, you can set CARGO_PROFILE_DEV_DEBUG=2 as an environment variable in your IDE.

    +

    This enables the additional debug information required for source-level debugging.

    +

    For more details, see the DataFusion Cargo.toml configuration.

    • diff --git a/contributor-guide/gsoc/gsoc_project_ideas_2025.html b/contributor-guide/gsoc/gsoc_project_ideas_2025.html index ea16376fff9a6..aede226246021 100644 --- a/contributor-guide/gsoc/gsoc_project_ideas_2025.html +++ b/contributor-guide/gsoc/gsoc_project_ideas_2025.html @@ -434,7 +434,7 @@

    GSoC Project Ideas (2025)#

    Introduction#

    -

    Welcome to the Apache DataFusion Google Summer of Code (GSoC) 2025 project ideas list. Below you can find information about the projects. Please refer to this page for application guidelines.

    +

    Welcome to the Apache DataFusion Google Summer of Code (GSoC) 2025 project ideas list. Below you can find information about the projects. Please refer to this page for application guidelines.

    Projects#

    diff --git a/contributor-guide/index.html b/contributor-guide/index.html index c6c7b18699069..04ff028490f57 100644 --- a/contributor-guide/index.html +++ b/contributor-guide/index.html @@ -536,7 +536,13 @@

    Before Submitting a PR# ./dev/rust_lint.sh --write --allow-dirty -

    You should also run any relevant commands from the testing quick start.

    +

    Please ensure your PR follows the testing guide. In particular:

    +

    Conventional Commits & Labeling PRs#

    diff --git a/contributor-guide/pr_review.html b/contributor-guide/pr_review.html index cd2999838e990..1cebf7ba0cbfd 100644 --- a/contributor-guide/pr_review.html +++ b/contributor-guide/pr_review.html @@ -545,7 +545,8 @@

    Review the Test Coverage

  • Prefer sqllogictest (.slt) tests or DataFrame API tests where possible, as they exercise user-visible behavior and are less coupled -to internal implementation details than unit tests.

    • +to internal implementation details than unit tests (see +    Choosing What Kind of Test to Write).

  • Verify tests cover edge cases and common failure scenarios, not just the common successful path. However, it is NOT necessary to test every possible error path, especially if it is difficult to trigger or unlikely to occur in diff --git a/contributor-guide/testing.html b/contributor-guide/testing.html index 8a3b2a9c7f8d6..44c5df3f9ad40 100644 --- a/contributor-guide/testing.html +++ b/contributor-guide/testing.html @@ -486,6 +486,27 @@

    Testing Overview +

    Choosing What Kind of Test to Write#

    +

    When adding tests, prefer tests that exercise user-visible behavior over +tests of internal implementation details. Doing so means your tests are more +likely to keep working (and keep catching regressions) while the code is +refactored. In rough order of preference:

    +
      +

    1. Use sqllogictest (.slt) tests for any behavior that can +be expressed in SQL.

      2. +

    2. End-to-end tests that use public APIs such as the DataFrame API, for behavior +that cannot be expressed in SQL. For example, DataFrame tests live in +datafusion/core/tests/dataframe and run as part of the core_integration +test binary:

      +
      cargo test -p datafusion --test core_integration -- dataframe
+
      +
      +
      3. +

    3. Rust unit tests for logic that cannot practically be reached through +SQL or DataFrame public APIs.

      4. +
    +

    • Unit tests#

    Tests for code in an individual module are defined in the same source file with a test module, following Rust convention.

    @@ -510,7 +531,7 @@

    sqllogictests Testssqllogictests may be less convenient for new contributors who are familiar with writing .rs tests as they require learning another tool. However, sqllogictest based tests are much easier to develop and maintain as they 1) do not require a slow recompile/link cycle and 2) can be automatically updated.

    Like similar systems such as DuckDB, DataFusion has chosen to trade off a slightly higher barrier to contribution for longer term maintainability.

    -

    DataFusion has integrated sqlite’s test suite as a supplemental test suite that is run whenever a PR is merged into DataFusion. To run it manually please refer to the README file for instructions.

    +

    DataFusion runs sqlite’s test suite in the merge queue before merging PRs into main. For local instructions, see Running Tests: sqlite.

    Snapshot testing (cargo insta)#

    @@ -524,13 +545,12 @@

    Snapshot testing (

    Extended Tests#

    -

    In addition to the standard CI test suite that is run on all PRs prior to merge, -DataFusion has “extended” tests (defined in extended.yml) that are run on each -commit to main. These tests rarely fail but take significantly longer to run -than the standard test suite and add important test coverage such as ensuring -correctness when there are hash collisions and running the relevant portions of -the entire sqlite test suite. You can run the extended tests -locally by following the instructions in the documentation.

    +

    DataFusion has extended tests (defined in extended.yml) that take significantly +longer to run than the standard suite. They provide additional correctness +coverage and must pass in the merge queue before a PR merges into main.

    +

    To conserve CI resources, these tests do not run on ordinary PR updates. They +also run on pushes to release branches (branch-*). You can run them manually.

    +

    For local SQLite test instructions, see the instructions in the documentation.

    Rust Integration Tests#

    @@ -550,7 +570,7 @@

    SQL “Fuzz” testing

    Documentation Examples#

    We use Rust doctest to verify examples from the documentation are correct and -up-to-date. These tests are run as part of our CI and you can run them them +up-to-date. These tests are run as part of our CI and you can run them locally with the following command:

    cargo test --doc
    @@ -559,6 +579,18 @@

    Documentation ExamplesAPI Documentation Examples#

    As with other Rust projects, examples in doc comments in .rs files are automatically checked to ensure they work and evolve along with the code.

    +

    Prefix a line with # (a # followed by a space) to keep it compiled and +run, but hidden from the rendered documentation. This is most useful for +use statements, since readers care about the API being demonstrated, not +its import boilerplate:

    +
    ///
+```
+/// # use std::sync::Arc;
+/// let value = Arc::new(42);
+/// ```
+
    +
    +

    Only the let value = ... line renders.

    User Guide Documentation#

    @@ -569,7 +601,11 @@

    User Guide Documentation

    Documentation Link Checks#

    -

    Run the internal markdown link check locally:

    +

    ./dev/rust_lint.sh runs the internal markdown link check. If lychee is +missing, the script installs the version pinned in +ci/scripts/utils/tool_versions.sh. It uses an existing installation as is, +even if the version differs from the pin.

    +

    To run the check on its own:

    source ci/scripts/utils/tool_versions.sh
 cargo install lychee --locked --version "${LYCHEE_VERSION}"
 bash ci/scripts/markdown_link_check.sh
@@ -579,6 +615,7 @@ 
    Documentation Link Checks
 
  • The script is run with bash and is compatible with the default Bash on macOS (no mapfile dependency).
    • 
 
  • The CI configuration currently checks internal markdown links only. External http(s) and mailto links are excluded to avoid flaky failures.
    • 
+
  • The check only reports broken links. ./dev/rust_lint.sh --write does not change them.
    • 
 
 
    When a link is broken, lychee prints the file and URL/path that failed. For example:
    
 
    [docs/source/user-guide/cli/overview.md]:
@@ -590,6 +627,91 @@ 
    Documentation Link Checks
+
    ASF Status Check Validation#
    
+
    ci/scripts/check_asf_yaml_status_checks.py checks that every required status
+check in .asf.yaml matches a job in .github/workflows, and that rust.yml
+skips only its listed jobs on pushes to main. ./dev/rust_lint.sh runs it
+and needs python3 with PyYAML. The uv workspace provides both:
    
+
    uv run ./dev/rust_lint.sh
+
    
+
    
+
    To run the check on its own:
    
+
    uv run python3 ci/scripts/check_asf_yaml_status_checks.py
+
    
+
    
+

    +
    +

    Security Audit#

    +

    ci/scripts/security_audit.sh runs cargo audit on the root Cargo.lock with +the advisory exceptions that CI uses. ./dev/rust_lint.sh runs it and installs +cargo-audit if it is missing. To run the audit on its own:

    +
    ./ci/scripts/security_audit.sh
+
    +
    +

    The audit fetches the RustSec advisory database. A new advisory or a different +cargo-audit version can change the result without any change to the repository.

    +
    +
    +

    Large File Check#

    +

    ci/scripts/check_large_files.sh fails if any file committed between a base +ref and a head ref is larger than 1.5 MB, the same check the “Large files PR +check” workflow runs on pull requests. ./dev/rust_lint.sh runs it against +the merge base of HEAD and main on the remote that points at +apache/datafusion, or on origin when there is no such remote. To run the +check on its own, or against a different range:

    +
    ./ci/scripts/check_large_files.sh
+./ci/scripts/check_large_files.sh --base upstream/main --head my-branch
+
    +
    +

    Only committed files are checked. Commit a change before running it.

    +
    +
    +

    Dependency Checks#

    +

    CI runs two dependency checks, and ./dev/rust_lint.sh runs both:

    +
      +

    • ci/scripts/check_circular_dependencies.sh builds and runs dev/depcheck, +which fails on dependency cycles between DataFusion crates.

      • +

    • ci/scripts/check_unused_dependencies.sh runs cargo machete --with-metadata +from the repository root. The lint suite installs cargo-machete with the +version in ci/scripts/utils/tool_versions.sh if it is missing.

      • +
    +

    To run either check on its own:

    +
    ./ci/scripts/check_circular_dependencies.sh
+./ci/scripts/check_unused_dependencies.sh
+
    +
    +
    +
    +

    Examples README Check#

    +

    datafusion-examples/README.md is generated from the documentation comments in +datafusion-examples/examples/<group>/main.rs. ci/scripts/check_examples_docs.sh +regenerates it and fails if the committed file differs. ./dev/rust_lint.sh +runs it and needs cargo and npx. To run the check on its own, or to update +the README:

    +
    ./ci/scripts/check_examples_docs.sh
+./ci/scripts/check_examples_docs.sh --write
+
    +
    +
    +
    +

    Config and Function Docs Check#

    +

    configs.md, aggregate_functions.md, scalar_functions.md, and +window_functions.md are generated by dev/update_config_docs.sh and +dev/update_function_docs.sh. To check that they are up to date, run the +ci/scripts/check_generated_docs.sh script, which also runs as part of +./dev/rust_lint.sh. Run it with --write to update the pages.

    +
    +
    +

    Documentation HTML Build#

    +

    ci/scripts/check_docs_html.sh builds the documentation website, the same +build the “Test doc build” job runs. Sphinx builds with -W, so one warning +fails the check. ./dev/rust_lint.sh runs it. Run the script with --help +for the tools it needs and the files it writes.

    +
    ./ci/scripts/check_docs_html.sh
+
    +
    +

    Benchmarks#

    @@ -684,6 +806,7 @@

    Upstream Benchmark Suites
  • Testing Quick Start
  • Testing Overview
    • +
  • Choosing What Kind of Test to Write
  • Unit tests
  • sqllogictests Tests
  • Snapshot testing (cargo insta)
    • @@ -696,6 +819,13 @@

    Upstream Benchmark SuitesDocumentation Link Checks +
  • ASF Status Check Validation
    • +
  • Security Audit
    • +
  • Large File Check
    • +
  • Dependency Checks
    • +
  • Examples README Check
    • +
  • Config and Function Docs Check
    • +
  • Documentation HTML Build
  • Benchmarks
    • Criterion Benchmarks
    • Parquet SQL Benchmarks
      • diff --git a/download.html b/download.html index 9bd8054b09854..44da789d6081b 100644 --- a/download.html +++ b/download.html @@ -452,7 +452,7 @@

      Download in their Cargo.toml file and downloading it from the Rust crates.io package registry.

      For example:

      [dependencies]
-datafusion = "55.0.0"
+datafusion = "55.1.0"

      While DataFusion is distributed via crates.io as a convenience, the diff --git a/library-user-guide/catalogs.html b/library-user-guide/catalogs.html index 310a6a0dbfc64..c8d015299cc70 100644 --- a/library-user-guide/catalogs.html +++ b/library-user-guide/catalogs.html @@ -642,6 +642,65 @@

      Implementing Me

      Like other traits, it also maintains the mapping of the Catalog’s name to the CatalogProvider.

    • +
    +

    Catalog Provider Factories#

    +

    The catalogs above are all registered programmatically, ahead of time, +before a query ever runs. Sometimes it is useful to let users attach a +catalog dynamically from SQL instead — for example, a catalog backed by a +remote catalog service such as an Iceberg REST catalog. This is exactly +analogous to how TableProviderFactory lets CREATE EXTERNAL TABLE +create a TableProvider “on the fly”: CatalogProviderFactory lets +CREATE EXTERNAL CATALOG ... STORED AS <catalog_type> ... create a +CatalogProvider on the fly.

    +

    To support CREATE EXTERNAL CATALOG, implement CatalogProviderFactory:

    +
    use std::sync::Arc;
+use async_trait::async_trait;
+use datafusion::catalog::{CatalogProvider, CatalogProviderFactory, MemoryCatalogProvider, Session};
+use datafusion::common::Result;
+use datafusion::logical_expr::CreateExternalCatalog;
+
+#[derive(Debug)]
+struct MyCatalogProviderFactory {}
+
+#[async_trait]
+impl CatalogProviderFactory for MyCatalogProviderFactory {
+    async fn create(
+        &self,
+        _state: &dyn Session,
+        cmd: &CreateExternalCatalog,
+    ) -> Result<Arc<dyn CatalogProvider>> {
+        // `cmd.location` and `cmd.options` carry whatever was supplied in the
+        // `LOCATION` and `OPTIONS` clauses of the SQL statement; a real
+        // implementation would use them to connect to the remote catalog
+        // service. This example just returns an empty in-memory catalog.
+        Ok(Arc::new(MemoryCatalogProvider::new()))
+    }
+}
+
    +
    +

    Then register it on the SessionState, keyed by the STORED AS value that +should resolve to it:

    +
    use datafusion::execution::session_state::SessionStateBuilder;
+
+let state = SessionStateBuilder::new()
+    .with_default_features()
+    .with_catalog_factory("MY_CATALOG_TYPE".to_string(), Arc::new(MyCatalogProviderFactory {}))
+    .build();
+
    +
    +

    With the factory registered, users can attach and detach the catalog from +SQL:

    +
    CREATE EXTERNAL CATALOG my_catalog
+STORED AS MY_CATALOG_TYPE
+LOCATION 's3://bucket/warehouse'
+OPTIONS ('warehouse' 'my_catalog');
+
+DROP CATALOG my_catalog;
+
    +
    +

    See DDL: CREATE EXTERNAL CATALOG +for the full SQL syntax.

    +

    Recap#

    To recap, you need to:

    @@ -708,6 +767,7 @@

    Recap#<
  • Implementing MemoryCatalogProvider
  • Implementing MemoryCatalogProviderList
    • +
  • Catalog Provider Factories
  • Recap
    • diff --git a/library-user-guide/custom-table-providers.html b/library-user-guide/custom-table-providers.html index 17db44ee1ba06..558d383b32092 100644 --- a/library-user-guide/custom-table-providers.html +++ b/library-user-guide/custom-table-providers.html @@ -1138,6 +1138,13 @@

    A Complete Filter Pushdown Exampleexecute().

    +
    +

    Row-Level DML: DELETE and UPDATE#

    +

    A custom table provider can support DELETE and UPDATE by implementing the optional TableProvider::delete_from() and TableProvider::update() methods. Their default implementations return a “not implemented” error.

    +

    Each method builds an ExecutionPlan that changes the matching rows and returns the number of affected rows in a count column. Your provider decides how to apply and persist the changes. Perform the changes when the returned plan executes: these methods run during physical planning, including for EXPLAIN.

    +

    The method documentation describes the filters, assignments, SQL semantics, and result schema that an implementation must support. MemTable provides an implementation of both methods over in-memory batches.

    +

    For SQL syntax and examples, see the DML section of the user guide.

    +

    Putting It All Together#

    Here is a minimal but complete example of a custom table provider that generates @@ -1385,6 +1392,7 @@

    Further ReadingA Complete Filter Pushdown Example +
  • Row-Level DML: DELETE and UPDATE
  • Putting It All Together
  • Using Your Table Provider
  • Further Reading
    • diff --git a/library-user-guide/extending-sql.html b/library-user-guide/extending-sql.html index 49eeb56984343..73f44c6d7faa1 100644 --- a/library-user-guide/extending-sql.html +++ b/library-user-guide/extending-sql.html @@ -708,10 +708,13 @@

    The RelationPlannerContext

    plan(relation)

    Recursively plan a nested relation

    -

    sql_to_expr(expr, schema)

    +

    get_cte(name)

    +

    Look up a CTE visible in the current query scope

    + +

    sql_to_expr(expr, schema)

    Convert SQL expression to DataFusion Expr

    -

    context_provider()

    +

    context_provider()

    Access session configuration, tables, functions

    @@ -719,6 +722,26 @@

    The RelationPlannerContextRelationPlanner API documentation for additional methods like normalize_ident() and object_name_to_table_reference().

    +

    Relation planners are called before DataFusion’s built-in CTE and catalog lookup. +If your planner recognizes ordinary relations by name, check get_cte() after +determining the relation is an ordinary name that your planner would otherwise +handle. Deliberately reserved names and custom syntax can keep their existing +precedence. Return RelationPlanning::Original when a CTE is visible so the +next planner can handle the relation. When every ordinary name-based planner in +the chain performs this check, planning reaches DataFusion’s built-in CTE +resolution. Normalize the parsed name first so the lookup follows DataFusion’s +existing name matching. Only do this for a table without function arguments: a +CTE named numbers shadows numbers, but it does not shadow a call such as +numbers(10).

    +
    // Run this after recognizing `relation` as an ordinary name handled here.
+if let TableFactor::Table { name, args: None, .. } = &relation {
+    let table_ref = ctx.object_name_to_table_reference(name.clone())?;
+    if ctx.get_cte(&table_ref).is_some() {
+        return Ok(RelationPlanning::Original(Box::new(relation)));
+    }
+}
+
    +

    Implementation Strategies#

    diff --git a/library-user-guide/functions/adding-udfs.html b/library-user-guide/functions/adding-udfs.html index 2df3be89aeda6..0a3fc34cfd656 100644 --- a/library-user-guide/functions/adding-udfs.html +++ b/library-user-guide/functions/adding-udfs.html @@ -1136,6 +1136,24 @@

    Adding an Aggregate UDF} +
    +

    Declaring how an Aggregate UDF treats DISTINCT#

    +

    By default DataFusion assumes an aggregate is sensitive to the DISTINCT modifier, which means the accumulator is expected to +read AccumulatorArgs::is_distinct and deduplicate its input. Override +AggregateUDFImpl::distinct_handling when that is not what your function does:

    +
      +

    • Return DistinctHandling::Insensitive when duplicates cannot change the result, that is, when merging a value the +accumulator has already seen is a no-op. min, max, bool_and and bit_or are all in this group. The optimizer +then plans f(DISTINCT x) as f(x), which skips both the per-group hash set and the extra grouping stage that +SingleDistinctToGroupBy would otherwise introduce.

      • +

    • Return DistinctHandling::Unsupported when the accumulator does not implement DISTINCT: it does not read +is_distinct, or it rejects DISTINCT with an error. The planner must then deduplicate the input first or reject +the query. Today this is a declaration only; rejecting such queries at planning time is a follow-up change.

      • +

    • Leave the default DistinctHandling::Sensitive when the accumulator reads AccumulatorArgs::is_distinct and +deduplicates its input itself.

      • +
    +

    Getting this wrong changes query results, so only claim Insensitive if your merge is genuinely idempotent.

    +

    Registering an Aggregate UDF#

    To register a Aggregate UDF, you need to wrap the function implementation in a AggregateUDF struct and then register @@ -1475,6 +1493,7 @@

    Implementing Custom Expression PlanningAdding an Aggregate UDF diff --git a/library-user-guide/upgrading/54.0.0.html b/library-user-guide/upgrading/54.0.0.html index e9fdf5636db0c..45f55539f5623 100644 --- a/library-user-guide/upgrading/54.0.0.html +++ b/library-user-guide/upgrading/54.0.0.html @@ -777,38 +777,6 @@

    Scalar subquery execution changesExecutionPlan trees, as well as EXPLAIN output, may need updating.

    -
    -

    Filter predicate evaluation order may differ from query text#

    -

    The logical optimizer now reorders filters so that cheap predicates (most binary -comparisons, IS NULL, Between, InList, etc.) evaluate before expensive -ones (LIKE, regex, scalar function calls, subqueries). For example, -WHERE col LIKE '%foo%' AND col2 = 5 may evaluate col2 = 5 before -col LIKE '%foo%'.

    -

    Evaluation order has never been guaranteed to match the order written in the -query. The SQL standard explicitly allows implementations to evaluate operands -in any order; major engines (PostgreSQL, SQL Server, Oracle, MySQL) document the -same. Queries should not rely on left-to-right evaluation or short-circuit -semantics for AND or OR. Previous versions of DataFusion already reordered -predicates (e.g., as part of expression simplification or predicate pushdown); -the new reordering pass just increases the scenarios where the optimizer will -change predicate evaluation order.

    -

    Fallible-predicate patterns are particularly affected. For example:

    -
    WHERE s ~ '^[0-9]+$' AND CAST(s AS INT) > 0
-
    -
    -

    The intent is likely to filter non-numeric strings before the CAST runs, -but this depends on evaluation-order behavior the SQL standard does not -guaranteed. The new reorder makes this kind of pattern more likely to fail -at runtime if the optimizer moves the CAST ahead of the regex. To force -conditional evaluation, rewrite using CASE, which has standardized -short-circuit semantics:

    -
    WHERE CASE WHEN s ~ '^[0-9]+$' THEN CAST(s AS INT) > 0 ELSE false END
-
    -
    -

    Volatile expressions (random(), now(), etc.) are exempt — their position -in the conjunct list is preserved so the number of times they evaluate per -query does not change.

    -

    datafusion-proto: expression deserialization now takes a TaskContext#

    Serializeable::from_bytes_with_registry is renamed to from_bytes_with_ctx @@ -1284,11 +1252,6 @@

    The RegisterFun

    -
    -

    New Dialect::Spark variant#

    -

    The Dialect enum in datafusion_common::config now includes a Spark variant. -If you match exhaustively on Dialect, add a Dialect::Spark arm.

    -
    @@ -1354,7 +1317,6 @@

    New Dialect::Sp
  • Avro API and timestamp decoding changes
  • lpad, rpad, and translate now operate on Unicode codepoints instead of grapheme clusters
  • Scalar subquery execution changes
    • -
  • Filter predicate evaluation order may differ from query text
  • datafusion-proto: expression deserialization now takes a TaskContext
  • datafusion-proto: PhysicalProtoConverterExtension reshaped
  • ExecutionProps has new fields
    • @@ -1377,7 +1339,6 @@

    New Dialect::Sp
  • The RegisterFunction enum has a new HigherOrder variant
    • -
  • New Dialect::Spark variant
    • diff --git a/library-user-guide/upgrading/55.0.0.html b/library-user-guide/upgrading/55.0.0.html index e9f3b7ed1774f..45ea19853282f 100644 --- a/library-user-guide/upgrading/55.0.0.html +++ b/library-user-guide/upgrading/55.0.0.html @@ -453,9 +453,38 @@

    Upgrade Guides#

    DataFusion 55.0.0#

    -

    Note: DataFusion 55.0.0 has not been released yet. The information provided -in this section pertains to features and changes that have already been merged -to the main branch and are awaiting release in this version.

    +
    +

    Filter predicate evaluation order may differ from query text#

    +

    The logical optimizer now reorders filters so that cheap predicates (most binary +comparisons, IS NULL, Between, InList, etc.) evaluate before expensive +ones (LIKE, regex, scalar function calls, subqueries). For example, +WHERE col LIKE '%foo%' AND col2 = 5 may evaluate col2 = 5 before +col LIKE '%foo%'.

    +

    Evaluation order has never been guaranteed to match the order written in the +query. The SQL standard explicitly allows implementations to evaluate operands +in any order; major engines (PostgreSQL, SQL Server, Oracle, MySQL) document the +same. Queries should not rely on left-to-right evaluation or short-circuit +semantics for AND or OR. Previous versions of DataFusion already reordered +predicates (e.g., as part of expression simplification or predicate pushdown); +the new reordering pass just increases the scenarios where the optimizer will +change predicate evaluation order.

    +

    Fallible-predicate patterns are particularly affected. For example:

    +
    WHERE s ~ '^[0-9]+$' AND CAST(s AS INT) > 0
+
    +
    +

    The intent is likely to filter non-numeric strings before the CAST runs, +but this depends on evaluation-order behavior the SQL standard does not +guaranteed. The new reorder makes this kind of pattern more likely to fail +at runtime if the optimizer moves the CAST ahead of the regex. To force +conditional evaluation, rewrite using CASE, which has standardized +short-circuit semantics:

    +
    WHERE CASE WHEN s ~ '^[0-9]+$' THEN CAST(s AS INT) > 0 ELSE false END
+
    +
    +

    Volatile expressions (random(), now(), etc.) are exempt — their position +in the conjunct list is preserved so the number of times they evaluate per +query does not change.

    +

    Map casts with nested Structs use schema-evolution rules#

    Casts of a Map whose key or value recursively contains a Struct now adapt @@ -614,6 +643,11 @@

    User SpillFile<

    Use temp_file.size() instead of RefCountedTempFile::current_disk_usage.

    +
    +

    New Dialect::Spark variant#

    +

    The Dialect enum in datafusion_common::config now includes a Spark variant. +If you match exhaustively on Dialect, add a Dialect::Spark arm.

    +

    Dialect::AVAILABLE replaced by Dialect::available()#

    datafusion_common::config::Dialect::AVAILABLE has been removed. Use @@ -1791,11 +1825,13 @@

    RANGE window frames with offsets now reject ORDER BY types without arithmeti

    +
    +

    Generated protobuf JsonWriterOptions gained a compression_level field#

    +

    The generated public protobuf JsonWriterOptions struct now carries an optional +compression_level so JSON sink plans preserve explicitly configured +compression levels across serialization.

    +

    Who is affected:

    +
      +

    • Users constructing generated JsonWriterOptions values with an exhaustive +struct literal.

      • +
    +

    Migration guide:

    +

    Set compression_level explicitly, or fill it from Default:

    +
    // Before
+JsonWriterOptions { compression }
+
+// After
+JsonWriterOptions {
+    compression,
+    compression_level: None,
+}
+// or
+JsonWriterOptions {
+    compression,
+    ..Default::default()
+}
+
    +
    +

    The protobuf wire format remains backward compatible.

    +

    See PR #24945 for details.

    +
    +
    +

    Generated protobuf Parquet structs gained state fields#

    +

    The generated public protobuf ParquetScanExecNode struct now carries an +optional metadata_size_hint, and ParquetSink now carries optional +sorting_columns. These fields preserve the corresponding Parquet source and +sink settings across physical-plan serialization.

    +

    Who is affected:

    +
      +

    • Users constructing either generated struct with an exhaustive struct literal.

      • +
    +

    Migration guide:

    +

    Add metadata_size_hint: None to ParquetScanExecNode literals and +sorting_columns: None to ParquetSink literals to retain the previous +behavior, or use ..Default::default() for unspecified fields.

    +

    The protobuf wire format remains backward compatible.

    +

    See PR #25057 for details.

    +
    +
    +

    Generated protobuf CsvWriterOptions changed#

    +

    The generated CsvWriterOptions now includes compression_level, +timestamp_tz_format, and terminator. Its four existing format fields changed +from String to Option<String> to distinguish unset from explicitly empty +formats. This affects exhaustive struct literals and direct format-field access.

    +

    Wrap existing formats in Some or use None, and initialize the new fields +(explicitly or through Default):

    +
    // Before (unchanged fields omitted)
+CsvWriterOptions {
+    date_format: "%Y-%m-%d".to_string(),
+    datetime_format: String::new(),
+    timestamp_format: String::new(),
+    time_format: String::new(),
+    // ...
+}
+
+// After
+CsvWriterOptions {
+    date_format: Some("%Y-%m-%d".to_string()),
+    datetime_format: None,
+    timestamp_format: None,
+    time_format: None,
+    compression_level: None,
+    timestamp_tz_format: None,
+    terminator: Vec::new(),
+    // ...
+}
+
    +
    +

    The protobuf wire format remains backward compatible.

    +

    See PR #25058 for details.

    +

    DefaultStatisticsProvider is deprecated#

    datafusion_physical_plan::operator_statistics::DefaultStatisticsProvider is @@ -501,6 +820,232 @@

    StatisticsRegis

    +
    +

    API change for floor and ceil UDF#

    +

    The output type of the floor and ceil UDFs has been changed from the exact input type to a rescaled type with the same bit width. For example, for input Decimal32(7,2) floor now returns Decimal32(6,0), where the new precision is p - s + 1, matching Spark’s behaviour. See [#24703] for more details.

    +

    Who is affected:

    +
      +

    • Users storing query result with these UDFs in a fixed schema

      • +

    • Users relying on arrow_typeof for these UDFs

      • +
    +

    Migration guide:

    +

    Change the expected type or wrap the expression in CAST. It’s recommended to avoid relying on decimal’s exact precision and scale.

    +
    +
    +

    map_extract / element_at return an empty list for absent keys#

    +

    map_extract (and its alias element_at) previously returned a single-element +list containing NULL when the key was not present in the map. It now returns +an empty list, matching the documented behavior and DuckDB. Two related cases +changed at the same time, also matching DuckDB:

    +
      +

    • A NULL map input now yields NULL instead of [NULL].

      • +

    • A NULL lookup key now yields [] instead of [NULL].

      • +
    +

    A key that is present with a NULL value still returns [NULL], so absent +keys and NULL values are now distinguishable.

    +

    Migration guide:

    +
    -- Before
+SELECT map_extract(MAP {'a': 1}, 'missing');  -- [NULL]
+
+-- After
+SELECT map_extract(MAP {'a': 1}, 'missing');  -- []
+
    +
    +

    Expressions that assumed the result always has exactly one element, for +example by checking its length, unnesting it, or comparing it to [NULL], +should treat an empty list as the absent-key case instead.

    +

    See issue #24981 and +issue #24983 for details.

    +
    +
    +

    DataFrame::from_columns accepts IntoIterator#

    +

    DataFrame::from_columns now accepts any IntoIterator<Item = (&str, ArrayRef)> +instead of specifically accepting a Vec<(&str, ArrayRef)>.

    +
    // Existing Vec usage continues to work
+let df = DataFrame::from_columns(vec![
+    ("id", id),
+    ("name", name),
+])?;
+
+// Arrays can now be used directly
+let df = DataFrame::from_columns([
+    ("id", id),
+    ("name", name),
+])?;
+
    +
    +

    Most existing call sites using Vec require no changes. Code that relies on +the exact non-generic function signature of DataFrame::from_columns may need +to be updated to account for the new generic API.

    +
    +
    +

    Aggregate ordering state field names are now namespaced#

    +

    Ordering expressions exposed as top-level aggregate state fields now use names +derived from the aggregate name and their ordering position.

    +

    For example:

    +
    first_value(value)[ordering_0]
+
    +
    +

    is now used instead of an unqualified ordering field name such as:

    +
    timestamp@0
+
    +
    +

    This guarantees unique aggregate state field names and allows +DFSchema::check_names() to be enforced.

    +

    Who is affected:

    +

    Users or integrations that inspect aggregate state field names directly, +including custom UDAFs and FFI integrations.

    +
    +
    +

    Physical filter pushdown resolves columns by position#

    +

    datafusion_physical_plan::filter_pushdown::ChildFilterDescription::from_child +and FilterDescription::from_children now resolve filter columns by position +instead of looking them up by name. This prevents incorrect results when a +child schema contains duplicate column names. from_child requires the child +field at each referenced position to have the same name as the filter column.

    +

    ChildFilterDescription::from_child_with_allowed_indices is deprecated but +preserves its previous name-based mapping to the first matching child field. +Migrate to from_child_with_column_mapping because name resolution is ambiguous +when the child schema contains duplicate field names.

    +

    Migration guide:

    +

    Use from_child (or from_children for multiple children) when the parent and +child schemas have matching column positions and names. When a node projects, +reorders, or renames columns, use from_child_with_column_mapping with an +explicit map from parent output indices to child input indices. Columns absent +from the mapping cannot be pushed down.

    +

    For example, if the parent outputs [a, b] and the child outputs [b, a], a +filter on a@0 must map to child column a@1:

    +
    use std::collections::{HashMap, HashSet};
+use datafusion_physical_plan::filter_pushdown::ChildFilterDescription;
+
+// Before: allow parent column 0 and resolve "a" by name in the child.
+let description = ChildFilterDescription::from_child_with_allowed_indices(
+    &parent_filters,
+    HashSet::from([0]),
+    &child,
+)?;
+
+// After: explicitly map parent column 0 to child column 1.
+let description = ChildFilterDescription::from_child_with_column_mapping(
+    &parent_filters,
+    HashMap::from([(0, 1)]),
+    &child,
+)?;
+
    +
    +
    +
    +

    Default datafusion.sql_parser.recursion_limit raised from 50 to 51#

    +

    DataFusion now uses sqlparser 0.63.0, which adds recursion guards to more +parse functions (for example, data type and INTERVAL parsing). As a result, +the same SQL statement consumes slightly more of the parser recursion budget +than it did with previous sqlparser versions. In particular, every expression +now needs one additional level of depth.

    +

    To avoid rejecting queries that previously parsed successfully, the default +value of the datafusion.sql_parser.recursion_limit configuration option, and +of DFParserBuilder::with_recursion_limit, has been raised from 50 to 51.

    +

    If you set datafusion.sql_parser.recursion_limit explicitly to a value close +to the depth of your queries, you may see RecursionLimitExceeded errors after +upgrading and should raise the limit accordingly.

    +

    See PR #25278 for details.

    +
    +
    +

    ensure_distribution is deprecated in favour of ensure_distribution_with_stats#

    +

    datafusion_physical_optimizer::enforce_distribution::ensure_distribution now +takes its StatisticsContext from the caller, so one context (and its +memoization cache) can be shared across a whole bottom-up traversal. The old +two-argument form is kept as a deprecated wrapper that allocates a fresh +context per call, which recomputes each shared subtree’s statistics once per +ancestor.

    +
    // Before
+let plan = DistributionContext::new_default(plan)
+    .transform_up(|ctx| ensure_distribution(ctx, config))
+    .data()?;
+
+// After: one context for the whole walk. `StatsCache` is keyed by raw plan-node
+// pointers, so reset it whenever a node's plan pointer actually changed.
+let stats_ctx = StatisticsContext::new();
+let plan = DistributionContext::new_default(plan)
+    .transform_up(|ctx| {
+        let before = Arc::clone(&ctx.plan);
+        let result = ensure_distribution_with_stats(ctx, config, &stats_ctx)?;
+        if !Arc::ptr_eq(&before, &result.data.plan) {
+            stats_ctx.reset_cache();
+        }
+        Ok(result)
+    })
+    .data()?;
+
    +
    +

    Callers that only need the previous behaviour can keep using the deprecated +form, or pass a freshly constructed StatisticsContext per call.

    +
    +
    +

    MergeIntoOp requires the SQL-visible target qualifier#

    +

    MergeIntoOp now stores the SQL-visible target qualifier separately from the +target provider identity. This prevents an aliased target from being confused +with a source relation that uses the target table’s name.

    +

    The struct is now non-exhaustive. Replace 55.0 struct literals with +MergeIntoOp::new:

    +
    // DataFusion 55.0
+let op = MergeIntoOp { on, clauses };
+
+// DataFusion 56.0
+let op = MergeIntoOp::new(target_qualifier, on, clauses);
+
    +
    +

    target_qualifier must be the relation name referenced by the MERGE expressions: +the target alias when one is present, otherwise the target table reference. +DmlStatement::table_name remains the provider/catalog identity.

    +
    +
    +

    MergeIntoOpNode protobuf literals require target_qualifier#

    +

    The generated MergeIntoOpNode type has a new optional field. Code constructing +the generated type with a struct literal must provide it:

    +
    // DataFusion 55.0
+let node = protobuf::MergeIntoOpNode { on, clauses };
+
+// DataFusion 56.0
+let node = protobuf::MergeIntoOpNode {
+    on,
+    clauses,
+    target_qualifier: Some(protobuf::TableReference::from(target_qualifier)),
+};
+
    +
    +

    Wire compatibility is directional:

    +
      +

    • A 56.0 reader accepts a 55.0 payload. When target_qualifier is absent, it +falls back to DmlNode.table_name, matching the 55.0 representation.

      • +

    • A 55.0 reader must not consume an alias-preserving 56.0 MERGE payload. It +ignores the unknown field but cannot preserve the qualifier required by the +expressions, which can cause resolution failure or incorrect rebinding.

      • +
    +
    +
    +

    FileSource::exact_filter controls scan equivalence properties#

    +

    FileScanConfig no longer derives equivalence properties (constant columns +and equal columns) from FileSource::filter. It uses the new +FileSource::exact_filter method instead. This method returns the part of the +filter that every output row satisfies. A filter that the source uses only for +pruning (for example, Parquet row group statistics when +datafusion.execution.parquet.pushdown_filters is false) lets +non-matching rows through, and equivalences derived from it gave wrong query +results.

    +

    exact_filter returns None by default. ParquetSource returns the pushed-down +conjuncts when pushdown_filters is enabled.

    +

    Migration guide:

    +

    If your custom FileSource applies its filter to every row, implement +exact_filter to keep the equivalence properties of the scan:

    +
    impl FileSource for MySource {
+    fn exact_filter(&self) -> Option<Arc<dyn PhysicalExpr>> {
+        self.filter()
+    }
+    // ...
+}
+
    +
    +
    @@ -552,9 +1097,35 @@

    StatisticsRegis diff --git a/user-guide/sql/index.html b/user-guide/sql/index.html index a2082cb2b33b2..856014c954368 100644 --- a/user-guide/sql/index.html +++ b/user-guide/sql/index.html @@ -479,7 +479,9 @@

    SQL ReferenceDDL