Merge branch 'main' into topk-mutable-borrow-slots

Author: MassivePizza

.github/dependabot.yml

@@ -62,8 +62,33 @@ updates:
         update-types:
           - "minor"
           - "patch"
+  # dev/depcheck is a standalone crate excluded from the workspace, so it is
+  # not covered by the workspace entry above
+  - package-ecosystem: cargo
+    directory: "/dev/depcheck"
+    schedule:
+      interval: weekly
+    target-branch: main
+    labels: [auto-dependencies]
+  # the wasm test app has its own npm lockfile
+  - package-ecosystem: npm
+    directory: "/datafusion/wasmtest/datafusion-wasm-app"
+    schedule:
+      interval: weekly
+    labels: [auto-dependencies]
+    groups:
+      # minor and patch bumps are grouped into a single PR
+      all-npm-deps:
+        applies-to: version-updates
+        patterns:
+          - "*"
+        update-types:
+          - "minor"
+          - "patch"
   - package-ecosystem: "github-actions"
-    directory: "/"
+    # "/" covers workflows in .github/workflows; the glob covers composite
+    # actions such as .github/actions/setup-builder
+    directories: ["/", "/.github/actions/*"]
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 10
@@ -72,8 +97,10 @@ updates:
       codeql-actions:
         patterns:
           - "github/codeql-action/*"
-  - package-ecosystem: "pip"
-    directory: "/docs"
+  # Python dependencies are managed as a uv workspace rooted at the top-level
+  # pyproject.toml with a shared uv.lock
+  - package-ecosystem: "uv"
+    directory: "/"
     schedule:
       interval: "weekly"
     labels: [auto-dependencies]

Cargo.toml

@@ -210,20 +210,71 @@ uuid = "1.23"
 zstd = { version = "0.13", default-features = false }
 
 # Keep this list sorted alphabetically.
+# See https://github.com/apache/datafusion/issues/18467 for the ongoing effort of
+# picking useful non-default lints.
 [workspace.lints.clippy]
 # https://github.com/apache/datafusion/issues/18881
 allow_attributes = "warn"
+as_ptr_cast_mut = "warn"
 assigning_clones = "warn"
+clear_with_drain = "warn"
+coerce_container_to_any = "warn"
+debug_assert_with_mut_call = "warn"
+decimal_bitwise_operands = "warn"
+default_union_representation = "warn"
+doc_include_without_cfg = "warn"
+empty_enum_variants_with_brackets = "warn"
+exit = "warn"
+flat_map_option = "warn"
+fn_to_numeric_cast_any = "warn"
+imprecise_flops = "warn"
+index_refutable_slice = "warn"
 inefficient_to_string = "warn"
+infinite_loop = "warn"
+invalid_upcast_comparisons = "warn"
+ip_constant = "warn"
+iter_filter_is_ok = "warn"
+iter_filter_is_some = "warn"
+iter_on_empty_collections = "warn"
 # Detects large stack-allocated futures that may cause stack overflow crashes (see threshold in clippy.toml)
 large_futures = "warn"
+large_include_file = "warn"
+macro_use_imports = "warn"
+manual_ilog2 = "warn"
+manual_instant_elapsed = "warn"
+manual_is_power_of_two = "warn"
+match_wild_err_arm = "warn"
+mem_forget = "warn"
+mismatching_type_param_order = "warn"
+mut_mut = "warn"
 # https://github.com/apache/datafusion/issues/18503
 needless_pass_by_value = "warn"
+negative_feature_names = "warn"
+non_zero_suggestions = "warn"
+nonstandard_macro_braces = "warn"
 or_fun_call = "warn"
+path_buf_push_overwrite = "warn"
+pathbuf_init_then_push = "warn"
+precedence_bits = "warn"
+pub_underscore_fields = "warn"
+rc_mutex = "warn"
+ref_option_ref = "warn"
+same_length_and_capacity = "warn"
+str_split_at_newline = "warn"
+string_add_assign = "warn"
+suspicious_xor_used_as_pow = "warn"
+trailing_empty_array = "warn"
+transmute_ptr_to_ptr = "warn"
+uninhabited_references = "warn"
 uninlined_format_args = "warn"
 unnecessary_lazy_evaluations = "warn"
+unnecessary_self_imports = "warn"
 unused_async = "warn"
+unused_rounding = "warn"
 used_underscore_binding = "warn"
+verbose_file_reads = "warn"
+wildcard_dependencies = "warn"
+zero_sized_map_values = "warn"
 
 [workspace.lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = [

benchmarks/src/clickbench.rs

@@ -111,9 +111,7 @@ pub struct RunOpt {
 
 /// Get the SQL file path
 pub fn get_query_path(query_dir: &Path, query: usize) -> PathBuf {
-    let mut query_path = query_dir.to_path_buf();
-    query_path.push(format!("q{query}.sql"));
-    query_path
+    query_dir.join(format!("q{query}.sql"))
 }
 
 /// Get the SQL statement from the specified query file

datafusion-cli/src/lib.rs

@@ -20,7 +20,7 @@
     html_favicon_url = "https://raw.githubusercontent.com/apache/datafusion/19fe44cf2f30cbdd63d4a4f52c74055163c6cc38/docs/logos/standalone_logo/logo_original.svg"
 )]
 #![cfg_attr(docsrs, feature(doc_cfg))]
-#![doc = include_str!("../README.md")]
+#![cfg_attr(doc, doc = include_str!("../README.md"))]
 pub const DATAFUSION_CLI_VERSION: &str = env!("CARGO_PKG_VERSION");
 
 pub mod catalog;

datafusion/common/src/cse.rs

@@ -808,7 +808,7 @@ mod test {
         ) -> HashSet<u64> {
             id_array
                 .iter_mut()
-                .flat_map(|(_, id_option)| {
+                .filter_map(|(_, id_option)| {
                     id_option.as_mut().map(|node_id| {
                         let hash = node_id.hash;
                         node_id.hash = 0;

datafusion/common/src/utils/proxy.rs

@@ -166,7 +166,7 @@ where
         if cfg!(debug_assertions) {
             // In debug mode, check that the element is not already present
             debug_assert!(
-                self.find_entry(hash, |y| y == &x).is_err(),
+                self.find(hash, |y| y == &x).is_none(),
                 "attempted to insert duplicate element into HashTableAllocExt::insert_accounted"
             );
         }

datafusion/core/benches/parquet_query_sql.rs

@@ -32,8 +32,6 @@ use parquet::file::properties::{WriterProperties, WriterVersion};
 use rand::distr::Alphanumeric;
 use rand::distr::uniform::SampleUniform;
 use rand::prelude::*;
-use std::fs::File;
-use std::io::Read;
 use std::ops::Range;
 use std::path::Path;
 use std::sync::Arc;
@@ -211,9 +209,7 @@ fn criterion_benchmark(c: &mut Criterion) {
         .unwrap();
 
     // We read the queries from a file so they can be changed without recompiling the benchmark
-    let mut queries_file = File::open("benches/parquet_query_sql.sql").unwrap();
-    let mut queries = String::new();
-    queries_file.read_to_string(&mut queries).unwrap();
+    let queries = std::fs::read_to_string("benches/parquet_query_sql.sql").unwrap();
 
     for query in queries.split(';') {
         let query = query.trim();

datafusion/core/src/lib.rs

@@ -761,7 +761,7 @@
 //! [`RecordBatch`]: arrow::array::RecordBatch
 //! [`RecordBatchReader`]: arrow::record_batch::RecordBatchReader
 //! [`Array`]: arrow::array::Array
-#![doc = include_str!("optimizer_rule_reference.md")]
+#![cfg_attr(doc, doc = include_str!("optimizer_rule_reference.md"))]
 
 extern crate core;
 #[cfg(feature = "sql")]

datafusion/core/tests/fuzz_cases/spilling_fuzz_in_memory_constrained_env.rs

@@ -373,16 +373,15 @@ async fn run_sort_preserving_merge_peak_memory_with_spilled_input(
     let mut partition_batches: Vec<Vec<RecordBatch>> = Vec::new();
 
     for stream_idx in 0..2usize {
-        // Each stream covers a non-overlapping key range so both are individually
-        // sorted: stream 0 → [0, 1000), stream 1 → [1000, 2000). When
-        // `tied_values` is set, every row of every batch in both streams
-        // instead carries the same sort key, so every comparison between the
-        // two streams is a tie.
+        // When `tied_values` is false, each stream covers a non-overlapping key range so both are
+        // individually sorted:
+        // stream 0 → [[0, 100), [200, 300),...]
+        // stream 1 → [[100, 200), [300, 400), ...]
+        //
+        // When `tied_values` is set, every row of every batch in both streams instead carries the
+        // same sort key, so every comparison between the two streams is a tie.
         let batches: Vec<RecordBatch> = (0..num_batches)
             .map(|b| {
-                // Interleave streams: stream 0 → even slots [0,200,400,...],
-                // stream 1 → odd slots [100,300,500,...] so the merge
-                // alternates between them on every batch.
                 let base = ((b * 2 + stream_idx) * num_rows_per_batch) as i32;
                 let sort_col: Int32Array = if tied_values {
                     std::iter::repeat_n(0, num_rows_per_batch).collect()
@@ -524,13 +523,7 @@ async fn run_sort_preserving_merge_peak_memory_with_spilled_input(
     // BatchBuilder needs to hold 3 Record batches simultaneously to merge two
     // streams (because a stream can cross a record batch boundary)
     // there is also one cursor needed per stream
-    let mut max_peak = 3 * ipc_batch_size + 2 * cursor_unit + converter_size;
-
-    // with round robin enabled, 2 extra cursors live in memory
-    // see https://github.com/apache/datafusion/issues/23604
-    if round_robin {
-        max_peak += 2 * cursor_unit;
-    };
+    let max_peak = 3 * ipc_batch_size + 2 * cursor_unit + converter_size;
 
     assert!(
         peak_bytes > 0,

datafusion/core/tests/physical_optimizer/pushdown_utils.rs

@@ -434,7 +434,7 @@ pub fn format_execution_plan(plan: &Arc<dyn ExecutionPlan>) -> Vec<String> {
 }
 
 fn format_lines(s: &str) -> Vec<String> {
-    s.trim().split('\n').map(|s| s.to_string()).collect()
+    s.trim().lines().map(|s| s.to_string()).collect()
 }
 
 pub fn format_plan_for_test(plan: &Arc<dyn ExecutionPlan>) -> String {