From 22243286233b738404753824f86d7b272aacb5fa Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Sat, 3 Oct 2026 17:43:58 +0800 Subject: [PATCH 1/3] test:Null-aware NOT IN behavior in decorrelate_predicate_subquery and mark join - Fixed null-aware evaluation for NULL constants in subquery comparisons within decorrelate_predicate_subquery. - Ensured NULL expressions are properly handled in LEFT ANTI JOIN filters over NOT NULL columns. - Improved handling of nullable expressions in mark joins involving NOT IN with NOT NULL keys. --- .../src/decorrelate_predicate_subquery.rs | 54 ++++++ .../test_files/null_aware_anti_join.slt | 174 ++++++++++++++++++ .../test_files/null_aware_mark_join.slt | 37 ++++ 3 files changed, 265 insertions(+) diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs index 4c6bbdc83a0ef..931bff6dc0bc9 100644 --- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs +++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs @@ -2050,6 +2050,60 @@ mod tests { ) } + /// A NULL constant has no column, but it is still nullable. The join must + /// be null-aware although the subquery column `sq.c` is not nullable + /// (). + #[test] + fn null_constant_not_in_non_nullable_subquery_is_null_aware() -> Result<()> { + let plan = LogicalPlanBuilder::from(test_table_scan()?) + .filter(not_in_subquery( + lit(ScalarValue::UInt32(None)), + test_subquery_with_name("sq")?, + ))? + .project(vec![col("test.b")])? + .build()?; + + assert_optimized_plan_equal!( + plan, + @r" + Projection: test.b [b:UInt32] + Projection: test.a, test.b, test.c [a:UInt32, b:UInt32, c:UInt32] + LeftAnti Join: Filter: __correlated_sq_1_value = __correlated_sq_1.c null_aware [a:UInt32, b:UInt32, c:UInt32, __correlated_sq_1_value:UInt32;N] + Projection: test.a, test.b, test.c, UInt32(NULL) AS __correlated_sq_1_value [a:UInt32, b:UInt32, c:UInt32, __correlated_sq_1_value:UInt32;N] + TableScan: test [a:UInt32, b:UInt32, c:UInt32] + SubqueryAlias: __correlated_sq_1 [c:UInt32] + Projection: sq.c [c:UInt32] + TableScan: sq [a:UInt32, b:UInt32, c:UInt32] + " + ) + } + + /// `test.c + 1` over a non-nullable `test.c` cannot be NULL, and neither can + /// `sq.c`, so the anti join stays plain + /// (). + #[test] + fn non_nullable_key_expr_not_in_is_not_null_aware() -> Result<()> { + let plan = LogicalPlanBuilder::from(test_table_scan()?) + .filter(not_in_subquery( + col("test.c") + lit(1u32), + test_subquery_with_name("sq")?, + ))? + .project(vec![col("test.b")])? + .build()?; + + assert_optimized_plan_equal!( + plan, + @r" + Projection: test.b [b:UInt32] + LeftAnti Join: Filter: test.c + UInt32(1) = __correlated_sq_1.c [a:UInt32, b:UInt32, c:UInt32] + TableScan: test [a:UInt32, b:UInt32, c:UInt32] + SubqueryAlias: __correlated_sq_1 [c:UInt32] + Projection: sq.c [c:UInt32] + TableScan: sq [a:UInt32, b:UInt32, c:UInt32] + " + ) + } + #[test] fn correlated_not_in_mark_join_is_null_aware_for_hashable_filter() -> Result<()> { let outer_scan = nullable_scalar_mark_scan("outer_t")?; diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt index 33e6099900203..ecdeaea46458a 100644 --- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt @@ -942,3 +942,177 @@ DROP TABLE nai_res_outer; statement ok DROP TABLE nai_res_inner; + +############# +## Regression: a NULL `NOT IN` value over a `NOT NULL` subquery column +## +## `NULL NOT IN (non-empty set)` is UNKNOWN, so a `WHERE` clause must remove +## every row, even when no column in the comparison is nullable. The join must +## be null-aware when the value *expression* can be NULL: a NULL constant +## (https://github.com/apache/datafusion/issues/25473) or a nullable expression +## over `NOT NULL` columns (https://github.com/apache/datafusion/issues/25474). +############# + +statement ok +CREATE TABLE nanull_o(x INT NOT NULL) AS VALUES (1), (2); + +statement ok +CREATE TABLE nanull_on(x INT) AS VALUES (1), (2); + +statement ok +CREATE TABLE nanull_i(id INT NOT NULL) AS VALUES (1), (3); + +statement ok +CREATE TABLE nanull_ie(id INT NOT NULL) AS SELECT * FROM (VALUES (1)) WHERE false; + +# NULL constants: UNKNOWN for every row, so no rows. +query I +SELECT x FROM nanull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i); +---- + +query I +SELECT x FROM nanull_o WHERE NULL NOT IN (SELECT id FROM nanull_i); +---- + +query I +SELECT x FROM nanull_o WHERE NULLIF(1, 1) NOT IN (SELECT id FROM nanull_i); +---- + +query I +SELECT x FROM nanull_o WHERE NOT (CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i)); +---- + +# The nullability of the outer column does not matter. +query I +SELECT x FROM nanull_on WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i); +---- + +# Control: against an empty subquery, a NULL value gives TRUE. +query I rowsort +SELECT x FROM nanull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_ie); +---- +1 +2 + +# Control: a non-NULL constant absent from the subquery gives TRUE. +query I rowsort +SELECT x FROM nanull_o WHERE 5 NOT IN (SELECT id FROM nanull_i); +---- +1 +2 + +# The NULL constant is projected as the key of a null-aware anti join, and no +# filter is pushed into the subquery. +query TT +EXPLAIN SELECT x FROM nanull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i); +---- +logical_plan +01)Projection: nanull_o.x +02)--LeftAnti Join: __correlated_sq_1_value = __correlated_sq_1.id null_aware +03)----Projection: nanull_o.x, Int32(NULL) AS __correlated_sq_1_value +04)------TableScan: nanull_o projection=[x] +05)----SubqueryAlias: __correlated_sq_1 +06)------TableScan: nanull_i projection=[id] +physical_plan +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(__correlated_sq_1_value@1, id@0)], projection=[x@0], null_aware +02)--ProjectionExec: expr=[x@0 as x, NULL as __correlated_sq_1_value] +03)----DataSourceExec: partitions=1, partition_sizes=[1] +04)--DataSourceExec: partitions=1, partition_sizes=[1] + +# Nullable expressions over `NOT NULL` columns: NULL for every row, so no rows. +query I +SELECT x FROM nanull_o WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nanull_i); +---- + +query I +SELECT x FROM nanull_o WHERE x + CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i); +---- + +# Control: the same expression over a nullable column. +query I +SELECT x FROM nanull_on WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nanull_i); +---- + +# Control: against an empty subquery, a NULL value gives TRUE. +query I rowsort +SELECT x FROM nanull_o WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nanull_ie); +---- +1 +2 + +# Control: a nullable expression on the subquery side. +query I +SELECT x FROM nanull_o WHERE x NOT IN (SELECT CASE WHEN id > 100 THEN id END FROM nanull_i); +---- + +query TT +EXPLAIN SELECT x FROM nanull_o WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nanull_i); +---- +logical_plan +01)LeftAnti Join: CASE WHEN nanull_o.x > Int32(100) THEN Int64(1) END = __correlated_sq_1.nanull_i.id null_aware +02)--TableScan: nanull_o projection=[x] +03)--SubqueryAlias: __correlated_sq_1 +04)----Projection: CAST(nanull_i.id AS Int64) +05)------TableScan: nanull_i projection=[id] +physical_plan +01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(nanull_i.id@0, CASE WHEN nanull_o.x > Int32(100) THEN Int64(1) END@1)], projection=[x@0], null_aware +02)--ProjectionExec: expr=[CAST(id@0 AS Int64) as nanull_i.id] +03)----DataSourceExec: partitions=1, partition_sizes=[1] +04)--ProjectionExec: expr=[x@0 as x, CASE WHEN x@0 > 100 THEN 1 END as CASE WHEN nanull_o.x > Int32(100) THEN Int64(1) END] +05)----DataSourceExec: partitions=1, partition_sizes=[1] + +# `x + 1` over `NOT NULL x` cannot be NULL, so the anti join stays plain. +query I +SELECT x FROM nanull_o WHERE x + 1 NOT IN (SELECT id FROM nanull_i); +---- +1 + +query TT +EXPLAIN SELECT x FROM nanull_o WHERE x + 1 NOT IN (SELECT id FROM nanull_i); +---- +logical_plan +01)LeftAnti Join: CAST(nanull_o.x AS Int64) + Int64(1) = __correlated_sq_1.nanull_i.id +02)--TableScan: nanull_o projection=[x] +03)--SubqueryAlias: __correlated_sq_1 +04)----Projection: CAST(nanull_i.id AS Int64) +05)------TableScan: nanull_i projection=[id] +physical_plan +01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(nanull_i.id@0, nanull_o.x + Int64(1)@1)], projection=[x@0] +02)--ProjectionExec: expr=[CAST(id@0 AS Int64) as nanull_i.id] +03)----DataSourceExec: partitions=1, partition_sizes=[1] +04)--ProjectionExec: expr=[x@0 as x, CAST(x@0 AS Int64) + 1 as nanull_o.x + Int64(1)] +05)----DataSourceExec: partitions=1, partition_sizes=[1] + +# The same results with a single partition. +statement ok +SET datafusion.execution.target_partitions = 1; + +query I +SELECT x FROM nanull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i); +---- + +query I +SELECT x FROM nanull_o WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nanull_i); +---- + +query I rowsort +SELECT x FROM nanull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_ie); +---- +1 +2 + +# The SLT runner sets target_partitions to 4, so restore that value explicitly. +statement ok +SET datafusion.execution.target_partitions = 4; + +statement ok +DROP TABLE nanull_o; + +statement ok +DROP TABLE nanull_on; + +statement ok +DROP TABLE nanull_i; + +statement ok +DROP TABLE nanull_ie; diff --git a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt index c45d87b51efc2..e0c7744a9b286 100644 --- a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt @@ -998,3 +998,40 @@ DROP TABLE nam_guard_outer; statement ok DROP TABLE nam_guard_inner; + +############# +## Regression: a NULL `NOT IN` value over a `NOT NULL` subquery column in a +## mark join (https://github.com/apache/datafusion/issues/25473, +## https://github.com/apache/datafusion/issues/25474). +## +## The mark must be NULL, not FALSE, so `NOT mark OR x = 99` keeps no row. +############# + +statement ok +CREATE TABLE nmnull_o(x INT NOT NULL) AS VALUES (1), (2); + +statement ok +CREATE TABLE nmnull_i(id INT NOT NULL) AS VALUES (1), (3); + +# NULL constant. +query I +SELECT x FROM nmnull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nmnull_i) OR x = 99; +---- + +# Nullable expression over a `NOT NULL` column. +query I +SELECT x FROM nmnull_o WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nmnull_i) OR x = 99; +---- + +# Control: the `SELECT`-list form gives a NULL mark. +query IB rowsort +SELECT x, CAST(NULL AS INT) NOT IN (SELECT id FROM nmnull_i) AS m FROM nmnull_o; +---- +1 NULL +2 NULL + +statement ok +DROP TABLE nmnull_o; + +statement ok +DROP TABLE nmnull_i; From 3d3ed4b2492bc4f37794bbdd34b1e038505f12d6 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Sat, 3 Oct 2026 17:48:44 +0800 Subject: [PATCH 2/3] test:Null-aware anti-join and mark join fixes - Fixed null-aware anti-join by simplifying expressions involving NULL in the outer query. - Improved null-aware mark join logic by properly handling NULL constants and subquery expressions. - Ensured correct nullability propagation in both anti-join and mark join test cases. --- .../test_files/null_aware_anti_join.slt | 12 +++++++++-- .../test_files/null_aware_mark_join.slt | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt index ecdeaea46458a..df6320760a100 100644 --- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt @@ -978,6 +978,11 @@ query I SELECT x FROM nanull_o WHERE NULLIF(1, 1) NOT IN (SELECT id FROM nanull_i); ---- +# `x + NULL` is simplified to a NULL constant before decorrelation. +query I +SELECT x FROM nanull_o WHERE x + CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i); +---- + query I SELECT x FROM nanull_o WHERE NOT (CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i)); ---- @@ -1024,8 +1029,10 @@ query I SELECT x FROM nanull_o WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nanull_i); ---- +# Not folded: the key keeps its column reference, so null-awareness comes from +# the nullability of the whole expression. query I -SELECT x FROM nanull_o WHERE x + CAST(NULL AS INT) NOT IN (SELECT id FROM nanull_i); +SELECT x FROM nanull_o WHERE x + (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nanull_i); ---- # Control: the same expression over a nullable column. @@ -1040,7 +1047,8 @@ SELECT x FROM nanull_o WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FR 1 2 -# Control: a nullable expression on the subquery side. +# Nullable expression on the subquery side: every inner value is NULL, so no +# rows. query I SELECT x FROM nanull_o WHERE x NOT IN (SELECT CASE WHEN id > 100 THEN id END FROM nanull_i); ---- diff --git a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt index e0c7744a9b286..0372e9ece3b43 100644 --- a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt @@ -1018,6 +1018,27 @@ query I SELECT x FROM nmnull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nmnull_i) OR x = 99; ---- +# The NULL constant is the key of a null-aware mark join. +query TT +EXPLAIN SELECT x FROM nmnull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nmnull_i) OR x = 99; +---- +logical_plan +01)Projection: nmnull_o.x +02)--Filter: NOT __correlated_sq_1.mark OR nmnull_o.x = Int32(99) +03)----Projection: nmnull_o.x, __correlated_sq_1.mark +04)------LeftMark Join: __correlated_sq_1_value = __correlated_sq_1.id null_aware +05)--------Projection: nmnull_o.x, Int32(NULL) AS __correlated_sq_1_value +06)----------TableScan: nmnull_o projection=[x] +07)--------SubqueryAlias: __correlated_sq_1 +08)----------TableScan: nmnull_i projection=[id] +physical_plan +01)FilterExec: NOT mark@1 OR x@0 = 99, projection=[x@0] +02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(__correlated_sq_1_value@1, id@0)], projection=[x@0, mark@2], null_aware +04)------ProjectionExec: expr=[x@0 as x, NULL as __correlated_sq_1_value] +05)--------DataSourceExec: partitions=1, partition_sizes=[1] +06)------DataSourceExec: partitions=1, partition_sizes=[1] + # Nullable expression over a `NOT NULL` column. query I SELECT x FROM nmnull_o WHERE (CASE WHEN x > 100 THEN 1 END) NOT IN (SELECT id FROM nmnull_i) OR x = 99; From 0a2612c1ed30bfdbddf498c161e5b49c835a3a82 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 6 Oct 2026 16:38:24 +0800 Subject: [PATCH 3/3] Update EXPLAIN blocks - Logical-plan-only was preserved. - Result tests were maintained. --- .../test_files/null_aware_anti_join.slt | 24 ++++++------------- .../test_files/null_aware_mark_join.slt | 14 +++++------ 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt index df6320760a100..2213f8bdabe6a 100644 --- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt @@ -1006,6 +1006,10 @@ SELECT x FROM nanull_o WHERE 5 NOT IN (SELECT id FROM nanull_i); 1 2 +# Assert decorrelation and null-awareness, not physical join choices. +statement ok +SET datafusion.explain.logical_plan_only = true; + # The NULL constant is projected as the key of a null-aware anti join, and no # filter is pushed into the subquery. query TT @@ -1018,11 +1022,6 @@ logical_plan 04)------TableScan: nanull_o projection=[x] 05)----SubqueryAlias: __correlated_sq_1 06)------TableScan: nanull_i projection=[id] -physical_plan -01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(__correlated_sq_1_value@1, id@0)], projection=[x@0], null_aware -02)--ProjectionExec: expr=[x@0 as x, NULL as __correlated_sq_1_value] -03)----DataSourceExec: partitions=1, partition_sizes=[1] -04)--DataSourceExec: partitions=1, partition_sizes=[1] # Nullable expressions over `NOT NULL` columns: NULL for every row, so no rows. query I @@ -1062,12 +1061,6 @@ logical_plan 03)--SubqueryAlias: __correlated_sq_1 04)----Projection: CAST(nanull_i.id AS Int64) 05)------TableScan: nanull_i projection=[id] -physical_plan -01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(nanull_i.id@0, CASE WHEN nanull_o.x > Int32(100) THEN Int64(1) END@1)], projection=[x@0], null_aware -02)--ProjectionExec: expr=[CAST(id@0 AS Int64) as nanull_i.id] -03)----DataSourceExec: partitions=1, partition_sizes=[1] -04)--ProjectionExec: expr=[x@0 as x, CASE WHEN x@0 > 100 THEN 1 END as CASE WHEN nanull_o.x > Int32(100) THEN Int64(1) END] -05)----DataSourceExec: partitions=1, partition_sizes=[1] # `x + 1` over `NOT NULL x` cannot be NULL, so the anti join stays plain. query I @@ -1084,12 +1077,9 @@ logical_plan 03)--SubqueryAlias: __correlated_sq_1 04)----Projection: CAST(nanull_i.id AS Int64) 05)------TableScan: nanull_i projection=[id] -physical_plan -01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(nanull_i.id@0, nanull_o.x + Int64(1)@1)], projection=[x@0] -02)--ProjectionExec: expr=[CAST(id@0 AS Int64) as nanull_i.id] -03)----DataSourceExec: partitions=1, partition_sizes=[1] -04)--ProjectionExec: expr=[x@0 as x, CAST(x@0 AS Int64) + 1 as nanull_o.x + Int64(1)] -05)----DataSourceExec: partitions=1, partition_sizes=[1] + +statement ok +RESET datafusion.explain.logical_plan_only; # The same results with a single partition. statement ok diff --git a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt index 0372e9ece3b43..a9fcef9ca5bd5 100644 --- a/datafusion/sqllogictest/test_files/null_aware_mark_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_mark_join.slt @@ -1018,6 +1018,10 @@ query I SELECT x FROM nmnull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nmnull_i) OR x = 99; ---- +# Assert decorrelation and null-awareness, not physical join choices. +statement ok +SET datafusion.explain.logical_plan_only = true; + # The NULL constant is the key of a null-aware mark join. query TT EXPLAIN SELECT x FROM nmnull_o WHERE CAST(NULL AS INT) NOT IN (SELECT id FROM nmnull_i) OR x = 99; @@ -1031,13 +1035,9 @@ logical_plan 06)----------TableScan: nmnull_o projection=[x] 07)--------SubqueryAlias: __correlated_sq_1 08)----------TableScan: nmnull_i projection=[id] -physical_plan -01)FilterExec: NOT mark@1 OR x@0 = 99, projection=[x@0] -02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(__correlated_sq_1_value@1, id@0)], projection=[x@0, mark@2], null_aware -04)------ProjectionExec: expr=[x@0 as x, NULL as __correlated_sq_1_value] -05)--------DataSourceExec: partitions=1, partition_sizes=[1] -06)------DataSourceExec: partitions=1, partition_sizes=[1] + +statement ok +RESET datafusion.explain.logical_plan_only; # Nullable expression over a `NOT NULL` column. query I