Skip to content

Commit 791660c

Browse files
authored
fix: Estimate string equality and LIKE filter selectivity (#25835)
## Which issue does this PR close? Closes #25614. ## Rationale for this change String equality, LIKE, and NOT LIKE currently receive the same fallback estimate. For a column with 1,000 rows, 200 NULLs, and 20 distinct non-null values, equality should estimate 40 matching rows and inequality 760, instead of estimating 200 for both. ## What changes are included in this PR? - Estimate direct string equality/inequality using the distinct count when available, under the existing uniform-distribution assumption. - Classify LIKE patterns as literal, all-percent, or wildcard patterns. Wildcard estimates use the configured fallback, halved for each anchored end. Recognize escaped wildcards and Arrow's literal trailing-backslash behavior. - Complement negated estimates over non-null rows and propagate zero null counts for LIKE operands. Literal ILIKE retains a heuristic because case folding can merge distinct values. - Retain existing estimates for compound or otherwise unsupported expressions. - Remove an unnecessary semicolon in the Windows rounding implementation so the required Clippy check can run without that pre-existing warning. ## What is the testing strategy for this PR? Public SQL planning tests in `datafusion/core/tests/custom_sources_cases/statistics.rs` inject known statistics through the existing table-provider fixture. They cover Utf8, LargeUtf8, Utf8View, equality/inequality, pattern forms, escaped characters, ILIKE, unknown distinct counts, and configured fallback selectivity. Direct physical-plan tests cover patterns eliminated by SQL simplification, NULL/empty inputs, and column-valued patterns. Validation: - 51 filter tests passed, including the unsupported-literal fallback. - 3 new public SQL regression tests passed. - `cargo fmt --all` passed. - Targeted physical-plan Clippy passed with warnings denied. - Ablation passed: all 3 new SQL tests fail with the original `main` filter implementation, with the expected row-count assertion failures. - TOML/Markdown formatting, license headers, typos, workflow checks, and ASF YAML checks passed. ## Are there any user-facing changes? String filter row estimates change. No public API changes. Wildcard estimates remain heuristics in the absence of string histograms.
1 parent 6a31aa8 commit 791660c

2 files changed

Lines changed: 443 additions & 7 deletions

File tree

‎datafusion/core/tests/custom_sources_cases/statistics.rs‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,128 @@ async fn sql_filter() -> Result<()> {
285285
Ok(())
286286
}
287287

288+
fn string_filter_ctx(
289+
data_type: DataType,
290+
distinct_count: Precision<usize>,
291+
) -> Result<SessionContext> {
292+
init_ctx(
293+
Statistics {
294+
num_rows: Precision::Exact(1000),
295+
total_byte_size: Precision::Absent,
296+
column_statistics: vec![ColumnStatistics {
297+
null_count: Precision::Exact(200),
298+
distinct_count,
299+
..ColumnStatistics::new_unknown()
300+
}],
301+
},
302+
Schema::new(vec![Field::new("c1", data_type, true)]),
303+
)
304+
}
305+
306+
async fn string_filter_rows(
307+
ctx: &SessionContext,
308+
predicate: &str,
309+
) -> Result<Precision<usize>> {
310+
let plan = ctx
311+
.sql(&format!("SELECT * FROM stats_table WHERE {predicate}"))
312+
.await?
313+
.create_physical_plan()
314+
.await?;
315+
Ok(StatisticsContext::new()
316+
.compute(plan.as_ref(), &StatisticsArgs::new())?
317+
.num_rows)
318+
}
319+
320+
#[tokio::test]
321+
async fn sql_string_filter_selectivity() -> Result<()> {
322+
// There are 800 non-null rows and 20 distinct strings. Equality estimates
323+
// 40 matches, and negated predicates exclude nulls as well as matches.
324+
// LIKE estimates distinguish unanchored patterns from patterns anchored
325+
// at one or both ends; escaped wildcards are literal characters.
326+
let cases = [
327+
("c1 = 'foo'", 40),
328+
("'foo' = c1", 40),
329+
("c1 != 'foo'", 760),
330+
("'foo' != c1", 760),
331+
("c1 LIKE 'foo'", 40),
332+
("c1 NOT LIKE 'foo'", 760),
333+
("c1 LIKE '%foo%'", 160),
334+
("c1 NOT LIKE '%foo%'", 640),
335+
("c1 LIKE 'foo%'", 80),
336+
("c1 NOT LIKE 'foo%'", 720),
337+
("c1 LIKE '%foo'", 80),
338+
("c1 NOT LIKE '%foo'", 720),
339+
("c1 LIKE 'f_o'", 40),
340+
("c1 NOT LIKE 'f_o'", 760),
341+
(r"c1 LIKE 'foo\%'", 40),
342+
(r"c1 NOT LIKE 'foo\%'", 760),
343+
(r"c1 LIKE 'foo\%%'", 80),
344+
(r"c1 NOT LIKE 'foo\%%'", 720),
345+
(r"c1 LIKE 'foo\'", 40),
346+
(r"c1 NOT LIKE 'foo\'", 760),
347+
(r"c1 LIKE '%foo\'", 80),
348+
(r"c1 NOT LIKE '%foo\'", 720),
349+
// Case-sensitive NDV cannot estimate a case-insensitive equality.
350+
("c1 ILIKE 'foo'", 160),
351+
("c1 NOT ILIKE 'foo'", 640),
352+
];
353+
for data_type in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] {
354+
let ctx = string_filter_ctx(data_type.clone(), Precision::Exact(20))?;
355+
for (predicate, expected_rows) in cases {
356+
assert_eq!(
357+
string_filter_rows(&ctx, predicate).await?,
358+
Precision::Inexact(expected_rows),
359+
"{data_type:?}: {predicate}"
360+
);
361+
}
362+
}
363+
Ok(())
364+
}
365+
366+
#[tokio::test]
367+
async fn sql_string_filter_without_distinct_count() -> Result<()> {
368+
let ctx = string_filter_ctx(DataType::Utf8View, Precision::Absent)?;
369+
// Without NDV, use the configured default (20%) over non-null rows.
370+
for (predicate, expected_rows) in [
371+
("c1 = 'foo'", 160),
372+
("c1 != 'foo'", 640),
373+
("c1 LIKE 'foo'", 160),
374+
("c1 NOT LIKE 'foo'", 640),
375+
] {
376+
assert_eq!(
377+
string_filter_rows(&ctx, predicate).await?,
378+
Precision::Inexact(expected_rows),
379+
"{predicate}"
380+
);
381+
}
382+
Ok(())
383+
}
384+
385+
#[tokio::test]
386+
async fn sql_string_filter_custom_selectivity() -> Result<()> {
387+
let ctx = string_filter_ctx(DataType::Utf8View, Precision::Absent)?;
388+
ctx.sql("SET datafusion.optimizer.default_filter_selectivity = 40")
389+
.await?
390+
.collect()
391+
.await?;
392+
393+
for (predicate, expected_rows) in [
394+
("c1 = 'foo'", 320),
395+
("c1 != 'foo'", 480),
396+
("c1 LIKE '%foo%'", 320),
397+
("c1 NOT LIKE '%foo%'", 480),
398+
("c1 LIKE 'foo%'", 160),
399+
("c1 NOT LIKE 'foo%'", 640),
400+
] {
401+
assert_eq!(
402+
string_filter_rows(&ctx, predicate).await?,
403+
Precision::Inexact(expected_rows),
404+
"{predicate}"
405+
);
406+
}
407+
Ok(())
408+
}
409+
288410
#[tokio::test]
289411
async fn sql_limit() -> Result<()> {
290412
let (stats, schema) = fully_defined();

0 commit comments

Comments
 (0)