Move expression analysis, rewrites and statistics pruning into vortex-expr (#10734)

Expression analysis, the rewrite transforms and statistics pruning are
only used by scan planning (vortex-layout, vortex-file, vortex-spatial
and one `replace` call in vortex-datafusion); nothing in vortex-array's
arrays or execution calls them. They move to a new `vortex-expr` crate:

- `vortex_expr::analysis` (annotations, labeling, field-access analysis,
`referenced_field_paths`), also re-exported at the crate root as before
- `vortex_expr::forms::conjuncts`
- `vortex_expr::transform::{partition_bound,
partition_bound_annotations, BoundPartitionedExpr, replace,
find_between}`
- `vortex_expr::stats`: the stat-expression helpers, `StatBinder`, the
falsify/satisfy rewrite rules and `StatsSession`, which turn filter
predicates into pruning predicates for zone maps and file stats

vortex-array keeps the statistics themselves (`StatsSet`, `ArrayStats`,
`Stat`, `Precision`) and the `StatFn` scalar function.
`vortex_array::expr::transform` is removed; `find_between` is no longer
part of `BoundExpression::optimize`.

- `BoundExpression::{falsify, satisfy}` were inherent methods; they
become the `BoundExpressionPruneExt` extension trait.
- `LikeVariant`, which only the LIKE pruning rule uses, moves with it.
- `DynamicComparisonExpr` gains `operator()`, `default()` and
`with_operator()`, since the pruning rules can no longer reach its
fields.
- The falsification tests for `is_null`, `is_not_null` and
`list_contains` move to `vortex_expr::stats::rewrite`, and the one
optimizer test that used `replace` builds the replaced expression
directly.

---------

Signed-off-by: Claude <noreply@anthropic.com>
Co-authored-by: Claude <noreply@anthropic.com>

Author: robert3005

Cargo.lock

@@ -13,6 +13,7 @@ members = [
     "vortex-io",
     "vortex-cloud",
     "vortex-array",
+    "vortex-expr",
     "vortex-arrow",
     "vortex-row",
     "vortex-tensor",
@@ -319,6 +320,7 @@ vortex-datetime-parts = { version = "0.1.0", path = "./encodings/datetime-parts"
 vortex-decimal-byte-parts = { version = "0.1.0", path = "encodings/decimal-byte-parts", default-features = false }
 vortex-edition = { version = "0.1.0", path = "./vortex-edition", default-features = false }
 vortex-error = { version = "0.1.0", path = "./vortex-error", default-features = false }
+vortex-expr = { version = "0.1.0", path = "./vortex-expr", default-features = false }
 vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false }
 vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false }
 vortex-fsst = { version = "0.1.0", path = "./encodings/fsst", default-features = false }

Cargo.toml

@@ -10,8 +10,8 @@ the proof was unknown, usually because a required stat was missing or inexact.
 
 The pruning pipeline has two phases:
 
-1. `Expression::falsify(scope, session)` asks the session's
-   `StatsRewriteRule`s to rewrite a filter into an abstract proof expression.
+1. `BoundExpressionPruneExt::falsify(session)`, from `vortex_expr::stats::rewrite`, asks the
+   session's `StatsRewriteRule`s to rewrite a bound filter into an abstract proof expression.
    Rules describe semantics in terms of `vortex.stat(input, aggregate_fn)`
    placeholders. These placeholders name the statistic needed by the proof, but
    not where that statistic is stored.

docs/developer-guide/internals/stats-pruning.md

@@ -22,12 +22,9 @@ use vortex_array::VortexSessionExecute;
 use vortex_array::aggregate_fn::session::AggregateFnSession;
 use vortex_array::arrays::PrimitiveArray;
 use vortex_array::dtype::NativePType;
-use vortex_array::dtype::session::DTypeSession;
 use vortex_array::memory::MemorySession;
 use vortex_array::optimizer::kernels::KernelSession;
-use vortex_array::scalar_fn::session::ScalarFnSession;
 use vortex_array::session::ArraySession;
-use vortex_array::stats::session::StatsSession;
 use vortex_array::validity::Validity;
 use vortex_buffer::Buffer;
 use vortex_buffer::buffer;
@@ -67,11 +64,8 @@ static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
     let session = VortexSession::empty()
         .with::<ArraySession>()
         .with::<KernelSession>()
-        .with::<DTypeSession>()
-        .with::<ScalarFnSession>()
         .with::<AggregateFnSession>()
-        .with::<MemorySession>()
-        .with::<StatsSession>();
+        .with::<MemorySession>();
     vortex_alp::initialize(&session);
     session
 });

encodings/alp/benches/alp_compress.rs

@@ -12,16 +12,10 @@ use rand::SeedableRng as _;
 use rand::rngs::StdRng;
 use vortex_array::IntoArray;
 use vortex_array::VortexSessionExecute;
-use vortex_array::aggregate_fn::session::AggregateFnSession;
 use vortex_array::arrays::PrimitiveArray;
 use vortex_array::arrays::TemporalArray;
-use vortex_array::dtype::session::DTypeSession;
 use vortex_array::extension::datetime::TimeUnit;
-use vortex_array::memory::MemorySession;
 use vortex_array::optimizer::kernels::KernelSession;
-use vortex_array::scalar_fn::session::ScalarFnSession;
-use vortex_array::session::ArraySession;
-use vortex_array::stats::session::StatsSession;
 use vortex_array::validity::Validity;
 use vortex_buffer::Buffer;
 use vortex_datetime_parts::split_temporal;
@@ -35,16 +29,8 @@ fn main() {
     divan::main();
 }
 
-static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
-    VortexSession::empty()
-        .with::<ArraySession>()
-        .with::<KernelSession>()
-        .with::<DTypeSession>()
-        .with::<ScalarFnSession>()
-        .with::<AggregateFnSession>()
-        .with::<MemorySession>()
-        .with::<StatsSession>()
-});
+static SESSION: LazyLock<VortexSession> =
+    LazyLock::new(|| VortexSession::empty().with::<KernelSession>());
 
 const BENCH_ARGS: &[(usize, TimeUnit)] = &[
     (65_536, TimeUnit::Seconds),

encodings/datetime-parts/benches/split_temporal.rs

@@ -111,15 +111,9 @@ mod arrays {
     use vortex_array::ArrayRef;
     use vortex_array::IntoArray;
     use vortex_array::VortexSessionExecute;
-    use vortex_array::aggregate_fn::session::AggregateFnSession;
     use vortex_array::dtype::DecimalDType;
     use vortex_array::dtype::DecimalType;
-    use vortex_array::dtype::session::DTypeSession;
-    use vortex_array::memory::MemorySession;
     use vortex_array::optimizer::kernels::KernelSession;
-    use vortex_array::scalar_fn::session::ScalarFnSession;
-    use vortex_array::session::ArraySession;
-    use vortex_array::stats::session::StatsSession;
     use vortex_array::validity::Validity;
     use vortex_buffer::buffer;
     use vortex_decimal_byte_parts::_benchmarking::assemble_decimal;
@@ -131,16 +125,8 @@ mod arrays {
     use crate::common::arrays::cases;
     use crate::common::arrays::decimal_array;
 
-    pub(super) static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
-        VortexSession::empty()
-            .with::<ArraySession>()
-            .with::<KernelSession>()
-            .with::<DTypeSession>()
-            .with::<ScalarFnSession>()
-            .with::<AggregateFnSession>()
-            .with::<MemorySession>()
-            .with::<StatsSession>()
-    });
+    pub(super) static SESSION: LazyLock<VortexSession> =
+        LazyLock::new(|| VortexSession::empty().with::<KernelSession>());
 
     #[divan::bench(args = cases())]
     fn dbp_assemble(bencher: Bencher, (values_type, len): (DecimalType, usize)) {

encodings/decimal-byte-parts/benches/dbp_assemble.rs

@@ -84,14 +84,8 @@ mod arrays {
     use rand::SeedableRng;
     use rand::rngs::StdRng;
     use vortex_array::VortexSessionExecute;
-    use vortex_array::aggregate_fn::session::AggregateFnSession;
     use vortex_array::dtype::DecimalType;
-    use vortex_array::dtype::session::DTypeSession;
-    use vortex_array::memory::MemorySession;
     use vortex_array::optimizer::kernels::KernelSession;
-    use vortex_array::scalar_fn::session::ScalarFnSession;
-    use vortex_array::session::ArraySession;
-    use vortex_array::stats::session::StatsSession;
     use vortex_array::validity::Validity;
     use vortex_decimal_byte_parts::split_decimal;
     use vortex_error::VortexExpect;
@@ -100,16 +94,8 @@ mod arrays {
     use crate::common::arrays::cases;
     use crate::common::arrays::decimal_array;
 
-    pub(super) static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
-        VortexSession::empty()
-            .with::<ArraySession>()
-            .with::<KernelSession>()
-            .with::<DTypeSession>()
-            .with::<ScalarFnSession>()
-            .with::<AggregateFnSession>()
-            .with::<MemorySession>()
-            .with::<StatsSession>()
-    });
+    pub(super) static SESSION: LazyLock<VortexSession> =
+        LazyLock::new(|| VortexSession::empty().with::<KernelSession>());
 
     #[divan::bench(args = cases())]
     fn dbp_split_all_valid(bencher: Bencher, (values_type, len): (DecimalType, usize)) {

encodings/decimal-byte-parts/benches/dbp_split.rs

@@ -13,12 +13,8 @@ use mimalloc::MiMalloc;
 use vortex_array::VortexSessionExecute;
 use vortex_array::aggregate_fn::session::AggregateFnSession;
 use vortex_array::arrays::PrimitiveArray;
-use vortex_array::dtype::session::DTypeSession;
-use vortex_array::memory::MemorySession;
 use vortex_array::optimizer::kernels::KernelSession;
-use vortex_array::scalar_fn::session::ScalarFnSession;
 use vortex_array::session::ArraySession;
-use vortex_array::stats::session::StatsSession;
 use vortex_fastlanes::bitpack_compress::bitpack_encode_blocked;
 use vortex_session::VortexSession;
 
@@ -34,11 +30,7 @@ static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
     let session = VortexSession::empty()
         .with::<ArraySession>()
         .with::<KernelSession>()
-        .with::<DTypeSession>()
-        .with::<ScalarFnSession>()
-        .with::<AggregateFnSession>()
-        .with::<MemorySession>()
-        .with::<StatsSession>();
+        .with::<AggregateFnSession>();
     vortex_fastlanes::initialize(&session);
     session
 });

encodings/fastlanes/benches/bitpack_blocked.rs

@@ -15,12 +15,9 @@ use vortex_array::VortexSessionExecute;
 use vortex_array::aggregate_fn::session::AggregateFnSession;
 use vortex_array::arrays::PrimitiveArray;
 use vortex_array::dtype::NativePType;
-use vortex_array::dtype::session::DTypeSession;
 use vortex_array::memory::MemorySession;
 use vortex_array::optimizer::kernels::KernelSession;
-use vortex_array::scalar_fn::session::ScalarFnSession;
 use vortex_array::session::ArraySession;
-use vortex_array::stats::session::StatsSession;
 use vortex_fastlanes::BitPackedArraySlotsExt;
 use vortex_fastlanes::bitpack_compress::bitpack_encode_blocked;
 use vortex_fastlanes::bitpack_decompress::unpack_array_blocked;
@@ -38,11 +35,8 @@ static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
     let session = VortexSession::empty()
         .with::<ArraySession>()
         .with::<KernelSession>()
-        .with::<DTypeSession>()
-        .with::<ScalarFnSession>()
         .with::<AggregateFnSession>()
-        .with::<MemorySession>()
-        .with::<StatsSession>();
+        .with::<MemorySession>();
     vortex_fastlanes::initialize(&session);
     session
 });

encodings/fastlanes/benches/bitpack_blocked_decompress.rs

@@ -24,13 +24,10 @@ use vortex_array::arrays::BoolArray;
 use vortex_array::arrays::ConstantArray;
 use vortex_array::arrays::PrimitiveArray;
 use vortex_array::builtins::ArrayBuiltins;
-use vortex_array::dtype::session::DTypeSession;
 use vortex_array::memory::MemorySession;
 use vortex_array::optimizer::kernels::KernelSession;
 use vortex_array::scalar_fn::fns::operators::Operator;
-use vortex_array::scalar_fn::session::ScalarFnSession;
 use vortex_array::session::ArraySession;
-use vortex_array::stats::session::StatsSession;
 use vortex_array::validity::Validity;
 use vortex_buffer::Alignment;
 use vortex_buffer::BufferMut;
@@ -52,11 +49,8 @@ static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
     let session = VortexSession::empty()
         .with::<ArraySession>()
         .with::<KernelSession>()
-        .with::<DTypeSession>()
-        .with::<ScalarFnSession>()
         .with::<AggregateFnSession>()
-        .with::<MemorySession>()
-        .with::<StatsSession>();
+        .with::<MemorySession>();
     vortex_fastlanes::initialize(&session);
     session
 });