perf(array): answer a constant IN list by probing a set, and falsify it by interval

encodings/fsst/src/compute/mod.rs

@@ -6,6 +6,7 @@ mod cast;
 mod compare;
 mod filter;
 mod like;
+mod list_contains;
 
 use vortex_array::ArrayRef;
 use vortex_array::ArrayView;

encodings/fsst/src/kernel.rs

@@ -16,6 +16,8 @@ use vortex_array::scalar_fn::fns::cast::Cast;
 use vortex_array::scalar_fn::fns::cast::CastExecuteAdaptor;
 use vortex_array::scalar_fn::fns::like::Like;
 use vortex_array::scalar_fn::fns::like::LikeExecuteAdaptor;
+use vortex_array::scalar_fn::fns::list_contains::ListContains;
+use vortex_array::scalar_fn::fns::list_contains::ListContainsElementExecuteAdaptor;
 use vortex_session::VortexSession;
 
 use crate::FSST;
@@ -24,6 +26,11 @@ pub(super) fn initialize(session: &VortexSession) {
     let kernels = session.kernels();
     kernels.register_execute_parent_kernel(Cast.id(), FSST, CastExecuteAdaptor(FSST));
     kernels.register_execute_parent_kernel(Binary.id(), FSST, CompareExecuteAdaptor(FSST));
+    kernels.register_execute_parent_kernel(
+        ListContains.id(),
+        FSST,
+        ListContainsElementExecuteAdaptor(FSST),
+    );
     kernels.register_execute_parent_kernel(Filter.id(), FSST, FilterExecuteAdaptor(FSST));
     kernels.register_execute_parent_kernel(Dict.id(), FSST, TakeExecuteAdaptor(FSST));
     kernels.register_execute_parent_kernel(Like.id(), FSST, LikeExecuteAdaptor(FSST));

vortex-array/src/arrays/varbinview/mod.rs

@@ -22,8 +22,10 @@ pub mod build_views;
 
 mod view;
 pub use view::BinaryView;
+mod views_side;
 pub use view::Inlined;
 pub use view::Ref;
+pub(crate) use views_side::ViewsSide;
 
 #[cfg(test)]
 mod tests;

vortex-array/src/arrays/varbinview/views_side.rs

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! A [`VarBinViewArray`] with its buffers resolved once, for callers that walk
+//! every row.
+
+use crate::arrays::VarBinViewArray;
+use crate::arrays::varbinview::BinaryView;
+
+/// A resolved view over a canonical [`VarBinViewArray`]: the view structs plus borrowed slices of
+/// every data buffer, supporting cheap per-lane byte access.
+pub(crate) struct ViewsSide<'a> {
+    views: &'a [BinaryView],
+    buffers: Vec<&'a [u8]>,
+}
+
+impl<'a> ViewsSide<'a> {
+    pub(crate) fn new(array: &'a VarBinViewArray) -> Self {
+        Self {
+            views: array.views(),
+            buffers: (0..array.data_buffers().len())
+                .map(|idx| array.buffer(idx).as_slice())
+                .collect(),
+        }
+    }
+
+    pub(crate) fn len(&self) -> usize {
+        self.views.len()
+    }
+
+    /// The view structs themselves, for a caller walking every row in order.
+    pub(crate) fn views(&self) -> &'a [BinaryView] {
+        self.views
+    }
+
+    /// The view at `index` without a bounds check.
+    ///
+    /// # Safety
+    ///
+    /// `index` must be strictly less than `self.len()`.
+    #[inline]
+    pub(crate) unsafe fn view_unchecked(&self, index: usize) -> &'a BinaryView {
+        // SAFETY: caller guarantees index < self.views.len().
+        unsafe { self.views.get_unchecked(index) }
+    }
+
+    /// The full bytes of `view`, which must belong to this side.
+    #[inline]
+    pub(crate) fn view_bytes(&self, view: &'a BinaryView) -> &'a [u8] {
+        if view.is_inlined() {
+            view.as_inlined().value()
+        } else {
+            let view = view.as_view();
+            &self.buffers[view.buffer_index as usize][view.as_range()]
+        }
+    }
+}

vortex-array/src/scalar/typed_view/list.rs

@@ -160,6 +160,17 @@ impl<'a> ListScalar<'a> {
         })
     }
 
+    /// Returns the element values, borrowed from the scalar itself.
+    ///
+    /// Returns None if the list is null. Unlike [`Self::elements`] this clones
+    /// nothing, so it is the form to reach for when a caller only reads the
+    /// values; the cost of materializing a `Vec<Scalar>` is proportional to the
+    /// length of the list and is paid again on drop.
+    #[inline]
+    pub fn element_values(&self) -> Option<&'a [Option<ScalarValue>]> {
+        self.elements
+    }
+
     /// Returns all elements in the list as a vector of scalars.
     ///
     /// Returns None if the list is null.

vortex-array/src/scalar_fn/fns/binary/compare/bytes.rs

@@ -22,6 +22,7 @@ use crate::arrays::BoolArray;
 use crate::arrays::Constant;
 use crate::arrays::VarBinViewArray;
 use crate::arrays::varbinview::BinaryView;
+use crate::arrays::varbinview::ViewsSide;
 use crate::dtype::DType;
 use crate::dtype::Nullability;
 use crate::scalar::Scalar;
@@ -79,50 +80,6 @@ fn constant_bytes(scalar: &Scalar) -> VortexResult<Vec<u8>> {
     value.ok_or_else(|| vortex_err!("null constant handled by execute_compare"))
 }
 
-/// A resolved view over a canonical [`VarBinViewArray`]: the view structs plus borrowed slices of
-/// every data buffer, supporting cheap per-lane byte access.
-struct ViewsSide<'a> {
-    views: &'a [BinaryView],
-    buffers: Vec<&'a [u8]>,
-}
-
-impl<'a> ViewsSide<'a> {
-    fn new(array: &'a VarBinViewArray) -> Self {
-        Self {
-            views: array.views(),
-            buffers: (0..array.data_buffers().len())
-                .map(|idx| array.buffer(idx).as_slice())
-                .collect(),
-        }
-    }
-
-    fn len(&self) -> usize {
-        self.views.len()
-    }
-
-    /// The view at `index` without a bounds check.
-    ///
-    /// # Safety
-    ///
-    /// `index` must be strictly less than `self.len()`.
-    #[inline]
-    unsafe fn view_unchecked(&self, index: usize) -> &'a BinaryView {
-        // SAFETY: caller guarantees index < self.views.len().
-        unsafe { self.views.get_unchecked(index) }
-    }
-
-    /// The full bytes of `view`, which must belong to this side.
-    #[inline]
-    fn view_bytes(&self, view: &'a BinaryView) -> &'a [u8] {
-        if view.is_inlined() {
-            view.as_inlined().value()
-        } else {
-            let view = view.as_view();
-            &self.buffers[view.buffer_index as usize][view.as_range()]
-        }
-    }
-}
-
 /// The leading 8 bytes of a view: the `u32` length plus the first 4 bytes of the value
 /// (zero-padded for values shorter than 4 bytes).
 #[inline]