Merge remote-tracking branch 'upstream/develop' into feat/direct-arrow-byte-arrays

Signed-off-by: 蔡略 <cailue@apache.org>

Author: ClSlaid

.github/workflows/bench.yml

@@ -171,31 +171,3 @@ jobs:
     secrets: inherit
     with:
       mode: "develop"
-
-  sql-vortex:
-    uses: ./.github/workflows/sql-benchmarks.yml
-    secrets: inherit
-    with:
-      mode: "develop"
-      benchmark_matrix: |
-        [
-          {
-            "id": "vortex-queries",
-            "subcommand": "vortex",
-            "name": "Vortex queries",
-            "data_formats": ["parquet", "vortex"],
-            "pr_targets": [
-              {"engine": "datafusion", "format": "parquet"},
-              {"engine": "datafusion", "format": "vortex"},
-              {"engine": "duckdb", "format": "parquet"},
-              {"engine": "duckdb", "format": "vortex"}
-            ],
-            "develop_targets": [
-              {"engine": "datafusion", "format": "parquet"},
-              {"engine": "datafusion", "format": "vortex"},
-              {"engine": "duckdb", "format": "parquet"},
-              {"engine": "duckdb", "format": "vortex"}
-            ],
-            "iterations": "100"
-          }
-        ]

.github/workflows/ci.yml

@@ -558,14 +558,25 @@ jobs:
           cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \
             --target x86_64-unknown-linux-gnu -Zbuild-std \
             -p vortex-ffi
-      - name: Build C++ library (asan)
+      - name: Build C++ tests and examples (asan)
         run: |
           mkdir -p build
-          cmake -S lang/cpp -Bbuild -DSANITIZER=asan -DBUILD_TESTS=1 -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
+          cmake -S lang/cpp -Bbuild -DSANITIZER=asan -DBUILD_TESTS=1 -DBUILD_EXAMPLES=1 -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
           cmake --build build --parallel $(nproc)
       - name: Run C++ tests
         run: |
           build/tests/vortex_cxx_test
+      - name: Run C++ examples
+        run: |
+          cd build/examples
+          ./writer people0.vortex
+          ./writer people1.vortex
+          ./writer me.vortex
+
+          ./scan '*.vortex'
+          ./dtype me.vortex
+
+          ./reader
 
   sqllogic-test:
     name: "SQL logic tests"

.github/workflows/rust-instrumented.yml

@@ -279,10 +279,6 @@ jobs:
         run: |
           set -o pipefail
 
-          # Failed to create data source: Object store error: Generic LocalFileSystem
-          # error: Unable to walk dir: File system loop found
-          rm -fr vortex-ffi/build/_deps/nanoarrow-src/python
-
           ./vortex-ffi/build/examples/write_sample file.vortex
           ./vortex-ffi/build/examples/write_sample file2.vortex
           ./vortex-ffi/build/examples/dtype '*.vortex'

.github/workflows/sql-benchmarks.yml

@@ -298,6 +298,25 @@ on:
                 {"engine": "duckdb", "format": "duckdb"}
               ],
               "iterations": "10"
+            },
+            {
+              "id": "vortex-queries",
+              "subcommand": "vortex",
+              "name": "Vortex queries",
+              "data_formats": ["parquet", "vortex"],
+              "pr_targets": [
+                {"engine": "datafusion", "format": "parquet"},
+                {"engine": "datafusion", "format": "vortex"},
+                {"engine": "duckdb", "format": "parquet"},
+                {"engine": "duckdb", "format": "vortex"}
+              ],
+              "develop_targets": [
+                {"engine": "datafusion", "format": "parquet"},
+                {"engine": "datafusion", "format": "vortex"},
+                {"engine": "duckdb", "format": "parquet"},
+                {"engine": "duckdb", "format": "vortex"}
+              ],
+              "iterations": "100"
             }
           ]
       base_benchmark_matrix:

.github/workflows/sql-vortex-pr.yml

@@ -26,6 +26,12 @@ use vortex_bench::runner::filter_queries;
 use vortex_bench::setup_logging_and_tracing;
 use vortex_bench::v3;
 
+const S3_HTTP_INIT_SQL: [&str; 3] = [
+    "SET http_retries = 8",
+    "SET http_retry_wait_ms = 250",
+    "SET http_retry_backoff = 2",
+];
+
 /// Common arguments shared across benchmarks
 #[derive(Parser)]
 struct Args {
@@ -159,6 +165,11 @@ fn main() -> anyhow::Result<()> {
     )?;
 
     let benchmark_name = benchmark.dataset().to_string();
+    let mut duckdb_init_sql = Vec::new();
+    if benchmark.data_url().scheme() == "s3" {
+        duckdb_init_sql.extend(S3_HTTP_INIT_SQL.map(String::from));
+    }
+    duckdb_init_sql.extend(benchmark.engine_init_sql(Engine::DuckDB));
 
     let mode = if args.explain {
         BenchmarkMode::Explain
@@ -178,7 +189,7 @@ fn main() -> anyhow::Result<()> {
                 args.delete_duckdb_database,
                 args.threads,
             )?;
-            ctx.set_init_sql(benchmark.engine_init_sql(Engine::DuckDB))?;
+            ctx.set_init_sql(duckdb_init_sql.clone())?;
             ctx.register_tables(&*benchmark, format)?;
 
             // Duckdb doesn't support octet_length for strings but we need this

benchmarks/duckdb-bench/src/main.rs

@@ -54,6 +54,13 @@ The postscript contains the locations of:
 2. a `layout` segment containing the root `Layout`
 3. a `statistics` segment containing file-level per-field statistics (e.g., minima and maxima of each field/column, for whole-file pruning)
 4. a `footer` segment containing a dictionary-encoded _segment map_, and other shared configuration such as compression and encryption schemes
+5. up to 16 user-defined `metadata` segments, each identified by a unique, non-empty UTF-8 key of at most 64 bytes
+
+The postscript carries a locator (offset, length, and alignment) for each metadata segment that is
+present; a file written without user metadata (and any file predating this feature) carries none.
+Readers do not load the opaque metadata values by default. Opt-in metadata reads resolve each
+locator separately, allowing values outside the initial file-tail read to be fetched without reading
+the intervening file contents.
 
 :::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs
 :start-after: [postscript]

docs/specs/file-format.md

@@ -16,6 +16,7 @@ mod tests {
     use vortex_array::VortexSessionExecute;
     use vortex_array::array_session;
     use vortex_array::arrays::PrimitiveArray;
+    use vortex_array::compute::conformance::binary_numeric::test_binary_numeric_array;
     use vortex_array::compute::conformance::consistency::test_array_consistency;
     use vortex_array::dtype::DecimalDType;
     use vortex_buffer::buffer;
@@ -74,4 +75,22 @@ mod tests {
         let ctx = &mut array_session().create_execution_ctx();
         test_array_consistency(&array.into_array(), ctx);
     }
+
+    #[rstest]
+    #[case::decimal_i32(DecimalByteParts::try_new(
+        buffer![100i32, 200, 300, 400, 500].into_array(),
+        DecimalDType::new(10, 2)
+    ).unwrap())]
+    #[case::decimal_nullable_i64(DecimalByteParts::try_new(
+        PrimitiveArray::from_option_iter([Some(1000i64), None, Some(3000), Some(4000), None]).into_array(),
+        DecimalDType::new(19, 4)
+    ).unwrap())]
+    #[case::decimal_negative(DecimalByteParts::try_new(
+        buffer![-100i32, -200, 300, -400, 500].into_array(),
+        DecimalDType::new(10, 2)
+    ).unwrap())]
+    fn test_decimal_byte_parts_binary_numeric(#[case] array: DecimalBytePartsArray) {
+        let ctx = &mut array_session().create_execution_ctx();
+        test_binary_numeric_array(&array.into_array(), ctx);
+    }
 }

encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs

@@ -49,6 +49,9 @@ pub fn fill_null_canonical_array(
         | Canonical::List(_)
         | Canonical::FixedSizeList(_)
         | Canonical::Extension(_) => canonical.into_array().fill_null(fill_value.clone())?,
+        Canonical::Union(_) => {
+            todo!("TODO(connor)[Union]: support Union arrays in the fill_null fuzzer")
+        }
         Canonical::Variant(_) => unreachable!("Variant arrays are not fuzzed"),
     })
 }

fuzz/src/array/fill_null.rs

@@ -149,6 +149,9 @@ pub fn mask_canonical_array(
                 .with_nullability(masked_storage.dtype().nullability());
             ExtensionArray::new(ext_dtype, masked_storage).into_array()
         }
+        Canonical::Union(_) => {
+            todo!("TODO(connor)[Union]: support Union arrays in the mask fuzzer")
+        }
         Canonical::Variant(_) => unreachable!("Variant arrays are not fuzzed"),
     })
 }