diff --git a/.gitmodules b/.gitmodules index 7cd6217..74dac07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "w3c-validation/rdf-tests"] path = w3c-validation/rdf-tests url = https://github.com/w3c/rdf-tests +[submodule "benches/trainmarks"] + path = benches/trainmarks + url = https://github.com/DeciSym/trainmarks.git diff --git a/Cargo.toml b/Cargo.toml index 233c170..ac66ea6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,9 @@ repository = "https://github.com/DeciSym/de" default-run = "de" keywords = ["hdt", "rdf", "semantic-web", "triple-store", "sparql"] categories = ["command-line-utilities"] +# The trainmarks benchmark submodule is a separate repo of fixtures and +# harnesses for other engines; it has no place in the published crate. +exclude = ["benches/trainmarks"] [[bin]] name = "de" @@ -20,7 +23,7 @@ path = "src/main.rs" bench = false [[bench]] -name = "benchmark" +name = "trainmarks" harness = false [dependencies] diff --git a/Makefile b/Makefile index 4cf6297..ddf5bf4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ HUB ?= decisym TAG ?= latest VERSION ?= 0.0.0-test +# Dataset scale for the trainmarks benchmarks: medium (~100K triples), +# large (~1M) or xlarge (~10M). Must match DE_BENCH_SCALE at bench time. +BENCH_SCALE ?= large init: scripts/download-sample-bench.sh @@ -21,8 +24,15 @@ test: init presubmit: lint test -bench: init - cargo bench +# Check out the trainmarks submodule and generate the N-Triples and Turtle +# fixtures the trainmarks bench target reads. Split out from `init` because +# they are ~150 MB at the default scale and only the benchmarks need them. +bench-init: init + git submodule update --init benches/trainmarks + python3 scripts/gen-trainmarks-data.py $(BENCH_SCALE) + +bench: bench-init + DE_BENCH_SCALE=$(BENCH_SCALE) cargo bench build: cargo build --features=server diff --git a/README.md b/README.md index 5c57289..20253d4 100644 --- a/README.md +++ b/README.md @@ -135,3 +135,49 @@ Run W3C RDF/SPARQL integration tests: ```sh cargo test --all-features --test w3c-sparql ``` + +## Benchmarks + +The `trainmarks` criterion suite covers `create` and `query` over the synthetic +e-commerce graph from [trainmarks](https://github.com/DeciSym/trainmarks), +checked out as a submodule at `benches/trainmarks`. It exists to catch +performance regressions at a realistic dataset size, not to compare `de` +against other engines. + +It measures four things: building an HDT from N-Triples, building one from +Turtle (the only path that runs the RDF parser), the five queries against a +prebuilt HDT, and one query given a Turtle file directly, which `de` converts +to a temporary package before evaluating. + +`make bench` first checks out the submodule and generates the fixtures the +suite reads: + +```sh +make bench +``` + +The dataset scale is `BENCH_SCALE` — `medium` (~100K triples), `large` (~1M, +the default) or `xlarge` (~10M): + +```sh +BENCH_SCALE=medium make bench +``` + +`make bench` passes the scale through to the suite as `DE_BENCH_SCALE`, which +is also what to set when driving `cargo bench` directly. The fixtures and the +benchmark must agree on it, since the scale is part of every benchmark id +(`trainmarks_query/large/q3_join_3_entities`) and criterion compares each run +against the stored baseline for that id: + +```sh +make bench-init # once, to lay down the fixtures +DE_BENCH_SCALE=large cargo bench --bench trainmarks +DE_BENCH_SCALE=large cargo bench --bench trainmarks -- q3_join_3_entities +``` + +Without the fixtures the suite prints how to get them and measures nothing, so +`cargo bench` still works on a fresh clone. + +The queries are trainmarks' own `q1`–`q5`, shared verbatim with the other +engines in that report. `q6_delete_insert` is omitted: it is a SPARQL Update, +and HDT packages are immutable. diff --git a/benches/benchmark.rs b/benches/benchmark.rs deleted file mode 100644 index 3926c48..0000000 --- a/benches/benchmark.rs +++ /dev/null @@ -1,133 +0,0 @@ -use criterion::{Criterion, criterion_group, criterion_main}; -use de::{create, query}; -#[cfg(target_os = "linux")] -use pprof::criterion::{Output, PProfProfiler}; -use std::{ - fs::{File, OpenOptions}, - io::BufWriter, - path::Path, - time::Duration, -}; -use tempfile::tempdir; - -fn devnull_writer() -> BufWriter { - let null_path = if cfg!(windows) { "NUL" } else { "/dev/null" }; - let file = OpenOptions::new() - .write(true) - .open(null_path) - .expect("failed to open null sink"); - BufWriter::new(file) -} - -fn query(c: &mut Criterion) { - // ######### NOTE ########### - // requires tests/resources/superhero.ttl, run 'make init' - // ########################## - let source_rdf = "tests/resources/superhero.ttl".to_string(); - let query_file = "tests/resources/hero-height.rq".to_string(); - assert!( - Path::new(&source_rdf).exists(), - "missing benchmark fixture {source_rdf}; run `make init`" - ); - assert!( - Path::new(&query_file).exists(), - "missing benchmark query fixture {query_file}; run `make init`" - ); - - let tmp_dir = tempdir().expect("failed to create benchmark tempdir"); - let test_hdt = tmp_dir.path().join("rdf.hdt"); - let test_hdt_path = test_hdt - .to_str() - .expect("temporary HDT path must be valid UTF-8") - .to_string(); - let query_files = vec![query_file]; - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("failed to build tokio runtime"); - let mut create_group = c.benchmark_group("create_hdt_from_ttl_file"); - create_group.sample_size(10); - create_group.measurement_time(Duration::from_mins(2)); - create_group.bench_function("create_hdt", |b| { - b.iter(|| { - runtime.block_on(async { - create::do_create(&test_hdt_path, std::slice::from_ref(&source_rdf)) - .await - .expect("failed to create HDT from benchmark fixture"); - }); - }); - }); - create_group.finish(); - - runtime.block_on(async { - create::do_create(&test_hdt_path, std::slice::from_ref(&source_rdf)) - .await - .expect("failed to prepare HDT fixture for query benchmarks"); - }); - - let hdt_data_files = vec![test_hdt_path]; - let mut hdt_writer = devnull_writer(); - let mut hdt_group = c.benchmark_group("query_single_hdt_file"); - hdt_group.sample_size(10); - hdt_group.measurement_time(Duration::from_secs(25)); - hdt_group.bench_function("query_hdt", |b| { - b.iter(|| { - runtime.block_on(async { - query::do_query( - &hdt_data_files, - &query_files, - query::EntailmentMode::Off, - &query::DeOutput::CSV, - &mut hdt_writer, - ) - .await - .expect("failed to query HDT benchmark fixture"); - }); - }); - }); - hdt_group.finish(); - - let rdf_data_files = vec![source_rdf]; - let mut rdf_writer = devnull_writer(); - let mut rdf_group = c.benchmark_group("query_single_rdf_file"); - rdf_group.sample_size(10); - rdf_group.measurement_time(Duration::from_secs(5)); - rdf_group.bench_function("query_rdf", |b| { - b.iter(|| { - runtime.block_on(async { - query::do_query( - &rdf_data_files, - &query_files, - query::EntailmentMode::Off, - &query::DeOutput::CSV, - &mut rdf_writer, - ) - .await - .expect("failed to query RDF benchmark fixture"); - }); - }); - }); - rdf_group.finish(); - - tmp_dir - .close() - .expect("failed to clean up benchmark tempdir"); -} - -#[cfg(target_os = "linux")] -criterion_group! { - name = benches; - config = Criterion::default() - .with_profiler(PProfProfiler::new(100, Output::Protobuf)) - .warm_up_time(Duration::from_millis(1)); - targets = query -} - -#[cfg(not(target_os = "linux"))] -criterion_group! { - name = benches; - config = Criterion::default().warm_up_time(Duration::from_millis(1)); - targets = query -} - -criterion_main!(benches); diff --git a/benches/trainmarks b/benches/trainmarks new file mode 160000 index 0000000..3430aa9 --- /dev/null +++ b/benches/trainmarks @@ -0,0 +1 @@ +Subproject commit 3430aa91688bfbc577b6666892eebd667883502e diff --git a/benches/trainmarks.rs b/benches/trainmarks.rs new file mode 100644 index 0000000..b5fd6ba --- /dev/null +++ b/benches/trainmarks.rs @@ -0,0 +1,331 @@ +// Copyright (c) 2025, Decisym, LLC +// Licensed under the BSD 3-Clause License (see LICENSE file in the project root). + +//! Regression benchmarks over the trainmarks e-commerce dataset. +//! +//! `benches/trainmarks` is a submodule of the `DeciSym` fork of the trainmarks +//! RDF benchmark suite, which compares thirteen triplestores on a synthetic +//! customers/orders/products graph. This target reuses two of its artifacts — +//! the data generator and the shared SPARQL queries — to track `de`'s own +//! numbers over time, rather than to compare `de` against other engines. +//! +//! Fixtures come from `make bench-init`, which checks out the submodule and +//! runs `scripts/gen-trainmarks-data.py` to write `benches/trainmarks/data/ +//! .{nt,ttl}`. Without them this target prints how to get them and +//! measures nothing, so `cargo bench` still works on a fresh clone. +//! +//! Both serialisations are measured, because they take different routes into +//! the engine. An `.nt` input is handed to `Hdt::read_nt` more or less as it +//! stands; a `.ttl` input goes through `oxrdfio` first, so it is the only one +//! of the two that puts the RDF parser on the measured path. `de query` given +//! a non-HDT file converts it to a temporary HDT before evaluating, and that +//! route is covered too. +//! +//! Scale is `DE_BENCH_SCALE` (`medium` ~100K, `large` ~1M, `xlarge` ~10M +//! triples), defaulting to `large`. That default is a wall-clock choice: +//! `large` builds its HDT in roughly two seconds against roughly seventeen for +//! `xlarge`, so a create group of ten samples costs half a minute rather than +//! three. Drop to `medium` when a query is slow enough that `large` will not +//! finish. + +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; +use de::{create, query}; +#[cfg(target_os = "linux")] +use pprof::criterion::{Output, PProfProfiler}; +use std::{ + io::{BufWriter, Sink, sink}, + path::{Path, PathBuf}, + time::Duration, +}; +use tempfile::{TempDir, tempdir}; + +/// The trainmarks queries `de` can answer, paired with the result format each +/// one needs. +/// +/// `q6_delete_insert` is deliberately absent: it is a SPARQL Update, HDT +/// packages are immutable, and `de`'s parser rejects DELETE/INSERT outright. +/// The trainmarks report records it as N/A for `de` for the same reason. +/// +/// `q5_construct` returns a graph rather than a solution sequence, so it is +/// serialised as Turtle; the other four are tabular and use CSV. +const QUERIES: [(&str, query::DeOutput); 5] = [ + ("q1_count", query::DeOutput::CSV), + ("q2_customer_orders", query::DeOutput::CSV), + ("q3_join_3_entities", query::DeOutput::CSV), + ("q4_optional_aggregation", query::DeOutput::CSV), + ("q5_construct", query::DeOutput::TURTLE), +]; + +const SCALE_VAR: &str = "DE_BENCH_SCALE"; +const DEFAULT_SCALE: &str = "large"; + +/// Results go to a sink rather than to a file or `/dev/null`. +/// +/// Serialisation still runs — `do_query` formats every solution through this +/// writer — but the write syscall that would follow is not part of what these +/// benchmarks are meant to track, and dropping it also avoids the +/// `/dev/null` vs `NUL` split. +fn null_writer() -> BufWriter { + BufWriter::new(sink()) +} + +fn trainmarks_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("benches") + .join("trainmarks") +} + +/// Build an HDT package from `nt` at `dest`, returning the path as the +/// `String` the `de` API takes. +fn build_hdt(runtime: &tokio::runtime::Runtime, nt: &str, dest: &Path) -> String { + let dest = dest + .to_str() + .expect("temporary HDT path must be valid UTF-8") + .to_string(); + let sources = vec![nt.to_string()]; + runtime.block_on(async { + create::do_create(&dest, &sources) + .await + .expect("failed to build HDT from trainmarks fixture"); + }); + dest +} + +fn bench_create(c: &mut Criterion, runtime: &tokio::runtime::Runtime, scale: &str, f: &Fixtures) { + // A dedicated tempdir, not the one the query fixture lives in: + // `write_hdt_to_path` clears `.index.*` sidecars before writing, so + // sharing a path with the query HDT would drop the warmed index cache + // partway through the run. + let tmp_dir = tempdir().expect("failed to create create-benchmark tempdir"); + let hdt_path = tmp_dir.path().join("trainmarks.hdt"); + let hdt_path = hdt_path + .to_str() + .expect("temporary HDT path must be valid UTF-8") + .to_string(); + + let mut group = c.benchmark_group(format!("trainmarks_create/{scale}")); + group.sample_size(10); + group.measurement_time(Duration::from_secs(30)); + + // Throughput is reported against each fixture's own size, so the two + // benches are not directly comparable as MiB/s: at `large` the same graph + // is 111 MB of N-Triples and 37 MB of Turtle. Each is only meaningful + // against its own history, which is what regression tracking needs. + for (name, source) in [ + ("create_hdt_from_nt", &f.nt), + ("create_hdt_from_ttl", &f.ttl), + ] { + let bytes = std::fs::metadata(source) + .expect("failed to stat trainmarks fixture") + .len(); + let sources = vec![source.clone()]; + group.throughput(Throughput::Bytes(bytes)); + group.bench_function(name, |b| { + b.iter(|| { + runtime.block_on(async { + create::do_create(&hdt_path, &sources) + .await + .expect("failed to create HDT from trainmarks fixture"); + }); + }); + }); + } + group.finish(); + + tmp_dir + .close() + .expect("failed to clean up create-benchmark tempdir"); +} + +/// Answer one query against a freshly built HDT, untimed. +/// +/// The first query to touch an HDT builds its wavelet-tree index sidecar and +/// faults the package in from disk. Both costs are per-package, not per-query, +/// so paying them once here keeps them out of whichever query group happens to +/// run first — including when `cargo bench -- ` runs only one of them. +/// `q1_count` is the cheapest of the five and scans the whole package, so it +/// warms the page cache as a side effect. +fn warm_hdt(runtime: &tokio::runtime::Runtime, hdt_path: &str, queries_dir: &Path) { + let data_files = vec![hdt_path.to_string()]; + let query_files = vec![ + queries_dir + .join("q1_count.rq") + .to_str() + .expect("trainmarks query path must be valid UTF-8") + .to_string(), + ]; + runtime.block_on(async { + query::do_query( + &data_files, + &query_files, + query::EntailmentMode::Off, + &query::DeOutput::CSV, + &mut null_writer(), + ) + .await + .expect("failed to warm the trainmarks HDT index"); + }); +} + +/// The `.nt` and `.ttl` serialisations of one scale. +struct Fixtures { + nt: String, + ttl: String, +} + +fn bench_queries( + c: &mut Criterion, + runtime: &tokio::runtime::Runtime, + scale: &str, + hdt_path: &str, + queries_dir: &Path, +) { + let data_files = vec![hdt_path.to_string()]; + + for (name, out) in &QUERIES { + let query_file = queries_dir.join(format!("{name}.rq")); + let query_files = vec![ + query_file + .to_str() + .expect("trainmarks query path must be valid UTF-8") + .to_string(), + ]; + + let mut group = c.benchmark_group(format!("trainmarks_query/{scale}")); + group.sample_size(10); + group.measurement_time(Duration::from_secs(20)); + group.bench_function(*name, |b| { + let mut writer = null_writer(); + b.iter(|| { + runtime.block_on(async { + query::do_query( + &data_files, + &query_files, + query::EntailmentMode::Off, + out, + &mut writer, + ) + .await + .unwrap_or_else(|e| panic!("failed to run trainmarks {name}: {e}")); + }); + }); + }); + group.finish(); + } +} + +/// Answer one query with a non-HDT file as the data source. +/// +/// `do_query` converts anything that is not already HDT into a temporary +/// package before evaluating, so this measures parse plus build plus query as +/// one number — the cost a caller actually pays for `de query -d graph.ttl`. +/// Only `q1_count` runs here: every iteration rebuilds the package, so the +/// conversion dominates and the other four queries would add minutes of +/// wall clock to re-measure the same conversion. +fn bench_query_from_rdf( + c: &mut Criterion, + runtime: &tokio::runtime::Runtime, + scale: &str, + ttl: &str, + queries_dir: &Path, +) { + let data_files = vec![ttl.to_string()]; + let query_files = vec![ + queries_dir + .join("q1_count.rq") + .to_str() + .expect("trainmarks query path must be valid UTF-8") + .to_string(), + ]; + + let mut group = c.benchmark_group(format!("trainmarks_query/{scale}")); + group.sample_size(10); + group.measurement_time(Duration::from_secs(30)); + group.bench_function("q1_count_from_ttl", |b| { + let mut writer = null_writer(); + b.iter(|| { + runtime.block_on(async { + query::do_query( + &data_files, + &query_files, + query::EntailmentMode::Off, + &query::DeOutput::CSV, + &mut writer, + ) + .await + .expect("failed to run trainmarks q1_count over Turtle"); + }); + }); + }); + group.finish(); +} + +fn trainmarks(c: &mut Criterion) { + let scale = std::env::var(SCALE_VAR).unwrap_or_else(|_| DEFAULT_SCALE.to_string()); + let root = trainmarks_dir(); + let data_dir = root.join("data"); + let nt = data_dir.join(format!("{scale}.nt")); + let ttl = data_dir.join(format!("{scale}.ttl")); + let queries_dir = root.join("queries"); + + if let Some(missing) = [&nt, &ttl, &queries_dir] + .into_iter() + .find(|path| !path.exists()) + { + eprintln!( + "skipping trainmarks benchmarks: {} is missing.\n\ + run `make bench-init` (or `BENCH_SCALE={scale} make bench-init`) to check out the \ + benches/trainmarks submodule and generate the fixtures.", + missing.display(), + ); + return; + } + let to_str = |path: &Path| { + path.to_str() + .expect("trainmarks fixture path must be valid UTF-8") + .to_string() + }; + let fixtures = Fixtures { + nt: to_str(&nt), + ttl: to_str(&ttl), + }; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build tokio runtime"); + + bench_create(c, &runtime, &scale, &fixtures); + + let query_dir: TempDir = tempdir().expect("failed to create query-benchmark tempdir"); + let hdt_path = build_hdt( + &runtime, + &fixtures.nt, + &query_dir.path().join("trainmarks.hdt"), + ); + warm_hdt(&runtime, &hdt_path, &queries_dir); + bench_queries(c, &runtime, &scale, &hdt_path, &queries_dir); + bench_query_from_rdf(c, &runtime, &scale, &fixtures.ttl, &queries_dir); + + query_dir + .close() + .expect("failed to clean up query-benchmark tempdir"); +} + +#[cfg(target_os = "linux")] +criterion_group! { + name = benches; + config = Criterion::default() + .with_profiler(PProfProfiler::new(100, Output::Protobuf)) + .warm_up_time(Duration::from_millis(1)); + targets = trainmarks +} + +#[cfg(not(target_os = "linux"))] +criterion_group! { + name = benches; + config = Criterion::default().warm_up_time(Duration::from_millis(1)); + targets = trainmarks +} + +criterion_main!(benches); diff --git a/scripts/gen-trainmarks-data.py b/scripts/gen-trainmarks-data.py new file mode 100755 index 0000000..7b742a4 --- /dev/null +++ b/scripts/gen-trainmarks-data.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Generate the N-Triples fixture the trainmarks benchmark suite runs against. + +The trainmarks submodule (``benches/trainmarks``) ships ``generate_data.py``, +which writes both Turtle and N-Triples at all three scales -- roughly 1.7 GB of +files. The benchmarks read one scale, so this wrapper imports the submodule's +generator and emits just that one. Both serialisations are written: the +benchmarks build HDT from the ``.nt`` and, separately, from the ``.ttl``, so +that the RDF parser stays on the measured path. + +The generator is deterministic (``random.seed(42)`` at import time), so the same +scale always yields byte-identical output and timings stay comparable across +machines and across runs. It is not, however, byte-identical to the file +trainmarks' own ``generate_data.py`` writes: that script draws all three scales +from one RNG stream, so its ``large.nt`` starts where ``medium`` left off. The +two are statistically the same graph and the query timings match, but do not +expect the files to hash the same. + +Usage: + scripts/gen-trainmarks-data.py [medium|large|xlarge] [--force] +""" + +import argparse +import os +import sys +import time + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TRAINMARKS = os.path.join(REPO_ROOT, "benches", "trainmarks") +DATA_DIR = os.path.join(TRAINMARKS, "data") + +# Customer/product/order counts per scale, copied from the trainmarks +# generator's __main__ block (which hard-codes them inline rather than +# exposing them as constants we could import). +SCALES = { + "medium": (1_000, 200, 13_000), + "large": (10_000, 2_000, 133_000), + "xlarge": (100_000, 10_000, 1_335_000), +} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("scale", nargs="?", default="large", choices=sorted(SCALES)) + parser.add_argument( + "--force", + action="store_true", + help="regenerate even if the .nt file is already present", + ) + args = parser.parse_args() + + if not os.path.isfile(os.path.join(TRAINMARKS, "generate_data.py")): + sys.exit( + f"trainmarks submodule not checked out at {TRAINMARKS}\n" + "run: git submodule update --init benches/trainmarks" + ) + + outputs = {ext: os.path.join(DATA_DIR, f"{args.scale}.{ext}") for ext in ("nt", "ttl")} + if all(os.path.isfile(p) for p in outputs.values()) and not args.force: + for path in outputs.values(): + size_mb = os.path.getsize(path) / 1024 / 1024 + print(f"{path} already present ({size_mb:.1f} MB)") + print("use --force to regenerate") + return + + sys.path.insert(0, TRAINMARKS) + import generate_data + + os.makedirs(DATA_DIR, exist_ok=True) + n_customers, n_products, n_orders = SCALES[args.scale] + + t0 = time.time() + triples = generate_data.generate_triples(n_customers, n_products, n_orders) + print(f"generated {len(triples)} triples in {time.time() - t0:.1f}s") + + for ext, writer in (("nt", generate_data.write_ntriples), ("ttl", generate_data.write_turtle)): + path = outputs[ext] + t0 = time.time() + writer(triples, path) + size_mb = os.path.getsize(path) / 1024 / 1024 + print(f"wrote {path} ({size_mb:.1f} MB) in {time.time() - t0:.1f}s") + + +if __name__ == "__main__": + main() diff --git a/src/sparql.rs b/src/sparql.rs index 23253eb..5b58f81 100644 --- a/src/sparql.rs +++ b/src/sparql.rs @@ -762,9 +762,11 @@ fn evaluate_query_with_debug_plan<'a, D>( where D: QueryableDataset<'a>, { - // Keep optimizer disabled for all execution paths: this matches current upstream patch behavior - // used to pass W3C suites in this repository and avoids optimizer-specific regressions. - let evaluator = QueryEvaluator::new().without_optimizations(); + // NOTE: previously this ran .without_optimizations() in order to pass W3C suites, but is no + // longer needed with current spareval revision. With optimizations, query responses are + // significantly reduced. + // TODO: monitor for optimizer-specific regressions, perhaps introducing toggle + let evaluator = QueryEvaluator::new(); if debug_plan { let (results, explanation) = evaluator.prepare(parsed).explain(dataset); let mut json = Vec::new();