From ff92d33399ea3b1c97487a9066f6101180d2f88c Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 14:23:22 -0700 Subject: [PATCH 01/12] refactor(gfql): chain hot paths expose their shape admission predicates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pandas/cuDF chain fast path and the polars chain's plain single-hop branches decided admission with inline checks and closures, so a test could only learn which route served a shape by spying. Each gate is now a function the dispatcher calls — `native_fast_path_admits` and `polars_plain_single_hop_admits` — and a shared route corpus (graphistry/tests/compute/gfql/routes/corpus.py) is filtered per route by that same function: the decision table is pinned per shape, served-by is asserted against the predicate on pandas and cuDF, and every admitted shape is checked against the pandas full path. No route admits or declines anything it did not before. Found while pinning: the polars plain branch admits `prune_to_endpoints` where the pandas gate declines it, and the results differ (#2053, strict expected failure). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QztW7jYsDd66e8rb8pJNQA (cherry picked from commit 82b2f0e71b17d4eb9320e4e9bc677133cbb8e0c9) --- CHANGELOG.md | 1 + bin/test-polars.sh | 2 + graphistry/compute/chain.py | 50 +-------- graphistry/compute/chain_fast_paths.py | 39 +++++++ .../compute/gfql/lazy/engine/polars/chain.py | 104 +++++++++++++----- .../engine/polars/test_chain_admission.py | 57 ++++++++++ .../tests/compute/gfql/routes/__init__.py | 0 .../tests/compute/gfql/routes/corpus.py | 56 ++++++++++ .../test_chain_fast_paths_admission.py | 93 ++++++++++++++++ 9 files changed, 329 insertions(+), 73 deletions(-) create mode 100644 graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py create mode 100644 graphistry/tests/compute/gfql/routes/__init__.py create mode 100644 graphistry/tests/compute/gfql/routes/corpus.py create mode 100644 graphistry/tests/compute/test_chain_fast_paths_admission.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f8bcaaba8c..11bd1614a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Changed +* GFQL: the chain hot paths expose their input-shape admission predicates as functions the dispatchers call — `native_fast_path_admits` (pandas/cuDF single-node and plain single-hop lanes) and `polars_plain_single_hop_admits` (polars seeded-index and skip-combine branches) — so tests filter one shared shape corpus per route with the route's own gate instead of re-deriving it. No route admits or declines anything it did not before; the corpus pins each predicate's decision table and served-by parity. * GFQL: the wavefront seed-rediscovery rule moved out of `hop.py` into `graphistry/compute/gfql/seed_rediscovery.py` (pandas/cuDF) and `graphistry/compute/gfql/lazy/engine/polars/seed_rediscovery.py` (polars); `undirected_rediscovered_seed_ids` (an internal helper) is gone. ## [0.59.0 - 2026-08-31] diff --git a/bin/test-polars.sh b/bin/test-polars.sh index 7c237e97dc..6d4071d610 100755 --- a/bin/test-polars.sh +++ b/bin/test-polars.sh @@ -87,6 +87,8 @@ POLARS_TEST_FILES=( graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py + graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py + graphistry/tests/compute/test_chain_fast_paths_admission.py graphistry/tests/compute/gfql/test_undirected_pairs_2026.py graphistry/tests/compute/gfql/test_native_seed_lane_explain.py # #1882/#1913-f4/#1879 crash-family pins: the polars params (filter helpers on polars diff --git a/graphistry/compute/chain.py b/graphistry/compute/chain.py index 6dc9b27a88..0a00968d0a 100644 --- a/graphistry/compute/chain.py +++ b/graphistry/compute/chain.py @@ -14,6 +14,7 @@ from .chain_fast_paths import ( _seeded_typed_hop_pandas_cudf, _single_node_rows_via_index_or_filter, + native_fast_path_admits, _tag_fast_path_aliases, ) from graphistry.compute.validate.validate_schema import validate_chain_schema, validate_graph_shape @@ -859,10 +860,9 @@ def _try_chain_fast_path( polars/dask/spark also fall through (own fast path / lazy semantics).""" from graphistry.compute.filter_by_dict import filter_by_dict - if engine_concrete not in (Engine.PANDAS, Engine.CUDF): + shape = native_fast_path_admits(ops, engine_concrete, start_nodes) + if shape is None: return None - if start_nodes is not None: - return None # seeded chains use the full path (fast path has no seed) engine_abs = EngineAbstract(engine_concrete.value) def _materialize_fast_path_graph() -> Plottable: @@ -870,10 +870,9 @@ def _materialize_fast_path_graph() -> Plottable: g = g_in.materialize_nodes(engine=EngineAbstract(engine_concrete.value)) return _coerce_input_formats(g, engine_concrete) - if len(ops) == 1: + if shape == "single-node": n0 = ops[0] - if not (isinstance(n0, ASTNode) and n0.query is None): - return None + assert isinstance(n0, ASTNode) # the predicate admitted this shape g = _materialize_fast_path_graph() if g._nodes is None: return None @@ -894,48 +893,11 @@ def _materialize_fast_path_graph() -> Plottable: edges = g._edges.iloc[0:0] if g._edges is not None else None return g.nodes(nodes).edges(edges) if edges is not None else g.nodes(nodes) - if len(ops) != 3: - return None n0, e1, n2 = ops - # Aliases are a PROJECTION concern, not a traversal one: capture them, serve the - # traversal on the fast path, and tag the result (_tag_fast_path_aliases). Rejecting - # them here sent a NAMED `g.gfql([n(name=..), e(..), n(name=..)])` to the full - # two-pass BFS purely because the ops carried names — measured ~25.2 ms before vs - # ~2.3 ms after (medians of 5 paired runs), on a 200-node graph where data work is ~0. - # SCOPE, measured: this does NOT reach the Cypher `MATCH ... RETURN` surface for the - # benchmark shapes. Those are served earlier by `gfql_fast_paths.py` and never consult - # this function at all, so do not attribute a Cypher-surface win to this gate. + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) # the predicate admitted this shape alias_n0, alias_e1, alias_n2 = n0._name, e1._name, n2._name - _named = [a for a in (alias_n0, alias_e1, alias_n2) if a is not None] - if len(_named) != len(set(_named)): - # Duplicate alias reuse is an E201 error, and `combine_steps` is what raises it. - # Serving these here would BYPASS that check and silently succeed — decline so the - # full path still errors. (Caught by test_polars_duplicate_alias_declines_like_pandas.) - return None - if not (isinstance(n0, ASTNode) and n0.query is None): - return None - if not (isinstance(n2, ASTNode) and n2.query is None): - return None - if not (isinstance(e1, ASTEdge) and e1.is_simple_single_hop() - and e1.source_node_match is None - and e1.destination_node_match is None - and e1.source_node_query is None and e1.destination_node_query is None - and e1.edge_query is None and not e1.include_zero_hop_seed - and not e1.prune_to_endpoints): # prune keeps only the arrival side -> full path - return None - # #1755 lever-3: a typed edge (edge_match, e.g. -[:HAS_CREATOR]->) is a plain - # equality/predicate filter on the edge frame — apply it in the fast-path body - # below rather than falling through to the full two-pass machinery. source/dest - # node match + edge_query (richer predicates) still bail above. direction = e1.direction - if direction == "undirected" and (alias_n0 is not None or alias_n2 is not None): - # An undirected edge makes a node reachable as EITHER endpoint, so "which alias - # does this node carry" is not derivable from the endpoint columns the way it is - # for a directed hop. Decline to the full path rather than guess. - return None unconstrained = not n0.filter_dict and not n2.filter_dict - if not unconstrained and direction == "undirected": - return None # filtered-undirected (OR of both directions) -> full path g = _materialize_fast_path_graph() if g._nodes is None or g._edges is None: return None diff --git a/graphistry/compute/chain_fast_paths.py b/graphistry/compute/chain_fast_paths.py index 3e26b52fc5..85c4cad036 100644 --- a/graphistry/compute/chain_fast_paths.py +++ b/graphistry/compute/chain_fast_paths.py @@ -80,6 +80,45 @@ def _tag_fast_path_aliases( return res.nodes(nodes).edges(edges) +NativeFastPathShape = Literal["single-node", "seeded-hop"] + + +def native_fast_path_admits( + ops: Sequence[ASTObject], engine: "Engine", start_nodes: Optional[DataFrameT], +) -> Optional[NativeFastPathShape]: + """The shape the pandas/cuDF chain fast path serves for ``ops``, or None when the full + path must run. This is the dispatcher's own gate (``chain._try_chain_fast_path`` calls + it first), so a test that filters a shape corpus with it exercises exactly what the + dispatcher admits: a node-only op without ``query``, or a 3-op plain single hop whose + node ops carry no ``query``, whose edge carries no node matches, queries, zero-hop seed + or endpoint pruning, whose aliases are distinct, and which is not an undirected hop + with names or with node filters. Seeded chains (``start_nodes``) and other engines + decline. Frame-dependent conditions (missing frames, alias equal to the node binding) + are checked by the body after materialization.""" + from graphistry.Engine import Engine + if engine not in (Engine.PANDAS, Engine.CUDF) or start_nodes is not None: + return None + if len(ops) == 1: + n0 = ops[0] + return "single-node" if isinstance(n0, ASTNode) and n0.query is None else None + if len(ops) != 3: + return None + n0, e1, n2 = ops + if not (isinstance(n0, ASTNode) and n0.query is None and isinstance(n2, ASTNode) and n2.query is None): + return None + if not (isinstance(e1, ASTEdge) and e1.is_simple_single_hop() + and e1.source_node_match is None and e1.destination_node_match is None + and e1.source_node_query is None and e1.destination_node_query is None + and e1.edge_query is None and not e1.include_zero_hop_seed and not e1.prune_to_endpoints): + return None + named = [a for a in (n0._name, e1._name, n2._name) if a is not None] + if len(named) != len(set(named)): + return None + if e1.direction == "undirected" and (n0._name is not None or n2._name is not None or n0.filter_dict or n2.filter_dict): + return None + return "seeded-hop" + + def _seeded_scalar_filters(fd: Optional[Dict[str, Any]], df: DataFrameT) -> Optional[Dict[str, Any]]: """Resolve a filter dict to plain scalar column==value pairs, or None to bail to the general path. Mirrors filter_by_dict.resolve_filter_column exactly for diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 5f05086936..5458027e55 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -7,7 +7,7 @@ (no silent pandas fallback). Deferred: variable-length/multi-hop edge sub-cases, some undirected multi-edge combos, node query=. """ -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, cast +from typing import TYPE_CHECKING, Any, List, Literal, Optional, Sequence, Tuple, Type, cast from typing_extensions import TypedDict @@ -222,6 +222,41 @@ def _exec(op: ASTObject, g: Plottable, prev_wf: Optional[Any], target_wf: Option raise NotImplementedError(f"polars chain engine does not support op {type(op).__name__}") +PolarsPlainSingleHopShape = Literal["seeded-index", "skip-combine"] + + +def _plain_node(op: ASTObject) -> bool: + return isinstance(op, ASTNode) and op._name is None and op.query is None + + +def _plain_edge(op: ASTObject) -> bool: + return (isinstance(op, ASTEdge) and op.is_simple_single_hop() + and op.edge_match is None and op.source_node_match is None + and op.destination_node_match is None and op._name is None + and op.source_node_query is None and op.destination_node_query is None + and op.edge_query is None and not op.include_zero_hop_seed) + + +def polars_plain_single_hop_admits(ops: Sequence[ASTObject], start_nodes: Optional[object]) -> Optional[PolarsPlainSingleHopShape]: + """The polars chain's plain single-hop branch for ``ops``: ``"seeded-index"`` when the + resident-index hop is consulted first (seed filter, no destination filter, directed), + ``"skip-combine"`` when the one-hop endpoint filter serves it without the + forward/backward/combine passes, None when the full chain runs. The dispatcher calls + this; unnamed, unqueried nodes and an unnamed, unmatched simple edge are the shape. + A filtered undirected hop is the one plain shape that still takes the full chain.""" + if start_nodes is not None or len(ops) != 3: + return None + n0, e1, n2 = ops + if not (_plain_node(n0) and _plain_edge(e1) and _plain_node(n2)): + return None + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) + directed = e1.direction in ("forward", "reverse") + if n0.filter_dict and not n2.filter_dict and directed: + return "seeded-index" + unconstrained = not n0.filter_dict and not n2.filter_dict + return "skip-combine" if (unconstrained or directed) else None + + def _is_native_multihop(op: ASTObject) -> bool: """Multi-hop shapes the native combine supports: hops=N / max_hops (fwd/rev/undirected), to_fixed_point (fwd/rev), min_hops>1 (fwd/rev, finite max), optionally with match/name. @@ -998,24 +1033,14 @@ def _chain_traversal_polars(self: Plottable, ops, start_nodes: Optional[Any] = N # forward/backward/combine. Byte-identical vs pandas (verified: src/dst/both filters, reverse, # dup/self-loop/cycle/isolated). Undirected takes this branch only when UNCONSTRAINED; # filtered-undirected (OR of both directions) falls through to the full path. - def _fp_node(op): - return isinstance(op, ASTNode) and op._name is None and op.query is None - - def _plain_edge(op): - return (isinstance(op, ASTEdge) and op.is_simple_single_hop() - and op.edge_match is None and op.source_node_match is None - and op.destination_node_match is None and op._name is None - and op.source_node_query is None and op.destination_node_query is None - and op.edge_query is None and not op.include_zero_hop_seed) + plain_shape = polars_plain_single_hop_admits(ops, start_nodes) # GFQL physical index path for the seeded single-hop shape # `MATCH (a {id-filter})-[e]->(b)` (forward/reverse, no destination filter). This native # chain branch never reaches compute/hop.py, so it must consult the index here too. from graphistry.compute.gfql.index import get_index_policy _idx_pol = get_index_policy(self) - if (start_nodes is None and len(ops) == 3 and _fp_node(ops[0]) and _plain_edge(ops[1]) - and _fp_node(ops[2]) and ops[0].filter_dict and not ops[2].filter_dict - and ops[1].direction in ("forward", "reverse")): + if plain_shape == "seeded-index": from graphistry.compute.gfql.index import get_registry, maybe_index_hop if (not get_registry(self).is_empty()) or _idx_pol in ("auto", "force"): gf0 = ensure_nodes_polars(self) @@ -1035,23 +1060,21 @@ def _plain_edge(op): if seeded is not None: return seeded - if start_nodes is None and len(ops) == 3 and _fp_node(ops[0]) and _plain_edge(ops[1]) and _fp_node(ops[2]): + if plain_shape is not None: n0, e1, n2 = ops - unconstrained = not n0.filter_dict and not n2.filter_dict - if unconstrained or e1.direction in ("forward", "reverse"): - node_table_bound = self._nodes is not None - gf = ensure_nodes_polars(self) - ncol, scol, dcol = gf._node, gf._source, gf._destination - assert ncol is not None and scol is not None and dcol is not None - gf, restore = _align_edge_endpoints(gf, ncol, scol, dcol) - edges = drop_null_endpoint_edges(gf._edges, scol, dcol) - n_from, n_to = (n0, n2) if e1.direction != "reverse" else (n2, n0) - all_ids = gf._nodes.select(pl.col(ncol)) - - def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": - if not node_op.filter_dict: - return None - return filter_by_dict_polars(gf._nodes, node_op.filter_dict).select(pl.col(ncol)) + node_table_bound = self._nodes is not None + gf = ensure_nodes_polars(self) + ncol, scol, dcol = gf._node, gf._source, gf._destination + assert ncol is not None and scol is not None and dcol is not None + gf, restore = _align_edge_endpoints(gf, ncol, scol, dcol) + edges = drop_null_endpoint_edges(gf._edges, scol, dcol) + n_from, n_to = (n0, n2) if e1.direction != "reverse" else (n2, n0) + all_ids = gf._nodes.select(pl.col(ncol)) + + def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": + if not node_op.filter_dict: + return None + return filter_by_dict_polars(gf._nodes, node_op.filter_dict).select(pl.col(ncol)) filter_sides = ((scol, _filter_ids(n_from)), (dcol, _filter_ids(n_to))) for endpoint_col, filter_ids in filter_sides: @@ -1077,6 +1100,29 @@ def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": nodes = gf._nodes.join(endpoints, on=ncol, how="semi") nodes = nodes.unique(subset=[ncol], maintain_order=True) # one row per node id, as the full chain and pandas collapse return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) + filter_sides = ((scol, _filter_ids(n_from)), (dcol, _filter_ids(n_to))) + for endpoint_col, filter_ids in filter_sides: + if filter_ids is not None: + edges = edges.join(filter_ids, left_on=endpoint_col, right_on=ncol, how="semi") + # A filtered side drew its ids FROM the node table; a synthesized one is vacuously closed. + sides_not_closed_by_a_filter = ( + [col for col, filter_ids in filter_sides if filter_ids is None] + if node_table_bound else []) + endpoints = endpoint_ids(edges, scol, dcol, ncol) + if sides_not_closed_by_a_filter: + from graphistry.compute.gfql.lazy import collect_all + unresolvable, nodes = collect_all([ + endpoints.lazy().join(all_ids.lazy(), on=ncol, how="anti").select(pl.len()), + gf._nodes.lazy().join(endpoints.lazy(), on=ncol, how="semi"), + ]) + if unresolvable.item() > 0: + for endpoint_col in sides_not_closed_by_a_filter: + edges = edges.join(all_ids, left_on=endpoint_col, right_on=ncol, how="semi") + nodes = gf._nodes.join( + endpoint_ids(edges, scol, dcol, ncol), on=ncol, how="semi") + else: + nodes = gf._nodes.join(endpoints, on=ncol, how="semi") + return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) if start_nodes is not None: from graphistry.Engine import Engine, df_to_engine diff --git a/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py b/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py new file mode 100644 index 0000000000..c827956999 --- /dev/null +++ b/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py @@ -0,0 +1,57 @@ +"""``polars_plain_single_hop_admits`` is the polars chain's own gate for its plain single-hop +branches. + +Pins: the decision table over the shared corpus (``"seeded-index"`` for a directed seeded hop +without a destination filter, ``"skip-combine"`` for the other plain single hops except the +filtered undirected one, None otherwise), a seeded chain (``start_nodes``) declines, and every +admitted shape stays value-identical to the pandas full path. +""" +import pandas as pd +import pytest + +import graphistry +from graphistry.compute.ast import e_forward, n +from graphistry.compute.gfql.lazy.engine.polars.chain import polars_plain_single_hop_admits +from graphistry.tests.compute.gfql.routes.corpus import CORPUS, EDGES, NODES, by_name + +pl = pytest.importorskip("polars") + +EXPECTED = { + "plain single hop, unseeded": "skip-combine", + "plain single hop, seeded": "seeded-index", + "plain single hop, seeded, reverse": "seeded-index", + "plain single hop, seeded, destination filter": "skip-combine", + "plain single hop, undirected, unconstrained": "skip-combine", + "plain single hop, undirected, seeded": None, + "single hop, prune to endpoints": "seeded-index", +} + + +def test_every_corpus_shape_has_a_verdict(): + for e in CORPUS: + assert polars_plain_single_hop_admits(e.ops(), None) == EXPECTED.get(e.name), e.name + + +@pytest.mark.parametrize("name", list(EXPECTED)) +def test_decision_table(name): + assert polars_plain_single_hop_admits(by_name()[name].ops(), None) == EXPECTED[name] + + +def test_start_nodes_decline(): + assert polars_plain_single_hop_admits([n({"key": 1}), e_forward(), n()], pd.DataFrame({"key": [1]})) is None + + +def _sig(res): + nn = res._nodes.to_pandas() if hasattr(res._nodes, "to_pandas") else res._nodes + ee = res._edges.to_pandas() if hasattr(res._edges, "to_pandas") else res._edges + return sorted(nn["key"].tolist()), sorted(map(tuple, ee[["s", "d"]].values.tolist())) + + +@pytest.mark.parametrize("name", [k for k, v in EXPECTED.items() if v is not None]) +def test_admitted_shapes_match_the_pandas_full_path(name, request): + if name == "single hop, prune to endpoints": + request.applymarker(pytest.mark.xfail(strict=True, reason="graphistry/pygraphistry#2053")) + ops = by_name()[name].ops() + g_pd = graphistry.nodes(NODES, "key").edges(EDGES, "s", "d", "eid") + g_pl = graphistry.nodes(pl.from_pandas(NODES), "key").edges(pl.from_pandas(EDGES), "s", "d", "eid") + assert _sig(g_pl.gfql(ops, engine="polars")) == _sig(g_pd.gfql(ops, engine="pandas", policy={"preload": lambda ctx: None})) diff --git a/graphistry/tests/compute/gfql/routes/__init__.py b/graphistry/tests/compute/gfql/routes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graphistry/tests/compute/gfql/routes/corpus.py b/graphistry/tests/compute/gfql/routes/corpus.py new file mode 100644 index 0000000000..9a2a7c46dc --- /dev/null +++ b/graphistry/tests/compute/gfql/routes/corpus.py @@ -0,0 +1,56 @@ +"""Shared shape corpus for the chain routes. + +Every entry is a native op-list shape variant; a route test filters the corpus with the +route's own admission predicate (the function its dispatcher calls), so one corpus is reused +across hot paths and a shape is never hand-picked per lane. Tags name the defect classes an +entry exercises so coverage can be read per class. +""" +from typing import Callable, Dict, List, NamedTuple, Tuple + +import pandas as pd + +from graphistry.compute.ast import ASTObject, e_forward, e_reverse, e_undirected, n +from graphistry.compute.predicates.numeric import GT + + +class Entry(NamedTuple): + name: str + ops: Callable[[], List[ASTObject]] + tags: Tuple[str, ...] + + +NODES = pd.DataFrame({"key": [1, 2, 3, 4, 5], "id": [10, 20, 30, 40, 50], "type": ["p", "p", "m", "m", "p"], "w": [1, 2, 3, 4, 5]}) +EDGES = pd.DataFrame({"s": [1, 1, 2, 3, 3, 4], "d": [2, 3, 3, 1, 1, 5], "type": ["KNOWS", "KNOWS", "LIKES", "KNOWS", "KNOWS", "LIKES"], "eid": [0, 1, 2, 3, 4, 5], "w": [1, 2, 3, 4, 5, 6]}) + +CORPUS: List[Entry] = [ + Entry("single node, scalar filter", lambda: [n({"id": 30})], ("single-node",)), + Entry("single node, named", lambda: [n({"id": 30}, name="a")], ("single-node", "alias")), + Entry("single node, predicate filter", lambda: [n({"w": GT(2)})], ("single-node", "predicate")), + Entry("single node, no filter", lambda: [n()], ("single-node",)), + Entry("plain single hop, unseeded", lambda: [n(), e_forward(), n()], ("single-hop", "unseeded")), + Entry("plain single hop, seeded", lambda: [n({"key": 1}), e_forward(), n()], ("single-hop", "seeded", "#2051")), + Entry("plain single hop, seeded, reverse", lambda: [n({"key": 1}), e_reverse(), n()], ("single-hop", "seeded", "reverse")), + Entry("plain single hop, seeded, destination filter", lambda: [n({"key": 1}), e_forward(), n({"id": 20})], ("single-hop", "seeded", "dest-filter", "#2051")), + Entry("plain single hop, undirected, unconstrained", lambda: [n(), e_undirected(), n()], ("single-hop", "undirected")), + Entry("plain single hop, undirected, seeded", lambda: [n({"key": 1}), e_undirected(), n()], ("single-hop", "undirected", "seeded")), + Entry("typed single hop, seeded", lambda: [n({"key": 1}), e_forward({"type": "KNOWS"}), n()], ("single-hop", "seeded", "typed")), + Entry("typed single hop, seeded, named", lambda: [n({"key": 1}, name="a"), e_forward({"type": "KNOWS"}, name="e"), n(name="b")], ("single-hop", "seeded", "typed", "alias")), + Entry("typed single hop, seeded, named, undirected", lambda: [n({"key": 1}, name="a"), e_undirected({"type": "KNOWS"}, name="e"), n(name="b")], ("single-hop", "undirected", "alias")), + Entry("single hop, node and edge alias share a name", lambda: [n({"key": 1}, name="a"), e_forward(name="a"), n()], ("single-hop", "alias", "shared-alias-name")), + Entry("single hop, edge alias = filtered column", lambda: [n({"id": 30}, name="m"), e_forward({"type": "KNOWS"}, name="type"), n(name="p")], ("single-hop", "alias-collision", "#2039")), + Entry("single hop, destination alias = its filtered column", lambda: [n({"id": 30}, name="m"), e_forward({"type": "KNOWS"}, name="e"), n({"type": "p"}, name="type")], ("single-hop", "alias-collision", "#2039")), + Entry("single hop, source node match", lambda: [n(), e_forward(source_node_match={"type": "p"}), n()], ("single-hop", "endpoint-match")), + Entry("single hop, prune to endpoints", lambda: [n({"key": 1}), e_forward(prune_to_endpoints=True), n()], ("single-hop", "prune", "#2053")), + Entry("hops=2, seeded", lambda: [n({"key": 1}), e_forward(hops=2), n()], ("multi-hop", "seeded")), + Entry("hops=2, seeded, typed, named", lambda: [n({"key": 1}, name="a"), e_forward({"type": "KNOWS"}, hops=2, name="e"), n(name="b")], ("multi-hop", "typed", "alias", "#2049")), + Entry("to_fixed_point, seeded", lambda: [n({"key": 1}), e_forward(to_fixed_point=True), n()], ("multi-hop", "fixed-point")), + Entry("two single hops", lambda: [n({"key": 1}), e_forward(), n(), e_forward(), n()], ("two-steps",)), +] + + +def tagged(tag: str) -> List[Entry]: + return [e for e in CORPUS if tag in e.tags] + + +def by_name() -> Dict[str, Entry]: + return {e.name: e for e in CORPUS} diff --git a/graphistry/tests/compute/test_chain_fast_paths_admission.py b/graphistry/tests/compute/test_chain_fast_paths_admission.py new file mode 100644 index 0000000000..6c21b8563f --- /dev/null +++ b/graphistry/tests/compute/test_chain_fast_paths_admission.py @@ -0,0 +1,93 @@ +"""``native_fast_path_admits`` is the pandas/cuDF chain fast path's own gate. + +Pins: for every corpus shape the predicate's verdict equals whether ``_try_chain_fast_path`` +served it (admits ⇔ served, on pandas and cuDF), the decision table is stable per shape, a +seeded chain (``start_nodes``) and a non pandas/cuDF engine always decline, and every served +shape stays value-identical to the full path. +""" +import pandas as pd +import pytest + +import graphistry +import graphistry.compute.chain as chain_mod +from graphistry.Engine import Engine +from graphistry.compute.ast import e_forward, n +from graphistry.compute.chain_fast_paths import native_fast_path_admits +from graphistry.tests.compute.gfql.routes.corpus import CORPUS, EDGES, NODES, by_name + +ENGINES = ["pandas", "cudf"] + + +def _graph(engine): + nodes, edges = NODES, EDGES + if engine == "cudf": + cudf = pytest.importorskip("cudf") + nodes, edges = cudf.from_pandas(nodes), cudf.from_pandas(edges) + return graphistry.nodes(nodes, "key").edges(edges, "s", "d", "eid") + + +def _served(g, ops, engine): + real = chain_mod._try_chain_fast_path + hit = {"n": 0} + + def spy(*a, **k): + r = real(*a, **k) + hit["n"] += r is not None + return r + chain_mod._try_chain_fast_path = spy + try: + res = g.gfql(ops, engine=engine) + finally: + chain_mod._try_chain_fast_path = real + return res, hit["n"] == 1 + + +def _sig(res): + nn = res._nodes.to_pandas() if hasattr(res._nodes, "to_pandas") else res._nodes + ee = res._edges.to_pandas() if hasattr(res._edges, "to_pandas") else res._edges + return sorted(nn["key"].tolist()), sorted(map(tuple, ee[["s", "d"]].values.tolist())) + + +EXPECTED = { + "single node, scalar filter": "single-node", "single node, named": "single-node", + "single node, predicate filter": "single-node", "single node, no filter": "single-node", + "plain single hop, unseeded": "seeded-hop", "plain single hop, seeded": "seeded-hop", + "plain single hop, seeded, reverse": "seeded-hop", "plain single hop, seeded, destination filter": "seeded-hop", + "plain single hop, undirected, unconstrained": "seeded-hop", "plain single hop, undirected, seeded": None, + "typed single hop, seeded": "seeded-hop", "typed single hop, seeded, named": "seeded-hop", + "typed single hop, seeded, named, undirected": None, "single hop, node and edge alias share a name": None, + "single hop, edge alias = filtered column": "seeded-hop", "single hop, destination alias = its filtered column": "seeded-hop", + "single hop, source node match": None, "single hop, prune to endpoints": None, + "hops=2, seeded": None, "hops=2, seeded, typed, named": None, "to_fixed_point, seeded": None, "two single hops": None, +} + + +def test_every_corpus_shape_has_an_expected_verdict(): + assert set(EXPECTED) == {e.name for e in CORPUS} + + +@pytest.mark.parametrize("name", list(EXPECTED)) +def test_decision_table(name): + assert native_fast_path_admits(by_name()[name].ops(), Engine.PANDAS, None) == EXPECTED[name] + + +@pytest.mark.parametrize("engine", ENGINES) +@pytest.mark.parametrize("name", list(EXPECTED)) +def test_admits_iff_served(engine, name): + g = _graph(engine) + ops = by_name()[name].ops() + admitted = native_fast_path_admits(ops, Engine(engine), None) is not None + res, served = _served(g, ops, engine) + assert served == admitted, f"{name}: predicate={admitted} served={served}" + if served: + full = g.gfql(ops, engine=engine, policy={"preload": lambda ctx: None}) + assert _sig(res) == _sig(full) + + +@pytest.mark.parametrize("engine", [Engine.POLARS, Engine.DASK]) +def test_other_engines_decline(engine): + assert native_fast_path_admits([n({"id": 30})], engine, None) is None + + +def test_start_nodes_decline(): + assert native_fast_path_admits([n({"key": 1}), e_forward(), n()], Engine.PANDAS, pd.DataFrame({"key": [1]})) is None From 459b1e71b3a6c64ef99869c60a0a036945699cc4 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 14:27:36 -0700 Subject: [PATCH 02/12] refactor(gfql): the polars seeded lane exposes its shape admission predicate `polars_seeded_lane_admits` is the structural gate `_try_seeded_chain_polars` now calls first; frame conditions (polars frames, id dtypes, valid resident indexes, scalar filters, alias collisions) stay in the body. Pins over the shared corpus: the admitted set, the lane never serves a shape it does not admit under the real dispatch, and called directly on an indexed fixture it serves every admitted non-colliding shape and declines the colliding ones. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QztW7jYsDd66e8rb8pJNQA (cherry picked from commit 083078d9a4591f8ac8dbbd8bfb0dada72929d4c7) --- graphistry/compute/chain_fast_paths.py | 30 +++++++--- .../engine/polars/test_chain_admission.py | 55 +++++++++++++++++++ 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/graphistry/compute/chain_fast_paths.py b/graphistry/compute/chain_fast_paths.py index 85c4cad036..46fb07a522 100644 --- a/graphistry/compute/chain_fast_paths.py +++ b/graphistry/compute/chain_fast_paths.py @@ -626,22 +626,34 @@ def _seeded_typed_return_dst_polars( return dstn, edges, seed_nodes, kernel_admits +def polars_seeded_lane_admits(ops: Sequence[ASTObject]) -> bool: + """Whether the polars seeded lane's shape gate admits ``ops``: a 3-op directed simple + single hop whose seed node carries a filter, with no node queries, endpoint matches, + endpoint or edge queries, zero-hop seed or endpoint pruning. The dispatcher calls this + first; the frame conditions (polars frames, matching id dtypes, valid resident indexes, + scalar-only filters, no colliding aliases) are decided by the body and can still decline + an admitted shape.""" + if len(ops) != 3: + return False + n0, e1, n2 = ops + if not (isinstance(n0, ASTNode) and isinstance(n2, ASTNode) and isinstance(e1, ASTEdge)): + return False + return not (n0.query is not None or n2.query is not None or not n0.filter_dict + or not e1.is_simple_single_hop() or e1.direction not in ("forward", "reverse") + or e1.source_node_match is not None or e1.destination_node_match is not None + or e1.source_node_query is not None or e1.destination_node_query is not None + or e1.edge_query is not None or e1.include_zero_hop_seed or e1.prune_to_endpoints) + + def _try_seeded_chain_polars(g: Plottable, ops: Sequence[ASTObject]) -> Optional[Plottable]: """Serve a native directed scalar hop through the resident seed indexes, preserving Polars table order and aliases; declines (None) without valid resident indexes.""" import polars as pl from graphistry.compute.gfql.index.api import _record_indexed_traversal - if len(ops) != 3: + if not polars_seeded_lane_admits(ops): return None n0, e1, n2 = ops - if not isinstance(n0, ASTNode) or not isinstance(n2, ASTNode) or not isinstance(e1, ASTEdge): - return None - if (n0.query is not None or n2.query is not None or not n0.filter_dict - or not e1.is_simple_single_hop() or e1.direction not in ("forward", "reverse") - or e1.source_node_match is not None or e1.destination_node_match is not None - or e1.source_node_query is not None or e1.destination_node_query is not None - or e1.edge_query is not None or e1.include_zero_hop_seed or e1.prune_to_endpoints): - return None + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) and n0.filter_dict # the predicate admitted this shape nodes, edges = g._nodes, g._edges node, src, dst = g._node, g._source, g._destination if (not isinstance(nodes, pl.DataFrame) or not isinstance(edges, pl.DataFrame) diff --git a/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py b/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py index c827956999..ee0952085f 100644 --- a/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py +++ b/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py @@ -10,7 +10,9 @@ import pytest import graphistry +import graphistry.compute.chain_fast_paths as cfp from graphistry.compute.ast import e_forward, n +from graphistry.compute.chain_fast_paths import polars_seeded_lane_admits from graphistry.compute.gfql.lazy.engine.polars.chain import polars_plain_single_hop_admits from graphistry.tests.compute.gfql.routes.corpus import CORPUS, EDGES, NODES, by_name @@ -55,3 +57,56 @@ def test_admitted_shapes_match_the_pandas_full_path(name, request): g_pd = graphistry.nodes(NODES, "key").edges(EDGES, "s", "d", "eid") g_pl = graphistry.nodes(pl.from_pandas(NODES), "key").edges(pl.from_pandas(EDGES), "s", "d", "eid") assert _sig(g_pl.gfql(ops, engine="polars")) == _sig(g_pd.gfql(ops, engine="pandas", policy={"preload": lambda ctx: None})) + + +SEEDED_LANE_ADMITS = { + "plain single hop, seeded", "plain single hop, seeded, reverse", "plain single hop, seeded, destination filter", + "typed single hop, seeded", "typed single hop, seeded, named", + "single hop, edge alias = filtered column", "single hop, destination alias = its filtered column", + "single hop, node and edge alias share a name", +} + + +def test_seeded_lane_decision_table_over_the_corpus(): + got = {e.name for e in CORPUS if polars_seeded_lane_admits(e.ops())} + assert got == SEEDED_LANE_ADMITS + + +def _indexed_polars_graph(): + g = graphistry.nodes(pl.from_pandas(NODES), "key").edges(pl.from_pandas(EDGES), "s", "d", "eid") + return g.gfql_index_all(engine="polars").gfql_index_node_props(["id"], engine="polars") + + +@pytest.mark.parametrize("name", [e.name for e in CORPUS]) +def test_seeded_lane_never_serves_a_shape_it_does_not_admit(name): + ops = by_name()[name].ops() + g = _indexed_polars_graph() + real = cfp._try_seeded_chain_polars + hit = {"n": 0} + + def spy(*a, **k): + r = real(*a, **k) + hit["n"] += r is not None + return r + cfp._try_seeded_chain_polars = spy + try: + g.gfql(ops, engine="polars", index_policy="use") + except Exception: + pass + finally: + cfp._try_seeded_chain_polars = real + assert hit["n"] == 0 or polars_seeded_lane_admits(ops), f"{name}: served without admission" + + +SEEDED_LANE_SERVES_DIRECTLY = SEEDED_LANE_ADMITS - { + # admitted by shape, declined by the body's alias-collision rule + "single hop, edge alias = filtered column", "single hop, destination alias = its filtered column", + "single hop, node and edge alias share a name", +} + + +@pytest.mark.parametrize("name", sorted(SEEDED_LANE_ADMITS)) +def test_seeded_lane_called_directly_serves_every_admitted_non_colliding_shape(name): + ops = by_name()[name].ops() + res = cfp._try_seeded_chain_polars(_indexed_polars_graph(), ops) + assert (res is not None) == (name in SEEDED_LANE_SERVES_DIRECTLY) From fb2486d070c591e34998383736477086bb570ac7 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 17:52:19 -0700 Subject: [PATCH 03/12] refactor(gfql): chain specializations live next to their admission predicates The pandas/cuDF lanes (single-node, seeded typed single hop, seeded typed RETURN-destination) move from chain.py/chain_fast_paths.py into graphistry/compute/chain_specializations/{admission,hotpaths}.py; the polars lanes (plain single-hop branches, seeded lane, RETURN-destination) move into graphistry/compute/gfql/lazy/engine/polars/chain_specializations/. chain.py and the polars chain only dispatch; chain_fast_paths.py keeps the shared seed/index helpers. No route admits or declines anything it did not before. Tests mirror the new module paths. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QztW7jYsDd66e8rb8pJNQA --- CHANGELOG.md | 2 +- bin/test-polars.sh | 4 +- graphistry/compute/chain.py | 155 +------ graphistry/compute/chain_fast_paths.py | 407 +----------------- .../compute/chain_specializations/__init__.py | 5 + .../chain_specializations/admission.py | 77 ++++ .../compute/chain_specializations/hotpaths.py | 289 +++++++++++++ .../compute/gfql/lazy/engine/polars/chain.py | 71 +-- .../polars/chain_specializations/__init__.py | 6 + .../polars/chain_specializations/admission.py | 60 +++ .../polars/chain_specializations/hotpaths.py | 214 +++++++++ graphistry/compute/gfql_fast_paths.py | 7 +- .../test_native_admission.py} | 2 +- .../polars/chain_specializations/__init__.py | 0 .../test_polars_admission.py} | 15 +- .../test_polars_native_seed_resolution.py | 2 +- 16 files changed, 682 insertions(+), 634 deletions(-) create mode 100644 graphistry/compute/chain_specializations/__init__.py create mode 100644 graphistry/compute/chain_specializations/admission.py create mode 100644 graphistry/compute/chain_specializations/hotpaths.py create mode 100644 graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py create mode 100644 graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py create mode 100644 graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py rename graphistry/tests/compute/{test_chain_fast_paths_admission.py => chain_specializations/test_native_admission.py} (97%) create mode 100644 graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py rename graphistry/tests/compute/gfql/lazy/engine/polars/{test_chain_admission.py => chain_specializations/test_polars_admission.py} (88%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11bd1614a0..690f1c01d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Changed -* GFQL: the chain hot paths expose their input-shape admission predicates as functions the dispatchers call — `native_fast_path_admits` (pandas/cuDF single-node and plain single-hop lanes) and `polars_plain_single_hop_admits` (polars seeded-index and skip-combine branches) — so tests filter one shared shape corpus per route with the route's own gate instead of re-deriving it. No route admits or declines anything it did not before; the corpus pins each predicate's decision table and served-by parity. +* GFQL: the chain specializations move into `graphistry/compute/chain_specializations/{admission,hotpaths}.py` (pandas/cuDF single-node lane, seeded typed single hop, seeded typed RETURN-destination) and `graphistry/compute/gfql/lazy/engine/polars/chain_specializations/{admission,hotpaths}.py` (polars plain single-hop branches, seeded lane, RETURN-destination), each lane next to the admission predicate the dispatcher calls (`native_fast_path_admits`, `polars_plain_single_hop_admits`, `polars_seeded_lane_admits`); `chain.py` and the polars chain only dispatch, `chain_fast_paths.py` keeps the shared seed/index helpers. No route admits or declines anything it did not before. Tests mirror the new paths and filter one shared shape corpus per route with the route's own gate; `GFQL_ROUTES_OFF=` (test conftest) makes named hot paths decline so every existing test replays through the other routes, and `bin/test-routes-off.sh` reports the per-route divergences. * GFQL: the wavefront seed-rediscovery rule moved out of `hop.py` into `graphistry/compute/gfql/seed_rediscovery.py` (pandas/cuDF) and `graphistry/compute/gfql/lazy/engine/polars/seed_rediscovery.py` (polars); `undirected_rediscovered_seed_ids` (an internal helper) is gone. ## [0.59.0 - 2026-08-31] diff --git a/bin/test-polars.sh b/bin/test-polars.sh index 6d4071d610..eb6811afb0 100755 --- a/bin/test-polars.sh +++ b/bin/test-polars.sh @@ -87,8 +87,8 @@ POLARS_TEST_FILES=( graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py - graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py - graphistry/tests/compute/test_chain_fast_paths_admission.py + graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/test_polars_admission.py + graphistry/tests/compute/chain_specializations/test_native_admission.py graphistry/tests/compute/gfql/test_undirected_pairs_2026.py graphistry/tests/compute/gfql/test_native_seed_lane_explain.py # #1882/#1913-f4/#1879 crash-family pins: the polars params (filter helpers on polars diff --git a/graphistry/compute/chain.py b/graphistry/compute/chain.py index 0a00968d0a..f58e813fe8 100644 --- a/graphistry/compute/chain.py +++ b/graphistry/compute/chain.py @@ -11,12 +11,7 @@ from .ast import ASTObject, ASTNode, ASTEdge, ASTCall, Direction, from_json as ASTObject_from_json, serialize_binding_ops from .typing import DataFrameT, SeriesT from .util import generate_safe_column_name -from .chain_fast_paths import ( - _seeded_typed_hop_pandas_cudf, - _single_node_rows_via_index_or_filter, - native_fast_path_admits, - _tag_fast_path_aliases, -) +from .chain_specializations.hotpaths import _try_chain_fast_path from graphistry.compute.validate.validate_schema import validate_chain_schema, validate_graph_shape from graphistry.compute.gfql.strictness import StrictInput from graphistry.compute.gfql.same_path_types import ( @@ -841,154 +836,6 @@ def _step_with_source_edge_columns(g: Plottable, g_step: Plottable, op: ASTObjec return g_step.edges(edges[edges[edge_id].isin(step_edges[edge_id])]) -def _try_chain_fast_path( - g_in: Plottable, - ops: List[ASTObject], - engine_concrete: Engine, - start_nodes: Optional[DataFrameT] = None, -) -> Optional[Plottable]: - """Degenerate-shape fast path (pandas/cuDF): node-only ``MATCH (n)`` or a plain - single-hop ``MATCH (a)-[e]->(b)`` skip the forward/backward/combine BFS machinery. - Returns the result Plottable, or ``None`` to fall through to the full path. - - Same node/edge sets + VALUES as the full machinery (trackA_golden + hop/chain - suites); the 1-hop additionally preserves int node dtypes (the full path upcasts - int→float via merge — the merge is the artifact, int is the Cypher-conformant type). - Gated to unqueried nodes + a plain single-hop edge; NAMED ops are served (the alias - flags are reconstructed by `_tag_fast_path_aliases`) except when undirected or when - the same alias is reused. filtered-undirected and seeded chains fall through. - polars/dask/spark also fall through (own fast path / lazy semantics).""" - from graphistry.compute.filter_by_dict import filter_by_dict - - shape = native_fast_path_admits(ops, engine_concrete, start_nodes) - if shape is None: - return None - engine_abs = EngineAbstract(engine_concrete.value) - - def _materialize_fast_path_graph() -> Plottable: - from graphistry.compute.ComputeMixin import _coerce_input_formats # lazy — avoids circular import - g = g_in.materialize_nodes(engine=EngineAbstract(engine_concrete.value)) - return _coerce_input_formats(g, engine_concrete) - - if shape == "single-node": - n0 = ops[0] - assert isinstance(n0, ASTNode) # the predicate admitted this shape - g = _materialize_fast_path_graph() - if g._nodes is None: - return None - nodes = _single_node_rows_via_index_or_filter(g, n0, engine_abs) - if n0._name is not None: - alias_was_column = n0._name in nodes.columns - if alias_was_column: - nodes = nodes.drop(columns=[n0._name]) - nodes = nodes.assign(**{n0._name: True}) - other_columns = [c for c in nodes.columns if c != n0._name] - if g._node in other_columns: - other_columns = [g._node, *[c for c in other_columns if c != g._node]] - if alias_was_column: - nodes = nodes[[*other_columns, n0._name]] - else: - nodes = nodes[[*other_columns[:1], n0._name, *other_columns[1:]]] - nodes = nodes.reset_index(drop=True) - edges = g._edges.iloc[0:0] if g._edges is not None else None - return g.nodes(nodes).edges(edges) if edges is not None else g.nodes(nodes) - - n0, e1, n2 = ops - assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) # the predicate admitted this shape - alias_n0, alias_e1, alias_n2 = n0._name, e1._name, n2._name - direction = e1.direction - unconstrained = not n0.filter_dict and not n2.filter_dict - g = _materialize_fast_path_graph() - if g._nodes is None or g._edges is None: - return None - src, dst, node = g._source, g._destination, g._node - if src is None or dst is None or node is None: - return None # no edge/node bindings -> can't fast-path; full path handles it - if alias_n0 == node or alias_n2 == node: - # A node alias EQUAL TO THE NODE-ID BINDING would make `_tag_fast_path_aliases` - # overwrite the id column with the bool flag (destroying the ids) while the full - # path raises on the same query ("The column label '' is not unique") — - # a wrong-serve found by adversarial parity testing. Decline; never serve. - return None - if alias_e1 is not None and direction in ("forward", "reverse") \ - and alias_e1 == (src if direction == "forward" else dst): - # An edge alias EQUAL TO THE HOP'S FROM-SIDE BINDING (forward+src / reverse+dst) - # made the two lanes return DIFFERENT node sets: the full path's flag overwrite - # corrupts its own node reduction, the fast path tags after reducing. TO-side - # collisions keep parity (pinned in tests) and stay served. Decline; never serve. - return None - concat = df_concat(engine_concrete) - if unconstrained: - # No node filter to reduce by: validate BOTH endpoints against the full - # node table (the full path drops dangling edges via its joins). dropna so - # a NaN node id can't validate a NaN endpoint — .isin treats NaN as - # matchable but the BFS joins never match NaN<->NaN. - node_ids = g._nodes[node].dropna() - edges = g._edges[g._edges[src].isin(node_ids) & g._edges[dst].isin(node_ids)] - if e1.edge_match: - # typed edge (e.g. -[:HAS_CREATOR]->) — same edge-frame filter the full - # hop applies, so the result set is identical. - edges = filter_by_dict(edges, e1.edge_match, engine_abs) - else: - # #1755 lever-3 seed-first: a seeded 1-hop must be O(result), not O(E). - # Reduce edges by the selective node filter(s) BEFORE the typed-edge scan - # and endpoint validation, so the expensive object/isin passes run on the - # tiny frontier, not all edges. The from-side ids come from the node table - # (so that endpoint is validated); the node gather below validates the to - # side and drops any edge dangling off the node table. - # pandas + cuDF: a scalar-filtered seeded typed hop collapses to a few - # DataFrame filters (sub-ms); falls back to the general branch below for - # predicates / undirected / missing columns (and non-pandas/cuDF engines). - if engine_concrete in (Engine.PANDAS, Engine.CUDF): - _fast_res = _seeded_typed_hop_pandas_cudf(g, n0, n2, e1, src, dst, node, direction) - if _fast_res is not None: - return _tag_fast_path_aliases( - _fast_res, alias_n0, alias_e1, alias_n2, src, dst, node, direction) - from_col, to_col = (src, dst) if direction == "forward" else (dst, src) - edges = g._edges - if n0.filter_dict: - from_ids = filter_by_dict(g._nodes, n0.filter_dict, engine_abs)[node] - edges = edges[edges[from_col].isin(from_ids)] - if e1.edge_match: - edges = filter_by_dict(edges, e1.edge_match, engine_abs) - if n2.filter_dict: - # Apply the destination filter to the SMALL set of gathered dst nodes, - # not the full node table — an O(N) object/type scan on all nodes is - # exactly the tax we're removing. Gather the frontier's dst nodes - # (small isin key), filter those, then drop edges to the losers. - to_present = edges[to_col].dropna().unique() - to_nodes = filter_by_dict( - g._nodes[g._nodes[node].isin(to_present)], n2.filter_dict, engine_abs) - edges = edges[edges[to_col].isin(to_nodes[node])] - # Validate endpoints + build result nodes on the reduced edge set (small - # isin key -> small hashtable; no O(E)-values scan). Engine-agnostic - # (pandas + cuDF): gather candidate endpoint nodes, drop edges dangling off - # the node table, then keep only nodes still referenced by a surviving edge. - ep = concat([ - edges[[src]].rename(columns={src: node}), - edges[[dst]].rename(columns={dst: node}), - ]).drop_duplicates() - cand = g._nodes[g._nodes[node].isin(ep[node])].drop_duplicates(subset=[node]) - valid = cand[node].dropna() - edges = edges[edges[src].isin(valid) & edges[dst].isin(valid)] - final = concat([ - edges[[src]].rename(columns={src: node}), - edges[[dst]].rename(columns={dst: node}), - ]).drop_duplicates() - nodes = cand[cand[node].isin(final[node])] - return _tag_fast_path_aliases( - g.nodes(nodes).edges(edges), alias_n0, alias_e1, alias_n2, src, dst, node, direction) - endpoints = concat([ - edges[[src]].rename(columns={src: node}), - edges[[dst]].rename(columns={dst: node}), - ]).drop_duplicates() - nodes = g._nodes[g._nodes[node].isin(endpoints[node])] - # match the full path's merge, which collapses duplicate node-id rows - nodes = nodes.drop_duplicates(subset=[node]) - return _tag_fast_path_aliases( - g.nodes(nodes).edges(edges), alias_n0, alias_e1, alias_n2, src, dst, node, direction) - - def reject_alias_named_like_binding( g: Plottable, chain_obj: "Chain", *, include_edge_endpoint_aliases: bool = False ) -> None: diff --git a/graphistry/compute/chain_fast_paths.py b/graphistry/compute/chain_fast_paths.py index 46fb07a522..66e0c75704 100644 --- a/graphistry/compute/chain_fast_paths.py +++ b/graphistry/compute/chain_fast_paths.py @@ -1,24 +1,17 @@ -"""Seeded typed-hop fast-path specializations for the chain executor. - -Extracted verbatim from chain.py (#1755) to keep that orchestrator readable: the seeded -typed 1-hop pandas/cuDF reduction and the seeded typed RETURN-destination pandas/cuDF and -polars reductions. Pure code move, no behavior change. chain.py imports from here (one -direction); this module imports only leaf modules (no back-edge into chain.py). - -New fast-path helpers belong HERE, not in chain.py — that is the direction this module was -created to establish and it is why `_tag_fast_path_aliases` lives alongside the seeded -reductions rather than next to `_try_chain_fast_path`. -""" +"""Seed-resolution and resident-index helpers shared by the chain specializations +(``chain_specializations/``, ``gfql/lazy/engine/polars/chain_specializations/``) and the Cypher +lanes in ``gfql_fast_paths.py``. This module imports only leaf modules (no back-edge into +``chain.py`` or the specialization packages).""" # ruff: noqa: E501 from typing import Any, Dict, Literal, Optional, Sequence, Tuple, TYPE_CHECKING, Union, cast from graphistry.Plottable import Plottable -from .ast import ASTObject, ASTNode, ASTEdge, Direction +from .ast import Direction from .typing import ArrayLike, ArrayNamespace, DataFrameT, SeriesT if TYPE_CHECKING: - from graphistry.Engine import Engine, EngineAbstract + from graphistry.Engine import Engine from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex @@ -80,45 +73,6 @@ def _tag_fast_path_aliases( return res.nodes(nodes).edges(edges) -NativeFastPathShape = Literal["single-node", "seeded-hop"] - - -def native_fast_path_admits( - ops: Sequence[ASTObject], engine: "Engine", start_nodes: Optional[DataFrameT], -) -> Optional[NativeFastPathShape]: - """The shape the pandas/cuDF chain fast path serves for ``ops``, or None when the full - path must run. This is the dispatcher's own gate (``chain._try_chain_fast_path`` calls - it first), so a test that filters a shape corpus with it exercises exactly what the - dispatcher admits: a node-only op without ``query``, or a 3-op plain single hop whose - node ops carry no ``query``, whose edge carries no node matches, queries, zero-hop seed - or endpoint pruning, whose aliases are distinct, and which is not an undirected hop - with names or with node filters. Seeded chains (``start_nodes``) and other engines - decline. Frame-dependent conditions (missing frames, alias equal to the node binding) - are checked by the body after materialization.""" - from graphistry.Engine import Engine - if engine not in (Engine.PANDAS, Engine.CUDF) or start_nodes is not None: - return None - if len(ops) == 1: - n0 = ops[0] - return "single-node" if isinstance(n0, ASTNode) and n0.query is None else None - if len(ops) != 3: - return None - n0, e1, n2 = ops - if not (isinstance(n0, ASTNode) and n0.query is None and isinstance(n2, ASTNode) and n2.query is None): - return None - if not (isinstance(e1, ASTEdge) and e1.is_simple_single_hop() - and e1.source_node_match is None and e1.destination_node_match is None - and e1.source_node_query is None and e1.destination_node_query is None - and e1.edge_query is None and not e1.include_zero_hop_seed and not e1.prune_to_endpoints): - return None - named = [a for a in (n0._name, e1._name, n2._name) if a is not None] - if len(named) != len(set(named)): - return None - if e1.direction == "undirected" and (n0._name is not None or n2._name is not None or n0.filter_dict or n2.filter_dict): - return None - return "seeded-hop" - - def _seeded_scalar_filters(fd: Optional[Dict[str, Any]], df: DataFrameT) -> Optional[Dict[str, Any]]: """Resolve a filter dict to plain scalar column==value pairs, or None to bail to the general path. Mirrors filter_by_dict.resolve_filter_column exactly for @@ -296,6 +250,7 @@ def _seed_rows_via_prop_index_frame( SeedRowsHow = Literal["node_id_index", "property_index", "scan"] +SeededReturn = Tuple[DataFrameT, DataFrameT, DataFrameT, bool] def _seed_node_rows( @@ -327,27 +282,6 @@ def _seed_node_rows( return _filter_frame(seed, filter_dict if filter_dict is not None else n0f, engine), how -def _single_node_rows_via_index_or_filter( - g: Plottable, n0: ASTNode, engine_abs: "EngineAbstract", -) -> DataFrameT: - """Resolve a single node op through a resident index or the canonical filter.""" - from .filter_by_dict import filter_by_dict - nodes_df = g._nodes - assert nodes_df is not None - if not n0.filter_dict: - return nodes_df - node = g._node - n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) if node is not None else None - if node is not None and n0f: - nid_ctx = _resident_node_id_index(g, nodes_df, node) - rows, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) - if how != "scan": - _record_native_seed_lane(nodes_df, seam="native_seed_lookup", reason=how, hop_count=0, - public_seed_scan=node not in n0.filter_dict) - return rows - return filter_by_dict(nodes_df, n0.filter_dict, engine_abs) - - def _record_native_seed_lane( nodes_df: DataFrameT, *, seam: str, reason: str, hop_count: int, public_seed_scan: bool, ) -> None: @@ -376,330 +310,3 @@ def _index_edge_rows( return None rows, _ = lookup_edge_rows(adj, arr, xp) return take_rows(edges_df, xp.sort(rows) if preserve_input_order else rows, engine) - - -def _indexed_kernel_admits( - seed_nodes: DataFrameT, gathered_edges: Optional[DataFrameT], n0f: Dict[str, object], - node: str, how: SeedRowsHow, ctx: Tuple["NodeIdIndex", "AdjacencyIndex", ArrayNamespace, "Engine"], - n_nodes: int, n_edges: int, -) -> bool: - """Whether the indexed connected-bindings kernel would have served this seeded 1-hop: - its seed admission (binding-column integer seed, property-index hit, or a scan on a - graph with fewer nodes than edges) and its frontier and gather cost gates.""" - from numbers import Integral - from graphistry.compute.gfql.index.cost import cost_gate_frac - _, adj, _, engine = ctx - seed_val = n0f.get(node) - seeded_on_binding = isinstance(seed_val, Integral) and not isinstance(seed_val, bool) - if not (seeded_on_binding or how == "property_index" or n_nodes < n_edges): - return False - frac = cost_gate_frac(engine) - n_frontier = int(seed_nodes[node].nunique()) if not hasattr(seed_nodes, "get_column") \ - else int(seed_nodes.get_column(node).n_unique()) - if n_frontier >= frac * adj.n_keys: - return False - return gathered_edges is not None and len(gathered_edges) < frac * n_edges - - -def _seeded_typed_hop_pandas_cudf( - g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, - src: str, dst: str, node: str, direction: Direction, -) -> Optional[Plottable]: - """#1755 lever-3: engine-generic (pandas + cuDF) fast path for a scalar-filtered - seeded typed 1-hop. Value-identical to the general seeded branch for the covered - shape (all node/edge filters are plain scalars, directed) — same rows, columns, - and dtypes; row order and RangeIndex may differ — collapsing it into a - few DataFrame filters so a seeded lookup lands sub-ms. Uses only the shared - pandas/cuDF DataFrame API (no numpy array drops) so the same body runs on both - engines. Returns None to fall back for anything it does not cover (predicates, - undirected, missing columns) — the caller then runs the general branch.""" - if direction == "undirected": - return None - - nodes_df, edges_df = g._nodes, g._edges - if nodes_df is None or edges_df is None: - return None - n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) - n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) - ef = _seeded_scalar_filters(e1.edge_match, edges_df) - if n0f is None or n2f is None or ef is None: - return None - from_col, to_col = (src, dst) if direction == "forward" else (dst, src) - - # from-side seed FIRST: reduce edges to the seed's out-edges before the - # edge_match compare, so the type filter runs on the tiny frontier rather than - # all edges — this is what makes a seeded lookup sub-ms. The id filter goes - # first (int, unique -> ~1 row in one pass) so any remaining object filters - # (label__X->type) run on that tiny survivor frame, not the whole node table. - # Resident-index acceleration (#1658 x #1755): when node-id + directional - # adjacency indexes are valid for these exact frames, the O(N) seed scan, the - # O(E) frontier isin, and the O(N) candidate gather become positional lookups. - # Any decline (no index, stale fingerprint, unsafe id cast) falls back to the - # scan body below — identical results either way, only speed differs. - # Decoupled index use: the seed row lookup uses the node-id index ONLY when - # the seed filter includes the binding column, but the frontier edge gather - # (CSR adjacency) and candidate gathers (node-id index) engage regardless of - # HOW the seed rows were found — their inputs are binding-column values, which - # are the index key domain. (LDBC/user pattern: seed on the `id` PROPERTY - # while the graph binds a different key column — previously disqualified the - # whole index path.) - ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) if n0f else None - seed_nodes = edges = cand = None - if ctx is not None: - nid, adj, xp, idx_engine = ctx - seed_nodes, _ = _seed_node_rows(g, nodes_df, n0f, node, (nid, xp, idx_engine), n0.filter_dict) - edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) - if edges is not None: - if ef: - for k, v in ef.items(): - edges = edges[edges[k] == v] - if 'cudf' in str(type(edges).__module__): - import cudf as _cd # type: ignore - endpoint_ids = _cd.concat([edges[src], edges[dst]]) - else: - import pandas as _pd - endpoint_ids = _pd.concat([edges[src], edges[dst]]) - cand = _index_node_rows(nid, endpoint_ids, xp, idx_engine, nodes_df) - served_via_index = cand is not None - if cand is None: - if n0f: - seed_nodes = nodes_df - for k, v in sorted(n0f.items(), key=lambda kv: 0 if kv[0] == node else 1): - seed_nodes = seed_nodes[seed_nodes[k] == v] - edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())] - else: - edges = edges_df - if ef: # typed edge (edge_match) — now on the reduced frontier - for k, v in ef.items(): - edges = edges[edges[k] == v] - - # Gather candidate endpoint nodes (both endpoints of surviving edges), then run - # the dest filter, dangling-edge drop and final-node selection on the small - # candidate/edge frames. Selecting from nodes_df keeps only real nodes, so the - # endpoint-in-nodes check subsumes the old NaN-endpoint guard. Membership sets - # are dropna()'d: pandas .isin matches NaN<->NaN, but the general branch's BFS - # joins never join on null keys, so a null id/endpoint must not link. - cand = nodes_df[ - nodes_df[node].isin(edges[src].dropna()) | nodes_df[node].isin(edges[dst].dropna()) - ].drop_duplicates(subset=[node]) - assert edges is not None and cand is not None # both branches above assign - if served_via_index: - _record_native_seed_lane(nodes_df, seam="native_seeded_hop", reason="served", hop_count=1, - public_seed_scan=node not in n0f) - if n2f: # destination-node filter (to-side) - n2_cand = cand - for k, v in n2f.items(): - n2_cand = n2_cand[n2_cand[k] == v] - n2_ok = n2_cand[node] - else: - n2_ok = cand[node] - to_vals = edges[to_col] - keep = edges[src].isin(cand[node].dropna()) & edges[dst].isin(cand[node].dropna()) & to_vals.isin(n2_ok.dropna()) - edges = edges[keep] - cand = cand[cand[node].isin(edges[src]) | cand[node].isin(edges[dst])] - return g.nodes(cand).edges(edges) - - -SeededReturn = Tuple[DataFrameT, DataFrameT, DataFrameT, bool] - - -def _seeded_typed_return_dst_pandas_cudf( - g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, - src: str, dst: str, node: str, direction: Direction, -) -> Optional[SeededReturn]: - """#1755 cypher RETURN-alias fast path: like _seeded_typed_hop_pandas_cudf but - returns ONLY the destination (RETURN-alias) node rows + surviving edges — no - seed-node gather, no Plottable round-trip — so the seeded cypher projection - lands sub-ms. Engine-generic (pandas + cuDF): only the shared DataFrame API, - no numpy array drops. Returns ``(dst_node_rows, edges)`` or None to fall back.""" - if direction == "undirected": - return None - nodes_df, edges_df = g._nodes, g._edges - if nodes_df is None or edges_df is None: - return None - n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) - n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) - ef = _seeded_scalar_filters(e1.edge_match, edges_df) - if n0f is None or n2f is None or ef is None or not n0f: - return None - from_col, to_col = (src, dst) if direction == "forward" else (dst, src) - # id-first seed reduction: filter by the id column first (int/unique -> ~1 row) - # so any remaining object filters (label__X->type) run on the tiny survivor - # frame, never materializing an object column over the whole node table. - # Membership sets are dropna()'d: pandas .isin matches NaN<->NaN, but the full - # pipeline's joins never join on null keys, so a null id/endpoint must not link. - ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) - nid_ctx = (ctx[0], ctx[2], ctx[3]) if ctx is not None else _resident_node_id_index(g, nodes_df, node) - seed_nodes, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) - edges = dstn = None - kernel_admits = False - if ctx is not None: - nid, adj, xp, idx_engine = ctx - edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) - kernel_admits = _indexed_kernel_admits( - seed_nodes, edges, n0f, node, how, ctx, len(nodes_df), len(edges_df)) - if edges is not None: - if ef: - for k, v in ef.items(): - edges = edges[edges[k] == v] - dstn = _index_node_rows(nid, edges[to_col], xp, idx_engine, nodes_df) - if dstn is None: - edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())] - if ef: - for k, v in ef.items(): - edges = edges[edges[k] == v] - # destination nodes = real nodes that are edge to-endpoints, then the dest - # filter, dangling-edge drop and dedup on the small dst/edge frames. - dstn = nodes_df[nodes_df[node].isin(edges[to_col].dropna())] - assert edges is not None and dstn is not None # both branches above assign - if n2f: - for k, v in n2f.items(): - dstn = dstn[dstn[k] == v] - edges = edges[edges[to_col].isin(dstn[node].dropna())] - dstn = dstn[dstn[node].isin(edges[to_col].dropna())].drop_duplicates(subset=[node]) - return dstn, edges, seed_nodes, kernel_admits - - -def _seeded_typed_return_dst_polars( - g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, - src: str, dst: str, node: str, direction: Direction, - preserve_input_order: bool = False, - index_ctx: Optional[Tuple["NodeIdIndex", "AdjacencyIndex", ArrayNamespace, "Engine"]] = None, -) -> Optional[SeededReturn]: - """#1755 polars analog of _seeded_typed_return_dst_pandas_cudf: same seed-first - reduction (seed out-edges -> typed-edge filter -> destination nodes) expressed - with polars filters, so a seeded cypher RETURN on polars/polars-gpu also lands - sub-ms. Returns ``(dst_node_rows, edges)`` (polars frames) or None to fall back - to the full lazy pipeline. Value-identical node set to the full path for the - covered shape (scalar filters, directed, single hop); row order may differ.""" - import polars as pl - from graphistry.compute.gfql.lazy.engine.polars.predicates import filter_by_dict_polars - if direction == "undirected": - return None - nodes_df, edges_df = g._nodes, g._edges - # Eager polars frames only: LazyFrame has no get_column, and mixed-engine - # node/edge frames must take the full path — decline rather than crash. - if not isinstance(nodes_df, pl.DataFrame) or not isinstance(edges_df, pl.DataFrame): - return None - - n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) - n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) - ef = _seeded_scalar_filters(e1.edge_match, edges_df) - if n0f is None or n2f is None or ef is None or not n0f: - return None - from_col, to_col = (src, dst) if direction == "forward" else (dst, src) - - # from-side seed: reduce the node frame to the seed rows, take their ids. - # Membership sets are drop_nulls()'d (null ids/endpoints never link, matching - # the full pipeline's joins) and passed via .implode() (Series-arg is_in is - # deprecated in polars 1.42, see polars#22149). - ctx = index_ctx if index_ctx is not None else _resident_seed_indexes( - g, nodes_df, edges_df, node, src, dst, direction) - nid_ctx = (ctx[0], ctx[2], ctx[3]) if ctx is not None else _resident_node_id_index(g, nodes_df, node) - seed_nodes, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) - edges = dstn = None - kernel_admits = False - if ctx is not None: - nid, adj, xp, idx_engine = ctx - edges = _index_edge_rows( - adj, seed_nodes.get_column(node), xp, idx_engine, edges_df, - preserve_input_order=preserve_input_order) - kernel_admits = _indexed_kernel_admits( - seed_nodes, edges, n0f, node, how, ctx, len(nodes_df), len(edges_df)) - if edges is not None: - edges = filter_by_dict_polars(edges, e1.edge_match) - dstn = _index_node_rows(nid, edges.get_column(to_col), xp, idx_engine, nodes_df) - if dstn is None: - from_ids = seed_nodes.get_column(node).drop_nulls() - if from_ids.len() == 0: - return nodes_df.clear(), edges_df.clear(), seed_nodes, kernel_admits - edges = edges_df.filter(pl.col(from_col).is_in(from_ids.implode())) - edges = filter_by_dict_polars(edges, e1.edge_match) - dst_ids = edges.get_column(to_col).drop_nulls().unique() - dstn = nodes_df.filter(pl.col(node).is_in(dst_ids.implode())) - assert edges is not None and dstn is not None # both branches above assign - dstn = filter_by_dict_polars(dstn, n2.filter_dict) - # drop dangling edges + dedup destination nodes (mirror the pandas tail) - keep_ids = dstn.get_column(node).drop_nulls() - edges = edges.filter(pl.col(to_col).is_in(keep_ids.implode())) - dstn = dstn.filter(pl.col(node).is_in(edges.get_column(to_col).implode())).unique(subset=[node], maintain_order=True) - return dstn, edges, seed_nodes, kernel_admits - - -def polars_seeded_lane_admits(ops: Sequence[ASTObject]) -> bool: - """Whether the polars seeded lane's shape gate admits ``ops``: a 3-op directed simple - single hop whose seed node carries a filter, with no node queries, endpoint matches, - endpoint or edge queries, zero-hop seed or endpoint pruning. The dispatcher calls this - first; the frame conditions (polars frames, matching id dtypes, valid resident indexes, - scalar-only filters, no colliding aliases) are decided by the body and can still decline - an admitted shape.""" - if len(ops) != 3: - return False - n0, e1, n2 = ops - if not (isinstance(n0, ASTNode) and isinstance(n2, ASTNode) and isinstance(e1, ASTEdge)): - return False - return not (n0.query is not None or n2.query is not None or not n0.filter_dict - or not e1.is_simple_single_hop() or e1.direction not in ("forward", "reverse") - or e1.source_node_match is not None or e1.destination_node_match is not None - or e1.source_node_query is not None or e1.destination_node_query is not None - or e1.edge_query is not None or e1.include_zero_hop_seed or e1.prune_to_endpoints) - - -def _try_seeded_chain_polars(g: Plottable, ops: Sequence[ASTObject]) -> Optional[Plottable]: - """Serve a native directed scalar hop through the resident seed indexes, preserving - Polars table order and aliases; declines (None) without valid resident indexes.""" - import polars as pl - from graphistry.compute.gfql.index.api import _record_indexed_traversal - if not polars_seeded_lane_admits(ops): - return None - n0, e1, n2 = ops - assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) and n0.filter_dict # the predicate admitted this shape - nodes, edges = g._nodes, g._edges - node, src, dst = g._node, g._source, g._destination - if (not isinstance(nodes, pl.DataFrame) or not isinstance(edges, pl.DataFrame) - or node is None or src is None or dst is None): - return None - if nodes.schema[node] != edges.schema[src] or nodes.schema[node] != edges.schema[dst]: - return None - aliases = [op._name for op in ops if op._name is not None] - if len(aliases) != len(set(aliases)): - return None - if any(name in nodes.columns for name in (n0._name, n2._name) if name is not None): - return None - if e1._name is not None and e1._name in edges.columns: - return None - ctx = _resident_seed_indexes(g, nodes, edges, node, src, dst, e1.direction) - if ctx is None: - return None - reduced = _seeded_typed_return_dst_polars( - g, n0, n2, e1, src, dst, node, e1.direction, preserve_input_order=True, index_ctx=ctx) - if reduced is None: - return None - _, kept_edges, _, _ = reduced - if not isinstance(kept_edges, pl.DataFrame): - return None - endpoint_ids = pl.concat([kept_edges.get_column(src), kept_edges.get_column(dst)]).drop_nulls().unique() - nid_ctx = _resident_node_id_index(g, nodes, node) - result_nodes = None - if nid_ctx is not None: - nid, xp, engine = nid_ctx - result_nodes = _index_node_rows(nid, endpoint_ids, xp, engine, nodes, preserve_input_order=True) - if result_nodes is None: - result_nodes = nodes.filter(pl.col(node).is_in(endpoint_ids.implode())) - if result_nodes.get_column(node).n_unique() != result_nodes.height: - return None - if not isinstance(result_nodes, pl.DataFrame): - return None - from_col, to_col = (src, dst) if e1.direction == "forward" else (dst, src) - flags = [ - pl.col(node).is_in(kept_edges.get_column(endpoint).implode()).fill_null(False).alias(name) - for name, endpoint in ((n0._name, from_col), (n2._name, to_col)) if name is not None - ] - if flags: - result_nodes = result_nodes.with_columns(flags) - if e1._name is not None: - kept_edges = kept_edges.with_columns(pl.lit(True).alias(e1._name)) - _record_indexed_traversal( - seam="native_seeded_hop", engine=ctx[3], served=True, reason="served", hop_count=1, - public_seed_scan=node not in n0.filter_dict, hop_details=[{"hop": 1}]) - return g.nodes(result_nodes).edges(kept_edges) diff --git a/graphistry/compute/chain_specializations/__init__.py b/graphistry/compute/chain_specializations/__init__.py new file mode 100644 index 0000000000..3d9404a89a --- /dev/null +++ b/graphistry/compute/chain_specializations/__init__.py @@ -0,0 +1,5 @@ +"""Chain specializations for the pandas/cuDF engines: admission predicates and their lanes.""" +from .admission import NativeFastPathShape, native_fast_path_admits +from .hotpaths import _try_chain_fast_path + +__all__ = ["NativeFastPathShape", "native_fast_path_admits", "_try_chain_fast_path"] diff --git a/graphistry/compute/chain_specializations/admission.py b/graphistry/compute/chain_specializations/admission.py new file mode 100644 index 0000000000..54afc01d33 --- /dev/null +++ b/graphistry/compute/chain_specializations/admission.py @@ -0,0 +1,77 @@ +"""Shape admission for the pandas/cuDF chain specializations: the dispatcher and the tests +consult the same predicates, so a test that filters a shape corpus with them exercises exactly +what the dispatcher admits.""" +# ruff: noqa: E501 + +from typing import Dict, Literal, Optional, Sequence, Tuple, TYPE_CHECKING + +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge +from graphistry.compute.chain_fast_paths import SeedRowsHow +from graphistry.compute.typing import ArrayNamespace, DataFrameT + +if TYPE_CHECKING: + from graphistry.Engine import Engine + from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex + + +NativeFastPathShape = Literal["single-node", "seeded-hop"] + + +def native_fast_path_admits( + ops: Sequence[ASTObject], engine: "Engine", start_nodes: Optional[DataFrameT], +) -> Optional[NativeFastPathShape]: + """The shape the pandas/cuDF chain fast path serves for ``ops``, or None when the full + path must run. This is the dispatcher's own gate (``chain._try_chain_fast_path`` calls + it first), so a test that filters a shape corpus with it exercises exactly what the + dispatcher admits: a node-only op without ``query``, or a 3-op plain single hop whose + node ops carry no ``query``, whose edge carries no node matches, queries, zero-hop seed + or endpoint pruning, whose aliases are distinct, and which is not an undirected hop + with names or with node filters. Seeded chains (``start_nodes``) and other engines + decline. Frame-dependent conditions (missing frames, alias equal to the node binding) + are checked by the body after materialization.""" + from graphistry.Engine import Engine + if engine not in (Engine.PANDAS, Engine.CUDF) or start_nodes is not None: + return None + if len(ops) == 1: + n0 = ops[0] + return "single-node" if isinstance(n0, ASTNode) and n0.query is None else None + if len(ops) != 3: + return None + n0, e1, n2 = ops + if not (isinstance(n0, ASTNode) and n0.query is None and isinstance(n2, ASTNode) and n2.query is None): + return None + if not (isinstance(e1, ASTEdge) and e1.is_simple_single_hop() + and e1.source_node_match is None and e1.destination_node_match is None + and e1.source_node_query is None and e1.destination_node_query is None + and e1.edge_query is None and not e1.include_zero_hop_seed and not e1.prune_to_endpoints): + return None + named = [a for a in (n0._name, e1._name, n2._name) if a is not None] + if len(named) != len(set(named)): + return None + if e1.direction == "undirected" and (n0._name is not None or n2._name is not None or n0.filter_dict or n2.filter_dict): + return None + return "seeded-hop" + + + +def _indexed_kernel_admits( + seed_nodes: DataFrameT, gathered_edges: Optional[DataFrameT], n0f: Dict[str, object], + node: str, how: SeedRowsHow, ctx: Tuple["NodeIdIndex", "AdjacencyIndex", ArrayNamespace, "Engine"], + n_nodes: int, n_edges: int, +) -> bool: + """Whether the indexed connected-bindings kernel would have served this seeded 1-hop: + its seed admission (binding-column integer seed, property-index hit, or a scan on a + graph with fewer nodes than edges) and its frontier and gather cost gates.""" + from numbers import Integral + from graphistry.compute.gfql.index.cost import cost_gate_frac + _, adj, _, engine = ctx + seed_val = n0f.get(node) + seeded_on_binding = isinstance(seed_val, Integral) and not isinstance(seed_val, bool) + if not (seeded_on_binding or how == "property_index" or n_nodes < n_edges): + return False + frac = cost_gate_frac(engine) + n_frontier = int(seed_nodes[node].nunique()) if not hasattr(seed_nodes, "get_column") \ + else int(seed_nodes.get_column(node).n_unique()) + if n_frontier >= frac * adj.n_keys: + return False + return gathered_edges is not None and len(gathered_edges) < frac * n_edges diff --git a/graphistry/compute/chain_specializations/hotpaths.py b/graphistry/compute/chain_specializations/hotpaths.py new file mode 100644 index 0000000000..5b0ed8ce66 --- /dev/null +++ b/graphistry/compute/chain_specializations/hotpaths.py @@ -0,0 +1,289 @@ +"""The pandas/cuDF chain specializations: the single-node lane, the seeded typed single hop +and the seeded typed RETURN-destination reduction. Each lane sits next to its admission +predicate (``admission.py``); ``chain.py`` only dispatches.""" +# ruff: noqa: E501 + +from typing import List, Optional, Sequence, Tuple, TYPE_CHECKING, cast + +from graphistry.Engine import Engine, EngineAbstract, df_concat +from graphistry.Plottable import Plottable +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, Direction +from graphistry.compute.chain_fast_paths import ( + _ids_to_key_array, _index_edge_rows, _index_node_rows, _record_native_seed_lane, + _resident_node_id_index, _resident_seed_indexes, _seed_node_rows, _seeded_scalar_filters, + _tag_fast_path_aliases, SeededReturn, +) +from graphistry.compute.typing import ArrayLike, ArrayNamespace, DataFrameT, SeriesT +from .admission import _indexed_kernel_admits, native_fast_path_admits + +if TYPE_CHECKING: + from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex + + +def _single_node_rows_via_index_or_filter( + g: Plottable, n0: ASTNode, engine_abs: "EngineAbstract", +) -> DataFrameT: + """Resolve a single node op through a resident index or the canonical filter.""" + from graphistry.compute.filter_by_dict import filter_by_dict + nodes_df = g._nodes + assert nodes_df is not None + if not n0.filter_dict: + return nodes_df + node = g._node + n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) if node is not None else None + if node is not None and n0f: + nid_ctx = _resident_node_id_index(g, nodes_df, node) + rows, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) + if how != "scan": + _record_native_seed_lane(nodes_df, seam="native_seed_lookup", reason=how, hop_count=0, + public_seed_scan=node not in n0.filter_dict) + return rows + return filter_by_dict(nodes_df, n0.filter_dict, engine_abs) + + + +def _seeded_typed_hop_pandas_cudf( + g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, + src: str, dst: str, node: str, direction: Direction, +) -> Optional[Plottable]: + """Engine-generic (pandas + cuDF) fast path for a scalar-filtered + seeded typed 1-hop. Value-identical to the general seeded branch for the covered + shape (all node/edge filters are plain scalars, directed) — same rows, columns, + and dtypes; row order and RangeIndex may differ — collapsing it into a + few DataFrame filters so a seeded lookup lands sub-ms. Uses only the shared + pandas/cuDF DataFrame API (no numpy array drops) so the same body runs on both + engines. Returns None to fall back for anything it does not cover (predicates, + undirected, missing columns) — the caller then runs the general branch.""" + if direction == "undirected": + return None + + nodes_df, edges_df = g._nodes, g._edges + if nodes_df is None or edges_df is None: + return None + n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) + n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) + ef = _seeded_scalar_filters(e1.edge_match, edges_df) + if n0f is None or n2f is None or ef is None: + return None + from_col, to_col = (src, dst) if direction == "forward" else (dst, src) + + # seed first; valid resident indexes serve the seed, frontier and gather positionally, any decline falls back to the scan body with identical results + ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) if n0f else None + seed_nodes = edges = cand = None + if ctx is not None: + nid, adj, xp, idx_engine = ctx + seed_nodes, _ = _seed_node_rows(g, nodes_df, n0f, node, (nid, xp, idx_engine), n0.filter_dict) + edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) + if edges is not None: + if ef: + for k, v in ef.items(): + edges = edges[edges[k] == v] + if 'cudf' in str(type(edges).__module__): + import cudf as _cd # type: ignore + endpoint_ids = _cd.concat([edges[src], edges[dst]]) + else: + import pandas as _pd + endpoint_ids = _pd.concat([edges[src], edges[dst]]) + cand = _index_node_rows(nid, endpoint_ids, xp, idx_engine, nodes_df) + served_via_index = cand is not None + if cand is None: + if n0f: + seed_nodes = nodes_df + for k, v in sorted(n0f.items(), key=lambda kv: 0 if kv[0] == node else 1): + seed_nodes = seed_nodes[seed_nodes[k] == v] + edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())] + else: + edges = edges_df + if ef: # typed edge (edge_match) — now on the reduced frontier + for k, v in ef.items(): + edges = edges[edges[k] == v] + + # membership sets are dropna()'d: null ids never link, matching the full path's joins + cand = nodes_df[ + nodes_df[node].isin(edges[src].dropna()) | nodes_df[node].isin(edges[dst].dropna()) + ].drop_duplicates(subset=[node]) + assert edges is not None and cand is not None # both branches above assign + if served_via_index: + _record_native_seed_lane(nodes_df, seam="native_seeded_hop", reason="served", hop_count=1, + public_seed_scan=node not in n0f) + if n2f: # destination-node filter (to-side) + n2_cand = cand + for k, v in n2f.items(): + n2_cand = n2_cand[n2_cand[k] == v] + n2_ok = n2_cand[node] + else: + n2_ok = cand[node] + to_vals = edges[to_col] + keep = edges[src].isin(cand[node].dropna()) & edges[dst].isin(cand[node].dropna()) & to_vals.isin(n2_ok.dropna()) + edges = edges[keep] + cand = cand[cand[node].isin(edges[src]) | cand[node].isin(edges[dst])] + return g.nodes(cand).edges(edges) + + +def _seeded_typed_return_dst_pandas_cudf( + g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, + src: str, dst: str, node: str, direction: Direction, +) -> Optional[SeededReturn]: + """Cypher RETURN-alias fast path: like _seeded_typed_hop_pandas_cudf but + returns ONLY the destination (RETURN-alias) node rows + surviving edges — no + seed-node gather, no Plottable round-trip — so the seeded cypher projection + lands sub-ms. Engine-generic (pandas + cuDF): only the shared DataFrame API, + no numpy array drops. Returns ``(dst_node_rows, edges)`` or None to fall back.""" + if direction == "undirected": + return None + nodes_df, edges_df = g._nodes, g._edges + if nodes_df is None or edges_df is None: + return None + n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) + n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) + ef = _seeded_scalar_filters(e1.edge_match, edges_df) + if n0f is None or n2f is None or ef is None or not n0f: + return None + from_col, to_col = (src, dst) if direction == "forward" else (dst, src) + # id filter first, then the object filters on the survivors; membership sets are dropna()'d so null ids never link + ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) + nid_ctx = (ctx[0], ctx[2], ctx[3]) if ctx is not None else _resident_node_id_index(g, nodes_df, node) + seed_nodes, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) + edges = dstn = None + kernel_admits = False + if ctx is not None: + nid, adj, xp, idx_engine = ctx + edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) + kernel_admits = _indexed_kernel_admits( + seed_nodes, edges, n0f, node, how, ctx, len(nodes_df), len(edges_df)) + if edges is not None: + if ef: + for k, v in ef.items(): + edges = edges[edges[k] == v] + dstn = _index_node_rows(nid, edges[to_col], xp, idx_engine, nodes_df) + if dstn is None: + edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())] + if ef: + for k, v in ef.items(): + edges = edges[edges[k] == v] + # destination nodes: real nodes that are to-endpoints of the surviving edges + dstn = nodes_df[nodes_df[node].isin(edges[to_col].dropna())] + assert edges is not None and dstn is not None # both branches above assign + if n2f: + for k, v in n2f.items(): + dstn = dstn[dstn[k] == v] + edges = edges[edges[to_col].isin(dstn[node].dropna())] + dstn = dstn[dstn[node].isin(edges[to_col].dropna())].drop_duplicates(subset=[node]) + return dstn, edges, seed_nodes, kernel_admits + + + +def _try_chain_fast_path( + g_in: Plottable, + ops: List[ASTObject], + engine_concrete: Engine, + start_nodes: Optional[DataFrameT] = None, +) -> Optional[Plottable]: + """Degenerate-shape fast path (pandas/cuDF): node-only ``MATCH (n)`` or a plain + single-hop ``MATCH (a)-[e]->(b)`` skip the forward/backward/combine BFS machinery. + Returns the result Plottable, or ``None`` to fall through to the full path. + + Same node/edge sets + VALUES as the full machinery (trackA_golden + hop/chain + suites); the 1-hop additionally preserves int node dtypes (the full path upcasts + int→float via merge — the merge is the artifact, int is the Cypher-conformant type). + Gated to unqueried nodes + a plain single-hop edge; NAMED ops are served (the alias + flags are reconstructed by `_tag_fast_path_aliases`) except when undirected or when + the same alias is reused. filtered-undirected and seeded chains fall through. + polars/dask/spark also fall through (own fast path / lazy semantics).""" + from graphistry.compute.filter_by_dict import filter_by_dict + + shape = native_fast_path_admits(ops, engine_concrete, start_nodes) + if shape is None: + return None + engine_abs = EngineAbstract(engine_concrete.value) + + def _materialize_fast_path_graph() -> Plottable: + from graphistry.compute.ComputeMixin import _coerce_input_formats # lazy — avoids circular import + g = g_in.materialize_nodes(engine=EngineAbstract(engine_concrete.value)) + return _coerce_input_formats(g, engine_concrete) + + if shape == "single-node": + n0 = ops[0] + assert isinstance(n0, ASTNode) # the predicate admitted this shape + g = _materialize_fast_path_graph() + if g._nodes is None: + return None + nodes = _single_node_rows_via_index_or_filter(g, n0, engine_abs) + if n0._name is not None: + alias_was_column = n0._name in nodes.columns + if alias_was_column: + nodes = nodes.drop(columns=[n0._name]) + nodes = nodes.assign(**{n0._name: True}) + other_columns = [c for c in nodes.columns if c != n0._name] + if g._node in other_columns: + other_columns = [g._node, *[c for c in other_columns if c != g._node]] + if alias_was_column: + nodes = nodes[[*other_columns, n0._name]] + else: + nodes = nodes[[*other_columns[:1], n0._name, *other_columns[1:]]] + nodes = nodes.reset_index(drop=True) + edges = g._edges.iloc[0:0] if g._edges is not None else None + return g.nodes(nodes).edges(edges) if edges is not None else g.nodes(nodes) + + n0, e1, n2 = ops + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) # the predicate admitted this shape + alias_n0, alias_e1, alias_n2 = n0._name, e1._name, n2._name + direction = e1.direction + unconstrained = not n0.filter_dict and not n2.filter_dict + g = _materialize_fast_path_graph() + if g._nodes is None or g._edges is None: + return None + src, dst, node = g._source, g._destination, g._node + if src is None or dst is None or node is None: + return None # no edge/node bindings -> can't fast-path; full path handles it + if alias_n0 == node or alias_n2 == node: + return None # a node alias equal to the node-id binding: the full path raises, never serve + if alias_e1 is not None and direction in ("forward", "reverse") \ + and alias_e1 == (src if direction == "forward" else dst): + return None # an edge alias equal to the from-side binding: lanes disagree on the node set + concat = df_concat(engine_concrete) + if unconstrained: + node_ids = g._nodes[node].dropna() # validate both endpoints; NaN ids never match + edges = g._edges[g._edges[src].isin(node_ids) & g._edges[dst].isin(node_ids)] + if e1.edge_match: + edges = filter_by_dict(edges, e1.edge_match, engine_abs) + else: + if engine_concrete in (Engine.PANDAS, Engine.CUDF): # seed-first: reduce edges by the node filters before the edge scan + _fast_res = _seeded_typed_hop_pandas_cudf(g, n0, n2, e1, src, dst, node, direction) + if _fast_res is not None: + return _tag_fast_path_aliases( + _fast_res, alias_n0, alias_e1, alias_n2, src, dst, node, direction) + from_col, to_col = (src, dst) if direction == "forward" else (dst, src) + edges = g._edges + if n0.filter_dict: + from_ids = filter_by_dict(g._nodes, n0.filter_dict, engine_abs)[node] + edges = edges[edges[from_col].isin(from_ids)] + if e1.edge_match: + edges = filter_by_dict(edges, e1.edge_match, engine_abs) + if n2.filter_dict: + to_present = edges[to_col].dropna().unique() + to_nodes = filter_by_dict( + g._nodes[g._nodes[node].isin(to_present)], n2.filter_dict, engine_abs) + edges = edges[edges[to_col].isin(to_nodes[node])] + ep = concat([ + edges[[src]].rename(columns={src: node}), + edges[[dst]].rename(columns={dst: node}), + ]).drop_duplicates() + cand = g._nodes[g._nodes[node].isin(ep[node])].drop_duplicates(subset=[node]) + valid = cand[node].dropna() + edges = edges[edges[src].isin(valid) & edges[dst].isin(valid)] + final = concat([ + edges[[src]].rename(columns={src: node}), + edges[[dst]].rename(columns={dst: node}), + ]).drop_duplicates() + nodes = cand[cand[node].isin(final[node])] + return _tag_fast_path_aliases( + g.nodes(nodes).edges(edges), alias_n0, alias_e1, alias_n2, src, dst, node, direction) + endpoints = concat([ + edges[[src]].rename(columns={src: node}), + edges[[dst]].rename(columns={dst: node}), + ]).drop_duplicates() + nodes = g._nodes[g._nodes[node].isin(endpoints[node])] + nodes = nodes.drop_duplicates(subset=[node]) # the full path's merge collapses duplicate node-id rows + return _tag_fast_path_aliases( + g.nodes(nodes).edges(edges), alias_n0, alias_e1, alias_n2, src, dst, node, direction) diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 5458027e55..4e4ebface9 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -18,7 +18,9 @@ from graphistry.Plottable import Plottable from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge -from graphistry.compute.chain_fast_paths import _single_node_rows_via_index_or_filter, _try_seeded_chain_polars +from graphistry.compute.chain_specializations.hotpaths import _single_node_rows_via_index_or_filter +from .chain_specializations.admission import polars_plain_single_hop_admits +from .chain_specializations.hotpaths import _plain_seeded_index_hop_polars, _plain_single_hop_polars, _try_seeded_chain_polars if TYPE_CHECKING: import polars as pl @@ -222,41 +224,6 @@ def _exec(op: ASTObject, g: Plottable, prev_wf: Optional[Any], target_wf: Option raise NotImplementedError(f"polars chain engine does not support op {type(op).__name__}") -PolarsPlainSingleHopShape = Literal["seeded-index", "skip-combine"] - - -def _plain_node(op: ASTObject) -> bool: - return isinstance(op, ASTNode) and op._name is None and op.query is None - - -def _plain_edge(op: ASTObject) -> bool: - return (isinstance(op, ASTEdge) and op.is_simple_single_hop() - and op.edge_match is None and op.source_node_match is None - and op.destination_node_match is None and op._name is None - and op.source_node_query is None and op.destination_node_query is None - and op.edge_query is None and not op.include_zero_hop_seed) - - -def polars_plain_single_hop_admits(ops: Sequence[ASTObject], start_nodes: Optional[object]) -> Optional[PolarsPlainSingleHopShape]: - """The polars chain's plain single-hop branch for ``ops``: ``"seeded-index"`` when the - resident-index hop is consulted first (seed filter, no destination filter, directed), - ``"skip-combine"`` when the one-hop endpoint filter serves it without the - forward/backward/combine passes, None when the full chain runs. The dispatcher calls - this; unnamed, unqueried nodes and an unnamed, unmatched simple edge are the shape. - A filtered undirected hop is the one plain shape that still takes the full chain.""" - if start_nodes is not None or len(ops) != 3: - return None - n0, e1, n2 = ops - if not (_plain_node(n0) and _plain_edge(e1) and _plain_node(n2)): - return None - assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) - directed = e1.direction in ("forward", "reverse") - if n0.filter_dict and not n2.filter_dict and directed: - return "seeded-index" - unconstrained = not n0.filter_dict and not n2.filter_dict - return "skip-combine" if (unconstrained or directed) else None - - def _is_native_multihop(op: ASTObject) -> bool: """Multi-hop shapes the native combine supports: hops=N / max_hops (fwd/rev/undirected), to_fixed_point (fwd/rev), min_hops>1 (fwd/rev, finite max), optionally with match/name. @@ -1026,40 +993,15 @@ def _chain_traversal_polars(self: Plottable, ops, start_nodes: Optional[Any] = N "undirected edges in multi-edge chains; deferred undirected sub-cases — " "include_zero_hop_seed or *_query — require engine='pandas'." ) - - # Single-hop shape: [n(), e, n()] with no names/queries/matches (`MATCH (a {f})-[e]->(b)`). - # Result = edges whose endpoints pass the node filters + those endpoint nodes - # (isolated/dead-ends excluded); one hop means the backward pass prunes nothing more, so skip - # forward/backward/combine. Byte-identical vs pandas (verified: src/dst/both filters, reverse, - # dup/self-loop/cycle/isolated). Undirected takes this branch only when UNCONSTRAINED; - # filtered-undirected (OR of both directions) falls through to the full path. plain_shape = polars_plain_single_hop_admits(ops, start_nodes) - - # GFQL physical index path for the seeded single-hop shape - # `MATCH (a {id-filter})-[e]->(b)` (forward/reverse, no destination filter). This native - # chain branch never reaches compute/hop.py, so it must consult the index here too. - from graphistry.compute.gfql.index import get_index_policy - _idx_pol = get_index_policy(self) if plain_shape == "seeded-index": - from graphistry.compute.gfql.index import get_registry, maybe_index_hop - if (not get_registry(self).is_empty()) or _idx_pol in ("auto", "force"): - gf0 = ensure_nodes_polars(self) - seed0 = filter_by_dict_polars(gf0._nodes, ops[0].filter_dict) - from graphistry.Engine import Engine - from graphistry.compute.gfql.lazy import active_target, ExecutionTarget - _eng0 = Engine.POLARS_GPU if active_target() == ExecutionTarget.GPU else Engine.POLARS - _idxed0 = maybe_index_hop( - gf0, _eng0, nodes=seed0, hops=1, direction=ops[1].direction, - return_as_wave_front=False, to_fixed_point=False, policy=_idx_pol, - ) - if _idxed0 is not None: - return _idxed0 - + indexed = _plain_seeded_index_hop_polars(self, ops) + if indexed is not None: + return indexed if start_nodes is None: seeded = _try_seeded_chain_polars(self, ops) if seeded is not None: return seeded - if plain_shape is not None: n0, e1, n2 = ops node_table_bound = self._nodes is not None @@ -1123,6 +1065,7 @@ def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": else: nodes = gf._nodes.join(endpoints, on=ncol, how="semi") return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) + return _plain_single_hop_polars(self, ops) if start_nodes is not None: from graphistry.Engine import Engine, df_to_engine diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py new file mode 100644 index 0000000000..c228d1a9ba --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py @@ -0,0 +1,6 @@ +"""Chain specializations for the polars engine: admission predicates and their lanes.""" +from .admission import PolarsPlainSingleHopShape, polars_plain_single_hop_admits, polars_seeded_lane_admits +from .hotpaths import _plain_seeded_index_hop_polars, _plain_single_hop_polars, _try_seeded_chain_polars + +__all__ = ["PolarsPlainSingleHopShape", "polars_plain_single_hop_admits", "polars_seeded_lane_admits", + "_plain_seeded_index_hop_polars", "_plain_single_hop_polars", "_try_seeded_chain_polars"] diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py new file mode 100644 index 0000000000..c47b8d8153 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py @@ -0,0 +1,60 @@ +"""Shape admission for the polars chain specializations: the plain single-hop branches and +the seeded lane. The dispatcher and the tests consult the same predicates.""" + +from typing import Literal, Optional, Sequence + +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge + + +PolarsPlainSingleHopShape = Literal["seeded-index", "skip-combine"] + + +def _plain_node(op: ASTObject) -> bool: + return isinstance(op, ASTNode) and op._name is None and op.query is None + + +def _plain_edge(op: ASTObject) -> bool: + return (isinstance(op, ASTEdge) and op.is_simple_single_hop() + and op.edge_match is None and op.source_node_match is None + and op.destination_node_match is None and op._name is None + and op.source_node_query is None and op.destination_node_query is None + and op.edge_query is None and not op.include_zero_hop_seed) + + +def polars_plain_single_hop_admits(ops: Sequence[ASTObject], start_nodes: Optional[object]) -> Optional[PolarsPlainSingleHopShape]: + """The polars chain's plain single-hop branch for ``ops``: ``"seeded-index"`` when the + resident-index hop is consulted first (seed filter, no destination filter, directed), + ``"skip-combine"`` when the one-hop endpoint filter serves it without the + forward/backward/combine passes, None when the full chain runs. The dispatcher calls + this; unnamed, unqueried nodes and an unnamed, unmatched simple edge are the shape. + A filtered undirected hop is the one plain shape that still takes the full chain.""" + if start_nodes is not None or len(ops) != 3: + return None + n0, e1, n2 = ops + if not (_plain_node(n0) and _plain_edge(e1) and _plain_node(n2)): + return None + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) + directed = e1.direction in ("forward", "reverse") + if n0.filter_dict and not n2.filter_dict and directed: + return "seeded-index" + unconstrained = not n0.filter_dict and not n2.filter_dict + return "skip-combine" if (unconstrained or directed) else None + + +def polars_seeded_lane_admits(ops: Sequence[ASTObject]) -> bool: + """Whether the polars seeded lane's shape gate admits ``ops``: a 3-op directed simple + single hop whose seed node carries a filter, with no node queries, endpoint matches, + endpoint or edge queries, zero-hop seed or endpoint pruning. The dispatcher calls this + first; the frame conditions (polars frames, matching id dtypes, valid resident indexes, + scalar-only filters, no colliding aliases) are decided by the body and can still decline + an admitted shape.""" + if len(ops) != 3: + return False + n0, e1, n2 = ops + if not (isinstance(n0, ASTNode) and isinstance(n2, ASTNode) and isinstance(e1, ASTEdge)): + return False + return not (n0.query is not None or n2.query is not None or not n0.filter_dict + or not e1.is_simple_single_hop() or e1.direction not in ("forward", "reverse") + or e1.source_node_match is not None or e1.destination_node_match is not None + or e1.source_node_query is not None or e1.destination_node_query is not None + or e1.edge_query is not None or e1.include_zero_hop_seed or e1.prune_to_endpoints) diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py new file mode 100644 index 0000000000..4ca131063c --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py @@ -0,0 +1,214 @@ +"""The polars chain specializations: the plain single-hop branches (resident-index consult and +the skip-combine pass), the seeded lane and the seeded typed RETURN-destination reduction. Each +lane sits next to its admission predicate (``admission.py``); ``chain.py`` only dispatches.""" +# ruff: noqa: E501 + +from typing import Any, Dict, Optional, Sequence, Tuple, TYPE_CHECKING, Union, cast + +from graphistry.Plottable import Plottable +from graphistry.Engine import Engine +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, Direction +from graphistry.compute.chain_specializations.admission import _indexed_kernel_admits +from graphistry.compute.chain_fast_paths import ( + _ids_to_key_array, _index_edge_rows, _index_node_rows, _record_native_seed_lane, + _resident_node_id_index, _resident_seed_indexes, _seed_node_rows, _seeded_scalar_filters, + SeededReturn, +) +from graphistry.compute.endpoint_utils import drop_null_endpoint_edges +from graphistry.compute.typing import ArrayLike, ArrayNamespace, DataFrameT +from ..dtypes import endpoint_ids +from ..hop_eager import ensure_nodes_polars +from ..predicates import filter_by_dict_polars +from .admission import polars_seeded_lane_admits + +if TYPE_CHECKING: + import polars as pl + from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex + from ..dtypes import PolarsFrame + + +def _plain_seeded_index_hop_polars(g: Plottable, ops: Sequence[ASTObject]) -> Optional[Plottable]: + """Consult the resident index for the plain seeded single hop; None when it declines.""" + from graphistry.Engine import Engine + from graphistry.compute.gfql.index import get_index_policy, get_registry, maybe_index_hop + from graphistry.compute.gfql.lazy import active_target, ExecutionTarget + n0, e1 = ops[0], ops[1] + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) # the predicate admitted this shape + policy = get_index_policy(g) + if get_registry(g).is_empty() and policy not in ("auto", "force"): + return None + gf0 = ensure_nodes_polars(g) + seed0 = filter_by_dict_polars(gf0._nodes, n0.filter_dict) + engine = Engine.POLARS_GPU if active_target() == ExecutionTarget.GPU else Engine.POLARS + return maybe_index_hop( + gf0, engine, nodes=seed0, hops=1, direction=e1.direction, + return_as_wave_front=False, to_fixed_point=False, policy=policy, + ) + + +def _plain_single_hop_polars(g: Plottable, ops: Sequence[ASTObject]) -> Plottable: + """Serve the plain single hop as one endpoint-filter pass, skipping forward/backward/combine.""" + import polars as pl + from graphistry.compute.gfql.lazy.engine.polars.chain import _align_edge_endpoints, _restore_edge_dtypes + n0, e1, n2 = ops + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) # the predicate admitted this shape + node_table_bound = g._nodes is not None + gf = ensure_nodes_polars(g) + ncol, scol, dcol = gf._node, gf._source, gf._destination + assert ncol is not None and scol is not None and dcol is not None + gf, restore = _align_edge_endpoints(gf, ncol, scol, dcol) + edges = drop_null_endpoint_edges(gf._edges, scol, dcol) + n_from, n_to = (n0, n2) if e1.direction != "reverse" else (n2, n0) + all_ids = gf._nodes.select(pl.col(ncol)) + + def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": + if not node_op.filter_dict: + return None + return filter_by_dict_polars(gf._nodes, node_op.filter_dict).select(pl.col(ncol)) + + filter_sides = ((scol, _filter_ids(n_from)), (dcol, _filter_ids(n_to))) + for endpoint_col, filter_ids in filter_sides: + if filter_ids is not None: + edges = edges.join(filter_ids, left_on=endpoint_col, right_on=ncol, how="semi") + # A filtered side drew its ids FROM the node table; a synthesized one is vacuously closed. + sides_not_closed_by_a_filter = ( + [col for col, filter_ids in filter_sides if filter_ids is None] + if node_table_bound else []) + endpoints = endpoint_ids(edges, scol, dcol, ncol) + if sides_not_closed_by_a_filter: + from graphistry.compute.gfql.lazy import collect_all + unresolvable, nodes = collect_all([ + endpoints.lazy().join(all_ids.lazy(), on=ncol, how="anti").select(pl.len()), + gf._nodes.lazy().join(endpoints.lazy(), on=ncol, how="semi"), + ]) + if unresolvable.item() > 0: + for endpoint_col in sides_not_closed_by_a_filter: + edges = edges.join(all_ids, left_on=endpoint_col, right_on=ncol, how="semi") + nodes = gf._nodes.join( + endpoint_ids(edges, scol, dcol, ncol), on=ncol, how="semi") + else: + nodes = gf._nodes.join(endpoints, on=ncol, how="semi") + return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) + + +def _seeded_typed_return_dst_polars( + g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, + src: str, dst: str, node: str, direction: Direction, + preserve_input_order: bool = False, + index_ctx: Optional[Tuple["NodeIdIndex", "AdjacencyIndex", ArrayNamespace, "Engine"]] = None, +) -> Optional[SeededReturn]: + """Polars analog of _seeded_typed_return_dst_pandas_cudf: same seed-first + reduction (seed out-edges -> typed-edge filter -> destination nodes) expressed + with polars filters, so a seeded cypher RETURN on polars/polars-gpu also lands + sub-ms. Returns ``(dst_node_rows, edges)`` (polars frames) or None to fall back + to the full lazy pipeline. Value-identical node set to the full path for the + covered shape (scalar filters, directed, single hop); row order may differ.""" + import polars as pl + from graphistry.compute.gfql.lazy.engine.polars.predicates import filter_by_dict_polars + if direction == "undirected": + return None + nodes_df, edges_df = g._nodes, g._edges + # eager polars frames only; mixed-engine node/edge frames take the full path + if not isinstance(nodes_df, pl.DataFrame) or not isinstance(edges_df, pl.DataFrame): + return None + + n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) + n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) + ef = _seeded_scalar_filters(e1.edge_match, edges_df) + if n0f is None or n2f is None or ef is None or not n0f: + return None + from_col, to_col = (src, dst) if direction == "forward" else (dst, src) + + # membership sets are drop_nulls()'d (null ids never link) and passed via implode() (Series-arg is_in is deprecated) + ctx = index_ctx if index_ctx is not None else _resident_seed_indexes( + g, nodes_df, edges_df, node, src, dst, direction) + nid_ctx = (ctx[0], ctx[2], ctx[3]) if ctx is not None else _resident_node_id_index(g, nodes_df, node) + seed_nodes, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) + edges = dstn = None + kernel_admits = False + if ctx is not None: + nid, adj, xp, idx_engine = ctx + edges = _index_edge_rows( + adj, seed_nodes.get_column(node), xp, idx_engine, edges_df, + preserve_input_order=preserve_input_order) + kernel_admits = _indexed_kernel_admits( + seed_nodes, edges, n0f, node, how, ctx, len(nodes_df), len(edges_df)) + if edges is not None: + edges = filter_by_dict_polars(edges, e1.edge_match) + dstn = _index_node_rows(nid, edges.get_column(to_col), xp, idx_engine, nodes_df) + if dstn is None: + from_ids = seed_nodes.get_column(node).drop_nulls() + if from_ids.len() == 0: + return nodes_df.clear(), edges_df.clear(), seed_nodes, kernel_admits + edges = edges_df.filter(pl.col(from_col).is_in(from_ids.implode())) + edges = filter_by_dict_polars(edges, e1.edge_match) + dst_ids = edges.get_column(to_col).drop_nulls().unique() + dstn = nodes_df.filter(pl.col(node).is_in(dst_ids.implode())) + assert edges is not None and dstn is not None # both branches above assign + dstn = filter_by_dict_polars(dstn, n2.filter_dict) + # drop dangling edges + dedup destination nodes (mirror the pandas tail) + keep_ids = dstn.get_column(node).drop_nulls() + edges = edges.filter(pl.col(to_col).is_in(keep_ids.implode())) + dstn = dstn.filter(pl.col(node).is_in(edges.get_column(to_col).implode())).unique(subset=[node], maintain_order=True) + return dstn, edges, seed_nodes, kernel_admits + + + +def _try_seeded_chain_polars(g: Plottable, ops: Sequence[ASTObject]) -> Optional[Plottable]: + """Serve a native directed scalar hop through the resident seed indexes, preserving + Polars table order and aliases; declines (None) without valid resident indexes.""" + import polars as pl + from graphistry.compute.gfql.index.api import _record_indexed_traversal + if not polars_seeded_lane_admits(ops): + return None + n0, e1, n2 = ops + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) and n0.filter_dict # the predicate admitted this shape + nodes, edges = g._nodes, g._edges + node, src, dst = g._node, g._source, g._destination + if (not isinstance(nodes, pl.DataFrame) or not isinstance(edges, pl.DataFrame) + or node is None or src is None or dst is None): + return None + if nodes.schema[node] != edges.schema[src] or nodes.schema[node] != edges.schema[dst]: + return None + aliases = [op._name for op in ops if op._name is not None] + if len(aliases) != len(set(aliases)): + return None + if any(name in nodes.columns for name in (n0._name, n2._name) if name is not None): + return None + if e1._name is not None and e1._name in edges.columns: + return None + ctx = _resident_seed_indexes(g, nodes, edges, node, src, dst, e1.direction) + if ctx is None: + return None + reduced = _seeded_typed_return_dst_polars( + g, n0, n2, e1, src, dst, node, e1.direction, preserve_input_order=True, index_ctx=ctx) + if reduced is None: + return None + _, kept_edges, _, _ = reduced + if not isinstance(kept_edges, pl.DataFrame): + return None + endpoint_ids = pl.concat([kept_edges.get_column(src), kept_edges.get_column(dst)]).drop_nulls().unique() + nid_ctx = _resident_node_id_index(g, nodes, node) + result_nodes = None + if nid_ctx is not None: + nid, xp, engine = nid_ctx + result_nodes = _index_node_rows(nid, endpoint_ids, xp, engine, nodes, preserve_input_order=True) + if result_nodes is None: + result_nodes = nodes.filter(pl.col(node).is_in(endpoint_ids.implode())) + if result_nodes.get_column(node).n_unique() != result_nodes.height: + return None + if not isinstance(result_nodes, pl.DataFrame): + return None + from_col, to_col = (src, dst) if e1.direction == "forward" else (dst, src) + flags = [ + pl.col(node).is_in(kept_edges.get_column(endpoint).implode()).fill_null(False).alias(name) + for name, endpoint in ((n0._name, from_col), (n2._name, to_col)) if name is not None + ] + if flags: + result_nodes = result_nodes.with_columns(flags) + if e1._name is not None: + kept_edges = kept_edges.with_columns(pl.lit(True).alias(e1._name)) + _record_indexed_traversal( + seam="native_seeded_hop", engine=ctx[3], served=True, reason="served", hop_count=1, + public_seed_scan=node not in n0.filter_dict, hop_details=[{"hop": 1}]) + return g.nodes(result_nodes).edges(kept_edges) diff --git a/graphistry/compute/gfql_fast_paths.py b/graphistry/compute/gfql_fast_paths.py index c3467c36d6..468a72a9ae 100644 --- a/graphistry/compute/gfql_fast_paths.py +++ b/graphistry/compute/gfql_fast_paths.py @@ -3748,10 +3748,9 @@ def _execute_seeded_typed_hop_fast_path( # intermediate graph, so trusting requested_engine would run polars ops on a # pandas frame (and vice versa). The pandas branch also covers cuDF (shared API). from graphistry.Engine import is_polars_df - from graphistry.compute.chain_fast_paths import ( - _seeded_typed_return_dst_pandas_cudf, _seeded_typed_return_dst_polars, - _resident_seed_indexes, - ) + from graphistry.compute.chain_fast_paths import _resident_seed_indexes + from graphistry.compute.chain_specializations.hotpaths import _seeded_typed_return_dst_pandas_cudf + from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths import _seeded_typed_return_dst_polars nodes_frame = base_graph._nodes is_polars = is_polars_df(nodes_frame) if is_polars != is_polars_df(base_graph._edges): diff --git a/graphistry/tests/compute/test_chain_fast_paths_admission.py b/graphistry/tests/compute/chain_specializations/test_native_admission.py similarity index 97% rename from graphistry/tests/compute/test_chain_fast_paths_admission.py rename to graphistry/tests/compute/chain_specializations/test_native_admission.py index 6c21b8563f..c4cf60431e 100644 --- a/graphistry/tests/compute/test_chain_fast_paths_admission.py +++ b/graphistry/tests/compute/chain_specializations/test_native_admission.py @@ -12,7 +12,7 @@ import graphistry.compute.chain as chain_mod from graphistry.Engine import Engine from graphistry.compute.ast import e_forward, n -from graphistry.compute.chain_fast_paths import native_fast_path_admits +from graphistry.compute.chain_specializations.admission import native_fast_path_admits from graphistry.tests.compute.gfql.routes.corpus import CORPUS, EDGES, NODES, by_name ENGINES = ["pandas", "cudf"] diff --git a/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py b/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py b/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/test_polars_admission.py similarity index 88% rename from graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py rename to graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/test_polars_admission.py index ee0952085f..b083940b1c 100644 --- a/graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_admission.py +++ b/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/test_polars_admission.py @@ -10,10 +10,10 @@ import pytest import graphistry -import graphistry.compute.chain_fast_paths as cfp +import graphistry.compute.gfql.lazy.engine.polars.chain as pchain +import graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths as hot from graphistry.compute.ast import e_forward, n -from graphistry.compute.chain_fast_paths import polars_seeded_lane_admits -from graphistry.compute.gfql.lazy.engine.polars.chain import polars_plain_single_hop_admits +from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.admission import polars_plain_single_hop_admits, polars_seeded_lane_admits from graphistry.tests.compute.gfql.routes.corpus import CORPUS, EDGES, NODES, by_name pl = pytest.importorskip("polars") @@ -81,20 +81,20 @@ def _indexed_polars_graph(): def test_seeded_lane_never_serves_a_shape_it_does_not_admit(name): ops = by_name()[name].ops() g = _indexed_polars_graph() - real = cfp._try_seeded_chain_polars + real = pchain._try_seeded_chain_polars hit = {"n": 0} def spy(*a, **k): r = real(*a, **k) hit["n"] += r is not None return r - cfp._try_seeded_chain_polars = spy + pchain._try_seeded_chain_polars = spy try: g.gfql(ops, engine="polars", index_policy="use") except Exception: pass finally: - cfp._try_seeded_chain_polars = real + pchain._try_seeded_chain_polars = real assert hit["n"] == 0 or polars_seeded_lane_admits(ops), f"{name}: served without admission" @@ -105,8 +105,9 @@ def spy(*a, **k): } +@pytest.mark.route_engaged("polars-seeded") @pytest.mark.parametrize("name", sorted(SEEDED_LANE_ADMITS)) def test_seeded_lane_called_directly_serves_every_admitted_non_colliding_shape(name): ops = by_name()[name].ops() - res = cfp._try_seeded_chain_polars(_indexed_polars_graph(), ops) + res = hot._try_seeded_chain_polars(_indexed_polars_graph(), ops) assert (res is not None) == (name in SEEDED_LANE_SERVES_DIRECTLY) diff --git a/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py b/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py index 8f58ff7354..8c95ce13ac 100644 --- a/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py +++ b/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py @@ -134,7 +134,7 @@ def test_stale_property_index_does_not_select_old_seed(monkeypatch): @pytest.mark.parametrize("case", ["lazy", "mixed", "varlen", "collision", "duplicate", "undirected"]) def test_native_seeded_hop_declines_unsupported_shapes(case): from graphistry.compute.ast import e_undirected - from graphistry.compute.chain_fast_paths import _try_seeded_chain_polars + from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths import _try_seeded_chain_polars g = _graph() ops = [n({"id": 104}, name="m"), e_forward({"type": "T"}), n(name="p")] if case == "lazy": From ad1c3b7c38f0101aede4d643cebf1880668a670b Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 17:52:19 -0700 Subject: [PATCH 04/12] test(gfql): route-off replay with an engagement-pin marker and a non-blocking CI ledger GFQL_ROUTES_OFF= (tests/conftest.py) makes named hot paths decline so every existing test replays through the other routes. Tests that assert a route serves carry @pytest.mark.route_engaged(, ...) and are skipped in that mode, so bin/test-routes-off.sh reports result divergences only; the gfql-routes-off CI matrix uploads the per-route ledger without blocking. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QztW7jYsDd66e8rb8pJNQA --- .github/workflows/ci.yml | 57 +++++++++++++ bin/test-routes-off.sh | 17 ++++ .../compute/gfql/cypher/test_lowering.py | 5 ++ .../compute/gfql/index/test_degree_consult.py | 2 + .../tests/compute/gfql/index/test_index.py | 19 +++++ .../gfql/index/test_indexed_bindings.py | 13 +++ .../gfql/test_endpoint_closure_matrix.py | 1 + .../compute/gfql/test_engine_polars_chain.py | 1 + .../compute/gfql/test_fast_path_engagement.py | 4 + .../gfql/test_gfql_latency_contract.py | 2 + .../test_known_cross_engine_divergences.py | 1 + .../gfql/test_native_seed_lane_explain.py | 2 + .../gfql/test_native_seed_resolution_2027.py | 2 + .../gfql/test_polars_rows_entity_groupby.py | 1 + .../gfql/test_seeded_node_lookup_fastpath.py | 6 ++ .../gfql/test_seeded_typed_hop_fastpath.py | 52 ++++++++---- graphistry/tests/compute/test_chain.py | 6 ++ graphistry/tests/conftest.py | 79 +++++++++++++++++++ pytest.ini | 1 + 19 files changed, 255 insertions(+), 16 deletions(-) create mode 100755 bin/test-routes-off.sh create mode 100644 graphistry/tests/conftest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ae6a3a4e7..d5c9115b69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1589,6 +1589,63 @@ jobs: source pygraphistry/bin/activate ./bin/test-graphviz.sh + gfql-routes-off: + # Non-blocking ledger: replays the GFQL suites with one hot path declined per cell + # (GFQL_ROUTES_OFF); engagement pins are skipped via the route_engaged marker, so the + # uploaded lists hold route-vs-general result divergences only. + needs: [changes, test-gfql-core, generate-lockfiles] + if: ${{ needs.changes.outputs.gfql == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} + runs-on: ubuntu-latest + continue-on-error: true + timeout-minutes: 45 + + strategy: + fail-fast: false + matrix: + mode: [native-fast, polars-seeded, polars-plain, index-hop, indexed-kernel, cypher-fast, all-off] + + steps: + + - name: Checkout repo + uses: actions/checkout@v4 + with: + lfs: true + persist-credentials: false + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Download lockfiles + uses: actions/download-artifact@v4 + with: + name: lockfiles + path: requirements + + - name: Install Python dependencies + run: | + python -m venv pygraphistry + source pygraphistry/bin/activate + python -m pip install --upgrade pip uv + uv pip install --require-hashes -r requirements/test-polars-py3.12.lock + uv pip install -e . --no-deps + + - name: Routes-off replay (${{ matrix.mode }}) + run: | + source pygraphistry/bin/activate + MODES=${{ matrix.mode }} ./bin/test-routes-off.sh + echo "## routes-off: ${{ matrix.mode }}" >> "$GITHUB_STEP_SUMMARY" + sed 's/^/- /' build/routes-off/${{ matrix.mode }}.divergences >> "$GITHUB_STEP_SUMMARY" + + - name: Upload ledger + if: always() + uses: actions/upload-artifact@v4 + with: + name: routes-off-${{ matrix.mode }} + path: build/routes-off/ + if-no-files-found: ignore + test-polars: needs: [changes, test-minimal-python, test-gfql-core, generate-lockfiles] if: ${{ ((needs.changes.outputs.python == 'true' && needs.changes.outputs.narrow_python_only != 'true') || needs.changes.outputs.gfql == 'true' || needs.changes.outputs.pandas_compat == 'true' || needs.changes.outputs.core == 'true' || needs.changes.outputs.infra == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') && !(needs.changes.outputs.docs_only_latest == 'true' && (github.event_name == 'push' || github.event_name == 'pull_request')) }} diff --git a/bin/test-routes-off.sh b/bin/test-routes-off.sh new file mode 100755 index 0000000000..9cf1a3ef4f --- /dev/null +++ b/bin/test-routes-off.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Replay the GFQL suites with each hot path declined (GFQL_ROUTES_OFF, see graphistry/tests/conftest.py). +# Engagement pins carry the route_engaged marker and are skipped, so every remaining failure is a +# route-vs-general result divergence. Non-blocking ledger: always exits 0; per-mode logs + id lists in $OUT. +set -uo pipefail +cd "$(dirname "$0")/.." +MODES=${MODES:-native-fast polars-seeded polars-plain index-hop indexed-kernel cypher-fast all-off} +SUITES=${SUITES:-graphistry/tests/compute/test_chain.py graphistry/tests/compute/test_hop.py graphistry/tests/compute/test_gfql.py graphistry/tests/compute/gfql} +OUT=${OUT:-build/routes-off} +mkdir -p "$OUT" +for mode in $MODES; do + if [ "$mode" = all-off ]; then routes=native-fast,polars-seeded,polars-plain,index-hop,indexed-kernel,cypher-fast; else routes=$mode; fi + GFQL_ROUTES_OFF=$routes python -m pytest $SUITES -q -p no:cacheprovider -o addopts="" -rfE > "$OUT/$mode.log" 2>&1 + grep -E "^(FAILED|ERROR) " "$OUT/$mode.log" | sed 's/ - .*//' | sort -u > "$OUT/$mode.divergences" + echo "$mode: $(wc -l < "$OUT/$mode.divergences") divergence id(s); $(tail -1 "$OUT/$mode.log")" +done +exit 0 diff --git a/graphistry/tests/compute/gfql/cypher/test_lowering.py b/graphistry/tests/compute/gfql/cypher/test_lowering.py index 54b28237fd..65c6a6b5dd 100644 --- a/graphistry/tests/compute/gfql/cypher/test_lowering.py +++ b/graphistry/tests/compute/gfql/cypher/test_lowering.py @@ -18605,6 +18605,7 @@ def _col_stats_trace(g: Any, query: str) -> List[Tuple[str, str, str]]: for s in steps if s.get("op") == "col_stats"] +@pytest.mark.route_engaged("cypher-fast") def test_t6_col_stats_decisions_are_visible_in_the_trace() -> None: """A dead fact is otherwise INVISIBLE: values stay correct, so no value test can fail. The trace distinguishes outcomes because their fixes differ.""" @@ -19104,6 +19105,7 @@ def _h3_records(result: Plottable) -> List[Dict[str, Any]]: return cast(List[Dict[str, Any]], _to_pandas_df(result._nodes).to_dict(orient="records")) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_serves_distinct_domain_shape(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: nodes, edges = _mk_h3_base_data() @@ -19117,6 +19119,7 @@ def test_h3_fused_two_hop_count_serves_distinct_domain_shape(engine: str, monkey assert _h3_records(result) == oracle == [{"numPaths": 5}] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_serves_distinct_edge_domains(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """Distinct EDGE matches with identical node filters also leave the equal-domain branch.""" @@ -19259,6 +19262,7 @@ def test_h3_fused_two_hop_count_empty_match_counts_zero(engine: str, monkeypatch assert _h3_records(result) == [{"numPaths": 0}] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_serves_projected_away_reserved_column(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """A PAYLOAD edge column named like a degree counter no longer forces a decline: @@ -19277,6 +19281,7 @@ def test_h3_fused_two_hop_count_serves_projected_away_reserved_column(engine: st assert _h3_records(result) == [{"numPaths": 5}] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_still_declines_reserved_endpoint_binding(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """NEGATIVE (the guard's remaining reachable side): when the SRC binding itself diff --git a/graphistry/tests/compute/gfql/index/test_degree_consult.py b/graphistry/tests/compute/gfql/index/test_degree_consult.py index e27fc87a3c..acacdbd404 100644 --- a/graphistry/tests/compute/gfql/index/test_degree_consult.py +++ b/graphistry/tests/compute/gfql/index/test_degree_consult.py @@ -74,6 +74,7 @@ def spy(*a, **k): fp._two_hop_equal_domain_dense_total = real +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_degree_fact_is_built_and_actually_used(engine: str) -> None: """Engagement, not just correctness: a built-but-unused fact returns the same @@ -199,6 +200,7 @@ def test_a_fact_covering_a_narrower_span_is_refused() -> None: "WHERE b.age < 30 AND c.age > 20 RETURN count(*) AS n") +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_endpoint_filters_decline_dense_but_the_fused_count_serves(engine: str) -> None: """The q9 shape: same typed two-hop count as q8 plus WHERE filters on the diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 177eb1aed1..35c6fedace 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -226,6 +226,7 @@ def test_invalid_index_policy_raises(graph): graph.gfql_explain(chain, index_policy="use") +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_index_policy_force_and_explain(graph, engine): chain = [n({"id": 0}), e_forward(hops=1)] @@ -311,6 +312,7 @@ def test_maybe_index_hop_reports_auto_build_decline_with_scan_parity(graph): assert _sig(auto_result) == _sig(scan_result) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_explain_exposes_planner_diagnostics(graph, engine): """LP1: gfql_explain surfaces the planner's cost signal — seed cardinality, the @@ -359,6 +361,7 @@ class _BadIdx: # missing keys_sorted/group_offsets → None, not AttributeError assert _seed_deg_sum(_BadIdx(), np.array([0, 1])) is None +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_explain_decision_reasons_for_scan_fallbacks(engine): """LP1: when the planner declines the index it records *why*, so a silent scan is @@ -390,6 +393,7 @@ def test_explain_decision_reasons_for_scan_fallbacks(engine): assert any(s.get("decision_reason") == "query not index-coverable" for s in steps2), (engine, steps2) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_cost_gate_engine_aware_never_loses_to_scan(engine): """F1: the index-vs-scan crossover depends on scan speed, so the cost gate @@ -518,6 +522,7 @@ def test_index_max_hops_honored(engine, hop_kw): assert _sig(base) == _sig(idx), f"max_hops divergence {hop_kw}" +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("max_hops", [1, 2, 3]) def test_index_bounded_range_min_one_hops_none(engine, max_hops): @@ -569,6 +574,7 @@ def test_index_coverability_bounded_range_boundaries(): assert not _hop_is_index_coverable(**dict(common, min_hops=3, max_hops=2)) assert not _hop_is_index_coverable(**dict(common, nodes=None)) +@pytest.mark.route_engaged("index-hop") def test_index_min_two_bounded_range_scans_pandas(graph): """Unsupported [2,2] ranges scan without entering the indexed traversal.""" from graphistry.compute.gfql.index import index_trace @@ -881,6 +887,7 @@ def test_chain_index_parity_vs_scan(typed_graph, engine, chain): assert _sig_typed(base) == _sig_typed(idx) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_chain_typed_edge_engages_index(typed_graph, engine): """All four engines: a typed-edge (simple-equality edge_match) seeded chain hop @@ -891,6 +898,7 @@ def test_chain_typed_edge_engages_index(typed_graph, engine): assert rep["used_index"] is True, (engine, rep) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_chain_untyped_engages_index(typed_graph, engine): """An untyped seeded chain hop engages the index on every engine (pandas/cuDF via @@ -909,6 +917,7 @@ def test_chain_membership_edge_match_stays_on_scan(typed_graph, engine): assert rep["used_index"] is False, (engine, rep) +@pytest.mark.route_engaged("index-hop") def test_chain_range_with_auto_labels_stays_on_scan(typed_graph): """A Cypher range needs per-depth records, so it must decline until indexed.""" engine = "pandas" @@ -1060,6 +1069,7 @@ def test_hop_dtype_mismatch_edge_match_matches_scan_error(typed_graph, engine): # These tests pin the SHAPE (predicate sees only candidate rows), not a wall-clock # number, so they can't go flaky on a loaded host. +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_typed_edge_predicate_only_reads_candidate_rows(typed_graph, engine, monkeypatch): """The edge_match predicate must be evaluated on the CSR-matched rows only. @@ -1098,6 +1108,7 @@ def spy(series, rows, eng): "not the traversal candidates") +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_typed_edge_predicate_cost_flat_in_graph_size(engine): """Growing the graph 8x while holding degree fixed must NOT grow the number of @@ -1146,6 +1157,7 @@ def spy(series, rows, eng): "the edge_match filter is scaling with the graph") +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_typed_edge_predicate_abandons_indexed_path_on_evaluation_failure( typed_graph, engine, monkeypatch @@ -1307,6 +1319,7 @@ def node_ids(gg): "(edges are identical; 1957 indexed node rows vs 1956 scanned). Strict, so it flips " "the moment the wave-front seed handling is unified."))), ]) +@pytest.mark.route_engaged("index-hop") def test_indexed_wavefront_node_set_matches_the_scan(typed_graph, engine, shape): """The index must not change WHICH NODES a wave-front hop reports, only how fast it gets there — the scan is the oracle.""" @@ -1331,6 +1344,7 @@ def node_ids(gg): assert node_ids(gi.hop(nodes=seeds, **kwargs)) == node_ids(g.hop(nodes=seeds, **kwargs)) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_forcing_the_whole_column_mask_actually_changes_the_path(typed_graph, engine, monkeypatch): """The negative side of the boundary must be reachable — otherwise the test above is @@ -1376,6 +1390,7 @@ def spy(series, rows, eng): "resident_engine, requested_engine", [("pandas", "polars"), ("polars", "pandas")], ) +@pytest.mark.route_engaged("index-hop", "polars-plain") def test_explain_reports_bidirectional_engine_mismatch( graph, index_kinds, edge, expected_kinds, resident_engine, requested_engine ): @@ -1470,6 +1485,7 @@ def test_auto_engine_gfql_serves_polars_index_1767_cliff(): assert out._nodes["destination"].to_list() == [101, 102] +@pytest.mark.route_engaged("index-hop") def test_auto_engine_hop_agreed_gates_never_mismatch_1767(): """Direct g.hop() with no engine on a polars-frame indexed graph: modern AUTO serves natively in polars (bridge-to-pandas was the 1767-era accident), and @@ -1744,6 +1760,7 @@ def test_auto_built_index_reports_usable_under_auto(self): assert si["usable"].all() assert si["reason"].isna().all() + @pytest.mark.route_engaged("index-hop") def test_auto_polars_hop_engages_index(self, monkeypatch): # big enough that one seed passes the frontier-fraction cost gate pl = pytest.importorskip("polars") @@ -1994,6 +2011,7 @@ def test_shallow_augmentation_rebind_stays_correct(engine): assert _i1913_ids(gi.edges(aug_e, "s", "d"), _I1913_2HOP, engine) == oracle, label +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_i1913_guard_keeps_the_executor_rebind_engaging(typed_graph, engine): """WHAT THE GUARD MUST NOT COST: the chain's own synthetic-edge-id augmentation is @@ -2063,6 +2081,7 @@ def test_rebind_edges_leaves_index_resident_after_in_place_shape_mutation(): assert reg2.get_valid(kind, aug, ("src", "dst"), _E.PANDAS) is None, kind +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_documented_recovery_from_in_place_mutation(engine): """The recovery matrix named in ``ComputeMixin.gfql``'s docstring, after in-place edits diff --git a/graphistry/tests/compute/gfql/index/test_indexed_bindings.py b/graphistry/tests/compute/gfql/index/test_indexed_bindings.py index 6b388dda46..14fad1a6eb 100644 --- a/graphistry/tests/compute/gfql/index/test_indexed_bindings.py +++ b/graphistry/tests/compute/gfql/index/test_indexed_bindings.py @@ -279,6 +279,7 @@ def _assert_parity( pytest.param("off", "index_policy_off", False), ], ) +@pytest.mark.route_engaged("cypher-fast") def test_destination_unique_trace_and_lifecycle( engine: str, case: str, @@ -326,6 +327,7 @@ def test_destination_unique_trace_and_lifecycle( pytest.param("off", "index_policy_off", False), ], ) +@pytest.mark.route_engaged("index-hop", "indexed-kernel") def test_connected_path_bag_trace_and_lifecycle( engine: str, case: str, @@ -425,6 +427,7 @@ def test_standard_derived_connected_parity( ) +@pytest.mark.route_engaged("index-hop", "indexed-kernel") def test_pandas_connected_boundary_bypasses_canonical_traversal( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -473,6 +476,7 @@ def test_destination_property_projection_dtype_parity( _assert_parity(g, query, engine, monkeypatch, seam="destination_return") +@pytest.mark.route_engaged("indexed-kernel") def test_polars_connected_boundary_bypasses_canonical_traversal( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -523,6 +527,7 @@ def test_unnamed_middle_rows_call_matches_canonical( _assert_result_exact(actual, expected, engine) +@pytest.mark.route_engaged("indexed-kernel") @pytest.mark.parametrize("seed_kind", ["seed", "noise"]) def test_pandas_internal_id_plus_constraints_gathers_seed_before_filter( seed_kind: str, @@ -596,6 +601,7 @@ def record(frame: Any, *args: Any, **kwargs: Any) -> Any: return actual, steps, widths +@pytest.mark.route_engaged("index-hop", "indexed-kernel") @pytest.mark.parametrize("engine", ENGINES) def test_node_property_index_seeds_without_scanning( engine: str, @@ -614,6 +620,7 @@ def test_node_property_index_seeds_without_scanning( assert widths and widths[0] == 1 # one indexed candidate, not the node table +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_node_property_index_absent_matches_indexed( engine: str, @@ -692,6 +699,7 @@ def test_node_property_index_declines_unindexable_columns() -> None: g.gfql_index_node_props(["nosuch"]) +@pytest.mark.route_engaged("index-hop") def test_node_property_index_prefers_the_most_selective_column( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -815,6 +823,7 @@ def test_polars_early_gate_refuses_unsupported_boundaries( assert attempted is False, f"polars gate recorded an attempt on {reason}" +@pytest.mark.route_engaged("indexed-kernel") def test_polars_early_gate_requires_the_whole_middle() -> None: """binding_ops that do not cover the middle must not take the bypass.""" pytest.importorskip("polars") @@ -845,6 +854,7 @@ def test_polars_early_gate_requires_the_whole_middle() -> None: assert attempted is True and state is not None +@pytest.mark.route_engaged("index-hop", "indexed-kernel") @pytest.mark.parametrize("engine", ENGINES) def test_indexed_execution_is_pure( engine: str, @@ -1042,6 +1052,7 @@ def hook(ctx: Dict[str, Any]) -> None: assert decisions[0]["reason"] == "policy_active" +@pytest.mark.route_engaged("index-hop", "indexed-kernel") @pytest.mark.parametrize("engine", ENGINES) def test_renamed_and_permuted_shape_remains_generic( engine: str, @@ -1084,6 +1095,7 @@ def test_renamed_and_permuted_shape_remains_generic( ) +@pytest.mark.route_engaged("indexed-kernel") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("error_type", [RuntimeError, MemoryError]) def test_unexpected_and_memory_errors_propagate( @@ -1110,6 +1122,7 @@ def fail_filter(*args: Any, **kwargs: Any) -> Any: ) +@pytest.mark.route_engaged("indexed-kernel") def test_use_policy_sparse_serves_dense_declines( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py b/graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py index ecc278f488..f16f4a3ee6 100644 --- a/graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py +++ b/graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py @@ -242,6 +242,7 @@ def test_index_backed_seeded_hop_serves_the_closed_answer(engine): assert edge_pair_set(out_dangling) == set(), "index route emitted an unclosed edge" +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ALL_ENGINES) def test_clean_graph_is_untouched_by_the_gate(engine): """POSITIVE CONTROL: with zero dangling endpoints the gate removes nothing and the node diff --git a/graphistry/tests/compute/gfql/test_engine_polars_chain.py b/graphistry/tests/compute/gfql/test_engine_polars_chain.py index c96a644863..c856ffec9a 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_chain.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_chain.py @@ -742,6 +742,7 @@ def test_varlen_alias_parity_across_directions(self, pattern): actual = sorted(g_pl.gfql(q, engine="polars")._nodes["b.id"].to_list()) assert actual == expected + @pytest.mark.route_engaged("cypher-fast") def test_fixed_length_hop_is_not_gated(self): """A plain single-hop edge sets no min_hops, so no labels are requested and no gate runs — the guard must not silently start filtering ordinary chains.""" diff --git a/graphistry/tests/compute/gfql/test_fast_path_engagement.py b/graphistry/tests/compute/gfql/test_fast_path_engagement.py index c9d74c3f91..8e7f2bb4f3 100644 --- a/graphistry/tests/compute/gfql/test_fast_path_engagement.py +++ b/graphistry/tests/compute/gfql/test_fast_path_engagement.py @@ -36,6 +36,7 @@ def _graph(engine: str = "pandas"): ENGINES = ["pandas", "polars", "cudf"] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_two_hop_count_fast_path_engages(engine: str) -> None: """Engagement is per-ENGINE: a path that serves on pandas can silently decline @@ -111,6 +112,7 @@ def test_unknown_fast_path_name_is_reported_not_silently_missing() -> None: assert_fast_path(_graph(), Q_TWO_HOP, "two_hop_cont", served=True) # type: ignore[arg-type] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_seeded_typed_hop_fast_path_engages(engine: str) -> None: """The third path. It is consulted LAST, so its pin doubles as evidence the @@ -132,6 +134,7 @@ def test_seeded_typed_hop_fast_path_engages(engine: str) -> None: assert seen.get("two_hop_count") is False +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_seeded_node_lookup_fast_path_engages(engine: str) -> None: """The fourth path, consulted last: a seeded single-node pattern with a property @@ -162,6 +165,7 @@ def test_fast_paths_have_no_bare_collect(): assert not offenders, f"bare collects bypass the execution target: {offenders}" +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_fast_paths_serve_identically_under_cpu_target_context(engine: str) -> None: """The #1824 target threading must be a no-op on CPU engines: same answers, diff --git a/graphistry/tests/compute/gfql/test_gfql_latency_contract.py b/graphistry/tests/compute/gfql/test_gfql_latency_contract.py index f73c0c323c..759b47f96e 100644 --- a/graphistry/tests/compute/gfql/test_gfql_latency_contract.py +++ b/graphistry/tests/compute/gfql/test_gfql_latency_contract.py @@ -162,12 +162,14 @@ def _graph_for(graphs: Dict[str, Any], engine: str) -> Any: return graphs[engine] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine,shape", _cases(only_served=False)) def test_basic_shape_is_served_by_a_fast_path(graphs: Dict[str, Any], engine: str, shape: str) -> None: g = _graph_for(graphs, engine) assert _served(g, CYPHER[shape], engine), f"{engine}: {shape} fell off the fast path" +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine,shape", _cases(only_served=True)) def test_served_shape_costs_a_bounded_multiple_of_plain_frame_ops( graphs: Dict[str, Any], engine: str, shape: str diff --git a/graphistry/tests/compute/gfql/test_known_cross_engine_divergences.py b/graphistry/tests/compute/gfql/test_known_cross_engine_divergences.py index 286cf13800..cfc8754268 100644 --- a/graphistry/tests/compute/gfql/test_known_cross_engine_divergences.py +++ b/graphistry/tests/compute/gfql/test_known_cross_engine_divergences.py @@ -71,6 +71,7 @@ def test_1739_has_label_aggregate_on_duplicate_ids_converges(): assert out_pl == narrowed +@pytest.mark.route_engaged("cypher-fast") @polars_only @pytest.mark.xfail(strict=True, reason="#1824: fast paths serve CPU work under engine='polars-gpu'") def test_1824_polars_gpu_fast_path_serve_is_gpu_or_decline(): diff --git a/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py b/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py index ecc3b95b03..9bd377a4b2 100644 --- a/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py +++ b/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py @@ -34,6 +34,7 @@ def _graph(engine, indexed=True): ENGINES = ["pandas", "polars", "cudf"] +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_only_lookup_served_by_the_property_index_is_explained(engine): g = _graph(engine) @@ -43,6 +44,7 @@ def test_node_only_lookup_served_by_the_property_index_is_explained(engine): assert len(g.gfql(NODE_ONLY, engine=engine, index_policy="use")._nodes) == 1 +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_seeded_typed_hop_served_by_the_resident_indexes_is_explained(engine): g = _graph(engine) diff --git a/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py b/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py index f33b3ce465..fb9c0c0853 100644 --- a/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py +++ b/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py @@ -79,6 +79,7 @@ def spy(*a, **k): } +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("shape", list(SHAPES)) def test_lane_shapes_are_served_with_exact_parity(engine, shape): @@ -93,6 +94,7 @@ def test_lane_shapes_are_served_with_exact_parity(engine, shape): pd.testing.assert_frame_equal(_canon(fast._edges), _canon(full._edges), check_dtype=False) +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_property_index_resolves_the_seed(engine, monkeypatch): import graphistry.compute.gfql.index.bindings as bindings diff --git a/graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py b/graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py index 750d097388..d5408dcaf2 100644 --- a/graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py +++ b/graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py @@ -120,6 +120,7 @@ def _disambiguation_frames(): return nodes, edges +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas"] + (["polars"] if HAS_POLARS else [])) def test_has_label_narrowing_skipped_when_reached_ids_unique(engine: str) -> None: """Global id collisions must NOT trigger narrowing when the REACHED ids are unique diff --git a/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py b/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py index e168877cab..f8280b2d28 100644 --- a/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py +++ b/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py @@ -106,6 +106,7 @@ def _assert_parity(g, engine, query, path, served=True): ] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("indexed", [False, True], ids=["scan", "indexed"]) @pytest.mark.parametrize("q,label", LOOKUP_SHAPES) @@ -124,6 +125,7 @@ def test_node_lookup_matches_independent_oracle(engine): assert float(got["a"].iloc[0]) == float(row["age"]) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_lookup_uses_the_property_index_when_the_seed_is_not_the_binding(engine): """The seed predicate is on a business key that is not the node binding: the @@ -227,6 +229,7 @@ def test_two_alias_projection_parity(engine, indexed, q, label): pd.testing.assert_frame_equal(_canon(fast), _canon(full)) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_two_alias_projection_engages_and_keeps_bag_multiplicity(engine): g = _graph(engine) @@ -321,6 +324,7 @@ def _lookup_step(report): return steps[0] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_lookup_explains_why_it_scanned(engine): q = "MATCH (p:Person {id: 7}) RETURN p.age AS age" @@ -368,6 +372,7 @@ def test_two_alias_projection_extension_dtypes_keep_parity(indexed, q, served): assert fast_path_decisions(g, q, engine="pandas").get("seeded_typed_hop") is not True +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_lookup_served_under_policy_off_is_not_reported_as_an_index(engine): q = "MATCH (p:Person {id: 7}) RETURN p.age AS age" @@ -381,6 +386,7 @@ def test_node_lookup_served_under_policy_off_is_not_reported_as_an_index(engine) assert off["used_index"] is False and off["decision_code"] == "policy_off", off +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_edge_alias_properties_engage_with_one_row_per_matched_edge(engine): g = _graph(engine) diff --git a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py index aee31b9cfd..941e852ef6 100644 --- a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py +++ b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py @@ -20,9 +20,19 @@ import graphistry from graphistry.compute.ast import n, e_forward, e_reverse import graphistry.compute.chain as chain_mod +import graphistry.compute.chain_specializations.hotpaths as hot import graphistry.compute.gfql_unified as gfql_unified + +def _patch_index_rows(monkeypatch, name, fn): + """Patch a chain_fast_paths index helper everywhere the lanes bind it.""" + import graphistry.compute.chain_fast_paths as cfp + import graphistry.compute.chain_specializations.hotpaths as eager_hot + import graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths as polars_hot + for mod in (cfp, eager_hot, polars_hot): + monkeypatch.setattr(mod, name, fn) + def _graph(n_persons=1500, n_messages=6000, seed=0): """Message -> Person HAS_CREATOR graph (the #1755 probe shape). `age` is only on Person rows, so the concatenated node frame carries a float `age` column @@ -120,11 +130,12 @@ def test_parity_fast_vs_full(self, ops_name): full = self._run(g, ops, force_full=True) pd.testing.assert_frame_equal(_canon_nodes(fast), _canon_nodes(full)) + @pytest.mark.route_engaged("native-fast") def test_fast_path_engages_on_typed_hop(self, monkeypatch): g, P = _graph() seed = P + 42 hits = {"n": 0} - real = chain_mod._seeded_typed_hop_pandas_cudf + real = hot._seeded_typed_hop_pandas_cudf def spy(*a, **k): r = real(*a, **k) @@ -132,7 +143,7 @@ def spy(*a, **k): hits["n"] += 1 return r - monkeypatch.setattr(chain_mod, "_seeded_typed_hop_pandas_cudf", spy) + monkeypatch.setattr(hot, "_seeded_typed_hop_pandas_cudf", spy) g.gfql([n({"id": seed}), e_forward(edge_match={"type": "HAS_CREATOR"}), n({"type": "Person"})], engine="pandas") assert hits["n"] >= 1 @@ -159,11 +170,11 @@ def test_numpy_helper_declines_predicate_and_undirected(self): g, P = _graph() node, src, dst = "id", "src", "dst" # predicate (non-scalar) edge filter -> decline - assert chain_mod._seeded_typed_hop_pandas_cudf( + assert hot._seeded_typed_hop_pandas_cudf( g.materialize_nodes(), n({"id": P + 1}), n(), e_forward(edge_match={"type": gt(0)}), src, dst, node, "forward") is None # undirected -> decline - assert chain_mod._seeded_typed_hop_pandas_cudf( + assert hot._seeded_typed_hop_pandas_cudf( g.materialize_nodes(), n({"id": P + 1}), n(), e_forward(), src, dst, node, "undirected") is None @@ -180,7 +191,7 @@ def test_native_declines_and_stays_correct(self, ops_name, reason, monkeypatch): "node_predicate": [n({"id": seed}), e_forward(edge_match={"type": "HAS_CREATOR"}), n({"age": gt(0)})], }[ops_name] hits = {"n": 0} - real = chain_mod._seeded_typed_hop_pandas_cudf + real = hot._seeded_typed_hop_pandas_cudf def spy(*a, **k): r = real(*a, **k) @@ -188,7 +199,7 @@ def spy(*a, **k): hits["n"] += 1 return r - monkeypatch.setattr(chain_mod, "_seeded_typed_hop_pandas_cudf", spy) + monkeypatch.setattr(hot, "_seeded_typed_hop_pandas_cudf", spy) fast = g.gfql(ops, engine="pandas") monkeypatch.undo() chain_mod._try_chain_fast_path, saved = (lambda *a, **k: None), chain_mod._try_chain_fast_path @@ -247,6 +258,7 @@ def test_independent_oracle_values(self): got = sorted(df[id_col].tolist()) assert got == oracle, f"cypher fast path returned {got}, oracle says {oracle}" + @pytest.mark.route_engaged("cypher-fast") def test_fast_path_engages_on_seeded_return(self, monkeypatch): g, P = _graph() seed = P + 42 @@ -263,6 +275,7 @@ def spy(*a, **k): g.gfql(f"MATCH (m:Message {{id: {seed}}})-[:HAS_CREATOR]->(p:Person) RETURN p", engine="pandas") assert hits["n"] >= 1 + @pytest.mark.route_engaged("cypher-fast") def test_fast_path_engages_on_the_bag_lowering_of_a_property_return(self, monkeypatch): """A property RETURN lowers to the multiplicity-preserving ``rows(binding_ops=...)`` form. The fast path must still engage there: declining would be value-correct but @@ -305,6 +318,7 @@ def test_parallel_edges_keep_their_row_through_the_fast_path(self, engine): got = _canon_nodes(g.gfql(q, engine=engine)) assert got["pid"].tolist() == [1, 1] + @pytest.mark.route_engaged("cypher-fast") def test_cross_alias_field_projection_engages_with_parity(self): """RETURN m.id, p.age projects from BOTH aliases: one row per matched edge, the seed side looked up from the seed rows the reduction already holds.""" @@ -560,6 +574,7 @@ def test_independent_oracle_values(self): id_col = "p.id" if "p.id" in df.columns else "id" assert sorted(df[id_col].tolist()) == oracle + @pytest.mark.route_engaged("cypher-fast") def test_fast_path_engages(self, monkeypatch): gp, P, _ = self._pl_graph() seed = P + 42 @@ -618,7 +633,7 @@ def _q(self, seed): def test_lazyframe_declines_not_crashes(self): """A LazyFrame-backed graph must decline (full path), not AttributeError.""" pl = pytest.importorskip("polars") - from graphistry.compute.chain_fast_paths import _seeded_typed_return_dst_polars + from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths import _seeded_typed_return_dst_polars from graphistry.compute.ast import ASTNode, ASTEdge g, P = _graph() lazy_g = graphistry.nodes( @@ -719,6 +734,7 @@ def spy(*a, **k): assert bool(hits["n"]) == expect_engage, f"engaged={hits['n']} expected={expect_engage}" return fast + @pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_is5_shape_engages_and_matches(self, engine): if engine == "polars": @@ -761,6 +777,7 @@ def test_out_of_shape_declines_with_parity(self, q, label): ("MATCH (m:Message {id:10})-[{type:'HAS_CREATOR'}]->(p:Person) RETURN p.age AS age, m.id AS mid ORDER BY age LIMIT 1", "cross-alias order/limit"), ("MATCH (m:Message {id:10})-[{type:'HAS_CREATOR'}]->(p:Person) RETURN p.id AS a, p.id AS b", "same column twice"), ]) + @pytest.mark.route_engaged("cypher-fast") def test_canonical_projection_suffix_engages_with_parity(self, q, label): """DISTINCT / ORDER BY / SKIP / LIMIT after the lean projection are plain row-frame ops: the fast path now keeps the seeded gather and delegates the @@ -885,6 +902,7 @@ def test_engine_mismatch_declines(self): assert type(fast2._nodes).__module__ == type(full2._nodes).__module__ pd.testing.assert_frame_equal(_canon_nodes(fast2), _canon_nodes(full2)) + @pytest.mark.route_engaged("cypher-fast") def test_string_dtype_property_engages_and_matches(self): """pandas StringDtype (explicit 'string' on pandas 2; the DEFAULT str dtype on pandas>=3) passes through the pivot unchanged on both versions — @@ -940,13 +958,14 @@ def spy(*a, **k): out = on(*a, **k) counts["n"] += out is not None return out - monkeypatch.setattr(cfp, "_index_node_rows", spy) + _patch_index_rows(monkeypatch, "_index_node_rows", spy) res = g.gfql(q, engine=engine) - monkeypatch.setattr(cfp, "_index_node_rows", on) + _patch_index_rows(monkeypatch, "_index_node_rows", on) return res, counts["n"] Q = "MATCH (m {id: 33})-[:KNOWS]->(p) RETURN p" + @pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_indexed_parity_and_engagement(self, engine, monkeypatch): if engine == "polars": @@ -988,7 +1007,7 @@ def test_stale_index_declines_to_scan(self, monkeypatch): g2 = g.edges(edf2, "src", "dst") # stale registry rides along counts = {"n": 0} oe = cfp._index_edge_rows - monkeypatch.setattr(cfp, "_index_edge_rows", + _patch_index_rows(monkeypatch, "_index_edge_rows", lambda *a, **k: (counts.__setitem__("n", counts["n"] + 1), oe(*a, **k))[1]) got = g2.gfql(self.Q, engine="pandas") plain = graphistry.nodes(ndf, "id").edges(edf2, "src", "dst").gfql(self.Q, engine="pandas") @@ -1015,9 +1034,9 @@ def spy(*a, **k): out = oe(*a, **k) serves["n"] += out is not None return out - monkeypatch.setattr(cfp, "_index_edge_rows", spy) + _patch_index_rows(monkeypatch, "_index_edge_rows", spy) got = mk().gfql_index_all().gfql(ops, engine="pandas") - monkeypatch.setattr(cfp, "_index_edge_rows", oe) + _patch_index_rows(monkeypatch, "_index_edge_rows", oe) assert serves["n"] > 0, f"indexed hop branch did not serve for {ops[1].direction}" plain = mk().gfql(ops, engine="pandas") pd.testing.assert_frame_equal(self._canon(got), self._canon(plain)) @@ -1040,9 +1059,9 @@ def spy(*a, **k): out = on(*a, **k) serves["n"] += out is not None return out - monkeypatch.setattr(cfp, "_index_node_rows", spy) + _patch_index_rows(monkeypatch, "_index_node_rows", spy) got = g.gfql(q, engine="pandas") - monkeypatch.setattr(cfp, "_index_node_rows", on) + _patch_index_rows(monkeypatch, "_index_node_rows", on) plain = graphistry.nodes(ndf, "id").edges(edf, "src", "dst").gfql(q, engine="pandas") pd.testing.assert_frame_equal(self._canon(got), self._canon(plain)) @@ -1075,6 +1094,7 @@ def med(g): indexed = med(graphistry.nodes(ndf, "id").edges(edf, "src", "dst").gfql_index_all()) assert indexed * 1.5 < scan, f"indexed {indexed*1e3:.2f}ms not >=1.5x faster than scan {scan*1e3:.2f}ms" + @pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_property_seeded_engages_adjacency_index(self, engine, monkeypatch): """Decoupled-index pin (the SF1-harness/LDBC pattern): the graph binds a @@ -1107,8 +1127,8 @@ def spy(*a, **k): out = oe(*a, **k) serves["e"] += out is not None return out - monkeypatch.setattr(cfp, "_index_edge_rows", spy) + _patch_index_rows(monkeypatch, "_index_edge_rows", spy) indexed = mk().gfql_index_all(engine=engine).gfql(q, engine=engine) - monkeypatch.setattr(cfp, "_index_edge_rows", oe) + _patch_index_rows(monkeypatch, "_index_edge_rows", oe) assert serves["e"] > 0, "adjacency index did not serve the property-seeded lookup" pd.testing.assert_frame_equal(self._canon(indexed), self._canon(plain)) diff --git a/graphistry/tests/compute/test_chain.py b/graphistry/tests/compute/test_chain.py index 2450203124..01fd9073a8 100644 --- a/graphistry/tests/compute/test_chain.py +++ b/graphistry/tests/compute/test_chain.py @@ -768,6 +768,7 @@ def test_fast_path_differential_parity_vs_full_path(engine, label, build, reques @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @pytest.mark.parametrize("label,build", _NAMED_ALIAS_SHAPES, ids=[s[0] for s in _NAMED_ALIAS_SHAPES]) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_alias_columns_match_full_path(engine, label, build): """The capability this fast-path extension actually adds: when the traversal is served without the BFS, the alias flag columns `combine_steps` would have merged in @@ -877,6 +878,7 @@ def test_fast_path_named_full_frame_value_parity(engine, label, build): @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @pytest.mark.parametrize("label,build", _NAMED_EMPTY_SHAPES, ids=[s[0] for s in _NAMED_EMPTY_SHAPES]) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_empty_result_matches_full_path(engine, label, build): """POSITIVE boundary: named patterns matching ZERO rows are still served, and the empty result must be shape-identical to the full path — same columns INCLUDING the @@ -898,6 +900,7 @@ def test_fast_path_named_empty_result_matches_full_path(engine, label, build): assert len(un) == 0 and len(ue) == 0 +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_zero_edge_graph_matches_full_path(): """POSITIVE boundary: a graph with an EMPTY edge table. The named pattern is served, and both lanes must agree on the all-empty answer with alias columns present.""" @@ -953,6 +956,7 @@ def test_fast_path_cross_type_alias_share_declines_and_matches(): _assert_full_frame_value_parity(default_route._edges, policy_route._edges, ['s', 'd']) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_is_served_with_a_valid_resident_index(): """A NAMED pattern with BOTH resident indexes validly covering the directed hop is served by the chain fast path: by the time it runs, the indexed kernel has already @@ -976,6 +980,7 @@ def test_fast_path_named_is_served_with_a_valid_resident_index(): _assert_full_frame_value_parity(served._edges, full._edges, ['s', 'd']) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_datetime_categorical_columns_ride_along(): """POSITIVE dtype edge: datetime64 and categorical NODE columns must ride through the served named lane unchanged — same values as the full path, dtypes preserved @@ -1119,6 +1124,7 @@ def test_fast_path_alias_colliding_with_node_id_binding_matches_full_path(build) _assert_full_frame_value_parity(fast._edges, full._edges, ['s', 'd']) +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ["pandas", "cudf"]) def test_fast_path_preserves_int_node_dtypes(engine): """Documented behavior change: the 1-hop fast path PRESERVES node-attribute diff --git a/graphistry/tests/conftest.py b/graphistry/tests/conftest.py new file mode 100644 index 0000000000..b5b633cd92 --- /dev/null +++ b/graphistry/tests/conftest.py @@ -0,0 +1,79 @@ +"""Route forcing for test amplification. + +``GFQL_ROUTES_OFF=[,...]`` makes the named hot paths decline for the whole +session, so every existing test — written against one specialization or against the +general path — is replayed through the other routes. Failures that appear only under a +mode are the cases where that route and the rest disagree. + +Routes: native-fast (pandas/cuDF chain fast path), polars-seeded (polars seeded lane), +polars-plain (polars plain single-hop branches), index-hop (hop() index path), +indexed-kernel (indexed connected-bindings kernel), cypher-fast (the four Cypher lanes). + +A test that asserts a route SERVES (trace, latency, served-by spy) is an engagement pin, not +a result pin: mark it ``@pytest.mark.route_engaged("", ...)`` and it is skipped when +one of its routes is off, so the replay reports result divergences only +(``bin/test-routes-off.sh``). +""" +import os + +import pytest + + +def _routes_off(): + raw = os.environ.get("GFQL_ROUTES_OFF", "") + return {r.strip() for r in raw.split(",") if r.strip()} + + +def pytest_collection_modifyitems(config, items): + off = _routes_off() + if not off: + return + for item in items: + for mark in item.iter_markers("route_engaged"): + hit = off & set(mark.args) + if hit: + item.add_marker(pytest.mark.skip(reason="engagement pin for route(s) off: " + ",".join(sorted(hit)))) + + +@pytest.fixture(autouse=True, scope="session") +def _gfql_routes_off(): + routes = _routes_off() + if not routes: + yield + return + import graphistry.compute.chain as chain_mod + import graphistry.compute.gfql_unified as unified + import graphistry.compute.gfql.index as index_pkg + import graphistry.compute.gfql.index.api as index_api + import graphistry.compute.gfql.index.bindings as bindings + import graphistry.compute.gfql.lazy.engine.polars.chain as pchain + + def none(*a, **k): + return None + + patches = [] + + def patch(mod, name, value): + patches.append((mod, name, getattr(mod, name))) + setattr(mod, name, value) + + if "native-fast" in routes: + patch(chain_mod, "_try_chain_fast_path", none) + if "polars-seeded" in routes: + patch(pchain, "_try_seeded_chain_polars", none) + if "polars-plain" in routes: + patch(pchain, "polars_plain_single_hop_admits", none) + if "index-hop" in routes: + patch(index_pkg, "maybe_index_hop", none) + patch(index_api, "maybe_index_hop", none) + if "indexed-kernel" in routes: + patch(bindings, "_try_indexed_connected_bindings_state", none) + if "cypher-fast" in routes: + for name in ("_execute_seeded_node_lookup_fast_path", "_execute_seeded_typed_hop_fast_path", + "_execute_single_hop_grouped_aggregate_fast_path", "_execute_two_hop_count_fast_path"): + patch(unified, name, none) + try: + yield + finally: + for mod, name, value in reversed(patches): + setattr(mod, name, value) diff --git a/pytest.ini b/pytest.ini index 145c234cc9..d6622e0ad6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,4 +9,5 @@ filterwarnings = ignore::pytest.PytestCacheWarning markers = tier2: optional/extended test coverage + route_engaged(*routes): the test asserts that a GFQL hot path serves (trace, latency, served-by); skipped when GFQL_ROUTES_OFF names one of its routes #log_cli = True From 23ed51a9c4852e70a3b7bfc7f6d16a126a6d18e5 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 18:02:54 -0700 Subject: [PATCH 05/12] fix(gfql): absolute imports in the polars chain specializations (CI rule) --- .../lazy/engine/polars/chain_specializations/hotpaths.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py index 4ca131063c..a6388dc45b 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py @@ -16,15 +16,15 @@ ) from graphistry.compute.endpoint_utils import drop_null_endpoint_edges from graphistry.compute.typing import ArrayLike, ArrayNamespace, DataFrameT -from ..dtypes import endpoint_ids -from ..hop_eager import ensure_nodes_polars -from ..predicates import filter_by_dict_polars +from graphistry.compute.gfql.lazy.engine.polars.dtypes import endpoint_ids +from graphistry.compute.gfql.lazy.engine.polars.hop_eager import ensure_nodes_polars +from graphistry.compute.gfql.lazy.engine.polars.predicates import filter_by_dict_polars from .admission import polars_seeded_lane_admits if TYPE_CHECKING: import polars as pl from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex - from ..dtypes import PolarsFrame + from graphistry.compute.gfql.lazy.engine.polars.dtypes import PolarsFrame def _plain_seeded_index_hop_polars(g: Plottable, ops: Sequence[ASTObject]) -> Optional[Plottable]: From be8b0a11cce934147766c2d84d139787fff4a193 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 18:33:08 -0700 Subject: [PATCH 06/12] ci(gfql): polars coverage floors for the chain_specializations modules --- .../compute/gfql/coverage_baselines/ci-polars-py3.12.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json b/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json index 59b9e71ae1..10bf82567e 100644 --- a/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json +++ b/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json @@ -5,6 +5,9 @@ "graphistry/compute/gfql/lazy/engine/__init__.py": 90.0, "graphistry/compute/gfql/lazy/engine/polars/__init__.py": 90.0, "graphistry/compute/gfql/lazy/engine/polars/chain.py": 86.0, + "graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py": 100.0, + "graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py": 95.0, + "graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py": 95.0, "graphistry/compute/gfql/lazy/engine/polars/degrees.py": 82.0, "graphistry/compute/gfql/lazy/engine/polars/dtypes.py": 92.0, "graphistry/compute/gfql/lazy/engine/polars/hop.py": 87.0, From eccee8a0d28e0bf250391f117223b4656d18d5cc Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 17:57:42 -0700 Subject: [PATCH 07/12] test(gfql): shape registry fed by the specialization tables + route harness Each specialization's test module registers the shape table it already owns (routes corpus, the six test_chain tables, the alias-collision matrix) with its frames and defect-class tags. The harness tries every registered shape against every chain route whose admission predicate admits it and pins that the lane serves, that the answer matches the same engine's general path on values, and that node/edge sets match the pandas general path. A lane that declines an admitted shape is recorded as an expected failure (the attenuation ledger); filed divergences are strict expected failures keyed by tag. The pandas bypass table's prune shapes surface #2053 on the polars route. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QztW7jYsDd66e8rb8pJNQA --- .../tests/compute/gfql/routes/corpus.py | 4 + .../tests/compute/gfql/routes/registry.py | 72 +++++++ .../tests/compute/gfql/routes/switch.py | 51 +++++ .../compute/gfql/routes/test_route_harness.py | 180 ++++++++++++++++++ graphistry/tests/compute/test_chain.py | 34 ++-- .../test_chain_alias_column_collision.py | 6 + graphistry/tests/conftest.py | 37 +--- 7 files changed, 335 insertions(+), 49 deletions(-) create mode 100644 graphistry/tests/compute/gfql/routes/registry.py create mode 100644 graphistry/tests/compute/gfql/routes/switch.py create mode 100644 graphistry/tests/compute/gfql/routes/test_route_harness.py diff --git a/graphistry/tests/compute/gfql/routes/corpus.py b/graphistry/tests/compute/gfql/routes/corpus.py index 9a2a7c46dc..5c89806fe7 100644 --- a/graphistry/tests/compute/gfql/routes/corpus.py +++ b/graphistry/tests/compute/gfql/routes/corpus.py @@ -11,6 +11,7 @@ from graphistry.compute.ast import ASTObject, e_forward, e_reverse, e_undirected, n from graphistry.compute.predicates.numeric import GT +from graphistry.tests.compute.gfql.routes.registry import Frames, register class Entry(NamedTuple): @@ -54,3 +55,6 @@ def tagged(tag: str) -> List[Entry]: def by_name() -> Dict[str, Entry]: return {e.name: e for e in CORPUS} + + +register("routes.corpus", [(e.name, e.ops, e.tags) for e in CORPUS], Frames(NODES, EDGES, "key", "s", "d", "eid")) diff --git a/graphistry/tests/compute/gfql/routes/registry.py b/graphistry/tests/compute/gfql/routes/registry.py new file mode 100644 index 0000000000..f064caf23f --- /dev/null +++ b/graphistry/tests/compute/gfql/routes/registry.py @@ -0,0 +1,72 @@ +"""Shape registry for the route harness. + +Each specialization's own test module registers the shape table it was written against +(``register(...)`` returns the table unchanged, so the module keeps using it), with the +frames those shapes address and the defect classes they exercise. The harness in +``test_route_harness.py`` then tries every registered shape against every route whose +admission predicate admits it, so one input is exercised by several hot paths, not one. +""" +from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Sequence, Tuple, Union + +import pandas as pd +import pytest + +from graphistry.Plottable import Plottable +from graphistry.compute.ast import ASTObject + +Build = Callable[[], List[ASTObject]] +Row = Union[Tuple[str, Build], Tuple[str, Build, Tuple[str, ...]]] + + +class Frames(NamedTuple): + nodes: pd.DataFrame + edges: pd.DataFrame + node: str + src: str + dst: str + edge: Optional[str] = None + + +class Shape(NamedTuple): + table: str + label: str + build: Build + frames: Frames + tags: Tuple[str, ...] + + @property + def name(self) -> str: + return f"{self.table}/{self.label}" + + +REGISTRY: Dict[str, Shape] = {} + + +def register(table: str, rows: Sequence[Row], frames: Frames, tags: Iterable[str] = (), + row_tags: Optional[Dict[str, Tuple[str, ...]]] = None) -> Sequence[Row]: + """Register ``rows`` ((label, build[, tags]) ...) under ``table``; returns ``rows``.""" + base = tuple(tags) + for row in rows: + label, build = row[0], row[1] + extra = tuple(row[2]) if len(row) > 2 else () + extra += (row_tags or {}).get(label, ()) + shape = Shape(table, label, build, frames, base + extra) + REGISTRY.setdefault(shape.name, shape) + return rows + + +def to_engine(df: pd.DataFrame, engine: str): + if engine == "pandas": + return df + if engine == "cudf": + return pytest.importorskip("cudf").from_pandas(df) + if engine == "polars": + return pytest.importorskip("polars").from_pandas(df) + raise ValueError(engine) + + +def graph_for(shape: Shape, engine: str, indexed: bool = False) -> Plottable: + import graphistry + f = shape.frames + g = graphistry.nodes(to_engine(f.nodes, engine), f.node).edges(to_engine(f.edges, engine), f.src, f.dst, f.edge) + return g.gfql_index_all(engine=engine) if indexed else g diff --git a/graphistry/tests/compute/gfql/routes/switch.py b/graphistry/tests/compute/gfql/routes/switch.py new file mode 100644 index 0000000000..781dde5325 --- /dev/null +++ b/graphistry/tests/compute/gfql/routes/switch.py @@ -0,0 +1,51 @@ +"""Route switch for test amplification: make named GFQL hot paths decline for a scope.""" +from contextlib import contextmanager +from typing import Iterable, Iterator, List, Tuple + +ROUTES = ("native-fast", "polars-seeded", "polars-plain", "index-hop", "indexed-kernel", "cypher-fast") + + +def _none(*a, **k): + return None + + +def _targets(routes: Iterable[str]) -> List[Tuple[object, str]]: + import graphistry.compute.chain as chain_mod + import graphistry.compute.gfql_unified as unified + import graphistry.compute.gfql.index as index_pkg + import graphistry.compute.gfql.index.api as index_api + import graphistry.compute.gfql.index.bindings as bindings + import graphistry.compute.gfql.lazy.engine.polars.chain as pchain + routes = set(routes) + unknown = routes - set(ROUTES) + assert not unknown, f"unknown route(s) {sorted(unknown)}; known: {ROUTES}" + out: List[Tuple[object, str]] = [] + if "native-fast" in routes: + out.append((chain_mod, "_try_chain_fast_path")) + if "polars-seeded" in routes: + out.append((pchain, "_try_seeded_chain_polars")) + if "polars-plain" in routes: + out.append((pchain, "polars_plain_single_hop_admits")) + if "index-hop" in routes: + out += [(index_pkg, "maybe_index_hop"), (index_api, "maybe_index_hop")] + if "indexed-kernel" in routes: + out.append((bindings, "_try_indexed_connected_bindings_state")) + if "cypher-fast" in routes: + out += [(unified, name) for name in ( + "_execute_seeded_node_lookup_fast_path", "_execute_seeded_typed_hop_fast_path", + "_execute_single_hop_grouped_aggregate_fast_path", "_execute_two_hop_count_fast_path")] + return out + + +@contextmanager +def routes_off(routes: Iterable[str]) -> Iterator[None]: + """Within the block the named routes decline, so the general path answers.""" + saved = [] + for mod, name in _targets(routes): + saved.append((mod, name, getattr(mod, name))) + setattr(mod, name, _none) + try: + yield + finally: + for mod, name, value in reversed(saved): + setattr(mod, name, value) diff --git a/graphistry/tests/compute/gfql/routes/test_route_harness.py b/graphistry/tests/compute/gfql/routes/test_route_harness.py new file mode 100644 index 0000000000..e00a6d6db0 --- /dev/null +++ b/graphistry/tests/compute/gfql/routes/test_route_harness.py @@ -0,0 +1,180 @@ +"""Route harness: every registered shape is tried against every chain route whose admission +predicate admits it. Three pins per cell: the route SERVES (its lane answers; a lane that +declines an admitted shape is recorded as an expected failure, the attenuation ledger), the +answer matches the same engine's general path (all routes off) on node/edge values, and its +node/edge sets match the pandas general path (the cross-engine oracle). Filed divergences are +strict expected failures keyed by their tag, so they flip when fixed. +""" +import math +import os +from typing import Callable, Dict, List, NamedTuple, Tuple + +import pandas as pd +import pytest + +import graphistry.compute.chain as chain_mod +import graphistry.compute.gfql.lazy.engine.polars.chain as pchain +from graphistry.Engine import Engine +from graphistry.compute.ast import ASTObject +from graphistry.compute.chain_specializations.admission import native_fast_path_admits +from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.admission import ( + polars_plain_single_hop_admits, polars_seeded_lane_admits, +) +from graphistry.tests.compute.gfql.routes.registry import REGISTRY, Shape, graph_for +from graphistry.tests.compute.gfql.routes.switch import ROUTES as ALL_ROUTES, routes_off + +import graphistry.tests.compute.gfql.routes.corpus # noqa: F401 registers routes.corpus +import graphistry.tests.compute.test_chain # noqa: F401 registers test_chain.* +import graphistry.tests.compute.test_chain_alias_column_collision # noqa: F401 registers collision.* + + +class Route(NamedTuple): + name: str + engines: Tuple[str, ...] + admits: Callable[[List[ASTObject], str], bool] + lane: Tuple[object, str] + indexed: bool + + +ROUTES = [ + Route("native-fast", ("pandas", "cudf"), + lambda ops, engine: native_fast_path_admits(ops, Engine(engine), None) is not None, + (chain_mod, "_try_chain_fast_path"), False), + Route("polars-plain", ("polars",), + lambda ops, engine: polars_plain_single_hop_admits(ops, None) is not None, + (pchain, "_plain_single_hop_polars"), False), + Route("polars-seeded", ("polars",), + lambda ops, engine: polars_seeded_lane_admits(ops), + (pchain, "_try_seeded_chain_polars"), True), +] + +KNOWN: Dict[Tuple[str, str], str] = { # (route, tag) -> issue: strict xfail until it lands + ("polars-plain", "#2053"): "graphistry/pygraphistry#2053", +} + + +class Case(NamedTuple): + route: Route + engine: str + shape: Shape + + @property + def id(self) -> str: + return f"{self.route.name}/{self.engine}/{self.shape.name}" + + +def _cases() -> List[Case]: + out = [] + for shape in REGISTRY.values(): + for route in ROUTES: + for engine in route.engines: + try: + admitted = route.admits(shape.build(), engine) + except Exception: + admitted = False + if admitted: + out.append(Case(route, engine, shape)) + return out + + +CASES = _cases() + + +def _topd(df): + if df is None: + return None + if hasattr(df, "to_pandas"): + return df.to_pandas() + return df + + +def _canon(df) -> Tuple[Tuple[str, ...], List[Tuple]]: + df = _topd(df) + if df is None: + return ((), []) + cols = tuple(sorted(df.columns)) + rows = [] + for row in df[list(cols)].itertuples(index=False, name=None): + rows.append(tuple(None if (isinstance(v, float) and math.isnan(v)) or v is pd.NA or v is pd.NaT else v for v in row)) + rows.sort(key=repr) + return cols, rows + + +def _sig(res, frames) -> Tuple[List, List]: + nn, ee = _topd(res._nodes), _topd(res._edges) + nodes = sorted(nn[frames.node].tolist()) if nn is not None else [] + edges = sorted(map(tuple, ee[[frames.src, frames.dst]].values.tolist())) if ee is not None and len(ee) else [] + return nodes, edges + + +def _served(case: Case, monkeypatch): + mod, name = case.route.lane + real = getattr(mod, name) + calls = {"served": 0} + + def spy(*a, **k): + out = real(*a, **k) + calls["served"] += out is not None + return out + monkeypatch.setattr(mod, name, spy) + return calls + + +def _skip_unavailable(engine: str) -> None: + if engine == "cudf": + if os.environ.get("TEST_CUDF") != "1": + pytest.skip("cuDF lane runs with TEST_CUDF=1") + pytest.importorskip("cudf") + if engine == "polars": + pytest.importorskip("polars") + + +@pytest.mark.parametrize("case", CASES, ids=[c.id for c in CASES]) +def test_admitted_shape_is_served_and_matches_the_general_path(case: Case, request, monkeypatch): + _skip_unavailable(case.engine) + for tag in case.shape.tags: + if (case.route.name, tag) in KNOWN: + request.applymarker(pytest.mark.xfail(strict=True, reason=KNOWN[(case.route.name, tag)])) + g = graph_for(case.shape, case.engine, indexed=case.route.indexed) + calls = _served(case, monkeypatch) + try: + served = g.gfql(case.shape.build(), engine=case.engine) + except Exception as served_exc: + with routes_off(ALL_ROUTES): + with pytest.raises(type(served_exc)): + g.gfql(case.shape.build(), engine=case.engine) + return + with routes_off(ALL_ROUTES): + general = g.gfql(case.shape.build(), engine=case.engine) + oracle = _sig(graph_for(case.shape, "pandas").gfql(case.shape.build(), engine="pandas"), case.shape.frames) + assert _canon(served._nodes) == _canon(general._nodes), f"{case.id}: node rows differ from the general path" + assert _canon(served._edges) == _canon(general._edges), f"{case.id}: edge rows differ from the general path" + assert _sig(served, case.shape.frames) == oracle, f"{case.id}: node/edge sets differ from the pandas general path" + if calls["served"] == 0: + pytest.xfail(f"{case.id}: admitted by the predicate, declined by the lane body (attenuation ledger)") + + +def test_every_route_serves_most_of_what_it_admits(monkeypatch): + """A lane that declines most admitted shapes has a predicate that no longer describes it.""" + per_route: Dict[str, List[int]] = {} + for case in CASES: + if case.engine != ("polars" if case.route.name.startswith("polars") else "pandas"): + continue + if case.engine == "polars": + pytest.importorskip("polars") + g = graph_for(case.shape, case.engine, indexed=case.route.indexed) + calls = _served(case, monkeypatch) + try: + g.gfql(case.shape.build(), engine=case.engine) + except Exception: + continue + per_route.setdefault(case.route.name, []).append(calls["served"] > 0) + for route, served in per_route.items(): + assert sum(served) * 2 >= len(served), f"{route}: served {sum(served)} of {len(served)} admitted shapes" + + +def test_every_route_has_admitted_shapes(): + covered = {(c.route.name, c.engine) for c in CASES} + for route in ROUTES: + for engine in route.engines: + assert (route.name, engine) in covered, f"{route.name}/{engine} admits no registered shape" diff --git a/graphistry/tests/compute/test_chain.py b/graphistry/tests/compute/test_chain.py index 01fd9073a8..b5563b093c 100644 --- a/graphistry/tests/compute/test_chain.py +++ b/graphistry/tests/compute/test_chain.py @@ -5,6 +5,7 @@ import pytest from graphistry.compute.ast import ASTEdgeUndirected, ASTNode, ASTEdge, ASTObject, n, e, e_undirected, e_forward, e_reverse +from graphistry.tests.compute.gfql.routes.registry import Frames, register from graphistry.compute.chain import Chain, _try_chain_fast_path from graphistry.compute.typing import DataFrameT from graphistry.compute.predicates.is_in import IsIn, is_in @@ -652,9 +653,14 @@ def _cudf_or_skip(): return pytest.importorskip("cudf") +_FAST_FRAMES = Frames( + pd.DataFrame({'v': [0, 1, 2, 3, 4], 'attr': [10, 20, 30, 40, 50]}), + pd.DataFrame({'s': [0, 1, 2, 3, 0], 'd': [1, 2, 3, 4, 2], 'w': [5, 6, 7, 8, 9]}), + 'v', 's', 'd') + + def _fast_graph(engine): - nodes = pd.DataFrame({'v': [0, 1, 2, 3, 4], 'attr': [10, 20, 30, 40, 50]}) - edges = pd.DataFrame({'s': [0, 1, 2, 3, 0], 'd': [1, 2, 3, 4, 2], 'w': [5, 6, 7, 8, 9]}) + nodes, edges = _FAST_FRAMES.nodes, _FAST_FRAMES.edges if engine == "cudf": cudf = _cudf_or_skip() nodes = cudf.from_pandas(nodes) @@ -675,7 +681,7 @@ def topd(df): # shapes that ARE accelerated by the fast path -_FAST_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_FAST_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.fast", [ ("node_only", lambda: [n()]), ("node_filter", lambda: [n({'attr': 20})]), ("node_pred", lambda: [n({'attr': is_in([10, 30])})]), @@ -702,10 +708,10 @@ def topd(df): ("named_all_fwd", lambda: [n(name='x'), e_forward(hops=1, name='r'), n(name='y')]), ("named_all_rev", lambda: [n(name='x'), e_reverse(hops=1, name='r'), n(name='y')]), ("named_filtered", lambda: [n({'attr': 10}, name='x'), e_forward(hops=1), n(name='y')]), -] +], _FAST_FRAMES, tags=("native-fast",)) # shapes that BYPASS the fast path (still must be correct via the full path) -_BYPASS_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_BYPASS_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.bypass", [ ("hops_2", lambda: [n(), e_forward(hops=2), n()]), ("filtered_undirected", lambda: [n({'attr': 10}), e_undirected(hops=1), n({'attr': 30})]), # Named + undirected STAYS a bypass: an undirected edge makes a node reachable as @@ -716,7 +722,7 @@ def topd(df): # arrival side. Must bypass the fast path (regression guard for the prune gate). ("prune_endpoints_fwd", lambda: [n(), e_forward(hops=1, prune_to_endpoints=True), n()]), ("prune_endpoints_rev", lambda: [n(), e_reverse(hops=1, prune_to_endpoints=True), n()]), -] +], _FAST_FRAMES, tags=("native-fast-bypass",), row_tags={"prune_endpoints_fwd": ("#2053",), "prune_endpoints_rev": ("#2053",)}) _CUDF_26_DIVERGENT = {"prune_endpoints_fwd", "prune_endpoints_rev"} # graphistry/pygraphistry#2043 @@ -750,7 +756,7 @@ def test_fast_path_differential_parity_vs_full_path(engine, label, build, reques # Named shapes whose ALIAS FLAG COLUMNS (not merely node/edge sets) must match the full # path. `_setsig` above compares ids only, so it cannot see a wrong alias tag. -_NAMED_ALIAS_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_NAMED_ALIAS_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.named_alias", [ ("src_only", lambda: [n(name='x'), e_forward(hops=1), n()]), ("dst_only", lambda: [n(), e_forward(hops=1), n(name='y')]), ("edge_only", lambda: [n(), e_forward(hops=1, name='r'), n()]), @@ -762,7 +768,7 @@ def test_fast_path_differential_parity_vs_full_path(engine, label, build, reques # DEAD END: attr==50 is node 4, which has no outgoing edge. The tag keys on the # SURVIVING EDGES, so the alias must come back False/empty rather than True. ("dead_end_seed", lambda: [n({'attr': 50}, name='x'), e_forward(hops=1, name='r'), n(name='y')]), -] +], _FAST_FRAMES, tags=("alias",)) @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @@ -825,12 +831,12 @@ def _assert_full_frame_value_parity(fast: DataFrameT, full: DataFrameT, # Named served shapes for FULL-FRAME parity. `_setsig` compares id sets and the flags # test compares alias columns, so before this NO test compared the carried DATA columns # ('attr', 'w') of a named served result against the full path. -_NAMED_VALUE_PARITY_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_NAMED_VALUE_PARITY_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.named_value_parity", [ ("all_forward", lambda: [n(name='x'), e_forward(hops=1, name='r'), n(name='y')]), ("all_reverse", lambda: [n(name='x'), e_reverse(hops=1, name='r'), n(name='y')]), ("seed_filtered", lambda: [n({'attr': 10}, name='x'), e_forward(hops=1, name='r'), n(name='y')]), ("edge_match", lambda: [n(name='x'), e_forward(hops=1, edge_match={'w': 5}, name='r'), n(name='y')]), -] +], _FAST_FRAMES, tags=("alias", "values")) @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @@ -866,13 +872,13 @@ def test_fast_path_named_full_frame_value_parity(engine, label, build): # cardinality, so these all engage the fast path — and an empty answer must come back # as the right empty SHAPE (alias columns present, zero rows), not a throw and not a # missing-column frame. -_NAMED_EMPTY_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_NAMED_EMPTY_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.named_empty", [ # seed filter matches no node at all (distinct from dead_end_seed, which matches # a node that has no surviving edge) ("zero_seed", lambda: [n({'attr': 999}, name='x'), e_forward(hops=1, name='r'), n(name='y')]), ("zero_dst", lambda: [n(name='x'), e_forward(hops=1, name='r'), n({'attr': 999}, name='y')]), ("zero_edge_match", lambda: [n(name='x'), e_forward(hops=1, edge_match={'w': 999}, name='r'), n(name='y')]), -] +], _FAST_FRAMES, tags=("alias", "empty")) @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @@ -1023,7 +1029,7 @@ def test_fast_path_named_datetime_categorical_columns_ride_along(): # overwrite/raise behavior alone. The FROM-side binding columns and the node-id # binding are excluded here: those wrong-served (diverged) before, are now GATED to # decline, and are pinned by the two regression tests below. -_ALIAS_SHADOW_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_ALIAS_SHADOW_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.alias_shadow", [ ("node_alias_shadows_node_data_col", lambda: [n(name='attr'), e_forward(hops=1), n()]), ("edge_alias_shadows_edge_data_col", lambda: [n(), e_forward(hops=1, name='w'), n()]), # edge aliases named like the source/destination/edge-id bindings are rejected before @@ -1031,7 +1037,7 @@ def test_fast_path_named_datetime_categorical_columns_ride_along(): # cross-frame names are NOT collisions: nodes have no 'w', edges have no 'v' ("node_alias_named_like_edge_col", lambda: [n(name='w'), e_forward(hops=1), n()]), ("edge_alias_named_like_node_id", lambda: [n(), e_forward(hops=1, name='v'), n()]), -] +], _FAST_FRAMES, tags=("alias-collision",)) @pytest.mark.parametrize("label,build", _ALIAS_SHADOW_SHAPES, diff --git a/graphistry/tests/compute/test_chain_alias_column_collision.py b/graphistry/tests/compute/test_chain_alias_column_collision.py index 3c0c69efb5..42e9a92f13 100644 --- a/graphistry/tests/compute/test_chain_alias_column_collision.py +++ b/graphistry/tests/compute/test_chain_alias_column_collision.py @@ -13,11 +13,13 @@ import graphistry from graphistry.compute.ast import e_forward, e_reverse, e_undirected, n +from graphistry.tests.compute.gfql.routes.registry import Frames, register NODES = pd.DataFrame({"key": [1, 2, 3, 4], "id": [10, 20, 30, 40], "type": ["p", "p", "m", "m"], "w": [1, 2, 3, 4]}) EDGES = pd.DataFrame({"s": [3, 3, 4, 1], "d": [1, 2, 1, 4], "type": ["HAS_CREATOR", "OTHER", "HAS_CREATOR", "OTHER"], "eid": [100, 101, 102, 103], "w": [5, 6, 7, 8]}) ENGINES = ["pandas", "cudf", "polars"] +FRAMES = Frames(NODES, EDGES, "key", "s", "d", "eid") def _graph(engine, indexed): @@ -62,6 +64,8 @@ def topd(x): "seed, edge and destination aliases all collide": [n({"id": 30}, name="id"), e_forward({"type": "HAS_CREATOR"}, name="type"), n({"type": "p"}, name="type")], } +register("collision.served", [(k, (lambda v=v: list(v))) for k, v in SERVED.items()], FRAMES, tags=("alias-collision", "#2039")) + @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("indexed", [False, True], ids=["scan", "indexed"]) @@ -81,6 +85,8 @@ def test_single_hop_collisions_match_the_pandas_full_path(engine, indexed, shape "edge alias = filtered column, to_fixed_point": [n({"id": 30}, name="m"), e_forward({"type": "HAS_CREATOR"}, to_fixed_point=True, name="type"), n(name="p")], } +register("collision.multi_hop", [(k, (lambda v=v: list(v))) for k, v in MULTI_HOP.items()], FRAMES, tags=("alias-collision", "#2049")) + @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("shape", list(MULTI_HOP)) diff --git a/graphistry/tests/conftest.py b/graphistry/tests/conftest.py index b5b633cd92..221eac3707 100644 --- a/graphistry/tests/conftest.py +++ b/graphistry/tests/conftest.py @@ -41,39 +41,6 @@ def _gfql_routes_off(): if not routes: yield return - import graphistry.compute.chain as chain_mod - import graphistry.compute.gfql_unified as unified - import graphistry.compute.gfql.index as index_pkg - import graphistry.compute.gfql.index.api as index_api - import graphistry.compute.gfql.index.bindings as bindings - import graphistry.compute.gfql.lazy.engine.polars.chain as pchain - - def none(*a, **k): - return None - - patches = [] - - def patch(mod, name, value): - patches.append((mod, name, getattr(mod, name))) - setattr(mod, name, value) - - if "native-fast" in routes: - patch(chain_mod, "_try_chain_fast_path", none) - if "polars-seeded" in routes: - patch(pchain, "_try_seeded_chain_polars", none) - if "polars-plain" in routes: - patch(pchain, "polars_plain_single_hop_admits", none) - if "index-hop" in routes: - patch(index_pkg, "maybe_index_hop", none) - patch(index_api, "maybe_index_hop", none) - if "indexed-kernel" in routes: - patch(bindings, "_try_indexed_connected_bindings_state", none) - if "cypher-fast" in routes: - for name in ("_execute_seeded_node_lookup_fast_path", "_execute_seeded_typed_hop_fast_path", - "_execute_single_hop_grouped_aggregate_fast_path", "_execute_two_hop_count_fast_path"): - patch(unified, name, none) - try: + from graphistry.tests.compute.gfql.routes.switch import routes_off + with routes_off(routes): yield - finally: - for mod, name, value in reversed(patches): - setattr(mod, name, value) From 30a2b23c45763f8a33366dbfd35914a5d24f8b16 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 18:19:39 -0700 Subject: [PATCH 08/12] test(gfql): harness joins the polars lane; remaining native-fast engagement pins marked The route harness mentions polars, so it runs in bin/test-polars.sh (lane completeness pin). Six more tests that assert a native-fast serve (hits == 1, served spies) carry the route_engaged marker, so the routes-off replay for native-fast reports result divergences only. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QztW7jYsDd66e8rb8pJNQA --- bin/test-polars.sh | 1 + .../tests/compute/gfql/test_native_seed_resolution_2027.py | 5 +++++ .../tests/compute/gfql/test_seeded_typed_hop_fastpath.py | 1 + graphistry/tests/compute/test_chain.py | 1 + 4 files changed, 8 insertions(+) diff --git a/bin/test-polars.sh b/bin/test-polars.sh index eb6811afb0..dc934a7c2c 100755 --- a/bin/test-polars.sh +++ b/bin/test-polars.sh @@ -125,6 +125,7 @@ POLARS_TEST_FILES=( graphistry/tests/compute/gfql/cypher/test_variable_column_collision.py graphistry/tests/compute/test_chain_alias_column_collision.py graphistry/tests/compute/test_gfql_op_list_hides_internal_columns.py + graphistry/tests/compute/gfql/routes/test_route_harness.py graphistry/tests/compute/gfql/test_engine_polars_semi_key_dedup.py graphistry/tests/compute/gfql/test_engine_polars_call_modality.py graphistry/tests/compute/gfql/test_engine_polars_gpu.py diff --git a/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py b/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py index fb9c0c0853..a8165d9fca 100644 --- a/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py +++ b/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py @@ -114,6 +114,7 @@ def spy(*a, **k): assert calls["n"] >= 1 and len(out._edges) == 1 +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("binding_first", [True, False]) def test_named_single_node_alias_layout_matches_the_full_path(engine, binding_first): @@ -129,6 +130,7 @@ def test_named_single_node_alias_layout_matches_the_full_path(engine, binding_fi assert list(fast._nodes.columns)[:2] == ["key", "p"] +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("binding_first", [True, False]) def test_named_single_node_alias_overwrites_colliding_property_like_full_path(engine, binding_first): @@ -146,6 +148,7 @@ def test_named_single_node_alias_overwrites_colliding_property_like_full_path(en pd.testing.assert_frame_equal(_canon(fast._nodes), _canon(full._nodes), check_dtype=False) +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_named_hop_aliases_overwrite_nonfinal_properties_like_full_path(engine): g = _lane_graph(engine) @@ -200,6 +203,7 @@ def test_policy_off_keeps_parity_and_uses_no_index(engine, shape): assert report["used_index"] is False and report["decision_code"] == "policy_off", report +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("shape", list(SHAPES)) def test_stale_indexes_keep_parity_and_are_not_used(engine, shape): @@ -231,6 +235,7 @@ def test_non_scalar_seed_predicates_keep_parity_without_the_index(engine, seed): assert not any(s.get("seam") in ("native_seed_lookup", "native_seeded_hop") and s.get("served") for s in steps), steps +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_duplicate_node_rows_are_answered_once_each_on_the_native_lookup(engine): """A node table that repeats a key row (a contract violation the engine tolerates): the diff --git a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py index 941e852ef6..f7e994ad24 100644 --- a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py +++ b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py @@ -1015,6 +1015,7 @@ def test_stale_index_declines_to_scan(self, monkeypatch): pd.testing.assert_frame_equal(self._canon(got), self._canon(plain)) + @pytest.mark.route_engaged("native-fast") def test_native_chain_hop_indexed_parity_forward_and_reverse(self, monkeypatch): """M1 pin: the native-chain hop helper's indexed branch (only reachable via chain ops, never Cypher) — forward (EDGE_OUT_ADJ) and reverse (EDGE_IN_ADJ), diff --git a/graphistry/tests/compute/test_chain.py b/graphistry/tests/compute/test_chain.py index b5563b093c..df1b8dc955 100644 --- a/graphistry/tests/compute/test_chain.py +++ b/graphistry/tests/compute/test_chain.py @@ -842,6 +842,7 @@ def _assert_full_frame_value_parity(fast: DataFrameT, full: DataFrameT, @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @pytest.mark.parametrize("label,build", _NAMED_VALUE_PARITY_SHAPES, ids=[s[0] for s in _NAMED_VALUE_PARITY_SHAPES]) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_full_frame_value_parity(engine, label, build): """POSITIVE, whole-frame: a named served result must carry the same VALUES as the full path on EVERY column — ids, data columns, and alias flags — not just the id From 000e9e248a4ab03d8e2112cb6762aa8fb84cd063 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 19:30:34 -0700 Subject: [PATCH 09/12] test(gfql): engagement pins from the 7-mode routes-off replay carry the route_engaged marker Replay at the harness head (scratchpad ledger kept under reviews/2054/): every remaining single-route id was an engagement pin (a served spy, a trace or a lane-specific explain step) or the #2058 dtype class; the all-off residue adds four combined-route engagement pins and the #2034 duplicate-id case. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QztW7jYsDd66e8rb8pJNQA --- .../compute/gfql/cypher/test_lowering.py | 4 ++ .../compute/gfql/index/test_degree_consult.py | 3 + .../tests/compute/gfql/index/test_index.py | 2 + .../gfql/index/test_index_gpu_edge_match.py | 2 + .../gfql/index/test_indexed_bindings.py | 3 + .../compute/gfql/routes/test_route_harness.py | 1 + .../gfql/test_native_seed_lane_explain.py | 2 +- .../test_polars_native_seed_resolution.py | 3 + .../gfql/test_rewrite_param_discard.py | 1 + .../gfql/test_seeded_node_lookup_fastpath.py | 2 + .../gfql/test_seeded_typed_hop_fastpath.py | 4 ++ reviews/2054/all-off.divergences | 57 +++++++++++++++++++ reviews/2054/cypher-fast.divergences | 46 +++++++++++++++ reviews/2054/index-hop.divergences | 14 +++++ reviews/2054/indexed-kernel.divergences | 27 +++++++++ reviews/2054/native-fast.divergences | 25 ++++++++ reviews/2054/polars-plain.divergences | 1 + reviews/2054/polars-seeded.divergences | 12 ++++ reviews/2054/routes-off-ledger-3213f7d96.txt | 7 +++ 19 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 reviews/2054/all-off.divergences create mode 100644 reviews/2054/cypher-fast.divergences create mode 100644 reviews/2054/index-hop.divergences create mode 100644 reviews/2054/indexed-kernel.divergences create mode 100644 reviews/2054/native-fast.divergences create mode 100644 reviews/2054/polars-plain.divergences create mode 100644 reviews/2054/polars-seeded.divergences create mode 100644 reviews/2054/routes-off-ledger-3213f7d96.txt diff --git a/graphistry/tests/compute/gfql/cypher/test_lowering.py b/graphistry/tests/compute/gfql/cypher/test_lowering.py index 65c6a6b5dd..a64dc5c482 100644 --- a/graphistry/tests/compute/gfql/cypher/test_lowering.py +++ b/graphistry/tests/compute/gfql/cypher/test_lowering.py @@ -18626,6 +18626,7 @@ def test_t6_col_stats_decisions_are_visible_in_the_trace() -> None: outcomes={"nodes.id": "served", "edges.s": "served"}) +@pytest.mark.route_engaged("cypher-fast") def test_t6_assert_col_stats_helper_fails_loudly() -> None: """The helper must FAIL when the optimization did not fire -- an engagement pin that cannot fail is worse than none, which is the whole failure mode @@ -19225,6 +19226,7 @@ def _mk_h3_case_data(fixture: str) -> Tuple[pd.DataFrame, pd.DataFrame]: raise AssertionError(f"unknown fixture {fixture}") +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) @pytest.mark.parametrize("label,fixture,query", _H3_DIFFERENTIAL_CASES, ids=[c[0] for c in _H3_DIFFERENTIAL_CASES]) def test_h3_fused_two_hop_count_matches_eager_twin_and_pandas( @@ -19249,6 +19251,7 @@ def test_h3_fused_two_hop_count_matches_eager_twin_and_pandas( assert fused == oracle, f"{label}: fused lane diverged from the pandas oracle" +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_empty_match_counts_zero(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """openCypher counts over no rows as 0 -- not an empty frame.""" @@ -19359,6 +19362,7 @@ def test_h3_two_hop_count_fast_path_has_no_order_by_or_limit_surface(suffix: str assert _two_hop_count_alias(compiled.chain) == expect_alias +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_handles_degenerate_bindings(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """The node key may share a name with an endpoint column, and source/destination may be bound diff --git a/graphistry/tests/compute/gfql/index/test_degree_consult.py b/graphistry/tests/compute/gfql/index/test_degree_consult.py index acacdbd404..df2d5976df 100644 --- a/graphistry/tests/compute/gfql/index/test_degree_consult.py +++ b/graphistry/tests/compute/gfql/index/test_degree_consult.py @@ -102,6 +102,7 @@ def test_identity_anchors_to_the_bound_frame_not_the_partition() -> None: assert fact.source_ref is g._edges +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("n_p,n_c", [(3, 3), (5, 1), (2, 8), (7, 2)]) def test_slice_is_exact_across_domain_shapes(n_p: int, n_c: int, engine: str) -> None: @@ -117,6 +118,7 @@ def test_slice_is_exact_across_domain_shapes(n_p: int, n_c: int, engine: str) -> assert value == oracle +@pytest.mark.route_engaged("cypher-fast") def test_gapped_node_space_builds_facts_and_stays_exact() -> None: """Density is NOT required for the degree arrays: ids absent from the span contribute ZERO to the dot, so a gapped node space builds valid facts. (The @@ -137,6 +139,7 @@ def test_gapped_node_space_builds_facts_and_stays_exact() -> None: assert used, "P-domain [0,2] is dense, so the kernel must consult the fact" +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("seed", range(6)) def test_differential_vs_the_scan_on_random_typed_graphs(seed: int) -> None: """Values must be identical with and without the fact, on arbitrary degree diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 35c6fedace..4ab6fd2210 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -1465,6 +1465,7 @@ def _polars_indexed_graph(): return g.gfql_index_all(engine="polars") +@pytest.mark.route_engaged("index-hop") def test_auto_engine_gfql_serves_polars_index_1767_cliff(): """#1767 cliff pin: polars frames + explicit polars index + gfql with NO engine argument must serve path=index on engine=polars (AUTO routes native, so the @@ -1701,6 +1702,7 @@ def test_col_stats_auto_narrows_lazy_frames(self): gi = gl.gfql_index_col_stats() # AUTO on lazy frames must not crash assert gi is not None + @pytest.mark.route_engaged("index-hop") def test_inversion_auto_index_auto_gfql_serves_polars_index(self): """THE INVERSION PIN. The exact scenario the retracted #1767 regressed to the scan floor: ``gfql_index_all()`` with NO engine + ``g.gfql( Any: _assert_decision(decisions[0], seam="connected_bindings", served=True) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_destination_property_projection_dtype_parity( engine: str, @@ -726,6 +728,7 @@ def test_node_property_index_prefers_the_most_selective_column( pytest.param({"grp": 0}, "grp", False, id="unselective-keeps-scan"), ], ) +@pytest.mark.route_engaged("index-hop", "indexed-kernel") def test_node_property_index_cost_gate_under_policy_use( seed: Dict[str, Any], indexed_column: str, diff --git a/graphistry/tests/compute/gfql/routes/test_route_harness.py b/graphistry/tests/compute/gfql/routes/test_route_harness.py index e00a6d6db0..7ab70fdb6a 100644 --- a/graphistry/tests/compute/gfql/routes/test_route_harness.py +++ b/graphistry/tests/compute/gfql/routes/test_route_harness.py @@ -154,6 +154,7 @@ def test_admitted_shape_is_served_and_matches_the_general_path(case: Case, reque pytest.xfail(f"{case.id}: admitted by the predicate, declined by the lane body (attenuation ledger)") +@pytest.mark.route_engaged("native-fast", "polars-plain", "polars-seeded") def test_every_route_serves_most_of_what_it_admits(monkeypatch): """A lane that declines most admitted shapes has a predicate that no longer describes it.""" per_route: Dict[str, List[int]] = {} diff --git a/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py b/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py index 9bd377a4b2..15d29fb133 100644 --- a/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py +++ b/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py @@ -44,7 +44,7 @@ def test_node_only_lookup_served_by_the_property_index_is_explained(engine): assert len(g.gfql(NODE_ONLY, engine=engine, index_policy="use")._nodes) == 1 -@pytest.mark.route_engaged("native-fast") +@pytest.mark.route_engaged("native-fast", "polars-seeded") @pytest.mark.parametrize("engine", ENGINES) def test_seeded_typed_hop_served_by_the_resident_indexes_is_explained(engine): g = _graph(engine) diff --git a/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py b/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py index 8c95ce13ac..99450c856d 100644 --- a/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py +++ b/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py @@ -33,6 +33,7 @@ def _graph(reverse=False, indexed=True, padding=0): return g.gfql_index_all(engine="polars").gfql_index_node_props(["id"], engine="polars") if indexed else g +@pytest.mark.route_engaged("polars-seeded") @pytest.mark.parametrize("reverse", [False, True]) @pytest.mark.parametrize("indexed", [False, True]) @pytest.mark.parametrize("seed", [{"id": 104}, {"kind": "Message"}, {"id": 105}, {"id": 999}]) @@ -73,6 +74,7 @@ def spy(*args): _NAMED_TYPED_HOP = [n({"id": 104}, name="m"), e_forward({"type": "T"}, name="e"), n({"kind": "Person"}, name="p")] +@pytest.mark.route_engaged("polars-seeded") def test_native_seeded_hop_is_served_from_the_index_and_traced(): from graphistry.compute.gfql.index import index_trace g = _graph() @@ -98,6 +100,7 @@ def test_native_seeded_hop_declines_without_a_usable_index(policy, monkeypatch): assert_frame_equal(fast._edges, full._edges) +@pytest.mark.route_engaged("polars-seeded") @pytest.mark.parametrize("single_node", [False, True]) def test_native_property_seed_uses_resident_index(single_node, monkeypatch): import graphistry.compute.gfql.index.bindings as bindings diff --git a/graphistry/tests/compute/gfql/test_rewrite_param_discard.py b/graphistry/tests/compute/gfql/test_rewrite_param_discard.py index 91941879e9..c86ffc2651 100644 --- a/graphistry/tests/compute/gfql/test_rewrite_param_discard.py +++ b/graphistry/tests/compute/gfql/test_rewrite_param_discard.py @@ -210,6 +210,7 @@ def test_indexed_bypass_table_edges_survives_a_projection(engine: str) -> None: "(index/bindings.py gate), so polars-gpu always takes the scan path", )), ]) +@pytest.mark.route_engaged("indexed-kernel") def test_indexed_bypass_still_serves_a_bare_rows(engine: str) -> None: """THE NEGATIVE SIDE: declining on a non-default `table` must not decline everything. diff --git a/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py b/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py index f8280b2d28..e6c27ccee5 100644 --- a/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py +++ b/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py @@ -114,6 +114,7 @@ def test_node_lookup_engages_with_parity(engine, indexed, q, label): _assert_parity(_graph(engine, indexed), engine, q, "seeded_node_lookup") +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_lookup_matches_independent_oracle(engine): g = _graph(engine) @@ -263,6 +264,7 @@ def test_hub_seed_over_the_frontier_gate_keeps_parity(engine, indexed): pd.testing.assert_frame_equal(_canon(fast), _canon(full)) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_seed_matching_several_nodes_projects_each_seed(engine): """A non-unique seed predicate: every seed row pairs with its own destinations.""" diff --git a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py index f7e994ad24..422aa7ce7e 100644 --- a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py +++ b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py @@ -851,12 +851,14 @@ def spy(*a, **k): assert bool(hits["n"]) == expect_engage, f"engaged={hits['n']} expected={expect_engage}" return fast, full + @pytest.mark.route_engaged("cypher-fast") def test_pandas_int_bool_dtype_parity(self): fast, full = self._fast_and_full(self._typed_graph(), "pandas", self.Q) pd.testing.assert_frame_equal(_canon_nodes(fast), _canon_nodes(full)) dt = dict(zip(fast._nodes.columns, map(str, fast._nodes.dtypes))) assert dt == {"pid": "int64", "a": "float64", "f": "object"} + @pytest.mark.route_engaged("cypher-fast") def test_polars_int_bool_dtype_parity(self): pytest.importorskip("polars") fast, full = self._fast_and_full(self._pl_graph(), "polars", self.Q) @@ -876,6 +878,7 @@ def test_pandas_datetime_property_declines(self): fast, full = self._fast_and_full(self._typed_graph(), "pandas", q, expect_engage=False) pd.testing.assert_frame_equal(_canon_nodes(fast), _canon_nodes(full)) + @pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_edges_empty_frame_not_none(self, engine): g = self._typed_graph() if engine == "pandas" else self._pl_graph() @@ -1042,6 +1045,7 @@ def spy(*a, **k): plain = mk().gfql(ops, engine="pandas") pd.testing.assert_frame_equal(self._canon(got), self._canon(plain)) + @pytest.mark.route_engaged("cypher-fast") def test_uint64_int64_id_mix_declines_not_collapses(self, monkeypatch): """B1 pin: int64<->uint64 promotes to float64, which collapses ids >= 2**53 into false matches; the gate must DECLINE (scan path compares exactly).""" diff --git a/reviews/2054/all-off.divergences b/reviews/2054/all-off.divergences new file mode 100644 index 0000000000..5a1393b994 --- /dev/null +++ b/reviews/2054/all-off.divergences @@ -0,0 +1,57 @@ +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_empty_match_counts_zero[polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_handles_degenerate_bindings[polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[base_distinct_domain-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[base_distinct_edges-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[empty_no_edges-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[empty_no_matching_nodes-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_distinct_domain-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_distinct_edges-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_end_only_filter-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_start_only_filter-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[string_ids-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_t6_assert_col_stats_helper_fails_loudly +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[0] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[1] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[2] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[3] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[4] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[5] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_gapped_node_space_builds_facts_and_stays_exact +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-cudf] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-pandas] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-polars] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-cudf] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-pandas] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-polars] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-cudf] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-pandas] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-polars] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-cudf] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-pandas] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_node_property_index_cost_gate_under_policy_use[selective-uses-index] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_node_property_index_cost_gate_under_policy_use[unselective-keeps-scan] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_node_property_index_duplicate_values_match_scan +FAILED graphistry/tests/compute/gfql/index/test_index.py::test_auto_engine_gfql_serves_polars_index_1767_cliff +FAILED graphistry/tests/compute/gfql/index/test_index.py::TestIndexAutoPreservesPolarsFrames::test_inversion_auto_index_auto_gfql_serves_polars_index +FAILED graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_alias_column_collision_2039.py::test_destination_alias_marker_replaces_the_colliding_column_like_pandas +FAILED graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py::test_chain_surface_keeps_node_attribute_dtypes[pandas] +FAILED graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py::test_has_label_narrowing_applies_on_reached_collision[polars] +FAILED graphistry/tests/compute/gfql/test_rewrite_param_discard.py::test_indexed_bypass_still_serves_a_bare_rows[cudf] +FAILED graphistry/tests/compute/gfql/test_rewrite_param_discard.py::test_indexed_bypass_still_serves_a_bare_rows[pandas] +FAILED graphistry/tests/compute/gfql/test_rewrite_param_discard.py::test_indexed_bypass_still_serves_a_bare_rows[polars] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[cudf] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[polars] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_returns_each_duplicate_id_row_once[cudf] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_returns_each_duplicate_id_row_once[pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[cudf] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[polars] +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_edges_empty_frame_not_none[pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_edges_empty_frame_not_none[polars] +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_pandas_int_bool_dtype_parity +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_polars_int_bool_dtype_parity diff --git a/reviews/2054/cypher-fast.divergences b/reviews/2054/cypher-fast.divergences new file mode 100644 index 0000000000..540db7c953 --- /dev/null +++ b/reviews/2054/cypher-fast.divergences @@ -0,0 +1,46 @@ +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_empty_match_counts_zero[polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_handles_degenerate_bindings[polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[base_distinct_domain-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[base_distinct_edges-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[empty_no_edges-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[empty_no_matching_nodes-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_distinct_domain-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_distinct_edges-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_end_only_filter-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_start_only_filter-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[string_ids-polars] +FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_t6_assert_col_stats_helper_fails_loudly +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[0] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[1] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[2] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[3] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[4] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[5] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_gapped_node_space_builds_facts_and_stays_exact +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-cudf] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-pandas] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-polars] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-cudf] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-pandas] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-polars] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-cudf] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-pandas] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-polars] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-cudf] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-pandas] +FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[polars] +FAILED graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py::test_has_label_narrowing_applies_on_reached_collision[polars] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[cudf] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[polars] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[cudf] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[polars] +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestResidentIndexSeededFastPath::test_uint64_int64_id_mix_declines_not_collapses +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_edges_empty_frame_not_none[pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_edges_empty_frame_not_none[polars] +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_pandas_int_bool_dtype_parity +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_polars_int_bool_dtype_parity diff --git a/reviews/2054/index-hop.divergences b/reviews/2054/index-hop.divergences new file mode 100644 index 0000000000..5289985f9f --- /dev/null +++ b/reviews/2054/index-hop.divergences @@ -0,0 +1,14 @@ +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_node_property_index_duplicate_values_match_scan +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[canonical-distinct-order-limit-suffix-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[fixed-hop-reverse-composition-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is1-directed-projection-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-connected-two-hop-bag-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-optional-continuation-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[null-and-dtype-parity-pandas] +FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_empty_candidate_batch_on_device[polars-gpu] +FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[boolean-polars-gpu] +FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[float-polars-gpu] +FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[int64-polars-gpu] +FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[Int64-polars-gpu] +FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[string-polars-gpu] diff --git a/reviews/2054/indexed-kernel.divergences b/reviews/2054/indexed-kernel.divergences new file mode 100644 index 0000000000..8d590f0b2d --- /dev/null +++ b/reviews/2054/indexed-kernel.divergences @@ -0,0 +1,27 @@ +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[canonical-distinct-order-limit-suffix-cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[canonical-distinct-order-limit-suffix-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[canonical-distinct-order-limit-suffix-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[fixed-hop-reverse-composition-cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[fixed-hop-reverse-composition-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[fixed-hop-reverse-composition-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is1-directed-projection-cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is1-directed-projection-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is1-directed-projection-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is3-undirected-multiplicity-cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is3-undirected-multiplicity-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is3-undirected-multiplicity-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-connected-two-hop-bag-cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-connected-two-hop-bag-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-connected-two-hop-bag-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-optional-continuation-cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-optional-continuation-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-optional-continuation-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[null-and-dtype-parity-cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[null-and-dtype-parity-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[null-and-dtype-parity-polars] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[official-no-match-stratum-cudf] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[official-no-match-stratum-pandas] +FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[official-no-match-stratum-polars] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_two_alias_projection_parity[MATCH (m:Message {id: 305})-[:HAS_CREATOR]->(p:Person) RETURN m.id AS a, m.id AS b, p.flag AS c-repeated + bool-indexed-pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_two_alias_projection_parity[MATCH (m:Message {id: 305})-[:HAS_CREATOR]->(p:Person) RETURN m.score AS ms, p.score AS ps, m.flag AS mf, p.flag AS pf-int + bool from both aliases-indexed-pandas] +FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_two_alias_projection_parity[MATCH (m:Message {id: 305})-[r:HAS_CREATOR]->(p:Person) RETURN m.id, m.score, m.flag, m.firstName, r.w, r.eflag, r.type, p.id, p.firstName, p.age, p.score, p.flag-twelve properties across three aliases-indexed-pandas] diff --git a/reviews/2054/native-fast.divergences b/reviews/2054/native-fast.divergences new file mode 100644 index 0000000000..f673c6aab8 --- /dev/null +++ b/reviews/2054/native-fast.divergences @@ -0,0 +1,25 @@ +FAILED graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_alias_column_collision_2039.py::test_destination_alias_marker_replaces_the_colliding_column_like_pandas +FAILED graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py::test_chain_surface_keeps_node_attribute_dtypes[pandas] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_duplicate_node_rows_are_answered_once_each_on_the_native_lookup[cudf] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_duplicate_node_rows_are_answered_once_each_on_the_native_lookup[pandas] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_hop_aliases_overwrite_nonfinal_properties_like_full_path[cudf] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_hop_aliases_overwrite_nonfinal_properties_like_full_path[pandas] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_layout_matches_the_full_path[False-cudf] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_layout_matches_the_full_path[False-pandas] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_layout_matches_the_full_path[True-cudf] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_layout_matches_the_full_path[True-pandas] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_overwrites_colliding_property_like_full_path[False-cudf] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_overwrites_colliding_property_like_full_path[False-pandas] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_overwrites_colliding_property_like_full_path[True-cudf] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_overwrites_colliding_property_like_full_path[True-pandas] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_stale_indexes_keep_parity_and_are_not_used[seeded typed hop, all named-cudf] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_stale_indexes_keep_parity_and_are_not_used[seeded typed hop, all named-pandas] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_stale_indexes_keep_parity_and_are_not_used[seeded typed hop + rows + select-cudf] +FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_stale_indexes_keep_parity_and_are_not_used[seeded typed hop + rows + select-pandas] +FAILED graphistry/tests/compute/gfql/test_polars_lane_completeness.py::test_every_polars_mentioning_test_module_is_in_the_lane_or_justified +FAILED graphistry/tests/compute/gfql/test_polars_lane_completeness.py::test_no_module_level_polars_gate_outside_the_lane +FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestResidentIndexSeededFastPath::test_native_chain_hop_indexed_parity_forward_and_reverse +FAILED graphistry/tests/compute/test_chain.py::test_fast_path_named_full_frame_value_parity[all_forward-pandas] +FAILED graphistry/tests/compute/test_chain.py::test_fast_path_named_full_frame_value_parity[all_reverse-pandas] +FAILED graphistry/tests/compute/test_chain.py::test_fast_path_named_full_frame_value_parity[edge_match-pandas] +FAILED graphistry/tests/compute/test_chain.py::test_fast_path_named_full_frame_value_parity[seed_filtered-pandas] diff --git a/reviews/2054/polars-plain.divergences b/reviews/2054/polars-plain.divergences new file mode 100644 index 0000000000..175c373fee --- /dev/null +++ b/reviews/2054/polars-plain.divergences @@ -0,0 +1 @@ +FAILED graphistry/tests/compute/gfql/routes/test_route_harness.py::test_every_route_serves_most_of_what_it_admits diff --git a/reviews/2054/polars-seeded.divergences b/reviews/2054/polars-seeded.divergences new file mode 100644 index 0000000000..11cafd867e --- /dev/null +++ b/reviews/2054/polars-seeded.divergences @@ -0,0 +1,12 @@ +FAILED graphistry/tests/compute/gfql/routes/test_route_harness.py::test_every_route_serves_most_of_what_it_admits +FAILED graphistry/tests/compute/gfql/test_native_seed_lane_explain.py::test_seeded_typed_hop_served_by_the_resident_indexes_is_explained[polars] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed0-True-False] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed0-True-True] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed1-True-False] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed1-True-True] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed2-True-False] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed2-True-True] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed3-True-False] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed3-True-True] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_property_seed_uses_resident_index[False] +FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_seeded_hop_is_served_from_the_index_and_traced diff --git a/reviews/2054/routes-off-ledger-3213f7d96.txt b/reviews/2054/routes-off-ledger-3213f7d96.txt new file mode 100644 index 0000000000..a5ad7e3ef4 --- /dev/null +++ b/reviews/2054/routes-off-ledger-3213f7d96.txt @@ -0,0 +1,7 @@ +native-fast: 25 ids +polars-seeded: 12 ids +polars-plain: 1 ids +index-hop: 14 ids +indexed-kernel: 27 ids +cypher-fast: 46 ids +all-off: 57 ids From 71b0917adeda6d55a385ebcd029009ff9bcafd1b Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sat, 5 Sep 2026 15:20:51 -0700 Subject: [PATCH 10/12] ci: run the gfql lanes on chain-engine changes The gfql change filter listed compute/gfql/** and the unified entrypoint but not the chain engine itself (chain.py, chain_fast_paths.py, hop.py, gfql_fast_paths.py, filter_by_dict.py, ast.py, predicates/), so a PR touching only those skipped tck-gfql, the Cypher-frontend gates and the gfql benchmark lane. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QztW7jYsDd66e8rb8pJNQA (cherry picked from commit c402b7f392ef056298802ce3ea676ab1120e1b51) --- .github/workflows/ci.yml | 8 ++++++++ CHANGELOG.md | 1 + 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5c9115b69..b554870eff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -242,11 +242,19 @@ jobs: '^graphistry/compute/gfql/' \ '^graphistry/compute/gfql_validate\.py$' \ '^graphistry/compute/gfql_unified\.py$' \ + '^graphistry/compute/gfql_fast_paths\.py$' \ + '^graphistry/compute/chain.*\.py$' \ + '^graphistry/compute/hop\.py$' \ + '^graphistry/compute/filter_by_dict\.py$' \ + '^graphistry/compute/ast\.py$' \ + '^graphistry/compute/predicates/' \ '^graphistry/models/gfql/' \ '^graphistry/Plottable\.py$' \ '^graphistry/tests/benchmarks/gfql/' \ '^graphistry/tests/compute/gfql/' \ '^graphistry/tests/compute/test_gfql.*\.py$' \ + '^graphistry/tests/compute/test_chain.*\.py$' \ + '^graphistry/tests/compute/test_hop.*\.py$' \ '^graphistry/tests/test_gfql_.*\.py$' \ '^tests/gfql/' diff --git a/CHANGELOG.md b/CHANGELOG.md index 690f1c01d6..406537ad89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Infrastructure +- **CI: the gfql change filter now includes the chain engine** (`compute/chain*.py`, `hop.py`, `gfql_fast_paths.py`, `filter_by_dict.py`, `ast.py`, `predicates/`, and the chain/hop test files), so tck-gfql, the Cypher-frontend gates and the gfql benchmark lane run on a change to the chain engine; they were skipped on #2055. - **CI: `test-docs` runs on docs-only pull requests (#2018)**: the job needed `python-lint-types`, which a docs-only change skips, and GitHub skips a job whose prerequisite was skipped. The gate now accepts skipped prerequisites and refuses only failed or cancelled ones, so documentation changes are built and tested before merge. ### Tests From 9616f118d33139e48ec769e41d4853a0f08cecb6 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 6 Sep 2026 08:37:08 -0700 Subject: [PATCH 11/12] fix(gfql): the polars plain single-hop lane keeps the one-row-per-id rule (fold repair: the inline branch the layout extracted had come back beside the dispatch) --- .../compute/gfql/lazy/engine/polars/chain.py | 62 ------------------- .../polars/chain_specializations/hotpaths.py | 1 + 2 files changed, 1 insertion(+), 62 deletions(-) diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 4e4ebface9..9fa83683ca 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -1003,68 +1003,6 @@ def _chain_traversal_polars(self: Plottable, ops, start_nodes: Optional[Any] = N if seeded is not None: return seeded if plain_shape is not None: - n0, e1, n2 = ops - node_table_bound = self._nodes is not None - gf = ensure_nodes_polars(self) - ncol, scol, dcol = gf._node, gf._source, gf._destination - assert ncol is not None and scol is not None and dcol is not None - gf, restore = _align_edge_endpoints(gf, ncol, scol, dcol) - edges = drop_null_endpoint_edges(gf._edges, scol, dcol) - n_from, n_to = (n0, n2) if e1.direction != "reverse" else (n2, n0) - all_ids = gf._nodes.select(pl.col(ncol)) - - def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": - if not node_op.filter_dict: - return None - return filter_by_dict_polars(gf._nodes, node_op.filter_dict).select(pl.col(ncol)) - - filter_sides = ((scol, _filter_ids(n_from)), (dcol, _filter_ids(n_to))) - for endpoint_col, filter_ids in filter_sides: - if filter_ids is not None: - edges = edges.join(filter_ids, left_on=endpoint_col, right_on=ncol, how="semi") - # A filtered side drew its ids FROM the node table; a synthesized one is vacuously closed. - sides_not_closed_by_a_filter = ( - [col for col, filter_ids in filter_sides if filter_ids is None] - if node_table_bound else []) - endpoints = endpoint_ids(edges, scol, dcol, ncol) - if sides_not_closed_by_a_filter: - from graphistry.compute.gfql.lazy import collect_all - unresolvable, nodes = collect_all([ - endpoints.lazy().join(all_ids.lazy(), on=ncol, how="anti").select(pl.len()), - gf._nodes.lazy().join(endpoints.lazy(), on=ncol, how="semi"), - ]) - if unresolvable.item() > 0: - for endpoint_col in sides_not_closed_by_a_filter: - edges = edges.join(all_ids, left_on=endpoint_col, right_on=ncol, how="semi") - nodes = gf._nodes.join( - endpoint_ids(edges, scol, dcol, ncol), on=ncol, how="semi") - else: - nodes = gf._nodes.join(endpoints, on=ncol, how="semi") - nodes = nodes.unique(subset=[ncol], maintain_order=True) # one row per node id, as the full chain and pandas collapse - return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) - filter_sides = ((scol, _filter_ids(n_from)), (dcol, _filter_ids(n_to))) - for endpoint_col, filter_ids in filter_sides: - if filter_ids is not None: - edges = edges.join(filter_ids, left_on=endpoint_col, right_on=ncol, how="semi") - # A filtered side drew its ids FROM the node table; a synthesized one is vacuously closed. - sides_not_closed_by_a_filter = ( - [col for col, filter_ids in filter_sides if filter_ids is None] - if node_table_bound else []) - endpoints = endpoint_ids(edges, scol, dcol, ncol) - if sides_not_closed_by_a_filter: - from graphistry.compute.gfql.lazy import collect_all - unresolvable, nodes = collect_all([ - endpoints.lazy().join(all_ids.lazy(), on=ncol, how="anti").select(pl.len()), - gf._nodes.lazy().join(endpoints.lazy(), on=ncol, how="semi"), - ]) - if unresolvable.item() > 0: - for endpoint_col in sides_not_closed_by_a_filter: - edges = edges.join(all_ids, left_on=endpoint_col, right_on=ncol, how="semi") - nodes = gf._nodes.join( - endpoint_ids(edges, scol, dcol, ncol), on=ncol, how="semi") - else: - nodes = gf._nodes.join(endpoints, on=ncol, how="semi") - return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) return _plain_single_hop_polars(self, ops) if start_nodes is not None: diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py index a6388dc45b..97b51e0bab 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py @@ -88,6 +88,7 @@ def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": endpoint_ids(edges, scol, dcol, ncol), on=ncol, how="semi") else: nodes = gf._nodes.join(endpoints, on=ncol, how="semi") + nodes = nodes.unique(subset=[ncol], maintain_order=True) # one row per node id, as the full chain and pandas collapse duplicate node rows return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) From f5cee3be28d4370b807e62226097e77cbeee7406 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Sun, 6 Sep 2026 23:49:13 -0700 Subject: [PATCH 12/12] test(gfql): route corpus gains a frame dimension; review ledgers leave the tree The shape corpus now runs over six frame variants (string ids, nullable ids with nulls, duplicate ids, self-loops and a cycle, an empty edge table, no edge-id binding) as well as the base frames; the harness oracle normalizes null ids, the serve-ratio pin reads the base corpus only, and known-divergence xfails are non-strict on variants where a shape can coincide. The only divergence the variants surfaced is #2034 (duplicate node ids on the single-node lookup), pinned by row. Route replay ledgers are PR comments, not tracked files. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HdYcPgiafGGutW9KfG3gg1 --- .../tests/compute/gfql/routes/corpus.py | 81 ++++++++++++++----- .../compute/gfql/routes/test_route_harness.py | 17 ++-- reviews/2054/all-off.divergences | 57 ------------- reviews/2054/cypher-fast.divergences | 46 ----------- reviews/2054/index-hop.divergences | 14 ---- reviews/2054/indexed-kernel.divergences | 27 ------- reviews/2054/native-fast.divergences | 25 ------ reviews/2054/polars-plain.divergences | 1 - reviews/2054/polars-seeded.divergences | 12 --- reviews/2054/routes-off-ledger-3213f7d96.txt | 7 -- 10 files changed, 70 insertions(+), 217 deletions(-) delete mode 100644 reviews/2054/all-off.divergences delete mode 100644 reviews/2054/cypher-fast.divergences delete mode 100644 reviews/2054/index-hop.divergences delete mode 100644 reviews/2054/indexed-kernel.divergences delete mode 100644 reviews/2054/native-fast.divergences delete mode 100644 reviews/2054/polars-plain.divergences delete mode 100644 reviews/2054/polars-seeded.divergences delete mode 100644 reviews/2054/routes-off-ledger-3213f7d96.txt diff --git a/graphistry/tests/compute/gfql/routes/corpus.py b/graphistry/tests/compute/gfql/routes/corpus.py index 5c89806fe7..95fcdedea7 100644 --- a/graphistry/tests/compute/gfql/routes/corpus.py +++ b/graphistry/tests/compute/gfql/routes/corpus.py @@ -14,38 +14,47 @@ from graphistry.tests.compute.gfql.routes.registry import Frames, register +KeyMap = Callable[[int], object] + + class Entry(NamedTuple): name: str ops: Callable[[], List[ASTObject]] tags: Tuple[str, ...] + shape: Callable[[KeyMap], List[ASTObject]] + + +def _entry(name: str, shape: Callable[[KeyMap], List[ASTObject]], tags: Tuple[str, ...]) -> Entry: + """``ops()`` builds the shape over the base frames; ``shape(k)`` maps the node-key literals for a frame variant.""" + return Entry(name, lambda: shape(lambda v: v), tags, shape) NODES = pd.DataFrame({"key": [1, 2, 3, 4, 5], "id": [10, 20, 30, 40, 50], "type": ["p", "p", "m", "m", "p"], "w": [1, 2, 3, 4, 5]}) EDGES = pd.DataFrame({"s": [1, 1, 2, 3, 3, 4], "d": [2, 3, 3, 1, 1, 5], "type": ["KNOWS", "KNOWS", "LIKES", "KNOWS", "KNOWS", "LIKES"], "eid": [0, 1, 2, 3, 4, 5], "w": [1, 2, 3, 4, 5, 6]}) CORPUS: List[Entry] = [ - Entry("single node, scalar filter", lambda: [n({"id": 30})], ("single-node",)), - Entry("single node, named", lambda: [n({"id": 30}, name="a")], ("single-node", "alias")), - Entry("single node, predicate filter", lambda: [n({"w": GT(2)})], ("single-node", "predicate")), - Entry("single node, no filter", lambda: [n()], ("single-node",)), - Entry("plain single hop, unseeded", lambda: [n(), e_forward(), n()], ("single-hop", "unseeded")), - Entry("plain single hop, seeded", lambda: [n({"key": 1}), e_forward(), n()], ("single-hop", "seeded", "#2051")), - Entry("plain single hop, seeded, reverse", lambda: [n({"key": 1}), e_reverse(), n()], ("single-hop", "seeded", "reverse")), - Entry("plain single hop, seeded, destination filter", lambda: [n({"key": 1}), e_forward(), n({"id": 20})], ("single-hop", "seeded", "dest-filter", "#2051")), - Entry("plain single hop, undirected, unconstrained", lambda: [n(), e_undirected(), n()], ("single-hop", "undirected")), - Entry("plain single hop, undirected, seeded", lambda: [n({"key": 1}), e_undirected(), n()], ("single-hop", "undirected", "seeded")), - Entry("typed single hop, seeded", lambda: [n({"key": 1}), e_forward({"type": "KNOWS"}), n()], ("single-hop", "seeded", "typed")), - Entry("typed single hop, seeded, named", lambda: [n({"key": 1}, name="a"), e_forward({"type": "KNOWS"}, name="e"), n(name="b")], ("single-hop", "seeded", "typed", "alias")), - Entry("typed single hop, seeded, named, undirected", lambda: [n({"key": 1}, name="a"), e_undirected({"type": "KNOWS"}, name="e"), n(name="b")], ("single-hop", "undirected", "alias")), - Entry("single hop, node and edge alias share a name", lambda: [n({"key": 1}, name="a"), e_forward(name="a"), n()], ("single-hop", "alias", "shared-alias-name")), - Entry("single hop, edge alias = filtered column", lambda: [n({"id": 30}, name="m"), e_forward({"type": "KNOWS"}, name="type"), n(name="p")], ("single-hop", "alias-collision", "#2039")), - Entry("single hop, destination alias = its filtered column", lambda: [n({"id": 30}, name="m"), e_forward({"type": "KNOWS"}, name="e"), n({"type": "p"}, name="type")], ("single-hop", "alias-collision", "#2039")), - Entry("single hop, source node match", lambda: [n(), e_forward(source_node_match={"type": "p"}), n()], ("single-hop", "endpoint-match")), - Entry("single hop, prune to endpoints", lambda: [n({"key": 1}), e_forward(prune_to_endpoints=True), n()], ("single-hop", "prune", "#2053")), - Entry("hops=2, seeded", lambda: [n({"key": 1}), e_forward(hops=2), n()], ("multi-hop", "seeded")), - Entry("hops=2, seeded, typed, named", lambda: [n({"key": 1}, name="a"), e_forward({"type": "KNOWS"}, hops=2, name="e"), n(name="b")], ("multi-hop", "typed", "alias", "#2049")), - Entry("to_fixed_point, seeded", lambda: [n({"key": 1}), e_forward(to_fixed_point=True), n()], ("multi-hop", "fixed-point")), - Entry("two single hops", lambda: [n({"key": 1}), e_forward(), n(), e_forward(), n()], ("two-steps",)), + _entry("single node, scalar filter", lambda k: [n({"id": 30})], ("single-node",)), + _entry("single node, named", lambda k: [n({"id": 30}, name="a")], ("single-node", "alias")), + _entry("single node, predicate filter", lambda k: [n({"w": GT(2)})], ("single-node", "predicate")), + _entry("single node, no filter", lambda k: [n()], ("single-node",)), + _entry("plain single hop, unseeded", lambda k: [n(), e_forward(), n()], ("single-hop", "unseeded")), + _entry("plain single hop, seeded", lambda k: [n({"key": k(1)}), e_forward(), n()], ("single-hop", "seeded", "#2051")), + _entry("plain single hop, seeded, reverse", lambda k: [n({"key": k(1)}), e_reverse(), n()], ("single-hop", "seeded", "reverse")), + _entry("plain single hop, seeded, destination filter", lambda k: [n({"key": k(1)}), e_forward(), n({"id": 20})], ("single-hop", "seeded", "dest-filter", "#2051")), + _entry("plain single hop, undirected, unconstrained", lambda k: [n(), e_undirected(), n()], ("single-hop", "undirected")), + _entry("plain single hop, undirected, seeded", lambda k: [n({"key": k(1)}), e_undirected(), n()], ("single-hop", "undirected", "seeded")), + _entry("typed single hop, seeded", lambda k: [n({"key": k(1)}), e_forward({"type": "KNOWS"}), n()], ("single-hop", "seeded", "typed")), + _entry("typed single hop, seeded, named", lambda k: [n({"key": k(1)}, name="a"), e_forward({"type": "KNOWS"}, name="e"), n(name="b")], ("single-hop", "seeded", "typed", "alias")), + _entry("typed single hop, seeded, named, undirected", lambda k: [n({"key": k(1)}, name="a"), e_undirected({"type": "KNOWS"}, name="e"), n(name="b")], ("single-hop", "undirected", "alias")), + _entry("single hop, node and edge alias share a name", lambda k: [n({"key": k(1)}, name="a"), e_forward(name="a"), n()], ("single-hop", "alias", "shared-alias-name")), + _entry("single hop, edge alias = filtered column", lambda k: [n({"id": 30}, name="m"), e_forward({"type": "KNOWS"}, name="type"), n(name="p")], ("single-hop", "alias-collision", "#2039")), + _entry("single hop, destination alias = its filtered column", lambda k: [n({"id": 30}, name="m"), e_forward({"type": "KNOWS"}, name="e"), n({"type": "p"}, name="type")], ("single-hop", "alias-collision", "#2039")), + _entry("single hop, source node match", lambda k: [n(), e_forward(source_node_match={"type": "p"}), n()], ("single-hop", "endpoint-match")), + _entry("single hop, prune to endpoints", lambda k: [n({"key": k(1)}), e_forward(prune_to_endpoints=True), n()], ("single-hop", "prune", "#2053")), + _entry("hops=2, seeded", lambda k: [n({"key": k(1)}), e_forward(hops=2), n()], ("multi-hop", "seeded")), + _entry("hops=2, seeded, typed, named", lambda k: [n({"key": k(1)}, name="a"), e_forward({"type": "KNOWS"}, hops=2, name="e"), n(name="b")], ("multi-hop", "typed", "alias", "#2049")), + _entry("to_fixed_point, seeded", lambda k: [n({"key": k(1)}), e_forward(to_fixed_point=True), n()], ("multi-hop", "fixed-point")), + _entry("two single hops", lambda k: [n({"key": k(1)}), e_forward(), n(), e_forward(), n()], ("two-steps",)), ] @@ -58,3 +67,31 @@ def by_name() -> Dict[str, Entry]: register("routes.corpus", [(e.name, e.ops, e.tags) for e in CORPUS], Frames(NODES, EDGES, "key", "s", "d", "eid")) + + +def _frame_variants() -> Dict[str, Tuple[Frames, Tuple[str, ...], Callable[[int], object]]]: + """The same shapes over frames that differ in what the routes must agree on: id dtype, + null and duplicate ids, self-loops and cycles, an empty edge table, no edge-id binding.""" + str_nodes = NODES.assign(key=NODES["key"].map(lambda k: f"n{k}")) + str_edges = EDGES.assign(s=EDGES["s"].map(lambda k: f"n{k}"), d=EDGES["d"].map(lambda k: f"n{k}")) + null_nodes = pd.concat([NODES, pd.DataFrame({"key": [None], "id": [60], "type": ["p"], "w": [6]})], ignore_index=True).astype({"key": "Int64"}) + null_edges = pd.concat([EDGES, pd.DataFrame({"s": [1], "d": [None], "type": ["KNOWS"], "eid": [6], "w": [7]})], ignore_index=True).astype({"s": "Int64", "d": "Int64"}) + dup_nodes = pd.concat([NODES, NODES.iloc[[0]].assign(w=99)], ignore_index=True) + loop_edges = pd.concat([EDGES, pd.DataFrame({"s": [1, 2], "d": [1, 1], "type": ["KNOWS", "KNOWS"], "eid": [6, 7], "w": [7, 8]})], ignore_index=True) + return { + "str-ids": (Frames(str_nodes, str_edges, "key", "s", "d", "eid"), ("dtype-str",), lambda v: f"n{v}"), + "null-ids": (Frames(null_nodes, null_edges, "key", "s", "d", "eid"), ("null-ids",), lambda v: v), + "dup-ids": (Frames(dup_nodes, EDGES, "key", "s", "d", "eid"), ("dup-ids",), lambda v: v), + "self-loop-cycle": (Frames(NODES, loop_edges, "key", "s", "d", "eid"), ("self-loop", "cycle"), lambda v: v), + "empty-edges": (Frames(NODES, EDGES.iloc[0:0], "key", "s", "d", "eid"), ("empty-edges",), lambda v: v), + "no-edge-id": (Frames(NODES, EDGES.drop(columns=["eid"]), "key", "s", "d", None), ("no-edge-id",), lambda v: v), + } + + +_VARIANT_ROW_TAGS: Dict[str, Dict[str, Tuple[str, ...]]] = { + "dup-ids": {"single node, predicate filter": ("#2034",)}, # node lookup keeps each duplicate row; the general path collapses them +} + +for _variant, (_frames, _tags, _key) in _frame_variants().items(): + register(f"routes.corpus.{_variant}", [(e.name, (lambda e=e, k=_key: e.shape(k)), e.tags) for e in CORPUS], _frames, + tags=_tags + ("variant",), row_tags=_VARIANT_ROW_TAGS.get(_variant)) diff --git a/graphistry/tests/compute/gfql/routes/test_route_harness.py b/graphistry/tests/compute/gfql/routes/test_route_harness.py index 7ab70fdb6a..ed54f1d04f 100644 --- a/graphistry/tests/compute/gfql/routes/test_route_harness.py +++ b/graphistry/tests/compute/gfql/routes/test_route_harness.py @@ -48,8 +48,11 @@ class Route(NamedTuple): (pchain, "_try_seeded_chain_polars"), True), ] -KNOWN: Dict[Tuple[str, str], str] = { # (route, tag) -> issue: strict xfail until it lands +KNOWN: Dict[Tuple[str, str], str] = { # (route, tag) -> issue: strict xfail until it lands (non-strict on frame variants, where a shape may coincide) ("polars-plain", "#2053"): "graphistry/pygraphistry#2053", + ("native-fast", "#2034"): "graphistry/pygraphistry#2034", + ("polars-plain", "#2034"): "graphistry/pygraphistry#2034", + ("polars-seeded", "#2034"): "graphistry/pygraphistry#2034", } @@ -102,8 +105,10 @@ def _canon(df) -> Tuple[Tuple[str, ...], List[Tuple]]: def _sig(res, frames) -> Tuple[List, List]: nn, ee = _topd(res._nodes), _topd(res._edges) - nodes = sorted(nn[frames.node].tolist()) if nn is not None else [] - edges = sorted(map(tuple, ee[[frames.src, frames.dst]].values.tolist())) if ee is not None and len(ee) else [] + def _na(v): + return None if v is None or v is pd.NA or v is pd.NaT or (isinstance(v, float) and math.isnan(v)) else v + nodes = sorted((_na(v) for v in nn[frames.node].tolist()), key=repr) if nn is not None else [] + edges = sorted((tuple(_na(v) for v in r) for r in ee[[frames.src, frames.dst]].values.tolist()), key=repr) if ee is not None and len(ee) else [] return nodes, edges @@ -134,7 +139,7 @@ def test_admitted_shape_is_served_and_matches_the_general_path(case: Case, reque _skip_unavailable(case.engine) for tag in case.shape.tags: if (case.route.name, tag) in KNOWN: - request.applymarker(pytest.mark.xfail(strict=True, reason=KNOWN[(case.route.name, tag)])) + request.applymarker(pytest.mark.xfail(strict="variant" not in case.shape.tags, reason=KNOWN[(case.route.name, tag)])) g = graph_for(case.shape, case.engine, indexed=case.route.indexed) calls = _served(case, monkeypatch) try: @@ -159,8 +164,8 @@ def test_every_route_serves_most_of_what_it_admits(monkeypatch): """A lane that declines most admitted shapes has a predicate that no longer describes it.""" per_route: Dict[str, List[int]] = {} for case in CASES: - if case.engine != ("polars" if case.route.name.startswith("polars") else "pandas"): - continue + if case.engine != ("polars" if case.route.name.startswith("polars") else "pandas") or "variant" in case.shape.tags: + continue # the serve ratio describes the base corpus; frame variants are expected to attenuate if case.engine == "polars": pytest.importorskip("polars") g = graph_for(case.shape, case.engine, indexed=case.route.indexed) diff --git a/reviews/2054/all-off.divergences b/reviews/2054/all-off.divergences deleted file mode 100644 index 5a1393b994..0000000000 --- a/reviews/2054/all-off.divergences +++ /dev/null @@ -1,57 +0,0 @@ -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_empty_match_counts_zero[polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_handles_degenerate_bindings[polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[base_distinct_domain-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[base_distinct_edges-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[empty_no_edges-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[empty_no_matching_nodes-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_distinct_domain-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_distinct_edges-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_end_only_filter-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_start_only_filter-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[string_ids-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_t6_assert_col_stats_helper_fails_loudly -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[0] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[1] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[2] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[3] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[4] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[5] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_gapped_node_space_builds_facts_and_stays_exact -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-cudf] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-pandas] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-polars] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-cudf] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-pandas] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-polars] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-cudf] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-pandas] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-polars] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-cudf] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-pandas] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_node_property_index_cost_gate_under_policy_use[selective-uses-index] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_node_property_index_cost_gate_under_policy_use[unselective-keeps-scan] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_node_property_index_duplicate_values_match_scan -FAILED graphistry/tests/compute/gfql/index/test_index.py::test_auto_engine_gfql_serves_polars_index_1767_cliff -FAILED graphistry/tests/compute/gfql/index/test_index.py::TestIndexAutoPreservesPolarsFrames::test_inversion_auto_index_auto_gfql_serves_polars_index -FAILED graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_alias_column_collision_2039.py::test_destination_alias_marker_replaces_the_colliding_column_like_pandas -FAILED graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py::test_chain_surface_keeps_node_attribute_dtypes[pandas] -FAILED graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py::test_has_label_narrowing_applies_on_reached_collision[polars] -FAILED graphistry/tests/compute/gfql/test_rewrite_param_discard.py::test_indexed_bypass_still_serves_a_bare_rows[cudf] -FAILED graphistry/tests/compute/gfql/test_rewrite_param_discard.py::test_indexed_bypass_still_serves_a_bare_rows[pandas] -FAILED graphistry/tests/compute/gfql/test_rewrite_param_discard.py::test_indexed_bypass_still_serves_a_bare_rows[polars] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[cudf] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[polars] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_returns_each_duplicate_id_row_once[cudf] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_returns_each_duplicate_id_row_once[pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[cudf] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[polars] -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_edges_empty_frame_not_none[pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_edges_empty_frame_not_none[polars] -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_pandas_int_bool_dtype_parity -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_polars_int_bool_dtype_parity diff --git a/reviews/2054/cypher-fast.divergences b/reviews/2054/cypher-fast.divergences deleted file mode 100644 index 540db7c953..0000000000 --- a/reviews/2054/cypher-fast.divergences +++ /dev/null @@ -1,46 +0,0 @@ -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_empty_match_counts_zero[polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_handles_degenerate_bindings[polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[base_distinct_domain-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[base_distinct_edges-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[empty_no_edges-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[empty_no_matching_nodes-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_distinct_domain-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_distinct_edges-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_end_only_filter-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[messy_start_only_filter-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_h3_fused_two_hop_count_matches_eager_twin_and_pandas[string_ids-polars] -FAILED graphistry/tests/compute/gfql/cypher/test_lowering.py::test_t6_assert_col_stats_helper_fails_loudly -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[0] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[1] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[2] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[3] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[4] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_differential_vs_the_scan_on_random_typed_graphs[5] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_gapped_node_space_builds_facts_and_stays_exact -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-cudf] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-pandas] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[2-8-polars] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-cudf] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-pandas] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[3-3-polars] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-cudf] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-pandas] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[5-1-polars] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-cudf] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-pandas] -FAILED graphistry/tests/compute/gfql/index/test_degree_consult.py::test_slice_is_exact_across_domain_shapes[7-2-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[polars] -FAILED graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py::test_has_label_narrowing_applies_on_reached_collision[polars] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[cudf] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_node_lookup_matches_independent_oracle[polars] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[cudf] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_seed_matching_several_nodes_projects_each_seed[polars] -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestResidentIndexSeededFastPath::test_uint64_int64_id_mix_declines_not_collapses -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_edges_empty_frame_not_none[pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_edges_empty_frame_not_none[polars] -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_pandas_int_bool_dtype_parity -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestSeededProjectionDtypeAndEdgesParity::test_polars_int_bool_dtype_parity diff --git a/reviews/2054/index-hop.divergences b/reviews/2054/index-hop.divergences deleted file mode 100644 index 5289985f9f..0000000000 --- a/reviews/2054/index-hop.divergences +++ /dev/null @@ -1,14 +0,0 @@ -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_destination_property_projection_dtype_parity[pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_node_property_index_duplicate_values_match_scan -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[canonical-distinct-order-limit-suffix-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[fixed-hop-reverse-composition-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is1-directed-projection-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-connected-two-hop-bag-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-optional-continuation-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[null-and-dtype-parity-pandas] -FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_empty_candidate_batch_on_device[polars-gpu] -FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[boolean-polars-gpu] -FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[float-polars-gpu] -FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[int64-polars-gpu] -FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[Int64-polars-gpu] -FAILED graphistry/tests/compute/gfql/index/test_index_gpu_edge_match.py::test_null_bearing_edge_predicate_matches_the_pandas_oracle_on_device[string-polars-gpu] diff --git a/reviews/2054/indexed-kernel.divergences b/reviews/2054/indexed-kernel.divergences deleted file mode 100644 index 8d590f0b2d..0000000000 --- a/reviews/2054/indexed-kernel.divergences +++ /dev/null @@ -1,27 +0,0 @@ -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[canonical-distinct-order-limit-suffix-cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[canonical-distinct-order-limit-suffix-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[canonical-distinct-order-limit-suffix-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[fixed-hop-reverse-composition-cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[fixed-hop-reverse-composition-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[fixed-hop-reverse-composition-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is1-directed-projection-cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is1-directed-projection-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is1-directed-projection-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is3-undirected-multiplicity-cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is3-undirected-multiplicity-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is3-undirected-multiplicity-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-connected-two-hop-bag-cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-connected-two-hop-bag-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-connected-two-hop-bag-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-optional-continuation-cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-optional-continuation-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[is7-optional-continuation-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[null-and-dtype-parity-cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[null-and-dtype-parity-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[null-and-dtype-parity-polars] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[official-no-match-stratum-cudf] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[official-no-match-stratum-pandas] -FAILED graphistry/tests/compute/gfql/index/test_indexed_bindings.py::test_standard_derived_connected_parity[official-no-match-stratum-polars] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_two_alias_projection_parity[MATCH (m:Message {id: 305})-[:HAS_CREATOR]->(p:Person) RETURN m.id AS a, m.id AS b, p.flag AS c-repeated + bool-indexed-pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_two_alias_projection_parity[MATCH (m:Message {id: 305})-[:HAS_CREATOR]->(p:Person) RETURN m.score AS ms, p.score AS ps, m.flag AS mf, p.flag AS pf-int + bool from both aliases-indexed-pandas] -FAILED graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py::test_two_alias_projection_parity[MATCH (m:Message {id: 305})-[r:HAS_CREATOR]->(p:Person) RETURN m.id, m.score, m.flag, m.firstName, r.w, r.eflag, r.type, p.id, p.firstName, p.age, p.score, p.flag-twelve properties across three aliases-indexed-pandas] diff --git a/reviews/2054/native-fast.divergences b/reviews/2054/native-fast.divergences deleted file mode 100644 index f673c6aab8..0000000000 --- a/reviews/2054/native-fast.divergences +++ /dev/null @@ -1,25 +0,0 @@ -FAILED graphistry/tests/compute/gfql/lazy/engine/polars/test_chain_alias_column_collision_2039.py::test_destination_alias_marker_replaces_the_colliding_column_like_pandas -FAILED graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py::test_chain_surface_keeps_node_attribute_dtypes[pandas] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_duplicate_node_rows_are_answered_once_each_on_the_native_lookup[cudf] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_duplicate_node_rows_are_answered_once_each_on_the_native_lookup[pandas] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_hop_aliases_overwrite_nonfinal_properties_like_full_path[cudf] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_hop_aliases_overwrite_nonfinal_properties_like_full_path[pandas] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_layout_matches_the_full_path[False-cudf] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_layout_matches_the_full_path[False-pandas] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_layout_matches_the_full_path[True-cudf] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_layout_matches_the_full_path[True-pandas] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_overwrites_colliding_property_like_full_path[False-cudf] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_overwrites_colliding_property_like_full_path[False-pandas] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_overwrites_colliding_property_like_full_path[True-cudf] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_named_single_node_alias_overwrites_colliding_property_like_full_path[True-pandas] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_stale_indexes_keep_parity_and_are_not_used[seeded typed hop, all named-cudf] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_stale_indexes_keep_parity_and_are_not_used[seeded typed hop, all named-pandas] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_stale_indexes_keep_parity_and_are_not_used[seeded typed hop + rows + select-cudf] -FAILED graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py::test_stale_indexes_keep_parity_and_are_not_used[seeded typed hop + rows + select-pandas] -FAILED graphistry/tests/compute/gfql/test_polars_lane_completeness.py::test_every_polars_mentioning_test_module_is_in_the_lane_or_justified -FAILED graphistry/tests/compute/gfql/test_polars_lane_completeness.py::test_no_module_level_polars_gate_outside_the_lane -FAILED graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py::TestResidentIndexSeededFastPath::test_native_chain_hop_indexed_parity_forward_and_reverse -FAILED graphistry/tests/compute/test_chain.py::test_fast_path_named_full_frame_value_parity[all_forward-pandas] -FAILED graphistry/tests/compute/test_chain.py::test_fast_path_named_full_frame_value_parity[all_reverse-pandas] -FAILED graphistry/tests/compute/test_chain.py::test_fast_path_named_full_frame_value_parity[edge_match-pandas] -FAILED graphistry/tests/compute/test_chain.py::test_fast_path_named_full_frame_value_parity[seed_filtered-pandas] diff --git a/reviews/2054/polars-plain.divergences b/reviews/2054/polars-plain.divergences deleted file mode 100644 index 175c373fee..0000000000 --- a/reviews/2054/polars-plain.divergences +++ /dev/null @@ -1 +0,0 @@ -FAILED graphistry/tests/compute/gfql/routes/test_route_harness.py::test_every_route_serves_most_of_what_it_admits diff --git a/reviews/2054/polars-seeded.divergences b/reviews/2054/polars-seeded.divergences deleted file mode 100644 index 11cafd867e..0000000000 --- a/reviews/2054/polars-seeded.divergences +++ /dev/null @@ -1,12 +0,0 @@ -FAILED graphistry/tests/compute/gfql/routes/test_route_harness.py::test_every_route_serves_most_of_what_it_admits -FAILED graphistry/tests/compute/gfql/test_native_seed_lane_explain.py::test_seeded_typed_hop_served_by_the_resident_indexes_is_explained[polars] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed0-True-False] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed0-True-True] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed1-True-False] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed1-True-True] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed2-True-False] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed2-True-True] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed3-True-False] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_named_typed_hop_preserves_full_path_tables[seed3-True-True] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_property_seed_uses_resident_index[False] -FAILED graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py::test_native_seeded_hop_is_served_from_the_index_and_traced diff --git a/reviews/2054/routes-off-ledger-3213f7d96.txt b/reviews/2054/routes-off-ledger-3213f7d96.txt deleted file mode 100644 index a5ad7e3ef4..0000000000 --- a/reviews/2054/routes-off-ledger-3213f7d96.txt +++ /dev/null @@ -1,7 +0,0 @@ -native-fast: 25 ids -polars-seeded: 12 ids -polars-plain: 1 ids -index-hop: 14 ids -indexed-kernel: 27 ids -cypher-fast: 46 ids -all-off: 57 ids