diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ae6a3a4e7..b554870eff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -242,11 +242,19 @@ jobs: '^graphistry/compute/gfql/' \ '^graphistry/compute/gfql_validate\.py$' \ '^graphistry/compute/gfql_unified\.py$' \ + '^graphistry/compute/gfql_fast_paths\.py$' \ + '^graphistry/compute/chain.*\.py$' \ + '^graphistry/compute/hop\.py$' \ + '^graphistry/compute/filter_by_dict\.py$' \ + '^graphistry/compute/ast\.py$' \ + '^graphistry/compute/predicates/' \ '^graphistry/models/gfql/' \ '^graphistry/Plottable\.py$' \ '^graphistry/tests/benchmarks/gfql/' \ '^graphistry/tests/compute/gfql/' \ '^graphistry/tests/compute/test_gfql.*\.py$' \ + '^graphistry/tests/compute/test_chain.*\.py$' \ + '^graphistry/tests/compute/test_hop.*\.py$' \ '^graphistry/tests/test_gfql_.*\.py$' \ '^tests/gfql/' @@ -1589,6 +1597,63 @@ jobs: source pygraphistry/bin/activate ./bin/test-graphviz.sh + gfql-routes-off: + # Non-blocking ledger: replays the GFQL suites with one hot path declined per cell + # (GFQL_ROUTES_OFF); engagement pins are skipped via the route_engaged marker, so the + # uploaded lists hold route-vs-general result divergences only. + needs: [changes, test-gfql-core, generate-lockfiles] + if: ${{ needs.changes.outputs.gfql == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} + runs-on: ubuntu-latest + continue-on-error: true + timeout-minutes: 45 + + strategy: + fail-fast: false + matrix: + mode: [native-fast, polars-seeded, polars-plain, index-hop, indexed-kernel, cypher-fast, all-off] + + steps: + + - name: Checkout repo + uses: actions/checkout@v4 + with: + lfs: true + persist-credentials: false + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Download lockfiles + uses: actions/download-artifact@v4 + with: + name: lockfiles + path: requirements + + - name: Install Python dependencies + run: | + python -m venv pygraphistry + source pygraphistry/bin/activate + python -m pip install --upgrade pip uv + uv pip install --require-hashes -r requirements/test-polars-py3.12.lock + uv pip install -e . --no-deps + + - name: Routes-off replay (${{ matrix.mode }}) + run: | + source pygraphistry/bin/activate + MODES=${{ matrix.mode }} ./bin/test-routes-off.sh + echo "## routes-off: ${{ matrix.mode }}" >> "$GITHUB_STEP_SUMMARY" + sed 's/^/- /' build/routes-off/${{ matrix.mode }}.divergences >> "$GITHUB_STEP_SUMMARY" + + - name: Upload ledger + if: always() + uses: actions/upload-artifact@v4 + with: + name: routes-off-${{ matrix.mode }} + path: build/routes-off/ + if-no-files-found: ignore + test-polars: needs: [changes, test-minimal-python, test-gfql-core, generate-lockfiles] if: ${{ ((needs.changes.outputs.python == 'true' && needs.changes.outputs.narrow_python_only != 'true') || needs.changes.outputs.gfql == 'true' || needs.changes.outputs.pandas_compat == 'true' || needs.changes.outputs.core == 'true' || needs.changes.outputs.infra == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') && !(needs.changes.outputs.docs_only_latest == 'true' && (github.event_name == 'push' || github.event_name == 'pull_request')) }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f8bcaaba8c..406537ad89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Infrastructure +- **CI: the gfql change filter now includes the chain engine** (`compute/chain*.py`, `hop.py`, `gfql_fast_paths.py`, `filter_by_dict.py`, `ast.py`, `predicates/`, and the chain/hop test files), so tck-gfql, the Cypher-frontend gates and the gfql benchmark lane run on a change to the chain engine; they were skipped on #2055. - **CI: `test-docs` runs on docs-only pull requests (#2018)**: the job needed `python-lint-types`, which a docs-only change skips, and GitHub skips a job whose prerequisite was skipped. The gate now accepts skipped prerequisites and refuses only failed or cancelled ones, so documentation changes are built and tested before merge. ### Tests @@ -40,6 +41,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Changed +* GFQL: the chain specializations move into `graphistry/compute/chain_specializations/{admission,hotpaths}.py` (pandas/cuDF single-node lane, seeded typed single hop, seeded typed RETURN-destination) and `graphistry/compute/gfql/lazy/engine/polars/chain_specializations/{admission,hotpaths}.py` (polars plain single-hop branches, seeded lane, RETURN-destination), each lane next to the admission predicate the dispatcher calls (`native_fast_path_admits`, `polars_plain_single_hop_admits`, `polars_seeded_lane_admits`); `chain.py` and the polars chain only dispatch, `chain_fast_paths.py` keeps the shared seed/index helpers. No route admits or declines anything it did not before. Tests mirror the new paths and filter one shared shape corpus per route with the route's own gate; `GFQL_ROUTES_OFF=` (test conftest) makes named hot paths decline so every existing test replays through the other routes, and `bin/test-routes-off.sh` reports the per-route divergences. * GFQL: the wavefront seed-rediscovery rule moved out of `hop.py` into `graphistry/compute/gfql/seed_rediscovery.py` (pandas/cuDF) and `graphistry/compute/gfql/lazy/engine/polars/seed_rediscovery.py` (polars); `undirected_rediscovered_seed_ids` (an internal helper) is gone. ## [0.59.0 - 2026-08-31] diff --git a/bin/test-polars.sh b/bin/test-polars.sh index 7c237e97dc..dc934a7c2c 100755 --- a/bin/test-polars.sh +++ b/bin/test-polars.sh @@ -87,6 +87,8 @@ POLARS_TEST_FILES=( graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py + graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/test_polars_admission.py + graphistry/tests/compute/chain_specializations/test_native_admission.py graphistry/tests/compute/gfql/test_undirected_pairs_2026.py graphistry/tests/compute/gfql/test_native_seed_lane_explain.py # #1882/#1913-f4/#1879 crash-family pins: the polars params (filter helpers on polars @@ -123,6 +125,7 @@ POLARS_TEST_FILES=( graphistry/tests/compute/gfql/cypher/test_variable_column_collision.py graphistry/tests/compute/test_chain_alias_column_collision.py graphistry/tests/compute/test_gfql_op_list_hides_internal_columns.py + graphistry/tests/compute/gfql/routes/test_route_harness.py graphistry/tests/compute/gfql/test_engine_polars_semi_key_dedup.py graphistry/tests/compute/gfql/test_engine_polars_call_modality.py graphistry/tests/compute/gfql/test_engine_polars_gpu.py diff --git a/bin/test-routes-off.sh b/bin/test-routes-off.sh new file mode 100755 index 0000000000..9cf1a3ef4f --- /dev/null +++ b/bin/test-routes-off.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Replay the GFQL suites with each hot path declined (GFQL_ROUTES_OFF, see graphistry/tests/conftest.py). +# Engagement pins carry the route_engaged marker and are skipped, so every remaining failure is a +# route-vs-general result divergence. Non-blocking ledger: always exits 0; per-mode logs + id lists in $OUT. +set -uo pipefail +cd "$(dirname "$0")/.." +MODES=${MODES:-native-fast polars-seeded polars-plain index-hop indexed-kernel cypher-fast all-off} +SUITES=${SUITES:-graphistry/tests/compute/test_chain.py graphistry/tests/compute/test_hop.py graphistry/tests/compute/test_gfql.py graphistry/tests/compute/gfql} +OUT=${OUT:-build/routes-off} +mkdir -p "$OUT" +for mode in $MODES; do + if [ "$mode" = all-off ]; then routes=native-fast,polars-seeded,polars-plain,index-hop,indexed-kernel,cypher-fast; else routes=$mode; fi + GFQL_ROUTES_OFF=$routes python -m pytest $SUITES -q -p no:cacheprovider -o addopts="" -rfE > "$OUT/$mode.log" 2>&1 + grep -E "^(FAILED|ERROR) " "$OUT/$mode.log" | sed 's/ - .*//' | sort -u > "$OUT/$mode.divergences" + echo "$mode: $(wc -l < "$OUT/$mode.divergences") divergence id(s); $(tail -1 "$OUT/$mode.log")" +done +exit 0 diff --git a/graphistry/compute/chain.py b/graphistry/compute/chain.py index 6dc9b27a88..f58e813fe8 100644 --- a/graphistry/compute/chain.py +++ b/graphistry/compute/chain.py @@ -11,11 +11,7 @@ from .ast import ASTObject, ASTNode, ASTEdge, ASTCall, Direction, from_json as ASTObject_from_json, serialize_binding_ops from .typing import DataFrameT, SeriesT from .util import generate_safe_column_name -from .chain_fast_paths import ( - _seeded_typed_hop_pandas_cudf, - _single_node_rows_via_index_or_filter, - _tag_fast_path_aliases, -) +from .chain_specializations.hotpaths import _try_chain_fast_path from graphistry.compute.validate.validate_schema import validate_chain_schema, validate_graph_shape from graphistry.compute.gfql.strictness import StrictInput from graphistry.compute.gfql.same_path_types import ( @@ -840,193 +836,6 @@ def _step_with_source_edge_columns(g: Plottable, g_step: Plottable, op: ASTObjec return g_step.edges(edges[edges[edge_id].isin(step_edges[edge_id])]) -def _try_chain_fast_path( - g_in: Plottable, - ops: List[ASTObject], - engine_concrete: Engine, - start_nodes: Optional[DataFrameT] = None, -) -> Optional[Plottable]: - """Degenerate-shape fast path (pandas/cuDF): node-only ``MATCH (n)`` or a plain - single-hop ``MATCH (a)-[e]->(b)`` skip the forward/backward/combine BFS machinery. - Returns the result Plottable, or ``None`` to fall through to the full path. - - Same node/edge sets + VALUES as the full machinery (trackA_golden + hop/chain - suites); the 1-hop additionally preserves int node dtypes (the full path upcasts - int→float via merge — the merge is the artifact, int is the Cypher-conformant type). - Gated to unqueried nodes + a plain single-hop edge; NAMED ops are served (the alias - flags are reconstructed by `_tag_fast_path_aliases`) except when undirected or when - the same alias is reused. filtered-undirected and seeded chains fall through. - polars/dask/spark also fall through (own fast path / lazy semantics).""" - from graphistry.compute.filter_by_dict import filter_by_dict - - if engine_concrete not in (Engine.PANDAS, Engine.CUDF): - return None - if start_nodes is not None: - return None # seeded chains use the full path (fast path has no seed) - engine_abs = EngineAbstract(engine_concrete.value) - - def _materialize_fast_path_graph() -> Plottable: - from graphistry.compute.ComputeMixin import _coerce_input_formats # lazy — avoids circular import - g = g_in.materialize_nodes(engine=EngineAbstract(engine_concrete.value)) - return _coerce_input_formats(g, engine_concrete) - - if len(ops) == 1: - n0 = ops[0] - if not (isinstance(n0, ASTNode) and n0.query is None): - return None - g = _materialize_fast_path_graph() - if g._nodes is None: - return None - nodes = _single_node_rows_via_index_or_filter(g, n0, engine_abs) - if n0._name is not None: - alias_was_column = n0._name in nodes.columns - if alias_was_column: - nodes = nodes.drop(columns=[n0._name]) - nodes = nodes.assign(**{n0._name: True}) - other_columns = [c for c in nodes.columns if c != n0._name] - if g._node in other_columns: - other_columns = [g._node, *[c for c in other_columns if c != g._node]] - if alias_was_column: - nodes = nodes[[*other_columns, n0._name]] - else: - nodes = nodes[[*other_columns[:1], n0._name, *other_columns[1:]]] - nodes = nodes.reset_index(drop=True) - edges = g._edges.iloc[0:0] if g._edges is not None else None - return g.nodes(nodes).edges(edges) if edges is not None else g.nodes(nodes) - - if len(ops) != 3: - return None - n0, e1, n2 = ops - # Aliases are a PROJECTION concern, not a traversal one: capture them, serve the - # traversal on the fast path, and tag the result (_tag_fast_path_aliases). Rejecting - # them here sent a NAMED `g.gfql([n(name=..), e(..), n(name=..)])` to the full - # two-pass BFS purely because the ops carried names — measured ~25.2 ms before vs - # ~2.3 ms after (medians of 5 paired runs), on a 200-node graph where data work is ~0. - # SCOPE, measured: this does NOT reach the Cypher `MATCH ... RETURN` surface for the - # benchmark shapes. Those are served earlier by `gfql_fast_paths.py` and never consult - # this function at all, so do not attribute a Cypher-surface win to this gate. - alias_n0, alias_e1, alias_n2 = n0._name, e1._name, n2._name - _named = [a for a in (alias_n0, alias_e1, alias_n2) if a is not None] - if len(_named) != len(set(_named)): - # Duplicate alias reuse is an E201 error, and `combine_steps` is what raises it. - # Serving these here would BYPASS that check and silently succeed — decline so the - # full path still errors. (Caught by test_polars_duplicate_alias_declines_like_pandas.) - return None - if not (isinstance(n0, ASTNode) and n0.query is None): - return None - if not (isinstance(n2, ASTNode) and n2.query is None): - return None - if not (isinstance(e1, ASTEdge) and e1.is_simple_single_hop() - and e1.source_node_match is None - and e1.destination_node_match is None - and e1.source_node_query is None and e1.destination_node_query is None - and e1.edge_query is None and not e1.include_zero_hop_seed - and not e1.prune_to_endpoints): # prune keeps only the arrival side -> full path - return None - # #1755 lever-3: a typed edge (edge_match, e.g. -[:HAS_CREATOR]->) is a plain - # equality/predicate filter on the edge frame — apply it in the fast-path body - # below rather than falling through to the full two-pass machinery. source/dest - # node match + edge_query (richer predicates) still bail above. - direction = e1.direction - if direction == "undirected" and (alias_n0 is not None or alias_n2 is not None): - # An undirected edge makes a node reachable as EITHER endpoint, so "which alias - # does this node carry" is not derivable from the endpoint columns the way it is - # for a directed hop. Decline to the full path rather than guess. - return None - unconstrained = not n0.filter_dict and not n2.filter_dict - if not unconstrained and direction == "undirected": - return None # filtered-undirected (OR of both directions) -> full path - g = _materialize_fast_path_graph() - if g._nodes is None or g._edges is None: - return None - src, dst, node = g._source, g._destination, g._node - if src is None or dst is None or node is None: - return None # no edge/node bindings -> can't fast-path; full path handles it - if alias_n0 == node or alias_n2 == node: - # A node alias EQUAL TO THE NODE-ID BINDING would make `_tag_fast_path_aliases` - # overwrite the id column with the bool flag (destroying the ids) while the full - # path raises on the same query ("The column label '' is not unique") — - # a wrong-serve found by adversarial parity testing. Decline; never serve. - return None - if alias_e1 is not None and direction in ("forward", "reverse") \ - and alias_e1 == (src if direction == "forward" else dst): - # An edge alias EQUAL TO THE HOP'S FROM-SIDE BINDING (forward+src / reverse+dst) - # made the two lanes return DIFFERENT node sets: the full path's flag overwrite - # corrupts its own node reduction, the fast path tags after reducing. TO-side - # collisions keep parity (pinned in tests) and stay served. Decline; never serve. - return None - concat = df_concat(engine_concrete) - if unconstrained: - # No node filter to reduce by: validate BOTH endpoints against the full - # node table (the full path drops dangling edges via its joins). dropna so - # a NaN node id can't validate a NaN endpoint — .isin treats NaN as - # matchable but the BFS joins never match NaN<->NaN. - node_ids = g._nodes[node].dropna() - edges = g._edges[g._edges[src].isin(node_ids) & g._edges[dst].isin(node_ids)] - if e1.edge_match: - # typed edge (e.g. -[:HAS_CREATOR]->) — same edge-frame filter the full - # hop applies, so the result set is identical. - edges = filter_by_dict(edges, e1.edge_match, engine_abs) - else: - # #1755 lever-3 seed-first: a seeded 1-hop must be O(result), not O(E). - # Reduce edges by the selective node filter(s) BEFORE the typed-edge scan - # and endpoint validation, so the expensive object/isin passes run on the - # tiny frontier, not all edges. The from-side ids come from the node table - # (so that endpoint is validated); the node gather below validates the to - # side and drops any edge dangling off the node table. - # pandas + cuDF: a scalar-filtered seeded typed hop collapses to a few - # DataFrame filters (sub-ms); falls back to the general branch below for - # predicates / undirected / missing columns (and non-pandas/cuDF engines). - if engine_concrete in (Engine.PANDAS, Engine.CUDF): - _fast_res = _seeded_typed_hop_pandas_cudf(g, n0, n2, e1, src, dst, node, direction) - if _fast_res is not None: - return _tag_fast_path_aliases( - _fast_res, alias_n0, alias_e1, alias_n2, src, dst, node, direction) - from_col, to_col = (src, dst) if direction == "forward" else (dst, src) - edges = g._edges - if n0.filter_dict: - from_ids = filter_by_dict(g._nodes, n0.filter_dict, engine_abs)[node] - edges = edges[edges[from_col].isin(from_ids)] - if e1.edge_match: - edges = filter_by_dict(edges, e1.edge_match, engine_abs) - if n2.filter_dict: - # Apply the destination filter to the SMALL set of gathered dst nodes, - # not the full node table — an O(N) object/type scan on all nodes is - # exactly the tax we're removing. Gather the frontier's dst nodes - # (small isin key), filter those, then drop edges to the losers. - to_present = edges[to_col].dropna().unique() - to_nodes = filter_by_dict( - g._nodes[g._nodes[node].isin(to_present)], n2.filter_dict, engine_abs) - edges = edges[edges[to_col].isin(to_nodes[node])] - # Validate endpoints + build result nodes on the reduced edge set (small - # isin key -> small hashtable; no O(E)-values scan). Engine-agnostic - # (pandas + cuDF): gather candidate endpoint nodes, drop edges dangling off - # the node table, then keep only nodes still referenced by a surviving edge. - ep = concat([ - edges[[src]].rename(columns={src: node}), - edges[[dst]].rename(columns={dst: node}), - ]).drop_duplicates() - cand = g._nodes[g._nodes[node].isin(ep[node])].drop_duplicates(subset=[node]) - valid = cand[node].dropna() - edges = edges[edges[src].isin(valid) & edges[dst].isin(valid)] - final = concat([ - edges[[src]].rename(columns={src: node}), - edges[[dst]].rename(columns={dst: node}), - ]).drop_duplicates() - nodes = cand[cand[node].isin(final[node])] - return _tag_fast_path_aliases( - g.nodes(nodes).edges(edges), alias_n0, alias_e1, alias_n2, src, dst, node, direction) - endpoints = concat([ - edges[[src]].rename(columns={src: node}), - edges[[dst]].rename(columns={dst: node}), - ]).drop_duplicates() - nodes = g._nodes[g._nodes[node].isin(endpoints[node])] - # match the full path's merge, which collapses duplicate node-id rows - nodes = nodes.drop_duplicates(subset=[node]) - return _tag_fast_path_aliases( - g.nodes(nodes).edges(edges), alias_n0, alias_e1, alias_n2, src, dst, node, direction) - - def reject_alias_named_like_binding( g: Plottable, chain_obj: "Chain", *, include_edge_endpoint_aliases: bool = False ) -> None: diff --git a/graphistry/compute/chain_fast_paths.py b/graphistry/compute/chain_fast_paths.py index 3e26b52fc5..66e0c75704 100644 --- a/graphistry/compute/chain_fast_paths.py +++ b/graphistry/compute/chain_fast_paths.py @@ -1,24 +1,17 @@ -"""Seeded typed-hop fast-path specializations for the chain executor. - -Extracted verbatim from chain.py (#1755) to keep that orchestrator readable: the seeded -typed 1-hop pandas/cuDF reduction and the seeded typed RETURN-destination pandas/cuDF and -polars reductions. Pure code move, no behavior change. chain.py imports from here (one -direction); this module imports only leaf modules (no back-edge into chain.py). - -New fast-path helpers belong HERE, not in chain.py — that is the direction this module was -created to establish and it is why `_tag_fast_path_aliases` lives alongside the seeded -reductions rather than next to `_try_chain_fast_path`. -""" +"""Seed-resolution and resident-index helpers shared by the chain specializations +(``chain_specializations/``, ``gfql/lazy/engine/polars/chain_specializations/``) and the Cypher +lanes in ``gfql_fast_paths.py``. This module imports only leaf modules (no back-edge into +``chain.py`` or the specialization packages).""" # ruff: noqa: E501 from typing import Any, Dict, Literal, Optional, Sequence, Tuple, TYPE_CHECKING, Union, cast from graphistry.Plottable import Plottable -from .ast import ASTObject, ASTNode, ASTEdge, Direction +from .ast import Direction from .typing import ArrayLike, ArrayNamespace, DataFrameT, SeriesT if TYPE_CHECKING: - from graphistry.Engine import Engine, EngineAbstract + from graphistry.Engine import Engine from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex @@ -257,6 +250,7 @@ def _seed_rows_via_prop_index_frame( SeedRowsHow = Literal["node_id_index", "property_index", "scan"] +SeededReturn = Tuple[DataFrameT, DataFrameT, DataFrameT, bool] def _seed_node_rows( @@ -288,27 +282,6 @@ def _seed_node_rows( return _filter_frame(seed, filter_dict if filter_dict is not None else n0f, engine), how -def _single_node_rows_via_index_or_filter( - g: Plottable, n0: ASTNode, engine_abs: "EngineAbstract", -) -> DataFrameT: - """Resolve a single node op through a resident index or the canonical filter.""" - from .filter_by_dict import filter_by_dict - nodes_df = g._nodes - assert nodes_df is not None - if not n0.filter_dict: - return nodes_df - node = g._node - n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) if node is not None else None - if node is not None and n0f: - nid_ctx = _resident_node_id_index(g, nodes_df, node) - rows, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) - if how != "scan": - _record_native_seed_lane(nodes_df, seam="native_seed_lookup", reason=how, hop_count=0, - public_seed_scan=node not in n0.filter_dict) - return rows - return filter_by_dict(nodes_df, n0.filter_dict, engine_abs) - - def _record_native_seed_lane( nodes_df: DataFrameT, *, seam: str, reason: str, hop_count: int, public_seed_scan: bool, ) -> None: @@ -337,318 +310,3 @@ def _index_edge_rows( return None rows, _ = lookup_edge_rows(adj, arr, xp) return take_rows(edges_df, xp.sort(rows) if preserve_input_order else rows, engine) - - -def _indexed_kernel_admits( - seed_nodes: DataFrameT, gathered_edges: Optional[DataFrameT], n0f: Dict[str, object], - node: str, how: SeedRowsHow, ctx: Tuple["NodeIdIndex", "AdjacencyIndex", ArrayNamespace, "Engine"], - n_nodes: int, n_edges: int, -) -> bool: - """Whether the indexed connected-bindings kernel would have served this seeded 1-hop: - its seed admission (binding-column integer seed, property-index hit, or a scan on a - graph with fewer nodes than edges) and its frontier and gather cost gates.""" - from numbers import Integral - from graphistry.compute.gfql.index.cost import cost_gate_frac - _, adj, _, engine = ctx - seed_val = n0f.get(node) - seeded_on_binding = isinstance(seed_val, Integral) and not isinstance(seed_val, bool) - if not (seeded_on_binding or how == "property_index" or n_nodes < n_edges): - return False - frac = cost_gate_frac(engine) - n_frontier = int(seed_nodes[node].nunique()) if not hasattr(seed_nodes, "get_column") \ - else int(seed_nodes.get_column(node).n_unique()) - if n_frontier >= frac * adj.n_keys: - return False - return gathered_edges is not None and len(gathered_edges) < frac * n_edges - - -def _seeded_typed_hop_pandas_cudf( - g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, - src: str, dst: str, node: str, direction: Direction, -) -> Optional[Plottable]: - """#1755 lever-3: engine-generic (pandas + cuDF) fast path for a scalar-filtered - seeded typed 1-hop. Value-identical to the general seeded branch for the covered - shape (all node/edge filters are plain scalars, directed) — same rows, columns, - and dtypes; row order and RangeIndex may differ — collapsing it into a - few DataFrame filters so a seeded lookup lands sub-ms. Uses only the shared - pandas/cuDF DataFrame API (no numpy array drops) so the same body runs on both - engines. Returns None to fall back for anything it does not cover (predicates, - undirected, missing columns) — the caller then runs the general branch.""" - if direction == "undirected": - return None - - nodes_df, edges_df = g._nodes, g._edges - if nodes_df is None or edges_df is None: - return None - n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) - n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) - ef = _seeded_scalar_filters(e1.edge_match, edges_df) - if n0f is None or n2f is None or ef is None: - return None - from_col, to_col = (src, dst) if direction == "forward" else (dst, src) - - # from-side seed FIRST: reduce edges to the seed's out-edges before the - # edge_match compare, so the type filter runs on the tiny frontier rather than - # all edges — this is what makes a seeded lookup sub-ms. The id filter goes - # first (int, unique -> ~1 row in one pass) so any remaining object filters - # (label__X->type) run on that tiny survivor frame, not the whole node table. - # Resident-index acceleration (#1658 x #1755): when node-id + directional - # adjacency indexes are valid for these exact frames, the O(N) seed scan, the - # O(E) frontier isin, and the O(N) candidate gather become positional lookups. - # Any decline (no index, stale fingerprint, unsafe id cast) falls back to the - # scan body below — identical results either way, only speed differs. - # Decoupled index use: the seed row lookup uses the node-id index ONLY when - # the seed filter includes the binding column, but the frontier edge gather - # (CSR adjacency) and candidate gathers (node-id index) engage regardless of - # HOW the seed rows were found — their inputs are binding-column values, which - # are the index key domain. (LDBC/user pattern: seed on the `id` PROPERTY - # while the graph binds a different key column — previously disqualified the - # whole index path.) - ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) if n0f else None - seed_nodes = edges = cand = None - if ctx is not None: - nid, adj, xp, idx_engine = ctx - seed_nodes, _ = _seed_node_rows(g, nodes_df, n0f, node, (nid, xp, idx_engine), n0.filter_dict) - edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) - if edges is not None: - if ef: - for k, v in ef.items(): - edges = edges[edges[k] == v] - if 'cudf' in str(type(edges).__module__): - import cudf as _cd # type: ignore - endpoint_ids = _cd.concat([edges[src], edges[dst]]) - else: - import pandas as _pd - endpoint_ids = _pd.concat([edges[src], edges[dst]]) - cand = _index_node_rows(nid, endpoint_ids, xp, idx_engine, nodes_df) - served_via_index = cand is not None - if cand is None: - if n0f: - seed_nodes = nodes_df - for k, v in sorted(n0f.items(), key=lambda kv: 0 if kv[0] == node else 1): - seed_nodes = seed_nodes[seed_nodes[k] == v] - edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())] - else: - edges = edges_df - if ef: # typed edge (edge_match) — now on the reduced frontier - for k, v in ef.items(): - edges = edges[edges[k] == v] - - # Gather candidate endpoint nodes (both endpoints of surviving edges), then run - # the dest filter, dangling-edge drop and final-node selection on the small - # candidate/edge frames. Selecting from nodes_df keeps only real nodes, so the - # endpoint-in-nodes check subsumes the old NaN-endpoint guard. Membership sets - # are dropna()'d: pandas .isin matches NaN<->NaN, but the general branch's BFS - # joins never join on null keys, so a null id/endpoint must not link. - cand = nodes_df[ - nodes_df[node].isin(edges[src].dropna()) | nodes_df[node].isin(edges[dst].dropna()) - ].drop_duplicates(subset=[node]) - assert edges is not None and cand is not None # both branches above assign - if served_via_index: - _record_native_seed_lane(nodes_df, seam="native_seeded_hop", reason="served", hop_count=1, - public_seed_scan=node not in n0f) - if n2f: # destination-node filter (to-side) - n2_cand = cand - for k, v in n2f.items(): - n2_cand = n2_cand[n2_cand[k] == v] - n2_ok = n2_cand[node] - else: - n2_ok = cand[node] - to_vals = edges[to_col] - keep = edges[src].isin(cand[node].dropna()) & edges[dst].isin(cand[node].dropna()) & to_vals.isin(n2_ok.dropna()) - edges = edges[keep] - cand = cand[cand[node].isin(edges[src]) | cand[node].isin(edges[dst])] - return g.nodes(cand).edges(edges) - - -SeededReturn = Tuple[DataFrameT, DataFrameT, DataFrameT, bool] - - -def _seeded_typed_return_dst_pandas_cudf( - g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, - src: str, dst: str, node: str, direction: Direction, -) -> Optional[SeededReturn]: - """#1755 cypher RETURN-alias fast path: like _seeded_typed_hop_pandas_cudf but - returns ONLY the destination (RETURN-alias) node rows + surviving edges — no - seed-node gather, no Plottable round-trip — so the seeded cypher projection - lands sub-ms. Engine-generic (pandas + cuDF): only the shared DataFrame API, - no numpy array drops. Returns ``(dst_node_rows, edges)`` or None to fall back.""" - if direction == "undirected": - return None - nodes_df, edges_df = g._nodes, g._edges - if nodes_df is None or edges_df is None: - return None - n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) - n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) - ef = _seeded_scalar_filters(e1.edge_match, edges_df) - if n0f is None or n2f is None or ef is None or not n0f: - return None - from_col, to_col = (src, dst) if direction == "forward" else (dst, src) - # id-first seed reduction: filter by the id column first (int/unique -> ~1 row) - # so any remaining object filters (label__X->type) run on the tiny survivor - # frame, never materializing an object column over the whole node table. - # Membership sets are dropna()'d: pandas .isin matches NaN<->NaN, but the full - # pipeline's joins never join on null keys, so a null id/endpoint must not link. - ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) - nid_ctx = (ctx[0], ctx[2], ctx[3]) if ctx is not None else _resident_node_id_index(g, nodes_df, node) - seed_nodes, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) - edges = dstn = None - kernel_admits = False - if ctx is not None: - nid, adj, xp, idx_engine = ctx - edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) - kernel_admits = _indexed_kernel_admits( - seed_nodes, edges, n0f, node, how, ctx, len(nodes_df), len(edges_df)) - if edges is not None: - if ef: - for k, v in ef.items(): - edges = edges[edges[k] == v] - dstn = _index_node_rows(nid, edges[to_col], xp, idx_engine, nodes_df) - if dstn is None: - edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())] - if ef: - for k, v in ef.items(): - edges = edges[edges[k] == v] - # destination nodes = real nodes that are edge to-endpoints, then the dest - # filter, dangling-edge drop and dedup on the small dst/edge frames. - dstn = nodes_df[nodes_df[node].isin(edges[to_col].dropna())] - assert edges is not None and dstn is not None # both branches above assign - if n2f: - for k, v in n2f.items(): - dstn = dstn[dstn[k] == v] - edges = edges[edges[to_col].isin(dstn[node].dropna())] - dstn = dstn[dstn[node].isin(edges[to_col].dropna())].drop_duplicates(subset=[node]) - return dstn, edges, seed_nodes, kernel_admits - - -def _seeded_typed_return_dst_polars( - g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, - src: str, dst: str, node: str, direction: Direction, - preserve_input_order: bool = False, - index_ctx: Optional[Tuple["NodeIdIndex", "AdjacencyIndex", ArrayNamespace, "Engine"]] = None, -) -> Optional[SeededReturn]: - """#1755 polars analog of _seeded_typed_return_dst_pandas_cudf: same seed-first - reduction (seed out-edges -> typed-edge filter -> destination nodes) expressed - with polars filters, so a seeded cypher RETURN on polars/polars-gpu also lands - sub-ms. Returns ``(dst_node_rows, edges)`` (polars frames) or None to fall back - to the full lazy pipeline. Value-identical node set to the full path for the - covered shape (scalar filters, directed, single hop); row order may differ.""" - import polars as pl - from graphistry.compute.gfql.lazy.engine.polars.predicates import filter_by_dict_polars - if direction == "undirected": - return None - nodes_df, edges_df = g._nodes, g._edges - # Eager polars frames only: LazyFrame has no get_column, and mixed-engine - # node/edge frames must take the full path — decline rather than crash. - if not isinstance(nodes_df, pl.DataFrame) or not isinstance(edges_df, pl.DataFrame): - return None - - n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) - n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) - ef = _seeded_scalar_filters(e1.edge_match, edges_df) - if n0f is None or n2f is None or ef is None or not n0f: - return None - from_col, to_col = (src, dst) if direction == "forward" else (dst, src) - - # from-side seed: reduce the node frame to the seed rows, take their ids. - # Membership sets are drop_nulls()'d (null ids/endpoints never link, matching - # the full pipeline's joins) and passed via .implode() (Series-arg is_in is - # deprecated in polars 1.42, see polars#22149). - ctx = index_ctx if index_ctx is not None else _resident_seed_indexes( - g, nodes_df, edges_df, node, src, dst, direction) - nid_ctx = (ctx[0], ctx[2], ctx[3]) if ctx is not None else _resident_node_id_index(g, nodes_df, node) - seed_nodes, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) - edges = dstn = None - kernel_admits = False - if ctx is not None: - nid, adj, xp, idx_engine = ctx - edges = _index_edge_rows( - adj, seed_nodes.get_column(node), xp, idx_engine, edges_df, - preserve_input_order=preserve_input_order) - kernel_admits = _indexed_kernel_admits( - seed_nodes, edges, n0f, node, how, ctx, len(nodes_df), len(edges_df)) - if edges is not None: - edges = filter_by_dict_polars(edges, e1.edge_match) - dstn = _index_node_rows(nid, edges.get_column(to_col), xp, idx_engine, nodes_df) - if dstn is None: - from_ids = seed_nodes.get_column(node).drop_nulls() - if from_ids.len() == 0: - return nodes_df.clear(), edges_df.clear(), seed_nodes, kernel_admits - edges = edges_df.filter(pl.col(from_col).is_in(from_ids.implode())) - edges = filter_by_dict_polars(edges, e1.edge_match) - dst_ids = edges.get_column(to_col).drop_nulls().unique() - dstn = nodes_df.filter(pl.col(node).is_in(dst_ids.implode())) - assert edges is not None and dstn is not None # both branches above assign - dstn = filter_by_dict_polars(dstn, n2.filter_dict) - # drop dangling edges + dedup destination nodes (mirror the pandas tail) - keep_ids = dstn.get_column(node).drop_nulls() - edges = edges.filter(pl.col(to_col).is_in(keep_ids.implode())) - dstn = dstn.filter(pl.col(node).is_in(edges.get_column(to_col).implode())).unique(subset=[node], maintain_order=True) - return dstn, edges, seed_nodes, kernel_admits - - -def _try_seeded_chain_polars(g: Plottable, ops: Sequence[ASTObject]) -> Optional[Plottable]: - """Serve a native directed scalar hop through the resident seed indexes, preserving - Polars table order and aliases; declines (None) without valid resident indexes.""" - import polars as pl - from graphistry.compute.gfql.index.api import _record_indexed_traversal - if len(ops) != 3: - return None - n0, e1, n2 = ops - if not isinstance(n0, ASTNode) or not isinstance(n2, ASTNode) or not isinstance(e1, ASTEdge): - return None - if (n0.query is not None or n2.query is not None or not n0.filter_dict - or not e1.is_simple_single_hop() or e1.direction not in ("forward", "reverse") - or e1.source_node_match is not None or e1.destination_node_match is not None - or e1.source_node_query is not None or e1.destination_node_query is not None - or e1.edge_query is not None or e1.include_zero_hop_seed or e1.prune_to_endpoints): - return None - nodes, edges = g._nodes, g._edges - node, src, dst = g._node, g._source, g._destination - if (not isinstance(nodes, pl.DataFrame) or not isinstance(edges, pl.DataFrame) - or node is None or src is None or dst is None): - return None - if nodes.schema[node] != edges.schema[src] or nodes.schema[node] != edges.schema[dst]: - return None - aliases = [op._name for op in ops if op._name is not None] - if len(aliases) != len(set(aliases)): - return None - if any(name in nodes.columns for name in (n0._name, n2._name) if name is not None): - return None - if e1._name is not None and e1._name in edges.columns: - return None - ctx = _resident_seed_indexes(g, nodes, edges, node, src, dst, e1.direction) - if ctx is None: - return None - reduced = _seeded_typed_return_dst_polars( - g, n0, n2, e1, src, dst, node, e1.direction, preserve_input_order=True, index_ctx=ctx) - if reduced is None: - return None - _, kept_edges, _, _ = reduced - if not isinstance(kept_edges, pl.DataFrame): - return None - endpoint_ids = pl.concat([kept_edges.get_column(src), kept_edges.get_column(dst)]).drop_nulls().unique() - nid_ctx = _resident_node_id_index(g, nodes, node) - result_nodes = None - if nid_ctx is not None: - nid, xp, engine = nid_ctx - result_nodes = _index_node_rows(nid, endpoint_ids, xp, engine, nodes, preserve_input_order=True) - if result_nodes is None: - result_nodes = nodes.filter(pl.col(node).is_in(endpoint_ids.implode())) - if result_nodes.get_column(node).n_unique() != result_nodes.height: - return None - if not isinstance(result_nodes, pl.DataFrame): - return None - from_col, to_col = (src, dst) if e1.direction == "forward" else (dst, src) - flags = [ - pl.col(node).is_in(kept_edges.get_column(endpoint).implode()).fill_null(False).alias(name) - for name, endpoint in ((n0._name, from_col), (n2._name, to_col)) if name is not None - ] - if flags: - result_nodes = result_nodes.with_columns(flags) - if e1._name is not None: - kept_edges = kept_edges.with_columns(pl.lit(True).alias(e1._name)) - _record_indexed_traversal( - seam="native_seeded_hop", engine=ctx[3], served=True, reason="served", hop_count=1, - public_seed_scan=node not in n0.filter_dict, hop_details=[{"hop": 1}]) - return g.nodes(result_nodes).edges(kept_edges) diff --git a/graphistry/compute/chain_specializations/__init__.py b/graphistry/compute/chain_specializations/__init__.py new file mode 100644 index 0000000000..3d9404a89a --- /dev/null +++ b/graphistry/compute/chain_specializations/__init__.py @@ -0,0 +1,5 @@ +"""Chain specializations for the pandas/cuDF engines: admission predicates and their lanes.""" +from .admission import NativeFastPathShape, native_fast_path_admits +from .hotpaths import _try_chain_fast_path + +__all__ = ["NativeFastPathShape", "native_fast_path_admits", "_try_chain_fast_path"] diff --git a/graphistry/compute/chain_specializations/admission.py b/graphistry/compute/chain_specializations/admission.py new file mode 100644 index 0000000000..54afc01d33 --- /dev/null +++ b/graphistry/compute/chain_specializations/admission.py @@ -0,0 +1,77 @@ +"""Shape admission for the pandas/cuDF chain specializations: the dispatcher and the tests +consult the same predicates, so a test that filters a shape corpus with them exercises exactly +what the dispatcher admits.""" +# ruff: noqa: E501 + +from typing import Dict, Literal, Optional, Sequence, Tuple, TYPE_CHECKING + +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge +from graphistry.compute.chain_fast_paths import SeedRowsHow +from graphistry.compute.typing import ArrayNamespace, DataFrameT + +if TYPE_CHECKING: + from graphistry.Engine import Engine + from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex + + +NativeFastPathShape = Literal["single-node", "seeded-hop"] + + +def native_fast_path_admits( + ops: Sequence[ASTObject], engine: "Engine", start_nodes: Optional[DataFrameT], +) -> Optional[NativeFastPathShape]: + """The shape the pandas/cuDF chain fast path serves for ``ops``, or None when the full + path must run. This is the dispatcher's own gate (``chain._try_chain_fast_path`` calls + it first), so a test that filters a shape corpus with it exercises exactly what the + dispatcher admits: a node-only op without ``query``, or a 3-op plain single hop whose + node ops carry no ``query``, whose edge carries no node matches, queries, zero-hop seed + or endpoint pruning, whose aliases are distinct, and which is not an undirected hop + with names or with node filters. Seeded chains (``start_nodes``) and other engines + decline. Frame-dependent conditions (missing frames, alias equal to the node binding) + are checked by the body after materialization.""" + from graphistry.Engine import Engine + if engine not in (Engine.PANDAS, Engine.CUDF) or start_nodes is not None: + return None + if len(ops) == 1: + n0 = ops[0] + return "single-node" if isinstance(n0, ASTNode) and n0.query is None else None + if len(ops) != 3: + return None + n0, e1, n2 = ops + if not (isinstance(n0, ASTNode) and n0.query is None and isinstance(n2, ASTNode) and n2.query is None): + return None + if not (isinstance(e1, ASTEdge) and e1.is_simple_single_hop() + and e1.source_node_match is None and e1.destination_node_match is None + and e1.source_node_query is None and e1.destination_node_query is None + and e1.edge_query is None and not e1.include_zero_hop_seed and not e1.prune_to_endpoints): + return None + named = [a for a in (n0._name, e1._name, n2._name) if a is not None] + if len(named) != len(set(named)): + return None + if e1.direction == "undirected" and (n0._name is not None or n2._name is not None or n0.filter_dict or n2.filter_dict): + return None + return "seeded-hop" + + + +def _indexed_kernel_admits( + seed_nodes: DataFrameT, gathered_edges: Optional[DataFrameT], n0f: Dict[str, object], + node: str, how: SeedRowsHow, ctx: Tuple["NodeIdIndex", "AdjacencyIndex", ArrayNamespace, "Engine"], + n_nodes: int, n_edges: int, +) -> bool: + """Whether the indexed connected-bindings kernel would have served this seeded 1-hop: + its seed admission (binding-column integer seed, property-index hit, or a scan on a + graph with fewer nodes than edges) and its frontier and gather cost gates.""" + from numbers import Integral + from graphistry.compute.gfql.index.cost import cost_gate_frac + _, adj, _, engine = ctx + seed_val = n0f.get(node) + seeded_on_binding = isinstance(seed_val, Integral) and not isinstance(seed_val, bool) + if not (seeded_on_binding or how == "property_index" or n_nodes < n_edges): + return False + frac = cost_gate_frac(engine) + n_frontier = int(seed_nodes[node].nunique()) if not hasattr(seed_nodes, "get_column") \ + else int(seed_nodes.get_column(node).n_unique()) + if n_frontier >= frac * adj.n_keys: + return False + return gathered_edges is not None and len(gathered_edges) < frac * n_edges diff --git a/graphistry/compute/chain_specializations/hotpaths.py b/graphistry/compute/chain_specializations/hotpaths.py new file mode 100644 index 0000000000..5b0ed8ce66 --- /dev/null +++ b/graphistry/compute/chain_specializations/hotpaths.py @@ -0,0 +1,289 @@ +"""The pandas/cuDF chain specializations: the single-node lane, the seeded typed single hop +and the seeded typed RETURN-destination reduction. Each lane sits next to its admission +predicate (``admission.py``); ``chain.py`` only dispatches.""" +# ruff: noqa: E501 + +from typing import List, Optional, Sequence, Tuple, TYPE_CHECKING, cast + +from graphistry.Engine import Engine, EngineAbstract, df_concat +from graphistry.Plottable import Plottable +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, Direction +from graphistry.compute.chain_fast_paths import ( + _ids_to_key_array, _index_edge_rows, _index_node_rows, _record_native_seed_lane, + _resident_node_id_index, _resident_seed_indexes, _seed_node_rows, _seeded_scalar_filters, + _tag_fast_path_aliases, SeededReturn, +) +from graphistry.compute.typing import ArrayLike, ArrayNamespace, DataFrameT, SeriesT +from .admission import _indexed_kernel_admits, native_fast_path_admits + +if TYPE_CHECKING: + from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex + + +def _single_node_rows_via_index_or_filter( + g: Plottable, n0: ASTNode, engine_abs: "EngineAbstract", +) -> DataFrameT: + """Resolve a single node op through a resident index or the canonical filter.""" + from graphistry.compute.filter_by_dict import filter_by_dict + nodes_df = g._nodes + assert nodes_df is not None + if not n0.filter_dict: + return nodes_df + node = g._node + n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) if node is not None else None + if node is not None and n0f: + nid_ctx = _resident_node_id_index(g, nodes_df, node) + rows, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) + if how != "scan": + _record_native_seed_lane(nodes_df, seam="native_seed_lookup", reason=how, hop_count=0, + public_seed_scan=node not in n0.filter_dict) + return rows + return filter_by_dict(nodes_df, n0.filter_dict, engine_abs) + + + +def _seeded_typed_hop_pandas_cudf( + g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, + src: str, dst: str, node: str, direction: Direction, +) -> Optional[Plottable]: + """Engine-generic (pandas + cuDF) fast path for a scalar-filtered + seeded typed 1-hop. Value-identical to the general seeded branch for the covered + shape (all node/edge filters are plain scalars, directed) — same rows, columns, + and dtypes; row order and RangeIndex may differ — collapsing it into a + few DataFrame filters so a seeded lookup lands sub-ms. Uses only the shared + pandas/cuDF DataFrame API (no numpy array drops) so the same body runs on both + engines. Returns None to fall back for anything it does not cover (predicates, + undirected, missing columns) — the caller then runs the general branch.""" + if direction == "undirected": + return None + + nodes_df, edges_df = g._nodes, g._edges + if nodes_df is None or edges_df is None: + return None + n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) + n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) + ef = _seeded_scalar_filters(e1.edge_match, edges_df) + if n0f is None or n2f is None or ef is None: + return None + from_col, to_col = (src, dst) if direction == "forward" else (dst, src) + + # seed first; valid resident indexes serve the seed, frontier and gather positionally, any decline falls back to the scan body with identical results + ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) if n0f else None + seed_nodes = edges = cand = None + if ctx is not None: + nid, adj, xp, idx_engine = ctx + seed_nodes, _ = _seed_node_rows(g, nodes_df, n0f, node, (nid, xp, idx_engine), n0.filter_dict) + edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) + if edges is not None: + if ef: + for k, v in ef.items(): + edges = edges[edges[k] == v] + if 'cudf' in str(type(edges).__module__): + import cudf as _cd # type: ignore + endpoint_ids = _cd.concat([edges[src], edges[dst]]) + else: + import pandas as _pd + endpoint_ids = _pd.concat([edges[src], edges[dst]]) + cand = _index_node_rows(nid, endpoint_ids, xp, idx_engine, nodes_df) + served_via_index = cand is not None + if cand is None: + if n0f: + seed_nodes = nodes_df + for k, v in sorted(n0f.items(), key=lambda kv: 0 if kv[0] == node else 1): + seed_nodes = seed_nodes[seed_nodes[k] == v] + edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())] + else: + edges = edges_df + if ef: # typed edge (edge_match) — now on the reduced frontier + for k, v in ef.items(): + edges = edges[edges[k] == v] + + # membership sets are dropna()'d: null ids never link, matching the full path's joins + cand = nodes_df[ + nodes_df[node].isin(edges[src].dropna()) | nodes_df[node].isin(edges[dst].dropna()) + ].drop_duplicates(subset=[node]) + assert edges is not None and cand is not None # both branches above assign + if served_via_index: + _record_native_seed_lane(nodes_df, seam="native_seeded_hop", reason="served", hop_count=1, + public_seed_scan=node not in n0f) + if n2f: # destination-node filter (to-side) + n2_cand = cand + for k, v in n2f.items(): + n2_cand = n2_cand[n2_cand[k] == v] + n2_ok = n2_cand[node] + else: + n2_ok = cand[node] + to_vals = edges[to_col] + keep = edges[src].isin(cand[node].dropna()) & edges[dst].isin(cand[node].dropna()) & to_vals.isin(n2_ok.dropna()) + edges = edges[keep] + cand = cand[cand[node].isin(edges[src]) | cand[node].isin(edges[dst])] + return g.nodes(cand).edges(edges) + + +def _seeded_typed_return_dst_pandas_cudf( + g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, + src: str, dst: str, node: str, direction: Direction, +) -> Optional[SeededReturn]: + """Cypher RETURN-alias fast path: like _seeded_typed_hop_pandas_cudf but + returns ONLY the destination (RETURN-alias) node rows + surviving edges — no + seed-node gather, no Plottable round-trip — so the seeded cypher projection + lands sub-ms. Engine-generic (pandas + cuDF): only the shared DataFrame API, + no numpy array drops. Returns ``(dst_node_rows, edges)`` or None to fall back.""" + if direction == "undirected": + return None + nodes_df, edges_df = g._nodes, g._edges + if nodes_df is None or edges_df is None: + return None + n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) + n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) + ef = _seeded_scalar_filters(e1.edge_match, edges_df) + if n0f is None or n2f is None or ef is None or not n0f: + return None + from_col, to_col = (src, dst) if direction == "forward" else (dst, src) + # id filter first, then the object filters on the survivors; membership sets are dropna()'d so null ids never link + ctx = _resident_seed_indexes(g, nodes_df, edges_df, node, src, dst, direction) + nid_ctx = (ctx[0], ctx[2], ctx[3]) if ctx is not None else _resident_node_id_index(g, nodes_df, node) + seed_nodes, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) + edges = dstn = None + kernel_admits = False + if ctx is not None: + nid, adj, xp, idx_engine = ctx + edges = _index_edge_rows(adj, seed_nodes[node], xp, idx_engine, edges_df) + kernel_admits = _indexed_kernel_admits( + seed_nodes, edges, n0f, node, how, ctx, len(nodes_df), len(edges_df)) + if edges is not None: + if ef: + for k, v in ef.items(): + edges = edges[edges[k] == v] + dstn = _index_node_rows(nid, edges[to_col], xp, idx_engine, nodes_df) + if dstn is None: + edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())] + if ef: + for k, v in ef.items(): + edges = edges[edges[k] == v] + # destination nodes: real nodes that are to-endpoints of the surviving edges + dstn = nodes_df[nodes_df[node].isin(edges[to_col].dropna())] + assert edges is not None and dstn is not None # both branches above assign + if n2f: + for k, v in n2f.items(): + dstn = dstn[dstn[k] == v] + edges = edges[edges[to_col].isin(dstn[node].dropna())] + dstn = dstn[dstn[node].isin(edges[to_col].dropna())].drop_duplicates(subset=[node]) + return dstn, edges, seed_nodes, kernel_admits + + + +def _try_chain_fast_path( + g_in: Plottable, + ops: List[ASTObject], + engine_concrete: Engine, + start_nodes: Optional[DataFrameT] = None, +) -> Optional[Plottable]: + """Degenerate-shape fast path (pandas/cuDF): node-only ``MATCH (n)`` or a plain + single-hop ``MATCH (a)-[e]->(b)`` skip the forward/backward/combine BFS machinery. + Returns the result Plottable, or ``None`` to fall through to the full path. + + Same node/edge sets + VALUES as the full machinery (trackA_golden + hop/chain + suites); the 1-hop additionally preserves int node dtypes (the full path upcasts + int→float via merge — the merge is the artifact, int is the Cypher-conformant type). + Gated to unqueried nodes + a plain single-hop edge; NAMED ops are served (the alias + flags are reconstructed by `_tag_fast_path_aliases`) except when undirected or when + the same alias is reused. filtered-undirected and seeded chains fall through. + polars/dask/spark also fall through (own fast path / lazy semantics).""" + from graphistry.compute.filter_by_dict import filter_by_dict + + shape = native_fast_path_admits(ops, engine_concrete, start_nodes) + if shape is None: + return None + engine_abs = EngineAbstract(engine_concrete.value) + + def _materialize_fast_path_graph() -> Plottable: + from graphistry.compute.ComputeMixin import _coerce_input_formats # lazy — avoids circular import + g = g_in.materialize_nodes(engine=EngineAbstract(engine_concrete.value)) + return _coerce_input_formats(g, engine_concrete) + + if shape == "single-node": + n0 = ops[0] + assert isinstance(n0, ASTNode) # the predicate admitted this shape + g = _materialize_fast_path_graph() + if g._nodes is None: + return None + nodes = _single_node_rows_via_index_or_filter(g, n0, engine_abs) + if n0._name is not None: + alias_was_column = n0._name in nodes.columns + if alias_was_column: + nodes = nodes.drop(columns=[n0._name]) + nodes = nodes.assign(**{n0._name: True}) + other_columns = [c for c in nodes.columns if c != n0._name] + if g._node in other_columns: + other_columns = [g._node, *[c for c in other_columns if c != g._node]] + if alias_was_column: + nodes = nodes[[*other_columns, n0._name]] + else: + nodes = nodes[[*other_columns[:1], n0._name, *other_columns[1:]]] + nodes = nodes.reset_index(drop=True) + edges = g._edges.iloc[0:0] if g._edges is not None else None + return g.nodes(nodes).edges(edges) if edges is not None else g.nodes(nodes) + + n0, e1, n2 = ops + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) # the predicate admitted this shape + alias_n0, alias_e1, alias_n2 = n0._name, e1._name, n2._name + direction = e1.direction + unconstrained = not n0.filter_dict and not n2.filter_dict + g = _materialize_fast_path_graph() + if g._nodes is None or g._edges is None: + return None + src, dst, node = g._source, g._destination, g._node + if src is None or dst is None or node is None: + return None # no edge/node bindings -> can't fast-path; full path handles it + if alias_n0 == node or alias_n2 == node: + return None # a node alias equal to the node-id binding: the full path raises, never serve + if alias_e1 is not None and direction in ("forward", "reverse") \ + and alias_e1 == (src if direction == "forward" else dst): + return None # an edge alias equal to the from-side binding: lanes disagree on the node set + concat = df_concat(engine_concrete) + if unconstrained: + node_ids = g._nodes[node].dropna() # validate both endpoints; NaN ids never match + edges = g._edges[g._edges[src].isin(node_ids) & g._edges[dst].isin(node_ids)] + if e1.edge_match: + edges = filter_by_dict(edges, e1.edge_match, engine_abs) + else: + if engine_concrete in (Engine.PANDAS, Engine.CUDF): # seed-first: reduce edges by the node filters before the edge scan + _fast_res = _seeded_typed_hop_pandas_cudf(g, n0, n2, e1, src, dst, node, direction) + if _fast_res is not None: + return _tag_fast_path_aliases( + _fast_res, alias_n0, alias_e1, alias_n2, src, dst, node, direction) + from_col, to_col = (src, dst) if direction == "forward" else (dst, src) + edges = g._edges + if n0.filter_dict: + from_ids = filter_by_dict(g._nodes, n0.filter_dict, engine_abs)[node] + edges = edges[edges[from_col].isin(from_ids)] + if e1.edge_match: + edges = filter_by_dict(edges, e1.edge_match, engine_abs) + if n2.filter_dict: + to_present = edges[to_col].dropna().unique() + to_nodes = filter_by_dict( + g._nodes[g._nodes[node].isin(to_present)], n2.filter_dict, engine_abs) + edges = edges[edges[to_col].isin(to_nodes[node])] + ep = concat([ + edges[[src]].rename(columns={src: node}), + edges[[dst]].rename(columns={dst: node}), + ]).drop_duplicates() + cand = g._nodes[g._nodes[node].isin(ep[node])].drop_duplicates(subset=[node]) + valid = cand[node].dropna() + edges = edges[edges[src].isin(valid) & edges[dst].isin(valid)] + final = concat([ + edges[[src]].rename(columns={src: node}), + edges[[dst]].rename(columns={dst: node}), + ]).drop_duplicates() + nodes = cand[cand[node].isin(final[node])] + return _tag_fast_path_aliases( + g.nodes(nodes).edges(edges), alias_n0, alias_e1, alias_n2, src, dst, node, direction) + endpoints = concat([ + edges[[src]].rename(columns={src: node}), + edges[[dst]].rename(columns={dst: node}), + ]).drop_duplicates() + nodes = g._nodes[g._nodes[node].isin(endpoints[node])] + nodes = nodes.drop_duplicates(subset=[node]) # the full path's merge collapses duplicate node-id rows + return _tag_fast_path_aliases( + g.nodes(nodes).edges(edges), alias_n0, alias_e1, alias_n2, src, dst, node, direction) diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain.py b/graphistry/compute/gfql/lazy/engine/polars/chain.py index 5f05086936..9fa83683ca 100644 --- a/graphistry/compute/gfql/lazy/engine/polars/chain.py +++ b/graphistry/compute/gfql/lazy/engine/polars/chain.py @@ -7,7 +7,7 @@ (no silent pandas fallback). Deferred: variable-length/multi-hop edge sub-cases, some undirected multi-edge combos, node query=. """ -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, cast +from typing import TYPE_CHECKING, Any, List, Literal, Optional, Sequence, Tuple, Type, cast from typing_extensions import TypedDict @@ -18,7 +18,9 @@ from graphistry.Plottable import Plottable from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge -from graphistry.compute.chain_fast_paths import _single_node_rows_via_index_or_filter, _try_seeded_chain_polars +from graphistry.compute.chain_specializations.hotpaths import _single_node_rows_via_index_or_filter +from .chain_specializations.admission import polars_plain_single_hop_admits +from .chain_specializations.hotpaths import _plain_seeded_index_hop_polars, _plain_single_hop_polars, _try_seeded_chain_polars if TYPE_CHECKING: import polars as pl @@ -991,92 +993,17 @@ def _chain_traversal_polars(self: Plottable, ops, start_nodes: Optional[Any] = N "undirected edges in multi-edge chains; deferred undirected sub-cases — " "include_zero_hop_seed or *_query — require engine='pandas'." ) - - # Single-hop shape: [n(), e, n()] with no names/queries/matches (`MATCH (a {f})-[e]->(b)`). - # Result = edges whose endpoints pass the node filters + those endpoint nodes - # (isolated/dead-ends excluded); one hop means the backward pass prunes nothing more, so skip - # forward/backward/combine. Byte-identical vs pandas (verified: src/dst/both filters, reverse, - # dup/self-loop/cycle/isolated). Undirected takes this branch only when UNCONSTRAINED; - # filtered-undirected (OR of both directions) falls through to the full path. - def _fp_node(op): - return isinstance(op, ASTNode) and op._name is None and op.query is None - - def _plain_edge(op): - return (isinstance(op, ASTEdge) and op.is_simple_single_hop() - and op.edge_match is None and op.source_node_match is None - and op.destination_node_match is None and op._name is None - and op.source_node_query is None and op.destination_node_query is None - and op.edge_query is None and not op.include_zero_hop_seed) - - # GFQL physical index path for the seeded single-hop shape - # `MATCH (a {id-filter})-[e]->(b)` (forward/reverse, no destination filter). This native - # chain branch never reaches compute/hop.py, so it must consult the index here too. - from graphistry.compute.gfql.index import get_index_policy - _idx_pol = get_index_policy(self) - if (start_nodes is None and len(ops) == 3 and _fp_node(ops[0]) and _plain_edge(ops[1]) - and _fp_node(ops[2]) and ops[0].filter_dict and not ops[2].filter_dict - and ops[1].direction in ("forward", "reverse")): - from graphistry.compute.gfql.index import get_registry, maybe_index_hop - if (not get_registry(self).is_empty()) or _idx_pol in ("auto", "force"): - gf0 = ensure_nodes_polars(self) - seed0 = filter_by_dict_polars(gf0._nodes, ops[0].filter_dict) - from graphistry.Engine import Engine - from graphistry.compute.gfql.lazy import active_target, ExecutionTarget - _eng0 = Engine.POLARS_GPU if active_target() == ExecutionTarget.GPU else Engine.POLARS - _idxed0 = maybe_index_hop( - gf0, _eng0, nodes=seed0, hops=1, direction=ops[1].direction, - return_as_wave_front=False, to_fixed_point=False, policy=_idx_pol, - ) - if _idxed0 is not None: - return _idxed0 - + plain_shape = polars_plain_single_hop_admits(ops, start_nodes) + if plain_shape == "seeded-index": + indexed = _plain_seeded_index_hop_polars(self, ops) + if indexed is not None: + return indexed if start_nodes is None: seeded = _try_seeded_chain_polars(self, ops) if seeded is not None: return seeded - - if start_nodes is None and len(ops) == 3 and _fp_node(ops[0]) and _plain_edge(ops[1]) and _fp_node(ops[2]): - n0, e1, n2 = ops - unconstrained = not n0.filter_dict and not n2.filter_dict - if unconstrained or e1.direction in ("forward", "reverse"): - node_table_bound = self._nodes is not None - gf = ensure_nodes_polars(self) - ncol, scol, dcol = gf._node, gf._source, gf._destination - assert ncol is not None and scol is not None and dcol is not None - gf, restore = _align_edge_endpoints(gf, ncol, scol, dcol) - edges = drop_null_endpoint_edges(gf._edges, scol, dcol) - n_from, n_to = (n0, n2) if e1.direction != "reverse" else (n2, n0) - all_ids = gf._nodes.select(pl.col(ncol)) - - def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": - if not node_op.filter_dict: - return None - return filter_by_dict_polars(gf._nodes, node_op.filter_dict).select(pl.col(ncol)) - - filter_sides = ((scol, _filter_ids(n_from)), (dcol, _filter_ids(n_to))) - for endpoint_col, filter_ids in filter_sides: - if filter_ids is not None: - edges = edges.join(filter_ids, left_on=endpoint_col, right_on=ncol, how="semi") - # A filtered side drew its ids FROM the node table; a synthesized one is vacuously closed. - sides_not_closed_by_a_filter = ( - [col for col, filter_ids in filter_sides if filter_ids is None] - if node_table_bound else []) - endpoints = endpoint_ids(edges, scol, dcol, ncol) - if sides_not_closed_by_a_filter: - from graphistry.compute.gfql.lazy import collect_all - unresolvable, nodes = collect_all([ - endpoints.lazy().join(all_ids.lazy(), on=ncol, how="anti").select(pl.len()), - gf._nodes.lazy().join(endpoints.lazy(), on=ncol, how="semi"), - ]) - if unresolvable.item() > 0: - for endpoint_col in sides_not_closed_by_a_filter: - edges = edges.join(all_ids, left_on=endpoint_col, right_on=ncol, how="semi") - nodes = gf._nodes.join( - endpoint_ids(edges, scol, dcol, ncol), on=ncol, how="semi") - else: - nodes = gf._nodes.join(endpoints, on=ncol, how="semi") - nodes = nodes.unique(subset=[ncol], maintain_order=True) # one row per node id, as the full chain and pandas collapse - return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) + if plain_shape is not None: + return _plain_single_hop_polars(self, ops) if start_nodes is not None: from graphistry.Engine import Engine, df_to_engine diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py new file mode 100644 index 0000000000..c228d1a9ba --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py @@ -0,0 +1,6 @@ +"""Chain specializations for the polars engine: admission predicates and their lanes.""" +from .admission import PolarsPlainSingleHopShape, polars_plain_single_hop_admits, polars_seeded_lane_admits +from .hotpaths import _plain_seeded_index_hop_polars, _plain_single_hop_polars, _try_seeded_chain_polars + +__all__ = ["PolarsPlainSingleHopShape", "polars_plain_single_hop_admits", "polars_seeded_lane_admits", + "_plain_seeded_index_hop_polars", "_plain_single_hop_polars", "_try_seeded_chain_polars"] diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py new file mode 100644 index 0000000000..c47b8d8153 --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py @@ -0,0 +1,60 @@ +"""Shape admission for the polars chain specializations: the plain single-hop branches and +the seeded lane. The dispatcher and the tests consult the same predicates.""" + +from typing import Literal, Optional, Sequence + +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge + + +PolarsPlainSingleHopShape = Literal["seeded-index", "skip-combine"] + + +def _plain_node(op: ASTObject) -> bool: + return isinstance(op, ASTNode) and op._name is None and op.query is None + + +def _plain_edge(op: ASTObject) -> bool: + return (isinstance(op, ASTEdge) and op.is_simple_single_hop() + and op.edge_match is None and op.source_node_match is None + and op.destination_node_match is None and op._name is None + and op.source_node_query is None and op.destination_node_query is None + and op.edge_query is None and not op.include_zero_hop_seed) + + +def polars_plain_single_hop_admits(ops: Sequence[ASTObject], start_nodes: Optional[object]) -> Optional[PolarsPlainSingleHopShape]: + """The polars chain's plain single-hop branch for ``ops``: ``"seeded-index"`` when the + resident-index hop is consulted first (seed filter, no destination filter, directed), + ``"skip-combine"`` when the one-hop endpoint filter serves it without the + forward/backward/combine passes, None when the full chain runs. The dispatcher calls + this; unnamed, unqueried nodes and an unnamed, unmatched simple edge are the shape. + A filtered undirected hop is the one plain shape that still takes the full chain.""" + if start_nodes is not None or len(ops) != 3: + return None + n0, e1, n2 = ops + if not (_plain_node(n0) and _plain_edge(e1) and _plain_node(n2)): + return None + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) + directed = e1.direction in ("forward", "reverse") + if n0.filter_dict and not n2.filter_dict and directed: + return "seeded-index" + unconstrained = not n0.filter_dict and not n2.filter_dict + return "skip-combine" if (unconstrained or directed) else None + + +def polars_seeded_lane_admits(ops: Sequence[ASTObject]) -> bool: + """Whether the polars seeded lane's shape gate admits ``ops``: a 3-op directed simple + single hop whose seed node carries a filter, with no node queries, endpoint matches, + endpoint or edge queries, zero-hop seed or endpoint pruning. The dispatcher calls this + first; the frame conditions (polars frames, matching id dtypes, valid resident indexes, + scalar-only filters, no colliding aliases) are decided by the body and can still decline + an admitted shape.""" + if len(ops) != 3: + return False + n0, e1, n2 = ops + if not (isinstance(n0, ASTNode) and isinstance(n2, ASTNode) and isinstance(e1, ASTEdge)): + return False + return not (n0.query is not None or n2.query is not None or not n0.filter_dict + or not e1.is_simple_single_hop() or e1.direction not in ("forward", "reverse") + or e1.source_node_match is not None or e1.destination_node_match is not None + or e1.source_node_query is not None or e1.destination_node_query is not None + or e1.edge_query is not None or e1.include_zero_hop_seed or e1.prune_to_endpoints) diff --git a/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py new file mode 100644 index 0000000000..97b51e0bab --- /dev/null +++ b/graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py @@ -0,0 +1,215 @@ +"""The polars chain specializations: the plain single-hop branches (resident-index consult and +the skip-combine pass), the seeded lane and the seeded typed RETURN-destination reduction. Each +lane sits next to its admission predicate (``admission.py``); ``chain.py`` only dispatches.""" +# ruff: noqa: E501 + +from typing import Any, Dict, Optional, Sequence, Tuple, TYPE_CHECKING, Union, cast + +from graphistry.Plottable import Plottable +from graphistry.Engine import Engine +from graphistry.compute.ast import ASTObject, ASTNode, ASTEdge, Direction +from graphistry.compute.chain_specializations.admission import _indexed_kernel_admits +from graphistry.compute.chain_fast_paths import ( + _ids_to_key_array, _index_edge_rows, _index_node_rows, _record_native_seed_lane, + _resident_node_id_index, _resident_seed_indexes, _seed_node_rows, _seeded_scalar_filters, + SeededReturn, +) +from graphistry.compute.endpoint_utils import drop_null_endpoint_edges +from graphistry.compute.typing import ArrayLike, ArrayNamespace, DataFrameT +from graphistry.compute.gfql.lazy.engine.polars.dtypes import endpoint_ids +from graphistry.compute.gfql.lazy.engine.polars.hop_eager import ensure_nodes_polars +from graphistry.compute.gfql.lazy.engine.polars.predicates import filter_by_dict_polars +from .admission import polars_seeded_lane_admits + +if TYPE_CHECKING: + import polars as pl + from graphistry.compute.gfql.index.registry import AdjacencyIndex, NodeIdIndex + from graphistry.compute.gfql.lazy.engine.polars.dtypes import PolarsFrame + + +def _plain_seeded_index_hop_polars(g: Plottable, ops: Sequence[ASTObject]) -> Optional[Plottable]: + """Consult the resident index for the plain seeded single hop; None when it declines.""" + from graphistry.Engine import Engine + from graphistry.compute.gfql.index import get_index_policy, get_registry, maybe_index_hop + from graphistry.compute.gfql.lazy import active_target, ExecutionTarget + n0, e1 = ops[0], ops[1] + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) # the predicate admitted this shape + policy = get_index_policy(g) + if get_registry(g).is_empty() and policy not in ("auto", "force"): + return None + gf0 = ensure_nodes_polars(g) + seed0 = filter_by_dict_polars(gf0._nodes, n0.filter_dict) + engine = Engine.POLARS_GPU if active_target() == ExecutionTarget.GPU else Engine.POLARS + return maybe_index_hop( + gf0, engine, nodes=seed0, hops=1, direction=e1.direction, + return_as_wave_front=False, to_fixed_point=False, policy=policy, + ) + + +def _plain_single_hop_polars(g: Plottable, ops: Sequence[ASTObject]) -> Plottable: + """Serve the plain single hop as one endpoint-filter pass, skipping forward/backward/combine.""" + import polars as pl + from graphistry.compute.gfql.lazy.engine.polars.chain import _align_edge_endpoints, _restore_edge_dtypes + n0, e1, n2 = ops + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) # the predicate admitted this shape + node_table_bound = g._nodes is not None + gf = ensure_nodes_polars(g) + ncol, scol, dcol = gf._node, gf._source, gf._destination + assert ncol is not None and scol is not None and dcol is not None + gf, restore = _align_edge_endpoints(gf, ncol, scol, dcol) + edges = drop_null_endpoint_edges(gf._edges, scol, dcol) + n_from, n_to = (n0, n2) if e1.direction != "reverse" else (n2, n0) + all_ids = gf._nodes.select(pl.col(ncol)) + + def _filter_ids(node_op: ASTNode) -> "Optional[PolarsFrame]": + if not node_op.filter_dict: + return None + return filter_by_dict_polars(gf._nodes, node_op.filter_dict).select(pl.col(ncol)) + + filter_sides = ((scol, _filter_ids(n_from)), (dcol, _filter_ids(n_to))) + for endpoint_col, filter_ids in filter_sides: + if filter_ids is not None: + edges = edges.join(filter_ids, left_on=endpoint_col, right_on=ncol, how="semi") + # A filtered side drew its ids FROM the node table; a synthesized one is vacuously closed. + sides_not_closed_by_a_filter = ( + [col for col, filter_ids in filter_sides if filter_ids is None] + if node_table_bound else []) + endpoints = endpoint_ids(edges, scol, dcol, ncol) + if sides_not_closed_by_a_filter: + from graphistry.compute.gfql.lazy import collect_all + unresolvable, nodes = collect_all([ + endpoints.lazy().join(all_ids.lazy(), on=ncol, how="anti").select(pl.len()), + gf._nodes.lazy().join(endpoints.lazy(), on=ncol, how="semi"), + ]) + if unresolvable.item() > 0: + for endpoint_col in sides_not_closed_by_a_filter: + edges = edges.join(all_ids, left_on=endpoint_col, right_on=ncol, how="semi") + nodes = gf._nodes.join( + endpoint_ids(edges, scol, dcol, ncol), on=ncol, how="semi") + else: + nodes = gf._nodes.join(endpoints, on=ncol, how="semi") + nodes = nodes.unique(subset=[ncol], maintain_order=True) # one row per node id, as the full chain and pandas collapse duplicate node rows + return gf.nodes(nodes, ncol).edges(_restore_edge_dtypes(edges, scol, dcol, restore), scol, dcol) + + +def _seeded_typed_return_dst_polars( + g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge, + src: str, dst: str, node: str, direction: Direction, + preserve_input_order: bool = False, + index_ctx: Optional[Tuple["NodeIdIndex", "AdjacencyIndex", ArrayNamespace, "Engine"]] = None, +) -> Optional[SeededReturn]: + """Polars analog of _seeded_typed_return_dst_pandas_cudf: same seed-first + reduction (seed out-edges -> typed-edge filter -> destination nodes) expressed + with polars filters, so a seeded cypher RETURN on polars/polars-gpu also lands + sub-ms. Returns ``(dst_node_rows, edges)`` (polars frames) or None to fall back + to the full lazy pipeline. Value-identical node set to the full path for the + covered shape (scalar filters, directed, single hop); row order may differ.""" + import polars as pl + from graphistry.compute.gfql.lazy.engine.polars.predicates import filter_by_dict_polars + if direction == "undirected": + return None + nodes_df, edges_df = g._nodes, g._edges + # eager polars frames only; mixed-engine node/edge frames take the full path + if not isinstance(nodes_df, pl.DataFrame) or not isinstance(edges_df, pl.DataFrame): + return None + + n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df) + n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df) + ef = _seeded_scalar_filters(e1.edge_match, edges_df) + if n0f is None or n2f is None or ef is None or not n0f: + return None + from_col, to_col = (src, dst) if direction == "forward" else (dst, src) + + # membership sets are drop_nulls()'d (null ids never link) and passed via implode() (Series-arg is_in is deprecated) + ctx = index_ctx if index_ctx is not None else _resident_seed_indexes( + g, nodes_df, edges_df, node, src, dst, direction) + nid_ctx = (ctx[0], ctx[2], ctx[3]) if ctx is not None else _resident_node_id_index(g, nodes_df, node) + seed_nodes, how = _seed_node_rows(g, nodes_df, n0f, node, nid_ctx, n0.filter_dict) + edges = dstn = None + kernel_admits = False + if ctx is not None: + nid, adj, xp, idx_engine = ctx + edges = _index_edge_rows( + adj, seed_nodes.get_column(node), xp, idx_engine, edges_df, + preserve_input_order=preserve_input_order) + kernel_admits = _indexed_kernel_admits( + seed_nodes, edges, n0f, node, how, ctx, len(nodes_df), len(edges_df)) + if edges is not None: + edges = filter_by_dict_polars(edges, e1.edge_match) + dstn = _index_node_rows(nid, edges.get_column(to_col), xp, idx_engine, nodes_df) + if dstn is None: + from_ids = seed_nodes.get_column(node).drop_nulls() + if from_ids.len() == 0: + return nodes_df.clear(), edges_df.clear(), seed_nodes, kernel_admits + edges = edges_df.filter(pl.col(from_col).is_in(from_ids.implode())) + edges = filter_by_dict_polars(edges, e1.edge_match) + dst_ids = edges.get_column(to_col).drop_nulls().unique() + dstn = nodes_df.filter(pl.col(node).is_in(dst_ids.implode())) + assert edges is not None and dstn is not None # both branches above assign + dstn = filter_by_dict_polars(dstn, n2.filter_dict) + # drop dangling edges + dedup destination nodes (mirror the pandas tail) + keep_ids = dstn.get_column(node).drop_nulls() + edges = edges.filter(pl.col(to_col).is_in(keep_ids.implode())) + dstn = dstn.filter(pl.col(node).is_in(edges.get_column(to_col).implode())).unique(subset=[node], maintain_order=True) + return dstn, edges, seed_nodes, kernel_admits + + + +def _try_seeded_chain_polars(g: Plottable, ops: Sequence[ASTObject]) -> Optional[Plottable]: + """Serve a native directed scalar hop through the resident seed indexes, preserving + Polars table order and aliases; declines (None) without valid resident indexes.""" + import polars as pl + from graphistry.compute.gfql.index.api import _record_indexed_traversal + if not polars_seeded_lane_admits(ops): + return None + n0, e1, n2 = ops + assert isinstance(n0, ASTNode) and isinstance(e1, ASTEdge) and isinstance(n2, ASTNode) and n0.filter_dict # the predicate admitted this shape + nodes, edges = g._nodes, g._edges + node, src, dst = g._node, g._source, g._destination + if (not isinstance(nodes, pl.DataFrame) or not isinstance(edges, pl.DataFrame) + or node is None or src is None or dst is None): + return None + if nodes.schema[node] != edges.schema[src] or nodes.schema[node] != edges.schema[dst]: + return None + aliases = [op._name for op in ops if op._name is not None] + if len(aliases) != len(set(aliases)): + return None + if any(name in nodes.columns for name in (n0._name, n2._name) if name is not None): + return None + if e1._name is not None and e1._name in edges.columns: + return None + ctx = _resident_seed_indexes(g, nodes, edges, node, src, dst, e1.direction) + if ctx is None: + return None + reduced = _seeded_typed_return_dst_polars( + g, n0, n2, e1, src, dst, node, e1.direction, preserve_input_order=True, index_ctx=ctx) + if reduced is None: + return None + _, kept_edges, _, _ = reduced + if not isinstance(kept_edges, pl.DataFrame): + return None + endpoint_ids = pl.concat([kept_edges.get_column(src), kept_edges.get_column(dst)]).drop_nulls().unique() + nid_ctx = _resident_node_id_index(g, nodes, node) + result_nodes = None + if nid_ctx is not None: + nid, xp, engine = nid_ctx + result_nodes = _index_node_rows(nid, endpoint_ids, xp, engine, nodes, preserve_input_order=True) + if result_nodes is None: + result_nodes = nodes.filter(pl.col(node).is_in(endpoint_ids.implode())) + if result_nodes.get_column(node).n_unique() != result_nodes.height: + return None + if not isinstance(result_nodes, pl.DataFrame): + return None + from_col, to_col = (src, dst) if e1.direction == "forward" else (dst, src) + flags = [ + pl.col(node).is_in(kept_edges.get_column(endpoint).implode()).fill_null(False).alias(name) + for name, endpoint in ((n0._name, from_col), (n2._name, to_col)) if name is not None + ] + if flags: + result_nodes = result_nodes.with_columns(flags) + if e1._name is not None: + kept_edges = kept_edges.with_columns(pl.lit(True).alias(e1._name)) + _record_indexed_traversal( + seam="native_seeded_hop", engine=ctx[3], served=True, reason="served", hop_count=1, + public_seed_scan=node not in n0.filter_dict, hop_details=[{"hop": 1}]) + return g.nodes(result_nodes).edges(kept_edges) diff --git a/graphistry/compute/gfql_fast_paths.py b/graphistry/compute/gfql_fast_paths.py index c3467c36d6..468a72a9ae 100644 --- a/graphistry/compute/gfql_fast_paths.py +++ b/graphistry/compute/gfql_fast_paths.py @@ -3748,10 +3748,9 @@ def _execute_seeded_typed_hop_fast_path( # intermediate graph, so trusting requested_engine would run polars ops on a # pandas frame (and vice versa). The pandas branch also covers cuDF (shared API). from graphistry.Engine import is_polars_df - from graphistry.compute.chain_fast_paths import ( - _seeded_typed_return_dst_pandas_cudf, _seeded_typed_return_dst_polars, - _resident_seed_indexes, - ) + from graphistry.compute.chain_fast_paths import _resident_seed_indexes + from graphistry.compute.chain_specializations.hotpaths import _seeded_typed_return_dst_pandas_cudf + from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths import _seeded_typed_return_dst_polars nodes_frame = base_graph._nodes is_polars = is_polars_df(nodes_frame) if is_polars != is_polars_df(base_graph._edges): diff --git a/graphistry/tests/compute/chain_specializations/test_native_admission.py b/graphistry/tests/compute/chain_specializations/test_native_admission.py new file mode 100644 index 0000000000..c4cf60431e --- /dev/null +++ b/graphistry/tests/compute/chain_specializations/test_native_admission.py @@ -0,0 +1,93 @@ +"""``native_fast_path_admits`` is the pandas/cuDF chain fast path's own gate. + +Pins: for every corpus shape the predicate's verdict equals whether ``_try_chain_fast_path`` +served it (admits ⇔ served, on pandas and cuDF), the decision table is stable per shape, a +seeded chain (``start_nodes``) and a non pandas/cuDF engine always decline, and every served +shape stays value-identical to the full path. +""" +import pandas as pd +import pytest + +import graphistry +import graphistry.compute.chain as chain_mod +from graphistry.Engine import Engine +from graphistry.compute.ast import e_forward, n +from graphistry.compute.chain_specializations.admission import native_fast_path_admits +from graphistry.tests.compute.gfql.routes.corpus import CORPUS, EDGES, NODES, by_name + +ENGINES = ["pandas", "cudf"] + + +def _graph(engine): + nodes, edges = NODES, EDGES + if engine == "cudf": + cudf = pytest.importorskip("cudf") + nodes, edges = cudf.from_pandas(nodes), cudf.from_pandas(edges) + return graphistry.nodes(nodes, "key").edges(edges, "s", "d", "eid") + + +def _served(g, ops, engine): + real = chain_mod._try_chain_fast_path + hit = {"n": 0} + + def spy(*a, **k): + r = real(*a, **k) + hit["n"] += r is not None + return r + chain_mod._try_chain_fast_path = spy + try: + res = g.gfql(ops, engine=engine) + finally: + chain_mod._try_chain_fast_path = real + return res, hit["n"] == 1 + + +def _sig(res): + nn = res._nodes.to_pandas() if hasattr(res._nodes, "to_pandas") else res._nodes + ee = res._edges.to_pandas() if hasattr(res._edges, "to_pandas") else res._edges + return sorted(nn["key"].tolist()), sorted(map(tuple, ee[["s", "d"]].values.tolist())) + + +EXPECTED = { + "single node, scalar filter": "single-node", "single node, named": "single-node", + "single node, predicate filter": "single-node", "single node, no filter": "single-node", + "plain single hop, unseeded": "seeded-hop", "plain single hop, seeded": "seeded-hop", + "plain single hop, seeded, reverse": "seeded-hop", "plain single hop, seeded, destination filter": "seeded-hop", + "plain single hop, undirected, unconstrained": "seeded-hop", "plain single hop, undirected, seeded": None, + "typed single hop, seeded": "seeded-hop", "typed single hop, seeded, named": "seeded-hop", + "typed single hop, seeded, named, undirected": None, "single hop, node and edge alias share a name": None, + "single hop, edge alias = filtered column": "seeded-hop", "single hop, destination alias = its filtered column": "seeded-hop", + "single hop, source node match": None, "single hop, prune to endpoints": None, + "hops=2, seeded": None, "hops=2, seeded, typed, named": None, "to_fixed_point, seeded": None, "two single hops": None, +} + + +def test_every_corpus_shape_has_an_expected_verdict(): + assert set(EXPECTED) == {e.name for e in CORPUS} + + +@pytest.mark.parametrize("name", list(EXPECTED)) +def test_decision_table(name): + assert native_fast_path_admits(by_name()[name].ops(), Engine.PANDAS, None) == EXPECTED[name] + + +@pytest.mark.parametrize("engine", ENGINES) +@pytest.mark.parametrize("name", list(EXPECTED)) +def test_admits_iff_served(engine, name): + g = _graph(engine) + ops = by_name()[name].ops() + admitted = native_fast_path_admits(ops, Engine(engine), None) is not None + res, served = _served(g, ops, engine) + assert served == admitted, f"{name}: predicate={admitted} served={served}" + if served: + full = g.gfql(ops, engine=engine, policy={"preload": lambda ctx: None}) + assert _sig(res) == _sig(full) + + +@pytest.mark.parametrize("engine", [Engine.POLARS, Engine.DASK]) +def test_other_engines_decline(engine): + assert native_fast_path_admits([n({"id": 30})], engine, None) is None + + +def test_start_nodes_decline(): + assert native_fast_path_admits([n({"key": 1}), e_forward(), n()], Engine.PANDAS, pd.DataFrame({"key": [1]})) is None diff --git a/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json b/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json index 59b9e71ae1..10bf82567e 100644 --- a/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json +++ b/graphistry/tests/compute/gfql/coverage_baselines/ci-polars-py3.12.json @@ -5,6 +5,9 @@ "graphistry/compute/gfql/lazy/engine/__init__.py": 90.0, "graphistry/compute/gfql/lazy/engine/polars/__init__.py": 90.0, "graphistry/compute/gfql/lazy/engine/polars/chain.py": 86.0, + "graphistry/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py": 100.0, + "graphistry/compute/gfql/lazy/engine/polars/chain_specializations/admission.py": 95.0, + "graphistry/compute/gfql/lazy/engine/polars/chain_specializations/hotpaths.py": 95.0, "graphistry/compute/gfql/lazy/engine/polars/degrees.py": 82.0, "graphistry/compute/gfql/lazy/engine/polars/dtypes.py": 92.0, "graphistry/compute/gfql/lazy/engine/polars/hop.py": 87.0, diff --git a/graphistry/tests/compute/gfql/cypher/test_lowering.py b/graphistry/tests/compute/gfql/cypher/test_lowering.py index 54b28237fd..a64dc5c482 100644 --- a/graphistry/tests/compute/gfql/cypher/test_lowering.py +++ b/graphistry/tests/compute/gfql/cypher/test_lowering.py @@ -18605,6 +18605,7 @@ def _col_stats_trace(g: Any, query: str) -> List[Tuple[str, str, str]]: for s in steps if s.get("op") == "col_stats"] +@pytest.mark.route_engaged("cypher-fast") def test_t6_col_stats_decisions_are_visible_in_the_trace() -> None: """A dead fact is otherwise INVISIBLE: values stay correct, so no value test can fail. The trace distinguishes outcomes because their fixes differ.""" @@ -18625,6 +18626,7 @@ def test_t6_col_stats_decisions_are_visible_in_the_trace() -> None: outcomes={"nodes.id": "served", "edges.s": "served"}) +@pytest.mark.route_engaged("cypher-fast") def test_t6_assert_col_stats_helper_fails_loudly() -> None: """The helper must FAIL when the optimization did not fire -- an engagement pin that cannot fail is worse than none, which is the whole failure mode @@ -19104,6 +19106,7 @@ def _h3_records(result: Plottable) -> List[Dict[str, Any]]: return cast(List[Dict[str, Any]], _to_pandas_df(result._nodes).to_dict(orient="records")) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_serves_distinct_domain_shape(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: nodes, edges = _mk_h3_base_data() @@ -19117,6 +19120,7 @@ def test_h3_fused_two_hop_count_serves_distinct_domain_shape(engine: str, monkey assert _h3_records(result) == oracle == [{"numPaths": 5}] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_serves_distinct_edge_domains(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """Distinct EDGE matches with identical node filters also leave the equal-domain branch.""" @@ -19222,6 +19226,7 @@ def _mk_h3_case_data(fixture: str) -> Tuple[pd.DataFrame, pd.DataFrame]: raise AssertionError(f"unknown fixture {fixture}") +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) @pytest.mark.parametrize("label,fixture,query", _H3_DIFFERENTIAL_CASES, ids=[c[0] for c in _H3_DIFFERENTIAL_CASES]) def test_h3_fused_two_hop_count_matches_eager_twin_and_pandas( @@ -19246,6 +19251,7 @@ def test_h3_fused_two_hop_count_matches_eager_twin_and_pandas( assert fused == oracle, f"{label}: fused lane diverged from the pandas oracle" +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_empty_match_counts_zero(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """openCypher counts over no rows as 0 -- not an empty frame.""" @@ -19259,6 +19265,7 @@ def test_h3_fused_two_hop_count_empty_match_counts_zero(engine: str, monkeypatch assert _h3_records(result) == [{"numPaths": 0}] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_serves_projected_away_reserved_column(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """A PAYLOAD edge column named like a degree counter no longer forces a decline: @@ -19277,6 +19284,7 @@ def test_h3_fused_two_hop_count_serves_projected_away_reserved_column(engine: st assert _h3_records(result) == [{"numPaths": 5}] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_still_declines_reserved_endpoint_binding(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """NEGATIVE (the guard's remaining reachable side): when the SRC binding itself @@ -19354,6 +19362,7 @@ def test_h3_two_hop_count_fast_path_has_no_order_by_or_limit_surface(suffix: str assert _two_hop_count_alias(compiled.chain) == expect_alias +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["polars", "polars-gpu"]) def test_h3_fused_two_hop_count_handles_degenerate_bindings(engine: str, monkeypatch: pytest.MonkeyPatch) -> None: """The node key may share a name with an endpoint column, and source/destination may be bound diff --git a/graphistry/tests/compute/gfql/index/test_degree_consult.py b/graphistry/tests/compute/gfql/index/test_degree_consult.py index e27fc87a3c..df2d5976df 100644 --- a/graphistry/tests/compute/gfql/index/test_degree_consult.py +++ b/graphistry/tests/compute/gfql/index/test_degree_consult.py @@ -74,6 +74,7 @@ def spy(*a, **k): fp._two_hop_equal_domain_dense_total = real +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_degree_fact_is_built_and_actually_used(engine: str) -> None: """Engagement, not just correctness: a built-but-unused fact returns the same @@ -101,6 +102,7 @@ def test_identity_anchors_to_the_bound_frame_not_the_partition() -> None: assert fact.source_ref is g._edges +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("n_p,n_c", [(3, 3), (5, 1), (2, 8), (7, 2)]) def test_slice_is_exact_across_domain_shapes(n_p: int, n_c: int, engine: str) -> None: @@ -116,6 +118,7 @@ def test_slice_is_exact_across_domain_shapes(n_p: int, n_c: int, engine: str) -> assert value == oracle +@pytest.mark.route_engaged("cypher-fast") def test_gapped_node_space_builds_facts_and_stays_exact() -> None: """Density is NOT required for the degree arrays: ids absent from the span contribute ZERO to the dot, so a gapped node space builds valid facts. (The @@ -136,6 +139,7 @@ def test_gapped_node_space_builds_facts_and_stays_exact() -> None: assert used, "P-domain [0,2] is dense, so the kernel must consult the fact" +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("seed", range(6)) def test_differential_vs_the_scan_on_random_typed_graphs(seed: int) -> None: """Values must be identical with and without the fact, on arbitrary degree @@ -199,6 +203,7 @@ def test_a_fact_covering_a_narrower_span_is_refused() -> None: "WHERE b.age < 30 AND c.age > 20 RETURN count(*) AS n") +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_endpoint_filters_decline_dense_but_the_fused_count_serves(engine: str) -> None: """The q9 shape: same typed two-hop count as q8 plus WHERE filters on the diff --git a/graphistry/tests/compute/gfql/index/test_index.py b/graphistry/tests/compute/gfql/index/test_index.py index 177eb1aed1..4ab6fd2210 100644 --- a/graphistry/tests/compute/gfql/index/test_index.py +++ b/graphistry/tests/compute/gfql/index/test_index.py @@ -226,6 +226,7 @@ def test_invalid_index_policy_raises(graph): graph.gfql_explain(chain, index_policy="use") +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_index_policy_force_and_explain(graph, engine): chain = [n({"id": 0}), e_forward(hops=1)] @@ -311,6 +312,7 @@ def test_maybe_index_hop_reports_auto_build_decline_with_scan_parity(graph): assert _sig(auto_result) == _sig(scan_result) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_explain_exposes_planner_diagnostics(graph, engine): """LP1: gfql_explain surfaces the planner's cost signal — seed cardinality, the @@ -359,6 +361,7 @@ class _BadIdx: # missing keys_sorted/group_offsets → None, not AttributeError assert _seed_deg_sum(_BadIdx(), np.array([0, 1])) is None +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_explain_decision_reasons_for_scan_fallbacks(engine): """LP1: when the planner declines the index it records *why*, so a silent scan is @@ -390,6 +393,7 @@ def test_explain_decision_reasons_for_scan_fallbacks(engine): assert any(s.get("decision_reason") == "query not index-coverable" for s in steps2), (engine, steps2) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_cost_gate_engine_aware_never_loses_to_scan(engine): """F1: the index-vs-scan crossover depends on scan speed, so the cost gate @@ -518,6 +522,7 @@ def test_index_max_hops_honored(engine, hop_kw): assert _sig(base) == _sig(idx), f"max_hops divergence {hop_kw}" +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("max_hops", [1, 2, 3]) def test_index_bounded_range_min_one_hops_none(engine, max_hops): @@ -569,6 +574,7 @@ def test_index_coverability_bounded_range_boundaries(): assert not _hop_is_index_coverable(**dict(common, min_hops=3, max_hops=2)) assert not _hop_is_index_coverable(**dict(common, nodes=None)) +@pytest.mark.route_engaged("index-hop") def test_index_min_two_bounded_range_scans_pandas(graph): """Unsupported [2,2] ranges scan without entering the indexed traversal.""" from graphistry.compute.gfql.index import index_trace @@ -881,6 +887,7 @@ def test_chain_index_parity_vs_scan(typed_graph, engine, chain): assert _sig_typed(base) == _sig_typed(idx) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_chain_typed_edge_engages_index(typed_graph, engine): """All four engines: a typed-edge (simple-equality edge_match) seeded chain hop @@ -891,6 +898,7 @@ def test_chain_typed_edge_engages_index(typed_graph, engine): assert rep["used_index"] is True, (engine, rep) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_chain_untyped_engages_index(typed_graph, engine): """An untyped seeded chain hop engages the index on every engine (pandas/cuDF via @@ -909,6 +917,7 @@ def test_chain_membership_edge_match_stays_on_scan(typed_graph, engine): assert rep["used_index"] is False, (engine, rep) +@pytest.mark.route_engaged("index-hop") def test_chain_range_with_auto_labels_stays_on_scan(typed_graph): """A Cypher range needs per-depth records, so it must decline until indexed.""" engine = "pandas" @@ -1060,6 +1069,7 @@ def test_hop_dtype_mismatch_edge_match_matches_scan_error(typed_graph, engine): # These tests pin the SHAPE (predicate sees only candidate rows), not a wall-clock # number, so they can't go flaky on a loaded host. +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_typed_edge_predicate_only_reads_candidate_rows(typed_graph, engine, monkeypatch): """The edge_match predicate must be evaluated on the CSR-matched rows only. @@ -1098,6 +1108,7 @@ def spy(series, rows, eng): "not the traversal candidates") +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_typed_edge_predicate_cost_flat_in_graph_size(engine): """Growing the graph 8x while holding degree fixed must NOT grow the number of @@ -1146,6 +1157,7 @@ def spy(series, rows, eng): "the edge_match filter is scaling with the graph") +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_typed_edge_predicate_abandons_indexed_path_on_evaluation_failure( typed_graph, engine, monkeypatch @@ -1307,6 +1319,7 @@ def node_ids(gg): "(edges are identical; 1957 indexed node rows vs 1956 scanned). Strict, so it flips " "the moment the wave-front seed handling is unified."))), ]) +@pytest.mark.route_engaged("index-hop") def test_indexed_wavefront_node_set_matches_the_scan(typed_graph, engine, shape): """The index must not change WHICH NODES a wave-front hop reports, only how fast it gets there — the scan is the oracle.""" @@ -1331,6 +1344,7 @@ def node_ids(gg): assert node_ids(gi.hop(nodes=seeds, **kwargs)) == node_ids(g.hop(nodes=seeds, **kwargs)) +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", _cpu_engines()) def test_forcing_the_whole_column_mask_actually_changes_the_path(typed_graph, engine, monkeypatch): """The negative side of the boundary must be reachable — otherwise the test above is @@ -1376,6 +1390,7 @@ def spy(series, rows, eng): "resident_engine, requested_engine", [("pandas", "polars"), ("polars", "pandas")], ) +@pytest.mark.route_engaged("index-hop", "polars-plain") def test_explain_reports_bidirectional_engine_mismatch( graph, index_kinds, edge, expected_kinds, resident_engine, requested_engine ): @@ -1450,6 +1465,7 @@ def _polars_indexed_graph(): return g.gfql_index_all(engine="polars") +@pytest.mark.route_engaged("index-hop") def test_auto_engine_gfql_serves_polars_index_1767_cliff(): """#1767 cliff pin: polars frames + explicit polars index + gfql with NO engine argument must serve path=index on engine=polars (AUTO routes native, so the @@ -1470,6 +1486,7 @@ def test_auto_engine_gfql_serves_polars_index_1767_cliff(): assert out._nodes["destination"].to_list() == [101, 102] +@pytest.mark.route_engaged("index-hop") def test_auto_engine_hop_agreed_gates_never_mismatch_1767(): """Direct g.hop() with no engine on a polars-frame indexed graph: modern AUTO serves natively in polars (bridge-to-pandas was the 1767-era accident), and @@ -1685,6 +1702,7 @@ def test_col_stats_auto_narrows_lazy_frames(self): gi = gl.gfql_index_col_stats() # AUTO on lazy frames must not crash assert gi is not None + @pytest.mark.route_engaged("index-hop") def test_inversion_auto_index_auto_gfql_serves_polars_index(self): """THE INVERSION PIN. The exact scenario the retracted #1767 regressed to the scan floor: ``gfql_index_all()`` with NO engine + ``g.gfql( None: @@ -451,6 +455,7 @@ def unexpected_traversal(*args: Any, **kwargs: Any) -> Any: _assert_decision(decisions[0], seam="connected_bindings", served=True) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_destination_property_projection_dtype_parity( engine: str, @@ -473,6 +478,7 @@ def test_destination_property_projection_dtype_parity( _assert_parity(g, query, engine, monkeypatch, seam="destination_return") +@pytest.mark.route_engaged("indexed-kernel") def test_polars_connected_boundary_bypasses_canonical_traversal( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -523,6 +529,7 @@ def test_unnamed_middle_rows_call_matches_canonical( _assert_result_exact(actual, expected, engine) +@pytest.mark.route_engaged("indexed-kernel") @pytest.mark.parametrize("seed_kind", ["seed", "noise"]) def test_pandas_internal_id_plus_constraints_gathers_seed_before_filter( seed_kind: str, @@ -596,6 +603,7 @@ def record(frame: Any, *args: Any, **kwargs: Any) -> Any: return actual, steps, widths +@pytest.mark.route_engaged("index-hop", "indexed-kernel") @pytest.mark.parametrize("engine", ENGINES) def test_node_property_index_seeds_without_scanning( engine: str, @@ -614,6 +622,7 @@ def test_node_property_index_seeds_without_scanning( assert widths and widths[0] == 1 # one indexed candidate, not the node table +@pytest.mark.route_engaged("index-hop") @pytest.mark.parametrize("engine", ENGINES) def test_node_property_index_absent_matches_indexed( engine: str, @@ -692,6 +701,7 @@ def test_node_property_index_declines_unindexable_columns() -> None: g.gfql_index_node_props(["nosuch"]) +@pytest.mark.route_engaged("index-hop") def test_node_property_index_prefers_the_most_selective_column( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -718,6 +728,7 @@ def test_node_property_index_prefers_the_most_selective_column( pytest.param({"grp": 0}, "grp", False, id="unselective-keeps-scan"), ], ) +@pytest.mark.route_engaged("index-hop", "indexed-kernel") def test_node_property_index_cost_gate_under_policy_use( seed: Dict[str, Any], indexed_column: str, @@ -815,6 +826,7 @@ def test_polars_early_gate_refuses_unsupported_boundaries( assert attempted is False, f"polars gate recorded an attempt on {reason}" +@pytest.mark.route_engaged("indexed-kernel") def test_polars_early_gate_requires_the_whole_middle() -> None: """binding_ops that do not cover the middle must not take the bypass.""" pytest.importorskip("polars") @@ -845,6 +857,7 @@ def test_polars_early_gate_requires_the_whole_middle() -> None: assert attempted is True and state is not None +@pytest.mark.route_engaged("index-hop", "indexed-kernel") @pytest.mark.parametrize("engine", ENGINES) def test_indexed_execution_is_pure( engine: str, @@ -1042,6 +1055,7 @@ def hook(ctx: Dict[str, Any]) -> None: assert decisions[0]["reason"] == "policy_active" +@pytest.mark.route_engaged("index-hop", "indexed-kernel") @pytest.mark.parametrize("engine", ENGINES) def test_renamed_and_permuted_shape_remains_generic( engine: str, @@ -1084,6 +1098,7 @@ def test_renamed_and_permuted_shape_remains_generic( ) +@pytest.mark.route_engaged("indexed-kernel") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("error_type", [RuntimeError, MemoryError]) def test_unexpected_and_memory_errors_propagate( @@ -1110,6 +1125,7 @@ def fail_filter(*args: Any, **kwargs: Any) -> Any: ) +@pytest.mark.route_engaged("indexed-kernel") def test_use_policy_sparse_serves_dense_declines( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py b/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/test_polars_admission.py b/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/test_polars_admission.py new file mode 100644 index 0000000000..b083940b1c --- /dev/null +++ b/graphistry/tests/compute/gfql/lazy/engine/polars/chain_specializations/test_polars_admission.py @@ -0,0 +1,113 @@ +"""``polars_plain_single_hop_admits`` is the polars chain's own gate for its plain single-hop +branches. + +Pins: the decision table over the shared corpus (``"seeded-index"`` for a directed seeded hop +without a destination filter, ``"skip-combine"`` for the other plain single hops except the +filtered undirected one, None otherwise), a seeded chain (``start_nodes``) declines, and every +admitted shape stays value-identical to the pandas full path. +""" +import pandas as pd +import pytest + +import graphistry +import graphistry.compute.gfql.lazy.engine.polars.chain as pchain +import graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths as hot +from graphistry.compute.ast import e_forward, n +from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.admission import polars_plain_single_hop_admits, polars_seeded_lane_admits +from graphistry.tests.compute.gfql.routes.corpus import CORPUS, EDGES, NODES, by_name + +pl = pytest.importorskip("polars") + +EXPECTED = { + "plain single hop, unseeded": "skip-combine", + "plain single hop, seeded": "seeded-index", + "plain single hop, seeded, reverse": "seeded-index", + "plain single hop, seeded, destination filter": "skip-combine", + "plain single hop, undirected, unconstrained": "skip-combine", + "plain single hop, undirected, seeded": None, + "single hop, prune to endpoints": "seeded-index", +} + + +def test_every_corpus_shape_has_a_verdict(): + for e in CORPUS: + assert polars_plain_single_hop_admits(e.ops(), None) == EXPECTED.get(e.name), e.name + + +@pytest.mark.parametrize("name", list(EXPECTED)) +def test_decision_table(name): + assert polars_plain_single_hop_admits(by_name()[name].ops(), None) == EXPECTED[name] + + +def test_start_nodes_decline(): + assert polars_plain_single_hop_admits([n({"key": 1}), e_forward(), n()], pd.DataFrame({"key": [1]})) is None + + +def _sig(res): + nn = res._nodes.to_pandas() if hasattr(res._nodes, "to_pandas") else res._nodes + ee = res._edges.to_pandas() if hasattr(res._edges, "to_pandas") else res._edges + return sorted(nn["key"].tolist()), sorted(map(tuple, ee[["s", "d"]].values.tolist())) + + +@pytest.mark.parametrize("name", [k for k, v in EXPECTED.items() if v is not None]) +def test_admitted_shapes_match_the_pandas_full_path(name, request): + if name == "single hop, prune to endpoints": + request.applymarker(pytest.mark.xfail(strict=True, reason="graphistry/pygraphistry#2053")) + ops = by_name()[name].ops() + g_pd = graphistry.nodes(NODES, "key").edges(EDGES, "s", "d", "eid") + g_pl = graphistry.nodes(pl.from_pandas(NODES), "key").edges(pl.from_pandas(EDGES), "s", "d", "eid") + assert _sig(g_pl.gfql(ops, engine="polars")) == _sig(g_pd.gfql(ops, engine="pandas", policy={"preload": lambda ctx: None})) + + +SEEDED_LANE_ADMITS = { + "plain single hop, seeded", "plain single hop, seeded, reverse", "plain single hop, seeded, destination filter", + "typed single hop, seeded", "typed single hop, seeded, named", + "single hop, edge alias = filtered column", "single hop, destination alias = its filtered column", + "single hop, node and edge alias share a name", +} + + +def test_seeded_lane_decision_table_over_the_corpus(): + got = {e.name for e in CORPUS if polars_seeded_lane_admits(e.ops())} + assert got == SEEDED_LANE_ADMITS + + +def _indexed_polars_graph(): + g = graphistry.nodes(pl.from_pandas(NODES), "key").edges(pl.from_pandas(EDGES), "s", "d", "eid") + return g.gfql_index_all(engine="polars").gfql_index_node_props(["id"], engine="polars") + + +@pytest.mark.parametrize("name", [e.name for e in CORPUS]) +def test_seeded_lane_never_serves_a_shape_it_does_not_admit(name): + ops = by_name()[name].ops() + g = _indexed_polars_graph() + real = pchain._try_seeded_chain_polars + hit = {"n": 0} + + def spy(*a, **k): + r = real(*a, **k) + hit["n"] += r is not None + return r + pchain._try_seeded_chain_polars = spy + try: + g.gfql(ops, engine="polars", index_policy="use") + except Exception: + pass + finally: + pchain._try_seeded_chain_polars = real + assert hit["n"] == 0 or polars_seeded_lane_admits(ops), f"{name}: served without admission" + + +SEEDED_LANE_SERVES_DIRECTLY = SEEDED_LANE_ADMITS - { + # admitted by shape, declined by the body's alias-collision rule + "single hop, edge alias = filtered column", "single hop, destination alias = its filtered column", + "single hop, node and edge alias share a name", +} + + +@pytest.mark.route_engaged("polars-seeded") +@pytest.mark.parametrize("name", sorted(SEEDED_LANE_ADMITS)) +def test_seeded_lane_called_directly_serves_every_admitted_non_colliding_shape(name): + ops = by_name()[name].ops() + res = hot._try_seeded_chain_polars(_indexed_polars_graph(), ops) + assert (res is not None) == (name in SEEDED_LANE_SERVES_DIRECTLY) diff --git a/graphistry/tests/compute/gfql/routes/__init__.py b/graphistry/tests/compute/gfql/routes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graphistry/tests/compute/gfql/routes/corpus.py b/graphistry/tests/compute/gfql/routes/corpus.py new file mode 100644 index 0000000000..95fcdedea7 --- /dev/null +++ b/graphistry/tests/compute/gfql/routes/corpus.py @@ -0,0 +1,97 @@ +"""Shared shape corpus for the chain routes. + +Every entry is a native op-list shape variant; a route test filters the corpus with the +route's own admission predicate (the function its dispatcher calls), so one corpus is reused +across hot paths and a shape is never hand-picked per lane. Tags name the defect classes an +entry exercises so coverage can be read per class. +""" +from typing import Callable, Dict, List, NamedTuple, Tuple + +import pandas as pd + +from graphistry.compute.ast import ASTObject, e_forward, e_reverse, e_undirected, n +from graphistry.compute.predicates.numeric import GT +from graphistry.tests.compute.gfql.routes.registry import Frames, register + + +KeyMap = Callable[[int], object] + + +class Entry(NamedTuple): + name: str + ops: Callable[[], List[ASTObject]] + tags: Tuple[str, ...] + shape: Callable[[KeyMap], List[ASTObject]] + + +def _entry(name: str, shape: Callable[[KeyMap], List[ASTObject]], tags: Tuple[str, ...]) -> Entry: + """``ops()`` builds the shape over the base frames; ``shape(k)`` maps the node-key literals for a frame variant.""" + return Entry(name, lambda: shape(lambda v: v), tags, shape) + + +NODES = pd.DataFrame({"key": [1, 2, 3, 4, 5], "id": [10, 20, 30, 40, 50], "type": ["p", "p", "m", "m", "p"], "w": [1, 2, 3, 4, 5]}) +EDGES = pd.DataFrame({"s": [1, 1, 2, 3, 3, 4], "d": [2, 3, 3, 1, 1, 5], "type": ["KNOWS", "KNOWS", "LIKES", "KNOWS", "KNOWS", "LIKES"], "eid": [0, 1, 2, 3, 4, 5], "w": [1, 2, 3, 4, 5, 6]}) + +CORPUS: List[Entry] = [ + _entry("single node, scalar filter", lambda k: [n({"id": 30})], ("single-node",)), + _entry("single node, named", lambda k: [n({"id": 30}, name="a")], ("single-node", "alias")), + _entry("single node, predicate filter", lambda k: [n({"w": GT(2)})], ("single-node", "predicate")), + _entry("single node, no filter", lambda k: [n()], ("single-node",)), + _entry("plain single hop, unseeded", lambda k: [n(), e_forward(), n()], ("single-hop", "unseeded")), + _entry("plain single hop, seeded", lambda k: [n({"key": k(1)}), e_forward(), n()], ("single-hop", "seeded", "#2051")), + _entry("plain single hop, seeded, reverse", lambda k: [n({"key": k(1)}), e_reverse(), n()], ("single-hop", "seeded", "reverse")), + _entry("plain single hop, seeded, destination filter", lambda k: [n({"key": k(1)}), e_forward(), n({"id": 20})], ("single-hop", "seeded", "dest-filter", "#2051")), + _entry("plain single hop, undirected, unconstrained", lambda k: [n(), e_undirected(), n()], ("single-hop", "undirected")), + _entry("plain single hop, undirected, seeded", lambda k: [n({"key": k(1)}), e_undirected(), n()], ("single-hop", "undirected", "seeded")), + _entry("typed single hop, seeded", lambda k: [n({"key": k(1)}), e_forward({"type": "KNOWS"}), n()], ("single-hop", "seeded", "typed")), + _entry("typed single hop, seeded, named", lambda k: [n({"key": k(1)}, name="a"), e_forward({"type": "KNOWS"}, name="e"), n(name="b")], ("single-hop", "seeded", "typed", "alias")), + _entry("typed single hop, seeded, named, undirected", lambda k: [n({"key": k(1)}, name="a"), e_undirected({"type": "KNOWS"}, name="e"), n(name="b")], ("single-hop", "undirected", "alias")), + _entry("single hop, node and edge alias share a name", lambda k: [n({"key": k(1)}, name="a"), e_forward(name="a"), n()], ("single-hop", "alias", "shared-alias-name")), + _entry("single hop, edge alias = filtered column", lambda k: [n({"id": 30}, name="m"), e_forward({"type": "KNOWS"}, name="type"), n(name="p")], ("single-hop", "alias-collision", "#2039")), + _entry("single hop, destination alias = its filtered column", lambda k: [n({"id": 30}, name="m"), e_forward({"type": "KNOWS"}, name="e"), n({"type": "p"}, name="type")], ("single-hop", "alias-collision", "#2039")), + _entry("single hop, source node match", lambda k: [n(), e_forward(source_node_match={"type": "p"}), n()], ("single-hop", "endpoint-match")), + _entry("single hop, prune to endpoints", lambda k: [n({"key": k(1)}), e_forward(prune_to_endpoints=True), n()], ("single-hop", "prune", "#2053")), + _entry("hops=2, seeded", lambda k: [n({"key": k(1)}), e_forward(hops=2), n()], ("multi-hop", "seeded")), + _entry("hops=2, seeded, typed, named", lambda k: [n({"key": k(1)}, name="a"), e_forward({"type": "KNOWS"}, hops=2, name="e"), n(name="b")], ("multi-hop", "typed", "alias", "#2049")), + _entry("to_fixed_point, seeded", lambda k: [n({"key": k(1)}), e_forward(to_fixed_point=True), n()], ("multi-hop", "fixed-point")), + _entry("two single hops", lambda k: [n({"key": k(1)}), e_forward(), n(), e_forward(), n()], ("two-steps",)), +] + + +def tagged(tag: str) -> List[Entry]: + return [e for e in CORPUS if tag in e.tags] + + +def by_name() -> Dict[str, Entry]: + return {e.name: e for e in CORPUS} + + +register("routes.corpus", [(e.name, e.ops, e.tags) for e in CORPUS], Frames(NODES, EDGES, "key", "s", "d", "eid")) + + +def _frame_variants() -> Dict[str, Tuple[Frames, Tuple[str, ...], Callable[[int], object]]]: + """The same shapes over frames that differ in what the routes must agree on: id dtype, + null and duplicate ids, self-loops and cycles, an empty edge table, no edge-id binding.""" + str_nodes = NODES.assign(key=NODES["key"].map(lambda k: f"n{k}")) + str_edges = EDGES.assign(s=EDGES["s"].map(lambda k: f"n{k}"), d=EDGES["d"].map(lambda k: f"n{k}")) + null_nodes = pd.concat([NODES, pd.DataFrame({"key": [None], "id": [60], "type": ["p"], "w": [6]})], ignore_index=True).astype({"key": "Int64"}) + null_edges = pd.concat([EDGES, pd.DataFrame({"s": [1], "d": [None], "type": ["KNOWS"], "eid": [6], "w": [7]})], ignore_index=True).astype({"s": "Int64", "d": "Int64"}) + dup_nodes = pd.concat([NODES, NODES.iloc[[0]].assign(w=99)], ignore_index=True) + loop_edges = pd.concat([EDGES, pd.DataFrame({"s": [1, 2], "d": [1, 1], "type": ["KNOWS", "KNOWS"], "eid": [6, 7], "w": [7, 8]})], ignore_index=True) + return { + "str-ids": (Frames(str_nodes, str_edges, "key", "s", "d", "eid"), ("dtype-str",), lambda v: f"n{v}"), + "null-ids": (Frames(null_nodes, null_edges, "key", "s", "d", "eid"), ("null-ids",), lambda v: v), + "dup-ids": (Frames(dup_nodes, EDGES, "key", "s", "d", "eid"), ("dup-ids",), lambda v: v), + "self-loop-cycle": (Frames(NODES, loop_edges, "key", "s", "d", "eid"), ("self-loop", "cycle"), lambda v: v), + "empty-edges": (Frames(NODES, EDGES.iloc[0:0], "key", "s", "d", "eid"), ("empty-edges",), lambda v: v), + "no-edge-id": (Frames(NODES, EDGES.drop(columns=["eid"]), "key", "s", "d", None), ("no-edge-id",), lambda v: v), + } + + +_VARIANT_ROW_TAGS: Dict[str, Dict[str, Tuple[str, ...]]] = { + "dup-ids": {"single node, predicate filter": ("#2034",)}, # node lookup keeps each duplicate row; the general path collapses them +} + +for _variant, (_frames, _tags, _key) in _frame_variants().items(): + register(f"routes.corpus.{_variant}", [(e.name, (lambda e=e, k=_key: e.shape(k)), e.tags) for e in CORPUS], _frames, + tags=_tags + ("variant",), row_tags=_VARIANT_ROW_TAGS.get(_variant)) diff --git a/graphistry/tests/compute/gfql/routes/registry.py b/graphistry/tests/compute/gfql/routes/registry.py new file mode 100644 index 0000000000..f064caf23f --- /dev/null +++ b/graphistry/tests/compute/gfql/routes/registry.py @@ -0,0 +1,72 @@ +"""Shape registry for the route harness. + +Each specialization's own test module registers the shape table it was written against +(``register(...)`` returns the table unchanged, so the module keeps using it), with the +frames those shapes address and the defect classes they exercise. The harness in +``test_route_harness.py`` then tries every registered shape against every route whose +admission predicate admits it, so one input is exercised by several hot paths, not one. +""" +from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Sequence, Tuple, Union + +import pandas as pd +import pytest + +from graphistry.Plottable import Plottable +from graphistry.compute.ast import ASTObject + +Build = Callable[[], List[ASTObject]] +Row = Union[Tuple[str, Build], Tuple[str, Build, Tuple[str, ...]]] + + +class Frames(NamedTuple): + nodes: pd.DataFrame + edges: pd.DataFrame + node: str + src: str + dst: str + edge: Optional[str] = None + + +class Shape(NamedTuple): + table: str + label: str + build: Build + frames: Frames + tags: Tuple[str, ...] + + @property + def name(self) -> str: + return f"{self.table}/{self.label}" + + +REGISTRY: Dict[str, Shape] = {} + + +def register(table: str, rows: Sequence[Row], frames: Frames, tags: Iterable[str] = (), + row_tags: Optional[Dict[str, Tuple[str, ...]]] = None) -> Sequence[Row]: + """Register ``rows`` ((label, build[, tags]) ...) under ``table``; returns ``rows``.""" + base = tuple(tags) + for row in rows: + label, build = row[0], row[1] + extra = tuple(row[2]) if len(row) > 2 else () + extra += (row_tags or {}).get(label, ()) + shape = Shape(table, label, build, frames, base + extra) + REGISTRY.setdefault(shape.name, shape) + return rows + + +def to_engine(df: pd.DataFrame, engine: str): + if engine == "pandas": + return df + if engine == "cudf": + return pytest.importorskip("cudf").from_pandas(df) + if engine == "polars": + return pytest.importorskip("polars").from_pandas(df) + raise ValueError(engine) + + +def graph_for(shape: Shape, engine: str, indexed: bool = False) -> Plottable: + import graphistry + f = shape.frames + g = graphistry.nodes(to_engine(f.nodes, engine), f.node).edges(to_engine(f.edges, engine), f.src, f.dst, f.edge) + return g.gfql_index_all(engine=engine) if indexed else g diff --git a/graphistry/tests/compute/gfql/routes/switch.py b/graphistry/tests/compute/gfql/routes/switch.py new file mode 100644 index 0000000000..781dde5325 --- /dev/null +++ b/graphistry/tests/compute/gfql/routes/switch.py @@ -0,0 +1,51 @@ +"""Route switch for test amplification: make named GFQL hot paths decline for a scope.""" +from contextlib import contextmanager +from typing import Iterable, Iterator, List, Tuple + +ROUTES = ("native-fast", "polars-seeded", "polars-plain", "index-hop", "indexed-kernel", "cypher-fast") + + +def _none(*a, **k): + return None + + +def _targets(routes: Iterable[str]) -> List[Tuple[object, str]]: + import graphistry.compute.chain as chain_mod + import graphistry.compute.gfql_unified as unified + import graphistry.compute.gfql.index as index_pkg + import graphistry.compute.gfql.index.api as index_api + import graphistry.compute.gfql.index.bindings as bindings + import graphistry.compute.gfql.lazy.engine.polars.chain as pchain + routes = set(routes) + unknown = routes - set(ROUTES) + assert not unknown, f"unknown route(s) {sorted(unknown)}; known: {ROUTES}" + out: List[Tuple[object, str]] = [] + if "native-fast" in routes: + out.append((chain_mod, "_try_chain_fast_path")) + if "polars-seeded" in routes: + out.append((pchain, "_try_seeded_chain_polars")) + if "polars-plain" in routes: + out.append((pchain, "polars_plain_single_hop_admits")) + if "index-hop" in routes: + out += [(index_pkg, "maybe_index_hop"), (index_api, "maybe_index_hop")] + if "indexed-kernel" in routes: + out.append((bindings, "_try_indexed_connected_bindings_state")) + if "cypher-fast" in routes: + out += [(unified, name) for name in ( + "_execute_seeded_node_lookup_fast_path", "_execute_seeded_typed_hop_fast_path", + "_execute_single_hop_grouped_aggregate_fast_path", "_execute_two_hop_count_fast_path")] + return out + + +@contextmanager +def routes_off(routes: Iterable[str]) -> Iterator[None]: + """Within the block the named routes decline, so the general path answers.""" + saved = [] + for mod, name in _targets(routes): + saved.append((mod, name, getattr(mod, name))) + setattr(mod, name, _none) + try: + yield + finally: + for mod, name, value in reversed(saved): + setattr(mod, name, value) diff --git a/graphistry/tests/compute/gfql/routes/test_route_harness.py b/graphistry/tests/compute/gfql/routes/test_route_harness.py new file mode 100644 index 0000000000..ed54f1d04f --- /dev/null +++ b/graphistry/tests/compute/gfql/routes/test_route_harness.py @@ -0,0 +1,186 @@ +"""Route harness: every registered shape is tried against every chain route whose admission +predicate admits it. Three pins per cell: the route SERVES (its lane answers; a lane that +declines an admitted shape is recorded as an expected failure, the attenuation ledger), the +answer matches the same engine's general path (all routes off) on node/edge values, and its +node/edge sets match the pandas general path (the cross-engine oracle). Filed divergences are +strict expected failures keyed by their tag, so they flip when fixed. +""" +import math +import os +from typing import Callable, Dict, List, NamedTuple, Tuple + +import pandas as pd +import pytest + +import graphistry.compute.chain as chain_mod +import graphistry.compute.gfql.lazy.engine.polars.chain as pchain +from graphistry.Engine import Engine +from graphistry.compute.ast import ASTObject +from graphistry.compute.chain_specializations.admission import native_fast_path_admits +from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.admission import ( + polars_plain_single_hop_admits, polars_seeded_lane_admits, +) +from graphistry.tests.compute.gfql.routes.registry import REGISTRY, Shape, graph_for +from graphistry.tests.compute.gfql.routes.switch import ROUTES as ALL_ROUTES, routes_off + +import graphistry.tests.compute.gfql.routes.corpus # noqa: F401 registers routes.corpus +import graphistry.tests.compute.test_chain # noqa: F401 registers test_chain.* +import graphistry.tests.compute.test_chain_alias_column_collision # noqa: F401 registers collision.* + + +class Route(NamedTuple): + name: str + engines: Tuple[str, ...] + admits: Callable[[List[ASTObject], str], bool] + lane: Tuple[object, str] + indexed: bool + + +ROUTES = [ + Route("native-fast", ("pandas", "cudf"), + lambda ops, engine: native_fast_path_admits(ops, Engine(engine), None) is not None, + (chain_mod, "_try_chain_fast_path"), False), + Route("polars-plain", ("polars",), + lambda ops, engine: polars_plain_single_hop_admits(ops, None) is not None, + (pchain, "_plain_single_hop_polars"), False), + Route("polars-seeded", ("polars",), + lambda ops, engine: polars_seeded_lane_admits(ops), + (pchain, "_try_seeded_chain_polars"), True), +] + +KNOWN: Dict[Tuple[str, str], str] = { # (route, tag) -> issue: strict xfail until it lands (non-strict on frame variants, where a shape may coincide) + ("polars-plain", "#2053"): "graphistry/pygraphistry#2053", + ("native-fast", "#2034"): "graphistry/pygraphistry#2034", + ("polars-plain", "#2034"): "graphistry/pygraphistry#2034", + ("polars-seeded", "#2034"): "graphistry/pygraphistry#2034", +} + + +class Case(NamedTuple): + route: Route + engine: str + shape: Shape + + @property + def id(self) -> str: + return f"{self.route.name}/{self.engine}/{self.shape.name}" + + +def _cases() -> List[Case]: + out = [] + for shape in REGISTRY.values(): + for route in ROUTES: + for engine in route.engines: + try: + admitted = route.admits(shape.build(), engine) + except Exception: + admitted = False + if admitted: + out.append(Case(route, engine, shape)) + return out + + +CASES = _cases() + + +def _topd(df): + if df is None: + return None + if hasattr(df, "to_pandas"): + return df.to_pandas() + return df + + +def _canon(df) -> Tuple[Tuple[str, ...], List[Tuple]]: + df = _topd(df) + if df is None: + return ((), []) + cols = tuple(sorted(df.columns)) + rows = [] + for row in df[list(cols)].itertuples(index=False, name=None): + rows.append(tuple(None if (isinstance(v, float) and math.isnan(v)) or v is pd.NA or v is pd.NaT else v for v in row)) + rows.sort(key=repr) + return cols, rows + + +def _sig(res, frames) -> Tuple[List, List]: + nn, ee = _topd(res._nodes), _topd(res._edges) + def _na(v): + return None if v is None or v is pd.NA or v is pd.NaT or (isinstance(v, float) and math.isnan(v)) else v + nodes = sorted((_na(v) for v in nn[frames.node].tolist()), key=repr) if nn is not None else [] + edges = sorted((tuple(_na(v) for v in r) for r in ee[[frames.src, frames.dst]].values.tolist()), key=repr) if ee is not None and len(ee) else [] + return nodes, edges + + +def _served(case: Case, monkeypatch): + mod, name = case.route.lane + real = getattr(mod, name) + calls = {"served": 0} + + def spy(*a, **k): + out = real(*a, **k) + calls["served"] += out is not None + return out + monkeypatch.setattr(mod, name, spy) + return calls + + +def _skip_unavailable(engine: str) -> None: + if engine == "cudf": + if os.environ.get("TEST_CUDF") != "1": + pytest.skip("cuDF lane runs with TEST_CUDF=1") + pytest.importorskip("cudf") + if engine == "polars": + pytest.importorskip("polars") + + +@pytest.mark.parametrize("case", CASES, ids=[c.id for c in CASES]) +def test_admitted_shape_is_served_and_matches_the_general_path(case: Case, request, monkeypatch): + _skip_unavailable(case.engine) + for tag in case.shape.tags: + if (case.route.name, tag) in KNOWN: + request.applymarker(pytest.mark.xfail(strict="variant" not in case.shape.tags, reason=KNOWN[(case.route.name, tag)])) + g = graph_for(case.shape, case.engine, indexed=case.route.indexed) + calls = _served(case, monkeypatch) + try: + served = g.gfql(case.shape.build(), engine=case.engine) + except Exception as served_exc: + with routes_off(ALL_ROUTES): + with pytest.raises(type(served_exc)): + g.gfql(case.shape.build(), engine=case.engine) + return + with routes_off(ALL_ROUTES): + general = g.gfql(case.shape.build(), engine=case.engine) + oracle = _sig(graph_for(case.shape, "pandas").gfql(case.shape.build(), engine="pandas"), case.shape.frames) + assert _canon(served._nodes) == _canon(general._nodes), f"{case.id}: node rows differ from the general path" + assert _canon(served._edges) == _canon(general._edges), f"{case.id}: edge rows differ from the general path" + assert _sig(served, case.shape.frames) == oracle, f"{case.id}: node/edge sets differ from the pandas general path" + if calls["served"] == 0: + pytest.xfail(f"{case.id}: admitted by the predicate, declined by the lane body (attenuation ledger)") + + +@pytest.mark.route_engaged("native-fast", "polars-plain", "polars-seeded") +def test_every_route_serves_most_of_what_it_admits(monkeypatch): + """A lane that declines most admitted shapes has a predicate that no longer describes it.""" + per_route: Dict[str, List[int]] = {} + for case in CASES: + if case.engine != ("polars" if case.route.name.startswith("polars") else "pandas") or "variant" in case.shape.tags: + continue # the serve ratio describes the base corpus; frame variants are expected to attenuate + if case.engine == "polars": + pytest.importorskip("polars") + g = graph_for(case.shape, case.engine, indexed=case.route.indexed) + calls = _served(case, monkeypatch) + try: + g.gfql(case.shape.build(), engine=case.engine) + except Exception: + continue + per_route.setdefault(case.route.name, []).append(calls["served"] > 0) + for route, served in per_route.items(): + assert sum(served) * 2 >= len(served), f"{route}: served {sum(served)} of {len(served)} admitted shapes" + + +def test_every_route_has_admitted_shapes(): + covered = {(c.route.name, c.engine) for c in CASES} + for route in ROUTES: + for engine in route.engines: + assert (route.name, engine) in covered, f"{route.name}/{engine} admits no registered shape" diff --git a/graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py b/graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py index ecc278f488..f16f4a3ee6 100644 --- a/graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py +++ b/graphistry/tests/compute/gfql/test_endpoint_closure_matrix.py @@ -242,6 +242,7 @@ def test_index_backed_seeded_hop_serves_the_closed_answer(engine): assert edge_pair_set(out_dangling) == set(), "index route emitted an unclosed edge" +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ALL_ENGINES) def test_clean_graph_is_untouched_by_the_gate(engine): """POSITIVE CONTROL: with zero dangling endpoints the gate removes nothing and the node diff --git a/graphistry/tests/compute/gfql/test_engine_polars_chain.py b/graphistry/tests/compute/gfql/test_engine_polars_chain.py index c96a644863..c856ffec9a 100644 --- a/graphistry/tests/compute/gfql/test_engine_polars_chain.py +++ b/graphistry/tests/compute/gfql/test_engine_polars_chain.py @@ -742,6 +742,7 @@ def test_varlen_alias_parity_across_directions(self, pattern): actual = sorted(g_pl.gfql(q, engine="polars")._nodes["b.id"].to_list()) assert actual == expected + @pytest.mark.route_engaged("cypher-fast") def test_fixed_length_hop_is_not_gated(self): """A plain single-hop edge sets no min_hops, so no labels are requested and no gate runs — the guard must not silently start filtering ordinary chains.""" diff --git a/graphistry/tests/compute/gfql/test_fast_path_engagement.py b/graphistry/tests/compute/gfql/test_fast_path_engagement.py index c9d74c3f91..8e7f2bb4f3 100644 --- a/graphistry/tests/compute/gfql/test_fast_path_engagement.py +++ b/graphistry/tests/compute/gfql/test_fast_path_engagement.py @@ -36,6 +36,7 @@ def _graph(engine: str = "pandas"): ENGINES = ["pandas", "polars", "cudf"] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_two_hop_count_fast_path_engages(engine: str) -> None: """Engagement is per-ENGINE: a path that serves on pandas can silently decline @@ -111,6 +112,7 @@ def test_unknown_fast_path_name_is_reported_not_silently_missing() -> None: assert_fast_path(_graph(), Q_TWO_HOP, "two_hop_cont", served=True) # type: ignore[arg-type] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_seeded_typed_hop_fast_path_engages(engine: str) -> None: """The third path. It is consulted LAST, so its pin doubles as evidence the @@ -132,6 +134,7 @@ def test_seeded_typed_hop_fast_path_engages(engine: str) -> None: assert seen.get("two_hop_count") is False +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_seeded_node_lookup_fast_path_engages(engine: str) -> None: """The fourth path, consulted last: a seeded single-node pattern with a property @@ -162,6 +165,7 @@ def test_fast_paths_have_no_bare_collect(): assert not offenders, f"bare collects bypass the execution target: {offenders}" +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_fast_paths_serve_identically_under_cpu_target_context(engine: str) -> None: """The #1824 target threading must be a no-op on CPU engines: same answers, diff --git a/graphistry/tests/compute/gfql/test_gfql_latency_contract.py b/graphistry/tests/compute/gfql/test_gfql_latency_contract.py index f73c0c323c..759b47f96e 100644 --- a/graphistry/tests/compute/gfql/test_gfql_latency_contract.py +++ b/graphistry/tests/compute/gfql/test_gfql_latency_contract.py @@ -162,12 +162,14 @@ def _graph_for(graphs: Dict[str, Any], engine: str) -> Any: return graphs[engine] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine,shape", _cases(only_served=False)) def test_basic_shape_is_served_by_a_fast_path(graphs: Dict[str, Any], engine: str, shape: str) -> None: g = _graph_for(graphs, engine) assert _served(g, CYPHER[shape], engine), f"{engine}: {shape} fell off the fast path" +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine,shape", _cases(only_served=True)) def test_served_shape_costs_a_bounded_multiple_of_plain_frame_ops( graphs: Dict[str, Any], engine: str, shape: str diff --git a/graphistry/tests/compute/gfql/test_known_cross_engine_divergences.py b/graphistry/tests/compute/gfql/test_known_cross_engine_divergences.py index 286cf13800..cfc8754268 100644 --- a/graphistry/tests/compute/gfql/test_known_cross_engine_divergences.py +++ b/graphistry/tests/compute/gfql/test_known_cross_engine_divergences.py @@ -71,6 +71,7 @@ def test_1739_has_label_aggregate_on_duplicate_ids_converges(): assert out_pl == narrowed +@pytest.mark.route_engaged("cypher-fast") @polars_only @pytest.mark.xfail(strict=True, reason="#1824: fast paths serve CPU work under engine='polars-gpu'") def test_1824_polars_gpu_fast_path_serve_is_gpu_or_decline(): diff --git a/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py b/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py index ecc3b95b03..15d29fb133 100644 --- a/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py +++ b/graphistry/tests/compute/gfql/test_native_seed_lane_explain.py @@ -34,6 +34,7 @@ def _graph(engine, indexed=True): ENGINES = ["pandas", "polars", "cudf"] +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_only_lookup_served_by_the_property_index_is_explained(engine): g = _graph(engine) @@ -43,6 +44,7 @@ def test_node_only_lookup_served_by_the_property_index_is_explained(engine): assert len(g.gfql(NODE_ONLY, engine=engine, index_policy="use")._nodes) == 1 +@pytest.mark.route_engaged("native-fast", "polars-seeded") @pytest.mark.parametrize("engine", ENGINES) def test_seeded_typed_hop_served_by_the_resident_indexes_is_explained(engine): g = _graph(engine) diff --git a/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py b/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py index f33b3ce465..a8165d9fca 100644 --- a/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py +++ b/graphistry/tests/compute/gfql/test_native_seed_resolution_2027.py @@ -79,6 +79,7 @@ def spy(*a, **k): } +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("shape", list(SHAPES)) def test_lane_shapes_are_served_with_exact_parity(engine, shape): @@ -93,6 +94,7 @@ def test_lane_shapes_are_served_with_exact_parity(engine, shape): pd.testing.assert_frame_equal(_canon(fast._edges), _canon(full._edges), check_dtype=False) +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_property_index_resolves_the_seed(engine, monkeypatch): import graphistry.compute.gfql.index.bindings as bindings @@ -112,6 +114,7 @@ def spy(*a, **k): assert calls["n"] >= 1 and len(out._edges) == 1 +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("binding_first", [True, False]) def test_named_single_node_alias_layout_matches_the_full_path(engine, binding_first): @@ -127,6 +130,7 @@ def test_named_single_node_alias_layout_matches_the_full_path(engine, binding_fi assert list(fast._nodes.columns)[:2] == ["key", "p"] +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("binding_first", [True, False]) def test_named_single_node_alias_overwrites_colliding_property_like_full_path(engine, binding_first): @@ -144,6 +148,7 @@ def test_named_single_node_alias_overwrites_colliding_property_like_full_path(en pd.testing.assert_frame_equal(_canon(fast._nodes), _canon(full._nodes), check_dtype=False) +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_named_hop_aliases_overwrite_nonfinal_properties_like_full_path(engine): g = _lane_graph(engine) @@ -198,6 +203,7 @@ def test_policy_off_keeps_parity_and_uses_no_index(engine, shape): assert report["used_index"] is False and report["decision_code"] == "policy_off", report +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("shape", list(SHAPES)) def test_stale_indexes_keep_parity_and_are_not_used(engine, shape): @@ -229,6 +235,7 @@ def test_non_scalar_seed_predicates_keep_parity_without_the_index(engine, seed): assert not any(s.get("seam") in ("native_seed_lookup", "native_seeded_hop") and s.get("served") for s in steps), steps +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ENGINES) def test_duplicate_node_rows_are_answered_once_each_on_the_native_lookup(engine): """A node table that repeats a key row (a contract violation the engine tolerates): the diff --git a/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py b/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py index 8f58ff7354..99450c856d 100644 --- a/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py +++ b/graphistry/tests/compute/gfql/test_polars_native_seed_resolution.py @@ -33,6 +33,7 @@ def _graph(reverse=False, indexed=True, padding=0): return g.gfql_index_all(engine="polars").gfql_index_node_props(["id"], engine="polars") if indexed else g +@pytest.mark.route_engaged("polars-seeded") @pytest.mark.parametrize("reverse", [False, True]) @pytest.mark.parametrize("indexed", [False, True]) @pytest.mark.parametrize("seed", [{"id": 104}, {"kind": "Message"}, {"id": 105}, {"id": 999}]) @@ -73,6 +74,7 @@ def spy(*args): _NAMED_TYPED_HOP = [n({"id": 104}, name="m"), e_forward({"type": "T"}, name="e"), n({"kind": "Person"}, name="p")] +@pytest.mark.route_engaged("polars-seeded") def test_native_seeded_hop_is_served_from_the_index_and_traced(): from graphistry.compute.gfql.index import index_trace g = _graph() @@ -98,6 +100,7 @@ def test_native_seeded_hop_declines_without_a_usable_index(policy, monkeypatch): assert_frame_equal(fast._edges, full._edges) +@pytest.mark.route_engaged("polars-seeded") @pytest.mark.parametrize("single_node", [False, True]) def test_native_property_seed_uses_resident_index(single_node, monkeypatch): import graphistry.compute.gfql.index.bindings as bindings @@ -134,7 +137,7 @@ def test_stale_property_index_does_not_select_old_seed(monkeypatch): @pytest.mark.parametrize("case", ["lazy", "mixed", "varlen", "collision", "duplicate", "undirected"]) def test_native_seeded_hop_declines_unsupported_shapes(case): from graphistry.compute.ast import e_undirected - from graphistry.compute.chain_fast_paths import _try_seeded_chain_polars + from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths import _try_seeded_chain_polars g = _graph() ops = [n({"id": 104}, name="m"), e_forward({"type": "T"}), n(name="p")] if case == "lazy": diff --git a/graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py b/graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py index 750d097388..d5408dcaf2 100644 --- a/graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py +++ b/graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py @@ -120,6 +120,7 @@ def _disambiguation_frames(): return nodes, edges +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas"] + (["polars"] if HAS_POLARS else [])) def test_has_label_narrowing_skipped_when_reached_ids_unique(engine: str) -> None: """Global id collisions must NOT trigger narrowing when the REACHED ids are unique diff --git a/graphistry/tests/compute/gfql/test_rewrite_param_discard.py b/graphistry/tests/compute/gfql/test_rewrite_param_discard.py index 91941879e9..c86ffc2651 100644 --- a/graphistry/tests/compute/gfql/test_rewrite_param_discard.py +++ b/graphistry/tests/compute/gfql/test_rewrite_param_discard.py @@ -210,6 +210,7 @@ def test_indexed_bypass_table_edges_survives_a_projection(engine: str) -> None: "(index/bindings.py gate), so polars-gpu always takes the scan path", )), ]) +@pytest.mark.route_engaged("indexed-kernel") def test_indexed_bypass_still_serves_a_bare_rows(engine: str) -> None: """THE NEGATIVE SIDE: declining on a non-default `table` must not decline everything. diff --git a/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py b/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py index e168877cab..e6c27ccee5 100644 --- a/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py +++ b/graphistry/tests/compute/gfql/test_seeded_node_lookup_fastpath.py @@ -106,6 +106,7 @@ def _assert_parity(g, engine, query, path, served=True): ] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("indexed", [False, True], ids=["scan", "indexed"]) @pytest.mark.parametrize("q,label", LOOKUP_SHAPES) @@ -113,6 +114,7 @@ def test_node_lookup_engages_with_parity(engine, indexed, q, label): _assert_parity(_graph(engine, indexed), engine, q, "seeded_node_lookup") +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_lookup_matches_independent_oracle(engine): g = _graph(engine) @@ -124,6 +126,7 @@ def test_node_lookup_matches_independent_oracle(engine): assert float(got["a"].iloc[0]) == float(row["age"]) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_lookup_uses_the_property_index_when_the_seed_is_not_the_binding(engine): """The seed predicate is on a business key that is not the node binding: the @@ -227,6 +230,7 @@ def test_two_alias_projection_parity(engine, indexed, q, label): pd.testing.assert_frame_equal(_canon(fast), _canon(full)) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_two_alias_projection_engages_and_keeps_bag_multiplicity(engine): g = _graph(engine) @@ -260,6 +264,7 @@ def test_hub_seed_over_the_frontier_gate_keeps_parity(engine, indexed): pd.testing.assert_frame_equal(_canon(fast), _canon(full)) +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_seed_matching_several_nodes_projects_each_seed(engine): """A non-unique seed predicate: every seed row pairs with its own destinations.""" @@ -321,6 +326,7 @@ def _lookup_step(report): return steps[0] +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_lookup_explains_why_it_scanned(engine): q = "MATCH (p:Person {id: 7}) RETURN p.age AS age" @@ -368,6 +374,7 @@ def test_two_alias_projection_extension_dtypes_keep_parity(indexed, q, served): assert fast_path_decisions(g, q, engine="pandas").get("seeded_typed_hop") is not True +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_node_lookup_served_under_policy_off_is_not_reported_as_an_index(engine): q = "MATCH (p:Person {id: 7}) RETURN p.age AS age" @@ -381,6 +388,7 @@ def test_node_lookup_served_under_policy_off_is_not_reported_as_an_index(engine) assert off["used_index"] is False and off["decision_code"] == "policy_off", off +@pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ENGINES) def test_edge_alias_properties_engage_with_one_row_per_matched_edge(engine): g = _graph(engine) diff --git a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py index aee31b9cfd..422aa7ce7e 100644 --- a/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py +++ b/graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py @@ -20,9 +20,19 @@ import graphistry from graphistry.compute.ast import n, e_forward, e_reverse import graphistry.compute.chain as chain_mod +import graphistry.compute.chain_specializations.hotpaths as hot import graphistry.compute.gfql_unified as gfql_unified + +def _patch_index_rows(monkeypatch, name, fn): + """Patch a chain_fast_paths index helper everywhere the lanes bind it.""" + import graphistry.compute.chain_fast_paths as cfp + import graphistry.compute.chain_specializations.hotpaths as eager_hot + import graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths as polars_hot + for mod in (cfp, eager_hot, polars_hot): + monkeypatch.setattr(mod, name, fn) + def _graph(n_persons=1500, n_messages=6000, seed=0): """Message -> Person HAS_CREATOR graph (the #1755 probe shape). `age` is only on Person rows, so the concatenated node frame carries a float `age` column @@ -120,11 +130,12 @@ def test_parity_fast_vs_full(self, ops_name): full = self._run(g, ops, force_full=True) pd.testing.assert_frame_equal(_canon_nodes(fast), _canon_nodes(full)) + @pytest.mark.route_engaged("native-fast") def test_fast_path_engages_on_typed_hop(self, monkeypatch): g, P = _graph() seed = P + 42 hits = {"n": 0} - real = chain_mod._seeded_typed_hop_pandas_cudf + real = hot._seeded_typed_hop_pandas_cudf def spy(*a, **k): r = real(*a, **k) @@ -132,7 +143,7 @@ def spy(*a, **k): hits["n"] += 1 return r - monkeypatch.setattr(chain_mod, "_seeded_typed_hop_pandas_cudf", spy) + monkeypatch.setattr(hot, "_seeded_typed_hop_pandas_cudf", spy) g.gfql([n({"id": seed}), e_forward(edge_match={"type": "HAS_CREATOR"}), n({"type": "Person"})], engine="pandas") assert hits["n"] >= 1 @@ -159,11 +170,11 @@ def test_numpy_helper_declines_predicate_and_undirected(self): g, P = _graph() node, src, dst = "id", "src", "dst" # predicate (non-scalar) edge filter -> decline - assert chain_mod._seeded_typed_hop_pandas_cudf( + assert hot._seeded_typed_hop_pandas_cudf( g.materialize_nodes(), n({"id": P + 1}), n(), e_forward(edge_match={"type": gt(0)}), src, dst, node, "forward") is None # undirected -> decline - assert chain_mod._seeded_typed_hop_pandas_cudf( + assert hot._seeded_typed_hop_pandas_cudf( g.materialize_nodes(), n({"id": P + 1}), n(), e_forward(), src, dst, node, "undirected") is None @@ -180,7 +191,7 @@ def test_native_declines_and_stays_correct(self, ops_name, reason, monkeypatch): "node_predicate": [n({"id": seed}), e_forward(edge_match={"type": "HAS_CREATOR"}), n({"age": gt(0)})], }[ops_name] hits = {"n": 0} - real = chain_mod._seeded_typed_hop_pandas_cudf + real = hot._seeded_typed_hop_pandas_cudf def spy(*a, **k): r = real(*a, **k) @@ -188,7 +199,7 @@ def spy(*a, **k): hits["n"] += 1 return r - monkeypatch.setattr(chain_mod, "_seeded_typed_hop_pandas_cudf", spy) + monkeypatch.setattr(hot, "_seeded_typed_hop_pandas_cudf", spy) fast = g.gfql(ops, engine="pandas") monkeypatch.undo() chain_mod._try_chain_fast_path, saved = (lambda *a, **k: None), chain_mod._try_chain_fast_path @@ -247,6 +258,7 @@ def test_independent_oracle_values(self): got = sorted(df[id_col].tolist()) assert got == oracle, f"cypher fast path returned {got}, oracle says {oracle}" + @pytest.mark.route_engaged("cypher-fast") def test_fast_path_engages_on_seeded_return(self, monkeypatch): g, P = _graph() seed = P + 42 @@ -263,6 +275,7 @@ def spy(*a, **k): g.gfql(f"MATCH (m:Message {{id: {seed}}})-[:HAS_CREATOR]->(p:Person) RETURN p", engine="pandas") assert hits["n"] >= 1 + @pytest.mark.route_engaged("cypher-fast") def test_fast_path_engages_on_the_bag_lowering_of_a_property_return(self, monkeypatch): """A property RETURN lowers to the multiplicity-preserving ``rows(binding_ops=...)`` form. The fast path must still engage there: declining would be value-correct but @@ -305,6 +318,7 @@ def test_parallel_edges_keep_their_row_through_the_fast_path(self, engine): got = _canon_nodes(g.gfql(q, engine=engine)) assert got["pid"].tolist() == [1, 1] + @pytest.mark.route_engaged("cypher-fast") def test_cross_alias_field_projection_engages_with_parity(self): """RETURN m.id, p.age projects from BOTH aliases: one row per matched edge, the seed side looked up from the seed rows the reduction already holds.""" @@ -560,6 +574,7 @@ def test_independent_oracle_values(self): id_col = "p.id" if "p.id" in df.columns else "id" assert sorted(df[id_col].tolist()) == oracle + @pytest.mark.route_engaged("cypher-fast") def test_fast_path_engages(self, monkeypatch): gp, P, _ = self._pl_graph() seed = P + 42 @@ -618,7 +633,7 @@ def _q(self, seed): def test_lazyframe_declines_not_crashes(self): """A LazyFrame-backed graph must decline (full path), not AttributeError.""" pl = pytest.importorskip("polars") - from graphistry.compute.chain_fast_paths import _seeded_typed_return_dst_polars + from graphistry.compute.gfql.lazy.engine.polars.chain_specializations.hotpaths import _seeded_typed_return_dst_polars from graphistry.compute.ast import ASTNode, ASTEdge g, P = _graph() lazy_g = graphistry.nodes( @@ -719,6 +734,7 @@ def spy(*a, **k): assert bool(hits["n"]) == expect_engage, f"engaged={hits['n']} expected={expect_engage}" return fast + @pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_is5_shape_engages_and_matches(self, engine): if engine == "polars": @@ -761,6 +777,7 @@ def test_out_of_shape_declines_with_parity(self, q, label): ("MATCH (m:Message {id:10})-[{type:'HAS_CREATOR'}]->(p:Person) RETURN p.age AS age, m.id AS mid ORDER BY age LIMIT 1", "cross-alias order/limit"), ("MATCH (m:Message {id:10})-[{type:'HAS_CREATOR'}]->(p:Person) RETURN p.id AS a, p.id AS b", "same column twice"), ]) + @pytest.mark.route_engaged("cypher-fast") def test_canonical_projection_suffix_engages_with_parity(self, q, label): """DISTINCT / ORDER BY / SKIP / LIMIT after the lean projection are plain row-frame ops: the fast path now keeps the seeded gather and delegates the @@ -834,12 +851,14 @@ def spy(*a, **k): assert bool(hits["n"]) == expect_engage, f"engaged={hits['n']} expected={expect_engage}" return fast, full + @pytest.mark.route_engaged("cypher-fast") def test_pandas_int_bool_dtype_parity(self): fast, full = self._fast_and_full(self._typed_graph(), "pandas", self.Q) pd.testing.assert_frame_equal(_canon_nodes(fast), _canon_nodes(full)) dt = dict(zip(fast._nodes.columns, map(str, fast._nodes.dtypes))) assert dt == {"pid": "int64", "a": "float64", "f": "object"} + @pytest.mark.route_engaged("cypher-fast") def test_polars_int_bool_dtype_parity(self): pytest.importorskip("polars") fast, full = self._fast_and_full(self._pl_graph(), "polars", self.Q) @@ -859,6 +878,7 @@ def test_pandas_datetime_property_declines(self): fast, full = self._fast_and_full(self._typed_graph(), "pandas", q, expect_engage=False) pd.testing.assert_frame_equal(_canon_nodes(fast), _canon_nodes(full)) + @pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_edges_empty_frame_not_none(self, engine): g = self._typed_graph() if engine == "pandas" else self._pl_graph() @@ -885,6 +905,7 @@ def test_engine_mismatch_declines(self): assert type(fast2._nodes).__module__ == type(full2._nodes).__module__ pd.testing.assert_frame_equal(_canon_nodes(fast2), _canon_nodes(full2)) + @pytest.mark.route_engaged("cypher-fast") def test_string_dtype_property_engages_and_matches(self): """pandas StringDtype (explicit 'string' on pandas 2; the DEFAULT str dtype on pandas>=3) passes through the pivot unchanged on both versions — @@ -940,13 +961,14 @@ def spy(*a, **k): out = on(*a, **k) counts["n"] += out is not None return out - monkeypatch.setattr(cfp, "_index_node_rows", spy) + _patch_index_rows(monkeypatch, "_index_node_rows", spy) res = g.gfql(q, engine=engine) - monkeypatch.setattr(cfp, "_index_node_rows", on) + _patch_index_rows(monkeypatch, "_index_node_rows", on) return res, counts["n"] Q = "MATCH (m {id: 33})-[:KNOWS]->(p) RETURN p" + @pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_indexed_parity_and_engagement(self, engine, monkeypatch): if engine == "polars": @@ -988,7 +1010,7 @@ def test_stale_index_declines_to_scan(self, monkeypatch): g2 = g.edges(edf2, "src", "dst") # stale registry rides along counts = {"n": 0} oe = cfp._index_edge_rows - monkeypatch.setattr(cfp, "_index_edge_rows", + _patch_index_rows(monkeypatch, "_index_edge_rows", lambda *a, **k: (counts.__setitem__("n", counts["n"] + 1), oe(*a, **k))[1]) got = g2.gfql(self.Q, engine="pandas") plain = graphistry.nodes(ndf, "id").edges(edf2, "src", "dst").gfql(self.Q, engine="pandas") @@ -996,6 +1018,7 @@ def test_stale_index_declines_to_scan(self, monkeypatch): pd.testing.assert_frame_equal(self._canon(got), self._canon(plain)) + @pytest.mark.route_engaged("native-fast") def test_native_chain_hop_indexed_parity_forward_and_reverse(self, monkeypatch): """M1 pin: the native-chain hop helper's indexed branch (only reachable via chain ops, never Cypher) — forward (EDGE_OUT_ADJ) and reverse (EDGE_IN_ADJ), @@ -1015,13 +1038,14 @@ def spy(*a, **k): out = oe(*a, **k) serves["n"] += out is not None return out - monkeypatch.setattr(cfp, "_index_edge_rows", spy) + _patch_index_rows(monkeypatch, "_index_edge_rows", spy) got = mk().gfql_index_all().gfql(ops, engine="pandas") - monkeypatch.setattr(cfp, "_index_edge_rows", oe) + _patch_index_rows(monkeypatch, "_index_edge_rows", oe) assert serves["n"] > 0, f"indexed hop branch did not serve for {ops[1].direction}" plain = mk().gfql(ops, engine="pandas") pd.testing.assert_frame_equal(self._canon(got), self._canon(plain)) + @pytest.mark.route_engaged("cypher-fast") def test_uint64_int64_id_mix_declines_not_collapses(self, monkeypatch): """B1 pin: int64<->uint64 promotes to float64, which collapses ids >= 2**53 into false matches; the gate must DECLINE (scan path compares exactly).""" @@ -1040,9 +1064,9 @@ def spy(*a, **k): out = on(*a, **k) serves["n"] += out is not None return out - monkeypatch.setattr(cfp, "_index_node_rows", spy) + _patch_index_rows(monkeypatch, "_index_node_rows", spy) got = g.gfql(q, engine="pandas") - monkeypatch.setattr(cfp, "_index_node_rows", on) + _patch_index_rows(monkeypatch, "_index_node_rows", on) plain = graphistry.nodes(ndf, "id").edges(edf, "src", "dst").gfql(q, engine="pandas") pd.testing.assert_frame_equal(self._canon(got), self._canon(plain)) @@ -1075,6 +1099,7 @@ def med(g): indexed = med(graphistry.nodes(ndf, "id").edges(edf, "src", "dst").gfql_index_all()) assert indexed * 1.5 < scan, f"indexed {indexed*1e3:.2f}ms not >=1.5x faster than scan {scan*1e3:.2f}ms" + @pytest.mark.route_engaged("cypher-fast") @pytest.mark.parametrize("engine", ["pandas", "polars"]) def test_property_seeded_engages_adjacency_index(self, engine, monkeypatch): """Decoupled-index pin (the SF1-harness/LDBC pattern): the graph binds a @@ -1107,8 +1132,8 @@ def spy(*a, **k): out = oe(*a, **k) serves["e"] += out is not None return out - monkeypatch.setattr(cfp, "_index_edge_rows", spy) + _patch_index_rows(monkeypatch, "_index_edge_rows", spy) indexed = mk().gfql_index_all(engine=engine).gfql(q, engine=engine) - monkeypatch.setattr(cfp, "_index_edge_rows", oe) + _patch_index_rows(monkeypatch, "_index_edge_rows", oe) assert serves["e"] > 0, "adjacency index did not serve the property-seeded lookup" pd.testing.assert_frame_equal(self._canon(indexed), self._canon(plain)) diff --git a/graphistry/tests/compute/test_chain.py b/graphistry/tests/compute/test_chain.py index 2450203124..df1b8dc955 100644 --- a/graphistry/tests/compute/test_chain.py +++ b/graphistry/tests/compute/test_chain.py @@ -5,6 +5,7 @@ import pytest from graphistry.compute.ast import ASTEdgeUndirected, ASTNode, ASTEdge, ASTObject, n, e, e_undirected, e_forward, e_reverse +from graphistry.tests.compute.gfql.routes.registry import Frames, register from graphistry.compute.chain import Chain, _try_chain_fast_path from graphistry.compute.typing import DataFrameT from graphistry.compute.predicates.is_in import IsIn, is_in @@ -652,9 +653,14 @@ def _cudf_or_skip(): return pytest.importorskip("cudf") +_FAST_FRAMES = Frames( + pd.DataFrame({'v': [0, 1, 2, 3, 4], 'attr': [10, 20, 30, 40, 50]}), + pd.DataFrame({'s': [0, 1, 2, 3, 0], 'd': [1, 2, 3, 4, 2], 'w': [5, 6, 7, 8, 9]}), + 'v', 's', 'd') + + def _fast_graph(engine): - nodes = pd.DataFrame({'v': [0, 1, 2, 3, 4], 'attr': [10, 20, 30, 40, 50]}) - edges = pd.DataFrame({'s': [0, 1, 2, 3, 0], 'd': [1, 2, 3, 4, 2], 'w': [5, 6, 7, 8, 9]}) + nodes, edges = _FAST_FRAMES.nodes, _FAST_FRAMES.edges if engine == "cudf": cudf = _cudf_or_skip() nodes = cudf.from_pandas(nodes) @@ -675,7 +681,7 @@ def topd(df): # shapes that ARE accelerated by the fast path -_FAST_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_FAST_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.fast", [ ("node_only", lambda: [n()]), ("node_filter", lambda: [n({'attr': 20})]), ("node_pred", lambda: [n({'attr': is_in([10, 30])})]), @@ -702,10 +708,10 @@ def topd(df): ("named_all_fwd", lambda: [n(name='x'), e_forward(hops=1, name='r'), n(name='y')]), ("named_all_rev", lambda: [n(name='x'), e_reverse(hops=1, name='r'), n(name='y')]), ("named_filtered", lambda: [n({'attr': 10}, name='x'), e_forward(hops=1), n(name='y')]), -] +], _FAST_FRAMES, tags=("native-fast",)) # shapes that BYPASS the fast path (still must be correct via the full path) -_BYPASS_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_BYPASS_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.bypass", [ ("hops_2", lambda: [n(), e_forward(hops=2), n()]), ("filtered_undirected", lambda: [n({'attr': 10}), e_undirected(hops=1), n({'attr': 30})]), # Named + undirected STAYS a bypass: an undirected edge makes a node reachable as @@ -716,7 +722,7 @@ def topd(df): # arrival side. Must bypass the fast path (regression guard for the prune gate). ("prune_endpoints_fwd", lambda: [n(), e_forward(hops=1, prune_to_endpoints=True), n()]), ("prune_endpoints_rev", lambda: [n(), e_reverse(hops=1, prune_to_endpoints=True), n()]), -] +], _FAST_FRAMES, tags=("native-fast-bypass",), row_tags={"prune_endpoints_fwd": ("#2053",), "prune_endpoints_rev": ("#2053",)}) _CUDF_26_DIVERGENT = {"prune_endpoints_fwd", "prune_endpoints_rev"} # graphistry/pygraphistry#2043 @@ -750,7 +756,7 @@ def test_fast_path_differential_parity_vs_full_path(engine, label, build, reques # Named shapes whose ALIAS FLAG COLUMNS (not merely node/edge sets) must match the full # path. `_setsig` above compares ids only, so it cannot see a wrong alias tag. -_NAMED_ALIAS_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_NAMED_ALIAS_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.named_alias", [ ("src_only", lambda: [n(name='x'), e_forward(hops=1), n()]), ("dst_only", lambda: [n(), e_forward(hops=1), n(name='y')]), ("edge_only", lambda: [n(), e_forward(hops=1, name='r'), n()]), @@ -762,12 +768,13 @@ def test_fast_path_differential_parity_vs_full_path(engine, label, build, reques # DEAD END: attr==50 is node 4, which has no outgoing edge. The tag keys on the # SURVIVING EDGES, so the alias must come back False/empty rather than True. ("dead_end_seed", lambda: [n({'attr': 50}, name='x'), e_forward(hops=1, name='r'), n(name='y')]), -] +], _FAST_FRAMES, tags=("alias",)) @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @pytest.mark.parametrize("label,build", _NAMED_ALIAS_SHAPES, ids=[s[0] for s in _NAMED_ALIAS_SHAPES]) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_alias_columns_match_full_path(engine, label, build): """The capability this fast-path extension actually adds: when the traversal is served without the BFS, the alias flag columns `combine_steps` would have merged in @@ -824,17 +831,18 @@ def _assert_full_frame_value_parity(fast: DataFrameT, full: DataFrameT, # Named served shapes for FULL-FRAME parity. `_setsig` compares id sets and the flags # test compares alias columns, so before this NO test compared the carried DATA columns # ('attr', 'w') of a named served result against the full path. -_NAMED_VALUE_PARITY_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_NAMED_VALUE_PARITY_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.named_value_parity", [ ("all_forward", lambda: [n(name='x'), e_forward(hops=1, name='r'), n(name='y')]), ("all_reverse", lambda: [n(name='x'), e_reverse(hops=1, name='r'), n(name='y')]), ("seed_filtered", lambda: [n({'attr': 10}, name='x'), e_forward(hops=1, name='r'), n(name='y')]), ("edge_match", lambda: [n(name='x'), e_forward(hops=1, edge_match={'w': 5}, name='r'), n(name='y')]), -] +], _FAST_FRAMES, tags=("alias", "values")) @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @pytest.mark.parametrize("label,build", _NAMED_VALUE_PARITY_SHAPES, ids=[s[0] for s in _NAMED_VALUE_PARITY_SHAPES]) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_full_frame_value_parity(engine, label, build): """POSITIVE, whole-frame: a named served result must carry the same VALUES as the full path on EVERY column — ids, data columns, and alias flags — not just the id @@ -865,18 +873,19 @@ def test_fast_path_named_full_frame_value_parity(engine, label, build): # cardinality, so these all engage the fast path — and an empty answer must come back # as the right empty SHAPE (alias columns present, zero rows), not a throw and not a # missing-column frame. -_NAMED_EMPTY_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_NAMED_EMPTY_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.named_empty", [ # seed filter matches no node at all (distinct from dead_end_seed, which matches # a node that has no surviving edge) ("zero_seed", lambda: [n({'attr': 999}, name='x'), e_forward(hops=1, name='r'), n(name='y')]), ("zero_dst", lambda: [n(name='x'), e_forward(hops=1, name='r'), n({'attr': 999}, name='y')]), ("zero_edge_match", lambda: [n(name='x'), e_forward(hops=1, edge_match={'w': 999}, name='r'), n(name='y')]), -] +], _FAST_FRAMES, tags=("alias", "empty")) @pytest.mark.parametrize("engine", ["pandas", "cudf"]) @pytest.mark.parametrize("label,build", _NAMED_EMPTY_SHAPES, ids=[s[0] for s in _NAMED_EMPTY_SHAPES]) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_empty_result_matches_full_path(engine, label, build): """POSITIVE boundary: named patterns matching ZERO rows are still served, and the empty result must be shape-identical to the full path — same columns INCLUDING the @@ -898,6 +907,7 @@ def test_fast_path_named_empty_result_matches_full_path(engine, label, build): assert len(un) == 0 and len(ue) == 0 +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_zero_edge_graph_matches_full_path(): """POSITIVE boundary: a graph with an EMPTY edge table. The named pattern is served, and both lanes must agree on the all-empty answer with alias columns present.""" @@ -953,6 +963,7 @@ def test_fast_path_cross_type_alias_share_declines_and_matches(): _assert_full_frame_value_parity(default_route._edges, policy_route._edges, ['s', 'd']) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_is_served_with_a_valid_resident_index(): """A NAMED pattern with BOTH resident indexes validly covering the directed hop is served by the chain fast path: by the time it runs, the indexed kernel has already @@ -976,6 +987,7 @@ def test_fast_path_named_is_served_with_a_valid_resident_index(): _assert_full_frame_value_parity(served._edges, full._edges, ['s', 'd']) +@pytest.mark.route_engaged("native-fast") def test_fast_path_named_datetime_categorical_columns_ride_along(): """POSITIVE dtype edge: datetime64 and categorical NODE columns must ride through the served named lane unchanged — same values as the full path, dtypes preserved @@ -1018,7 +1030,7 @@ def test_fast_path_named_datetime_categorical_columns_ride_along(): # overwrite/raise behavior alone. The FROM-side binding columns and the node-id # binding are excluded here: those wrong-served (diverged) before, are now GATED to # decline, and are pinned by the two regression tests below. -_ALIAS_SHADOW_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = [ +_ALIAS_SHADOW_SHAPES: List[Tuple[str, Callable[[], List[ASTObject]]]] = register("test_chain.alias_shadow", [ ("node_alias_shadows_node_data_col", lambda: [n(name='attr'), e_forward(hops=1), n()]), ("edge_alias_shadows_edge_data_col", lambda: [n(), e_forward(hops=1, name='w'), n()]), # edge aliases named like the source/destination/edge-id bindings are rejected before @@ -1026,7 +1038,7 @@ def test_fast_path_named_datetime_categorical_columns_ride_along(): # cross-frame names are NOT collisions: nodes have no 'w', edges have no 'v' ("node_alias_named_like_edge_col", lambda: [n(name='w'), e_forward(hops=1), n()]), ("edge_alias_named_like_node_id", lambda: [n(), e_forward(hops=1, name='v'), n()]), -] +], _FAST_FRAMES, tags=("alias-collision",)) @pytest.mark.parametrize("label,build", _ALIAS_SHADOW_SHAPES, @@ -1119,6 +1131,7 @@ def test_fast_path_alias_colliding_with_node_id_binding_matches_full_path(build) _assert_full_frame_value_parity(fast._edges, full._edges, ['s', 'd']) +@pytest.mark.route_engaged("native-fast") @pytest.mark.parametrize("engine", ["pandas", "cudf"]) def test_fast_path_preserves_int_node_dtypes(engine): """Documented behavior change: the 1-hop fast path PRESERVES node-attribute diff --git a/graphistry/tests/compute/test_chain_alias_column_collision.py b/graphistry/tests/compute/test_chain_alias_column_collision.py index 3c0c69efb5..42e9a92f13 100644 --- a/graphistry/tests/compute/test_chain_alias_column_collision.py +++ b/graphistry/tests/compute/test_chain_alias_column_collision.py @@ -13,11 +13,13 @@ import graphistry from graphistry.compute.ast import e_forward, e_reverse, e_undirected, n +from graphistry.tests.compute.gfql.routes.registry import Frames, register NODES = pd.DataFrame({"key": [1, 2, 3, 4], "id": [10, 20, 30, 40], "type": ["p", "p", "m", "m"], "w": [1, 2, 3, 4]}) EDGES = pd.DataFrame({"s": [3, 3, 4, 1], "d": [1, 2, 1, 4], "type": ["HAS_CREATOR", "OTHER", "HAS_CREATOR", "OTHER"], "eid": [100, 101, 102, 103], "w": [5, 6, 7, 8]}) ENGINES = ["pandas", "cudf", "polars"] +FRAMES = Frames(NODES, EDGES, "key", "s", "d", "eid") def _graph(engine, indexed): @@ -62,6 +64,8 @@ def topd(x): "seed, edge and destination aliases all collide": [n({"id": 30}, name="id"), e_forward({"type": "HAS_CREATOR"}, name="type"), n({"type": "p"}, name="type")], } +register("collision.served", [(k, (lambda v=v: list(v))) for k, v in SERVED.items()], FRAMES, tags=("alias-collision", "#2039")) + @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("indexed", [False, True], ids=["scan", "indexed"]) @@ -81,6 +85,8 @@ def test_single_hop_collisions_match_the_pandas_full_path(engine, indexed, shape "edge alias = filtered column, to_fixed_point": [n({"id": 30}, name="m"), e_forward({"type": "HAS_CREATOR"}, to_fixed_point=True, name="type"), n(name="p")], } +register("collision.multi_hop", [(k, (lambda v=v: list(v))) for k, v in MULTI_HOP.items()], FRAMES, tags=("alias-collision", "#2049")) + @pytest.mark.parametrize("engine", ENGINES) @pytest.mark.parametrize("shape", list(MULTI_HOP)) diff --git a/graphistry/tests/conftest.py b/graphistry/tests/conftest.py new file mode 100644 index 0000000000..221eac3707 --- /dev/null +++ b/graphistry/tests/conftest.py @@ -0,0 +1,46 @@ +"""Route forcing for test amplification. + +``GFQL_ROUTES_OFF=[,...]`` makes the named hot paths decline for the whole +session, so every existing test — written against one specialization or against the +general path — is replayed through the other routes. Failures that appear only under a +mode are the cases where that route and the rest disagree. + +Routes: native-fast (pandas/cuDF chain fast path), polars-seeded (polars seeded lane), +polars-plain (polars plain single-hop branches), index-hop (hop() index path), +indexed-kernel (indexed connected-bindings kernel), cypher-fast (the four Cypher lanes). + +A test that asserts a route SERVES (trace, latency, served-by spy) is an engagement pin, not +a result pin: mark it ``@pytest.mark.route_engaged("", ...)`` and it is skipped when +one of its routes is off, so the replay reports result divergences only +(``bin/test-routes-off.sh``). +""" +import os + +import pytest + + +def _routes_off(): + raw = os.environ.get("GFQL_ROUTES_OFF", "") + return {r.strip() for r in raw.split(",") if r.strip()} + + +def pytest_collection_modifyitems(config, items): + off = _routes_off() + if not off: + return + for item in items: + for mark in item.iter_markers("route_engaged"): + hit = off & set(mark.args) + if hit: + item.add_marker(pytest.mark.skip(reason="engagement pin for route(s) off: " + ",".join(sorted(hit)))) + + +@pytest.fixture(autouse=True, scope="session") +def _gfql_routes_off(): + routes = _routes_off() + if not routes: + yield + return + from graphistry.tests.compute.gfql.routes.switch import routes_off + with routes_off(routes): + yield diff --git a/pytest.ini b/pytest.ini index 145c234cc9..d6622e0ad6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,4 +9,5 @@ filterwarnings = ignore::pytest.PytestCacheWarning markers = tier2: optional/extended test coverage + route_engaged(*routes): the test asserts that a GFQL hot path serves (trace, latency, served-by); skipped when GFQL_ROUTES_OFF names one of its routes #log_cli = True