From 1832a58b84391aa20310556c3e234be9804efaf3 Mon Sep 17 00:00:00 2001 From: Neonsy <118444485+Neonsy@users.noreply.github.com> Date: Thu, 20 Aug 2026 04:05:57 +0200 Subject: [PATCH 1/4] fix: preserve JavaScript graph relationships --- README.md | 9 + graphify/__main__.py | 33 +-- graphify/analyze.py | 9 +- graphify/build.py | 169 ++++++++++++-- graphify/cli.py | 149 ++++++++++--- graphify/cluster.py | 13 +- graphify/detect.py | 5 + graphify/diagnostics.py | 42 +++- graphify/export.py | 6 +- graphify/extract.py | 118 ++++++++++ graphify/extractors/resolution.py | 257 ++++++++++++++++++---- graphify/paths.py | 99 ++++++++- graphify/watch.py | 90 ++++++-- tests/test_build.py | 124 ++++++++++- tests/test_endpoint_classification.py | 79 +++++++ tests/test_export.py | 16 ++ tests/test_extract_code_only_cli.py | 90 ++++++++ tests/test_import_extension_resolution.py | 35 +++ tests/test_js_import_resolution.py | 99 +++++++++ tests/test_paths.py | 57 +++++ tests/test_watch.py | 24 ++ 21 files changed, 1385 insertions(+), 138 deletions(-) create mode 100644 tests/test_endpoint_classification.py diff --git a/README.md b/README.md index 0c14d207c9..0e3e26ee2f 100644 --- a/README.md +++ b/README.md @@ -746,6 +746,7 @@ graphify extract ./docs --google-workspace # export .gdoc/.gsheet/.gslides v graphify extract ./src --no-gitignore # include git-ignored source; still honor .graphifyignore graphify extract ./docs --mode deep # richer semantic extraction via extended system prompt graphify extract ./docs --no-cluster # raw extraction only, skip clustering +graphify extract ./src --code-only --multigraph --out D:/graphs/my-project # preserve parallel edges outside the repo graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) graphify extract ./docs --force # overwrite graph.json even if new graph has fewer nodes (use after refactors or to clear ghost duplicates) graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) @@ -753,6 +754,14 @@ graphify extract ./src --no-dedup # skip entity dedup; on an increm graphify extract ./docs --global --as myrepo # extract and register into the cross-project global graph GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # raise output cap for dense corpora +`--multigraph` stores the authoritative graph as a directed `MultiDiGraph`, so +distinct relations and source locations between the same ordered node pair are +not overwritten. `--out` can point anywhere; Graphify remembers an explicit +selection per scanned source root under `~/.graphify/output-roots/`. Later +`extract`, `update`, watch, query, and export commands reuse that location, and +another explicit `--out` replaces it. Build settings such as `--multigraph` are +saved with the graph. The scanned repository remains unchanged. + graphify export callflow-html # graphify-out/-callflow.html graphify export callflow-html --max-sections 8 # cap generated architecture sections graphify export callflow-html --output docs/arch.html diff --git a/graphify/__main__.py b/graphify/__main__.py index 155501a98d..deb38bb84a 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -510,12 +510,12 @@ def _run_cli() -> None: print(" uninstall remove graphify from all detected platforms in one shot") print(" --purge also delete graphify-out/ directory") print(" path \"A\" \"B\" shortest path between two nodes in graph.json") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --graph path to graph.json (default: configured source output)") print(" explain \"X\" plain-language explanation of a node and its neighbors") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --graph path to graph.json (default: configured source output)") print(" diagnose multigraph report same-endpoint edge collapse risk in graph.json") print(" --graph path to graph/extraction JSON") - print(" (default graphify-out/graph.json)") + print(" (default: configured source output)") print(" --json emit machine-readable JSON") print(" --max-examples N max same-endpoint examples to print (default 5)") print(" --directed force directed post-build simulation") @@ -538,9 +538,10 @@ def _run_cli() -> None: print(" --force overwrite graph.json even if the rebuild has fewer nodes") print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") print(" --no-cluster skip clustering, write raw extraction only") + print(" --multigraph enable and persist parallel directed edge storage") print(" cluster-only rerun clustering on an existing graph.json and regenerate report") print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") - print(" --graph path to graph.json (default /graphify-out/graph.json)") + print(" --graph path to graph.json (default: configured source output)") print(" --no-label keep 'Community N' placeholders (skip LLM community naming)") print(" --backend= backend to use for community naming (default: auto-detect)") print(" --model= model to use for community naming") @@ -556,16 +557,16 @@ def _run_cli() -> None: print(" --dfs use depth-first instead of breadth-first") print(" --context C explicit edge-context filter (repeatable)") print(" --budget N cap output at N tokens (default 2000)") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --graph path to graph.json (default: configured source output)") print(" affected \"X\" reverse traversal to find nodes impacted by X") print(" --relation R edge relation to traverse in reverse (repeatable)") print(" --depth N reverse traversal depth (default 2)") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --graph path to graph.json (default: configured source output)") print(" god-nodes list the most connected nodes (architectural hubs)") print(" --top N how many to show (default 10)") - print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --graph path to graph.json (default: configured source output)") print(" --json emit JSON instead of text") - print(" save-result save a Q&A result to graphify-out/memory/ for graph feedback loop") + print(" save-result save a Q&A result to the configured output memory") print(" --question Q the question asked") print(" --answer A the answer to save") print( @@ -574,10 +575,10 @@ def _run_cli() -> None: print(" --nodes N1 N2 ... source node labels cited in the answer") print(" --outcome O work-memory signal: useful|dead_end|corrected") print(" --correction TEXT what the right answer was (pairs with --outcome corrected)") - print(" --memory-dir DIR memory directory (default: graphify-out/memory)") - print(" reflect aggregate graphify-out/memory/ outcomes into a deterministic lessons doc") - print(" --memory-dir DIR memory directory (default: graphify-out/memory)") - print(" --out FILE output path (default: graphify-out/reflections/LESSONS.md)") + print(" --memory-dir DIR memory directory (default: configured source output)") + print(" reflect aggregate configured output memory into a deterministic lessons doc") + print(" --memory-dir DIR memory directory (default: configured source output)") + print(" --out FILE output path (default: configured source output)") print(" --graph PATH graph.json, for community grouping + dropping stale nodes (optional)") print(" --analysis PATH .graphify_analysis.json (optional, auto-detected next to --graph)") print(" --labels PATH .graphify_labels.json (optional, auto-detected next to --graph)") @@ -585,8 +586,8 @@ def _run_cli() -> None: print(" --min-corroboration N distinct useful results to prefer a node (default 2)") print(" check-update check needs_update flag and notify if semantic re-extraction is pending (cron-safe)") print(" tree emit a D3 v7 collapsible-tree HTML for graph.json") - print(" --graph PATH path to graph.json (default graphify-out/graph.json)") - print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") + print(" --graph PATH path to graph.json (default: configured source output)") + print(" --output HTML output path (default: configured source output)") print(" --root PATH filesystem root for the hierarchy") print(" --max-children N cap children per node (default 200)") print(" --top-k-edges N per-symbol outbound edges in inspector (default 12)") @@ -606,10 +607,12 @@ def _run_cli() -> None: print(" --token-budget N per-chunk token cap for semantic extraction (default: 60000)") print(" --max-concurrency N parallel semantic chunks in flight (default: 4; set 1 for local LLMs)") print(" --api-timeout S per-request timeout in seconds for the LLM client (default: 600)") - print(" --out DIR, --output DIR output dir (default: ); writes /graphify-out/") + print(" --out DIR, --output DIR output dir; remembered per source root") + print(" writes /graphify-out/ (default: )") print(" --google-workspace export .gdoc/.gsheet/.gslides shortcuts via gws before extraction") print(" --no-gitignore ignore .gitignore and .git/info/exclude (prioritizes .graphifyignore)") print(" --no-cluster skip clustering, write raw extraction only") + print(" --multigraph preserve parallel directed edge records (persisted per output)") print(" --code-only index code (local AST, no API key) and skip doc/paper/image files") print(" --postgres DSN extract schema from a live PostgreSQL database") print(" maps tables, views, functions + FK relationships;") diff --git a/graphify/analyze.py b/graphify/analyze.py index 0707e2be78..a112c692e9 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -3,7 +3,7 @@ from pathlib import Path import networkx as nx -from graphify.build import edge_data +from graphify.build import analysis_projection, edge_data # Builtin/mock names that can appear as annotation-derived nodes in pre-existing # graphs. Excluded from god-node ranking so they don't displace real abstractions @@ -112,7 +112,8 @@ def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: File-level hub nodes are excluded: they accumulate import/contains edges mechanically and don't represent meaningful architectural abstractions. """ - degree = dict(G.degree()) + topology = analysis_projection(G) + degree = {node: topology.degree(node) for node in G.nodes} sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True) result = [] for node_id, deg in sorted_nodes: @@ -289,7 +290,7 @@ def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: Each result includes a 'why' field explaining what makes it non-obvious. """ node_community = _node_community_map(communities) - degrees = dict(G.degree()) + degrees = dict(analysis_projection(G).degree()) candidates = [] for u, v, data in G.edges(data=True): @@ -355,7 +356,7 @@ def _cross_community_surprises( return [] if G.number_of_nodes() > 5000: return [] - betweenness = nx.edge_betweenness_centrality(G) + betweenness = nx.edge_betweenness_centrality(analysis_projection(G)) top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n] result = [] for (u, v), score in top_edges: diff --git a/graphify/build.py b/graphify/build.py index 8efdcbd6e2..3e8775d542 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -21,6 +21,7 @@ # before any graph construction happens. # from __future__ import annotations +import hashlib import json import math import os @@ -552,6 +553,64 @@ def edge_datas(G: nx.Graph, u: str, v: str) -> list[dict]: return [raw] +def _canonical_json_value(value: object) -> object: + """Return a JSON-stable value for persistent edge identity.""" + if isinstance(value, dict): + return { + str(key): _canonical_json_value(item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + } + if isinstance(value, (list, tuple)): + return [_canonical_json_value(item) for item in value] + if isinstance(value, set): + return sorted( + (_canonical_json_value(item) for item in value), + key=lambda item: json.dumps(item, sort_keys=True, default=str), + ) + if value is None or isinstance(value, (str, int, float, bool)): + return value + return str(value) + + +def canonical_edge_key(source: str, target: str, attrs: dict) -> str: + """Return a stable identity for one directed relationship record.""" + semantic = { + key: value + for key, value in attrs.items() + if key not in {"key", "occurrence_count", "_src", "_tgt"} + } + encoded = json.dumps( + { + "source": source, + "target": target, + "attributes": _canonical_json_value(semantic), + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ).encode("utf-8") + return "edge_" + hashlib.sha256(encoded).hexdigest() + + +def analysis_projection(G: nx.Graph) -> nx.Graph: + """Return a simple undirected topology without mutating the canonical graph.""" + projected = nx.Graph() + projected.add_nodes_from( + (node, dict(attrs)) + for node, attrs in sorted(G.nodes(data=True), key=lambda row: str(row[0])) + ) + pairs = { + (source, target) if str(source) <= str(target) else (target, source) + for source, target in G.edges() + } + projected.add_edges_from( + (source, target, {"weight": 1.0}) + for source, target in sorted(pairs, key=lambda pair: (str(pair[0]), str(pair[1]))) + ) + return projected + + def dedupe_nodes(nodes: list[dict]) -> list[dict]: """Collapse nodes sharing an ``id``, last-writer-wins on attributes. @@ -795,14 +854,26 @@ def _doc_twin_remap(nodes: list) -> dict[str, str]: return remap -def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: +def build_from_json( + extraction: dict, + *, + directed: bool = False, + multigraph: bool = False, + root: str | Path | None = None, +) -> nx.Graph: """Build a NetworkX graph from an extraction dict. + multigraph=True produces a MultiDiGraph and implies directed=True. directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. root: if given, absolute source_file paths from semantic subagents are made relative to root so all nodes share a consistent path key (#932). """ + if multigraph: + from graphify.multigraph_compat import require_multigraph_capabilities + + require_multigraph_capabilities() + directed = True _root = str(Path(root).resolve()) if root else None # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. if "edges" not in extraction and "links" in extraction: @@ -952,7 +1023,13 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat _doc_remap.get(n, n) if _hashable(n) else n for n in he["nodes"] ] - G: nx.Graph = nx.DiGraph() if directed else nx.Graph() + G: nx.Graph + if multigraph: + G = nx.MultiDiGraph() + elif directed: + G = nx.DiGraph() + else: + G = nx.Graph() for node in extraction.get("nodes", []): # Skip dict nodes with a missing or non-hashable id (e.g. a list emitted # by a buggy LLM extraction) so NetworkX add_node never raises @@ -1132,6 +1209,8 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat str(e.get("source", e.get("from", ""))), str(e.get("target", e.get("to", ""))), str(e.get("relation", "")), + json.dumps(e, sort_keys=True, ensure_ascii=False, default=str) + if multigraph else "", ), ): if "source" not in edge and "from" in edge: @@ -1181,7 +1260,11 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # strings, NaN/inf, negatives — while numeric strings coerce cleanly. # Repair (not drop) the key so graph.json round-trips a clean value and a # cluster-only/--update reload never re-ingests the null. - attrs = {k: v for k, v in edge.items() if k not in ("source", "target", "target_file", "local_alias")} + attrs = { + k: v + for k, v in edge.items() + if k not in ("source", "target", "target_file", "local_alias", "key") + } for _num_key in ("weight", "confidence_score"): if _num_key in attrs: try: @@ -1265,7 +1348,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # entirely. Alphabetical order carries no meaning; keeping the specific # fact does. The reverse (specific arriving after generic) still # overwrites, so the outcome no longer depends on edge order at all. - if G.has_edge(src, tgt): + if not multigraph and G.has_edge(src, tgt): existing_rel = edge_data(G, src, tgt).get("relation") if ( attrs.get("relation") in _GENERIC_RELATIONS @@ -1273,7 +1356,23 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat and existing_rel not in _GENERIC_RELATIONS ): continue - G.add_edge(src, tgt, **attrs) + if multigraph: + edge_key = canonical_edge_key(str(src), str(tgt), attrs) + incoming_count = attrs.pop("occurrence_count", 1) + try: + incoming_count = max(1, int(incoming_count)) + except (TypeError, ValueError): + incoming_count = 1 + if G.has_edge(src, tgt, edge_key): + existing = G[src][tgt][edge_key] + existing["occurrence_count"] = ( + int(existing.get("occurrence_count", 1)) + incoming_count + ) + else: + attrs["occurrence_count"] = incoming_count + G.add_edge(src, tgt, key=edge_key, **attrs) + else: + G.add_edge(src, tgt, **attrs) hyperedges = extraction.get("hyperedges", []) if hyperedges: # Relativize hyperedge source_file the same way nodes and edges are @@ -1341,12 +1440,14 @@ def build( extractions: list[dict], *, directed: bool = False, + multigraph: bool = False, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, ) -> nx.Graph: """Merge multiple extraction results into one graph. + multigraph=True produces a MultiDiGraph and implies directed=True. directed=True produces a DiGraph that preserves edge direction (source→target). directed=False (default) produces an undirected Graph for backward compatibility. dedup=True (default) runs entity deduplication before building the graph. @@ -1387,7 +1488,12 @@ def build( # survivor rewiring the edges get (#2805). hyperedges=combined.get("hyperedges"), ) - return build_from_json(combined, directed=directed, root=root) + return build_from_json( + combined, + directed=directed, + multigraph=multigraph, + root=root, + ) def _norm_label(label: str | None) -> str: @@ -1452,8 +1558,8 @@ def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dic return deduped_nodes, deduped_edges -def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | None": - """Load (nodes, edges, hyperedges, directed) from an existing graph.json for +def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool, bool] | None": + """Load (nodes, edges, hyperedges, directed, multigraph) from graph.json for an incremental merge, accepting both the ``links`` and ``edges`` spellings. Reads the JSON directly instead of going through node_link_graph(). @@ -1499,6 +1605,7 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N edges, list(data.get("hyperedges", [])), bool(data.get("directed", False)), + bool(data.get("multigraph", False)), ) @@ -1538,7 +1645,7 @@ def merge_raw_extraction( loaded = _load_existing_graph(graph_path) if loaded is None: return new - existing_nodes, existing_edges, existing_hyperedges, _ = loaded + existing_nodes, existing_edges, existing_hyperedges, _, _ = loaded _eff_root = ( str(Path(root).resolve()) if root is not None @@ -1629,6 +1736,7 @@ def build_merge( prune_sources: list[str] | None = None, *, directed: bool | None = None, + multigraph: bool | None = None, dedup: bool = True, dedup_llm_backend: str | None = None, root: str | Path | None = None, @@ -1652,16 +1760,33 @@ def build_merge( graph_path = Path(graph_path if graph_path is not None else _default_graph_json()) _loaded = _load_existing_graph(graph_path) if _loaded is not None: - existing_nodes, existing_edges, existing_hyperedges, existing_directed = _loaded + ( + existing_nodes, + existing_edges, + existing_hyperedges, + existing_directed, + existing_multigraph, + ) = _loaded had_graph = True else: existing_nodes = [] existing_edges = [] existing_hyperedges = [] existing_directed = False + existing_multigraph = False had_graph = False if directed is None: directed = existing_directed if had_graph else False + if existing_multigraph: + if multigraph is False: + raise ValueError( + "graphify: refusing to simplify an existing MultiDiGraph; " + "re-run with --multigraph or omit the explicit downgrade." + ) + multigraph = True + directed = True + elif multigraph is None: + multigraph = False # Effective root for relativizing absolute source_file / prune paths back to the # stored relative source_file keys. When the caller passes root we use it; @@ -1729,7 +1854,14 @@ def _kept(item: dict) -> bool: base = [{"nodes": existing_nodes, "edges": existing_edges}] if had_graph else [] all_chunks = base + list(new_chunks) - G = build(all_chunks, directed=directed, dedup=dedup, dedup_llm_backend=dedup_llm_backend, root=root) + G = build( + all_chunks, + directed=directed, + multigraph=multigraph, + dedup=dedup, + dedup_llm_backend=dedup_llm_backend, + root=root, + ) # Prune set for deleted source files — both the raw form (matches nodes that # kept absolute source_file) and the normalised relative form (matches nodes @@ -1836,10 +1968,17 @@ def _prune_match(sf: "str | None") -> bool: G.remove_nodes_from(to_remove) n_nodes = len(to_remove) - edges_to_remove = [ - (u, v) for u, v, d in G.edges(data=True) - if _prune_match(d.get("source_file")) - ] + if G.is_multigraph(): + edges_to_remove = [ + (u, v, key) + for u, v, key, d in G.edges(keys=True, data=True) + if _prune_match(d.get("source_file")) + ] + else: + edges_to_remove = [ + (u, v) for u, v, d in G.edges(data=True) + if _prune_match(d.get("source_file")) + ] if edges_to_remove: G.remove_edges_from(edges_to_remove) diff --git a/graphify/cli.py b/graphify/cli.py index 5b73397266..1cfaa459c4 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -11,7 +11,12 @@ import re import sys import time -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import ( + GRAPHIFY_OUT as _GRAPHIFY_OUT, + graphify_out_dir, + remember_output_root, + resolve_output_root, +) from pathlib import Path, PurePosixPath, PureWindowsPath @@ -82,7 +87,7 @@ def _default_graph_path() -> str: - return str(Path(_GRAPHIFY_OUT) / "graph.json") + return str(graphify_out_dir(Path.cwd()) / "graph.json") def _stamped_manifest_files( @@ -1322,7 +1327,10 @@ def dispatch_command(cmd: str) -> None: p.add_argument("--nodes", nargs="*", default=[]) p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) p.add_argument("--correction", default=None) - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) + p.add_argument( + "--memory-dir", + default=str(graphify_out_dir(Path.cwd()) / "memory"), + ) opts = p.parse_args(sys.argv[2:]) if opts.answer_file: opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() @@ -1343,11 +1351,12 @@ def dispatch_command(cmd: str) -> None: elif cmd == "reflect": import argparse as _ap + default_reflect_dir = graphify_out_dir(Path.cwd()) p = _ap.ArgumentParser(prog="graphify reflect") - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) + p.add_argument("--memory-dir", default=str(default_reflect_dir / "memory")) p.add_argument( "--out", - default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), + default=str(default_reflect_dir / "reflections" / "LESSONS.md"), ) p.add_argument("--graph", default=None) p.add_argument("--analysis", default=None) @@ -1364,7 +1373,7 @@ def dispatch_command(cmd: str) -> None: graph_arg = opts.graph if graph_arg is None: - default_graph = Path(_GRAPHIFY_OUT) / "graph.json" + default_graph = default_reflect_dir / "graph.json" if default_graph.exists(): graph_arg = str(default_graph) @@ -1908,7 +1917,11 @@ def dispatch_command(cmd: str) -> None: i_arg += 1 if watch_path is None: watch_path = Path(".") - graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" + graph_json = ( + graph_override + if graph_override is not None + else graphify_out_dir(watch_path) / "graph.json" + ) if not graph_json.exists(): print( f"error: no graph found at {graph_json} — run /graphify first", @@ -1946,7 +1959,11 @@ def dispatch_command(cmd: str) -> None: ) _raw = json.loads(graph_json.read_text(encoding="utf-8")) _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) + G = build_from_json( + _raw, + directed=_directed, + multigraph=bool(_raw.get("multigraph", False)), + ) print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") stages.mark("load") print("Re-clustering...") @@ -1980,7 +1997,7 @@ def dispatch_command(cmd: str) -> None: if graph_override is not None and graph_json.parent.name == _out_name: out = graph_json.parent else: - out = watch_path / _GRAPHIFY_OUT + out = graphify_out_dir(watch_path) out.mkdir(parents=True, exist_ok=True) labels_path = out / ".graphify_labels.json" existing_labels: dict[int, str] = {} @@ -2251,6 +2268,7 @@ def _clear_html_stale_marker() -> None: elif cmd == "update": force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") no_cluster = False + multigraph: bool | None = None args = sys.argv[2:] watch_arg: str | None = None for a in args: @@ -2260,6 +2278,9 @@ def _clear_html_stale_marker() -> None: if a == "--no-cluster": no_cluster = True continue + if a == "--multigraph": + multigraph = True + continue if a.startswith("-"): print(f"error: unknown update option: {a}", file=sys.stderr) sys.exit(2) @@ -2272,7 +2293,7 @@ def _clear_html_stale_marker() -> None: watch_path = Path(watch_arg) else: # Try to recover the scan root saved by the last full build - saved = Path(_GRAPHIFY_OUT) / ".graphify_root" + saved = graphify_out_dir(Path.cwd()) / ".graphify_root" if saved.exists(): watch_path = Path(saved.read_text(encoding="utf-8").strip()) else: @@ -2286,7 +2307,13 @@ def _clear_html_stale_marker() -> None: # Interactive CLI: block on the per-repo lock rather than skip, so the # user sees their explicit `graphify update` complete instead of # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) + ok = _rebuild_code( + watch_path, + force=force, + no_cluster=no_cluster, + multigraph=multigraph, + block_on_lock=True, + ) if ok: print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") if not ( @@ -2337,7 +2364,7 @@ def _clear_html_stale_marker() -> None: # showing top-K outbound edges per symbol. from typing import Optional as _Opt from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + graph_path = Path(_default_graph_path()) output_path: "_Opt[Path]" = None root: "_Opt[str]" = None max_children = DEFAULT_MAX_CHILDREN @@ -2622,11 +2649,12 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": # Parse shared args args = sys.argv[3:] - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + default_export_dir = graphify_out_dir(Path.cwd()) + graph_path = default_export_dir / "graph.json" graph_path_explicit = False - labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" + labels_path = default_export_dir / ".graphify_labels.json" labels_path_explicit = False - report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" + report_path = default_export_dir / "GRAPH_REPORT.md" report_path_explicit = False sections_path: Path | None = None callflow_output: Path | None = None @@ -2635,10 +2663,10 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": callflow_diagram_scale = 1.0 callflow_max_diagram_nodes = 18 callflow_max_diagram_edges = 24 - analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" + analysis_path = default_export_dir / ".graphify_analysis.json" node_limit = 5000 no_viz = False - obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" + obsidian_dir = default_export_dir / "obsidian" # Shared push-connection settings for the graph-database sinks (neo4j, # falkordb), parsed from the generic --push/--user/--password flags below. push_uri: str | None = None @@ -2687,7 +2715,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") print(" --report PATH path to GRAPH_REPORT.md") print(" --sections PATH JSON section definitions") - print(" --output HTML output path (default graphify-out/-callflow.html)") + print(" --output HTML output path (default: configured source output)") print(" --lang LANG auto, zh-CN, en, etc. (default auto)") print(" --max-sections N maximum auto-derived sections (default 15)") print(" --diagram-scale N Mermaid diagram scale (default 1.0)") @@ -2713,7 +2741,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": elif (candidate / "graph.json").exists(): graph_path = candidate / "graph.json" else: - graph_path = candidate / _GRAPHIFY_OUT / "graph.json" + graph_path = graphify_out_dir(candidate) / "graph.json" graph_path_explicit = True i += 1 else: @@ -2994,7 +3022,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print( "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " "[--model M] [--mode deep] [--out DIR|--output DIR] [--google-workspace] [--no-cluster] " - "[--no-gitignore] [--code-only] [--no-dedup] " + "[--multigraph] [--no-gitignore] [--code-only] [--no-dedup] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " "[--api-timeout S] [--postgres DSN] [--cargo] [--allow-partial] [--timing]", file=sys.stderr, @@ -3019,6 +3047,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": cli_cargo: bool = False cli_allow_partial: bool = False no_cluster = False + cli_multigraph = False dedup_llm = False # --no-dedup: skip entity deduplication entirely. On an incremental # merge the fuzzy pass runs over the COMBINED node set (existing graph + @@ -3093,6 +3122,8 @@ def _parse_float(name: str, raw: str) -> float: out_dir = Path(a.split("=", 1)[1]); i += 1 elif a == "--no-cluster": no_cluster = True; i += 1 + elif a == "--multigraph": + cli_multigraph = True; i += 1 elif a == "--dedup-llm": dedup_llm = True; i += 1 elif a == "--no-dedup": @@ -3187,15 +3218,27 @@ def _parse_float(name: str, raw: str) -> float: # Resolve output dir. The user-facing contract is "/graphify-out/" # so a fresh checkout writes graphify-out/ at the project root, matching # the skill.md pipeline. - out_root = (out_dir.resolve() if out_dir else target) + out_root = resolve_output_root(target, out_dir) graphify_out = out_root / _GRAPHIFY_OUT graphify_out.mkdir(parents=True, exist_ok=True) + + def _remember_selected_output() -> None: + if out_dir is None: + return + try: + remember_output_root(target, out_root) + except OSError as exc: + print( + f"[graphify extract] warning: could not remember --out: {exc}", + file=sys.stderr, + ) # Persist corpus-shaping options so later update/watch/hook rebuilds # use the same file set as the initial extraction (#1886). from graphify.watch import ( _write_build_config as _write_build_cfg, _read_build_excludes as _read_build_ex, _read_build_gitignore as _read_build_gi, + _read_build_multigraph as _read_build_mg, ) # #1971 persistence: an explicit --no-gitignore persists False; a later # flag-less `graphify extract` must NOT clobber it back to True, which @@ -3207,10 +3250,12 @@ def _parse_float(name: str, raw: str) -> float: _effective_gitignore = False if no_gitignore else _read_build_gi(graphify_out) # An explicit list replaces the persisted one; omission reuses it. _effective_excludes = cli_excludes or _read_build_ex(graphify_out) + _effective_multigraph = cli_multigraph or _read_build_mg(graphify_out) _write_build_cfg( graphify_out, excludes=cli_excludes or None, gitignore=False if no_gitignore else None, + multigraph=True if cli_multigraph else None, ) stages = _StageTimer(cli_timing) @@ -3275,6 +3320,7 @@ def _parse_float(name: str, raw: str) -> float: manifest_path=str(manifest_path), google_workspace=google_workspace or None, extra_excludes=_effective_excludes or None, + cache_root=out_root, gitignore=_effective_gitignore, ) files_by_type = detection.get("files", {}) @@ -4021,8 +4067,18 @@ def _invalidate_file_manifest_for_db_graph() -> None: # raw-dump this run's partial extraction over it. print(f"error: {exc}", file=sys.stderr) sys.exit(1) - merged["nodes"] = _dedupe_nodes(merged["nodes"]) - merged["edges"] = _dedupe_edges(merged["edges"]) + raw_multigraph = None + if _effective_multigraph: + from graphify.build import build as _build_raw_multigraph + raw_multigraph = _build_raw_multigraph( + [merged], + multigraph=True, + dedup=True, + root=target, + ) + else: + merged["nodes"] = _dedupe_nodes(merged["nodes"]) + merged["edges"] = _dedupe_edges(merged["edges"]) # Disambiguate colliding-basename file-node labels (#2032). This raw # --no-cluster path bypasses build_from_json (where the clustered path # gets this), so apply it directly on the merged node list. @@ -4045,14 +4101,19 @@ def _invalidate_file_manifest_for_db_graph() -> None: from graphify.export import MALFORMED_GRAPH as _MALFORMED_GRAPH _existing_n = _existing_graph_node_count(graph_json_path) _malformed = _existing_n is _MALFORMED_GRAPH - _shrinks = isinstance(_existing_n, int) and len(merged["nodes"]) < _existing_n + _candidate_nodes = ( + raw_multigraph.number_of_nodes() + if raw_multigraph is not None + else len(merged["nodes"]) + ) + _shrinks = isinstance(_existing_n, int) and _candidate_nodes < _existing_n if _malformed or _shrinks: _detail = ( f"the existing {graph_json_path} is present but unparseable " "(corrupt or a mid-write), so a shrink cannot be ruled out" if _malformed else f"smaller than the existing {graph_json_path} " - f"({len(merged['nodes'])} < {_existing_n} nodes)" + f"({_candidate_nodes} < {_existing_n} nodes)" ) print( "[graphify extract] error: extraction was incomplete (an AST/" @@ -4064,8 +4125,19 @@ def _invalidate_file_manifest_for_db_graph() -> None: sys.exit(1) _backup(graphify_out) _invalidate_file_manifest_for_db_graph() - from graphify.paths import write_json_atomic as _write_json_atomic - _write_json_atomic(graph_json_path, merged, indent=2) + if raw_multigraph is not None: + from graphify.export import to_json as _write_raw_multigraph + from graphify.watch import _git_head as _raw_git_head + _write_raw_multigraph( + raw_multigraph, + {}, + str(graph_json_path), + force=True, + built_at_commit=_raw_git_head(cwd=Path(target).resolve()), + ) + else: + from graphify.paths import write_json_atomic as _write_json_atomic + _write_json_atomic(graph_json_path, merged, indent=2) try: # Record the scan root so a later build_merge / update runbook can # relativize deleted-file paths correctly even for a custom --out @@ -4076,13 +4148,15 @@ def _invalidate_file_manifest_for_db_graph() -> None: ) except OSError: pass + _remember_selected_output() stages.mark("write") cost = _estimate_cost( backend, merged["input_tokens"], merged["output_tokens"] ) print( f"[graphify extract] wrote {graph_json_path} — " - f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " + f"{raw_multigraph.number_of_nodes() if raw_multigraph is not None else len(merged['nodes'])} nodes, " + f"{raw_multigraph.number_of_edges() if raw_multigraph is not None else len(merged['edges'])} edges " f"(no clustering)" ) if merged["input_tokens"] or merged["output_tokens"]: @@ -4136,6 +4210,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: [merged], graph_path=existing_graph_path, prune_sources=_prune_sources or None, + multigraph=True if _effective_multigraph else None, dedup=not no_dedup, dedup_llm_backend=dedup_backend, root=target, @@ -4148,7 +4223,13 @@ def _invalidate_file_manifest_for_db_graph() -> None: print(f"[graphify extract] {exc}", file=sys.stderr) sys.exit(1) else: - G = _build([merged], dedup=not no_dedup, dedup_llm_backend=dedup_backend, root=target) + G = _build( + [merged], + multigraph=_effective_multigraph, + dedup=not no_dedup, + dedup_llm_backend=dedup_backend, + root=target, + ) stages.mark("build") if G.number_of_nodes() == 0: print( @@ -4228,6 +4309,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: ) except OSError: pass + _remember_selected_output() stages.mark("export") if merged.get("output_tokens", 0) > 0: (graphify_out / ".graphify_semantic_marker").write_text( @@ -4343,10 +4425,15 @@ def _invalidate_file_manifest_for_db_graph() -> None: else: i += 1 files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] + cache_output_root = resolve_output_root(root) cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( - files, root, mode=cache_mode, prompt_file=prompt_file + files, + root, + mode=cache_mode, + prompt_file=prompt_file, + cache_root=cache_output_root, ) - out = root / _GRAPHIFY_OUT + out = cache_output_root / _GRAPHIFY_OUT out.mkdir(parents=True, exist_ok=True) if cached_nodes or cached_edges or cached_hyperedges: (out / ".graphify_cached.json").write_text( diff --git a/graphify/cluster.py b/graphify/cluster.py index 6822107001..37bc670b77 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -6,6 +6,7 @@ import json import sys import networkx as nx +from graphify.build import analysis_projection def _suppress_output(): @@ -95,6 +96,7 @@ def label_communities_by_hub( Used as the default (no-backend) labeler; an LLM naming pass, when configured, overrides these with richer names. """ + topology = analysis_projection(G) labels: dict[int, str] = {} for cid, members in communities.items(): present = [n for n in members if n in G] @@ -102,7 +104,7 @@ def label_communities_by_hub( labels[cid] = f"Community {cid}" continue # highest degree wins; ties broken by node id (ascending) for determinism - hub = min(present, key=lambda n: (-G.degree(n), str(n))) + hub = min(present, key=lambda n: (-topology.degree(n), str(n))) name = str(G.nodes[hub].get("label") or hub).strip() if name.endswith("()"): name = name[:-2] @@ -154,8 +156,7 @@ def cluster( """ if G.number_of_nodes() == 0: return {} - if G.is_directed(): - G = G.to_undirected() + G = analysis_projection(G) if G.number_of_edges() == 0: return {i: [n] for i, n in enumerate(sorted(G.nodes))} @@ -259,14 +260,16 @@ def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float: n = len(community_nodes) if n <= 1: return 1.0 - subgraph = G.subgraph(community_nodes) + topology = analysis_projection(G) if (G.is_directed() or G.is_multigraph()) else G + subgraph = topology.subgraph(community_nodes) actual = subgraph.number_of_edges() possible = n * (n - 1) / 2 return actual / possible if possible > 0 else 0.0 def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]: - return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()} + topology = analysis_projection(G) + return {cid: cohesion_score(topology, nodes) for cid, nodes in communities.items()} def remap_communities_to_previous( diff --git a/graphify/detect.py b/graphify/detect.py index d16b5800ce..1f556b2db7 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -2218,6 +2218,7 @@ def detect_incremental( google_workspace: bool | None = None, kind: str = "semantic", extra_excludes: list[str] | None = None, + cache_root: Path | None = None, gitignore: bool = True, ) -> dict: """Like detect(), but returns only new or modified files since the last run. @@ -2234,6 +2235,9 @@ def detect_incremental( beyond stat). Slow path: mtime bumped → compare MD5 against the relevant hash field before re-extracting. + ``cache_root`` is forwarded to :func:`detect` so repeated scans configured + with an external output keep the stat/word-count cache outside the corpus. + Backwards compatible with legacy manifests storing plain float mtime values or {mtime, hash} dicts (treated as ast_hash only; semantic_hash = miss). @@ -2247,6 +2251,7 @@ def detect_incremental( follow_symlinks=follow_symlinks, google_workspace=google_workspace, extra_excludes=extra_excludes, + cache_root=cache_root, gitignore=gitignore, ) # Pass ``root`` so a manifest written with relative keys (post-#777) is diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index fcb9a11cf2..cd883bbda4 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -157,6 +157,7 @@ def diagnose_extraction( extraction: dict[str, Any], *, directed: bool = True, + multigraph: bool = False, root: str | Path | None = None, max_examples: int = 5, extract_path: str | Path | None = None, @@ -184,20 +185,36 @@ def diagnose_extraction( non_object_edges = 0 missing_endpoint_edges = 0 dangling_endpoint_edges = 0 + external_endpoint_edges = 0 + excluded_local_endpoint_edges = 0 + unresolved_internal_endpoint_edges = 0 + unclassified_endpoint_edges = 0 self_loop_edges = 0 valid_candidate_edges = 0 - for edge in canonical_edges: + for raw_edge, edge in zip(raw_edges, canonical_edges): if edge["_invalid"]: non_object_edges += 1 continue source = edge["source"] target = edge["target"] + if isinstance(raw_edge, dict): + if raw_edge.get("external") is True: + external_endpoint_edges += 1 + elif raw_edge.get("excluded_local") is True: + excluded_local_endpoint_edges += 1 + elif raw_edge.get("unresolved_internal") is True: + unresolved_internal_endpoint_edges += 1 if not source or not target: missing_endpoint_edges += 1 continue if source not in node_ids or target not in node_ids: dangling_endpoint_edges += 1 + if not isinstance(raw_edge, dict) or not any( + raw_edge.get(flag) is True + for flag in ("external", "excluded_local", "unresolved_internal") + ): + unclassified_endpoint_edges += 1 continue if source == target: self_loop_edges += 1 @@ -234,7 +251,12 @@ def diagnose_extraction( post_build_node_count: int | None = None try: graph_input = deepcopy(extraction) - graph: nx.Graph = build_from_json(graph_input, directed=directed, root=root) + graph: nx.Graph = build_from_json( + graph_input, + directed=directed, + multigraph=multigraph, + root=root, + ) graph_type = type(graph).__name__ post_build_edge_count = graph.number_of_edges() post_build_node_count = graph.number_of_nodes() @@ -252,6 +274,10 @@ def diagnose_extraction( "non_object_edges": non_object_edges, "missing_endpoint_edges": missing_endpoint_edges, "dangling_endpoint_edges": dangling_endpoint_edges, + "external_endpoint_edges": external_endpoint_edges, + "excluded_local_endpoint_edges": excluded_local_endpoint_edges, + "unresolved_internal_endpoint_edges": unresolved_internal_endpoint_edges, + "unclassified_endpoint_edges": unclassified_endpoint_edges, "self_loop_edges": self_loop_edges, "valid_candidate_edges": valid_candidate_edges, "exact_duplicate_edges": _count_extra(exact_counts), @@ -299,6 +325,7 @@ def diagnose_file( path: str | Path, *, directed: bool | None = None, + multigraph: bool | None = None, root: str | Path | None = None, max_examples: int = 5, extract_path: str | Path | None = None, @@ -314,16 +341,22 @@ def diagnose_file( effective_directed = raw_directed if isinstance(raw_directed, bool) else True else: effective_directed = directed + if multigraph is None: + effective_multigraph = data.get("multigraph") is True + else: + effective_multigraph = multigraph summary = diagnose_extraction( data, directed=effective_directed, + multigraph=effective_multigraph, root=root, max_examples=max_examples, extract_path=extract_path, ) summary["input_path"] = str(path) summary["effective_directed"] = effective_directed + summary["effective_multigraph"] = effective_multigraph return summary @@ -352,12 +385,17 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: f"input: {summary.get('input_path', '')}", "input_stage: provided JSON (normal graph.json is post-build)", f"effective_directed: {summary.get('effective_directed', '')}", + f"effective_multigraph: {summary.get('effective_multigraph', '')}", f"nodes: {summary['node_count']}", f"unverified_code_nodes: {summary.get('unverified_node_count', 0)}", f"raw_edges: {summary['raw_edge_count']}", f"valid_candidate_edges: {summary['valid_candidate_edges']}", f"missing_endpoint_edges: {summary['missing_endpoint_edges']}", f"dangling_endpoint_edges: {summary['dangling_endpoint_edges']}", + f"external_endpoint_edges: {summary.get('external_endpoint_edges', 0)}", + f"excluded_local_endpoint_edges: {summary.get('excluded_local_endpoint_edges', 0)}", + f"unresolved_internal_endpoint_edges: {summary.get('unresolved_internal_endpoint_edges', 0)}", + f"unclassified_endpoint_edges: {summary.get('unclassified_endpoint_edges', 0)}", f"self_loop_edges: {summary['self_loop_edges']}", f"exact_duplicate_edges: {summary['exact_duplicate_edges']}", f"directed_unique_endpoint_pairs: {summary['directed_unique_endpoint_pairs']}", diff --git a/graphify/export.py b/graphify/export.py index 69136befce..236bb0407e 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -1259,9 +1259,9 @@ def _graphml_safe(val): for node_id in H.nodes(): for key, val in list(H.nodes[node_id].items()): H.nodes[node_id][key] = _graphml_safe(val) - for u, v in H.edges(): - for key, val in list(H.edges[u, v].items()): - H.edges[u, v][key] = _graphml_safe(val) + for _, _, attrs in H.edges(data=True): + for key, val in list(attrs.items()): + attrs[key] = _graphml_safe(val) # Write atomically: a mid-serialization error otherwise leaves a 0-byte # .graphml on disk that downstream tooling mistakes for a completed export diff --git a/graphify/extract.py b/graphify/extract.py index 89082af878..37c7170835 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -430,6 +430,7 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p resolved = _resolve_js_import_target(raw, str_path) if resolved is not None: tgt_nid, resolved_path = resolved + attempted_path = resolved_path # `_resolve_js_import_path` returns the attempted path when no # local file exists. Static ES imports must treat that as unresolved # rather than minting a checkout-specific target ID (#2457). @@ -445,6 +446,7 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, + "specifier": raw, } # Stamp the resolved target file so a same-basename cross-extension # sibling (foo.ts importing/re-exporting ./foo.mjs) keys its target salt @@ -453,6 +455,24 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p # back onto the importer's own variant, a phantom self-loop (#1814). if resolved_path is not None: edge["target_file"] = str(resolved_path) + elif attempted_path is not None: + edge["target_file"] = str(attempted_path) + edge["unresolved_internal"] = True + else: + aliases = _load_tsconfig_aliases(Path(str_path).parent) + local_alias = any( + _match_tsconfig_alias(raw, pattern) is not None + for pattern in aliases + ) + workspace_packages = _load_workspace_packages(Path(str_path).parent) + local_workspace = any( + raw == name or raw.startswith(name + "/") + for name in workspace_packages + ) + if raw.startswith((".", "/", "#")) or local_alias or local_workspace: + edge["unresolved_internal"] = True + else: + edge["external"] = True edges.append(edge) # Emit symbol-level edges for named imports/re-exports from local/aliased files. @@ -486,6 +506,7 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, + "specifier_symbol": sym, # Which file this symbol target was synthesized # from, so the id-remap post-pass can repoint a # target the candidates rewrite never learns — @@ -513,6 +534,7 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, + "specifier_symbol": sym, # See the re_exports stamp above (#1983). "target_file": str(resolved_path), }) @@ -6388,6 +6410,102 @@ def _learn(e: dict) -> None: _repoint_python_package_imports(paths, all_nodes, all_edges, root) _merge_swift_extensions(per_file, all_nodes, all_edges) _merge_csharp_partial_class_nodes(per_file, all_nodes, all_edges, paths, root) + + # Preserve JS/TS dependency records whose target is intentionally outside + # the scanned node set. Explicit endpoint nodes keep those edges available + # to queries while distinguishing dependencies, excluded resources, and + # genuinely unresolved project-local imports. + _owned_endpoint_ids = {node.get("id") for node in all_nodes} + _owned_endpoint_ids.update( + node.get("id") + for node in (resolution_context_nodes or []) + if isinstance(node, dict) + ) + _owned_labels = { + node.get("id"): str(node.get("label", "")).strip().strip("()").lstrip(".") + for node in [*all_nodes, *(resolution_context_nodes or [])] + if isinstance(node, dict) and node.get("id") + } + def _resolved_path_key(value) -> str: + if not value: + return "" + try: + return str(Path(str(value)).resolve()) + except (OSError, RuntimeError): + return str(value) + + _resolved_symbol_evidence = { + ( + edge.get("source"), + edge.get("relation"), + edge.get("source_location"), + edge.get("resolved_specifier_symbol") + or _owned_labels.get(edge.get("target")), + _resolved_path_key(edge.get("resolved_specifier_file")), + ) + for edge in all_edges + if edge.get("target") in _owned_endpoint_ids + and not edge.get("specifier_symbol") + } + _redundant_symbol_edges = { + id(edge) + for edge in all_edges + if edge.get("specifier_symbol") + and ( + edge.get("source"), + edge.get("relation"), + edge.get("source_location"), + edge.get("specifier_symbol"), + _resolved_path_key(edge.get("target_file")), + ) in _resolved_symbol_evidence + } + if _redundant_symbol_edges: + all_edges[:] = [ + edge for edge in all_edges if id(edge) not in _redundant_symbol_edges + ] + for _edge in all_edges: + _edge.pop("resolved_specifier_file", None) + _edge.pop("resolved_specifier_symbol", None) + for _edge in all_edges: + _target = _edge.get("target") + if ( + not _target + or _target in _owned_endpoint_ids + or _edge.get("relation") not in ("imports", "imports_from", "re_exports") + ): + continue + _specifier = _edge.get("specifier") or _edge.get("specifier_symbol") + _target_file = _edge.get("target_file") + if _edge.get("external") is True: + _kind = "external" + elif _edge.get("unresolved_internal") is True: + _kind = "unresolved_internal" + elif _target_file and _edge.get("relation") == "imports_from": + _kind = "excluded_local" + _edge["excluded_local"] = True + elif _target_file: + _kind = "unresolved_internal" + _edge["unresolved_internal"] = True + else: + continue + + _stub = { + "id": _target, + "label": str(_specifier or Path(str(_target_file or _target)).name or _target), + "file_type": "concept", + "source_file": "", + _kind: True, + } + if _kind == "excluded_local" and _target_file: + try: + _stub["source_file"] = Path(_target_file).resolve().relative_to(root).as_posix() + except (ValueError, OSError): + pass + all_nodes.append(_stub) + _owned_endpoint_ids.add(_target) + for _edge in all_edges: + _edge.pop("specifier_symbol", None) + _disambiguate_colliding_node_ids(all_nodes, all_edges, all_raw_calls, root) _canonicalize_csharp_namespace_nodes(all_nodes, all_edges) # PHP namespace/use disambiguation must run BEFORE the unique-stub rewire: diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 5369637aeb..fe3d86d5ec 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -24,9 +24,22 @@ _WORKSPACE_MANIFEST_NAMES = ("pnpm-workspace.yaml", "package.json") -_JS_RESOLVE_EXTS = (".ts", ".tsx", ".mts", ".cts", ".svelte", ".js", ".jsx", ".mjs", ".cjs") +_JS_RESOLVE_EXTS = ( + ".ts", ".tsx", ".mts", ".cts", ".d.ts", + ".svelte", ".js", ".jsx", ".mjs", ".cjs", +) + +_JS_INDEX_FILES = ( + "index.ts", "index.tsx", "index.mts", "index.cts", "index.d.ts", + "index.svelte", "index.js", "index.jsx", "index.mjs", "index.cjs", +) -_JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs") +_JS_SPECIFIER_REPLACEMENTS = { + ".js": (".ts", ".tsx", ".mts", ".cts", ".d.ts"), + ".jsx": (".tsx", ".ts", ".d.ts"), + ".mjs": (".mts", ".ts", ".tsx", ".d.mts", ".d.ts"), + ".cjs": (".cts", ".ts", ".tsx", ".d.cts", ".d.ts"), +} def _resolve_js_import_path(candidate: Path) -> Path: """Resolve a JS/TS/Svelte import target to a local file when it exists.""" @@ -34,15 +47,12 @@ def _resolve_js_import_path(candidate: Path) -> Path: if candidate.is_file(): return candidate - # TS ESM convention: imports often spell .js/.jsx while source is .ts/.tsx. - if candidate.suffix == ".js": - ts_candidate = candidate.with_suffix(".ts") - if ts_candidate.is_file(): - return ts_candidate - elif candidate.suffix == ".jsx": - tsx_candidate = candidate.with_suffix(".tsx") - if tsx_candidate.is_file(): - return tsx_candidate + # TypeScript ESM convention: source imports spell the runtime extension. + # Replace that extension rather than appending to it (`foo.js.tsx`). + for replacement in _JS_SPECIFIER_REPLACEMENTS.get(candidate.suffix.lower(), ()): + source_candidate = candidate.with_suffix(replacement) + if source_candidate.is_file(): + return source_candidate # Append extensions to the full filename, which covers extensionless imports, # multi-dot helpers, and Svelte 5 rune files like Foo.svelte.ts. @@ -409,25 +419,29 @@ def _load_workspace_packages(start_dir: Path) -> dict[str, Path]: return packages _EXPORT_CONDITION_PRIORITY = ( - "source", "import", "module", "svelte", "types", "require", "default", + "source", "development", "types", "import", "module", "svelte", "require", "default", ) + +def _resolve_export_targets(value: Any) -> list[str]: + """Return every eligible export target in source-analysis priority order.""" + if isinstance(value, str): + return [value] + if not isinstance(value, dict): + return [] + targets: list[str] = [] + for condition in _EXPORT_CONDITION_PRIORITY: + for target in _resolve_export_targets(value.get(condition)): + if target not in targets: + targets.append(target) + return targets + def _resolve_export_target(value: Any) -> str | None: """Resolve an `exports` map value (string or condition object) to a relative target string, honouring _EXPORT_CONDITION_PRIORITY for objects and recursing into nested condition objects.""" - if isinstance(value, str): - return value - if isinstance(value, dict): - for cond in _EXPORT_CONDITION_PRIORITY: - v = value.get(cond) - if isinstance(v, str): - return v - if isinstance(v, dict): - nested = _resolve_export_target(v) - if nested: - return nested - return None + targets = _resolve_export_targets(value) + return targets[0] if targets else None def _contained_in_package(resolved: Path, package_dir: Path) -> bool: """Guard against `exports` targets that escape the package directory @@ -454,11 +468,14 @@ def _package_entry_candidates(package_dir: Path, subpath: str) -> list[Path]: exports = manifest_data.get("exports") if isinstance(exports, dict): subpath_key = "./" + subpath - target = _resolve_export_target(exports.get(subpath_key)) - if target: - candidate = package_dir / target - if _contained_in_package(candidate, package_dir): - return [candidate] + targets = _resolve_export_targets(exports.get(subpath_key)) + candidates = [ + package_dir / target + for target in targets + if _contained_in_package(package_dir / target, package_dir) + ] + if candidates: + return candidates else: for pattern, pattern_value in exports.items(): if "*" in pattern and pattern.count("*") == 1: @@ -466,20 +483,29 @@ def _package_entry_candidates(package_dir: Path, subpath: str) -> list[Path]: if (subpath_key.startswith(prefix) and (not suffix or subpath_key.endswith(suffix))): matched = subpath_key[len(prefix):len(subpath_key) - len(suffix) if suffix else None] - resolved = _resolve_export_target(pattern_value) - if resolved and "*" in resolved: + candidates = [] + for resolved in _resolve_export_targets(pattern_value): + if "*" not in resolved: + continue candidate = package_dir / resolved.replace("*", matched) if _contained_in_package(candidate, package_dir): - return [candidate] + candidates.append(candidate) + if candidates: + return candidates return [package_dir / subpath] exports = manifest_data.get("exports") if isinstance(exports, str): return [package_dir / exports] if isinstance(exports, dict): - dot_target = _resolve_export_target(exports.get(".")) - if dot_target: - return [package_dir / dot_target] + dot_targets = _resolve_export_targets(exports.get(".")) + candidates = [ + package_dir / target + for target in dot_targets + if _contained_in_package(package_dir / target, package_dir) + ] + if candidates: + return candidates candidates: list[Path] = [] for key in ("svelte", "module", "main", "types"): @@ -505,6 +531,45 @@ def _resolve_workspace_import(raw: str, start_dir: Path) -> Path | None: return resolved return None +def _resolve_package_import(raw: str, start_dir: Path) -> Path | None: + """Resolve a package-local ``imports`` alias such as ``#/models``.""" + if not raw.startswith("#"): + return None + for package_dir in (start_dir, *start_dir.parents): + manifest = package_dir / "package.json" + if not manifest.is_file(): + continue + try: + imports = json.loads(manifest.read_text(encoding="utf-8")).get("imports") + except Exception: + return None + if not isinstance(imports, dict): + return None + matches: list[tuple[Any, str | None]] = [] + if raw in imports: + matches.append((imports[raw], None)) + for pattern, value in imports.items(): + if not isinstance(pattern, str) or pattern.count("*") != 1: + continue + prefix, suffix = pattern.split("*", 1) + if raw.startswith(prefix) and (not suffix or raw.endswith(suffix)): + end = len(raw) - len(suffix) if suffix else None + matches.append((value, raw[len(prefix):end])) + for value, wildcard in matches: + for target in _resolve_export_targets(value): + if wildcard is not None: + if "*" not in target: + continue + target = target.replace("*", wildcard) + candidate = package_dir / target + if not _contained_in_package(candidate, package_dir): + continue + resolved = _resolve_js_import_path(candidate) + if resolved.is_file(): + return resolved + return None + return None + def _resolve_js_module_path(raw: str | Path, start_dir: Path | None = None) -> Path | None: """Resolve a JS/TS module path or specifier to a local source file. @@ -517,9 +582,20 @@ def _resolve_js_module_path(raw: str | Path, start_dir: Path | None = None) -> P return _resolve_js_import_path(raw) if start_dir is None: return _resolve_js_import_path(Path(raw)) + # Bundlers attach loader/resource modifiers to the specifier, not the path. + # A leading ``#`` is instead a Node package-import-map key and must survive. + raw = raw.split("?", 1)[0] + if not raw.startswith("#"): + raw = raw.split("#", 1)[0] + if not raw: + return None if raw.startswith("."): return _resolve_js_import_path(start_dir / raw) + package_import = _resolve_package_import(raw, start_dir) + if package_import is not None: + return package_import + aliases = _load_tsconfig_aliases(start_dir) hit = _resolve_tsconfig_alias(raw, aliases, base_url=_load_tsconfig_base_url(start_dir)) @@ -876,7 +952,7 @@ def ensure_symbol_node(path: Path, name: str, line: int) -> str: for edge in edges } - def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path, target_file: str | None = None, local_alias: str | None = None) -> None: + def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path, target_file: str | None = None, local_alias: str | None = None, resolved_specifier_file: str | None = None, resolved_specifier_symbol: str | None = None) -> None: key = (source, target, relation, context or "") if key in existing_edges: return @@ -901,6 +977,10 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s # cross-file member-call resolver match `alias.func()` (#2082). if local_alias is not None: edge["local_alias"] = local_alias + if resolved_specifier_file is not None: + edge["resolved_specifier_file"] = resolved_specifier_file + if resolved_specifier_symbol is not None: + edge["resolved_specifier_symbol"] = resolved_specifier_symbol edges.append(edge) for declaration in facts.declarations: @@ -932,6 +1012,7 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s changed = True named_exports_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + ambiguous_named_exports: set[tuple[Path, str]] = set() star_exports_by_file: dict[Path, list[Path]] = {} for star_fact in facts.star_exports: @@ -992,7 +1073,15 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s origin = (file_path, export_fact.local_name) if origin is None: continue - named_exports_by_file.setdefault(file_path, {})[export_fact.exported_name] = origin + export_key = (file_path, export_fact.exported_name) + prior_origin = named_exports_by_file.setdefault(file_path, {}).get( + export_fact.exported_name + ) + if prior_origin is not None and prior_origin != origin: + ambiguous_named_exports.add(export_key) + named_exports_by_file[file_path].pop(export_fact.exported_name, None) + elif export_key not in ambiguous_named_exports: + named_exports_by_file[file_path][export_fact.exported_name] = origin if origin[0] != file_path: source_id = source_file_id.get(file_path) if source_id is not None: @@ -1006,6 +1095,22 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s target_file=str(path_by_resolved.get(origin[0], origin[0])), ) + def declaration_companions(target_path: Path) -> list[Path]: + suffix = target_path.suffix.lower() + if suffix in (".js", ".jsx"): + return [target_path.with_suffix(".d.ts").resolve()] + if suffix == ".mjs": + return [ + target_path.with_suffix(".d.mts").resolve(), + target_path.with_suffix(".d.ts").resolve(), + ] + if suffix == ".cjs": + return [ + target_path.with_suffix(".d.cts").resolve(), + target_path.with_suffix(".d.ts").resolve(), + ] + return [] + def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: target_path = target_path.resolve() key = (target_path, imported_name) @@ -1014,18 +1119,77 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup if key in seen: return key seen.add(key) + if key in ambiguous_named_exports: + return key origin = named_exports_by_file.get(target_path, {}).get(imported_name) if origin is not None: return resolve_exported_origin(origin[0], origin[1], seen) + # A runtime JavaScript file and its generated declaration file form one + # module surface. Keep runtime declarations on the runtime node, but + # fall through to declaration-only members such as imported types. + for companion in declaration_companions(target_path): + companion_key = (companion, imported_name) + if companion_key in ambiguous_named_exports: + continue + if companion_key in symbol_nodes: + return companion_key + companion_origin = named_exports_by_file.get(companion, {}).get(imported_name) + if companion_origin is not None: + return resolve_exported_origin( + companion_origin[0], companion_origin[1], seen + ) + star_origins: set[tuple[Path, str]] = set() for star_target in star_exports_by_file.get(target_path, []): star_key = (star_target, imported_name) if star_key in symbol_nodes: - return star_key + star_origins.add(star_key) + continue resolved = resolve_exported_origin(star_target, imported_name, seen) if resolved in symbol_nodes: - return resolved + star_origins.add(resolved) + if len(star_origins) == 1: + return next(iter(star_origins)) return key + # Emit the symbol-level half of named re-exports after the complete export + # table is known. The syntax extractor initially targets the immediate + # barrel; this pass follows star/named chains to the defining declaration. + for export_fact in facts.exports: + if export_fact.target_path is None or export_fact.target_name is None: + continue + source_id = source_file_id.get(export_fact.file_path.resolve()) + if source_id is None: + continue + # The syntax pass may already point at an owned symbol. Preserve that + # authoritative edge, including the collision guard exercised by the + # existing alias-remap tests, instead of manufacturing a second target. + if any( + edge.get("source") == source_id + and edge.get("relation") == "re_exports" + and edge.get("source_location") == f"L{export_fact.line}" + and edge.get("specifier_symbol") == export_fact.target_name + and edge.get("target") in symbol_nodes.values() + for edge in edges + ): + continue + origin_path, origin_symbol = resolve_exported_origin( + export_fact.target_path, + export_fact.target_name, + ) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None: + continue + add_edge( + source_id, + target_id, + "re_exports", + "re-export", + export_fact.line, + export_fact.file_path, + resolved_specifier_file=str(export_fact.target_path.resolve()), + resolved_specifier_symbol=export_fact.target_name, + ) + for import_fact in facts.imports: source_id = source_file_id.get(import_fact.file_path.resolve()) if source_id is None: @@ -1044,6 +1208,8 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup "import", import_fact.line, import_fact.file_path, + resolved_specifier_file=str(import_fact.target_path.resolve()), + resolved_specifier_symbol=import_fact.imported_name, ) # #1146: emit file-to-file imports_from edges for package-form submodule imports. @@ -1216,8 +1382,19 @@ def _js_exported_declaration_names(node, source: bytes) -> list[str]: if declaration is None: return names + if declaration.type == "ambient_declaration": + declaration = next( + (child for child in declaration.children if child.is_named), + declaration, + ) + if declaration.type == "lexical_declaration": - names.extend(alias for alias, _target in _js_lexical_aliases(declaration, source)) + for child in declaration.children: + if child.type != "variable_declarator": + continue + name_node = child.child_by_field_name("name") + if name_node is not None: + names.append(_read_text(name_node, source)) return names if declaration.type in ( diff --git a/graphify/paths.py b/graphify/paths.py index ba15b32cc7..1c03d82d96 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -16,6 +16,7 @@ from __future__ import annotations +import hashlib import json import os import re @@ -100,6 +101,95 @@ def write_json_atomic(path: "str | Path", obj, *, indent: "int | None" = None, e UTF-8 (non-ASCII labels/paths) keep byte-for-byte output. See :func:`_atomic_replace`.""" _atomic_replace(path, lambda f: json.dump(obj, f, indent=indent, ensure_ascii=ensure_ascii)) + +_OUTPUT_ROOT_BINDINGS_DIR = "output-roots" + + +def _canonical_local_path(path: "str | Path") -> Path: + """Return a stable absolute identity for a path on this machine.""" + return Path(path).expanduser().resolve() + + +def _local_path_identity(path: "str | Path") -> str: + """Return a comparison/hash identity using the host filesystem's casing.""" + return os.path.normcase(str(_canonical_local_path(path))) + + +def _output_root_binding_path(source_root: "str | Path") -> Path: + source = _canonical_local_path(source_root) + digest = hashlib.sha256(_local_path_identity(source).encode("utf-8")).hexdigest() + return Path.home() / ".graphify" / _OUTPUT_ROOT_BINDINGS_DIR / f"{digest}.json" + + +def remember_output_root(source_root: "str | Path", output_root: "str | Path") -> None: + """Remember the output root explicitly selected for one source tree. + + One binding file per canonical source root avoids lost updates when separate + Graphify processes configure different repositories concurrently. Source + contents are never stored, only the two absolute directory paths. + """ + source = _canonical_local_path(source_root) + output = _canonical_local_path(output_root) + write_json_atomic( + _output_root_binding_path(source), + {"source_root": str(source), "output_root": str(output)}, + indent=2, + ) + + +def persisted_output_root(source_root: "str | Path") -> "Path | None": + """Return the nearest remembered output root for a source path. + + Exact bindings win. Walking parents lets commands launched from a project + subdirectory find the output configured at the project root. Malformed or + mismatched binding files are ignored rather than redirecting output. + """ + source = _canonical_local_path(source_root) + for candidate in (source, *source.parents): + path = _output_root_binding_path(candidate) + try: + data = json.loads(path.read_text(encoding="utf-8")) + stored_source = _canonical_local_path(data["source_root"]) + stored_output = _canonical_local_path(data["output_root"]) + except ( + OSError, + KeyError, + TypeError, + ValueError, + RuntimeError, + json.JSONDecodeError, + ): + continue + if _local_path_identity(stored_source) == _local_path_identity(candidate): + return stored_output + return None + + +def resolve_output_root( + source_root: "str | Path", + explicit_output_root: "str | Path | None" = None, +) -> Path: + """Resolve the parent directory that owns a source tree's Graphify output. + + An explicit CLI selection wins and a non-default ``GRAPHIFY_OUT`` environment + override keeps its established behavior. Otherwise a remembered selection + is reused, falling back to the source root for an unconfigured project. + """ + source = _canonical_local_path(source_root) + if explicit_output_root is not None: + return _canonical_local_path(explicit_output_root) + if os.environ.get("GRAPHIFY_OUT", "graphify-out") != "graphify-out": + return source + return persisted_output_root(source) or source + + +def graphify_out_dir( + source_root: "str | Path", + explicit_output_root: "str | Path | None" = None, +) -> Path: + """Return the effective Graphify artifact directory for a source tree.""" + return resolve_output_root(source_root, explicit_output_root) / GRAPHIFY_OUT + # Directory segments that, when they appear as a whole path component, mark the # whole path as a test location. Matched against path *segments* (not raw # substrings) so "src/contest.py" / "latest/x.py" / "src/greatest/x.py" do NOT @@ -293,12 +383,13 @@ def disambiguate_ambiguous_candidates( def out_path(*parts: str) -> Path: - """A path inside the configured output dir, e.g. ``out_path("cache")``. + """A path inside the effective output dir, e.g. ``out_path("cache")``. - ``Path(GRAPHIFY_OUT) / ...`` resolves correctly for both a relative name - ("graphify-out") and an absolute override ("/shared/graphify-out"). + The current directory is treated as the source location, so project commands + reuse an output root previously selected with ``extract --out``. A custom + ``GRAPHIFY_OUT`` environment value keeps precedence. """ - return Path(GRAPHIFY_OUT, *parts) + return graphify_out_dir(Path.cwd(), None).joinpath(*parts) def default_graph_json() -> str: diff --git a/graphify/watch.py b/graphify/watch.py index 8ad02c4df8..85fdcf661c 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -11,7 +11,12 @@ from typing import Callable # Single source of truth in graphify.paths (#1423); re-exported as _GRAPHIFY_OUT. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT, is_absolute_any_platform +from graphify.paths import ( + GRAPHIFY_OUT as _GRAPHIFY_OUT, + graphify_out_dir, + is_absolute_any_platform, + resolve_output_root, +) _PENDING_FILENAME = ".pending_changes" _PENDING_DRAIN_MAX_PASSES = 20 @@ -82,12 +87,13 @@ def _write_build_config( *, excludes: "list[str] | None", gitignore: bool | None = None, + multigraph: bool | None = None, ) -> None: """Persist corpus-shaping options under ``out_dir``. Best effort and non clobbering: omitted options retain their existing values. """ - if not excludes and gitignore is None: + if not excludes and gitignore is None and multigraph is None: return try: out_dir.mkdir(parents=True, exist_ok=True) @@ -102,6 +108,8 @@ def _write_build_config( config["excludes"] = list(excludes) if gitignore is not None: config["gitignore"] = gitignore + if multigraph is not None: + config["multigraph"] = multigraph path.write_text(json.dumps(config), encoding="utf-8") except OSError: pass @@ -134,6 +142,19 @@ def _read_build_gitignore(out_dir: Path) -> bool: return True +def _read_build_multigraph(out_dir: Path) -> bool: + """Return whether this output is configured for canonical parallel edges.""" + try: + path = out_dir / _BUILD_CONFIG_FILENAME + if path.is_file(): + cfg = json.loads(path.read_text(encoding="utf-8")) + if isinstance(cfg, dict) and isinstance(cfg.get("multigraph"), bool): + return cfg["multigraph"] + except (OSError, json.JSONDecodeError): + pass + return False + + def _merge_changed_paths(*sources: "list[Path] | None") -> list[Path]: """Concatenate path lists, preserving order and dropping duplicates. @@ -1090,6 +1111,7 @@ def _rebuild_code( follow_symlinks: bool = False, force: bool = False, no_cluster: bool = False, + multigraph: bool | None = None, acquire_lock: bool = True, block_on_lock: bool = False, ) -> bool: @@ -1119,7 +1141,11 @@ def _rebuild_code( if not _stabilize_rebuild_cwd(watch_path): return False - out = watch_path / _GRAPHIFY_OUT + output_root = resolve_output_root(watch_path) + out = output_root / _GRAPHIFY_OUT + if multigraph is True: + _write_build_config(out, excludes=None, multigraph=True) + effective_multigraph = bool(multigraph or _read_build_multigraph(out)) if acquire_lock: # #1059: incremental (changed_paths is not None) hooks must not drop # their change set when another rebuild is already running. Queue @@ -1150,6 +1176,7 @@ def _rebuild_code( follow_symlinks=follow_symlinks, force=force, no_cluster=no_cluster, + multigraph=effective_multigraph, acquire_lock=False, ) # Late-arrival drain: another hook may have queued work while we @@ -1167,6 +1194,7 @@ def _rebuild_code( follow_symlinks=follow_symlinks, force=force, no_cluster=no_cluster, + multigraph=effective_multigraph, acquire_lock=False, ) and ok return ok @@ -1196,6 +1224,7 @@ def _rebuild_code( detected = detect( watch_path, follow_symlinks=follow_symlinks, extra_excludes=_persisted_excludes or None, + cache_root=output_root, gitignore=_gitignore_enabled, ) code_files = [Path(f) for f in detected['files']['code']] @@ -1453,7 +1482,7 @@ def _add_deleted_source(path: Path) -> None: commit = _git_head(cwd=watch_root) result = extract( extract_targets, - cache_root=watch_root, + cache_root=output_root, resolution_context_nodes=resolution_context_nodes or None, resolution_context_edges=resolution_context_edges or None, ) if extract_targets else { @@ -1555,16 +1584,34 @@ def _failed(f: str) -> bool: # Dedupe parallel edges (the clustered path's DiGraph collapses them implicitly); # without it, --no-cluster + repeated `update` accumulate duplicates and edge # counts diverge across build modes (#1317). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - candidate_graph_data = { - **{k: v for k, v in result.items() if k not in ("edges", "nodes")}, - "nodes": _dedupe_nodes(result.get("nodes", [])), - "links": _dedupe_edges(result.get("edges", [])), - # Inherit the existing graph's directed flag (#2342) so - # `graphify update --no-cluster` can't silently drop it - - # `result` (the raw merged extraction) never carries one. - "directed": bool((existing_graph_data or {}).get("directed", False)), - } + if effective_multigraph: + from networkx.readwrite import json_graph + from graphify.build import build as _build_raw_multigraph + + raw_graph = _build_raw_multigraph( + [result], + multigraph=True, + dedup=True, + root=watch_root, + ) + try: + candidate_graph_data = json_graph.node_link_data( + raw_graph, edges="links" + ) + except TypeError: + candidate_graph_data = json_graph.node_link_data(raw_graph) + else: + from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes + candidate_graph_data = { + **{k: v for k, v in result.items() if k not in ("edges", "nodes")}, + "nodes": _dedupe_nodes(result.get("nodes", [])), + "links": _dedupe_edges(result.get("edges", [])), + # Inherit the existing graph's directed flag (#2342) so + # `graphify update --no-cluster` can't silently drop it - + # `result` (the raw merged extraction) never carries one. + "directed": bool((existing_graph_data or {}).get("directed", False)), + "multigraph": False, + } candidate_graph_text = _json_text(candidate_graph_data) same_graph = False if existing_graph.exists(): @@ -1652,7 +1699,14 @@ def _failed(f: str) -> bool: # Inherit the existing graph's directed flag (#2342) so `graphify # update` can't silently downgrade a directed graph to undirected - # build_from_json defaults to directed=False otherwise. - G = build_from_json(result, directed=bool((existing_graph_data or {}).get("directed", False))) + G = build_from_json( + result, + directed=bool((existing_graph_data or {}).get("directed", False)), + multigraph=( + effective_multigraph + or bool((existing_graph_data or {}).get("multigraph", False)) + ), + ) candidate_topology = _topology_from_graph(G) if existing_graph_data: try: @@ -1894,7 +1948,7 @@ def check_update(watch_path: Path) -> bool: re-extraction via `/graphify --update` — this function only signals that the update is needed. """ - flag = Path(watch_path) / _GRAPHIFY_OUT / "needs_update" + flag = graphify_out_dir(watch_path) / "needs_update" if flag.exists(): print(f"[graphify check-update] Pending non-code changes in {watch_path}.") print("[graphify check-update] Run `/graphify --update` to apply semantic re-extraction.") @@ -1903,7 +1957,7 @@ def check_update(watch_path: Path) -> bool: def _notify_only(watch_path: Path) -> None: """Write a flag file and print a notification (fallback for non-code-only corpora).""" - flag = watch_path / _GRAPHIFY_OUT / "needs_update" + flag = graphify_out_dir(watch_path) / "needs_update" flag.parent.mkdir(parents=True, exist_ok=True) flag.write_text("1", encoding="utf-8") print(f"\n[graphify watch] New or changed files detected in {watch_path}") @@ -1971,7 +2025,7 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None: watch_root_for_ignore = watch_path.resolve() ignore_patterns = _load_graphifyignore( watch_root_for_ignore, - gitignore=_read_build_gitignore(watch_path / _GRAPHIFY_OUT), + gitignore=_read_build_gitignore(graphify_out_dir(watch_path)), ) class Handler(FileSystemEventHandler): diff --git a/tests/test_build.py b/tests/test_build.py index b376b173be..50c19708bc 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -3,11 +3,133 @@ import networkx as nx import pytest from networkx.readwrite import json_graph -from graphify.build import build_from_json, build, build_merge, edge_data, edge_datas, dedupe_edges, dedupe_nodes +from graphify.build import ( + analysis_projection, + build_from_json, + build, + build_merge, + edge_data, + edge_datas, + dedupe_edges, + dedupe_nodes, +) FIXTURES = Path(__file__).parent / "fixtures" +def test_multigraph_preserves_distinct_parallel_records_and_counts_duplicates(): + edge = { + "source": "a", + "target": "b", + "relation": "imports_from", + "source_location": "L1", + } + extraction = { + "nodes": [{"id": "a"}, {"id": "b"}], + "edges": [ + edge, + dict(edge), + { + "source": "a", + "target": "b", + "relation": "re_exports", + "source_location": "L2", + }, + { + "source": "a", + "target": "b", + "relation": "imports_from", + "source_location": "L3", + }, + ], + } + + graph = build_from_json(extraction, multigraph=True) + + assert isinstance(graph, nx.MultiDiGraph) + assert graph.number_of_edges("a", "b") == 3 + rows = edge_datas(graph, "a", "b") + assert {(row["relation"], row["source_location"]) for row in rows} == { + ("imports_from", "L1"), + ("re_exports", "L2"), + ("imports_from", "L3"), + } + assert next(row for row in rows if row["source_location"] == "L1")["occurrence_count"] == 2 + + +def test_multigraph_edge_keys_survive_node_link_roundtrip(): + graph = build_from_json( + { + "nodes": [{"id": "a"}, {"id": "b"}], + "edges": [ + {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, + {"source": "a", "target": "b", "relation": "calls", "source_location": "L2"}, + ], + }, + multigraph=True, + ) + + payload = json_graph.node_link_data(graph, edges="links") + restored = json_graph.node_link_graph(payload, edges="links") + + assert payload["directed"] is True + assert payload["multigraph"] is True + assert set(graph["a"]["b"]) == set(restored["a"]["b"]) + + +def test_analysis_projection_counts_each_directed_pair_once(): + graph = nx.MultiDiGraph() + graph.add_nodes_from(["a", "b"]) + graph.add_edge("a", "b", key="one") + graph.add_edge("a", "b", key="two") + graph.add_edge("b", "a", key="reverse") + + projected = analysis_projection(graph) + + assert isinstance(projected, nx.Graph) + assert projected.number_of_edges() == 1 + assert projected["a"]["b"]["weight"] == 1.0 + + +def test_build_merge_inherits_existing_multigraph_mode(tmp_path): + graph_path = tmp_path / "graph.json" + existing = build_from_json( + { + "nodes": [{"id": "a"}, {"id": "b"}], + "edges": [ + {"source": "a", "target": "b", "relation": "calls", "source_location": "L1"}, + {"source": "a", "target": "b", "relation": "references", "source_location": "L2"}, + ], + }, + multigraph=True, + ) + graph_path.write_text( + json.dumps(json_graph.node_link_data(existing, edges="links")), + encoding="utf-8", + ) + + merged = build_merge([], graph_path=graph_path, dedup=False) + + assert isinstance(merged, nx.MultiDiGraph) + assert merged.number_of_edges("a", "b") == 2 + + +def test_build_merge_refuses_explicit_multigraph_downgrade(tmp_path): + graph_path = tmp_path / "graph.json" + graph_path.write_text( + json.dumps({ + "directed": True, + "multigraph": True, + "nodes": [{"id": "a"}], + "links": [], + }), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="refusing to simplify"): + build_merge([], graph_path=graph_path, multigraph=False, dedup=False) + + def test_dedupe_edges_collapses_exact_parallels(): # #1317: --no-cluster / incremental update concatenate edge lists raw. edges = [ diff --git a/tests/test_endpoint_classification.py b/tests/test_endpoint_classification.py new file mode 100644 index 0000000000..7ffd55ee38 --- /dev/null +++ b/tests/test_endpoint_classification.py @@ -0,0 +1,79 @@ +from pathlib import Path + +from graphify.diagnostics import diagnose_extraction +from graphify.extract import extract + + +def test_js_import_endpoints_are_explicitly_classified_and_materialized( + tmp_path: Path, +) -> None: + source = tmp_path / "src/app.ts" + source.parent.mkdir(parents=True) + source.write_text( + "import React from 'react'\n" + "import { Missing } from './missing.js'\n" + "import stylesheet from './styles.css?url'\n", + encoding="utf-8", + ) + (source.parent / "styles.css").write_text("body {}", encoding="utf-8") + + result = extract( + [source], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + imports = [ + edge for edge in result["edges"] + if edge.get("relation") == "imports_from" + ] + node_ids = {node["id"] for node in result["nodes"]} + + assert len(imports) == 3 + assert sum(edge.get("external") is True for edge in imports) == 1 + assert sum(edge.get("unresolved_internal") is True for edge in imports) == 1 + assert sum(edge.get("excluded_local") is True for edge in imports) == 1 + assert all(edge["target"] in node_ids for edge in imports) + + summary = diagnose_extraction(result, root=tmp_path) + assert summary["external_endpoint_edges"] == 1 + assert summary["unresolved_internal_endpoint_edges"] == 1 + assert summary["excluded_local_endpoint_edges"] == 1 + assert summary["dangling_endpoint_edges"] == 0 + assert summary["unclassified_endpoint_edges"] == 0 + + +def test_existing_but_unscanned_tsconfig_alias_is_excluded_local( + tmp_path: Path, +) -> None: + source = tmp_path / "src/app.ts" + source.parent.mkdir(parents=True) + source.write_text( + "import { browser } from 'collections/browser'\n", + encoding="utf-8", + ) + hidden = tmp_path / ".source/browser.ts" + hidden.parent.mkdir() + hidden.write_text("export const browser = true\n", encoding="utf-8") + (tmp_path / "tsconfig.json").write_text( + '{"compilerOptions":{"paths":{"collections/*":["./.source/*"]}}}', + encoding="utf-8", + ) + + result = extract( + [source], + cache_root=tmp_path / "cache", + root=tmp_path, + parallel=False, + ) + edge = next( + edge + for edge in result["edges"] + if edge.get("relation") == "imports_from" + ) + endpoint = next(node for node in result["nodes"] if node["id"] == edge["target"]) + + assert edge.get("excluded_local") is True + assert edge.get("unresolved_internal") is not True + assert endpoint.get("excluded_local") is True + assert endpoint["source_file"] == ".source/browser.ts" diff --git a/tests/test_export.py b/tests/test_export.py index d957b87957..4c3fa1b4b3 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -288,6 +288,22 @@ def test_to_graphml_preserves_native_scalar_types(): assert H.nodes["a"]["name"] == "x" +def test_to_graphml_preserves_parallel_multigraph_edges(tmp_path): + import networkx as nx + + graph = nx.MultiDiGraph() + graph.add_node("a", label="A") + graph.add_node("b", label="B") + graph.add_edge("a", "b", key="calls", relation="calls") + graph.add_edge("a", "b", key="imports", relation="imports") + output = tmp_path / "parallel.graphml" + + to_graphml(graph, {0: ["a", "b"]}, str(output)) + restored = nx.read_graphml(output) + + assert restored.number_of_edges("a", "b") == 2 + + def test_to_html_creates_file(): G = make_graph() communities = cluster(G) diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py index 9853bbdd1f..0e6a9ba493 100644 --- a/tests/test_extract_code_only_cli.py +++ b/tests/test_extract_code_only_cli.py @@ -80,6 +80,15 @@ def _run_relative_out(repo: Path, *extra: str): ) +def _run_graphify(repo: Path, *args: str): + env = {k: v for k, v in os.environ.items() if k not in _KEY_VARS} + env["GRAPHIFY_OUT"] = "graphify-out" + return subprocess.run( + [PYTHON, "-m", "graphify", *args], + cwd=repo, capture_output=True, text=True, env=env, + ) + + def test_output_flag_is_alias_of_out(tmp_path): """#2004 part 3: `--output DIR` was silently ignored on extract (output went to the default `/graphify-out/`). It is now an alias of `--out`.""" @@ -104,6 +113,87 @@ def test_output_flag_inline_form(tmp_path): assert (custom / "graphify-out" / "graph.json").exists() +def test_external_output_is_reused_by_extract_update_and_query(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "origin.ts").write_text("export const value = 1\n", encoding="utf-8") + (repo / "barrel.ts").write_text( + "import { value } from './origin.js'\n" + "export { value } from './origin.js'\n", + encoding="utf-8", + ) + (repo / "README.md").write_text("# External output\n", encoding="utf-8") + external = tmp_path / "external" + + first = _run_relative_out( + repo, + "--code-only", + "--no-cluster", + "--multigraph", + "--out", + str(external), + ) + assert first.returncode == 0, first.stderr + + graph_out = external / "graphify-out" + graph = json.loads((graph_out / "graph.json").read_text(encoding="utf-8")) + config = json.loads( + (graph_out / ".graphify_build.json").read_text(encoding="utf-8") + ) + assert graph["directed"] is True + assert graph["multigraph"] is True + assert all(node.get("community") is None for node in graph["nodes"]) + assert not (graph_out / ".graphify_analysis.json").exists() + assert config["multigraph"] is True + assert not (repo / "graphify-out").exists() + + (repo / "later.ts").write_text( + "export const persistedOutput = true\n", + encoding="utf-8", + ) + second = _run_relative_out(repo, "--code-only", "--no-cluster", "--force") + assert second.returncode == 0, second.stderr + persisted = json.loads((graph_out / "graph.json").read_text(encoding="utf-8")) + assert persisted["directed"] is True + assert persisted["multigraph"] is True + assert any(node.get("label") == "persistedOutput" for node in persisted["nodes"]) + assert not (repo / "graphify-out").exists() + + (repo / "updated.ts").write_text( + "export const updatedThroughPersistentOut = true\n", + encoding="utf-8", + ) + updated = _run_graphify(repo, "update", ".", "--no-cluster") + assert updated.returncode == 0, updated.stderr + after_update = json.loads((graph_out / "graph.json").read_text(encoding="utf-8")) + assert any( + node.get("label") == "updatedThroughPersistentOut" + for node in after_update["nodes"] + ) + assert after_update["multigraph"] is True + assert not (repo / "graphify-out").exists() + + queried = _run_graphify(repo, "query", "persistedOutput") + assert queried.returncode == 0, queried.stderr + assert "persistedOutput" in queried.stdout + + exported = _run_graphify(repo, "export", "graphml") + assert exported.returncode == 0, exported.stderr + assert (graph_out / "graph.graphml").exists() + + callflow_path = graph_out / "callflow.html" + callflow = _run_graphify( + repo, + "export", + "callflow-html", + str(repo), + "--output", + str(callflow_path), + ) + assert callflow.returncode == 0, callflow.stderr + assert callflow_path.exists() + + def test_no_gitignore_indexes_vcs_ignored_code_but_keeps_graphifyignore(tmp_path): repo = tmp_path / "repo" generated = repo / "proj" / "deep" / "generated" diff --git a/tests/test_import_extension_resolution.py b/tests/test_import_extension_resolution.py index 0d1222c0a0..9d8f22be34 100644 --- a/tests/test_import_extension_resolution.py +++ b/tests/test_import_extension_resolution.py @@ -8,6 +8,8 @@ so `build_from_json` dropped the edge as external. """ +import json + from pathlib import Path from graphify.extract import ( @@ -155,6 +157,39 @@ def test_resolve_js_to_ts_when_real_file_is_ts(tmp_path): assert _resolve_js_module_path(written_as) == target +def test_resolve_js_to_tsx_when_real_file_is_tsx(tmp_path): + target = _write(tmp_path / "button.tsx", "export const Button = () => null") + assert _resolve_js_module_path(tmp_path / "button.js") == target + + +def test_resolve_js_to_declaration_when_runtime_file_is_absent(tmp_path): + target = _write(tmp_path / "server.d.ts", "export type MutationCtx = object") + assert _resolve_js_module_path(tmp_path / "server.js") == target + + +def test_resource_query_is_ignored_for_filesystem_resolution(tmp_path): + target = _write(tmp_path / "styles.css", "body {}") + assert _resolve_js_module_path("./styles.css?url", tmp_path) == target + + +def test_package_import_map_prefers_scanned_source(tmp_path): + source = _write(tmp_path / "src" / "models.ts", "export const model = 1\n") + _write(tmp_path / "dist" / "models.js", "export const model = 1\n") + (tmp_path / "package.json").write_text( + json.dumps({ + "imports": { + "#/*": { + "development": "./src/*.ts", + "default": "./dist/*.js", + } + } + }), + encoding="utf-8", + ) + + assert _resolve_js_module_path("#/models?raw", source.parent) == source + + def test_resolve_jsx_to_tsx_when_real_file_is_tsx(tmp_path): target = _write(tmp_path / "Component.tsx", "export const x = 1") written_as = tmp_path / "Component.jsx" diff --git a/tests/test_js_import_resolution.py b/tests/test_js_import_resolution.py index cc464e4b69..64ad70f3a6 100644 --- a/tests/test_js_import_resolution.py +++ b/tests/test_js_import_resolution.py @@ -726,6 +726,105 @@ def test_workspace_subpath_export_condition_object_resolves(tmp_path: Path): assert _has_edge(result, "apps/web/src/consumer.ts", "packages/pkg-a/src/browser.ts") +def test_workspace_export_prefers_source_when_runtime_dist_exists(tmp_path: Path): + _write(tmp_path / "pnpm-workspace.yaml", "packages:\n - 'apps/*'\n - 'packages/*'\n") + _write( + tmp_path / "packages/db/package.json", + json.dumps({ + "name": "@acme/db", + "exports": { + ".": { + "types": "./src/index.ts", + "development": "./src/index.ts", + "import": "./dist/index.js", + }, + }, + }), + ) + source_entry = _write( + tmp_path / "packages/db/src/index.ts", + "export const value = 1\n", + ) + dist_entry = _write( + tmp_path / "packages/db/dist/index.js", + "export const value = 1\n", + ) + importer = _write( + tmp_path / "apps/bot/src/app.ts", + "import { value } from '@acme/db'\nexport const result = value\n", + ) + + result = _extract_for([source_entry, dist_entry, importer], tmp_path) + + assert _has_edge(result, "apps/bot/src/app.ts", "packages/db/src/index.ts") + assert not _has_edge(result, "apps/bot/src/app.ts", "packages/db/dist/index.js") + + +def test_runtime_module_uses_companion_declaration_exports(tmp_path: Path): + runtime = _write( + tmp_path / "generated/server.js", + "export const mutation = mutationGeneric\n", + ) + declaration = _write( + tmp_path / "generated/server.d.ts", + "export declare const mutation: unknown\nexport type MutationCtx = object\n", + ) + consumer = _write( + tmp_path / "consumer.ts", + "import { mutation, type MutationCtx } from './generated/server.js'\n" + "export type Ctx = MutationCtx\nexport const run = mutation\n", + ) + + result = _extract_for([runtime, declaration, consumer], tmp_path) + + assert _has_symbol_edge(result, "consumer.ts", "generated/server.js", "mutation") + assert _has_symbol_edge(result, "consumer.ts", "generated/server.d.ts", "MutationCtx") + assert all("specifier_symbol" not in edge for edge in result["edges"]) + + +def test_ambient_const_declaration_is_a_resolvable_export(tmp_path: Path): + declaration = _write( + tmp_path / "generated/api.d.ts", + "export declare const api: Record;\n", + ) + consumer = _write( + tmp_path / "consumer.ts", + "import { api } from './generated/api.js'\nexport const value = api\n", + ) + + result = _extract_for([declaration, consumer], tmp_path) + + assert _has_symbol_edge(result, "consumer.ts", "generated/api.d.ts", "api") + + +def test_named_reexport_through_star_barrel_targets_defining_symbol(tmp_path: Path): + origin = _write( + tmp_path / "origin.ts", + "export function sourceName() { return 1 }\n", + ) + barrel = _write(tmp_path / "index.ts", "export * from './origin.js'\n") + wrapper = _write( + tmp_path / "wrapper.ts", + "export { sourceName as publicName } from './index.js'\n", + ) + + result = _extract_for([origin, barrel, wrapper], tmp_path) + wrapper_id = _file_node_id(Path("wrapper.ts")) + origin_symbol = _make_id(_file_stem(Path("origin.ts")), "sourceName") + reexports = [ + edge + for edge in result["edges"] + if edge.get("source") == wrapper_id + and edge.get("relation") == "re_exports" + and edge.get("source_location") == "L1" + ] + + assert [ + edge["target"] for edge in reexports if edge["target"] == origin_symbol + ] == [origin_symbol] + assert not any(edge.get("unresolved_internal") is True for edge in reexports) + + def test_workspace_subpath_export_wildcard_resolves(tmp_path: Path): _write( tmp_path / "pnpm-workspace.yaml", diff --git a/tests/test_paths.py b/tests/test_paths.py index 12359006fb..0f20e32c6f 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -144,3 +144,60 @@ def test_is_absolute_any_platform_is_host_independent(): from graphify.paths import is_absolute_any_platform assert is_absolute_any_platform("/home/ci/x.md") assert is_absolute_any_platform("C:/Users/u/x.md") + + +def test_output_root_binding_applies_to_project_subdirectories(tmp_path) -> None: + from graphify.paths import persisted_output_root, remember_output_root + + project = tmp_path / "project" + nested = project / "packages/app" + output = tmp_path / "graphs/project" + nested.mkdir(parents=True) + + remember_output_root(project, output) + + assert persisted_output_root(project) == output.resolve() + assert persisted_output_root(nested) == output.resolve() + + +def test_nearest_output_root_binding_wins(tmp_path) -> None: + from graphify.paths import persisted_output_root, remember_output_root + + project = tmp_path / "project" + package = project / "packages/app" + project_output = tmp_path / "graphs/project" + package_output = tmp_path / "graphs/app" + package.mkdir(parents=True) + + remember_output_root(project, project_output) + remember_output_root(package, package_output) + + assert persisted_output_root(package / "src") == package_output.resolve() + assert persisted_output_root(project / "other") == project_output.resolve() + + +def test_explicit_output_root_replaces_persisted_selection(tmp_path) -> None: + from graphify.paths import persisted_output_root, remember_output_root + + project = tmp_path / "project" + project.mkdir() + first = tmp_path / "first" + second = tmp_path / "second" + + remember_output_root(project, first) + remember_output_root(project, second) + + assert persisted_output_root(project) == second.resolve() + + +def test_nondefault_graphify_out_environment_overrides_binding( + tmp_path, monkeypatch +) -> None: + from graphify.paths import remember_output_root, resolve_output_root + + project = tmp_path / "project" + project.mkdir() + remember_output_root(project, tmp_path / "remembered") + monkeypatch.setenv("GRAPHIFY_OUT", "graphify-out-worktree") + + assert resolve_output_root(project) == project.resolve() diff --git a/tests/test_watch.py b/tests/test_watch.py index 25d0cd9673..d9cdabc0eb 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -635,6 +635,30 @@ def test_rebuild_honors_persisted_no_gitignore(tmp_path): assert any(source.endswith("generated/gen.py") for source in sources) +def test_rebuild_honors_persisted_multigraph_mode(tmp_path): + import json + from graphify.watch import _rebuild_code, _write_build_config + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "lib.py").write_text("def value(): return 1\n", encoding="utf-8") + _write_build_config( + corpus / "graphify-out", excludes=None, multigraph=True + ) + + assert _rebuild_code( + corpus, + no_cluster=True, + acquire_lock=False, + ) is True + + graph = json.loads( + (corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8") + ) + assert graph["directed"] is True + assert graph["multigraph"] is True + + def test_graphify_root_preserves_absolute_when_user_supplied(tmp_path): """When the caller supplies an absolute path, ``.graphify_root`` stores that absolute form verbatim — preserving explicit-absolute intent.""" From 1de265d24d584bff709bdf954f9b8f90c254f9ca Mon Sep 17 00:00:00 2001 From: Neonsy <118444485+Neonsy@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:39:17 +0200 Subject: [PATCH 2/4] fix: preserve directed edges in surprise analysis --- graphify/analyze.py | 16 ++++++++++++---- tests/test_analyze.py | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/graphify/analyze.py b/graphify/analyze.py index a112c692e9..6f2de097d3 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -360,13 +360,21 @@ def _cross_community_surprises( top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n] result = [] for (u, v), score in top_edges: + if G.is_directed() and not G.has_edge(u, v): + u, v = v, u data = edge_data(G, u, v) + src_id = data.get("_src", u) + if src_id not in G.nodes: + src_id = u + tgt_id = data.get("_tgt", v) + if tgt_id not in G.nodes: + tgt_id = v result.append({ - "source": G.nodes[u].get("label", u), - "target": G.nodes[v].get("label", v), + "source": G.nodes[src_id].get("label", src_id), + "target": G.nodes[tgt_id].get("label", tgt_id), "source_files": [ - G.nodes[u].get("source_file", ""), - G.nodes[v].get("source_file", ""), + G.nodes[src_id].get("source_file", ""), + G.nodes[tgt_id].get("source_file", ""), ], "confidence": data.get("confidence", "EXTRACTED"), "relation": data.get("relation", ""), diff --git a/tests/test_analyze.py b/tests/test_analyze.py index 7bff432cf7..1ec2d17053 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -87,6 +87,27 @@ def test_surprising_connections_single_file_uses_community_bridges(): assert len(surprises) > 0 +def test_surprising_connections_without_communities_preserves_directed_edge_order(): + G = nx.MultiDiGraph() + G.add_node("z", label="Caller", file_type="code", source_file="single.py") + G.add_node("a", label="Callee", file_type="code", source_file="single.py") + G.add_edge( + "z", + "a", + relation="calls", + confidence="INFERRED", + source_file="single.py", + ) + + surprises = surprising_connections(G, {}) + + assert len(surprises) == 1 + assert surprises[0]["source"] == "Caller" + assert surprises[0]["target"] == "Callee" + assert surprises[0]["relation"] == "calls" + assert surprises[0]["confidence"] == "INFERRED" + + def test_surprising_connections_ambiguous_scores_higher_than_extracted(): """AMBIGUOUS edge should score higher than an otherwise identical EXTRACTED edge.""" G = nx.Graph() From af4505a2553c320ce0e3440b30ac3f3ee4664fdc Mon Sep 17 00:00:00 2001 From: Neonsy <118444485+Neonsy@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:39:49 +0200 Subject: [PATCH 3/4] fix: preserve simple graph edge keys --- graphify/build.py | 5 ++++- tests/test_build.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/graphify/build.py b/graphify/build.py index 3e8775d542..0e8b775e68 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -1260,10 +1260,13 @@ def build_from_json( # strings, NaN/inf, negatives — while numeric strings coerce cleanly. # Repair (not drop) the key so graph.json round-trips a clean value and a # cluster-only/--update reload never re-ingests the null. + excluded_attrs = {"source", "target", "target_file", "local_alias"} + if multigraph: + excluded_attrs.add("key") attrs = { k: v for k, v in edge.items() - if k not in ("source", "target", "target_file", "local_alias", "key") + if k not in excluded_attrs } for _num_key in ("weight", "confidence_score"): if _num_key in attrs: diff --git a/tests/test_build.py b/tests/test_build.py index 50c19708bc..9306e70416 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -77,6 +77,24 @@ def test_multigraph_edge_keys_survive_node_link_roundtrip(): assert set(graph["a"]["b"]) == set(restored["a"]["b"]) +def test_simple_graph_preserves_edge_attribute_named_key(): + graph = build_from_json( + { + "nodes": [{"id": "a"}, {"id": "b"}], + "edges": [ + { + "source": "a", + "target": "b", + "relation": "calls", + "key": "domain-key", + }, + ], + } + ) + + assert graph["a"]["b"]["key"] == "domain-key" + + def test_analysis_projection_counts_each_directed_pair_once(): graph = nx.MultiDiGraph() graph.add_nodes_from(["a", "b"]) From 698d37e2c1d1f0223001028cfbfdb9df7aed3c65 Mon Sep 17 00:00:00 2001 From: Neonsy <118444485+Neonsy@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:42:09 +0200 Subject: [PATCH 4/4] fix: validate persisted update roots --- graphify/cli.py | 29 ++++++++++++++++++++++----- graphify/paths.py | 42 +++++++++++++++++++++++++++++---------- tests/test_paths.py | 48 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/graphify/cli.py b/graphify/cli.py index 1cfaa459c4..b80d878018 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -15,6 +15,7 @@ GRAPHIFY_OUT as _GRAPHIFY_OUT, graphify_out_dir, remember_output_root, + resolve_output_binding, resolve_output_root, ) from pathlib import Path, PurePosixPath, PureWindowsPath @@ -90,6 +91,24 @@ def _default_graph_path() -> str: return str(graphify_out_dir(Path.cwd()) / "graph.json") +def _default_update_path(source_path: Path) -> Path: + """Recover and validate the scan root for ``graphify update`` without a path.""" + saved = graphify_out_dir(source_path) / ".graphify_root" + if not saved.exists(): + return Path(".") + + watch_path = Path(saved.read_text(encoding="utf-8").strip()) + binding = resolve_output_binding(source_path) + if binding is not None: + expected_source, _ = binding + if watch_path.expanduser().resolve() != expected_source: + raise ValueError( + f"saved scan root {watch_path} does not match configured project " + f"{expected_source}; pass the source path explicitly" + ) + return watch_path + + def _stamped_manifest_files( files_by_type: dict[str, list[str]], sem_result: dict, @@ -2293,11 +2312,11 @@ def _clear_html_stale_marker() -> None: watch_path = Path(watch_arg) else: # Try to recover the scan root saved by the last full build - saved = graphify_out_dir(Path.cwd()) / ".graphify_root" - if saved.exists(): - watch_path = Path(saved.read_text(encoding="utf-8").strip()) - else: - watch_path = Path(".") + try: + watch_path = _default_update_path(Path.cwd()) + except (OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) if not watch_path.exists(): print(f"error: path not found: {watch_path}", file=sys.stderr) sys.exit(1) diff --git a/graphify/paths.py b/graphify/paths.py index 1c03d82d96..dc9f107477 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -137,13 +137,10 @@ def remember_output_root(source_root: "str | Path", output_root: "str | Path") - ) -def persisted_output_root(source_root: "str | Path") -> "Path | None": - """Return the nearest remembered output root for a source path. - - Exact bindings win. Walking parents lets commands launched from a project - subdirectory find the output configured at the project root. Malformed or - mismatched binding files are ignored rather than redirecting output. - """ +def _persisted_output_binding( + source_root: "str | Path", +) -> "tuple[Path, Path] | None": + """Return the nearest valid ``(source, output)`` binding.""" source = _canonical_local_path(source_root) for candidate in (source, *source.parents): path = _output_root_binding_path(candidate) @@ -161,10 +158,34 @@ def persisted_output_root(source_root: "str | Path") -> "Path | None": ): continue if _local_path_identity(stored_source) == _local_path_identity(candidate): - return stored_output + return candidate, stored_output return None +def resolve_output_binding( + source_root: "str | Path", +) -> "tuple[Path, Path] | None": + """Return the remembered binding used by default output resolution. + + A non-default ``GRAPHIFY_OUT`` is an explicit override and therefore has no + active remembered binding. + """ + if os.environ.get("GRAPHIFY_OUT", "graphify-out") != "graphify-out": + return None + return _persisted_output_binding(source_root) + + +def persisted_output_root(source_root: "str | Path") -> "Path | None": + """Return the nearest remembered output root for a source path. + + Exact bindings win. Walking parents lets commands launched from a project + subdirectory find the output configured at the project root. Malformed or + mismatched binding files are ignored rather than redirecting output. + """ + binding = _persisted_output_binding(source_root) + return binding[1] if binding is not None else None + + def resolve_output_root( source_root: "str | Path", explicit_output_root: "str | Path | None" = None, @@ -178,9 +199,10 @@ def resolve_output_root( source = _canonical_local_path(source_root) if explicit_output_root is not None: return _canonical_local_path(explicit_output_root) - if os.environ.get("GRAPHIFY_OUT", "graphify-out") != "graphify-out": + binding = resolve_output_binding(source) + if binding is None: return source - return persisted_output_root(source) or source + return binding[1] def graphify_out_dir( diff --git a/tests/test_paths.py b/tests/test_paths.py index 0f20e32c6f..33b2c67647 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -147,7 +147,11 @@ def test_is_absolute_any_platform_is_host_independent(): def test_output_root_binding_applies_to_project_subdirectories(tmp_path) -> None: - from graphify.paths import persisted_output_root, remember_output_root + from graphify.paths import ( + persisted_output_root, + remember_output_root, + resolve_output_binding, + ) project = tmp_path / "project" nested = project / "packages/app" @@ -158,6 +162,7 @@ def test_output_root_binding_applies_to_project_subdirectories(tmp_path) -> None assert persisted_output_root(project) == output.resolve() assert persisted_output_root(nested) == output.resolve() + assert resolve_output_binding(nested) == (project.resolve(), output.resolve()) def test_nearest_output_root_binding_wins(tmp_path) -> None: @@ -193,7 +198,11 @@ def test_explicit_output_root_replaces_persisted_selection(tmp_path) -> None: def test_nondefault_graphify_out_environment_overrides_binding( tmp_path, monkeypatch ) -> None: - from graphify.paths import remember_output_root, resolve_output_root + from graphify.paths import ( + remember_output_root, + resolve_output_binding, + resolve_output_root, + ) project = tmp_path / "project" project.mkdir() @@ -201,3 +210,38 @@ def test_nondefault_graphify_out_environment_overrides_binding( monkeypatch.setenv("GRAPHIFY_OUT", "graphify-out-worktree") assert resolve_output_root(project) == project.resolve() + assert resolve_output_binding(project) is None + + +def test_default_update_path_accepts_marker_for_bound_source(tmp_path) -> None: + from graphify.cli import _default_update_path + from graphify.paths import remember_output_root + + project = tmp_path / "project" + nested = project / "packages/app" + output = tmp_path / "graphs/project" + nested.mkdir(parents=True) + graphify_out = output / "graphify-out" + graphify_out.mkdir(parents=True) + (graphify_out / ".graphify_root").write_text(str(project.resolve()), encoding="utf-8") + remember_output_root(project, output) + + assert _default_update_path(nested) == project.resolve() + + +def test_default_update_path_rejects_marker_for_another_source(tmp_path) -> None: + from graphify.cli import _default_update_path + from graphify.paths import remember_output_root + + project = tmp_path / "project" + other = tmp_path / "other" + output = tmp_path / "graphs/project" + project.mkdir() + other.mkdir() + graphify_out = output / "graphify-out" + graphify_out.mkdir(parents=True) + (graphify_out / ".graphify_root").write_text(str(other.resolve()), encoding="utf-8") + remember_output_root(project, output) + + with pytest.raises(ValueError, match="does not match configured project"): + _default_update_path(project)