diff --git a/graphify/extract.py b/graphify/extract.py index c8a4f0bbff..bd237c07a6 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -147,6 +147,27 @@ from graphify.extractors.objc import _objc_local_var_types, extract_objc # noqa: E402,F401 from graphify.extractors.julia import extract_julia # noqa: E402,F401 +from graphify.extractors.lazarus_package import extract_lazarus_package # noqa: E402,F401 +from graphify.extractors.slnx import extract_slnx # noqa: E402,F401 +from graphify.extractors.csproj import extract_csproj # noqa: E402,F401 +from graphify.extractors.base import _shorten_rationale_label # noqa: E402,F401 +from graphify.extractors.xaml import extract_xaml # noqa: E402,F401 +from graphify.extractors.python import _PYTHON_CONFIG, _RATIONALE_PREFIXES, _import_python, _is_autogenerated_python, _extract_python_rationale, extract_python # noqa: E402,F401 +from graphify.extractors.js import _JS_CONFIG, _TS_CONFIG, _TSX_CONFIG, _import_js, _rescue_js_dynamic_imports, _JS_RATIONALE_PREFIXES, _JS_DOC_REF_RE, _JS_COMMENT_LINE_RE, _extract_js_rationale, _resolve_rescued_specifier, _emit_rescued_import, extract_js # noqa: E402,F401 +from graphify.extractors.svelte import extract_svelte # noqa: E402,F401 +from graphify.extractors.astro import extract_astro # noqa: E402,F401 +from graphify.extractors.vue import extract_vue # noqa: E402,F401 +from graphify.extractors.java import _JAVA_CONFIG, _import_java, extract_java # noqa: E402,F401 +from graphify.extractors.groovy import _GROOVY_CONFIG, _is_spock_file, _extract_spock_fallback, extract_groovy # noqa: E402,F401 +from graphify.extractors.c import _C_CONFIG, _get_c_func_name, _import_c, extract_c # noqa: E402,F401 +from graphify.extractors.cpp import _CPP_CONFIG, extract_cpp # noqa: E402,F401 +from graphify.extractors.ruby import _RUBY_CONFIG, extract_ruby # noqa: E402,F401 +from graphify.extractors.csharp import _CSHARP_CONFIG, _import_csharp, extract_csharp # noqa: E402,F401 +from graphify.extractors.kotlin import _KOTLIN_CONFIG, _import_kotlin, extract_kotlin # noqa: E402,F401 +from graphify.extractors.scala import _SCALA_CONFIG, _import_scala, extract_scala # noqa: E402,F401 +from graphify.extractors.php import _PHP_CONFIG, _import_php, extract_php # noqa: E402,F401 +from graphify.extractors.lua import _LUA_CONFIG, _import_lua, extract_lua # noqa: E402,F401 +from graphify.extractors.swift import _SWIFT_CONFIG, _import_swift, extract_swift # noqa: E402,F401 _RECURSION_LIMIT = 10_000 @@ -310,1590 +331,6 @@ def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None: return None -# ── Import handlers ─────────────────────────────────────────────────────────── - -def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - t = node.type - if t == "import_statement": - for child in node.children: - if child.type in ("dotted_name", "aliased_import"): - raw = _read_text(child, source) - raw_module, _, raw_alias = raw.partition(" as ") - module_name = raw_module.strip().lstrip(".") - tgt_nid = _make_id(module_name) - edge = { - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - } - if raw_alias: - # `import pkg.mod as alias` binds the local name `alias`, not - # `mod`'s own stem, to the module -- stash it so the cross-file - # member-call resolver can match `alias.func()` against this - # edge instead of dropping it (#2082). - edge["local_alias"] = raw_alias.strip() - edges.append(edge) - elif t == "import_from_statement": - module_node = node.child_by_field_name("module_name") - if module_node: - raw = _read_text(module_node, source) - target_path: "Path | None" = None - if raw.startswith("."): - # Relative import - resolve to full path so IDs match file node IDs - dots = len(raw) - len(raw.lstrip(".")) - module_name = raw.lstrip(".") - base = Path(str_path).parent - for _ in range(dots - 1): - base = base.parent - rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py" - target_path = base / rel - tgt_nid = _make_id(str(target_path)) - else: - tgt_nid = _make_id(raw) - edge = { - "source": file_nid, - "target": tgt_nid, - "relation": "imports_from", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - } - # Stamp the resolved target file (mirroring _import_js, #1814) so - # the #2169 remap pass can canonicalize this edge's target on an - # incremental run where the target file itself is not in the - # batch — without it the target keeps an absolute-path-derived id - # that matches no node in the merged graph and dangles (#2213). - # Existence-gated: a speculative import of a nonexistent sibling - # must stay dangling, exactly as before. The stamp is transient - # and popped before graph.json ships. - if target_path is not None: - try: - if target_path.is_file(): - edge["target_file"] = str(target_path) - except OSError: - pass - edges.append(edge) - - -def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - is_reexport = node.type == "export_statement" - # Only handle export_statement if it has a `from` clause (re-export). - # Pure exports like `export const x = 1` or `export { localVar }` have no source module. - if is_reexport: - has_from = any(child.type == "from" or (_read_text(child, source) == "from") for child in node.children if child.type in ("from", "identifier")) - if not has_from: - # Check for string child (source path) as a more reliable indicator - has_from = any(child.type == "string" for child in node.children) - if not has_from: - return - - resolved_path: "Path | None" = None - module_string = None - for child in node.children: - if child.type == "string": - module_string = child - break - if child.type == "import_require_clause": - # TS import-equals form: `import x = require("./m")`. The module - # string sits inside the clause, not on the import_statement - # itself, so the direct-child scan above never sees it. - module_string = next( - (sub for sub in child.children if sub.type == "string"), None - ) - break - if module_string is not None: - raw = _read_text(module_string, source).strip("'\"` ") - resolved = _resolve_js_import_target(raw, str_path) - if resolved is not None: - tgt_nid, resolved_path = resolved - # `_resolve_js_import_path` returns the attempted path when no - # local file exists. Static ES imports must treat that as unresolved - # rather than minting a checkout-specific target ID (#2457). - if resolved_path is not None and not resolved_path.is_file(): - tgt_nid = _make_id("ref", raw) - resolved_path = None - edge = { - "source": file_nid, - "target": tgt_nid, - "relation": "imports_from", - "context": "re-export" if is_reexport else "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - } - # Stamp the resolved target file so a same-basename cross-extension - # sibling (foo.ts importing/re-exporting ./foo.mjs) keys its target salt - # by the TARGET's file rather than the importer's. Both files collapse to - # the base id `foo`; without this the salted lookup mis-points the target - # back onto the importer's own variant, a phantom self-loop (#1814). - if resolved_path is not None: - edge["target_file"] = str(resolved_path) - edges.append(edge) - - # Emit symbol-level edges for named imports/re-exports from local/aliased files. - # e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED) - # e.g. `export { Foo } from './bar'` → file → Foo (re_exports edge) - # Uses the same _make_id(target_stem, name) key that _extract_generic emits when - # defining the symbol, so these edges wire importers directly to existing symbol nodes. - if resolved_path is not None: - target_stem = _file_stem(resolved_path) - line = node.start_point[0] + 1 - - if is_reexport: - # Handle: export { foo, bar } from './module' - # export { default as baz } from './module' - for child in node.children: - if child.type == "export_clause": - for spec in child.children: - if spec.type == "export_specifier": - # The exported name is the local name from the source module - name_node = spec.child_by_field_name("name") - if name_node: - sym = _read_text(name_node, source) - if sym == "default": - continue # skip default re-exports for ID matching - edges.append({ - "source": file_nid, - "target": _make_id(target_stem, sym), - "relation": "re_exports", - "context": "re-export", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - # Which file this symbol target was synthesized - # from, so the id-remap post-pass can repoint a - # target the candidates rewrite never learns — - # a barrel defines no symbols (#1983). Transient, - # stripped at build like the #1814 stamp. - "target_file": str(resolved_path), - }) - else: - # Handle: import { Foo, type Bar } from './bar' - for child in node.children: - if child.type == "import_clause": - for sub in child.children: - if sub.type == "named_imports": - for spec in sub.children: - if spec.type == "import_specifier": - name_node = spec.child_by_field_name("name") - if name_node: - sym = _read_text(name_node, source) - edges.append({ - "source": file_nid, - "target": _make_id(target_stem, sym), - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - # See the re_exports stamp above (#1983). - "target_file": str(resolved_path), - }) - - -def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - def _walk_scoped(n) -> str: - parts: list[str] = [] - cur = n - while cur: - if cur.type == "scoped_identifier": - name_node = cur.child_by_field_name("name") - if name_node: - parts.append(_read_text(name_node, source)) - cur = cur.child_by_field_name("scope") - elif cur.type == "identifier": - parts.append(_read_text(cur, source)) - break - else: - break - parts.reverse() - return ".".join(parts) - - for child in node.children: - if child.type in ("scoped_identifier", "identifier"): - path_str = _walk_scoped(child) - module_name = path_str.split(".")[-1].strip("*").strip(".") or ( - path_str.split(".")[-2] if len(path_str.split(".")) > 1 else path_str - ) - if module_name: - tgt_nid = _make_id(module_name) - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) - break - - -def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - for child in node.children: - if child.type in ("string_literal", "system_lib_string", "string"): - raw = _read_text(child, source).strip('"<> ') - # Quoted includes: try to resolve to a real file so the target ID - # matches the node ID _extract_generic creates for that file. - if child.type != "system_lib_string": - resolved = _resolve_c_include_path(raw, str_path) - if resolved is not None: - tgt_nid = _make_id(str(resolved)) - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - # Stamp the resolved target, mirroring _import_python (#1814): - # without it, an include whose header lives outside this - # batch's paths keeps the raw absolute-path id no later pass - # ever learns to relativize (#2243). - "target_file": str(resolved), - }) - break - module_name = raw.split("/")[-1].split(".")[0] - if module_name: - tgt_nid = _make_id(module_name) - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) - break - - -def _import_csharp(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - text = _read_text(node, source).strip().rstrip(";") - if text.startswith("global "): - text = text[len("global "):].strip() - if not text.startswith("using"): - return - body = text[len("using"):].strip() - using_kind, alias, target_fqn = "namespace", None, body - if body.startswith("static "): - using_kind, target_fqn = "static", body[len("static "):].strip() - elif "=" in body: - lhs, rhs = body.split("=", 1) - using_kind, alias, target_fqn = "alias", lhs.strip(), rhs.strip() - if not target_fqn: - return - edges.append({ - "source": file_nid, - "target": _make_id(target_fqn), - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - "metadata": sanitize_metadata({k: v for k, v in - {"using_kind": using_kind, "alias": alias, "target_fqn": target_fqn, - "scope_kind": "namespace" if scope_stack else "file", - "scope_id": scope_stack[-1] if scope_stack else None}.items() if v is not None}), - }) - - -def _import_kotlin(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - # Grammar 1.1.0 (PyPI tree_sitter_kotlin) emits an `import` node whose - # children are the `import` keyword and a `qualified_identifier` (the dotted - # path), optionally followed by `.` `*` (wildcard) or `as` + `identifier` - # (alias). There is no `path` field. Older forks emit `import_header` with a - # `path` field or a bare `identifier` child; keep those branches so the - # extractor works across grammar generations (#2526, adapted from PR #2531 - # by @Mustaqeem66). - path_node = node.child_by_field_name("path") - if path_node is None: - path_node = next( - (c for c in node.children if c.type == "qualified_identifier"), None - ) - if path_node is not None: - raw = _read_text(path_node, source).strip() - else: - raw = next( - (_read_text(c, source).strip() for c in node.children - if c.type == "identifier"), - "", - ) - if not raw: - return - # Wildcard (`import a.b.*`): imports a whole package, not a symbol. The last - # path segment is a PACKAGE name, so a symbol-level edge would dangle on (or - # collide with) an unrelated node that happens to share the package's name. - if raw.endswith(".*") or raw == "*" or any(c.type == "*" for c in node.children): - return - # Alias (`import a.b.C as D`): the alias is the identifier child after `as`. - alias = None - saw_as = False - for child in node.children: - if not saw_as: - saw_as = child.type == "as" - elif child.type in ("identifier", "simple_identifier"): - alias = _read_text(child, source).strip() or None - break - module_name = raw.split(".")[-1].strip() - if not module_name: - return - # Target is the bare last segment for now; _resolve_kotlin_import_targets - # rewrites it to the real node id via the target_fqn stamped here, once the - # per-file package index exists. Unresolved targets stay dangling like other - # languages' external imports. - edges.append({ - "source": file_nid, - "target": _make_id(module_name), - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - "metadata": sanitize_metadata({k: v for k, v in - {"target_fqn": raw, "alias": alias}.items() if v is not None}), - }) - - -def _import_scala(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - for child in node.children: - if child.type in ("stable_id", "identifier"): - raw = _read_text(child, source) - module_name = raw.split(".")[-1].strip("{} ") - if module_name and module_name != "_": - tgt_nid = _make_id(module_name) - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) - break - - -def _import_php(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - for child in node.children: - if child.type in ("qualified_name", "name", "identifier"): - raw = _read_text(child, source) - module_name = raw.split("\\")[-1].strip() - if module_name: - tgt_nid = _make_id(module_name) - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) - break - - -# ── C/C++ function name helpers ─────────────────────────────────────────────── - -def _get_c_func_name(node, source: bytes) -> str | None: - """Recursively unwrap declarator to find the innermost identifier (C).""" - if node.type == "identifier": - return _read_text(node, source) - decl = node.child_by_field_name("declarator") - if decl: - return _get_c_func_name(decl, source) - for child in node.children: - if child.type == "identifier": - return _read_text(child, source) - return None - - -# ── JS/TS extra walk for arrow functions ────────────────────────────────────── - - -# Node types whose value is a callable, for the JS/TS assignment / class-field -# / function-expression forms below. Older tree-sitter-javascript grammars -# label a function expression `function`; current ones use `function_expression`. - - -# ── TS extra walk for namespace / module declarations ───────────────────────── - - -# ── C# extra walk for namespace declarations ────────────────────────────────── - - -# ── Swift extra walk for enum cases ────────────────────────────────────────── - - -# ── Java extra walk for enum constants ─────────────────────────────────────── - - -# ── Language configs ────────────────────────────────────────────────────────── - -_PYTHON_CONFIG = LanguageConfig( - ts_module="tree_sitter_python", - class_types=frozenset({"class_definition"}), - function_types=frozenset({"function_definition"}), - import_types=frozenset({"import_statement", "import_from_statement"}), - call_types=frozenset({"call"}), - call_function_field="function", - call_accessor_node_types=frozenset({"attribute"}), - call_accessor_field="attribute", - call_accessor_object_field="object", - function_boundary_types=frozenset({"function_definition"}), - import_handler=_import_python, -) - -_JS_CONFIG = LanguageConfig( - ts_module="tree_sitter_javascript", - class_types=frozenset({"class_declaration"}), - function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition"}), - import_types=frozenset({"import_statement", "export_statement"}), - call_types=frozenset({"call_expression", "new_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"member_expression"}), - call_accessor_field="property", - call_accessor_object_field="object", - function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition"}), - import_handler=_import_js, -) - -_TS_CONFIG = LanguageConfig( - ts_module="tree_sitter_typescript", - ts_language_fn="language_typescript", - class_types=frozenset({ - "class_declaration", - "abstract_class_declaration", # TS abstract class - "interface_declaration", # parity with Java/C# - "enum_declaration", # named enums - "type_alias_declaration", # named type aliases - }), - function_types=frozenset({"function_declaration", "generator_function_declaration", "method_definition", "method_signature"}), - import_types=frozenset({"import_statement", "export_statement"}), - call_types=frozenset({"call_expression", "new_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"member_expression"}), - call_accessor_field="property", - call_accessor_object_field="object", - function_boundary_types=frozenset({"function_declaration", "generator_function_declaration", "arrow_function", "method_definition"}), - import_handler=_import_js, -) - -# .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar. -# tree-sitter-typescript ships two languages: language_typescript (for .ts) and -# language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on -# JSX expressions, dropping any call_expression nested inside JSX (e.g. {fmtDate(x)}). -_TSX_CONFIG = LanguageConfig( - ts_module="tree_sitter_typescript", - ts_language_fn="language_tsx", - class_types=_TS_CONFIG.class_types, - function_types=_TS_CONFIG.function_types, - import_types=_TS_CONFIG.import_types, - call_types=_TS_CONFIG.call_types, - call_function_field=_TS_CONFIG.call_function_field, - call_accessor_node_types=_TS_CONFIG.call_accessor_node_types, - call_accessor_field=_TS_CONFIG.call_accessor_field, - call_accessor_object_field=_TS_CONFIG.call_accessor_object_field, - function_boundary_types=_TS_CONFIG.function_boundary_types, - import_handler=_TS_CONFIG.import_handler, -) - -_JAVA_CONFIG = LanguageConfig( - ts_module="tree_sitter_java", - # record_declaration shares class_declaration's name/body/interfaces fields, - # so it becomes a first-class type node instead of an isolated file (#1373). - # Enums and annotation declarations use the same name/body contract. - class_types=frozenset({ - "class_declaration", "interface_declaration", "record_declaration", - "enum_declaration", "annotation_type_declaration", - }), - function_types=frozenset({"method_declaration", "constructor_declaration"}), - import_types=frozenset({"import_declaration"}), - # object_creation_expression (`new Foo(...)`) is handled by a dedicated Java - # branch in walk_calls below — its callee is in the `type` field, not `name`. - call_types=frozenset({"method_invocation", "object_creation_expression"}), - call_function_field="name", - call_accessor_node_types=frozenset(), - function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), - import_handler=_import_java, -) - -_GROOVY_CONFIG = LanguageConfig( - ts_module="tree_sitter_groovy", - class_types=frozenset({"class_declaration", "interface_declaration"}), - function_types=frozenset({"method_declaration", "constructor_declaration"}), - import_types=frozenset({"import_declaration"}), - call_types=frozenset({"method_invocation"}), - call_function_field="name", - call_accessor_node_types=frozenset(), - function_boundary_types=frozenset({"method_declaration", "constructor_declaration"}), - import_handler=_import_java, -) - -_C_CONFIG = LanguageConfig( - ts_module="tree_sitter_c", - class_types=frozenset(), - function_types=frozenset({"function_definition"}), - import_types=frozenset({"preproc_include"}), - call_types=frozenset({"call_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"field_expression"}), - call_accessor_field="field", - function_boundary_types=frozenset({"function_definition"}), - import_handler=_import_c, - resolve_function_name_fn=_get_c_func_name, -) - -_CPP_CONFIG = LanguageConfig( - ts_module="tree_sitter_cpp", - class_types=frozenset({"class_specifier", "struct_specifier"}), - function_types=frozenset({"function_definition"}), - import_types=frozenset({"preproc_include"}), - call_types=frozenset({"call_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"field_expression", "qualified_identifier"}), - call_accessor_field="field", - function_boundary_types=frozenset({"function_definition"}), - import_handler=_import_c, - resolve_function_name_fn=_get_cpp_func_name, -) - -_RUBY_CONFIG = LanguageConfig( - ts_module="tree_sitter_ruby", - # `module Foo` is a container node just like `class Foo` in tree-sitter's - # Ruby grammar (name in a `constant` child, body in `body_statement`), so it - # gets a node and its methods attach via `method` (#1640). Without it, plain - # utility/`module_function` modules produced no node and their methods hung - # off the file via `contains` with dot-less labels. - class_types=frozenset({"class", "module"}), - function_types=frozenset({"method", "singleton_method"}), - import_types=frozenset(), - call_types=frozenset({"call"}), - call_function_field="method", - call_accessor_node_types=frozenset(), - name_fallback_child_types=("constant", "scope_resolution", "identifier"), - body_fallback_child_types=("body_statement",), - function_boundary_types=frozenset({"method", "singleton_method"}), -) - -_CSHARP_CONFIG = LanguageConfig( - ts_module="tree_sitter_c_sharp", - class_types=frozenset({ - "class_declaration", - "interface_declaration", - "enum_declaration", - "struct_declaration", - "record_declaration", - }), - function_types=frozenset({"method_declaration"}), - import_types=frozenset({"using_directive"}), - call_types=frozenset({"invocation_expression"}), - call_function_field="function", - call_accessor_node_types=frozenset({"member_access_expression"}), - call_accessor_field="name", - body_fallback_child_types=("declaration_list",), - function_boundary_types=frozenset({"method_declaration"}), - import_handler=_import_csharp, -) - -_KOTLIN_CONFIG = LanguageConfig( - ts_module="tree_sitter_kotlin", - class_types=frozenset({"class_declaration", "object_declaration"}), - function_types=frozenset({"function_declaration"}), - # Grammar 1.1.0 (PyPI tree_sitter_kotlin) names the import node `import`; - # older forks use `import_header`. Accept both (#2526). - import_types=frozenset({"import_header", "import"}), - call_types=frozenset({"call_expression"}), - call_function_field="", - call_accessor_node_types=frozenset({"navigation_expression"}), - call_accessor_field="", - # Different tree-sitter-kotlin grammar versions name plain identifier - # nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`, - # older forks use `simple_identifier`. Accept both so the extractor - # works across grammar generations. - name_fallback_child_types=("simple_identifier", "identifier"), - body_fallback_child_types=("function_body", "class_body", "enum_class_body"), - function_boundary_types=frozenset({"function_declaration"}), - import_handler=_import_kotlin, -) - -_SCALA_CONFIG = LanguageConfig( - ts_module="tree_sitter_scala", - class_types=frozenset({"class_definition", "object_definition"}), - function_types=frozenset({"function_definition"}), - import_types=frozenset({"import_declaration"}), - call_types=frozenset({"call_expression"}), - call_function_field="", - call_accessor_node_types=frozenset({"field_expression"}), - call_accessor_field="field", - name_fallback_child_types=("identifier",), - body_fallback_child_types=("template_body",), - function_boundary_types=frozenset({"function_definition"}), - import_handler=_import_scala, -) - -_PHP_CONFIG = LanguageConfig( - ts_module="tree_sitter_php", - ts_language_fn="language_php", - class_types=frozenset({"class_declaration"}), - function_types=frozenset({"function_definition", "method_declaration"}), - import_types=frozenset({"namespace_use_clause"}), - call_types=frozenset({"function_call_expression", "member_call_expression", "scoped_call_expression", "class_constant_access_expression"}), - static_prop_types=frozenset({"scoped_property_access_expression"}), - helper_fn_names=frozenset({"config"}), - container_bind_methods=frozenset({"bind", "singleton", "scoped", "instance"}), - event_listener_properties=frozenset({"listen", "subscribe"}), - call_function_field="function", - call_accessor_node_types=frozenset({"member_call_expression"}), - call_accessor_field="name", - name_fallback_child_types=("name",), - body_fallback_child_types=("declaration_list", "compound_statement"), - function_boundary_types=frozenset({"function_definition", "method_declaration"}), - import_handler=_import_php, -) - - -def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: - """Extract require('module') from Lua variable_declaration nodes.""" - text = _read_text(node, source) - import re - m = re.search(r"""require\s*[\('"]\s*['"]?([^'")\s]+)""", text) - if m: - raw_module = m.group(1) - if raw_module: - tgt_nid = _resolve_lua_import_target(raw_module, str_path) - if tgt_nid: - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "confidence_score": 1.0, - "source_file": str_path, - "source_location": str(node.start_point[0] + 1), - "weight": 1.0, - }) - - -_LUA_CONFIG = LanguageConfig( - ts_module="tree_sitter_lua", - ts_language_fn="language", - class_types=frozenset(), - function_types=frozenset({"function_declaration"}), - import_types=frozenset({"variable_declaration"}), - call_types=frozenset({"function_call"}), - call_function_field="name", - call_accessor_node_types=frozenset({"method_index_expression"}), - call_accessor_field="name", - name_fallback_child_types=("identifier", "method_index_expression"), - body_fallback_child_types=("block",), - function_boundary_types=frozenset({"function_declaration"}), - import_handler=_import_lua, -) - - -def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> list[tuple[str, str]]: - """Emit module-level ``imports`` edges and report the imported modules. - - A Swift ``import CoreKit`` names a module, not a file path, so — unlike the - file-resolving JS/TS handlers — there is no existing node for the edge to - point at. The returned ``(id, label)`` pairs let the extractor materialize a - ``type=module`` anchor node so the edge survives; without it ``build_from_json`` - prunes every Swift import edge as a dangling/external reference (#1327). - """ - modules: list[tuple[str, str]] = [] - for child in node.children: - if child.type == "identifier": - raw = _read_text(child, source) - tgt_nid = _make_id(raw) - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) - modules.append((tgt_nid, raw)) - break - return modules - - -_SWIFT_CONFIG = LanguageConfig( - ts_module="tree_sitter_swift", - class_types=frozenset({"class_declaration", "protocol_declaration"}), - function_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), - import_types=frozenset({"import_declaration"}), - call_types=frozenset({"call_expression"}), - call_function_field="", - call_accessor_node_types=frozenset({"navigation_expression"}), - call_accessor_field="", - name_fallback_child_types=("simple_identifier", "type_identifier", "user_type"), - body_fallback_child_types=("class_body", "protocol_body", "function_body", "enum_class_body"), - function_boundary_types=frozenset({"function_declaration", "init_declaration", "deinit_declaration", "subscript_declaration"}), - import_handler=_import_swift, -) - -# ── Ruby local type inference (for member-call resolution) ───────────────────── - - -# `Const = (...)` shapes that define a lightweight class named after the -# constant. tree-sitter parses each as an `assignment`, not a `class`, so the -# generic class branch never saw them (#1640). - - -# ── Generic extractor ───────────────────────────────────────────────────────── - - -# ── Python rationale extraction ─────────────────────────────────────────────── - -_RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:") - - -def _shorten_rationale_label(text: str, width: int = 80) -> str: - """Collapse whitespace and truncate ``text`` to ``width`` chars for a - rationale node label, cutting on a word boundary rather than mid-word. - Shared by the Python and JS/TS rationale extractors (#2206). - - ``textwrap.shorten`` collapses to just the placeholder when the first - "word" alone exceeds ``width`` (e.g. a docstring/comment that opens with - an unbroken URL) -- that would emit a content-free label, so fall back to - a plain character truncation of the normalized text in that case. - """ - label = textwrap.shorten(text, width=width, placeholder="…") - if label in ("", "…"): - flat = " ".join(text.split()) - label = flat if len(flat) <= width else flat[: width - 1] + "…" - return label - - -def _is_autogenerated_python(source: bytes) -> bool: - """Return True if this Python file is auto-generated and its module docstring is noise. - - Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs. - Module docstrings in these files are change annotations or boilerplate, not rationale. - """ - head = source[:2048].decode("utf-8", errors="replace") - # Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.) - if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")): - return True - # Alembic / Flask-Migrate revision files - if (re.search(r"^revision\s*[:=]", head, re.MULTILINE) - and "def upgrade(" in head - and "down_revision" in head): - return True - # Django migrations - if "class Migration(migrations.Migration)" in head and "operations" in head: - return True - return False - - -def _extract_python_rationale(path: Path, result: dict) -> None: - """Post-pass: extract docstrings and rationale comments from Python source. - Mutates result in-place by appending to result['nodes'] and result['edges']. - """ - try: - import tree_sitter_python as tspython - from tree_sitter import Language, Parser - language = Language(tspython.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception: - return - - stem = _file_stem(path) - str_path = str(path) - nodes = result["nodes"] - edges = result["edges"] - seen_ids = {n["id"] for n in nodes} - file_nid = _make_id(str(path)) - - def _get_docstring(body_node) -> tuple[str, int] | None: - if not body_node: - return None - for child in body_node.children: - if child.type == "expression_statement": - for sub in child.children: - if sub.type in ("string", "concatenated_string"): - text = source[sub.start_byte:sub.end_byte].decode("utf-8", errors="replace") - text = text.strip("\"'").strip('"""').strip("'''").strip() - if len(text) > 20: - return text, child.start_point[0] + 1 - break - return None - - def _add_rationale(text: str, line: int, parent_nid: str) -> None: - # Normalize whitespace before truncating, not after: slicing raw text - # first can land mid-word, leave a run of literal spaces where a - # newline + indentation used to be, or end on a "." that turns into - # an Obsidian "..md" filename once export.py appends the extension. - label = _shorten_rationale_label(text) - rid = _make_id(stem, "rationale", str(line)) - if rid not in seen_ids: - seen_ids.add(rid) - nodes.append({ - "id": rid, - "label": label, - "file_type": "rationale", - "source_file": str_path, - "source_location": f"L{line}", - }) - edges.append({ - "source": rid, - "target": parent_nid, - "relation": "rationale_for", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - - # Module-level docstring — skip for auto-generated files (Alembic, Django - # migrations, protobuf stubs, etc.) whose module docstrings are revision - # annotations, not architectural rationale. - if not _is_autogenerated_python(source): - ds = _get_docstring(root) - if ds: - _add_rationale(ds[0], ds[1], file_nid) - - # Class and function docstrings - def walk_docstrings(node, parent_nid: str) -> None: - t = node.type - if t == "class_definition": - name_node = node.child_by_field_name("name") - body = node.child_by_field_name("body") - if name_node and body: - class_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace") - nid = _make_id(stem, class_name) - ds = _get_docstring(body) - if ds: - _add_rationale(ds[0], ds[1], nid) - for child in body.children: - walk_docstrings(child, nid) - return - if t == "function_definition": - name_node = node.child_by_field_name("name") - body = node.child_by_field_name("body") - if name_node and body: - func_name = source[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace") - nid = _make_id(parent_nid, func_name) if parent_nid != file_nid else _make_id(stem, func_name) - ds = _get_docstring(body) - if ds: - _add_rationale(ds[0], ds[1], nid) - return - for child in node.children: - walk_docstrings(child, parent_nid) - - walk_docstrings(root, file_nid) - - # Rationale comments (# NOTE:, # IMPORTANT:, etc.) - source_text = source.decode("utf-8", errors="replace") - for lineno, line_text in enumerate(source_text.splitlines(), start=1): - stripped = line_text.strip() - if any(stripped.startswith(p) for p in _RATIONALE_PREFIXES): - _add_rationale(stripped, lineno, file_nid) - - -# ── Public API ──────────────────────────────────────────────────────────────── - -def extract_python(path: Path) -> dict: - """Extract classes, functions, and imports from a .py file via tree-sitter AST.""" - result = _extract_generic(path, _PYTHON_CONFIG) - if "error" not in result: - _extract_python_rationale(path, result) - return result - - -def extract_js(path: Path) -> dict: - """Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx/.mts/.cts file.""" - suffix = path.suffix.lower() - if suffix == ".tsx": - config = _TSX_CONFIG - elif suffix in (".ts", ".mts", ".cts"): - config = _TS_CONFIG - else: - config = _JS_CONFIG - result = _extract_generic(path, config) - if "error" not in result: - _extract_js_rationale(path, result) - _rescue_js_dynamic_imports(path, result) - return result - - -def _rescue_js_dynamic_imports(path: Path, result: dict) -> None: - """Recover ``import('…')`` edges the AST pass does not emit for plain JS/TS. - - tree-sitter models ``await import('x')`` as a ``call_expression``, not an - ``import_statement``, so the specifier only reaches the graph when - ``walk_calls`` visits that call — which it never does at module scope - (only function bodies are walked for calls). The Svelte/Astro/Vue - extractors already patch the same gap by regex because their AST pass - fails wholesale; plain ``.ts``/``.js`` was left out on the reasoning that - its AST pass "works". It works for STATIC imports; dynamic ones outside a - walked body fell through silently (#2575), and because they cluster under - hub modules the loss compounds with ``affected`` traversal depth. - - Dedupe: a dynamic import the AST pass DID capture is already in the graph - as an ``imports_from`` edge marked ``deferred`` (``_dynamic_import_js``). - Re-emitting it here as a second ``dynamic_import`` edge would state the - same fact twice, so a match whose resolved target already has a deferred - edge FROM THIS FILE'S NODE is skipped. The source check matters: the AST - pass anchors the edge on the enclosing function when the ``import()`` is - written inside one, and that is a different fact from "this file depends on - that module" — the only one file-level traversal can use (#2584). - - Regex false positives in comments/strings are the precedented trade of - the Svelte/Vue rescues; a ``//``-prefix guard covers the common case. - """ - try: - import re as _re - src = path.read_text(encoding="utf-8", errors="replace") - if "import(" not in src: # cheap bail — most files have none - return - existing_ids = {n["id"] for n in result.get("nodes", [])} - file_node_id = _make_id(str(path)) - aliases = _load_tsconfig_aliases(path.parent) - base_url = _load_tsconfig_base_url(path.parent) - deferred_ids: set[str] = set() - deferred_files: set[str] = set() - rescued_targets: set[str] = set() - for e in result.get("edges", []): - # Only a FILE-level deferred edge makes the rescue redundant (#2584). - # - # `_dynamic_import_js` emits `caller_nid -> target`, and `caller_nid` is this - # file's node only when the `import()` sits at module scope. Written inside a - # function it is that function's node — a different fact, at a granularity - # `affected` does not walk. Matching on target alone treated the two as one and - # skipped the rescue, so a dynamic import inside a function ended up with no - # file-level edge at all. The reverse walk then reached the enclosing function - # and stopped: the only edge pointing at it is `contains`, deliberately kept out - # of DEFAULT_AFFECTED_RELATIONS. - # - # Measured on a ~700-file TS repo: `affected --depth 3` returned 39 of 49 truly - # affected files (recall 0.80, precision 1.00) and deeper traversal did not help, - # which is a dead end rather than a depth limit. It stayed hidden because the - # usual case still resolves — when the next importer imports that exact symbol - # by name there IS an edge into the function. Switch that importer to - # `import * as ns` or a side-effect `import './dyn'` and the same graph goes - # silent. - if (e.get("deferred") and e.get("relation") == "imports_from" - and e.get("source") == file_node_id): - deferred_ids.add(e.get("target")) - tf = e.get("target_file") - if tf: - try: - deferred_files.add(str(Path(tf).resolve())) - except OSError: - deferred_files.add(str(tf)) - # `(?ADR edges never form even when the code cites the ADR. - -_JS_RATIONALE_PREFIXES = ( - "// NOTE:", "// IMPORTANT:", "// HACK:", "// WHY:", "// RATIONALE:", - "// TODO:", "// FIXME:", - "* NOTE:", "* IMPORTANT:", "* HACK:", "* WHY:", "* RATIONALE:", - "* TODO:", "* FIXME:", -) - -# Doc-reference tokens worth first-classing as graph nodes. Deliberately -# conservative: ADR-NNNN (Architecture Decision Records, any zero padding) -# and RFC NNNN / RFC-NNNN. -_JS_DOC_REF_RE = re.compile(r"\b(ADR[- ]?\d{1,5}|RFC[- ]?\d{1,5})\b", re.IGNORECASE) - -# Only look for doc references inside comments, not string literals or code. -_JS_COMMENT_LINE_RE = re.compile(r"^\s*(//|/\*|\*)") - - -def _extract_js_rationale(path: Path, result: dict) -> None: - """Post-pass: extract rationale comments and doc references from JS/TS source. - Mutates result in-place by appending to result['nodes'] and result['edges']. - """ - try: - source_text = path.read_text(encoding="utf-8", errors="replace") - except Exception: - return - - stem = _file_stem(path) - str_path = str(path) - nodes = result["nodes"] - edges = result["edges"] - seen_ids = {n["id"] for n in nodes} - file_nid = _make_id(str(path)) - seen_doc_refs: set[str] = set() - - def _add_rationale(text: str, line: int) -> None: - # Normalize whitespace before truncating, not after: slicing raw text - # first can land mid-word, leave a run of literal spaces where a - # newline + indentation used to be, or end on a "." that turns into - # an Obsidian "..md" filename once export.py appends the extension. - label = _shorten_rationale_label(text) - rid = _make_id(stem, "rationale", str(line)) - if rid not in seen_ids: - seen_ids.add(rid) - nodes.append({ - "id": rid, - "label": label, - "file_type": "rationale", - "source_file": str_path, - "source_location": f"L{line}", - }) - edges.append({ - "source": rid, - "target": file_nid, - "relation": "rationale_for", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - - def _add_doc_ref(token: str, line: int) -> None: - # Normalize "adr 11" / "ADR-0011" spellings to a canonical "ADR-0011" - # style label so references to the same document collapse to one node. - kind, num = re.match(r"([A-Za-z]+)[- ]?(\d+)", token).groups() - kind = kind.upper() - label = f"{kind}-{num.zfill(4)}" if kind == "ADR" else f"{kind}-{num}" - if label in seen_doc_refs: - return - seen_doc_refs.add(label) - rid = _make_id("docref", label) - if rid not in seen_ids: - seen_ids.add(rid) - nodes.append({ - "id": rid, - "label": label, - "file_type": "doc_ref", - "source_file": str_path, - "source_location": f"L{line}", - }) - edges.append({ - "source": file_nid, - "target": rid, - "relation": "cites", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - - for lineno, line_text in enumerate(source_text.splitlines(), start=1): - stripped = line_text.strip() - if any(stripped.startswith(p) for p in _JS_RATIONALE_PREFIXES): - _add_rationale(stripped.lstrip("/* "), lineno) - if _JS_COMMENT_LINE_RE.match(line_text): - for m in _JS_DOC_REF_RE.finditer(stripped): - _add_doc_ref(m.group(1), lineno) - - -def _resolve_rescued_specifier( - path: Path, - raw: str, - aliases, - base_url, -) -> "tuple[str, str, Path | None] | None": - """Resolve a regex-rescued import specifier the way ``_import_js`` does. - - Returns ``(node_id, stub_source_file, resolved_file)`` — ``resolved_file`` - is the target as a real on-disk file, or None when the specifier is - external or dangling. Returns None when no target can be minted at all - (empty bare-import segment). Split out of :func:`_emit_rescued_import` so - :func:`_rescue_js_dynamic_imports` can resolve a match FIRST and skip - specifiers the AST pass already emitted, without duplicating the - resolution rules. - """ - if raw.startswith("."): - resolved = _resolve_js_module_path( - Path(os.path.normpath(path.parent / raw)) - ) - resolved_file = resolved if resolved is not None and resolved.is_file() else None - return _make_id(str(resolved)), str(resolved), resolved_file - # Check tsconfig.json path aliases (e.g. "$lib/" -> "src/lib/", - # "@/" -> "src/") before treating as external. Mirrors _import_js - # logic so alias imports resolve to the same file node IDs the - # extractor creates (#701). - resolved_alias = _resolve_tsconfig_alias(raw, aliases, base_url=base_url) - if resolved_alias is not None: - resolved_alias = _resolve_js_module_path(resolved_alias) - resolved_file = (resolved_alias if resolved_alias is not None - and resolved_alias.is_file() else None) - return _make_id(str(resolved_alias)), str(resolved_alias), resolved_file - # Bare/scoped import (node_modules) - use last segment; - # build_from_json drops as external if no matching node exists. - module_name = raw.split("/")[-1] - if not module_name: - return None - return _make_id(module_name), raw, None - - -def _emit_rescued_import( - result: dict, - existing_ids: set, - file_node_id: str, - path: Path, - raw: str, - relation: str, - aliases, - base_url, -) -> None: - """Shared edge/stub emit for the Svelte/Astro/Vue regex-rescue import passes. - - Resolves the specifier the same way ``_import_js`` does — relative paths and - tsconfig aliases both go through :func:`_resolve_js_module_path` so - extensionless specifiers probe real on-disk extensions (``../lib/content`` - -> ``content.ts``) instead of a naive ``.js``->``.ts`` suffix swap. - - When the resolved target is a real file on disk, mirror ``_import_js``: - emit ONLY the edge, stamped with ``target_file``, and mint no stub node. - The #2169 canonicalization loop in :func:`extract` reads the stamp and - repoints the edge at the real file node's canonical id. Minting a stub - here would carry an absolute-path-derived id when the input path is - absolute — a ghost node (e.g. ``private_tmp_..._src_lib_content``) - duplicating the real ``src_lib_content`` node and clobbering its label on - dedupe (#2195). Stub nodes are still minted for unresolved specifiers - (externals, not-yet-created files) so prior behavior is preserved. - """ - resolution = _resolve_rescued_specifier(path, raw, aliases, base_url) - if resolution is None: - return - node_id, stub_source_file, resolved_file = resolution - edge = { - "source": file_node_id, "target": node_id, - "relation": relation, "confidence": "EXTRACTED", - "source_file": str(path), - } - if resolved_file is not None: - # Real file on disk: edge only (no stub node), stamped so the #2169 - # canonicalization pass repoints it at the real node (#2195). - edge["target_file"] = str(resolved_file) - result.setdefault("edges", []).append(edge) - return - if node_id in existing_ids: - # Edge target already a real node - just add the edge, don't add a node. - result.setdefault("edges", []).append(edge) - return - result.setdefault("nodes", []).append({ - "id": node_id, "label": raw, - "file_type": "code", "source_file": stub_source_file, - "confidence": "EXTRACTED", - }) - result.setdefault("edges", []).append(edge) - existing_ids.add(node_id) - - -def extract_svelte(path: Path) -> dict: - """Extract imports from .svelte files: script-block via JS AST + template regex fallback. - - Tree-sitter only sees the