refactor(extract): complete extractor migration to extractors/ package

graphify/extractors/MIGRATION.md

@@ -25,8 +25,31 @@ written so an AI agent can execute it in a single session.
 | sln | yes |
 | pascal_forms (dfm + lfm) | yes |
 | json_config | yes |
-| (config-driven core: python, js, java, c, cpp, csharp, kotlin, scala, php, lua, swift, groovy, vue, svelte, astro, xaml, groovy) | no — shared _extract_generic core, move as one batch |
-| (other bespoke: julia, verilog, markdown, objc, csproj, slnx, lazarus_package, pascal) | no |
+| python | yes |
+| js (js/ts/tsx/mts/cts) | yes |
+| svelte | yes |
+| astro | yes |
+| vue | yes |
+| java | yes |
+| groovy | yes |
+| c | yes |
+| cpp | yes |
+| ruby | yes |
+| csharp (extractor appended to existing csharp.py) | yes |
+| kotlin | yes |
+| scala | yes |
+| php | yes |
+| lua | yes |
+| swift | yes |
+| lazarus_package | yes |
+| slnx | yes |
+| csproj | yes |
+| objc | yes |
+| pascal | yes |
+| julia | yes |
+| verilog | yes |
+| markdown | yes |
+| xaml | yes |
 
 Note: config-driven extractors (python, js, java, c, cpp, ruby, csharp,
 kotlin, scala, php, lua, swift, groovy) depend on the shared

graphify/extractors/__init__.py

@@ -11,31 +11,56 @@
 from typing import Callable
 
 from graphify.extractors.apex import extract_apex
+from graphify.extractors.astro import extract_astro
 from graphify.extractors.bash import extract_bash
 from graphify.extractors.blade import extract_blade
+from graphify.extractors.c import extract_c
+from graphify.extractors.cpp import extract_cpp
+from graphify.extractors.csharp import extract_csharp
+from graphify.extractors.csproj import extract_csproj
 from graphify.extractors.dart import extract_dart
 from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm
 from graphify.extractors.elixir import extract_elixir
 from graphify.extractors.fortran import extract_fortran
 from graphify.extractors.go import extract_go
+from graphify.extractors.groovy import extract_groovy
+from graphify.extractors.java import extract_java
+from graphify.extractors.js import extract_js
 from graphify.extractors.json_config import extract_json
 from graphify.extractors.julia import extract_julia
+from graphify.extractors.kotlin import extract_kotlin
+from graphify.extractors.lazarus_package import extract_lazarus_package
+from graphify.extractors.lua import extract_lua
 from graphify.extractors.markdown import extract_markdown
 from graphify.extractors.objc import extract_objc
 from graphify.extractors.pascal import extract_pascal
 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form
+from graphify.extractors.php import extract_php
 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest
+from graphify.extractors.python import extract_python
 from graphify.extractors.razor import extract_razor
+from graphify.extractors.ruby import extract_ruby
 from graphify.extractors.rust import extract_rust
+from graphify.extractors.scala import extract_scala
 from graphify.extractors.sln import extract_sln
+from graphify.extractors.slnx import extract_slnx
 from graphify.extractors.sql import extract_sql
+from graphify.extractors.svelte import extract_svelte
+from graphify.extractors.swift import extract_swift
 from graphify.extractors.terraform import extract_terraform
 from graphify.extractors.verilog import extract_verilog
+from graphify.extractors.vue import extract_vue
+from graphify.extractors.xaml import extract_xaml
 from graphify.extractors.zig import extract_zig
 
 LANGUAGE_EXTRACTORS: dict[str, Callable[[Path], dict]] = {
     "apex": extract_apex,
+    "astro": extract_astro,
     "bash": extract_bash,
+    "c": extract_c,
+    "cpp": extract_cpp,
+    "csharp": extract_csharp,
+    "csproj": extract_csproj,
     "blade": extract_blade,
     "dart": extract_dart,
     "delphi_form": extract_delphi_form,
@@ -46,19 +71,34 @@
     "elixir": extract_elixir,
     "fortran": extract_fortran,
     "go": extract_go,
+    "groovy": extract_groovy,
+    "java": extract_java,
+    "js": extract_js,
     "json": extract_json,
     "julia": extract_julia,
+    "kotlin": extract_kotlin,
     "lazarus_form": extract_lazarus_form,
+    "lazarus_package": extract_lazarus_package,
+    "lua": extract_lua,
     "markdown": extract_markdown,
     "objc": extract_objc,
     "pascal": extract_pascal,
+    "php": extract_php,
     "powershell": extract_powershell,
     "powershell_manifest": extract_powershell_manifest,
+    "python": extract_python,
     "razor": extract_razor,
+    "ruby": extract_ruby,
     "rust": extract_rust,
+    "scala": extract_scala,
     "sln": extract_sln,
+    "slnx": extract_slnx,
     "sql": extract_sql,
+    "svelte": extract_svelte,
+    "swift": extract_swift,
     "terraform": extract_terraform,
     "verilog": extract_verilog,
+    "vue": extract_vue,
+    "xaml": extract_xaml,
     "zig": extract_zig,
 }

graphify/extractors/astro.py

@@ -0,0 +1,72 @@
+"""Astro extractor. Moved verbatim from graphify/extract.py."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from graphify.extractors.engine import _extract_generic
+from graphify.extractors.base import _make_id
+from graphify.extractors.resolution import _load_tsconfig_aliases, _load_tsconfig_base_url
+from graphify.extractors.js import _JS_CONFIG, _emit_rescued_import
+
+
+def extract_astro(path: Path) -> dict:
+    """Extract imports from .astro files: frontmatter (TS) + template regex fallback.
+
+    Astro files start with a ``---\\n...\\n---`` frontmatter block of TypeScript
+    setup code (where almost all imports live), followed by an HTML-with-expressions
+    template body, and optionally ``<script>`` blocks for client-side JS. Tree-sitter
+    only sees the file usefully through the frontmatter — feeding the whole file to
+    the JS parser produces a top-level ERROR node because the template is not valid
+    JS, so ``import_statement`` nodes are never reached and static imports are
+    silently dropped (#850). Mirrors :func:`extract_svelte` — same regex-rescue
+    approach, scanning the frontmatter block and any client-side ``<script>`` blocks
+    for static and dynamic imports.
+    """
+    result = _extract_generic(path, _JS_CONFIG)
+    try:
+        import re as _re
+        src = path.read_text(encoding="utf-8", errors="replace")
+        existing_ids = {n["id"] for n in result.get("nodes", [])}
+        file_node_id = _make_id(str(path))
+        aliases = _load_tsconfig_aliases(path.parent)
+        base_url = _load_tsconfig_base_url(path.parent)
+        # Dynamic imports anywhere in the file: `import('./X.astro')` is legal in
+        # frontmatter setup code and inside expression slots.
+        for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
+            raw = m.group(1)
+            if not raw:
+                continue
+            _emit_rescued_import(
+                result, existing_ids, file_node_id, path, raw,
+                "dynamic_import", aliases, base_url,
+            )
+        # Static imports: scan the `---...---` frontmatter at the file head plus any
+        # client-side <script> blocks. Both are TS/JS regions but live inside a file
+        # the JS tree-sitter parser cannot validate as a whole.
+        frontmatter_re = _re.compile(
+            r"\A\s*---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|\Z)"
+        )
+        script_re = _re.compile(
+            r"<script\b[^>]*>([\s\S]*?)</script\s*>", _re.IGNORECASE
+        )
+        static_import_re = _re.compile(
+            r"""import\s+(?:[^'"`;]+?\s+from\s+)?['"]([^'"]+)['"]"""
+        )
+        regions: list[str] = []
+        fm = frontmatter_re.search(src)
+        if fm:
+            regions.append(fm.group(1))
+        for script_match in script_re.finditer(src):
+            regions.append(script_match.group(1))
+        for region in regions:
+            for m in static_import_re.finditer(region):
+                raw = m.group(1)
+                if not raw:
+                    continue
+                _emit_rescued_import(
+                    result, existing_ids, file_node_id, path, raw,
+                    "imports_from", aliases, base_url,
+                )
+    except Exception:
+        pass
+    return result

graphify/extractors/base.py

@@ -1,6 +1,7 @@
 # DO NOT import from graphify.extract here — direction is extract.py → extractors/ only.
 from __future__ import annotations
 
+import textwrap
 from pathlib import Path
 
 from graphify.ids import make_id
@@ -83,3 +84,41 @@ def _file_stem(path: Path) -> str:
 
 def _read_text(node, source: bytes) -> str:
     return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+
+
+# Size cap for project XML files we parse with stdlib ElementTree.
+# Real .csproj/.fsproj/.vbproj/.lpk files are well under 2 MiB; anything
+# larger is either malformed or hostile.
+_PROJECT_XML_MAX_BYTES = 2 * 1024 * 1024
+
+
+def _shorten_rationale_label(text: str, width: int = 80) -> str:
+    """Collapse whitespace and truncate ``text`` to ``width`` chars for a
+    rationale node label, cutting on a word boundary rather than mid-word.
+    Shared by the Python and JS/TS rationale extractors (#2206).
+
+    ``textwrap.shorten`` collapses to just the placeholder when the first
+    "word" alone exceeds ``width`` (e.g. a docstring/comment that opens with
+    an unbroken URL) -- that would emit a content-free label, so fall back to
+    a plain character truncation of the normalized text in that case.
+    """
+    label = textwrap.shorten(text, width=width, placeholder="…")
+    if label in ("", "…"):
+        flat = " ".join(text.split())
+        label = flat if len(flat) <= width else flat[: width - 1] + "…"
+    return label
+
+
+def _project_xml_is_safe(src: bytes) -> bool:
+    """Reject XML that declares DTDs or entities.
+
+    Stdlib ``xml.etree.ElementTree`` does not cap entity expansion, so a
+    crafted project file could trigger a billion-laughs style DoS. External
+    entity resolution is already disabled by pyexpat defaults, but rejecting
+    ``<!DOCTYPE`` / ``<!ENTITY`` outright is defense in depth.
+
+    Legitimate MSBuild and Lazarus package files never contain a DOCTYPE
+    or ENTITY declaration, so this is a zero-false-positive screen.
+    """
+    lowered = src.lower()
+    return b"<!doctype" not in lowered and b"<!entity" not in lowered

graphify/extractors/c.py

@@ -0,0 +1,84 @@
+"""C extractor. Moved verbatim from graphify/extract.py."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from graphify.extractors.models import LanguageConfig
+from graphify.extractors.engine import _extract_generic
+from graphify.extractors.base import _make_id, _read_text
+from graphify.extractors.resolution import _resolve_c_include_path
+
+
+def _get_c_func_name(node, source: bytes) -> str | None:
+    """Recursively unwrap declarator to find the innermost identifier (C)."""
+    if node.type == "identifier":
+        return _read_text(node, source)
+    decl = node.child_by_field_name("declarator")
+    if decl:
+        return _get_c_func_name(decl, source)
+    for child in node.children:
+        if child.type == "identifier":
+            return _read_text(child, source)
+    return None
+
+
+def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None:
+    for child in node.children:
+        if child.type in ("string_literal", "system_lib_string", "string"):
+            raw = _read_text(child, source).strip('"<> ')
+            # Quoted includes: try to resolve to a real file so the target ID
+            # matches the node ID _extract_generic creates for that file.
+            if child.type != "system_lib_string":
+                resolved = _resolve_c_include_path(raw, str_path)
+                if resolved is not None:
+                    tgt_nid = _make_id(str(resolved))
+                    edges.append({
+                        "source": file_nid,
+                        "target": tgt_nid,
+                        "relation": "imports",
+                        "context": "import",
+                        "confidence": "EXTRACTED",
+                        "source_file": str_path,
+                        "source_location": f"L{node.start_point[0] + 1}",
+                        "weight": 1.0,
+                        # Stamp the resolved target, mirroring _import_python (#1814):
+                        # without it, an include whose header lives outside this
+                        # batch's paths keeps the raw absolute-path id no later pass
+                        # ever learns to relativize (#2243).
+                        "target_file": str(resolved),
+                    })
+                    break
+            module_name = raw.split("/")[-1].split(".")[0]
+            if module_name:
+                tgt_nid = _make_id(module_name)
+                edges.append({
+                    "source": file_nid,
+                    "target": tgt_nid,
+                    "relation": "imports",
+                    "context": "import",
+                    "confidence": "EXTRACTED",
+                    "source_file": str_path,
+                    "source_location": f"L{node.start_point[0] + 1}",
+                    "weight": 1.0,
+                })
+            break
+
+
+_C_CONFIG = LanguageConfig(
+    ts_module="tree_sitter_c",
+    class_types=frozenset(),
+    function_types=frozenset({"function_definition"}),
+    import_types=frozenset({"preproc_include"}),
+    call_types=frozenset({"call_expression"}),
+    call_function_field="function",
+    call_accessor_node_types=frozenset({"field_expression"}),
+    call_accessor_field="field",
+    function_boundary_types=frozenset({"function_definition"}),
+    import_handler=_import_c,
+    resolve_function_name_fn=_get_c_func_name,
+)
+
+
+def extract_c(path: Path) -> dict:
+    """Extract functions and includes from a .c/.h file."""
+    return _extract_generic(path, _C_CONFIG)

graphify/extractors/cpp.py

@@ -0,0 +1,30 @@
+"""C++ extractor. Moved verbatim from graphify/extract.py."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from graphify.extractors.models import LanguageConfig
+from graphify.extractors.engine import _extract_generic, _get_cpp_func_name
+from graphify.extractors.base import _make_id, _read_text
+from graphify.extractors.resolution import _resolve_c_include_path
+from graphify.extractors.c import _import_c
+
+
+_CPP_CONFIG = LanguageConfig(
+    ts_module="tree_sitter_cpp",
+    class_types=frozenset({"class_specifier", "struct_specifier"}),
+    function_types=frozenset({"function_definition"}),
+    import_types=frozenset({"preproc_include"}),
+    call_types=frozenset({"call_expression"}),
+    call_function_field="function",
+    call_accessor_node_types=frozenset({"field_expression", "qualified_identifier"}),
+    call_accessor_field="field",
+    function_boundary_types=frozenset({"function_definition"}),
+    import_handler=_import_c,
+    resolve_function_name_fn=_get_cpp_func_name,
+)
+
+
+def extract_cpp(path: Path) -> dict:
+    """Extract functions, classes, and includes from a .cpp/.cc/.cxx/.hpp file."""
+    return _extract_generic(path, _CPP_CONFIG)