fix(export): adopt a pre-manifest vault's own notes instead of orphaning them (#2863)

graphify/export.py

@@ -544,6 +544,73 @@ def _cap_filename(s: str, limit: int = 200) -> str:
     return f"{truncated}_{digest}"
 
 
+# A frontmatter tag entry in graphify's own namespace, e.g. "  - graphify/document".
+_GRAPHIFY_TAG_RE = re.compile(r"^\s*-\s+graphify/\S")
+
+# Frontmatter sits at the very top of a note; reading this much is enough to see
+# the whole block without pulling a large note into memory.
+_NOTE_FRONTMATTER_PROBE_BYTES = 4096
+
+# Community notes carry no frontmatter; graphify identifies its own by the
+# Dataview query it writes into every one of them.
+_COMMUNITY_QUERY_MARKER = "FROM #community/"
+
+
+def _is_graphify_note(path: Path) -> bool:
+    """Whether a vault note carries graphify's own frontmatter signature.
+
+    Every note graphify writes opens with a YAML frontmatter block tagging it in
+    the ``graphify/`` namespace::
+
+        ---
+        source_file: "d0.md"
+        tags:
+          - graphify/document
+          - graphify/EXTRACTED
+        ---
+
+    Only that block is inspected, and only a tag entry inside it counts — a
+    user's note that merely mentions graphify in its prose is not adopted.
+
+    Community overview notes are recognised separately: they carry no
+    frontmatter at all, so they are identified by graphify's own filename prefix
+    together with the Dataview query it writes into the body. Requiring both
+    keeps a user's own ``_COMMUNITY_*.md`` from being adopted on the name alone.
+    """
+    try:
+        with path.open("r", encoding="utf-8", errors="replace") as fh:
+            head = fh.read(_NOTE_FRONTMATTER_PROBE_BYTES)
+    except OSError:
+        return False
+    if path.name.startswith(_COMMUNITY_PREFIX) and _COMMUNITY_QUERY_MARKER in head:
+        return True
+    if not head.startswith("---"):
+        return False
+    for line in head.splitlines()[1:]:
+        if line.strip() == "---":
+            return False  # frontmatter closed without a graphify tag
+        if _GRAPHIFY_TAG_RE.match(line):
+            return True
+    return False
+
+
+def _adopt_pre_manifest_notes(out: Path) -> set[str]:
+    """Names of notes in *out* that graphify itself wrote before manifests existed.
+
+    Deliberately limited to top-level ``*.md``: those are the only files graphify
+    can identify as its own from their content. ``.obsidian/graph.json`` is NOT
+    adopted — graphify writes one, but so does Obsidian, and with no manifest
+    there is no way to tell whose it is. Leaving it unowned keeps the
+    conservative behaviour for the one file where guessing wrong would cost the
+    user their own vault configuration.
+    """
+    try:
+        candidates = sorted(out.glob("*.md"))
+    except OSError:
+        return set()
+    return {p.name for p in candidates if _is_graphify_note(p)}
+
+
 def _obsidian_safe_stem(label: str, limit: int = 200) -> str:
     """Filename stem for an Obsidian note / canvas card from a node label.
 
@@ -636,8 +703,18 @@ def to_obsidian(
     _manifest_path = out / ".graphify_obsidian_manifest.json"
     try:
         _owned: set[str] = set(json.loads(_manifest_path.read_text(encoding="utf-8")).get("files", []))
+        _manifest_existed = True
     except (OSError, ValueError):
         _owned = set()
+        _manifest_existed = False
+    if not _manifest_existed:
+        # A vault written before the manifest existed has no record of what
+        # graphify owns, so every note it wrote last time reads as the user's and
+        # is skipped. The re-export then writes fresh notes BESIDE the stale ones
+        # and the vault carries two generations, with a warning claiming graphify
+        # "did not create" files it did (#2863). Adopt the notes that carry
+        # graphify's own frontmatter, once, so the manifest starts out honest.
+        _owned |= _adopt_pre_manifest_notes(out)
     _written: list[str] = []
     _skipped: list[str] = []
 

tests/test_obsidian_vault_migration.py

@@ -0,0 +1,182 @@
+"""A vault written before the ownership manifest must not end up with two generations.
+
+#1506 added `.graphify_obsidian_manifest.json` so a re-export can update its own
+notes while refusing to touch the user's. A vault created by an EARLIER graphify
+has no manifest, so `_owned` starts empty and every note graphify itself wrote
+last time reads as a user file: the re-export writes fresh notes beside the
+stale ones, and warns that graphify "did not create" files it did (#2863).
+
+The fix adopts, once, the notes carrying graphify's own signature:
+
+  * node notes    - YAML frontmatter with a tag in the ``graphify/`` namespace
+  * community notes - no frontmatter at all, so matched on graphify's own
+                      filename prefix AND the Dataview query it writes
+
+`.obsidian/graph.json` is deliberately NOT adopted: graphify writes one, but so
+does Obsidian, and without a manifest there is no way to tell whose it is.
+"""
+import io
+import json
+from contextlib import redirect_stderr
+from pathlib import Path
+
+import pytest
+
+from graphify.build import build_from_json
+from graphify.export import to_obsidian
+
+try:
+    from graphify.export import _adopt_pre_manifest_notes, _is_graphify_note
+except ImportError:  # pre-fix tree
+    _adopt_pre_manifest_notes = _is_graphify_note = None
+
+MANIFEST = ".graphify_obsidian_manifest.json"
+
+
+def _graph(labels):
+    nodes = [{"id": f"n{i}", "label": l, "file_type": "document",
+              "source_file": f"d{i}.md"} for i, l in enumerate(labels)]
+    edges = [{"source": f"n{i}", "target": f"n{i+1}", "relation": "references",
+              "confidence": "EXTRACTED", "source_file": f"d{i}.md"}
+             for i in range(len(labels) - 1)]
+    return build_from_json({"nodes": nodes, "edges": edges, "hyperedges": []})
+
+
+def _export(vault, labels):
+    ids = [f"n{i}" for i in range(len(labels))]
+    buf = io.StringIO()
+    with redirect_stderr(buf):
+        to_obsidian(_graph(labels), {0: ids}, str(vault))
+    return buf.getvalue()
+
+
+def _notes(vault):
+    return {p.name for p in Path(vault).glob("*.md")}
+
+
+@pytest.fixture
+def pre_manifest_vault(tmp_path):
+    """A vault as an older graphify would have left it: its notes, no manifest."""
+    vault = tmp_path / "vault"
+    _export(vault, ["Alpha", "Beta", "Gamma"])
+    (vault / MANIFEST).unlink()
+    return vault
+
+
+# ---------------------------------------------------------------------------
+# The bug
+# ---------------------------------------------------------------------------
+
+def test_a_reexport_leaves_no_second_generation(pre_manifest_vault):
+    _export(pre_manifest_vault, ["Alpha", "Beta renamed", "Gamma renamed"])
+    assert _notes(pre_manifest_vault) == {
+        "Alpha.md", "Beta renamed.md", "Gamma renamed.md",
+        "_COMMUNITY_Community 0.md",
+    }
+
+
+def test_stale_notes_for_renamed_nodes_are_gone(pre_manifest_vault):
+    _export(pre_manifest_vault, ["Alpha", "Beta renamed", "Gamma renamed"])
+    left = _notes(pre_manifest_vault)
+    assert "Beta.md" not in left and "Gamma.md" not in left
+
+
+def test_graphify_no_longer_claims_it_did_not_write_its_own_notes(pre_manifest_vault):
+    err = _export(pre_manifest_vault, ["Alpha", "Beta renamed", "Gamma renamed"])
+    for name in ("Alpha.md", "Beta.md", "_COMMUNITY_"):
+        assert name not in err, err
+
+
+def test_the_manifest_is_written_so_migration_happens_once(pre_manifest_vault):
+    _export(pre_manifest_vault, ["Alpha", "Beta", "Gamma"])
+    owned = json.loads((pre_manifest_vault / MANIFEST).read_text(encoding="utf-8"))["files"]
+    assert "Alpha.md" in owned
+    assert any(f.startswith("_COMMUNITY_") for f in owned)
+
+
+# ---------------------------------------------------------------------------
+# The user's own notes must still be safe
+# ---------------------------------------------------------------------------
+
+def test_a_users_own_note_is_never_adopted(pre_manifest_vault):
+    mine = pre_manifest_vault / "Alpha.md"
+    mine.write_text("# My own note about Alpha\n\nI wrote this.\n", encoding="utf-8")
+    _export(pre_manifest_vault, ["Alpha", "Beta", "Gamma"])
+    assert "I wrote this." in mine.read_text(encoding="utf-8")
+
+
+def test_a_note_merely_mentioning_graphify_is_not_adopted(pre_manifest_vault):
+    mine = pre_manifest_vault / "Notes.md"
+    mine.write_text("# Notes\n\nI use graphify/document tags manually.\n", encoding="utf-8")
+    _export(pre_manifest_vault, ["Alpha", "Beta", "Gamma"])
+    assert "I use graphify" in mine.read_text(encoding="utf-8")
+
+
+def test_a_user_community_named_file_needs_the_query_marker_too(pre_manifest_vault):
+    """The filename prefix alone must not be enough to adopt a file."""
+    mine = pre_manifest_vault / "_COMMUNITY_mine.md"
+    mine.write_text("# My own community summary\n\nhand written\n", encoding="utf-8")
+    _export(pre_manifest_vault, ["Alpha", "Beta", "Gamma"])
+    assert "hand written" in mine.read_text(encoding="utf-8")
+
+
+def test_obsidian_config_is_not_adopted(pre_manifest_vault):
+    """graphify writes .obsidian/graph.json, but so does Obsidian. With no
+    manifest there is no way to tell, so it stays unowned."""
+    cfg = pre_manifest_vault / ".obsidian" / "graph.json"
+    cfg.parent.mkdir(exist_ok=True)
+    cfg.write_text('{"mine": true}', encoding="utf-8")
+    _export(pre_manifest_vault, ["Alpha", "Beta", "Gamma"])
+    assert json.loads(cfg.read_text(encoding="utf-8")) == {"mine": True}
+
+
+# ---------------------------------------------------------------------------
+# Unchanged behaviour
+# ---------------------------------------------------------------------------
+
+def test_a_vault_with_a_manifest_is_untouched_by_the_migration(tmp_path):
+    vault = tmp_path / "v"
+    _export(vault, ["Alpha", "Beta"])
+    before = json.loads((vault / MANIFEST).read_text(encoding="utf-8"))["files"]
+    _export(vault, ["Alpha", "Beta"])
+    after = json.loads((vault / MANIFEST).read_text(encoding="utf-8"))["files"]
+    assert sorted(before) == sorted(after)
+
+
+def test_a_fresh_directory_still_works(tmp_path):
+    vault = tmp_path / "brand-new"
+    _export(vault, ["Alpha", "Beta"])
+    assert "Alpha.md" in _notes(vault)
+
+
+# ---------------------------------------------------------------------------
+# The detector itself
+# ---------------------------------------------------------------------------
+
+@pytest.mark.skipif(_is_graphify_note is None, reason="pre-fix tree")
+def test_detector_accepts_a_graphify_node_note(tmp_path):
+    p = tmp_path / "n.md"
+    p.write_text('---\nsource_file: "a.md"\ntags:\n  - graphify/document\n---\n\n# N\n',
+                 encoding="utf-8")
+    assert _is_graphify_note(p)
+
+
+@pytest.mark.skipif(_is_graphify_note is None, reason="pre-fix tree")
+@pytest.mark.parametrize("body", [
+    "# plain note\n",
+    "---\ntitle: mine\ntags:\n  - personal\n---\n\n# mine\n",
+    "---\nnot even closed\n",
+    "",
+])
+def test_detector_rejects_everything_else(tmp_path, body):
+    p = tmp_path / "x.md"
+    p.write_text(body, encoding="utf-8")
+    assert not _is_graphify_note(p)
+
+
+@pytest.mark.skipif(_adopt_pre_manifest_notes is None, reason="pre-fix tree")
+def test_adoption_only_looks_at_top_level_markdown(tmp_path):
+    (tmp_path / "sub").mkdir()
+    (tmp_path / "sub" / "deep.md").write_text(
+        "---\ntags:\n  - graphify/document\n---\n", encoding="utf-8")
+    assert _adopt_pre_manifest_notes(tmp_path) == set()