Search- and selection-driven curation (curate --ids/--from, search --save)

CHANGELOG.md

@@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- **Search- and selection-driven curation.** `forge curate` now selects
+  episodes three ways — a SQL `--where` predicate, an explicit `--ids` list, or
+  a `--from <selection.json>` file — so semantic search and Forge Studio can
+  drive curation, not just SQL. `forge search --save sel.json` writes the result
+  ids (with provenance) and Studio's dedup tab exports a matching decisions file;
+  `forge curate --from sel.json` applies them and records the source in
+  `labeled_by`. Explicit ids are filtered to episodes that exist (no dangling
+  labels) and can be intersected with `--where`.
+
 - **Catalog dedup, curation + Forge Studio (Phase 3).** Find near-duplicate
   episodes from the embeddings and curate a clean, labeled training set.
 

README.md

@@ -253,6 +253,13 @@ forge curate -c ./forge-catalog \
     --dedup 0.97 --dedup-policy keep-higher-quality --label approved
 ```
 
+Curation isn't only SQL-driven — **semantic search and visual review can drive it too**. Save a search result (or Studio's keep/reject decisions) to a selection file and label it, provenance included:
+
+```bash
+forge search "regrasp after a failed pick" -c ./forge-catalog --top 20 --save sel.json
+forge curate -c ./forge-catalog --from sel.json --label approved     # or --ids e1,e2,e3
+```
+
 Curation is an append-log (`curation_labels`, latest-wins) — nothing is deleted. Policies: `keep-higher-quality`, `keep-longer`, `keep-first`.
 
 ## 4. Forge Studio

forge/catalog/README.md

@@ -132,9 +132,13 @@ cat.sql("SELECT task, count(*) FROM episodes GROUP BY task")   # -> pyarrow.Tabl
   ANN pass is the scale path (skipped-with-warning until then).
 - **`curation_labels`** — an append-log of decisions (`approved` / `rejected` /
   `held`); latest row wins per episode, history preserved. `forge curate`
-  selects with a WHERE filter, resolves near-dup losers under a **policy**
-  (`keep-higher-quality` / `keep-longer` / `keep-first`), and labels survivors
-  approved, losers rejected.
+  selects episodes three ways — a SQL `--where` predicate, an explicit `--ids`
+  list, or a `--from` selection file — so **semantic search and Forge Studio can
+  drive curation, not just SQL**. It then resolves near-dup losers under a
+  **policy** (`keep-higher-quality` / `keep-longer` / `keep-first`) and labels
+  survivors approved, losers rejected. `forge search --save sel.json` and
+  Studio's dedup tab both emit the `{episode_ids, source}` selection file that
+  `--from` consumes, and the `source` is recorded in `labeled_by` for provenance.
 - **`forge studio`** ([studio.py](studio.py)) renders a self-contained, themed
   HTML app (Overview · Corpus · Dedup review · Snapshot) from real catalog data
   and embedded video thumbnails — the design system matches the Forge Studio

forge/catalog/_studio_template.py

@@ -281,7 +281,8 @@
   el.innerHTML=`<div class="crumb">curation / <b>dedup review</b></div><h1>Dedup review</h1>
    <div class="sub">${fmt(pairs.length)} pairs above ${D.dedup.threshold} cosine &middot; review, then apply a policy with <span class="mono" style="color:var(--cyan)">forge curate</span></div>
    <div class="dedup-head"><div class="lozenge num">decided <b id="dc">0</b> / ${pairs.length}</div>
-     <button class="btn" id="export">Copy forge curate command</button>
+     <button class="btn" id="download">Download decisions</button>
+     <button class="btn" id="export">Copy policy command</button>
      <div class="kbd-hint"><b>&larr;</b> keep left &nbsp;<b>&rarr;</b> keep right &nbsp;<b>x</b> reject both</div></div>
    <div id="pairs"></div>`;
   const q=(v)=>`<span class="qq num" style="background:${v>=7?'var(--good-soft)':'var(--mid-soft)'};color:${qColor(v)}">${v!=null?v.toFixed(1):'—'}</span>`;
@@ -315,7 +316,19 @@
   $('#export').addEventListener('click',()=>{
     const cmd=`forge curate --catalog ${D.catalog} \\\n  --dedup ${D.dedup.threshold} --dedup-policy keep-higher-quality \\\n  --label approved`;
     navigator.clipboard&&navigator.clipboard.writeText(cmd);
-    $('#export').textContent='Copied ✓';setTimeout(()=>$('#export').textContent='Copy forge curate command',1400);});
+    $('#export').textContent='Copied ✓';setTimeout(()=>$('#export').textContent='Copy policy command',1400);});
+  // Hand-picked decisions -> a selection file for `forge curate --from`.
+  $('#download').addEventListener('click',()=>{
+    const rejected=new Set();
+    pairs.forEach((p,i)=>{const a=decisions[i];
+      if(a==='a')rejected.add(p.b); if(a==='b')rejected.add(p.a);
+      if(a==='reject'){rejected.add(p.a);rejected.add(p.b);}});
+    const sel={episode_ids:[...rejected],label:'rejected',source:'studio-dedup'};
+    const blob=new Blob([JSON.stringify(sel,null,2)],{type:'application/json'});
+    const a=document.createElement('a');a.href=URL.createObjectURL(blob);
+    a.download='studio_decisions.json';a.click();URL.revokeObjectURL(a.href);
+    $('#download').textContent=`Saved ${rejected.size} ✓`;
+    setTimeout(()=>$('#download').textContent='Download decisions',1600);});
 })();
 
 /* ── SNAPSHOT (Phase 4 preview) ── */

forge/catalog/cli.py

@@ -367,12 +367,16 @@ def search_cmd(
     output_format: str = typer.Option(
         "table", "--format", "-f", help="table | json | csv"
     ),
+    save: str = typer.Option(
+        None, "--save", help="Write the result ids to a selection JSON for `forge curate --from`"
+    ),
 ) -> None:
     """Semantic search over a catalog's embeddings.
 
     Examples:
         forge search "picks up the red cup" -c ./forge-catalog --top 10
-        forge search --like <episode_id> -c ./forge-catalog
+        forge search "regrasp after a failed pick" -c ./cat --save sel.json
+        forge curate -c ./cat --from sel.json --label approved
     """
     _require_catalog_deps()
     from forge.catalog import Catalog
@@ -396,6 +400,17 @@ def search_cmd(
         console.print(f"[red]Error:[/red] {e}")
         raise typer.Exit(1)
 
+    if save:
+        ids = result.column("episode_id").to_pylist()
+        source = f"search: {query!r}" if query else f"search: like {like}"
+        with open(save, "w") as f:
+            json.dump({"episode_ids": ids, "source": source}, f, indent=2)
+        console.print(
+            f"[green]Saved[/green] {len(ids)} ids → [cyan]{save}[/cyan] "
+            f"[dim](feed it to `forge curate --from {save}`)[/dim]"
+        )
+        return
+
     _render_result(result, output_format)
 
 
@@ -439,6 +454,12 @@ def curate_cmd(
     where: str = typer.Option(
         None, "--where", help="SQL filter over episodes + quality (e.g. \"overall_score > 6\")"
     ),
+    ids: str = typer.Option(
+        None, "--ids", help="Comma-separated episode ids to label (from search / Studio)"
+    ),
+    from_file: str = typer.Option(
+        None, "--from", help="Selection JSON with episode_ids (from `forge search --save` or Studio)"
+    ),
     label: str = typer.Option("approved", "--label", help="Label for survivors"),
     reason: str = typer.Option(None, "--reason", help="Free-text reason"),
     dedup: float = typer.Option(
@@ -452,25 +473,49 @@ def curate_cmd(
 ) -> None:
     """Label a selection of episodes, optionally dropping near-duplicates.
 
+    The selection is a SQL `--where` predicate, an explicit `--ids` list, or a
+    `--from` selection file (produced by `forge search --save` or Forge Studio) —
+    so semantic search and visual review can drive curation, not just SQL.
     Appends to the curation_labels log (latest-row-wins); never deletes history.
 
     Examples:
         forge curate -c ./cat --where "overall_score > 6 AND task='pick_place'" --label approved
         forge curate -c ./cat --dedup 0.97 --dedup-policy keep-higher-quality --label approved
+        forge curate -c ./cat --from sel.json --label approved
     """
     _require_catalog_deps()
     from forge.catalog import Catalog
     from forge.catalog.dedup import compute_dedup_edges, curate
     from forge.core.exceptions import ForgeError
 
+    id_list = None
+    labeled_by = f"user:{by}"
+    if from_file:
+        try:
+            with open(from_file) as f:
+                sel = json.load(f)
+        except (OSError, ValueError) as e:
+            console.print(f"[red]Error:[/red] can't read selection file {from_file}: {e}")
+            raise typer.Exit(1)
+        id_list = sel.get("episode_ids") or []
+        source = sel.get("source")
+        if reason is None:
+            reason = sel.get("reason") or (f"from {source}" if source else None)
+        if source:
+            labeled_by = source
+        if label == "approved" and sel.get("label"):
+            label = sel["label"]
+    elif ids:
+        id_list = [x.strip() for x in ids.split(",") if x.strip()]
+
     try:
         cat = Catalog.open(catalog)
         if dedup is not None:
             # Ensure edges exist at this threshold (idempotent).
             compute_dedup_edges(cat, threshold=dedup)
         stats = curate(
-            cat, where=where, label=label, reason=reason, labeled_by=f"user:{by}",
-            dedup_threshold=dedup, dedup_policy=dedup_policy,
+            cat, where=where, ids=id_list, label=label, reason=reason,
+            labeled_by=labeled_by, dedup_threshold=dedup, dedup_policy=dedup_policy,
         )
     except (ForgeError, ValueError) as e:
         console.print(f"[red]Error:[/red] {e}")

forge/catalog/dedup.py

@@ -140,13 +140,19 @@ def curate(
     catalog: Catalog,
     *,
     where: str | None = None,
+    ids: list[str] | None = None,
     label: str = "approved",
     reason: str | None = None,
     labeled_by: str = "policy:curate",
     dedup_threshold: float | None = None,
     dedup_policy: str = DEFAULT_POLICY,
 ) -> CurateStats:
-    """Label a WHERE-selected set, dropping near-dup losers under a policy.
+    """Label a selection, dropping near-dup losers under a policy.
+
+    The selection is either an explicit ``ids`` list (from ``forge search`` or
+    Forge Studio) or a SQL ``where`` predicate — or both, in which case ``ids``
+    is filtered by the predicate. ``ids`` are filtered to episodes that actually
+    exist, so typos/stale ids never create dangling labels.
 
     Survivors get ``label`` (default ``approved``); if ``dedup_threshold`` is
     given, the dedup losers within the selection get ``rejected``. Appends to the
@@ -161,9 +167,22 @@ def curate(
         "SELECT e.episode_id FROM episodes e "
         "LEFT JOIN v_latest_quality q USING(episode_id)"
     )
-    if where:
-        base += f" WHERE {where}"
-    selected = [r["episode_id"] for r in catalog.sql(base).to_pylist()]
+    if ids is not None:
+        # Explicit selection: keep order, de-dup, and intersect with the
+        # predicate (if any) and with episodes that actually exist.
+        matched = {
+            r["episode_id"]
+            for r in catalog.sql(base + (f" WHERE {where}" if where else "")).to_pylist()
+        }
+        seen: set[str] = set()
+        selected = [
+            e for e in ids
+            if e in matched and not (e in seen or seen.add(e))
+        ]
+    else:
+        if where:
+            base += f" WHERE {where}"
+        selected = [r["episode_id"] for r in catalog.sql(base).to_pylist()]
     stats.selected = len(selected)
     if not selected:
         return stats

tests/test_catalog_dedup.py

@@ -170,6 +170,26 @@ def test_unknown_policy_raises(self, dup_catalog):
         with pytest.raises(ValueError, match="unknown dedup policy"):
             curate(dup_catalog, dedup_threshold=0.97, dedup_policy="bogus")
 
+    def test_curate_by_ids(self, dup_catalog):
+        st = curate(dup_catalog, ids=["e0", "e2"], label="approved")
+        assert st.selected == 2 and st.approved == 2
+        labels = {r["episode_id"]: r["label"] for r in dup_catalog.sql(
+            "SELECT episode_id, label FROM v_curation").to_pylist()}
+        assert labels == {"e0": "approved", "e2": "approved"}
+
+    def test_curate_ids_filtered_to_existing(self, dup_catalog):
+        # a typo'd / stale id never creates a dangling label
+        st = curate(dup_catalog, ids=["e0", "does-not-exist"], label="held")
+        assert st.selected == 1
+        assert dup_catalog.sql("SELECT count(*) n FROM curation_labels").to_pylist()[0]["n"] == 1
+
+    def test_curate_ids_intersect_where(self, dup_catalog):
+        # ids [e0(q8), e1(q6)] filtered by score>=7 -> only e0
+        st = curate(dup_catalog, ids=["e0", "e1"], where="overall_score >= 7", label="approved")
+        assert st.selected == 1
+        assert dup_catalog.sql(
+            "SELECT episode_id FROM v_curation").to_pylist()[0]["episode_id"] == "e0"
+
 
 class TestStudio:
     def test_generate_html(self, dup_catalog):
@@ -208,3 +228,37 @@ def test_dedup_curate_studio(self, tmp_path: Path, dup_catalog):
         r = runner.invoke(app, ["studio", "-c", root, "-o", out, "--max-thumbnails", "0"])
         assert r.exit_code == 0, r.output
         assert Path(out).exists()
+
+    def test_search_save_then_curate_from(self, tmp_path: Path, dup_catalog):
+        from typer.testing import CliRunner
+
+        from forge.cli import app
+
+        runner = CliRunner()
+        root = dup_catalog.root_uri
+        sel = str(tmp_path / "sel.json")
+
+        # search --like needs no model; --save writes a selection file
+        r = runner.invoke(app, ["search", "--like", "e0", "-c", root, "--save", sel])
+        assert r.exit_code == 0, r.output
+        saved = json.loads(Path(sel).read_text())
+        assert "e0" in saved["episode_ids"] and "like e0" in saved["source"]
+
+        # curate --from applies it, recording provenance in labeled_by
+        r = runner.invoke(app, ["curate", "-c", root, "--from", sel, "--label", "approved"])
+        assert r.exit_code == 0, r.output
+        rows = dup_catalog.sql(
+            "SELECT DISTINCT labeled_by FROM curation_labels WHERE label='approved'"
+        ).to_pylist()
+        assert any("like e0" in r["labeled_by"] for r in rows)
+
+    def test_curate_ids_flag(self, dup_catalog):
+        from typer.testing import CliRunner
+
+        from forge.cli import app
+
+        r = CliRunner().invoke(
+            app, ["curate", "-c", dup_catalog.root_uri, "--ids", "e0,e2", "--label", "held"]
+        )
+        assert r.exit_code == 0, r.output
+        assert "2" in r.output