Add ability for block plugins to provide custom vue components and npm deps

.docker/app/Dockerfile

@@ -5,6 +5,12 @@ FROM plugin_environment AS plugin-panels
 COPY webapp /webapp
 RUN /opt/.venv/bin/invoke dev.collect-plugin-panels
 
+# Copy plugin's package.json files alone (yarn workspaces), so that editing a plugin's
+# components does not invalidate the dependency install below
+RUN mkdir /plugin-packages \
+    && cd /webapp/src/plugins \
+    && find . -mindepth 2 -maxdepth 2 -name package.json -exec cp --parents {} /plugin-packages/ \;
+
 FROM node:22.22-trixie-slim AS base
 SHELL ["/bin/bash", "--login", "-c"]
 
@@ -15,16 +21,28 @@ EXPOSE 8081
 RUN apt update && apt install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
 
 COPY webapp/package.json webapp/yarn.lock ./
-
-# Using a custom node_modules location to avoid mounting it outside of docker
-RUN --mount=type=cache,target=/root/.cache/yarn yarn install --frozen-lockfile --modules-folder /node_modules
+COPY --from=plugin-panels /plugin-packages ./src/plugins
+
+# Using a custom node_modules location to avoid mounting it outside of docker.
+# Plugin dependencies are not in the committed yarn.lock, so when plugins declare
+# any, install with --pure-lockfile (core pins are kept; the lockfile is not
+# written). Plugin-specific versions that conflict with the webapp's are nested
+# in src/plugins/<package>/node_modules rather than in /node_modules.
+RUN --mount=type=cache,target=/root/.cache/yarn \
+    if compgen -G "src/plugins/*/package.json" > /dev/null; then \
+        yarn install --pure-lockfile --modules-folder /node_modules; \
+    else \
+        yarn install --frozen-lockfile --modules-folder /node_modules; \
+    fi
 
 # dev target: deps installed, source mounted at runtime — no production build.
 # Also installs the system libraries the Cypress test runner needs (GTK/X11
 # runtime libs plus xvfb for a virtual display), so the webapp test suites can
 # run inside the container.
 FROM base AS dev
 COPY --from=plugin-panels /webapp/src/plugins /opt/datalab-plugin-panels
+# Add any nested plugin node_modules, as /app/src is bind-mounted at runtime
+RUN cp -a /app/src/plugins/. /opt/datalab-plugin-panels/
 COPY .docker/app_dev_entrypoint.sh /usr/local/bin/datalab-app-dev
 RUN apt update \
     && apt install -y --no-install-recommends \

pydatalab/docs/plugins.md

@@ -17,7 +17,9 @@ Plugins can also be kept private and installed from e.g., a private git reposito
 
 At present, a *datalab* plugin is a Python package that registers one or more [data block](blocks/index.md) classes or item types via a Python entry point.
 Data blocks ingest a file (or set of files) attached to an item and render an interactive view of the parsed data, e.g. an NMR spectrum, an electrochemistry cycler trace, or an XRD pattern.
-Plugins (and deployments) can also register **custom item types**, new top-level item models served through the generic item endpoints (see [below](#custom-item-types)). Further plugin types, e.g., ingestion hooks and webapp components, are planned in the future (see [roadmap.md](roadmap.md)); please reach out if you have a specific use case.
+Plugins (and deployments) can also register **custom item types**, new top-level item models served through the generic item endpoints (see [below](#custom-item-types)).
+Plugins can ship Vue components for their blocks and item types (see [below](#webapp-components)).
+Further plugin types, e.g., ingestion hooks, are planned in the future (see [roadmap.md](roadmap.md)); please reach out if you have a specific use case.
 *datalab* discovers them at server startup by enumerating the relevant entry point group, with no changes required to the core code.
 
 ## Writing a plugin
@@ -320,6 +322,49 @@ resulting mixture live — none of which the core panel can do on its own.
     perform collection before the development or production frontend is compiled. Only install
     panels from sources you trust.
 
+## Webapp components
+
+A plugin can ship Vue components by placing a `webapp/` folder containing an `index.js` manifest
+inside its Python package. The manifest declares components for two extension points: `blocks`,
+keyed by blocktype, and `itemPanels`, keyed by item type (as with the panel convention above,
+with a leading underscore, e.g. `_mixed_solutions`):
+
+```js
+// my_plugin/webapp/index.js
+export default {
+  apiVersion: 1,
+  blocks: {
+    my_block: () => import("./MyBlock.vue"),
+  },
+  itemPanels: {},
+};
+```
+
+The whole folder is collected into the webapp by `invoke dev.install` (as for custom panels above),
+so components can import each other relatively and datalab's own components via `@/…`. A block
+component receives `item_id` and `block_id` props and typically wraps `DataBlockBase`. Plugin
+blocks cannot replace built-in block components, and a manifest takes precedence over the
+`<ClassName>Panel.vue` convention for its package.
+The plugin's `pyproject.toml` must include the folder as package data.
+
+A plugin can declare npm dependencies with a `package.json` (requiring `name` and `version`)
+in the same folder. Each collected plugin becomes a [yarn workspace](https://classic.yarnpkg.com/en/docs/workspaces/)
+of the webapp, and `invoke dev.install` runs `yarn install --pure-lockfile`, which keeps the
+webapp's locked versions without writing the plugin's dependencies into `yarn.lock`. Versions
+that conflict with the webapp's are installed alongside it for the plugin alone. Packages that
+must be shared with the webapp (`vue`, `vuex`, `vue-router`, `primevue`, `@primeuix/themes`,
+`@bokeh/bokehjs` and `@fortawesome/*`) may only be listed under `peerDependencies`:
+
+```json
+{
+  "name": "my-plugin-webapp",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": { "markdown-it": "^14.1.0" },
+  "peerDependencies": { "vue": "^3.5.0" }
+}
+```
+
 ## Deployment integration
 
 The same `invoke dev.install` task is used by both Docker API images (`.docker/server/Dockerfile`): a `plugins.toml` at the repository root is picked up automatically at build time, and both frontend profiles collect panels from the corresponding plugin-enabled API image. Plugins can therefore be baked into a complete deployment without modifying either Dockerfile.

pydatalab/docs/roadmap.md

@@ -11,7 +11,6 @@ The plugin entry points exposed today only cover [data blocks](plugins.md). Plan
 
 - **Custom item types** [in-progress #1767](https://github.com/datalab-org/datalab/issues/1767) — register new top-level item models (beyond the built-in samples, cells, and starting materials) from a plugin package.
 - **Ingestion hooks** — allow plugins to register handlers that run on file upload, item creation, or other lifecycle events.
-- **Webapp components** — distribute Vue components alongside the Python plugin package so that custom blocks and item types can ship their own UI.
 
 ## Other planned work
 

pydatalab/tasks.py

@@ -130,7 +130,7 @@ def generate_schemas(_):
             json.dump(hint_schema, f, indent=2)
 
     with open(schemas_path / "plugin_config.json", "w") as f:
-        json.dump(load_plugin_schema().schema(), f, indent=2)
+        json.dump(load_plugin_schema().model_json_schema(), f, indent=2)
 
 
 dev.add_task(generate_schemas)
@@ -151,21 +151,87 @@ def _panel_item_type(entry_point) -> str:
     return item_type
 
 
-def _write_plugin_panel_index(path: pathlib.Path, registered: dict[str, str]) -> None:
-    """Write the deterministic generated Vue panel registry."""
+def _write_plugin_panel_index(
+    path: pathlib.Path, registered: dict[str, str], manifests: list[str] | None = None
+) -> None:
+    """Write the deterministic generated Vue panel registry.
+
+    ``manifests`` lists the packages that ship a ``webapp/index.js`` manifest;
+    each is imported and exported in ``PLUGINS`` for ``plugins/index.js`` to merge.
+    """
     lines = [
         "// Auto-generated by `invoke dev.collect-plugin-panels` — do not edit manually.",
         "// Re-run plugin installation after installing or updating plugins.",
-        "export const PLUGIN_PANELS = {",
     ]
+    if manifests:
+        for package_name in manifests:
+            lines.append(f'import {package_name} from "./{package_name}/index.js";')
+        lines.append(f"export const PLUGINS = [{', '.join(manifests)}];")
+    lines.append("export const PLUGIN_PANELS = {")
     for type_name, import_path in sorted(registered.items()):
         lines.append(f'  {type_name}: () => import("{import_path}"),')
     lines.append("};")
     path.write_text("\n".join(lines) + "\n", encoding="utf-8")
 
 
+# Packages holding global state that plugins must share with the webapp, so
+# they may only be declared as `peerDependencies` of a plugin's package.json.
+SHARED_WEBAPP_PACKAGES = {
+    "vue",
+    "vuex",
+    "vue-router",
+    "primevue",
+    "@primeuix/themes",
+    "@bokeh/bokehjs",
+}
+SHARED_WEBAPP_PACKAGE_PREFIXES = ("@fortawesome/",)
+
+
+def _check_plugin_package_json(path: pathlib.Path, package_name: str) -> None:
+    """Validate the package.json shipped in a plugin's webapp folder."""
+    data = json.loads(path.read_text(encoding="utf-8"))
+    for key in ("name", "version"):
+        if not data.get(key):
+            raise ValueError(f"{package_name}/webapp/package.json must set `{key}`.")
+    shared = sorted(
+        dep
+        for dep in data.get("dependencies", {})
+        if dep in SHARED_WEBAPP_PACKAGES or dep.startswith(SHARED_WEBAPP_PACKAGE_PREFIXES)
+    )
+    if shared:
+        raise ValueError(
+            f"{package_name}/webapp/package.json lists {shared} in `dependencies`; "
+            "declare them in `peerDependencies` so the webapp's copies are shared."
+        )
+
+
+def _install_plugin_webapp_dependencies(src_path: pathlib.Path) -> bool:
+    """Install npm dependencies declared by collected plugins.
+
+    Each plugin folder with a package.json is a yarn workspace of the webapp.
+    ``--pure-lockfile`` keeps the committed ``yarn.lock`` pins for the core webapp
+    without writing the plugins' entries back into it.
+    """
+    if not any((src_path / "plugins").glob("*/package.json")):
+        return False
+    yarn = shutil.which("yarn")
+    if yarn is None:
+        print(
+            "Plugins declare webapp dependencies but `yarn` was not found; "
+            "run `yarn install --pure-lockfile` in webapp/ to install them."
+        )
+        return False
+    subprocess.run([yarn, "install", "--pure-lockfile"], cwd=src_path.parent, check=True)  # noqa: S603
+    return True
+
+
 def _collect_plugin_panels(src_path: pathlib.Path | None = None) -> dict[str, str]:
-    """Collect installed plugin Vue panels into ``webapp/src/plugins``.
+    """Collect installed plugin Vue components into ``webapp/src/plugins``.
+
+    Packages registering ``pydatalab.item_types`` or ``pydatalab.apps.plugins``
+    entry points are scanned for a ``webapp/`` folder. If it contains an
+    ``index.js`` manifest, the whole folder is copied and the manifest is
+    registered; otherwise, the legacy ``<ClassName>Panel.vue`` convention applies.
 
     The destination is treated as generated output except for the committed
     ``index.js`` loader. Rebuilding therefore cannot retain panels from plugins
@@ -188,15 +254,20 @@ def _collect_plugin_panels(src_path: pathlib.Path | None = None) -> dict[str, st
             generated_path.unlink()
 
     registered: dict[str, str] = {}
+    manifests: list[str] = []
     eps = sorted(
         entry_points(group="pydatalab.item_types"),
         key=lambda entry_point: (entry_point.name, entry_point.value),
     )
-    if not eps:
-        print("No pydatalab.item_types entry points found.")
+    block_eps = sorted(
+        entry_points(group="pydatalab.apps.plugins"),
+        key=lambda entry_point: (entry_point.name, entry_point.value),
+    )
+    if not eps and not block_eps:
+        print("No pydatalab.item_types or pydatalab.apps.plugins entry points found.")
 
     seen_packages: set[str] = set()
-    for ep in eps:
+    for ep in eps + block_eps:
         package_name = ep.value.split(":")[0].split(".")[0]
         if package_name in seen_packages:
             continue
@@ -218,6 +289,18 @@ def _collect_plugin_panels(src_path: pathlib.Path | None = None) -> dict[str, st
             print(f"  [skip] {package_name}: no webapp/ folder found")
             continue
 
+        if (webapp_dir / "index.js").is_file():
+            shutil.copytree(
+                webapp_dir,
+                plugins_dir / package_name,
+                ignore=shutil.ignore_patterns("__pycache__", "node_modules"),
+            )
+            if (webapp_dir / "package.json").is_file():
+                _check_plugin_package_json(webapp_dir / "package.json", package_name)
+            manifests.append(package_name)
+            print(f"  Copied  {package_name}/webapp/ (manifest)")
+            continue
+
         vue_files = sorted(webapp_dir.glob("*.vue"))
         if not vue_files:
             print(f"  [skip] {package_name}: no Vue components found")
@@ -232,14 +315,18 @@ def _collect_plugin_panels(src_path: pathlib.Path | None = None) -> dict[str, st
 
     for ep in eps:
         package_name = ep.value.split(":")[0].split(".")[0]
+        if package_name in manifests:
+            continue
         panel_name = _panel_name(ep.value)
         dest_file = plugins_dir / package_name / f"{panel_name}.vue"
         if dest_file.is_file():
             registered[_panel_item_type(ep)] = f"./{package_name}/{panel_name}.vue"
 
     generated_index = plugins_dir / "panels.generated.js"
-    _write_plugin_panel_index(generated_index, registered)
+    _write_plugin_panel_index(generated_index, registered, manifests)
     print(f"\nGenerated  {generated_index}")
+    if manifests:
+        print(f"Registered plugin manifests: {', '.join(manifests)}")
     if registered:
         print(f"Registered panels: {', '.join(sorted(registered))}")
     else:
@@ -248,27 +335,30 @@ def _collect_plugin_panels(src_path: pathlib.Path | None = None) -> dict[str, st
 
 
 def _collect_plugin_panels_if_available() -> bool:
-    """Collect panels when a complete source checkout includes the webapp."""
+    """Collect panels, and install their npm dependencies, when a complete source checkout
+    includes the webapp."""
     src_path = _webapp_src_path()
     if not src_path.is_dir():
         print(f"No webapp/src found at {src_path}; skipping plugin panel collection.")
         return False
     _collect_plugin_panels(src_path)
+    _install_plugin_webapp_dependencies(src_path)
     return True
 
 
 @task
 def collect_plugin_panels(_):
-    """Discover installed plugin Vue panels and wire them into the webapp.
+    """Discover installed plugin Vue components and wire them into the webapp.
 
-    Scans all installed ``pydatalab.item_types`` entry points. For each plugin
-    package that ships a ``webapp/<ClassName>Panel.vue`` file, the component is
-    copied into ``webapp/src/plugins/<package_name>/`` and
+    Scans all installed ``pydatalab.item_types`` and ``pydatalab.apps.plugins``
+    entry points. A plugin package whose ``webapp/`` folder contains an
+    ``index.js`` manifest (declaring ``blocks`` and/or ``itemPanels``) has the
+    folder copied into ``webapp/src/plugins/<package_name>/``, and
     ``webapp/src/plugins/panels.generated.js`` is regenerated automatically
     (the committed ``plugins/index.js`` shim loads it when present).
 
-    Convention: the model class ``MixedSolution`` → ``MixedSolutionPanel.vue``
-    (class name + "Panel").
+    Packages without a manifest fall back to the panel naming convention: the
+    model class ``MixedSolution`` → ``webapp/MixedSolutionPanel.vue``.
 
     ``dev.install`` runs this automatically for a complete source checkout. The
     explicit task remains useful for troubleshooting or refreshing generated
@@ -576,13 +666,13 @@ def install(_, dev=True):
             raw_plugin_data = tomlkit.load(f)
 
         try:
-            plugin_data = load_plugin_schema().parse_obj(raw_plugin_data.unwrap())
+            plugin_data = load_plugin_schema().model_validate(raw_plugin_data.unwrap())
         except Exception as exc:
             raise SystemExit(f"Invalid plugins.toml at {plugin_cfg}:\n{exc}") from None
 
         deps = list(plugin_data.dependencies)
         sources = {
-            name: source.dict(exclude_none=True)
+            name: source.model_dump(exclude_none=True)
             for name, source in plugin_data.tool.uv.sources.items()
         }
 
@@ -612,8 +702,11 @@ def install(_, dev=True):
 
         pyproject_data["project"]["optional-dependencies"]["plugins"] = deps
 
-        original_sources = pyproject_data.get("tool", {}).get("uv", {}).get("sources", {})
-        original_sources.update(sources)
+        # Create the table if the base pyproject.toml has no sources of its own,
+        # otherwise the plugin sources would be written to a throwaway dict
+        pyproject_data.setdefault("tool", {}).setdefault("uv", {}).setdefault("sources", {}).update(
+            sources
+        )
 
     build_dir = pathlib.Path(__file__).parent / "build"
     build_dir.mkdir(exist_ok=True)