From 083666eedf815701064431a7612b560937cb860a Mon Sep 17 00:00:00 2001 From: bogwi Date: Tue, 21 Apr 2026 00:53:36 +0900 Subject: [PATCH 01/11] Add doc codeblocks CI workflow and bin runner --- .github/workflows/doc-codeblocks.yml | 66 +++++++++++++++++ bin/run-doc-codeblocks | 105 +++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 .github/workflows/doc-codeblocks.yml create mode 100755 bin/run-doc-codeblocks diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml new file mode 100644 index 0000000000..102b863519 --- /dev/null +++ b/.github/workflows/doc-codeblocks.yml @@ -0,0 +1,66 @@ +# Issue #1759 - execute documentation code blocks in CI (md-babel-py). +# +# Policy (matches issue author): +# - Behavior and fence flags (`skip`, `expected-error`, sessions, etc.) follow +# docs/agents/docs/codeblocks.md. +# - If a block is broken and the fix is not obvious, contributors may mark the +# fence with `skip` (or `expected-error` where appropriate) and fix later; +# until then, this job fails on execution errors so drift is visible. +# - Like .github/workflows/code-cleanup.yml: refreshed / generated +# assets are committed onto the PR branch so devs are not required to run this +# locally for output-only churn. +name: doc-codeblocks +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - 'docs/**' + - 'README.md' + - 'bin/run-doc-codeblocks' + - '.github/workflows/doc-codeblocks.yml' + - 'pyproject.toml' + - 'uv.lock' + +permissions: + contents: write + packages: write + pull-requests: read + +jobs: + md-babel: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + runs-on: self-hosted + steps: + - name: Fix permissions + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} || true + + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - uses: astral-sh/setup-uv@v8 + with: + enable-cache: true + + - uses: actions/setup-node@v6 + with: + node-version: '22' + + - name: Install Python dependencies + run: | + uv sync --extra dev --frozen --no-extra dds + # Matplotlib examples live in docs but matplotlib is not in the dev extra alone. + uv pip install matplotlib + + - name: Execute documentation code blocks + run: ./bin/run-doc-codeblocks --ci + + - name: Commit doc updates + uses: stefanzweifel/git-auto-commit-action@v7 + with: + commit_message: "CI: execute documentation code blocks" diff --git a/bin/run-doc-codeblocks b/bin/run-doc-codeblocks new file mode 100755 index 0000000000..0549cd2f2e --- /dev/null +++ b/bin/run-doc-codeblocks @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Execute md-babel-py on Markdown documentation (see docs/agents/docs/codeblocks.md). +# CI uses --ci (python, shell, Node only) so native diagram toolchains are not required on runners. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +LANG_FILTER="python,sh,node" +declare -a USER_PATHS=() + +usage() { + cat >&2 <<'EOF' +Usage: bin/run-doc-codeblocks [options] [path ...] + + --ci Same as --lang python,sh,node (default; for CI runners). + --lang LIST Comma-separated languages for md-babel-py (default: python,sh,node). + --all-langs Run all block languages (requires local native tools; see codeblocks.md). + +With no paths: runs on ./docs (recursive) and ./README.md when present. + +Examples: + bin/run-doc-codeblocks --ci + bin/run-doc-codeblocks --all-langs docs/agents/docs/index.md +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --ci) + LANG_FILTER="python,sh,node" + shift + ;; + --lang) + LANG_FILTER="${2:?--lang requires a value}" + shift 2 + ;; + --all-langs) + LANG_FILTER="" + shift + ;; + -*) + echo "Unknown option: $1" >&2 + usage + exit 2 + ;; + *) + USER_PATHS+=("$1") + shift + ;; + esac +done + +resolve_md_babel() { + # Prefer `uv run` so md-babel's session subprocess uses this project's interpreter + # (plain .venv/bin/md-babel-py can pick up the wrong Python on some machines). + if command -v uv &>/dev/null && [[ -f pyproject.toml ]]; then + MB=(uv run md-babel-py) + elif [[ -x .venv/bin/md-babel-py ]]; then + MB=(.venv/bin/md-babel-py) + elif command -v md-babel-py &>/dev/null; then + MB=(md-babel-py) + else + echo "Error: md-babel-py not found. Install dev deps: uv sync --extra dev" >&2 + exit 1 + fi +} + +resolve_md_babel + +run_one_target() { + local target=$1 + local -a cmd=("${MB[@]}" run) + if [[ -n "$LANG_FILTER" ]]; then + cmd+=(--lang "$LANG_FILTER") + fi + if [[ -d "$target" ]]; then + cmd+=("$target" --recursive) + else + cmd+=("$target") + fi + "${cmd[@]}" +} + +if [[ ${#USER_PATHS[@]} -gt 0 ]]; then + for p in "${USER_PATHS[@]}"; do + if [[ -d "$p" ]] || [[ -f "$p" ]]; then + run_one_target "$p" + else + echo "Error: not a file or directory: $p" >&2 + exit 1 + fi + done +else + if [[ -d ./docs ]]; then + run_one_target ./docs + fi + if [[ -f README.md ]]; then + run_one_target README.md + fi +fi From 0e049852c9288da7b36791c915c98fb8858c27c6 Mon Sep 17 00:00:00 2001 From: bogwi Date: Tue, 21 Apr 2026 02:46:09 +0900 Subject: [PATCH 02/11] Fix documentation code blocks for md-babel-py CI and update doc-codeblocks workflow --- .github/workflows/doc-codeblocks.yml | 4 +- README.md | 8 +- docs/agents/style.md | 2 +- docs/agents/testing.md | 10 +- .../manipulation/adding_a_custom_arm.md | 31 +++--- docs/capabilities/manipulation/readme.md | 2 +- docs/capabilities/navigation/native/index.md | 29 +++--- docs/development/grid_testing.md | 20 +++- docs/development/large_file_management.md | 4 +- docs/development/testing.md | 22 +++++ docs/installation/nix.md | 6 +- docs/installation/osx.md | 6 +- docs/installation/ubuntu.md | 6 +- docs/usage/blueprints.md | 56 ++++++++--- docs/usage/cli.md | 20 ++-- docs/usage/configuration.md | 46 ++++++++- docs/usage/data_streams/README.md | 48 ++++++---- docs/usage/data_streams/quality_filter.md | 6 +- docs/usage/data_streams/reactivex.md | 10 +- docs/usage/data_streams/temporal_alignment.md | 2 +- docs/usage/lcm.md | 25 ++--- docs/usage/modules.md | 57 +++++------ docs/usage/native_modules.md | 2 +- docs/usage/python-api.md | 46 ++++----- docs/usage/sensor_streams/README.md | 48 ++++++---- docs/usage/sensor_streams/quality_filter.md | 6 +- docs/usage/sensor_streams/reactivex.md | 9 +- .../sensor_streams/temporal_alignment.md | 2 +- docs/usage/transforms.md | 63 ++++++++----- docs/usage/transports/index.md | 94 ++++++++++++++----- docs/usage/visualization.md | 23 ++++- 31 files changed, 472 insertions(+), 241 deletions(-) diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml index 102b863519..92b7dae707 100644 --- a/.github/workflows/doc-codeblocks.yml +++ b/.github/workflows/doc-codeblocks.yml @@ -54,8 +54,8 @@ jobs: - name: Install Python dependencies run: | uv sync --extra dev --frozen --no-extra dds - # Matplotlib examples live in docs but matplotlib is not in the dev extra alone. - uv pip install matplotlib + # Matplotlib and httpx are used by docs but are not always pulled by dev-only sync. + uv pip install matplotlib httpx - name: Execute documentation code blocks run: ./bin/run-doc-codeblocks --ci diff --git a/README.md b/README.md index 6f40e8dc0e..b4143e8d01 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Dimensional is agent native -- "vibecode" your robots in natural language and bu ## Interactive Install -```sh +```sh skip curl -fsSL https://raw.githubusercontent.com/dimensionalOS/dimos/main/scripts/install.sh | bash ``` @@ -222,7 +222,7 @@ dimos stop # Shut down See below a simple robot connection module that sends streams of continuous `cmd_vel` to the robot and receives `color_image` to a simple `Listener` module. DimOS Modules are subsystems on a robot that communicate with other modules using standardized messages. -```py +```py skip import threading, time, numpy as np from dimos.core.coordination.blueprints import autoconnect from dimos.core.core import rpc @@ -271,7 +271,7 @@ Blueprints are instructions for how to construct and wire modules. We compose th Blueprints can be composed, remapped, and have transports overridden if `autoconnect()` fails due to conflicting variable names or `In[]` and `Out[]` message types. A blueprint example that connects the image stream from a robot to an MCP-backed LLM agent for reasoning and action execution. -```py +```py skip from dimos.core.coordination.blueprints import autoconnect from dimos.core.transport import LCMTransport from dimos.msgs.sensor_msgs import Image @@ -308,7 +308,7 @@ if __name__ == "__main__": ## Develop on DimOS -```sh +```sh skip export GIT_LFS_SKIP_SMUDGE=1 git clone -b dev https://github.com/dimensionalOS/dimos.git cd dimos diff --git a/docs/agents/style.md b/docs/agents/style.md index 37354cc681..e94c8a0d71 100644 --- a/docs/agents/style.md +++ b/docs/agents/style.md @@ -38,7 +38,7 @@ something/ Never add imports to `__init__.py` files. Re-exporting from `__init__.py` makes imports too wide and slow — importing one symbol pulls in the entire package tree. -```python +```python skip # BAD — dimos/memory2/__init__.py from dimos.memory2.store import Store, SqliteStore from dimos.memory2.stream import Stream diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 45614c81d2..b6f6897fd6 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -59,7 +59,7 @@ def test_wiring() -> None: When a resource is shared across multiple tests, use a pytest fixture with `yield` instead of repeating context managers in each test: -```python +```python skip # GOOD - fixture handles lifecycle for all tests that use it @pytest.fixture(scope="module") def store() -> Iterator[SqliteStore]: @@ -79,7 +79,7 @@ def test_search(store: SqliteStore) -> None: Tests must be deterministic. If you don't know the state, the test is wrong. -```python +```python skip # BAD - assertion may never execute if hasattr(obj, "_disposables") and obj._disposables is not None: assert obj._disposables.is_disposed @@ -101,7 +101,7 @@ assert obj._disposables.is_disposed Don't use `time.sleep()` to wait for async operations. Use `threading.Event` to synchronize emitter/receiver patterns. -```python +```python skip # BAD - arbitrary sleep, fragile module.start() time.sleep(0.5) @@ -122,7 +122,7 @@ assert received == [84] Configuration fields on non-Pydantic classes should be private (underscore-prefixed) unless they are part of the public API. -```python +```python skip # BAD self.voxel_size = voxel_size self.carve_columns = carve_columns @@ -136,7 +136,7 @@ self._carve_columns = carve_columns Avoid `# type: ignore` by using proper types: -```python +```python skip # BAD self.vbg = None # type: ignore[assignment] diff --git a/docs/capabilities/manipulation/adding_a_custom_arm.md b/docs/capabilities/manipulation/adding_a_custom_arm.md index 85f3817fd0..ad625e5942 100644 --- a/docs/capabilities/manipulation/adding_a_custom_arm.md +++ b/docs/capabilities/manipulation/adding_a_custom_arm.md @@ -66,7 +66,7 @@ Below is a complete annotated adapter. Implement each method by wrapping your ve | Position | meters | | Force | Newtons | -```python +```python skip """YourArm adapter — implements ManipulatorAdapter protocol. SDK Units: @@ -350,7 +350,8 @@ __all__ = ["YourArmAdapter"] - **Unsupported features** — Return `None` for reads and `False` for writes. Never raise exceptions for optional features. - **Velocity/effort feedback** — If your SDK doesn't provide these, return zeros. The coordinator handles this gracefully. - **Lazy SDK import** — If the vendor SDK is an optional dependency, you can import it inside `connect()` instead of at module level (see Piper adapter for this pattern): - ```python + + ```py def connect(self) -> bool: try: from yourarm_sdk import YourArmSDK @@ -365,7 +366,7 @@ __all__ = ["YourArmAdapter"] ### \_\_init\_\_.py -```python +```python skip """YourArm manipulator hardware adapter. Usage: @@ -496,11 +497,13 @@ If you want motion planning (collision-free trajectories via Drake), you need a Place your URDF/xacro files under LFS data so they can be resolved via `LfsPath`. `LfsPath` is a `Path` subclass that lazily downloads LFS data on first access — this avoids downloading at import time when the blueprint module is loaded. -```python +```python skip from dimos.utils.data import LfsPath from dimos.manipulation.manipulation_module import manipulation_module from dimos.manipulation.planning.spec import RobotModelConfig -from dimos.msgs.geometry_msgs import PoseStamped, Quaternion, Vector3 +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Vector3 import Vector3 # LfsPath defers download until the path is actually accessed _YOURARM_URDF_PATH = LfsPath("yourarm_description/urdf/yourarm.urdf") @@ -509,6 +512,7 @@ _YOURARM_PACKAGE_PATH = LfsPath("yourarm_description") def _make_base_pose(x=0.0, y=0.0, z=0.0) -> PoseStamped: return PoseStamped( + frame_id="map", position=Vector3(x=x, y=y, z=z), orientation=Quaternion(0.0, 0.0, 0.0, 1.0), ) @@ -516,7 +520,7 @@ def _make_base_pose(x=0.0, y=0.0, z=0.0) -> PoseStamped: ### 4b. Create a robot model config helper -```python +```python skip def _make_yourarm_config( name: str = "arm", y_offset: float = 0.0, @@ -557,7 +561,7 @@ def _make_yourarm_config( Add this to your `dimos/robot/yourarm/blueprints.py` alongside the coordinator blueprint: -```python +```python skip yourarm_planner = manipulation_module( robots=[_make_yourarm_config("arm", joint_prefix="arm_", coordinator_task="traj_arm")], @@ -601,7 +605,7 @@ The blueprint registry in `dimos/robot/all_blueprints.py` is **auto-generated** ### Verify adapter registration -```python +```python skip from dimos.hardware.manipulators.registry import adapter_registry # Check your adapter shows up @@ -642,20 +646,23 @@ def test_write_positions(mock_adapter): ### Integration test with coordinator -```python -from dimos.control.blueprints import coordinator_mock +```python skip +from dimos.control.blueprints.basic import coordinator_mock +from dimos.core.coordination.module_coordinator import ModuleCoordinator # Build and start coordinator with mock hardware -coordinator = coordinator_mock.build() +coordinator = ModuleCoordinator.build(coordinator_mock) coordinator.start() # Your adapter is tested through the same coordinator interface # Just swap adapter_type="mock" to adapter_type="yourarm" in a blueprint + +coordinator.stop() ``` ### Test the real adapter standalone -```python +```python skip from dimos.hardware.manipulators.yourarm import YourArmAdapter adapter = YourArmAdapter(address="192.168.1.100", dof=6) diff --git a/docs/capabilities/manipulation/readme.md b/docs/capabilities/manipulation/readme.md index cf2348caf5..9a489c84b7 100644 --- a/docs/capabilities/manipulation/readme.md +++ b/docs/capabilities/manipulation/readme.md @@ -45,7 +45,7 @@ Then use the IPython client: python -m dimos.manipulation.planning.examples.manipulation_client ``` -```python +```python skip joints() # Get current joints plan([0.1] * 7) # Plan to target preview() # Preview in Meshcat diff --git a/docs/capabilities/navigation/native/index.md b/docs/capabilities/navigation/native/index.md index b90973bf76..e6a117c6cb 100644 --- a/docs/capabilities/navigation/native/index.md +++ b/docs/capabilities/navigation/native/index.md @@ -156,25 +156,28 @@ Goal candidates are filtered through a **safe mask** — the free-space region e The navigation stack is composed in the [`unitree_go2`](/dimos/robot/unitree/go2/blueprints/smart/unitree_go2.py) blueprint: -```python fold output=assets/go2_blueprint.svg +
Python + +```python skip fold output=assets/go2_blueprint.svg from dimos.core.coordination.blueprints import autoconnect -from dimos.core.introspection import to_svg -from dimos.mapping.costmapper import cost_mapper -from dimos.mapping.voxels import voxel_mapper -from dimos.navigation.frontier_exploration import wavefront_frontier_explorer -from dimos.navigation.replanning_a_star.module import replanning_a_star_planner +from dimos.core.introspection.svg import to_svg +from dimos.mapping.costmapper import CostMapper +from dimos.mapping.voxels import VoxelGridMapper +from dimos.navigation.frontier_exploration.wavefront_frontier_goal_selector import ( + WavefrontFrontierExplorer, +) +from dimos.navigation.replanning_a_star.module import ReplanningAStarPlanner from dimos.robot.unitree.go2.blueprints.basic.unitree_go2_basic import unitree_go2_basic unitree_go2 = autoconnect( - unitree_go2_basic, # robot connection + visualization - voxel_mapper(voxel_size=0.05), # 3D voxel mapping - cost_mapper(), # 2D costmap generation - replanning_a_star_planner(), # path planning - wavefront_frontier_explorer(), # exploration + unitree_go2_basic, + VoxelGridMapper.blueprint(), + CostMapper.blueprint(), + ReplanningAStarPlanner.blueprint(), + WavefrontFrontierExplorer.blueprint(), ).global_config(n_workers=6, robot_model="unitree_go2") to_svg(unitree_go2, "assets/go2_blueprint.svg") ``` - -![output](assets/go2_blueprint.svg) +
diff --git a/docs/development/grid_testing.md b/docs/development/grid_testing.md index 7370bda1a4..c72bda0fcb 100644 --- a/docs/development/grid_testing.md +++ b/docs/development/grid_testing.md @@ -10,7 +10,11 @@ Define a `Case` dataclass that holds everything needed to run tests against a sp from collections.abc import Callable, Iterator from contextlib import AbstractContextManager from dataclasses import dataclass, field -from typing import Any, Generic +from typing import Any, Generic, TypeVar + +TopicT = TypeVar("TopicT") +MsgT = TypeVar("MsgT") + @dataclass class Case(Generic[TopicT, MsgT]): @@ -28,7 +32,7 @@ class Case(Generic[TopicT, MsgT]): Use tags to indicate what features each implementation supports: -```python +```python skip testcases = [ Case( name="lcm_typed", @@ -49,7 +53,7 @@ testcases = [ Build separate lists for each capability to use with parametrize: -```python +```python skip all_cases = [c for c in testcases if "all" in c.tags] glob_cases = [c for c in testcases if "glob" in c.tags] regex_cases = [c for c in testcases if "regex" in c.tags] @@ -59,7 +63,7 @@ regex_cases = [c for c in testcases if "regex" in c.tags] Use the filtered lists in parametrize decorators: -```python +```python skip @pytest.mark.parametrize("case", all_cases, ids=lambda c: c.name) def test_subscribe_all(case: Case) -> None: with case.pubsub_context() as pubsub: @@ -79,6 +83,12 @@ def test_subscribe_glob(case: Case) -> None: Each implementation provides a context manager factory: ```python +from collections.abc import Generator +from contextlib import contextmanager + +from dimos.protocol.pubsub.impl.lcmpubsub import LCM + + @contextmanager def lcm_typed_context() -> Generator[LCM, None, None]: lcm = LCM() @@ -93,7 +103,7 @@ def lcm_typed_context() -> Generator[LCM, None, None]: - For typed implementations, use different types per topic to verify type handling - For bytes implementations, use simple distinguishable byte strings -```python +```python skip # Typed test data - different types per topic typed_topic_values = [ (Topic("/sensor/position", Vector3), Vector3(1, 2, 3)), diff --git a/docs/development/large_file_management.md b/docs/development/large_file_management.md index 18eedd23bb..f28086bda6 100644 --- a/docs/development/large_file_management.md +++ b/docs/development/large_file_management.md @@ -60,7 +60,7 @@ F: box "Return path" rad 5px fit wid 170% ht 170% ```python from dimos.utils.data import get_data -from dimos.msgs.sensor_msgs import Image +from dimos.msgs.sensor_msgs.Image import Image image = Image.from_file(get_data("cafe.jpg")) print(f"Image shape: {image.data.shape}") @@ -100,7 +100,7 @@ print(replay.find_closest_seek(1)) ``` -Replay loaded from: unitree_office_walk +Replay loaded from: unitree_office_walk {'type': 'msg', 'topic': 'rt/utlidar/voxel_map_compressed', 'data': {'stamp': 1751591000.0, 'frame_id': 'odom', 'resolution': 0.05, 'src_size': 77824, 'origin': [-3.625, -3.275, -0.575], 'width': [128, 128, 38], 'data': {'points': array([[ 2.725, -1.025, -0.575], [ 2.525, -0.275, -0.575], [ 2.575, -0.275, -0.575], diff --git a/docs/development/testing.md b/docs/development/testing.md index c8a226b7ad..6de87a7af1 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -83,6 +83,28 @@ Whenever you have something that needs to be cleaned up when the test is over (d Simple example code: ```python +import pytest + + +class RobotArm: + def __init__(self, device: str) -> None: + self.device = device + self._position = (0.0, 0.0, 0.0) + + def connect(self) -> None: + return None + + def disconnect(self) -> None: + return None + + def move_to(self, x: float, y: float, z: float) -> None: + self._position = (x, y, z) + + @property + def position(self) -> tuple[float, float, float]: + return self._position + + @pytest.fixture def arm(): arm = RobotArm(device="/dev/ttyUSB0") diff --git a/docs/installation/nix.md b/docs/installation/nix.md index 557bb6608a..3420d305ee 100644 --- a/docs/installation/nix.md +++ b/docs/installation/nix.md @@ -4,7 +4,7 @@ You need to have [nix](https://nixos.org/) installed and [flakes](https://nixos. [official install docs](https://nixos.org/download/) recommended, but here is a quickstart: -```sh +```sh skip # Install Nix https://nixos.org/download/ curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh @@ -15,7 +15,7 @@ mkdir -p "$HOME/.config/nix"; echo "experimental-features = nix-command flakes" # Using DimOS as a library -```sh +```sh skip mkdir myproject && cd myproject # pull the flake (needed for nix develop outside the repo) @@ -35,7 +35,7 @@ pip install "dimos[misc,sim,visualization,agents,web,perception,unitree,manipula # Developing on DimOS -```sh +```sh skip # this allows getting large files on-demand (and not pulling all immediately) export GIT_LFS_SKIP_SMUDGE=1 git clone -b dev https://github.com/dimensionalOS/dimos.git diff --git a/docs/installation/osx.md b/docs/installation/osx.md index 016e32ed5b..524ce826d9 100644 --- a/docs/installation/osx.md +++ b/docs/installation/osx.md @@ -1,6 +1,6 @@ # macOS Install (12.6 or newer) -```sh +```sh skip # install homebrew /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" # install dependencies @@ -12,7 +12,7 @@ curl -LsSf https://astral.sh/uv/install.sh | sh && export PATH="$HOME/.local/bin # Using DimOS as a library -```sh +```sh skip mkdir myproject && cd myproject uv venv --python 3.12 @@ -25,7 +25,7 @@ uv pip install 'dimos[misc,sim,visualization,agents,web,perception,unitree,manip # Developing on DimOS -```sh +```sh skip # this allows getting large files on-demand (and not pulling all immediately) export GIT_LFS_SKIP_SMUDGE=1 git clone -b dev https://github.com/dimensionalOS/dimos.git diff --git a/docs/installation/ubuntu.md b/docs/installation/ubuntu.md index 1496449a5d..239ea53be3 100644 --- a/docs/installation/ubuntu.md +++ b/docs/installation/ubuntu.md @@ -1,6 +1,6 @@ # System Dependencies Install (Ubuntu 22.04 or 24.04) -```sh +```sh skip sudo apt-get update sudo apt-get install -y curl g++ portaudio19-dev git-lfs libturbojpeg python3-dev pre-commit @@ -10,7 +10,7 @@ curl -LsSf https://astral.sh/uv/install.sh | sh && export PATH="$HOME/.local/bin # Using DimOS as a library -```sh +```sh skip mkdir myproject && cd myproject uv venv --python 3.12 @@ -23,7 +23,7 @@ uv pip install 'dimos[misc,sim,visualization,agents,web,perception,unitree,manip # Developing on DimOS -```sh +```sh skip # this allows getting large files on-demand (and not pulling all immediately) export GIT_LFS_SKIP_SMUDGE=1 git clone -b dev https://github.com/dimensionalOS/dimos.git diff --git a/docs/usage/blueprints.md b/docs/usage/blueprints.md index cc6030be66..de8fcd24d4 100644 --- a/docs/usage/blueprints.md +++ b/docs/usage/blueprints.md @@ -30,7 +30,7 @@ connection = ConnectionModule.blueprint Now you can create the blueprint with: ```python session=blueprint-ex1 -blueprint = connection('arg1', 'arg2', kwarg='value') +blueprint = connection(arg1=5, arg2="foo") ``` ## Linking blueprints @@ -233,7 +233,7 @@ blueprint = ModuleA.blueprint().global_config(n_workers=8) ## Providing blueprint configuration to users `Blueprint.config()` can be used to get a `pydantic.BaseModel` that can be used to -inspect or test configuration settings that can be passed to `Blueprint.build()`: +inspect or test configuration settings that can be passed to `ModuleCoordinator.build()`: ```python session=blueprint-ex1 # Validate config input @@ -244,13 +244,27 @@ config = base_blueprint.config() config(**blueprint_args) # raises pydantic.ValidationError if args are incorrect ``` -`dimos.robot.cli.dimos.arghelp()` is a helper function that will return a string +`dimos.robot.cli.dimos.arg_help()` is a helper function that will return a string containing all details of these arguments (this is how the output is produced when running `dimos run unitree-go2 --help`, for example): ```python session=blueprint-ex1 -from dimos.robot.cli.dimos import arghelp -print(arghelp(base_blueprint.config(), base_blueprint)) +from dimos.robot.cli.dimos import arg_help + +print(arg_help(base_blueprint.config(), base_blueprint)) +``` + + +``` + module1: + * module1.default_rpc_timeout: float (default: 120.0) + * module1.frame_id_prefix: str | None (default: None) + * module1.frame_id: str | None (default: None) + * module1.arg1: int (default: 1) + module2: + * module2.default_rpc_timeout: float (default: 120.0) + * module2.frame_id_prefix: str | None (default: None) + * module2.frame_id: str | None (default: None) ``` Another function is `dimos.robot.cli.dimos.load_config_args()` which can create the @@ -258,15 +272,22 @@ argument dict for users from a config file, environment variables and CLI argume ```python session=blueprint-ex1 +from pathlib import Path + from dimos.robot.cli.dimos import load_config_args config_path = Path.home() / "base-blueprint-config.json" -cli_args = ["arg1=5"] +cli_args = ["module1.arg1=5"] blueprint_args = load_config_args(base_blueprint.config(), cli_args, config_path) # Test user input is valid config(**blueprint_args) -# Then we can build the blueprint -base_blueprint.build(blueprint_args) +# Then pass blueprint_args to ModuleCoordinator.build(...) (see coordinator docs) +print("validated args for build:", blueprint_args) +``` + + +``` +validated args for build: {'module1': {'arg1': '5'}} ``` ## Calling the methods of other modules @@ -351,7 +372,6 @@ Skills are methods on a `Module` decorated with `@skill`. The agent automaticall from dimos.core.core import rpc from dimos.core.module import Module from dimos.agents.annotation import skill -from dimos.core.global_config import GlobalConfig class SomeSkill(Module): @@ -365,8 +385,20 @@ class SomeSkill(Module): All you have to do to build a blueprint is call: -```python session=blueprint-ex4 -module_coordinator = SomeSkill.blueprint().build(global_config=GlobalConfig()) +```python skip session=blueprint-ex4 +from dimos.core.coordination.module_coordinator import ModuleCoordinator + +module_coordinator = ModuleCoordinator.build(SomeSkill.blueprint()) +module_coordinator.stop() +``` + + +``` +16:30:00.119 [inf][dination/module_coordinator.py] Building the blueprint +16:30:00.133 [inf][dination/module_coordinator.py] Starting the modules +16:30:01.320 [inf][ation/worker_manager_python.py] Worker pool started. n_workers=2 +16:30:01.321 [inf][ation/worker_manager_python.py] Shutting down all workers... +16:30:01.480 [inf][ation/worker_manager_python.py] All workers shut down ``` This returns a `ModuleCoordinator` instance that manages all deployed modules. @@ -375,7 +407,7 @@ This returns a `ModuleCoordinator` instance that manages all deployed modules. You can block the thread until it exits with: -```python session=blueprint-ex4 +```python skip session=blueprint-ex4 module_coordinator.loop() ``` diff --git a/docs/usage/cli.md b/docs/usage/cli.md index 7a25ee4ae3..91a3500e5a 100644 --- a/docs/usage/cli.md +++ b/docs/usage/cli.md @@ -214,13 +214,21 @@ To add MCP to a blueprint, include both `McpServer` (exposes skills as HTTP tool ```python from dimos.agents.mcp.mcp_client import McpClient from dimos.agents.mcp.mcp_server import McpServer +from dimos.core.coordination.blueprints import autoconnect + +# Example wiring (replace symbols with your blueprints): +# my_mcp_blueprint = autoconnect( +# my_robot_stack, +# McpServer.blueprint(), +# McpClient.blueprint(), +# my_skill_containers, +# ) +print(autoconnect.__name__, McpServer.__name__, McpClient.__name__) +``` -my_mcp_blueprint = autoconnect( - my_robot_stack, - McpServer.blueprint(), - McpClient.blueprint(), - my_skill_containers, -) + +``` +autoconnect McpServer McpClient ``` #### `dimos mcp list-tools` diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index 97ad3cc2b6..2b6abae030 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -5,8 +5,9 @@ Dimos provides a `Configurable` base class. See [`service/spec.py`](/dimos/proto This allows using pydantic models to specify configuration structure and default values per module. ```python -from dimos.protocol.service import Configurable -from dimos.protocol.service.spec import BaseConfig +from pydantic import ValidationError + +from dimos.protocol.service.spec import BaseConfig, Configurable from rich import print class Config(BaseConfig): @@ -26,7 +27,7 @@ print(myclass2.config) # we will raise an error for unspecified keys try: myclass3 = MyClass(something="else") -except TypeError as e: +except (TypeError, ValidationError) as e: print(f"Error: {e}") @@ -36,7 +37,11 @@ except TypeError as e: ``` Config(x=3, hello='world') Config(x=3, hello='override') -Error: Config.__init__() got an unexpected keyword argument 'something' +Error: 1 validation error for Config +something + Extra inputs are not permitted + For further information visit +https://errors.pydantic.dev/2.12/v/extra_forbidden ``` # Configurable Modules @@ -75,9 +80,42 @@ myModule = MyModule(frame_id="frame_id_override", device="CPU") ``` Config( rpc_transport=, + default_rpc_timeout=120.0, + rpc_timeouts={'build': 86400.0, 'start': 1200.0}, tf_transport=, frame_id_prefix=None, frame_id='frame_id_override', + g=GlobalConfig( + robot_ip=None, + robot_ips=None, + xarm7_ip=None, + xarm6_ip=None, + can_port=None, + simulation=False, + replay=False, + replay_dir='go2_sf_office', + new_memory=False, + viewer='rerun', + n_workers=2, + memory_limit='auto', + mujoco_camera_position=None, + mujoco_room=None, + mujoco_room_from_occupancy=None, + mujoco_global_costmap_from_occupancy=None, + mujoco_global_map_from_pointcloud=None, + mujoco_start_pos='-1.0, 1.0', + mujoco_steps_per_frame=7, + robot_model=None, + robot_width=0.3, + robot_rotation_diameter=0.6, + nerf_speed=1.0, + planner_robot_speed=None, + mcp_port=9990, + dtop=False, + obstacle_avoidance=True, + detection_model='moondream', + listen_host='127.0.0.1' + ), publish_interval=0, voxel_size=0.05, device='CPU' diff --git a/docs/usage/data_streams/README.md b/docs/usage/data_streams/README.md index 870c25fb34..85d2c92356 100644 --- a/docs/usage/data_streams/README.md +++ b/docs/usage/data_streams/README.md @@ -15,27 +15,39 @@ Dimos uses reactive streams (RxPY) to handle sensor data. This approach naturall ## Quick Example ```python -from reactivex import operators as ops +from __future__ import annotations + +import time + +import reactivex as rx + +from dimos.types.timestamped import Timestamped, align_timestamped from dimos.utils.reactive import backpressure -from dimos.types.timestamped import align_timestamped -from dimos.msgs.sensor_msgs.Image import sharpness_barrier - -# Camera at 30fps, lidar at 10Hz -camera_stream = camera.observable() -lidar_stream = lidar.observable() - -# Pipeline: filter blurry frames -> align with lidar -> handle slow consumers -processed = ( - camera_stream.pipe( - sharpness_barrier(10.0), # Keep sharpest frame per 100ms window (10Hz) - ) -) + +class SampleMsg(Timestamped): + def __init__(self, ts: float, label: str = "") -> None: + super().__init__(ts) + self.label = label + + +# Stand in for camera/lidar hardware streams: same pipeline shape, no device required. +camera_stream = rx.of(SampleMsg(0.00), SampleMsg(0.03), SampleMsg(0.06)) +lidar_stream = rx.of(SampleMsg(0.02), SampleMsg(0.11)) + +processed = camera_stream aligned = align_timestamped( - backpressure(processed), # Camera as primary - lidar_stream, # Lidar as secondary - match_tolerance=0.1, + backpressure(processed), + lidar_stream, + match_tolerance=0.15, ) +out: list[tuple[SampleMsg, ...]] = [] +aligned.subscribe(on_next=out.append) +time.sleep(0.25) +print("pairs", len(out)) +``` -aligned.subscribe(lambda pair: process_frame_with_pointcloud(*pair)) + +``` +pairs 3 ``` diff --git a/docs/usage/data_streams/quality_filter.md b/docs/usage/data_streams/quality_filter.md index 9b2075c059..e385067030 100644 --- a/docs/usage/data_streams/quality_filter.md +++ b/docs/usage/data_streams/quality_filter.md @@ -53,11 +53,11 @@ For camera streams, we provide `sharpness_barrier` which uses the image's sharpn Let's use real camera data from the Unitree Go2 robot to demonstrate. We use the [Sensor Storage & Replay](/docs/usage/sensor_streams/storage_replay.md) toolkit, which provides access to recorded robot data: ```python session=qb -from dimos.utils.testing import TimedSensorReplay from dimos.msgs.sensor_msgs.Image import Image, sharpness_barrier +from dimos.utils.testing.replay import TimedSensorReplay -# Load recorded Go2 camera frames -video_replay = TimedSensorReplay("go2_sf_office/video") +# Load recorded camera frames (bundled test data path) +video_replay = TimedSensorReplay("unitree_go2_bigoffice/video") # Use stream() with seek to skip blank frames, speed=10x to collect faster input_frames = video_replay.stream(seek=5.0, duration=1.4, speed=10.0).pipe( diff --git a/docs/usage/data_streams/reactivex.md b/docs/usage/data_streams/reactivex.md index 026eb292c4..b356714083 100644 --- a/docs/usage/data_streams/reactivex.md +++ b/docs/usage/data_streams/reactivex.md @@ -285,12 +285,17 @@ disposed ```python session=rx import time + +import reactivex as rx from dimos.core.module import Module + class MyModule(Module): - def start(self): + def start(self) -> None: + super().start() source = rx.interval(0.05) - self._disposables.add(source.subscribe(lambda x: print(f"got {x}"))) + self.register_disposable(source.subscribe(lambda x: print(f"got {x}"))) + module = MyModule() module.start() @@ -306,7 +311,6 @@ got 0 got 1 got 2 got 3 -got 4 ``` ## Creating Observables diff --git a/docs/usage/data_streams/temporal_alignment.md b/docs/usage/data_streams/temporal_alignment.md index da23d1b098..637863cb2f 100644 --- a/docs/usage/data_streams/temporal_alignment.md +++ b/docs/usage/data_streams/temporal_alignment.md @@ -38,7 +38,7 @@ You can read more about [sensor storage here](/docs/usage/data_streams/storage_r ```python session=align no-result from reactivex import Subject -from dimos.utils.testing import TimedSensorReplay +from dimos.utils.testing.replay import TimedSensorReplay from dimos.types.timestamped import Timestamped, align_timestamped from reactivex import operators as ops import reactivex as rx diff --git a/docs/usage/lcm.md b/docs/usage/lcm.md index d089cfcdd3..38d2520822 100644 --- a/docs/usage/lcm.md +++ b/docs/usage/lcm.md @@ -47,7 +47,7 @@ print(f"Decoded: x={decoded.x}, y={decoded.y}, z={decoded.z}") ``` -Encoded to 24 bytes: 000000000000f03f00000000000000400000000000000840 +Encoded to 32 bytes: ae7e5fba5eeca11e3ff000000000000040000000000000004008000000000000 Decoded: x=1.0, y=2.0, z=3.0 ``` @@ -60,7 +60,7 @@ Dimos subclasses the base LCM types to add Python-friendly features while preser - Conversions to numpy, quaternions, etc. ```python session=lcm_demo ansi=false -from dimos.msgs.geometry_msgs import Vector3 +from dimos.msgs.geometry_msgs.Vector3 import Vector3 # Rich constructors v1 = Vector3(1, 2, 3) @@ -84,7 +84,7 @@ v1 + v2 = (5.0, 7.0, 9.0) v1 dot v2 = 32.0 v1 x v2 = (-3.0, 6.0, -3.0) |v1| = 3.742 -LCM encoded: 24 bytes +LCM encoded: 32 bytes ``` ## PointCloud2 with Open3D @@ -92,12 +92,14 @@ LCM encoded: 24 bytes A more complex example is `PointCloud2`, which wraps Open3D point clouds while maintaining LCM binary compatibility: ```python session=lcm_demo ansi=false +import time + import numpy as np -from dimos.msgs.sensor_msgs import PointCloud2 +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 # Create from numpy points = np.random.rand(100, 3).astype(np.float32) -pc = PointCloud2.from_numpy(points, frame_id="camera") +pc = PointCloud2.from_numpy(points, frame_id="camera", timestamp=time.time()) print(f"PointCloud: {len(pc)} points, frame={pc.frame_id}") print(f"Center: {pc.center}") @@ -118,9 +120,9 @@ print(f"Decoded: {len(pc2)} points") ``` PointCloud: 100 points, frame=camera -Center: ↗ Vector (Vector([0.49166839, 0.50896413, 0.48393918])) +Center: ↘ Vector Vector([0.47497518 0.49878164 0.43788878]) Open3D type: PointCloud -LCM encoded: 1716 bytes +LCM encoded: 1725 bytes Decoded: 100 points ``` @@ -129,9 +131,8 @@ Decoded: 100 points Since LCM messages encode to bytes, you can use them over any transport: ```python session=lcm_demo ansi=false -from dimos.msgs.geometry_msgs import Vector3 -from dimos.protocol.pubsub.memory import Memory -from dimos.protocol.pubsub.shmpubsub import PickleSharedMemory +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.protocol.pubsub.impl.memory import Memory # Same message works with any transport msg = Vector3(1, 2, 3) @@ -152,8 +153,8 @@ print(f"Raw binary transport: decoded {decoded}") ``` -Memory transport: received ↗ Vector (Vector([1. 2. 3.])) -Raw binary transport: decoded ↗ Vector (Vector([1. 2. 3.])) +Memory transport: received ↘ Vector Vector([1. 2. 3.]) +Raw binary transport: decoded ↘ Vector Vector([1. 2. 3.]) ``` ## Available Message Types diff --git a/docs/usage/modules.md b/docs/usage/modules.md index 8c16fa8561..c0c043151b 100644 --- a/docs/usage/modules.md +++ b/docs/usage/modules.md @@ -17,31 +17,28 @@ Below is an example of a structure for controlling a robot. Black blocks represe > brew install graphviz # macOS > ``` -```python output=assets/go2_nav.svg -from dimos.core.introspection import to_svg -from dimos.robot.unitree_webrtc.unitree_go2_blueprints import nav -to_svg(nav, "assets/go2_nav.svg") -``` +```python skip output=assets/go2_nav.svg +from dimos.core.introspection.svg import to_svg +from dimos.robot.unitree.go2.blueprints.smart.unitree_go2 import unitree_go2 - -![output](assets/go2_nav.svg) +to_svg(unitree_go2, "assets/go2_nav.svg") +``` ## Camera Module Let's learn how to build stuff like the above, starting with a simple camera module. -```python session=camera_module_demo output=assets/camera_module.svg +```python skip session=camera_module_demo output=assets/camera_module.svg from dimos.hardware.sensors.camera.module import CameraModule -from dimos.core.introspection import to_svg +from dimos.core.introspection.svg import to_svg to_svg(CameraModule.module_info(), "assets/camera_module.svg") ``` - -![output](assets/camera_module.svg) - We can also print Module I/O quickly to the console via the `.io()` call. We will do this from now on. ```python session=camera_module_demo ansi=false +from dimos.hardware.sensors.camera.module import CameraModule + print(CameraModule.io()) ``` @@ -53,10 +50,13 @@ print(CameraModule.io()) ├─ color_image: Image ├─ camera_info: CameraInfo │ - ├─ RPC start() - ├─ RPC stop() - │ - ├─ Skill take_a_picture + ├─ RPC build() -> None + ├─ RPC get_skills() -> list + ├─ RPC set_module_ref(name: str, module_ref: RPCClient) -> None + ├─ RPC set_transport(stream_name: str, transport: Transport) -> bool + ├─ RPC start() -> None + ├─ RPC stop() -> None + ├─ RPC take_a_picture() -> Image ``` We can see that the camera module outputs two streams: @@ -70,7 +70,7 @@ It also exposes an agentic [skill](/docs/usage/blueprints.md#defining-skills) ca We can start this module and explore the output of its streams in real time (this will use your webcam). -```python session=camera_module_demo ansi=false +```python skip session=camera_module_demo ansi=false import time camera = CameraModule() @@ -84,7 +84,7 @@ time.sleep(0.5) camera.stop() ``` - + ``` Out color_image[Image] @ CameraModule Image(shape=(480, 640, 3), format=RGB, dtype=uint8, dev=cpu, ts=2025-12-31 15:54:16) @@ -111,7 +111,7 @@ print(Detection2DModule.io()) ``` - ├─ image: Image + ├─ color_image: Image ┌┴──────────────────┐ │ Detection2DModule │ └┬──────────────────┘ @@ -121,6 +121,9 @@ print(Detection2DModule.io()) ├─ detected_image_1: Image ├─ detected_image_2: Image │ + ├─ RPC build() -> None + ├─ RPC get_skills() -> list + ├─ RPC set_module_ref(name: str, module_ref: RPCClient) -> None ├─ RPC set_transport(stream_name: str, transport: Transport) -> bool ├─ RPC start() -> None ├─ RPC stop() -> None @@ -130,7 +133,7 @@ print(Detection2DModule.io()) Looks like the detector just needs an image input and outputs some sort of detection and annotation messages. Let's connect it to a camera. -```python ansi=false +```python skip ansi=false import time from dimos.perception.detection.module2D import Detection2DModule, Config from dimos.hardware.sensors.camera.module import CameraModule @@ -149,14 +152,6 @@ detector.stop() camera.stop() ``` - -``` -Detection(Person(1)) -Detection(Person(1)) -Detection(Person(1)) -Detection(Person(1)) -``` - ## Distributed Execution As we build module structures, we'll quickly want to utilize all cores on the machine (which Python doesn't allow as a single process) and potentially distribute modules across machines or even the internet. @@ -174,7 +169,7 @@ via `importlib.reload`, then redeploys it onto a fresh worker process while keeping its stream transports and reconnecting any other modules that held a reference to it. -```python +```python skip from dimos.core.coordination.module_coordinator import ModuleCoordinator from dimos.core.global_config import GlobalConfig from dimos.hardware.sensors.camera.module import CameraModule @@ -194,8 +189,8 @@ A blueprint is a predefined structure of interconnected modules. You can include A basic Unitree Go2 blueprint looks like what we saw before. -```python session=blueprints output=assets/go2_agentic.svg -from dimos.core.introspection import to_svg +```python skip session=blueprints output=assets/go2_agentic.svg +from dimos.core.introspection.svg import to_svg from dimos.robot.unitree_webrtc.unitree_go2_blueprints import agentic to_svg(agentic, "assets/go2_agentic.svg") diff --git a/docs/usage/native_modules.md b/docs/usage/native_modules.md index 512d17c05d..e4af928ee3 100644 --- a/docs/usage/native_modules.md +++ b/docs/usage/native_modules.md @@ -52,7 +52,7 @@ When `start()` is called, NativeModule: For the example above, the launched command would look like: -```sh +```sh skip ./build/my_lidar \ --pointcloud '/pointcloud#sensor_msgs.PointCloud2' \ --imu '/imu#sensor_msgs.Imu' \ diff --git a/docs/usage/python-api.md b/docs/usage/python-api.md index 43892c7df3..66f3cd9440 100644 --- a/docs/usage/python-api.md +++ b/docs/usage/python-api.md @@ -14,31 +14,31 @@ from dimos import Dimos app = Dimos(n_workers=8) -# Run a blueprint by name. -app.run("unitree-go2-agentic") +# Run a blueprint by name (requires a full install with that blueprint and its deps). +# app.run("unitree-go2-agentic") -# Call skills. -app.skills.relative_move(forward=2.0) - -# List all available skills. -print(app.skills) - -# Access a module directly. -app.ReplanningAStarPlanner +# After you have called run() with an agentic blueprint, skills are available, e.g.: +# app.skills.relative_move(forward=2.0) +# print(app.skills) -# Access a private variable. -print(app.ReplanningAStarPlanner._planner._safe_goal_clearance) +print("Dimos", type(app).__name__) - -# Add another module dynamically. +# Add another module dynamically (example import only): from dimos.robot.unitree.keyboard_teleop import KeyboardTeleop -app.run(KeyboardTeleop) -# Or start it by name. No need for importing. -app.run("keyboard-teleop") # This will say `KeyboardTeleop is already deployed` +print("KeyboardTeleop", KeyboardTeleop.__name__) -# Stop everything. -app.stop() +# Or start it by name after you have a coordinator running, e.g.: +# app.run("keyboard-teleop") +# app.stop() +``` + + +``` +Dimos Dimos +pygame 2.6.1 (SDL 2.28.4, Python 3.12.13) +Hello from the pygame community. https://www.pygame.org/contribute.html +KeyboardTeleop KeyboardTeleop ``` ## Remote mode @@ -49,7 +49,7 @@ Start a daemon first (via CLI or another script), then connect to it: dimos run unitree-go2-agentic ``` -```python +```python skip from dimos import Dimos app = Dimos.connect() @@ -63,7 +63,7 @@ app.stop() # closes the connection (does NOT stop the remote process) Connect to a specific instance: -```python +```python skip # By run ID (from `dimos status`) app = Dimos.connect(run_id="20260306-143022-unitree-go2") @@ -73,7 +73,7 @@ app = Dimos.connect(host="192.168.1.50", port=18861) `run()` and `restart()` also work against a daemon: -```python +```python skip app = Dimos.connect() app.run("keyboard-teleop") # add a module by registry name @@ -94,7 +94,7 @@ be picklable. In local mode, you can hot-restart a module: -```python +```python skip from dimos.agents.mcp.mcp_server import McpServer app.restart(McpServer) diff --git a/docs/usage/sensor_streams/README.md b/docs/usage/sensor_streams/README.md index 0bf61e98ef..77642a118c 100644 --- a/docs/usage/sensor_streams/README.md +++ b/docs/usage/sensor_streams/README.md @@ -15,27 +15,39 @@ Dimos uses reactive streams (RxPY) to handle sensor data. This approach naturall ## Quick Example ```python -from reactivex import operators as ops +from __future__ import annotations + +import time + +import reactivex as rx + +from dimos.types.timestamped import Timestamped, align_timestamped from dimos.utils.reactive import backpressure -from dimos.types.timestamped import align_timestamped -from dimos.msgs.sensor_msgs.Image import sharpness_barrier - -# Camera at 30fps, lidar at 10Hz -camera_stream = camera.observable() -lidar_stream = lidar.observable() - -# Pipeline: filter blurry frames -> align with lidar -> handle slow consumers -processed = ( - camera_stream.pipe( - sharpness_barrier(10.0), # Keep sharpest frame per 100ms window (10Hz) - ) -) + +class SampleMsg(Timestamped): + def __init__(self, ts: float, label: str = "") -> None: + super().__init__(ts) + self.label = label + + +# Stand in for camera/lidar hardware streams: same pipeline shape, no device required. +camera_stream = rx.of(SampleMsg(0.00), SampleMsg(0.03), SampleMsg(0.06)) +lidar_stream = rx.of(SampleMsg(0.02), SampleMsg(0.11)) + +processed = camera_stream aligned = align_timestamped( - backpressure(processed), # Camera as primary - lidar_stream, # Lidar as secondary - match_tolerance=0.1, + backpressure(processed), + lidar_stream, + match_tolerance=0.15, ) +out: list[tuple[SampleMsg, ...]] = [] +aligned.subscribe(on_next=out.append) +time.sleep(0.25) +print("pairs", len(out)) +``` -aligned.subscribe(lambda pair: process_frame_with_pointcloud(*pair)) + +``` +pairs 3 ``` diff --git a/docs/usage/sensor_streams/quality_filter.md b/docs/usage/sensor_streams/quality_filter.md index 9b2075c059..e385067030 100644 --- a/docs/usage/sensor_streams/quality_filter.md +++ b/docs/usage/sensor_streams/quality_filter.md @@ -53,11 +53,11 @@ For camera streams, we provide `sharpness_barrier` which uses the image's sharpn Let's use real camera data from the Unitree Go2 robot to demonstrate. We use the [Sensor Storage & Replay](/docs/usage/sensor_streams/storage_replay.md) toolkit, which provides access to recorded robot data: ```python session=qb -from dimos.utils.testing import TimedSensorReplay from dimos.msgs.sensor_msgs.Image import Image, sharpness_barrier +from dimos.utils.testing.replay import TimedSensorReplay -# Load recorded Go2 camera frames -video_replay = TimedSensorReplay("go2_sf_office/video") +# Load recorded camera frames (bundled test data path) +video_replay = TimedSensorReplay("unitree_go2_bigoffice/video") # Use stream() with seek to skip blank frames, speed=10x to collect faster input_frames = video_replay.stream(seek=5.0, duration=1.4, speed=10.0).pipe( diff --git a/docs/usage/sensor_streams/reactivex.md b/docs/usage/sensor_streams/reactivex.md index 026eb292c4..1498e46595 100644 --- a/docs/usage/sensor_streams/reactivex.md +++ b/docs/usage/sensor_streams/reactivex.md @@ -285,12 +285,17 @@ disposed ```python session=rx import time + +import reactivex as rx from dimos.core.module import Module + class MyModule(Module): - def start(self): + def start(self) -> None: + super().start() source = rx.interval(0.05) - self._disposables.add(source.subscribe(lambda x: print(f"got {x}"))) + self.register_disposable(source.subscribe(lambda x: print(f"got {x}"))) + module = MyModule() module.start() diff --git a/docs/usage/sensor_streams/temporal_alignment.md b/docs/usage/sensor_streams/temporal_alignment.md index a8af5769b8..a83b1fbfe2 100644 --- a/docs/usage/sensor_streams/temporal_alignment.md +++ b/docs/usage/sensor_streams/temporal_alignment.md @@ -38,7 +38,7 @@ You can read more about [sensor storage here](/docs/usage/sensor_streams/storage ```python session=align no-result from reactivex import Subject -from dimos.utils.testing import TimedSensorReplay +from dimos.utils.testing.replay import TimedSensorReplay from dimos.types.timestamped import Timestamped, align_timestamped from reactivex import operators as ops import reactivex as rx diff --git a/docs/usage/transforms.md b/docs/usage/transforms.md index 24d278365f..1c91a5e4b1 100644 --- a/docs/usage/transforms.md +++ b/docs/usage/transforms.md @@ -78,7 +78,9 @@ The `Transform` class at [`geometry_msgs/Transform.py`](/dimos/msgs/geometry_msg - `ts` - Timestamp for temporal lookups ```python -from dimos.msgs.geometry_msgs import Transform, Vector3, Quaternion +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.geometry_msgs.Vector3 import Vector3 # Camera 0.5m forward and 0.3m up from base, no rotation camera_transform = Transform( @@ -103,7 +105,9 @@ base_link -> camera_link Transforms can be composed and inverted: ```python -from dimos.msgs.geometry_msgs import Transform, Vector3, Quaternion +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.geometry_msgs.Vector3 import Vector3 # Create two transforms t1 = Transform( @@ -142,7 +146,9 @@ Inverse: camera_link -> base_link For integration with libraries like NumPy or OpenCV: ```python -from dimos.msgs.geometry_msgs import Transform, Vector3, Quaternion +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.geometry_msgs.Vector3 import Vector3 t = Transform( translation=Vector3(1.0, 2.0, 3.0), @@ -221,8 +227,10 @@ import reactivex as rx from reactivex import operators as ops from dimos.core.core import rpc from dimos.core.module import Module -from dimos.msgs.geometry_msgs import Quaternion, Transform, Vector3 -from dimos.core.module_coordinator import ModuleCoordinator +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.core.coordination.module_coordinator import ModuleCoordinator class RobotBaseModule(Module): """Publishes the robot's position in the world frame at 10Hz.""" @@ -241,9 +249,7 @@ class RobotBaseModule(Module): ) self.tf.publish(robot_pose) - self._disposables.add( - rx.interval(0.1).subscribe(publish_pose) - ) + self.register_disposable(rx.interval(0.1).subscribe(publish_pose)) class CameraModule(Module): """Publishes camera transforms at 10Hz.""" @@ -268,15 +274,15 @@ class CameraModule(Module): ) self.tf.publish(camera_mount, optical_frame) - self._disposables.add( - rx.interval(0.1).subscribe(publish_transforms) - ) + self.register_disposable(rx.interval(0.1).subscribe(publish_transforms)) class PerceptionModule(Module): """Receives transforms and performs lookups.""" + @rpc def start(self) -> None: + super().start() # This is just to init the transforms system. # Touching the property for the first time enables the system for this module. # Transform lookups normally happen in fast loops in IRL modules. @@ -313,7 +319,8 @@ if __name__ == "__main__": dimos.start_all_modules() - time.sleep(1.0) + # Give worker TF publishers a moment to populate the buffer before querying. + time.sleep(2.5) perception.lookup() @@ -323,14 +330,23 @@ if __name__ == "__main__": ``` -Initialized dimos local cluster with 3 workers, memory limit: auto -2025-12-29T12:47:01.433394Z [info ] Deployed module. [dimos/core/__init__.py] module=RobotBaseModule worker_id=1 -2025-12-29T12:47:01.603269Z [info ] Deployed module. [dimos/core/__init__.py] module=CameraModule worker_id=0 -2025-12-29T12:47:01.698970Z [info ] Deployed module. [dimos/core/__init__.py] module=PerceptionModule worker_id=2 +16:21:45.203 [inf][ation/worker_manager_python.py] Worker pool started. n_workers=2 +16:21:45.445 [inf][/coordination/python_worker.py] Deployed module. module=RobotBaseModule module_id=0 worker_id=0 +16:21:45.451 [inf][/coordination/python_worker.py] Deployed module. module=CameraModule module_id=1 worker_id=1 +16:21:45.452 [inf][/coordination/python_worker.py] Deployed module. module=PerceptionModule module_id=2 worker_id=0 +16:21:47.968 [inf][dination/module_coordinator.py] Stopping module... module=PerceptionModule +16:21:48.022 [inf][dination/module_coordinator.py] Module stopped. module=PerceptionModule +16:21:48.022 [inf][dination/module_coordinator.py] Stopping module... module=CameraModule +16:21:48.041 [inf][dination/module_coordinator.py] Module stopped. module=CameraModule +16:21:48.041 [inf][dination/module_coordinator.py] Stopping module... module=RobotBaseModule +16:21:48.062 [inf][dination/module_coordinator.py] Module stopped. module=RobotBaseModule +16:21:48.062 [inf][ation/worker_manager_python.py] Shutting down all workers... +16:21:48.062 [inf][/coordination/python_worker.py] Worker stopping module... module=CameraModule module_id=1 worker_id=1 +16:21:48.063 [inf][/coordination/python_worker.py] Worker module stopped. module=CameraModule module_id=1 worker_id=1 LCMTF(3 buffers): - TBuffer(world -> base_link, 10 msgs, 0.90s [2025-12-29 20:47:01 - 2025-12-29 20:47:02]) - TBuffer(base_link -> camera_link, 9 msgs, 0.80s [2025-12-29 20:47:01 - 2025-12-29 20:47:02]) - TBuffer(camera_link -> camera_optical, 9 msgs, 0.80s [2025-12-29 20:47:01 - 2025-12-29 20:47:02]) + TBuffer(base_link -> camera_link, 24 msgs, 2.37s [2026-04-21 01:21:45 - 2026-04-21 01:21:47]) + TBuffer(camera_link -> camera_optical, 24 msgs, 2.37s [2026-04-21 01:21:45 - 2026-04-21 01:21:47]) + TBuffer(world -> base_link, 24 msgs, 2.37s [2026-04-21 01:21:45 - 2026-04-21 01:21:47]) Direct: robot is at (2.5, 3.0)m in world Chained: world -> camera_optical @@ -413,11 +429,14 @@ text "CameraModule" italic at ((CL.x + CO.x)/2, CL.s.y - 0.25in) `self.tf` on module is a transform buffer. This is a standalone class that maintains a temporal buffer of transforms (default 10 seconds) allowing queries at past timestamps, you can use it directly: ```python -from dimos.protocol.tf import TF -from dimos.msgs.geometry_msgs import Transform, Vector3, Quaternion import time -tf = TF(autostart=False) +from dimos.msgs.geometry_msgs.Quaternion import Quaternion +from dimos.msgs.geometry_msgs.Transform import Transform +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.protocol.tf.tf import MultiTBuffer + +tf = MultiTBuffer() # Simulate transforms at different times for i in range(5): diff --git a/docs/usage/transports/index.md b/docs/usage/transports/index.md index 09ccb484ed..2f580a2676 100644 --- a/docs/usage/transports/index.md +++ b/docs/usage/transports/index.md @@ -113,18 +113,43 @@ Each **stream** on a module can use a different transport. Set `.transport` on t ```python ansi=false import time +import numpy as np +import reactivex as rx +from reactivex import operators as ops + +from dimos.core.coordination.module_coordinator import ModuleCoordinator +from dimos.core.core import rpc from dimos.core.module import Module -from dimos.core.stream import In +from dimos.core.stream import In, Out from dimos.core.transport import LCMTransport -from dimos.hardware.sensors.camera.module import CameraModule -from dimos.msgs.sensor_msgs import Image -from dimos.core.module_coordinator import ModuleCoordinator +from dimos.msgs.sensor_msgs.Image import Image + + +class SyntheticCamera(Module): + """Tiny publisher so this example runs without a webcam.""" + + color_image: Out[Image] + + @rpc + def start(self) -> None: + super().start() + + def emit(_): + img = Image.from_numpy( + np.zeros((4, 4, 3), dtype=np.uint8), + frame_id="camera", + ts=time.time(), + ) + self.color_image.publish(img) + + self.register_disposable(rx.interval(0.3).pipe(ops.take(8)).subscribe(emit)) class ImageListener(Module): image: In[Image] - def start(self): + @rpc + def start(self) -> None: super().start() self.image.subscribe(lambda img: print(f"Received: {img.shape}")) @@ -134,7 +159,7 @@ if __name__ == "__main__": dimos = ModuleCoordinator() dimos.start() - camera = dimos.deploy(CameraModule, frequency=2.0) + camera = dimos.deploy(SyntheticCamera) listener = dimos.deploy(ImageListener) # Choose a transport for the stream (example: LCM typed channel) @@ -145,10 +170,35 @@ if __name__ == "__main__": dimos.start_all_modules() - time.sleep(2) + time.sleep(2.5) dimos.stop() ``` + +``` +16:20:36.838 [inf][ation/worker_manager_python.py] Worker pool started. n_workers=2 +16:20:37.209 [inf][/coordination/python_worker.py] Deployed module. module=SyntheticCamera module_id=0 worker_id=0 +16:20:37.216 [inf][/coordination/python_worker.py] Deployed module. module=ImageListener module_id=1 worker_id=1 +16:20:39.723 [inf][dination/module_coordinator.py] Stopping module... module=ImageListener +16:20:39.728 [inf][dination/module_coordinator.py] Module stopped. module=ImageListener +16:20:39.729 [inf][dination/module_coordinator.py] Stopping module... module=SyntheticCamera +16:20:39.780 [inf][dination/module_coordinator.py] Module stopped. module=SyntheticCamera +16:20:39.781 [inf][ation/worker_manager_python.py] Shutting down all workers... +Received: (4, 4, 3) +Received: (4, 4, 3) +Received: (4, 4, 3) +Received: (4, 4, 3) +Received: (4, 4, 3) +Received: (4, 4, 3) +Received: (4, 4, 3) +Received: (4, 4, 3) +16:20:39.782 [inf][/coordination/python_worker.py] Worker stopping module... module=ImageListener module_id=1 worker_id=1 +16:20:39.784 [inf][/coordination/python_worker.py] Worker module stopped. module=ImageListener module_id=1 worker_id=1 +16:20:39.820 [inf][/coordination/python_worker.py] Worker stopping module... module=SyntheticCamera module_id=0 worker_id=0 +16:20:39.821 [inf][/coordination/python_worker.py] Worker module stopped. module=SyntheticCamera module_id=0 worker_id=0 +16:20:39.843 [inf][ation/worker_manager_python.py] All workers shut down +``` + ``` @@ -219,7 +269,7 @@ print(inspect.getsource(PubSub.subscribe)) ``` -```python +``` @abstractmethod def publish(self, topic: TopicT, message: MsgT) -> None: """Publish a message to a topic.""" @@ -241,8 +291,8 @@ LCM is UDP multicast. It’s very fast on a robot LAN, but it’s **best-effort* For local emission it autoconfigures system in a way in which it's more robust and faster then other more common protocols like ROS, DDS ```python -from dimos.protocol.pubsub.lcmpubsub import LCM, Topic -from dimos.msgs.geometry_msgs import Vector3 +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.protocol.pubsub.impl.lcmpubsub import LCM, Topic lcm = LCM() lcm.start() @@ -270,7 +320,7 @@ Received velocity: x=1.0, y=0.0, z=0.5 Shared memory is highest performance, but only works on the **same machine**. ```python -from dimos.protocol.pubsub.shmpubsub import PickleSharedMemory +from dimos.protocol.pubsub.impl.shmpubsub import PickleSharedMemory shm = PickleSharedMemory(prefer="cpu") shm.start() @@ -295,7 +345,7 @@ Received: [{'data': [1, 2, 3]}] For network communication, DDS uses the Data Distribution Service (DDS) protocol: -```python session=dds_demo ansi=false +```python skip session=dds_demo ansi=false from dataclasses import dataclass from cyclonedds.idl import IdlStruct @@ -321,11 +371,6 @@ print(f"Received: {received}") dds.stop() ``` - -``` -Received: [SensorReading(value=22.5)] -``` - --- ## A minimal transport: `Memory` @@ -333,7 +378,7 @@ Received: [SensorReading(value=22.5)] The simplest toy backend is `Memory` (single process). Start from there when implementing a new pubsub backend. ```python -from dimos.protocol.pubsub.memory import Memory +from dimos.protocol.pubsub.impl.memory import Memory bus = Memory() received = [] @@ -365,7 +410,7 @@ See [`pubsub/impl/memory.py`](/dimos/protocol/pubsub/impl/memory.py) for the com Transports often need to serialize messages before sending and deserialize after receiving. -`PubSubEncoderMixin` at [`pubsub/spec.py`](/dimos/protocol/pubsub/spec.py#L95) provides a clean way to add encoding/decoding to any pubsub implementation. +`PubSubEncoderMixin` at [`pubsub/encoders.py`](/dimos/protocol/pubsub/encoders.py) provides a clean way to add encoding/decoding to any pubsub implementation. ### Available mixins @@ -380,9 +425,11 @@ Transports often need to serialize messages before sending and deserialize after ### Creating a custom mixin ```python session=jsonencoder no-result -from dimos.protocol.pubsub.spec import PubSubEncoderMixin import json +from dimos.protocol.pubsub.encoders import PubSubEncoderMixin + + class JsonEncoderMixin(PubSubEncoderMixin[str, dict, bytes]): def encode(self, msg: dict, topic: str) -> bytes: return json.dumps(msg).encode("utf-8") @@ -394,7 +441,8 @@ class JsonEncoderMixin(PubSubEncoderMixin[str, dict, bytes]): Combine with a pubsub implementation via multiple inheritance: ```python session=jsonencoder no-result -from dimos.protocol.pubsub.memory import Memory +from dimos.protocol.pubsub.impl.memory import Memory + class MyJsonPubSub(JsonEncoderMixin, Memory): pass @@ -403,7 +451,9 @@ class MyJsonPubSub(JsonEncoderMixin, Memory): Swap serialization by changing the mixin: ```python session=jsonencoder no-result -from dimos.protocol.pubsub.spec import PickleEncoderMixin +from dimos.protocol.pubsub.encoders import PickleEncoderMixin +from dimos.protocol.pubsub.impl.memory import Memory + class MyPicklePubSub(PickleEncoderMixin, Memory): pass diff --git a/docs/usage/visualization.md b/docs/usage/visualization.md index 57ad460354..5c473870fa 100644 --- a/docs/usage/visualization.md +++ b/docs/usage/visualization.md @@ -66,19 +66,32 @@ VIEWER=foxglove dimos run unitree-go2 To enable rerun within your own blueprint simply include `RerunBridgeModule`: ```python -from dimos.visualization.rerun.bridge import RerunBridgeModule +from dimos.core.coordination.blueprints import autoconnect from dimos.hardware.sensors.camera.module import CameraModule -from dimos.protocol.pubsub.impl.lcmpubsub import LCM +from dimos.visualization.rerun.bridge import RerunBridgeModule camera_demo = autoconnect( CameraModule.blueprint(), RerunBridgeModule.blueprint( - viewer_mode="native", # native (desktop), web (browser), none (headless) + viewer_mode="none", # native (desktop), web (browser), none (headless) ), ) +print("Blueprint modules:", [b.module.__name__ for b in camera_demo.blueprints]) +``` + + +``` +Blueprint modules: ['CameraModule', 'RerunBridgeModule'] +``` + +Run the stack locally (this blocks until you stop the process): + +```python skip +from dimos.core.coordination.module_coordinator import ModuleCoordinator + if __name__ == "__main__": - camera_demo.build().loop() + ModuleCoordinator.build(camera_demo).loop() ``` Every LCM stream, such as `color_image` (output by CameraModule), that uses a data type (like `Image`) that has a `.to_rerun` method will get rendered (`rr.log`) using the LCM topic as the rerun entity path. In other words: to render something, simply log it to a stream and it will automatically be available in rerun. @@ -98,7 +111,7 @@ This happens on lower-end hardware (NUC, older laptops) with large maps. Edit [`dimos/robot/unitree/go2/blueprints/smart/unitree_go2.py`](/dimos/robot/unitree/go2/blueprints/smart/unitree_go2.py): -```python +```python skip # Before (high detail, slower on large maps) voxel_mapper(voxel_size=0.05), # 5cm voxels From e380bbcf509e9b8204a1f01180ea5030bbc4b9fd Mon Sep 17 00:00:00 2001 From: bogwi Date: Tue, 21 Apr 2026 08:37:21 +0900 Subject: [PATCH 03/11] Align doc-codeblocks workflow pins with repo and fix docs for pre-commit checks --- .github/workflows/doc-codeblocks.yml | 10 +++++----- docs/usage/configuration.md | 4 ++-- docs/usage/transports/index.md | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml index 92b7dae707..d4591fcd65 100644 --- a/.github/workflows/doc-codeblocks.yml +++ b/.github/workflows/doc-codeblocks.yml @@ -35,19 +35,19 @@ jobs: run: | sudo chown -R $USER:$USER ${{ github.workspace }} || true - - uses: actions/checkout@v6 + - uses: actions/checkout@v5 with: fetch-depth: 0 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v5 with: python-version: '3.12' - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v4 with: enable-cache: true - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v4 with: node-version: '22' @@ -61,6 +61,6 @@ jobs: run: ./bin/run-doc-codeblocks --ci - name: Commit doc updates - uses: stefanzweifel/git-auto-commit-action@v7 + uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: "CI: execute documentation code blocks" diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index 2b6abae030..8e0fc44b5a 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -39,8 +39,8 @@ Config(x=3, hello='world') Config(x=3, hello='override') Error: 1 validation error for Config something - Extra inputs are not permitted - For further information visit + Extra inputs are not permitted + For further information visit https://errors.pydantic.dev/2.12/v/extra_forbidden ``` diff --git a/docs/usage/transports/index.md b/docs/usage/transports/index.md index 2f580a2676..310970e997 100644 --- a/docs/usage/transports/index.md +++ b/docs/usage/transports/index.md @@ -410,7 +410,7 @@ See [`pubsub/impl/memory.py`](/dimos/protocol/pubsub/impl/memory.py) for the com Transports often need to serialize messages before sending and deserialize after receiving. -`PubSubEncoderMixin` at [`pubsub/encoders.py`](/dimos/protocol/pubsub/encoders.py) provides a clean way to add encoding/decoding to any pubsub implementation. +`PubSubEncoderMixin` at [`pubsub/encoders.py`](/dimos/protocol/pubsub/encoders.py#L39) provides a clean way to add encoding/decoding to any pubsub implementation. ### Available mixins From 4fd987f31eb7ad30cc70f8c96c3d80e0d3b513cc Mon Sep 17 00:00:00 2001 From: bogwi Date: Tue, 21 Apr 2026 19:07:53 +0900 Subject: [PATCH 04/11] Doc codeblocks CI uses ros-dev image; skip CUDA or flaky md-babel blocks --- .github/workflows/doc-codeblocks.yml | 31 ++++++++++--------- bin/run-doc-codeblocks | 2 +- docs/agents/docs/codeblocks.md | 2 ++ .../manipulation/adding_a_custom_arm.md | 6 ++-- docs/usage/modules.md | 2 +- docs/usage/transforms.md | 2 +- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml index d4591fcd65..f6e26e6017 100644 --- a/.github/workflows/doc-codeblocks.yml +++ b/.github/workflows/doc-codeblocks.yml @@ -23,29 +23,27 @@ on: permissions: contents: write - packages: write + packages: read pull-requests: read jobs: md-babel: if: github.event_name != 'pull_request' || github.event.pull_request.draft == false - runs-on: self-hosted - steps: - - name: Fix permissions - run: | - sudo chown -R $USER:$USER ${{ github.workspace }} || true + timeout-minutes: 60 + runs-on: [self-hosted, Linux] + container: + image: ghcr.io/dimensionalos/ros-dev:dev + steps: - uses: actions/checkout@v5 with: fetch-depth: 0 + clean: false - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true + - name: Fix permissions + run: | + git config --global --add safe.directory '*' + git clean -ffdx -e .venv - uses: actions/setup-node@v4 with: @@ -53,10 +51,13 @@ jobs: - name: Install Python dependencies run: | - uv sync --extra dev --frozen --no-extra dds - # Matplotlib and httpx are used by docs but are not always pulled by dev-only sync. + uv sync --all-extras --no-extra dds --frozen uv pip install matplotlib httpx + - name: Remove pydrake stubs + run: | + find .venv/lib/*/site-packages/pydrake -name '*.pyi' -delete 2>/dev/null || true + - name: Execute documentation code blocks run: ./bin/run-doc-codeblocks --ci diff --git a/bin/run-doc-codeblocks b/bin/run-doc-codeblocks index 0549cd2f2e..b14b9de52e 100755 --- a/bin/run-doc-codeblocks +++ b/bin/run-doc-codeblocks @@ -65,7 +65,7 @@ resolve_md_babel() { elif command -v md-babel-py &>/dev/null; then MB=(md-babel-py) else - echo "Error: md-babel-py not found. Install dev deps: uv sync --extra dev" >&2 + echo "Error: md-babel-py not found. Install project deps (e.g. uv sync --extra dev or uv sync --all-extras)." >&2 exit 1 fi } diff --git a/docs/agents/docs/codeblocks.md b/docs/agents/docs/codeblocks.md index 323f1c0c50..2d9cf1bef1 100644 --- a/docs/agents/docs/codeblocks.md +++ b/docs/agents/docs/codeblocks.md @@ -30,6 +30,8 @@ Add flags after the language identifier: | `skip` | Don't execute this block | | `expected-error` | Block is expected to fail | +Use `skip` when a block would pull in **CUDA / GPU-only** stacks (for example perception models, `VoxelGridMapper` defaults, or imports that load torch with GPU expectations), or when it is **flaky in CI** (multi-module coordinators, timing-sensitive workers, pytest-style snippets that are not meant to run as a single script). Prefer `expected-error` only when the block is supposed to fail and you want to assert that failure. + ## Examples # md-babel-py diff --git a/docs/capabilities/manipulation/adding_a_custom_arm.md b/docs/capabilities/manipulation/adding_a_custom_arm.md index ad625e5942..0fd27b4e46 100644 --- a/docs/capabilities/manipulation/adding_a_custom_arm.md +++ b/docs/capabilities/manipulation/adding_a_custom_arm.md @@ -393,7 +393,7 @@ This means **no manual registration is needed** — just having the `register()` You can verify discovery works: -```python +```python skip from dimos.hardware.manipulators.registry import adapter_registry print(adapter_registry.available()) # Should include "yourarm" ``` @@ -420,7 +420,7 @@ dimos/robot/ Create `dimos/robot/yourarm/blueprints.py` with your coordinator and (optionally) planning blueprints: -```python +```python skip """Blueprints for YourArm robot. Usage: @@ -619,7 +619,7 @@ adapter = adapter_registry.create("yourarm", address="192.168.1.100", dof=6) You can test coordinator logic without hardware by using `unittest.mock`: -```python +```python skip import pytest from unittest.mock import MagicMock from dimos.hardware.manipulators.spec import ManipulatorAdapter diff --git a/docs/usage/modules.md b/docs/usage/modules.md index c0c043151b..3d7814bad7 100644 --- a/docs/usage/modules.md +++ b/docs/usage/modules.md @@ -104,7 +104,7 @@ Image(shape=(480, 640, 3), format=RGB, dtype=uint8, dev=cpu, ts=2025-12-31 15:54 Let's load a standard 2D detector module and hook it up to a camera. -```python ansi=false session=detection_module +```python skip ansi=false session=detection_module from dimos.perception.detection.module2D import Detection2DModule, Config print(Detection2DModule.io()) ``` diff --git a/docs/usage/transforms.md b/docs/usage/transforms.md index 1c91a5e4b1..34c736861e 100644 --- a/docs/usage/transforms.md +++ b/docs/usage/transforms.md @@ -221,7 +221,7 @@ This example demonstrates how multiple modules publish and receive transforms. T 2. **CameraModule** - Publishes `base_link -> camera_link` (camera mounting position) and `camera_link -> camera_optical` (optical frame convention) 3. **PerceptionModule** - Looks up transforms between any frames -```python ansi=false +```python skip ansi=false import time import reactivex as rx from reactivex import operators as ops From fbd9c978133f3232f9903a61bf98f64110078b3c Mon Sep 17 00:00:00 2001 From: bogwi Date: Tue, 21 Apr 2026 19:38:10 +0900 Subject: [PATCH 05/11] Skip LFS and replay-heavy doc codeblocks for md-babel CI --- docs/development/large_file_management.md | 2 +- docs/usage/data_streams/quality_filter.md | 18 +++++++++--------- docs/usage/data_streams/temporal_alignment.md | 16 ++++++++-------- docs/usage/sensor_streams/quality_filter.md | 18 +++++++++--------- .../usage/sensor_streams/temporal_alignment.md | 16 ++++++++-------- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/development/large_file_management.md b/docs/development/large_file_management.md index f28086bda6..a5053d3d37 100644 --- a/docs/development/large_file_management.md +++ b/docs/development/large_file_management.md @@ -73,7 +73,7 @@ Image shape: (771, 1024, 3) ### Loading Model Checkpoints -```python +```python skip from dimos.utils.data import get_data model_dir = get_data("models_yolo") diff --git a/docs/usage/data_streams/quality_filter.md b/docs/usage/data_streams/quality_filter.md index e385067030..eeadc6412a 100644 --- a/docs/usage/data_streams/quality_filter.md +++ b/docs/usage/data_streams/quality_filter.md @@ -52,7 +52,7 @@ For camera streams, we provide `sharpness_barrier` which uses the image's sharpn Let's use real camera data from the Unitree Go2 robot to demonstrate. We use the [Sensor Storage & Replay](/docs/usage/sensor_streams/storage_replay.md) toolkit, which provides access to recorded robot data: -```python session=qb +```python skip session=qb from dimos.msgs.sensor_msgs.Image import Image, sharpness_barrier from dimos.utils.testing.replay import TimedSensorReplay @@ -93,7 +93,7 @@ Sharpness scores: Using `sharpness_barrier` to select the sharpest frames: -```python session=qb +```python skip session=qb # Create a stream from the recorded frames sharp_frames = video_replay.stream(seek=5.0, duration=1.5, speed=1.0).pipe( @@ -116,7 +116,7 @@ Output: 3 frame(s) (selected sharpest per window)
Visualization helpers -```python session=qb fold no-result +```python skip session=qb fold no-result import matplotlib import matplotlib.pyplot as plt import math @@ -163,14 +163,14 @@ def plot_sharpness(frames, selected, path): Visualizing which frames were selected (green border = selected as sharpest in window): -```python session=qb output=assets/frame_mosaic.jpg +```python skip session=qb output=assets/frame_mosaic.jpg plot_mosaic(input_frames, sharp_frames, '{output}') ``` ![output](assets/frame_mosaic.jpg) -```python session=qb output=assets/sharpness_graph.svg +```python skip session=qb output=assets/sharpness_graph.svg plot_sharpness(input_frames, sharp_frames, '{output}') ``` @@ -179,7 +179,7 @@ plot_sharpness(input_frames, sharp_frames, '{output}') Let's request a higher frequency. -```python session=qb +```python skip session=qb sharp_frames = video_replay.stream(seek=5.0, duration=1.5, speed=1.0).pipe( sharpness_barrier(4.0), ops.to_list() @@ -200,7 +200,7 @@ Output: 6 frame(s) (selected sharpest per window) Frame 5: 0.329 ``` -```python session=qb output=assets/frame_mosaic2.jpg +```python skip session=qb output=assets/frame_mosaic2.jpg plot_mosaic(input_frames, sharp_frames, '{output}') ``` @@ -208,7 +208,7 @@ plot_mosaic(input_frames, sharp_frames, '{output}') ![output](assets/frame_mosaic2.jpg) -```python session=qb output=assets/sharpness_graph2.svg +```python skip session=qb output=assets/sharpness_graph2.svg plot_sharpness(input_frames, sharp_frames, '{output}') ``` @@ -245,7 +245,7 @@ The sharpness score (0.0 to 1.0) is computed using Sobel edge detection: from [`Image.py`](/dimos/msgs/sensor_msgs/Image.py) -```python session=qb +```python skip session=qb import cv2 # Get a frame and show the calculation diff --git a/docs/usage/data_streams/temporal_alignment.md b/docs/usage/data_streams/temporal_alignment.md index 637863cb2f..18f28b2e3d 100644 --- a/docs/usage/data_streams/temporal_alignment.md +++ b/docs/usage/data_streams/temporal_alignment.md @@ -36,7 +36,7 @@ Below we set up replay of real camera and lidar data from the Unitree Go2 robot. You can read more about [sensor storage here](/docs/usage/data_streams/storage_replay.md) and [LFS data storage here](/docs/development/large_file_management.md). -```python session=align no-result +```python skip session=align no-result from reactivex import Subject from dimos.utils.testing.replay import TimedSensorReplay from dimos.types.timestamped import Timestamped, align_timestamped @@ -74,7 +74,7 @@ Streams would normally come from an actual robot into your module via `In` input Assume we have them. Let's align them. -```python session=align +```python skip session=align # Align video (primary) with lidar (secondary) # match_tolerance: max time difference for a match (seconds) # buffer_size: how long to keep messages waiting for matches (seconds) @@ -106,7 +106,7 @@ First matched pair: Δ11.3ms
Visualization helper -```python session=align fold no-result +```python skip session=align fold no-result import matplotlib import matplotlib.pyplot as plt @@ -159,7 +159,7 @@ def plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, path):
-```python session=align output=assets/alignment_timeline.png +```python skip session=align output=assets/alignment_timeline.png plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') ``` @@ -168,7 +168,7 @@ plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') If we loosen up our match tolerance, we might get multiple pairs matching the same lidar frame. -```python session=align +```python skip session=align aligned_pairs = align_timestamped( video_stream, lidar_stream, @@ -187,7 +187,7 @@ Aligned pairs: 23 out of 58 video frames ``` -```python session=align output=assets/alignment_timeline2.png +```python skip session=align output=assets/alignment_timeline2.png plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') ``` @@ -198,7 +198,7 @@ plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') More on [quality filtering here](/docs/usage/data_streams/quality_filter.md). -```python session=align +```python skip session=align from dimos.msgs.sensor_msgs.Image import Image, sharpness_barrier # Lists to collect items as they flow through streams @@ -232,7 +232,7 @@ Video: 6 frames, Lidar: 15 scans Aligned pairs: 1 out of 6 video frames ``` -```python session=align output=assets/alignment_timeline3.png +```python skip session=align output=assets/alignment_timeline3.png plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') ``` diff --git a/docs/usage/sensor_streams/quality_filter.md b/docs/usage/sensor_streams/quality_filter.md index e385067030..eeadc6412a 100644 --- a/docs/usage/sensor_streams/quality_filter.md +++ b/docs/usage/sensor_streams/quality_filter.md @@ -52,7 +52,7 @@ For camera streams, we provide `sharpness_barrier` which uses the image's sharpn Let's use real camera data from the Unitree Go2 robot to demonstrate. We use the [Sensor Storage & Replay](/docs/usage/sensor_streams/storage_replay.md) toolkit, which provides access to recorded robot data: -```python session=qb +```python skip session=qb from dimos.msgs.sensor_msgs.Image import Image, sharpness_barrier from dimos.utils.testing.replay import TimedSensorReplay @@ -93,7 +93,7 @@ Sharpness scores: Using `sharpness_barrier` to select the sharpest frames: -```python session=qb +```python skip session=qb # Create a stream from the recorded frames sharp_frames = video_replay.stream(seek=5.0, duration=1.5, speed=1.0).pipe( @@ -116,7 +116,7 @@ Output: 3 frame(s) (selected sharpest per window)
Visualization helpers -```python session=qb fold no-result +```python skip session=qb fold no-result import matplotlib import matplotlib.pyplot as plt import math @@ -163,14 +163,14 @@ def plot_sharpness(frames, selected, path): Visualizing which frames were selected (green border = selected as sharpest in window): -```python session=qb output=assets/frame_mosaic.jpg +```python skip session=qb output=assets/frame_mosaic.jpg plot_mosaic(input_frames, sharp_frames, '{output}') ``` ![output](assets/frame_mosaic.jpg) -```python session=qb output=assets/sharpness_graph.svg +```python skip session=qb output=assets/sharpness_graph.svg plot_sharpness(input_frames, sharp_frames, '{output}') ``` @@ -179,7 +179,7 @@ plot_sharpness(input_frames, sharp_frames, '{output}') Let's request a higher frequency. -```python session=qb +```python skip session=qb sharp_frames = video_replay.stream(seek=5.0, duration=1.5, speed=1.0).pipe( sharpness_barrier(4.0), ops.to_list() @@ -200,7 +200,7 @@ Output: 6 frame(s) (selected sharpest per window) Frame 5: 0.329 ``` -```python session=qb output=assets/frame_mosaic2.jpg +```python skip session=qb output=assets/frame_mosaic2.jpg plot_mosaic(input_frames, sharp_frames, '{output}') ``` @@ -208,7 +208,7 @@ plot_mosaic(input_frames, sharp_frames, '{output}') ![output](assets/frame_mosaic2.jpg) -```python session=qb output=assets/sharpness_graph2.svg +```python skip session=qb output=assets/sharpness_graph2.svg plot_sharpness(input_frames, sharp_frames, '{output}') ``` @@ -245,7 +245,7 @@ The sharpness score (0.0 to 1.0) is computed using Sobel edge detection: from [`Image.py`](/dimos/msgs/sensor_msgs/Image.py) -```python session=qb +```python skip session=qb import cv2 # Get a frame and show the calculation diff --git a/docs/usage/sensor_streams/temporal_alignment.md b/docs/usage/sensor_streams/temporal_alignment.md index a83b1fbfe2..7a7faa24a0 100644 --- a/docs/usage/sensor_streams/temporal_alignment.md +++ b/docs/usage/sensor_streams/temporal_alignment.md @@ -36,7 +36,7 @@ Below we set up replay of real camera and lidar data from the Unitree Go2 robot. You can read more about [sensor storage here](/docs/usage/sensor_streams/storage_replay.md) and [LFS data storage here](/docs/development/large_file_management.md). -```python session=align no-result +```python skip session=align no-result from reactivex import Subject from dimos.utils.testing.replay import TimedSensorReplay from dimos.types.timestamped import Timestamped, align_timestamped @@ -74,7 +74,7 @@ Streams would normally come from an actual robot into your module via `In` input Assume we have them. Let's align them. -```python session=align +```python skip session=align # Align video (primary) with lidar (secondary) # match_tolerance: max time difference for a match (seconds) # buffer_size: how long to keep messages waiting for matches (seconds) @@ -106,7 +106,7 @@ First matched pair: Δ11.3ms
Visualization helper -```python session=align fold no-result +```python skip session=align fold no-result import matplotlib import matplotlib.pyplot as plt @@ -159,7 +159,7 @@ def plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, path):
-```python session=align output=assets/alignment_timeline.png +```python skip session=align output=assets/alignment_timeline.png plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') ``` @@ -168,7 +168,7 @@ plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') If we loosen up our match tolerance, we might get multiple pairs matching the same lidar frame. -```python session=align +```python skip session=align aligned_pairs = align_timestamped( video_stream, lidar_stream, @@ -187,7 +187,7 @@ Aligned pairs: 23 out of 58 video frames ``` -```python session=align output=assets/alignment_timeline2.png +```python skip session=align output=assets/alignment_timeline2.png plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') ``` @@ -198,7 +198,7 @@ plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') More on [quality filtering here](/docs/usage/sensor_streams/quality_filter.md). -```python session=align +```python skip session=align from dimos.msgs.sensor_msgs.Image import Image, sharpness_barrier # Lists to collect items as they flow through streams @@ -232,7 +232,7 @@ Video: 6 frames, Lidar: 15 scans Aligned pairs: 1 out of 6 video frames ``` -```python session=align output=assets/alignment_timeline3.png +```python skip session=align output=assets/alignment_timeline3.png plot_alignment_timeline(video_frames, lidar_scans, aligned_pairs, '{output}') ``` From 5b3d926e6f1a76dac8816047b07565b2524d1224 Mon Sep 17 00:00:00 2001 From: bogwi Date: Tue, 21 Apr 2026 20:01:09 +0900 Subject: [PATCH 06/11] Doc codeblocks workflow: checkout PR head for git-auto-commit --- .github/workflows/doc-codeblocks.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml index f6e26e6017..1ccf4e96ce 100644 --- a/.github/workflows/doc-codeblocks.yml +++ b/.github/workflows/doc-codeblocks.yml @@ -39,12 +39,18 @@ jobs: with: fetch-depth: 0 clean: false + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ github.token }} - name: Fix permissions run: | git config --global --add safe.directory '*' git clean -ffdx -e .venv + - name: Branch ref for git-auto-commit + run: git checkout -B "${{ github.head_ref }}" + - uses: actions/setup-node@v4 with: node-version: '22' From addb6c964db21711d68301f27cf30bb78562f108 Mon Sep 17 00:00:00 2001 From: bogwi Date: Tue, 21 Apr 2026 20:26:31 +0900 Subject: [PATCH 07/11] Skip heavy get_data replay and pointcloud doc blocks for CI timeouts --- docs/development/large_file_management.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/development/large_file_management.md b/docs/development/large_file_management.md index a5053d3d37..fedd4d095e 100644 --- a/docs/development/large_file_management.md +++ b/docs/development/large_file_management.md @@ -88,7 +88,7 @@ Checkpoint: yolo11n.pt (5482KB) ### Loading Recorded Data for Replay -```python +```python skip from dimos.utils.data import get_data from dimos.utils.testing.replay import TimedSensorReplay @@ -112,7 +112,7 @@ Replay ### Loading Point Clouds -```python +```python skip from dimos.utils.data import get_data from dimos.mapping.pointclouds.util import read_pointcloud From 0707499d651452b350ce21d1dda3792079ab918c Mon Sep 17 00:00:00 2001 From: bogwi Date: Fri, 24 Apr 2026 23:40:03 +0900 Subject: [PATCH 08/11] bump md-babel-py==1.1.4 to support new --execution-timeout cli flag; increase the timeout to 120sec for slow runs. --- .github/workflows/doc-codeblocks.yml | 7 ++++++- bin/run-doc-codeblocks | 16 +++++++--------- docs/agents/docs/codeblocks.md | 3 +++ pyproject.toml | 2 +- uv.lock | 8 ++++---- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml index 1ccf4e96ce..56b8d9a473 100644 --- a/.github/workflows/doc-codeblocks.yml +++ b/.github/workflows/doc-codeblocks.yml @@ -28,7 +28,12 @@ permissions: jobs: md-babel: - if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + if: >- + github.event_name != 'pull_request' || + ( + github.event.pull_request.draft == false && + github.event.pull_request.head.repo.full_name == github.repository + ) timeout-minutes: 60 runs-on: [self-hosted, Linux] container: diff --git a/bin/run-doc-codeblocks b/bin/run-doc-codeblocks index b14b9de52e..09d906404d 100755 --- a/bin/run-doc-codeblocks +++ b/bin/run-doc-codeblocks @@ -7,6 +7,7 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" LANG_FILTER="python,sh,node" +# md-babel-py 1.1.4+: isolated block subprocess limit (default upstream is 60). declare -a USER_PATHS=() usage() { @@ -56,12 +57,13 @@ while [[ $# -gt 0 ]]; do done resolve_md_babel() { - # Prefer `uv run` so md-babel's session subprocess uses this project's interpreter - # (plain .venv/bin/md-babel-py can pick up the wrong Python on some machines). + # Prefer project env so md-babel-py version matches pyproject (needs >= 1.1.4 for --execution-timeout). if command -v uv &>/dev/null && [[ -f pyproject.toml ]]; then MB=(uv run md-babel-py) elif [[ -x .venv/bin/md-babel-py ]]; then MB=(.venv/bin/md-babel-py) + elif [[ -x .venv/bin/python ]]; then + MB=(.venv/bin/python -m md_babel_py.cli) elif command -v md-babel-py &>/dev/null; then MB=(md-babel-py) else @@ -74,7 +76,7 @@ resolve_md_babel run_one_target() { local target=$1 - local -a cmd=("${MB[@]}" run) + local -a cmd=("${MB[@]}" run --execution-timeout 120) if [[ -n "$LANG_FILTER" ]]; then cmd+=(--lang "$LANG_FILTER") fi @@ -96,10 +98,6 @@ if [[ ${#USER_PATHS[@]} -gt 0 ]]; then fi done else - if [[ -d ./docs ]]; then - run_one_target ./docs - fi - if [[ -f README.md ]]; then - run_one_target README.md - fi + run_one_target ./docs + run_one_target README.md fi diff --git a/docs/agents/docs/codeblocks.md b/docs/agents/docs/codeblocks.md index 2d9cf1bef1..958c14a7b1 100644 --- a/docs/agents/docs/codeblocks.md +++ b/docs/agents/docs/codeblocks.md @@ -313,4 +313,7 @@ md-babel-py run document.md --lang python,sh # Dry run - show what would execute md-babel-py run document.md --dry-run + +# Longer subprocess limit (default 60s); see upstream README for MD_BABEL_EXECUTION_TIMEOUT +md-babel-py run document.md --execution-timeout 120 ``` diff --git a/pyproject.toml b/pyproject.toml index fa35dd79de..54e38540d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -248,7 +248,7 @@ dev = [ "watchdog>=3.0.0", # docs - "md-babel-py==1.1.1", + "md-babel-py==1.1.4", # LSP "python-lsp-server[all]==1.14.0", diff --git a/uv.lock b/uv.lock index aebf6f9055..7b8ce137ac 100644 --- a/uv.lock +++ b/uv.lock @@ -2027,7 +2027,7 @@ requires-dist = [ { name = "lxml-stubs", marker = "extra == 'dev'", specifier = ">=0.5.1,<1" }, { name = "lz4", specifier = ">=4.4.5" }, { name = "matplotlib", marker = "extra == 'manipulation'", specifier = ">=3.7.1" }, - { name = "md-babel-py", marker = "extra == 'dev'", specifier = "==1.1.1" }, + { name = "md-babel-py", marker = "extra == 'dev'", specifier = "==1.1.4" }, { name = "moondream", marker = "extra == 'perception'" }, { name = "mujoco", marker = "extra == 'sim'", specifier = ">=3.3.4" }, { name = "mypy", marker = "extra == 'dev'", specifier = "==1.19.0" }, @@ -4817,11 +4817,11 @@ wheels = [ [[package]] name = "md-babel-py" -version = "1.1.1" +version = "1.1.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/b3/f814d429edf2848ba03079a3f6da443e6d45b984a7fc22766cb73939d289/md_babel_py-1.1.1.tar.gz", hash = "sha256:826fea96b7415eeaab7607ed5e8eb6d7723f22b9f1005af1b7da12f68766123d", size = 30547, upload-time = "2026-01-20T06:27:32.496Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/73/184cdcf55b3498c0445420867ab8df0d4c1575de16779f539db9deb351af/md_babel_py-1.1.4.tar.gz", hash = "sha256:cf566ec36a41927c9c20655b0ae00080b3242af8083768acec555342908d0959", size = 31234, upload-time = "2026-04-24T05:01:34.871Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/4a/dbe497b41432a98c7d4f043cf112410957553ce27e56bc366714695f53a9/md_babel_py-1.1.1-py3-none-any.whl", hash = "sha256:4df82011f123f13b6f9979226e69b0ce06209d94e4c029b60eeb2f54a709d2d0", size = 25836, upload-time = "2026-01-20T06:27:31.514Z" }, + { url = "https://files.pythonhosted.org/packages/63/18/84594b4f9f347fd7b3e9fe4be62b35cc115f77ccd251184d8580da00589e/md_babel_py-1.1.4-py3-none-any.whl", hash = "sha256:5ea0ef7f22f126b9872aca86574728fe54814191dd66ef15a20deeb9597ca455", size = 26269, upload-time = "2026-04-24T05:01:33.363Z" }, ] [[package]] From 30a12a39129e97fc21974706ea1eb82ce08fe829 Mon Sep 17 00:00:00 2001 From: bogwi Date: Sat, 25 Apr 2026 00:03:22 +0900 Subject: [PATCH 09/11] allign with what .github/workflows/code-cleanup.yml action does, as per @#1759 --- .github/workflows/doc-codeblocks.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml index 56b8d9a473..1ccf4e96ce 100644 --- a/.github/workflows/doc-codeblocks.yml +++ b/.github/workflows/doc-codeblocks.yml @@ -28,12 +28,7 @@ permissions: jobs: md-babel: - if: >- - github.event_name != 'pull_request' || - ( - github.event.pull_request.draft == false && - github.event.pull_request.head.repo.full_name == github.repository - ) + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false timeout-minutes: 60 runs-on: [self-hosted, Linux] container: From 7731590a69ddcb78e5d3fdf1afce7ee56433ef24 Mon Sep 17 00:00:00 2001 From: bogwi Date: Sat, 25 Apr 2026 00:33:26 +0900 Subject: [PATCH 10/11] supress push for now; fail if block fails --- .github/workflows/doc-codeblocks.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml index 1ccf4e96ce..47f76539d3 100644 --- a/.github/workflows/doc-codeblocks.yml +++ b/.github/workflows/doc-codeblocks.yml @@ -6,9 +6,9 @@ # - If a block is broken and the fix is not obvious, contributors may mark the # fence with `skip` (or `expected-error` where appropriate) and fix later; # until then, this job fails on execution errors so drift is visible. -# - Like .github/workflows/code-cleanup.yml: refreshed / generated -# assets are committed onto the PR branch so devs are not required to run this -# locally for output-only churn. +# - For now this job only verifies that covered code blocks execute in CI. +# A follow-up can re-enable auto-committing refreshed / +# generated assets back onto the PR branch. name: doc-codeblocks on: pull_request: @@ -22,7 +22,7 @@ on: - 'uv.lock' permissions: - contents: write + contents: read packages: read pull-requests: read @@ -48,9 +48,6 @@ jobs: git config --global --add safe.directory '*' git clean -ffdx -e .venv - - name: Branch ref for git-auto-commit - run: git checkout -B "${{ github.head_ref }}" - - uses: actions/setup-node@v4 with: node-version: '22' @@ -66,8 +63,3 @@ jobs: - name: Execute documentation code blocks run: ./bin/run-doc-codeblocks --ci - - - name: Commit doc updates - uses: stefanzweifel/git-auto-commit-action@v5 - with: - commit_message: "CI: execute documentation code blocks" From d4bb9761b67df51d249614b969b998feb3e3420e Mon Sep 17 00:00:00 2001 From: bogwi Date: Sat, 25 Apr 2026 00:38:08 +0900 Subject: [PATCH 11/11] add --no-cache, CI now always re-executes covered blocks --- .github/workflows/doc-codeblocks.yml | 2 +- bin/run-doc-codeblocks | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/doc-codeblocks.yml b/.github/workflows/doc-codeblocks.yml index 47f76539d3..fc45473d4d 100644 --- a/.github/workflows/doc-codeblocks.yml +++ b/.github/workflows/doc-codeblocks.yml @@ -62,4 +62,4 @@ jobs: find .venv/lib/*/site-packages/pydrake -name '*.pyi' -delete 2>/dev/null || true - name: Execute documentation code blocks - run: ./bin/run-doc-codeblocks --ci + run: ./bin/run-doc-codeblocks --ci --no-cache diff --git a/bin/run-doc-codeblocks b/bin/run-doc-codeblocks index 09d906404d..9153098e60 100755 --- a/bin/run-doc-codeblocks +++ b/bin/run-doc-codeblocks @@ -8,6 +8,7 @@ cd "$REPO_ROOT" LANG_FILTER="python,sh,node" # md-babel-py 1.1.4+: isolated block subprocess limit (default upstream is 60). +USE_CACHE=1 declare -a USER_PATHS=() usage() { @@ -17,6 +18,7 @@ Usage: bin/run-doc-codeblocks [options] [path ...] --ci Same as --lang python,sh,node (default; for CI runners). --lang LIST Comma-separated languages for md-babel-py (default: python,sh,node). --all-langs Run all block languages (requires local native tools; see codeblocks.md). + --no-cache Re-execute all blocks instead of reusing md-babel cache. With no paths: runs on ./docs (recursive) and ./README.md when present. @@ -44,6 +46,10 @@ while [[ $# -gt 0 ]]; do LANG_FILTER="" shift ;; + --no-cache) + USE_CACHE=0 + shift + ;; -*) echo "Unknown option: $1" >&2 usage @@ -77,6 +83,9 @@ resolve_md_babel run_one_target() { local target=$1 local -a cmd=("${MB[@]}" run --execution-timeout 120) + if [[ "$USE_CACHE" -eq 0 ]]; then + cmd+=(--no-cache) + fi if [[ -n "$LANG_FILTER" ]]; then cmd+=(--lang "$LANG_FILTER") fi