feat(image): add synchronized annotation and scene mesh overlays

src/features/layout/autoLayout/applyDefaultRosDockLayout.test.ts

@@ -115,14 +115,15 @@ describe('buildDefaultRosFoxgloveLayoutData', () => {
     expect(panelTypes.filter((type) => type === 'Image')).toHaveLength(3);
   });
 
-  it('builds two image rows for six CompressedVideo streams without 3D', () => {
+  it('builds two image rows for six CompressedVideo streams and binds one scene mesh topic', () => {
     const topics: TopicInfo[] = [
       { name: '/head/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' },
       { name: '/head/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' },
       { name: '/left/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' },
       { name: '/left/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' },
       { name: '/right/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' },
       { name: '/right/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' },
+      { name: '/robot0/perception/mano/scene', type: 'foxglove.SceneUpdate' },
     ];
     const data = buildDefaultRosFoxgloveLayoutData(topics);
     const ids = collectMosaicPanelIds(data.layout);
@@ -143,6 +144,17 @@ describe('buildDefaultRosFoxgloveLayoutData', () => {
       '/right/color/image',
       '/right/depth/image',
     ]);
+
+    const imageConfigs = Object.values(data.configById) as Array<{
+      topic?: string;
+      meshTopic?: string;
+      meshVisible?: boolean;
+    }>;
+    expect(imageConfigs).toHaveLength(6);
+    expect(imageConfigs.every((config) => config.meshTopic === '/robot0/perception/mano/scene')).toBe(
+      true,
+    );
+    expect(imageConfigs.every((config) => config.meshVisible === true)).toBe(true);
   });
 
   it('BVH-only dataset: single 3D panel only; dockview root still wraps to branch for fromJSON', () => {

src/features/layout/autoLayout/applyDefaultRosDockLayout.ts

@@ -22,6 +22,7 @@ import { heuristicAudioInfoTopics } from '@/features/panels/Audio/core/resolveAu
 import { getPanelDefinition } from '@/features/panels/registry';
 import { isAudioCommonInfoSchema, isJointStateSchema, isRawAudioSchema, isRosImageSchema, normalizeRosSchemaName } from '@/shared/ros/rosMessageTypes';
 import { pickDefaultRawMessagesTopic } from '@/features/layout/autoLayout/pickDefaultRawMessagesTopic';
+import { isSceneUpdateSchema } from '@/features/panels/Image/core/sceneMesh';
 
 function imageTabTitle(topic: string): string {
   const parts = topic.split('/').filter(Boolean);
@@ -132,6 +133,7 @@ function appendFallbackRawMessagesPanel(
 function appendImagePanelsForRow(
   row: (string | null)[],
   configById: Record<string, FoxgloveConfig>,
+  meshTopic?: string,
 ): string[] {
   const ids: string[] = [];
   for (const topic of row) {
@@ -140,6 +142,7 @@ function appendImagePanelsForRow(
     ids.push(id);
     configById[id] = {
       topic,
+      ...(meshTopic ? { meshTopic, meshVisible: true } : {}),
       [FOXGLOVE_PANEL_TITLE_KEY]: imageTabTitle(topic),
     };
   }
@@ -237,6 +240,8 @@ export function buildDefaultRosFoxgloveLayoutData(
 
   const stackParts: FoxgloveMosaicNode[] = [];
   const usedImageTopics = new Set<string>();
+  const sceneMeshTopics = topics.filter((topic) => isSceneUpdateSchema(topic.type));
+  const meshTopic = sceneMeshTopics.length === 1 ? sceneMeshTopics[0]?.name : undefined;
 
   const pickedImageTopics = selectImageTopicsForAutoLayout(topics);
   const hasDepthImageStreams = topics.some(
@@ -245,8 +250,8 @@ export function buildDefaultRosFoxgloveLayoutData(
 
   if (hasDepthImageStreams) {
     const { colorRow, depthRow } = planColorDepthCameraRows(topics);
-    const colorImageIds = appendImagePanelsForRow(colorRow, configById);
-    const depthImageIds = appendImagePanelsForRow(depthRow, configById);
+    const colorImageIds = appendImagePanelsForRow(colorRow, configById, meshTopic);
+    const depthImageIds = appendImagePanelsForRow(depthRow, configById, meshTopic);
     for (const topic of colorRow) {
       if (topic) usedImageTopics.add(topic);
     }
@@ -259,7 +264,7 @@ export function buildDefaultRosFoxgloveLayoutData(
     if (depthMosaic) stackParts.push(depthMosaic);
   } else if (pickedImageTopics.length > 0) {
     for (const row of buildImageRows(pickedImageTopics)) {
-      const imageIds = appendImagePanelsForRow(row, configById);
+      const imageIds = appendImagePanelsForRow(row, configById, meshTopic);
       for (const topic of row) {
         usedImageTopics.add(topic);
       }

src/features/panels/Image/ImagePanel.tsx

@@ -4,7 +4,7 @@ import type { Player } from '@/core/types/player';
 import { useMessagePipeline } from '@/core/pipeline/useMessagePipeline';
 import type { MessageEvent as RosMessageEvent } from '@/core/types/ros';
 import { scheduleFrame } from '@/shared/utils/rafScheduler';
-import { toNano } from '@/shared/utils/time';
+import { addMs, toNano } from '@/shared/utils/time';
 import type { RawImageDecodeOptions } from './core/imageColorMode';
 import type {
   ImageRenderMetrics,
@@ -19,6 +19,14 @@ import {
 import { repairH264Seek } from './core/h264SeekRepair';
 import { isH264MessageEvent, toWorkerFrame } from './core/messageFrameAdapter';
 import { applyDepthTopicPreset } from './core/depthColorDefaults';
+import { parseImageAnnotations } from './core/imageAnnotations';
+import { SceneMeshOverlayRenderer } from './core/SceneMeshOverlayRenderer';
+import {
+  inferCameraCalibrationTopic,
+  parseDoubleSphereCalibration,
+  parseSceneMeshes,
+} from './core/sceneMesh';
+import { subscribeSceneMeshTopic } from './core/sceneMeshTopicBroker';
 import type { ImageConfig } from './defaults';
 import { TopicQuickPicker } from '../framework/TopicQuickPicker';
 import { PanelTopicBar } from '../framework/PanelTopicBar';
@@ -54,6 +62,10 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
     panelId,
     setConfig,
     topic,
+    annotationTopic,
+    annotationVisible,
+    meshTopic,
+    meshVisible,
     backgroundColor,
     showStatusText,
     fitMode,
@@ -71,17 +83,46 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
   } = props;
 
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
+  const meshCanvasRef = useRef<HTMLCanvasElement | null>(null);
+  const meshRendererRef = useRef<SceneMeshOverlayRenderer | null>(null);
   const viewportRef = useRef<HTMLDivElement | null>(null);
   const workerRef = useRef<Worker | null>(null);
   const workerDisposeTimerRef = useRef<number | null>(null);
   const transferredCanvasRef = useRef<HTMLCanvasElement | null>(null);
   const lastPlaybackTimeNsRef = useRef<bigint | null>(null);
+  const seekRepairGenerationRef = useRef(0);
   const lastUiStatusRef = useRef<ImageSurfaceStatus>({ phase: 'idle' });
   const h264ModeRef = useRef(false);
+  const meshRenderOptionsRef = useRef<ImageRenderOptions>({
+    backgroundColor,
+    flipHorizontal,
+    flipVertical,
+    rotationDeg: rotation,
+    smoothing,
+    fitMode,
+  });
   const [status, setStatus] = useState<ImageSurfaceStatus>({ phase: 'idle' });
   const [metrics, setMetrics] = useState<ImageRenderMetrics | null>(null);
   const mainConsumerId = `${panelId}:image-main`;
   const h264ConsumerId = `${panelId}:image-main-h264`;
+  const annotationConsumerId = `${panelId}:image-annotations`;
+  const calibrationConsumerId = `${panelId}:image-scene-mesh-calibration`;
+  const selectedAnnotationTopic = annotationVisible ? annotationTopic.trim() : '';
+  const selectedMeshTopic = meshVisible ? meshTopic.trim() : '';
+  const selectedCalibrationTopic = inferCameraCalibrationTopic(topic);
+
+  useEffect(() => {
+    const canvas = meshCanvasRef.current;
+    if (!canvas) return;
+    const renderer = new SceneMeshOverlayRenderer(canvas, meshRenderOptionsRef.current);
+    meshRendererRef.current = renderer;
+    return () => {
+      renderer.dispose();
+      if (meshRendererRef.current === renderer) {
+        meshRendererRef.current = null;
+      }
+    };
+  }, []);
 
   // Worker lifecycle: init on mount, dispose on unmount
   useEffect(() => {
@@ -134,6 +175,14 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
         setMetrics(data.metrics);
         return;
       }
+      if (data.type === 'rendered') {
+        meshRendererRef.current?.renderImageFrame(
+          data.timestampNs,
+          data.width,
+          data.height,
+        );
+        return;
+      }
       if (data.type !== 'status') {
         return;
       }
@@ -254,6 +303,66 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
     };
   }, [player, mainConsumerId, h264ConsumerId, topic]);
 
+  // Keep annotation delivery on the video lane. The worker selects the
+  // closest publish-time match before drawing over each image frame.
+  useEffect(() => {
+    const worker = workerRef.current;
+    if (!selectedAnnotationTopic || !worker) return;
+
+    player.registerHighFrequencyConsumer(annotationConsumerId, {
+      topic: selectedAnnotationTopic,
+      lane: 'video',
+      mode: 'all',
+      onMessageBatch: (messages) => {
+        for (const event of messages) {
+          const overlay = parseImageAnnotations(event.message);
+          if (overlay) {
+            worker.postMessage({ type: 'overlay', overlay } satisfies ImageRenderWorkerRequest);
+          }
+        }
+      },
+    });
+
+    return () => {
+      player.unregisterHighFrequencyConsumer(annotationConsumerId);
+      worker.postMessage({ type: 'overlay', overlay: null } satisfies ImageRenderWorkerRequest);
+    };
+  }, [annotationConsumerId, player, selectedAnnotationTopic]);
+
+  useEffect(() => {
+    const renderer = meshRendererRef.current;
+    renderer?.clearFrames();
+    if (!renderer || !selectedMeshTopic) return;
+    const unsubscribe = subscribeSceneMeshTopic(
+      player,
+      selectedMeshTopic,
+      (frame) => renderer.addFrame(frame),
+    );
+    return () => {
+      unsubscribe();
+      renderer.clearFrames();
+    };
+  }, [player, selectedMeshTopic]);
+
+  useEffect(() => {
+    const renderer = meshRendererRef.current;
+    renderer?.setCalibration(null);
+    if (!renderer || !selectedCalibrationTopic) return;
+    player.registerHighFrequencyConsumer(calibrationConsumerId, {
+      topic: selectedCalibrationTopic,
+      lane: 'pointcloud',
+      mode: 'latest',
+      onLatestMessage: (event) => {
+        const calibration = parseDoubleSphereCalibration(event.message);
+        if (calibration) renderer.setCalibration(calibration);
+      },
+    });
+    return () => {
+      player.unregisterHighFrequencyConsumer(calibrationConsumerId);
+      renderer.setCalibration(null);
+    };
+  }, [calibrationConsumerId, player, selectedCalibrationTopic]);
+
   // Keep the worker's media deadline current. On rewind, rebuild H.264 state
   // from the nearest complete random-access point.
   useEffect(() => {
@@ -265,21 +374,59 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
       } satisfies ImageRenderWorkerRequest);
       const nowNs = toNano(time);
       const previousNs = lastPlaybackTimeNsRef.current;
+      if (previousNs !== nowNs) {
+        seekRepairGenerationRef.current += 1;
+      }
       if (previousNs != null && nowNs + 5_000_000n < previousNs) {
+        const repairGeneration = seekRepairGenerationRef.current;
         const worker = workerRef.current;
+        const renderer = meshRendererRef.current;
+        renderer?.clearFrames();
+        const stillImageTopic = worker && topic && !h264ModeRef.current ? topic : null;
         if (worker && topic && h264ModeRef.current) {
           worker.postMessage({
             type: 'reset',
             preserveFrame: true,
           } satisfies ImageRenderWorkerRequest);
           void repairH264Seek(player, worker, topic, time);
         } else {
-          workerRef.current?.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest);
+          worker?.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest);
+        }
+        const repairTopics = new Set<string>();
+        if (stillImageTopic) repairTopics.add(stillImageTopic);
+        if (renderer && selectedMeshTopic) repairTopics.add(selectedMeshTopic);
+        if (repairTopics.size > 0 && player.getMessagesInTimeRange) {
+          void player.getMessagesInTimeRange({
+            start: addMs(time, -2000),
+            end: time,
+            topics: [...repairTopics],
+          }).then((messages) => {
+            if (
+              seekRepairGenerationRef.current !== repairGeneration ||
+              (renderer && meshRendererRef.current !== renderer)
+            ) return;
+            let latestImage: RosMessageEvent | undefined;
+            for (const event of messages) {
+              if (
+                stillImageTopic &&
+                event.topic === stillImageTopic &&
+                toNano(event.receiveTime) <= nowNs &&
+                (!latestImage || toNano(event.receiveTime) > toNano(latestImage.receiveTime))
+              ) {
+                latestImage = event;
+              }
+              if (renderer && event.topic === selectedMeshTopic) {
+                const frame = parseSceneMeshes(event.message, toNano(event.publishTime));
+                if (frame) renderer.addFrame(frame);
+              }
+            }
+            if (worker && latestImage) postImageFrame(worker, latestImage);
+          });
         }
       }
       lastPlaybackTimeNsRef.current = nowNs;
     });
-  }, [isPlaying, player, topic]);
+  }, [isPlaying, player, selectedMeshTopic, topic]);
 
   // Send color/depth decode options when they change — triggers immediate redraw in worker
   useEffect(() => {
@@ -303,10 +450,6 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
 
   // Send render options (flip/rotation/smoothing/fitMode) — triggers immediate redraw
   useEffect(() => {
-    const worker = workerRef.current;
-    if (!worker) {
-      return;
-    }
     const options: ImageRenderOptions = {
       backgroundColor,
       flipHorizontal,
@@ -315,7 +458,12 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
       smoothing,
       fitMode,
     };
-    worker.postMessage({ type: 'renderOptions', options } satisfies ImageRenderWorkerRequest);
+    meshRenderOptionsRef.current = options;
+    meshRendererRef.current?.setOptions(options);
+    workerRef.current?.postMessage({
+      type: 'renderOptions',
+      options,
+    } satisfies ImageRenderWorkerRequest);
   }, [backgroundColor, flipHorizontal, flipVertical, rotation, smoothing, fitMode]);
 
   const statusText = getStatusText(status);
@@ -352,6 +500,11 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
           className="w-full h-full block"
           data-testid="image-panel-canvas"
         />
+        <canvas
+          ref={meshCanvasRef}
+          className="absolute inset-0 z-[1] h-full w-full pointer-events-none"
+          data-testid="image-panel-scene-mesh-overlay"
+        />
         {showStatusText && statusText && (
           <div className="absolute inset-0 flex items-center justify-center pointer-events-none text-white/40 italic text-xs">
             {statusText}