Improve H.264 accelerated playback

docs/H264_PERFORMANCE.md

@@ -13,12 +13,16 @@ npm run benchmark:h264 -- \
   --file /absolute/path/representative.mcap \
   --base-url http://127.0.0.1:4173 \
   --duration 60 \
+  --speed 8 \
+  --no-seeks \
   --output benchmark-h264.json
 ```
 
 The benchmark exposes the original file through a temporary read-only Range/CORS server. It records
-progress updates, random seeks, Image panel H.264 metrics, decode/page/console errors, and Chromium
-JS heap when available. Run `npm run benchmark:h264 -- --help` for all options.
+progress updates, optional random seeks, aggregate metrics for every Image panel, decode/page/console
+errors, and Chromium JS heap when available. `--speed` is a load input only: adaptive behavior is
+driven by measured media lag and queue pressure, not by fixed playback-rate branches. Run
+`npm run benchmark:h264 -- --help` for all options.
 
 ## Manual browser pass
 
@@ -35,6 +39,8 @@ Suggested acceptance targets:
   movement in about one second.
 - No decode, page, or console errors; Image metrics are non-negative and pressure is `normal`,
   `degraded`, or `recovery`.
+- Decoder input remains bounded, media lag repeatedly recovers after pressure, rendered frames keep
+  increasing, and resync count does not grow continuously under a steady workload.
 - The H.264 pending queue is hard-bounded at 120 frames and a 1,000 ms media-time span. Soft
   pressure keeps a sole complete GOP intact; if either hard bound is exceeded without a newer IDR
   suffix that fits, the worker keeps the current picture, drops the complete backlog, and waits for

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ioai/rosview",
-  "version": "1.7.4",
+  "version": "1.7.5",
   "description": "High-performance robotics data visualization for MCAP, ROS bag, ROS2 db3, HDF5 and BVH — embeddable React component and standalone SPA",
   "keywords": [
     "ros",

scripts/benchmark-h264.mjs

@@ -18,6 +18,8 @@ Options:
   --file <path>       Absolute path to a local MCAP file (required)
   --base-url <url>    Running rosview URL (default: ${DEFAULT_BASE_URL})
   --duration <sec>    Benchmark duration in seconds (default: ${DEFAULT_DURATION_SECONDS})
+  --speed <rate>      Playback speed used as load input (default: 1)
+  --no-seeks          Disable random seek stress during sampling
   --output <path>     Also write the JSON result to this path
   --help              Show this help
 `;
@@ -27,6 +29,8 @@ function parseArgs(argv) {
   const options = {
     baseUrl: DEFAULT_BASE_URL,
     durationSeconds: DEFAULT_DURATION_SECONDS,
+    speed: 1,
+    seeks: true,
     file: undefined,
     output: undefined,
     help: false,
@@ -35,6 +39,7 @@ function parseArgs(argv) {
     ['--file', 'file'],
     ['--base-url', 'baseUrl'],
     ['--duration', 'durationSeconds'],
+    ['--speed', 'speed'],
     ['--output', 'output'],
   ]);
 
@@ -44,6 +49,10 @@ function parseArgs(argv) {
       options.help = true;
       continue;
     }
+    if (argument === '--no-seeks') {
+      options.seeks = false;
+      continue;
+    }
     const separator = argument.indexOf('=');
     const name = separator >= 0 ? argument.slice(0, separator) : argument;
     const key = valueOptions.get(name);
@@ -54,7 +63,7 @@ function parseArgs(argv) {
     if (!value || value.startsWith('--')) {
       throw new Error(`${name} requires a value`);
     }
-    options[key] = key === 'durationSeconds' ? Number(value) : value;
+    options[key] = key === 'durationSeconds' || key === 'speed' ? Number(value) : value;
   }
   return options;
 }
@@ -85,6 +94,9 @@ async function validateOptions(options) {
   if (!Number.isFinite(options.durationSeconds) || options.durationSeconds < 2) {
     throw new Error('--duration must be a number of at least 2 seconds');
   }
+  if (!Number.isFinite(options.speed) || options.speed < 0.1 || options.speed > 8) {
+    throw new Error('--speed must be between 0.1 and 8');
+  }
   if (options.output) {
     options.output = path.resolve(options.output);
   }
@@ -206,12 +218,9 @@ async function readProgress(fill) {
   });
 }
 
-async function readImagePanel(page) {
-  const panel = page.getByTestId('image-panel').first();
-  if (!(await panel.isVisible().catch(() => false))) {
-    return null;
-  }
-  return panel.evaluate((element) => {
+async function readImagePanels(page) {
+  const panels = page.getByTestId('image-panel');
+  return panels.evaluateAll((elements) => elements.map((element) => {
     const numberAttribute = (name) => {
       const value = element.getAttribute(name);
       if (value === null) return null;
@@ -222,8 +231,12 @@ async function readImagePanel(page) {
       pressure: element.getAttribute('data-h264-pressure'),
       queueFrames: numberAttribute('data-h264-queue-frames'),
       droppedFrames: numberAttribute('data-h264-dropped-frames'),
+      decodeQueueSize: numberAttribute('data-h264-decode-queue'),
+      mediaLagMs: numberAttribute('data-h264-media-lag-ms'),
+      resyncCount: numberAttribute('data-h264-resync-count'),
+      renderedFrames: numberAttribute('data-h264-rendered-frames'),
     };
-  });
+  }));
 }
 
 function summarizeProgress(samples) {
@@ -276,6 +289,10 @@ async function runBenchmark(options, fixtureUrl) {
       .waitFor({ state: 'visible', timeout: 90_000 });
     const play = page.getByRole('button', { name: 'Play playback' });
     await play.waitFor({ state: 'visible', timeout: 30_000 });
+    if (options.speed !== 1) {
+      await page.getByTestId('playback-speed-trigger').click();
+      await page.getByRole('menuitem', { name: `${options.speed}x`, exact: true }).click();
+    }
     await play.click();
 
     const fill = page.getByTestId('playback-progress-fill');
@@ -286,7 +303,9 @@ async function runBenchmark(options, fixtureUrl) {
     const heapSamples = [];
     const seeks = [];
     const durationMs = options.durationSeconds * 1_000;
-    const seekTimes = [0.35, 0.6, 0.82].map((ratio) => durationMs * ratio);
+    const seekTimes = options.seeks
+      ? [0.35, 0.6, 0.82].map((ratio) => durationMs * ratio)
+      : [];
     let nextSeek = 0;
     const samplingStartedAt = Date.now();
 
@@ -312,8 +331,8 @@ async function runBenchmark(options, fixtureUrl) {
         await resume.click();
       }
       samples.push({ elapsedMs, width: await readProgress(fill) });
-      const image = await readImagePanel(page);
-      if (image) imageSamples.push({ elapsedMs, ...image });
+      const panels = await readImagePanels(page);
+      if (panels.length > 0) imageSamples.push({ elapsedMs, panels });
       const heapBytes = await readHeap(page);
       if (heapBytes !== null) heapSamples.push({ elapsedMs, bytes: heapBytes });
       await page.waitForTimeout(SAMPLE_INTERVAL_MS);
@@ -326,13 +345,23 @@ async function runBenchmark(options, fixtureUrl) {
     const statusTexts = await page.getByTestId('image-panel-status').allTextContents().catch(() => []);
     const heapValues = heapSamples.map(({ bytes }) => bytes);
     const latestImage = imageSamples.at(-1) ?? null;
+    const latestPanels = latestImage?.panels ?? [];
     const metricsReasonable =
-      latestImage === null ||
-      (['normal', 'degraded', 'recovery'].includes(latestImage.pressure) &&
-        Number.isInteger(latestImage.queueFrames) &&
-        latestImage.queueFrames >= 0 &&
-        Number.isInteger(latestImage.droppedFrames) &&
-        latestImage.droppedFrames >= 0);
+      latestPanels.every((panel) =>
+        ['normal', 'degraded', 'recovery'].includes(panel.pressure) &&
+        Number.isInteger(panel.queueFrames) &&
+        panel.queueFrames >= 0 &&
+        Number.isInteger(panel.droppedFrames) &&
+        panel.droppedFrames >= 0 &&
+        Number.isInteger(panel.decodeQueueSize) &&
+        panel.decodeQueueSize >= 0 &&
+        Number.isFinite(panel.mediaLagMs) &&
+        panel.mediaLagMs >= 0 &&
+        Number.isInteger(panel.resyncCount) &&
+        panel.resyncCount >= 0 &&
+        Number.isInteger(panel.renderedFrames) &&
+        panel.renderedFrames >= 0
+      );
 
     return {
       schemaVersion: 1,
@@ -342,6 +371,7 @@ async function runBenchmark(options, fixtureUrl) {
         fileBytes: options.fileSize,
         baseUrl: options.baseUrl,
         durationSeconds: options.durationSeconds,
+        speed: options.speed,
       },
       environment: {
         platform: process.platform,

src/core/players/IterablePlayer.test.ts

@@ -120,6 +120,25 @@ afterEach(() => {
   messageBus.reset();
 });
 
+describe('IterablePlayer playback speed', () => {
+  it('uses a deterministic 10x maximum instead of a best-effort sentinel', async () => {
+    const player = new IterablePlayer(makeSource([]));
+    let latestState: PlayerState | undefined;
+    player.setListener((state) => {
+      latestState = state;
+    });
+    await player.initialize({});
+
+    player.setSpeed(10);
+    expect(latestState?.activeData?.speed).toBe(10);
+
+    player.setSpeed(64);
+    expect(latestState?.activeData?.speed).toBe(10);
+
+    player.close();
+  });
+});
+
 describe('IterablePlayer high-frequency lane', () => {
   it('routes video-only topics outside the generic message bus', async () => {
     const source = makeSource([makeImageMessage()]);

src/core/players/IterablePlayer.ts

@@ -6,7 +6,7 @@ import type {
   StreamMessagesInTimeRangeArgs,
   Subscription,
 } from '@/core/types/player';
-import { PLAYBACK_SPEED_MAX } from '@/core/types/player';
+import { MAX_PLAYBACK_SPEED } from '@/core/types/player';
 import type { DataQualityReport, Time, Initialization, MessageEvent, TimeRange } from '@/core/types/ros';
 import type { ISourceHandle } from '@/infra/workers/ISourceHandle';
 import type { IMessageCursor } from '@/infra/workers/types';
@@ -469,7 +469,7 @@ export class IterablePlayer implements Player {
     this._advancePlaybackEpoch();
     this._isPlaying = true;
     const now = performance.now();
-    this._clock.play(this._currentTime, this._speedFactor(), now);
+    this._clock.play(this._currentTime, this._speed, now);
     this._lastTickWallMs = now;
     this._pageSuspended = typeof document !== "undefined" && document.hidden;
     if (this._pageSuspended) {
@@ -600,19 +600,11 @@ export class IterablePlayer implements Player {
   }
 
   setSpeed(speed: number): void {
-    if (speed === PLAYBACK_SPEED_MAX) {
-      this._speed = PLAYBACK_SPEED_MAX;
-    } else {
-      this._speed = Math.min(8, Math.max(0.1, speed));
-    }
-    this._clock.setSpeed(this._speedFactor(), performance.now());
+    this._speed = Math.min(MAX_PLAYBACK_SPEED, Math.max(0.1, speed));
+    this._clock.setSpeed(this._speed, performance.now());
     this._emitState();
   }
 
-  private _speedFactor(): number {
-    return this._speed === PLAYBACK_SPEED_MAX ? 64 : this._speed;
-  }
-
   setSamplingFps(fps: number): void {
     const clamped = Math.max(1, Math.min(MAX_SAMPLING_FPS, Math.round(fps)));
     this._samplingFps = clamped;

src/core/types/player.ts

@@ -97,8 +97,8 @@ export interface HighFrequencyConsumer {
   onMessageBatch?: (messages: MessageEvent[]) => void;
 }
 
-/** Sentinel playback speed for “as fast as possible” (see IterablePlayer / PlaybackBar). */
-export const PLAYBACK_SPEED_MAX = -1;
+/** Highest deterministic playback rate exposed by the visualization player. */
+export const MAX_PLAYBACK_SPEED = 10;
 
 export interface Player {
   setListener(listener: (state: PlayerState) => void): void;

src/features/panels/Audio/AudioPanel.tsx

@@ -3,7 +3,6 @@ import { useIntl } from 'react-intl';
 import { useShallow } from 'zustand/react/shallow';
 import { timeToNs } from '@/core/analysis/timeSeries';
 import type { Player } from '@/core/types/player';
-import { PLAYBACK_SPEED_MAX } from '@/core/types/player';
 import { messageBus } from '@/core/pipeline/messageBus';
 import { useSubscriberSeq } from '@/core/pipeline/useMessageBus';
 import { useMessagePipeline } from '@/core/pipeline/useMessagePipeline';
@@ -100,7 +99,6 @@ export const AudioPanel: React.FC<AudioPanelProps> = (props) => {
 
   const allowPlayback = useMemo(() => {
     if (!isPlaying || config.mute) return false;
-    if (speed === PLAYBACK_SPEED_MAX) return false;
     return Math.abs(speed - 1) < 1e-4;
   }, [isPlaying, config.mute, speed]);
 
@@ -259,7 +257,7 @@ export const AudioPanel: React.FC<AudioPanelProps> = (props) => {
   const statusLabel = useMemo(() => {
     if (!config.topic) return formatMessage({ id: 'panels.audio.status.waitingTopic' });
     if (!allowPlayback && isPlaying && !config.mute) {
-      if (speed === PLAYBACK_SPEED_MAX || Math.abs(speed - 1) >= 1e-4) {
+      if (Math.abs(speed - 1) >= 1e-4) {
         return formatMessage({ id: 'panels.audio.status.mutedNon1x' });
       }
     }

src/features/panels/Image/ImagePanel.tsx

@@ -1,6 +1,7 @@
 import React, { useEffect, useRef, useState } from 'react';
 import { useIntl } from 'react-intl';
 import type { Player } from '@/core/types/player';
+import { useMessagePipeline } from '@/core/pipeline/useMessagePipeline';
 import type { MessageEvent as RosMessageEvent } from '@/core/types/ros';
 import { scheduleFrame } from '@/shared/utils/rafScheduler';
 import { toNano } from '@/shared/utils/time';
@@ -45,6 +46,9 @@ export type ImagePanelProps = ImageConfig & {
 
 export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
   const { formatMessage } = useIntl();
+  const isPlaying = useMessagePipeline(
+    (state) => state.playerState.activeData?.isPlaying ?? false,
+  );
   const {
     player,
     panelId,
@@ -250,23 +254,32 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
     };
   }, [player, mainConsumerId, h264ConsumerId, topic]);
 
-  // Reset on playback rewind; for H264, rebuild decoder state from the nearest keyframe.
+  // Keep the worker's media deadline current. On rewind, rebuild H.264 state
+  // from the nearest complete random-access point.
   useEffect(() => {
     return player.subscribeCurrentTime((time) => {
+      workerRef.current?.postMessage({
+        type: 'playback',
+        currentTime: time,
+        isPlaying,
+      } satisfies ImageRenderWorkerRequest);
       const nowNs = toNano(time);
       const previousNs = lastPlaybackTimeNsRef.current;
       if (previousNs != null && nowNs + 5_000_000n < previousNs) {
         const worker = workerRef.current;
         if (worker && topic && h264ModeRef.current) {
-          worker.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest);
+          worker.postMessage({
+            type: 'reset',
+            preserveFrame: true,
+          } satisfies ImageRenderWorkerRequest);
           void repairH264Seek(player, worker, topic, time);
         } else {
           workerRef.current?.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest);
         }
       }
       lastPlaybackTimeNsRef.current = nowNs;
     });
-  }, [player, topic]);
+  }, [isPlaying, player, topic]);
 
   // Send color/depth decode options when they change — triggers immediate redraw in worker
   useEffect(() => {
@@ -315,6 +328,10 @@ export const ImagePanel: React.FC<ImagePanelProps> = (props) => {
       data-h264-pressure={metrics?.pressureMode}
       data-h264-queue-frames={metrics?.queueFrames}
       data-h264-dropped-frames={metrics?.droppedFrames}
+      data-h264-decode-queue={metrics?.decodeQueueSize}
+      data-h264-media-lag-ms={metrics?.mediaLagMs}
+      data-h264-resync-count={metrics?.resyncCount}
+      data-h264-rendered-frames={metrics?.renderedFrames}
     >
       <PanelTopicBar className="border-zinc-800 bg-zinc-950">
         <TopicQuickPicker