feat(stream): experimental Framegen WebGPU frame interpolation

locales/en.json

@@ -608,6 +608,16 @@
       "filmGrain": "Film Grain",
       "reset": "Reset to defaults"
     },
+    "frameInterpolation": {
+      "title": "Frame Interpolation",
+      "hint": "Experimental neural frame interpolation (Framegen WebGPU runtime) that synthesizes extra frames between decoded stream frames for smoother motion. Adds display latency and GPU load. Web client mode only; requires WebGPU + shader-f16.",
+      "nativeUnavailable": "Frame interpolation does not apply while native streaming renders the video outside the app window.",
+      "factor": "Factor",
+      "quality": "Model quality",
+      "qualityHint": "Internal resolution of interpolated frames. Lower uses less GPU; 480p is a good starting point.",
+      "weightsNotice": "Runtime code is MIT. Model weights are non-commercial research/personal use only (Framegen weight license).",
+      "reset": "Reset to defaults"
+    },
     "codecDiagnostics": {
       "advanced": "Advanced - Codec Diagnostics",
       "title": "Codec Diagnostics",

opennow-stable/.gitignore

@@ -3,3 +3,6 @@ dist
 dist-electron
 dist-release
 *.log
+
+# Framegen weights copied at postinstall (non-commercial; see third_party/framegen)
+src/renderer/public/framegen-weights/

opennow-stable/electron.vite.config.ts

@@ -53,5 +53,8 @@ export default defineConfig({
         "@shared": resolve("src/shared"),
       },
     },
+    optimizeDeps: {
+      include: ["framegen"],
+    },
   },
 });

opennow-stable/package.json

@@ -15,7 +15,7 @@
   "type": "module",
   "main": "dist-electron/main/index.js",
   "scripts": {
-    "postinstall": "node scripts/ensure-electron-installed.mjs",
+    "postinstall": "node scripts/ensure-electron-installed.mjs && node scripts/copy-framegen-weights.mjs",
     "electron:install": "node scripts/ensure-electron-installed.mjs",
     "dev": "electron-vite dev",
     "build": "electron-vite build",
@@ -34,6 +34,7 @@
     "@react-three/fiber": "^9.6.1",
     "discord-rpc": "^4.0.1",
     "electron-updater": "^6.8.9",
+    "framegen": "^1.4.0",
     "lucide-react": "^1.25.0",
     "motion": "^12.42.2",
     "posthog-js": "^1.406.2",
@@ -53,6 +54,7 @@
     "@types/three": "^0.185.1",
     "@types/ws": "^8.18.1",
     "@vitejs/plugin-react": "^5.2.0",
+    "@webgpu/types": "^0.1.71",
     "cross-env": "^10.1.0",
     "electron": "^43.1.1",
     "electron-builder": "^26.15.3",

opennow-stable/scripts/copy-framegen-weights.mjs

@@ -0,0 +1,41 @@
+/**
+ * Copy Framegen v7-small weights into the renderer public folder so the
+ * Electron app can load them offline from the app origin.
+ *
+ * Weight license is non-commercial (see third_party/framegen/WEIGHTS_LICENSE.md).
+ */
+import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+const src = join(root, "node_modules", "framegen", "weights");
+const dest = join(root, "src", "renderer", "public", "framegen-weights");
+const noticeDir = join(root, "third_party", "framegen");
+
+if (!existsSync(src)) {
+  console.warn(`[copy-framegen-weights] skip: ${src} not found (install framegen first)`);
+  process.exit(0);
+}
+
+mkdirSync(dest, { recursive: true });
+cpSync(src, dest, { recursive: true });
+
+mkdirSync(noticeDir, { recursive: true });
+writeFileSync(
+  join(noticeDir, "WEIGHTS_LICENSE.md"),
+  `# Framegen model weights
+
+The files under \`src/renderer/public/framegen-weights/\` are the Framegen
+v7-small neural frame-interpolation weights, redistributed from the
+[\`framegen\`](https://www.npmjs.com/package/framegen) npm package.
+
+**Non-commercial research and personal use only.** See the upstream notice:
+https://github.com/MONZikWasTaken/Framegen/blob/main/WEIGHTS_LICENSE.md
+
+Framegen **runtime code** (JavaScript/WGSL) is MIT-licensed separately.
+`,
+  "utf8",
+);
+
+console.log(`[copy-framegen-weights] copied weights to ${dest}`);

opennow-stable/src/main/settings.ts

@@ -17,6 +17,7 @@ import {
   normalizeStreamPreferences,
   normalizeTransportModeForPlatform,
   normalizeVideoShaderSettings,
+  normalizeFrameInterpolationSettings,
   normalizeUpdateChannel,
 } from "@shared/gfn";
 
@@ -294,6 +295,12 @@ export class SettingsManager {
       migrated = true;
     }
 
+    const frameInterpolation = normalizeFrameInterpolationSettings(settings.frameInterpolation);
+    if (JSON.stringify(settings.frameInterpolation) !== JSON.stringify(frameInterpolation)) {
+      settings.frameInterpolation = frameInterpolation;
+      migrated = true;
+    }
+
     const consentBefore = settings.errorReportingConsent;
     settings.errorReportingConsent = normalizeErrorReportingConsent(settings.errorReportingConsent);
     if (settings.errorReportingConsent !== consentBefore) {

opennow-stable/src/renderer/public/.gitkeep

@@ -0,0 +1 @@
+

opennow-stable/src/renderer/src/App.tsx

@@ -2754,6 +2754,7 @@ export function App(): JSX.Element {
               allowEscapeToExitFullscreen={settings.allowEscapeToExitFullscreen}
               videoShader={settings.videoShader}
               onVideoShaderChange={handleVideoShaderChange}
+              frameInterpolation={settings.frameInterpolation}
             />
           </m.div>
         )}

opennow-stable/src/renderer/src/components/StreamView.tsx

@@ -9,8 +9,14 @@ import type { StreamDiagnosticsStore } from "../utils/streamDiagnosticsStore";
 import { useStreamDiagnosticsSelector } from "../utils/streamDiagnosticsStore";
 import { getStoreDisplayName, getStoreIconComponent } from "./GameCard";
 import { SessionElapsedIndicator } from "./ElapsedSessionIndicators";
-import type { MicrophoneMode, SubscriptionInfo, VideoShaderSettings } from "@shared/gfn";
+import type {
+  FrameInterpolationSettings,
+  MicrophoneMode,
+  SubscriptionInfo,
+  VideoShaderSettings,
+} from "@shared/gfn";
 import { VideoShaderPipeline } from "../platforms/gfn/videoShaderPipeline";
+import { FrameInterpolationPipeline } from "../platforms/gfn/frameInterpolationPipeline";
 import { formatShortcutForDisplay } from "../shortcuts";
 import { useScreenshotGallery } from "../hooks/useScreenshotGallery";
 import { useStreamMenuNavigation } from "../hooks/useStreamMenuNavigation";
@@ -101,6 +107,7 @@ interface StreamViewProps {
   allowEscapeToExitFullscreen?: boolean;
   videoShader: VideoShaderSettings;
   onVideoShaderChange: (value: VideoShaderSettings) => void;
+  frameInterpolation: FrameInterpolationSettings;
 }
 
 export function StreamView({
@@ -156,6 +163,7 @@ export function StreamView({
   className,
   videoShader,
   onVideoShaderChange,
+  frameInterpolation,
 }: StreamViewProps): JSX.Element {
   const [showHints, setShowHints] = useState(true);
   const [showSessionClock, setShowSessionClock] = useState(false);
@@ -170,6 +178,7 @@ export function StreamView({
   const localVideoRef = useRef<HTMLVideoElement | null>(null);
   const localAudioRef = useRef<HTMLAudioElement | null>(null);
   const shaderPipelineRef = useRef<VideoShaderPipeline | null>(null);
+  const frameInterpolationPipelineRef = useRef<FrameInterpolationPipeline | null>(null);
   const streamHasVideo = useStreamDiagnosticsSelector(
     diagnosticsStore,
     (stats) => hasVisibleStreamVideo(stats),
@@ -390,9 +399,26 @@ export function StreamView({
     }
   }, [videoShader, gstreamerEnabled, nativeRendererActive]);
 
+  // Experimental Framegen WebGPU frame interpolation (web client path only).
+  useEffect(() => {
+    const video = localVideoRef.current;
+    if (!video) return;
+    const effective = gstreamerEnabled || nativeRendererActive
+      ? { ...frameInterpolation, enabled: false }
+      : frameInterpolation;
+    if (!frameInterpolationPipelineRef.current) {
+      if (!effective.enabled) return;
+      frameInterpolationPipelineRef.current = new FrameInterpolationPipeline(video, effective);
+    } else {
+      frameInterpolationPipelineRef.current.updateSettings(effective);
+    }
+  }, [frameInterpolation, gstreamerEnabled, nativeRendererActive]);
+
   useEffect(() => () => {
     shaderPipelineRef.current?.dispose();
     shaderPipelineRef.current = null;
+    frameInterpolationPipelineRef.current?.dispose();
+    frameInterpolationPipelineRef.current = null;
   }, []);
 
   const setVideoRef = useCallback((element: HTMLVideoElement | null) => {