From 38526f24386056114f0589a497d5c824c60d0a98 Mon Sep 17 00:00:00 2001 From: Monzik Allen Date: Sun, 9 Aug 2026 01:41:32 +0300 Subject: [PATCH 1/5] feat(stream): experimental Framegen WebGPU frame interpolation Add optional client-side neural frame interpolation for the embedded WebRTC stream path using the Framegen WGSL runtime (npm framegen). - New frameInterpolation settings (factor 2-4x, quality 360/480/720) - WebGPU pipeline overlay (web client only; disabled for native streamer) - Settings UI under Video with license notice for non-commercial weights - postinstall copies weights into public/; CDN fallback if missing - Unit tests for settings normalization Off by default. Adds display latency and GPU load when enabled. --- locales/en.json | 10 + opennow-stable/.gitignore | 3 + opennow-stable/electron.vite.config.ts | 3 + opennow-stable/package-lock.json | 18 + opennow-stable/package.json | 4 +- .../scripts/copy-framegen-weights.mjs | 41 ++ opennow-stable/src/main/settings.ts | 7 + opennow-stable/src/renderer/public/.gitkeep | 1 + opennow-stable/src/renderer/src/App.tsx | 1 + .../renderer/src/components/StreamView.tsx | 28 +- .../stream/FrameInterpolationControls.tsx | 142 +++++ .../settings/stream/StreamVideoSection.tsx | 5 + .../gfn/frameInterpolationPipeline.ts | 580 ++++++++++++++++++ .../src/renderer/src/platforms/gfn/index.ts | 1 + .../src/renderer/src/types/framegen.d.ts | 54 ++ opennow-stable/src/renderer/src/vite-env.d.ts | 1 + .../src/shared/gfn/frameInterpolation.test.ts | 58 ++ .../src/shared/gfn/frameInterpolation.ts | 59 ++ opennow-stable/src/shared/gfn/index.ts | 1 + .../src/shared/gfn/settings.test.ts | 3 + opennow-stable/src/shared/gfn/settings.ts | 10 + opennow-stable/third_party/framegen/README.md | 14 + .../third_party/framegen/WEIGHTS_LICENSE.md | 10 + opennow-stable/tsconfig.json | 6 +- 24 files changed, 1057 insertions(+), 3 deletions(-) create mode 100644 opennow-stable/scripts/copy-framegen-weights.mjs create mode 100644 opennow-stable/src/renderer/public/.gitkeep create mode 100644 opennow-stable/src/renderer/src/components/settings/stream/FrameInterpolationControls.tsx create mode 100644 opennow-stable/src/renderer/src/platforms/gfn/frameInterpolationPipeline.ts create mode 100644 opennow-stable/src/renderer/src/types/framegen.d.ts create mode 100644 opennow-stable/src/shared/gfn/frameInterpolation.test.ts create mode 100644 opennow-stable/src/shared/gfn/frameInterpolation.ts create mode 100644 opennow-stable/third_party/framegen/README.md create mode 100644 opennow-stable/third_party/framegen/WEIGHTS_LICENSE.md diff --git a/locales/en.json b/locales/en.json index e59d1fd7..eca5985a 100644 --- a/locales/en.json +++ b/locales/en.json @@ -608,6 +608,16 @@ "filmGrain": "Film Grain", "reset": "Reset to defaults" }, + "frameInterpolation": { + "title": "Frame Interpolation", + "hint": "Experimental neural frame interpolation (Framegen WebGPU runtime) that synthesizes extra frames between decoded stream frames for smoother motion. Adds display latency and GPU load. Web client mode only; requires WebGPU + shader-f16.", + "nativeUnavailable": "Frame interpolation does not apply while native streaming renders the video outside the app window.", + "factor": "Factor", + "quality": "Model quality", + "qualityHint": "Internal resolution of interpolated frames. Lower uses less GPU; 480p is a good starting point.", + "weightsNotice": "Runtime code is MIT. Model weights are non-commercial research/personal use only (Framegen weight license).", + "reset": "Reset to defaults" + }, "codecDiagnostics": { "advanced": "Advanced - Codec Diagnostics", "title": "Codec Diagnostics", diff --git a/opennow-stable/.gitignore b/opennow-stable/.gitignore index 5152f10c..613e8ab4 100644 --- a/opennow-stable/.gitignore +++ b/opennow-stable/.gitignore @@ -3,3 +3,6 @@ dist dist-electron dist-release *.log + +# Framegen weights copied at postinstall (non-commercial; see third_party/framegen) +src/renderer/public/framegen-weights/ diff --git a/opennow-stable/electron.vite.config.ts b/opennow-stable/electron.vite.config.ts index 9c8f13f7..19910376 100644 --- a/opennow-stable/electron.vite.config.ts +++ b/opennow-stable/electron.vite.config.ts @@ -53,5 +53,8 @@ export default defineConfig({ "@shared": resolve("src/shared"), }, }, + optimizeDeps: { + include: ["framegen"], + }, }, }); diff --git a/opennow-stable/package-lock.json b/opennow-stable/package-lock.json index 0c33fbd7..a95cfcfe 100644 --- a/opennow-stable/package-lock.json +++ b/opennow-stable/package-lock.json @@ -13,6 +13,7 @@ "@react-three/fiber": "^9.6.1", "discord-rpc": "^4.0.1", "electron-updater": "^6.8.9", + "framegen": "^1.4.0", "lucide-react": "^1.25.0", "motion": "^12.42.2", "posthog-js": "^1.406.2", @@ -32,6 +33,7 @@ "@types/three": "^0.185.1", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^5.2.0", + "@webgpu/types": "^0.1.71", "cross-env": "^10.1.0", "electron": "^43.1.1", "electron-builder": "^26.15.3", @@ -3713,6 +3715,13 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/@webgpu/types": { + "version": "0.1.71", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.71.tgz", + "integrity": "sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/@xmldom/xmldom": { "version": "0.8.13", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", @@ -5941,6 +5950,15 @@ "node": ">= 6" } }, + "node_modules/framegen": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/framegen/-/framegen-1.4.0.tgz", + "integrity": "sha512-NqK6nckQMqW6l29mKlNzVlbwH89YLPi7wqiPbrmLSm3p9LMqWEnURTjubNQ8k8F4ogz2OWpygT5MSTmxtvojQQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/framer-motion": { "version": "12.42.2", "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.42.2.tgz", diff --git a/opennow-stable/package.json b/opennow-stable/package.json index 4efa71df..2a5d67ff 100644 --- a/opennow-stable/package.json +++ b/opennow-stable/package.json @@ -15,7 +15,7 @@ "type": "module", "main": "dist-electron/main/index.js", "scripts": { - "postinstall": "node scripts/ensure-electron-installed.mjs", + "postinstall": "node scripts/ensure-electron-installed.mjs && node scripts/copy-framegen-weights.mjs", "electron:install": "node scripts/ensure-electron-installed.mjs", "dev": "electron-vite dev", "build": "electron-vite build", @@ -34,6 +34,7 @@ "@react-three/fiber": "^9.6.1", "discord-rpc": "^4.0.1", "electron-updater": "^6.8.9", + "framegen": "^1.4.0", "lucide-react": "^1.25.0", "motion": "^12.42.2", "posthog-js": "^1.406.2", @@ -53,6 +54,7 @@ "@types/three": "^0.185.1", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^5.2.0", + "@webgpu/types": "^0.1.71", "cross-env": "^10.1.0", "electron": "^43.1.1", "electron-builder": "^26.15.3", diff --git a/opennow-stable/scripts/copy-framegen-weights.mjs b/opennow-stable/scripts/copy-framegen-weights.mjs new file mode 100644 index 00000000..2e749444 --- /dev/null +++ b/opennow-stable/scripts/copy-framegen-weights.mjs @@ -0,0 +1,41 @@ +/** + * Copy Framegen v7-small weights into the renderer public folder so the + * Electron app can load them offline from the app origin. + * + * Weight license is non-commercial (see third_party/framegen/WEIGHTS_LICENSE.md). + */ +import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const src = join(root, "node_modules", "framegen", "weights"); +const dest = join(root, "src", "renderer", "public", "framegen-weights"); +const noticeDir = join(root, "third_party", "framegen"); + +if (!existsSync(src)) { + console.warn(`[copy-framegen-weights] skip: ${src} not found (install framegen first)`); + process.exit(0); +} + +mkdirSync(dest, { recursive: true }); +cpSync(src, dest, { recursive: true }); + +mkdirSync(noticeDir, { recursive: true }); +writeFileSync( + join(noticeDir, "WEIGHTS_LICENSE.md"), + `# Framegen model weights + +The files under \`src/renderer/public/framegen-weights/\` are the Framegen +v7-small neural frame-interpolation weights, redistributed from the +[\`framegen\`](https://www.npmjs.com/package/framegen) npm package. + +**Non-commercial research and personal use only.** See the upstream notice: +https://github.com/MONZikWasTaken/Framegen/blob/main/WEIGHTS_LICENSE.md + +Framegen **runtime code** (JavaScript/WGSL) is MIT-licensed separately. +`, + "utf8", +); + +console.log(`[copy-framegen-weights] copied weights to ${dest}`); diff --git a/opennow-stable/src/main/settings.ts b/opennow-stable/src/main/settings.ts index 6279c023..859bc662 100644 --- a/opennow-stable/src/main/settings.ts +++ b/opennow-stable/src/main/settings.ts @@ -17,6 +17,7 @@ import { normalizeStreamPreferences, normalizeTransportModeForPlatform, normalizeVideoShaderSettings, + normalizeFrameInterpolationSettings, normalizeUpdateChannel, } from "@shared/gfn"; @@ -294,6 +295,12 @@ export class SettingsManager { migrated = true; } + const frameInterpolation = normalizeFrameInterpolationSettings(settings.frameInterpolation); + if (JSON.stringify(settings.frameInterpolation) !== JSON.stringify(frameInterpolation)) { + settings.frameInterpolation = frameInterpolation; + migrated = true; + } + const consentBefore = settings.errorReportingConsent; settings.errorReportingConsent = normalizeErrorReportingConsent(settings.errorReportingConsent); if (settings.errorReportingConsent !== consentBefore) { diff --git a/opennow-stable/src/renderer/public/.gitkeep b/opennow-stable/src/renderer/public/.gitkeep new file mode 100644 index 00000000..e02abfc9 --- /dev/null +++ b/opennow-stable/src/renderer/public/.gitkeep @@ -0,0 +1 @@ + diff --git a/opennow-stable/src/renderer/src/App.tsx b/opennow-stable/src/renderer/src/App.tsx index 43637b3e..54b750db 100644 --- a/opennow-stable/src/renderer/src/App.tsx +++ b/opennow-stable/src/renderer/src/App.tsx @@ -2754,6 +2754,7 @@ export function App(): JSX.Element { allowEscapeToExitFullscreen={settings.allowEscapeToExitFullscreen} videoShader={settings.videoShader} onVideoShaderChange={handleVideoShaderChange} + frameInterpolation={settings.frameInterpolation} /> )} diff --git a/opennow-stable/src/renderer/src/components/StreamView.tsx b/opennow-stable/src/renderer/src/components/StreamView.tsx index 8d7261e7..e01f585c 100644 --- a/opennow-stable/src/renderer/src/components/StreamView.tsx +++ b/opennow-stable/src/renderer/src/components/StreamView.tsx @@ -9,8 +9,14 @@ import type { StreamDiagnosticsStore } from "../utils/streamDiagnosticsStore"; import { useStreamDiagnosticsSelector } from "../utils/streamDiagnosticsStore"; import { getStoreDisplayName, getStoreIconComponent } from "./GameCard"; import { SessionElapsedIndicator } from "./ElapsedSessionIndicators"; -import type { MicrophoneMode, SubscriptionInfo, VideoShaderSettings } from "@shared/gfn"; +import type { + FrameInterpolationSettings, + MicrophoneMode, + SubscriptionInfo, + VideoShaderSettings, +} from "@shared/gfn"; import { VideoShaderPipeline } from "../platforms/gfn/videoShaderPipeline"; +import { FrameInterpolationPipeline } from "../platforms/gfn/frameInterpolationPipeline"; import { formatShortcutForDisplay } from "../shortcuts"; import { useScreenshotGallery } from "../hooks/useScreenshotGallery"; import { useStreamMenuNavigation } from "../hooks/useStreamMenuNavigation"; @@ -101,6 +107,7 @@ interface StreamViewProps { allowEscapeToExitFullscreen?: boolean; videoShader: VideoShaderSettings; onVideoShaderChange: (value: VideoShaderSettings) => void; + frameInterpolation: FrameInterpolationSettings; } export function StreamView({ @@ -156,6 +163,7 @@ export function StreamView({ className, videoShader, onVideoShaderChange, + frameInterpolation, }: StreamViewProps): JSX.Element { const [showHints, setShowHints] = useState(true); const [showSessionClock, setShowSessionClock] = useState(false); @@ -170,6 +178,7 @@ export function StreamView({ const localVideoRef = useRef(null); const localAudioRef = useRef(null); const shaderPipelineRef = useRef(null); + const frameInterpolationPipelineRef = useRef(null); const streamHasVideo = useStreamDiagnosticsSelector( diagnosticsStore, (stats) => hasVisibleStreamVideo(stats), @@ -390,9 +399,26 @@ export function StreamView({ } }, [videoShader, gstreamerEnabled, nativeRendererActive]); + // Experimental Framegen WebGPU frame interpolation (web client path only). + useEffect(() => { + const video = localVideoRef.current; + if (!video) return; + const effective = gstreamerEnabled || nativeRendererActive + ? { ...frameInterpolation, enabled: false } + : frameInterpolation; + if (!frameInterpolationPipelineRef.current) { + if (!effective.enabled) return; + frameInterpolationPipelineRef.current = new FrameInterpolationPipeline(video, effective); + } else { + frameInterpolationPipelineRef.current.updateSettings(effective); + } + }, [frameInterpolation, gstreamerEnabled, nativeRendererActive]); + useEffect(() => () => { shaderPipelineRef.current?.dispose(); shaderPipelineRef.current = null; + frameInterpolationPipelineRef.current?.dispose(); + frameInterpolationPipelineRef.current = null; }, []); const setVideoRef = useCallback((element: HTMLVideoElement | null) => { diff --git a/opennow-stable/src/renderer/src/components/settings/stream/FrameInterpolationControls.tsx b/opennow-stable/src/renderer/src/components/settings/stream/FrameInterpolationControls.tsx new file mode 100644 index 00000000..17bc8d1a --- /dev/null +++ b/opennow-stable/src/renderer/src/components/settings/stream/FrameInterpolationControls.tsx @@ -0,0 +1,142 @@ +import { type JSX } from "react"; +import type { + FrameInterpolationFactor, + FrameInterpolationQuality, + Settings, +} from "@shared/gfn"; +import { DEFAULT_FRAME_INTERPOLATION_SETTINGS } from "@shared/gfn"; +import { useTranslation } from "../../../i18n"; +import type { SettingsChangeHandler } from "./streamSettingsTypes"; + +const FACTOR_OPTIONS: FrameInterpolationFactor[] = [2, 3, 4]; +const QUALITY_OPTIONS: FrameInterpolationQuality[] = [360, 480, 720]; + +interface FrameInterpolationControlsProps { + settings: Settings; + handleChange: SettingsChangeHandler; +} + +export function FrameInterpolationControls({ + settings, + handleChange, +}: FrameInterpolationControlsProps): JSX.Element { + const { t } = useTranslation(); + const fi = settings.frameInterpolation; + const nativeBlocked = settings.streamClientMode === "native"; + + return ( +
+
+ + +
+ + {nativeBlocked + ? t("settings.frameInterpolation.nativeUnavailable") + : t("settings.frameInterpolation.hint")} + + {fi.enabled && !nativeBlocked && ( + <> +
+
+ + {fi.factor}× +
+
+ {FACTOR_OPTIONS.map((factor) => ( + + ))} +
+
+
+
+ + {fi.quality}p +
+
+ {QUALITY_OPTIONS.map((quality) => ( + + ))} +
+ + {t("settings.frameInterpolation.qualityHint")} + +
+

+ {t("settings.frameInterpolation.weightsNotice")} +

+
+ +
+ + )} +
+ ); +} diff --git a/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx b/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx index 04666735..5879d76d 100644 --- a/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx +++ b/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx @@ -7,6 +7,7 @@ import type { SettingsChangeHandler } from "./streamSettingsTypes"; import { StreamQualityControls } from "./StreamQualityControls"; import { SessionProxySettings } from "./SessionProxySettings"; import { VideoShaderControls } from "./VideoShaderControls"; +import { FrameInterpolationControls } from "./FrameInterpolationControls"; import { StreamSessionOptions } from "./StreamSessionOptions"; interface StreamVideoSectionProps { @@ -65,6 +66,10 @@ export function StreamVideoSection({ handleChange={handleChange} handlePreview={handlePreview} /> + ); diff --git a/opennow-stable/src/renderer/src/platforms/gfn/frameInterpolationPipeline.ts b/opennow-stable/src/renderer/src/platforms/gfn/frameInterpolationPipeline.ts new file mode 100644 index 00000000..23b03ba1 --- /dev/null +++ b/opennow-stable/src/renderer/src/platforms/gfn/frameInterpolationPipeline.ts @@ -0,0 +1,580 @@ +import type { FrameInterpolationSettings } from "@shared/gfn"; +import { frameInterpolationIsActive } from "@shared/gfn"; +import { createRT, type RT } from "framegen"; + +/** + * WebGPU neural frame-interpolation overlay for the embedded WebRTC stream. + * + * Uses the Framegen WGSL runtime (`framegen` npm, MIT code) to synthesize + * intermediate frames between consecutive decoded