diff --git a/locales/en.json b/locales/en.json index e59d1fd7d..eca5985a7 100644 --- a/locales/en.json +++ b/locales/en.json @@ -608,6 +608,16 @@ "filmGrain": "Film Grain", "reset": "Reset to defaults" }, + "frameInterpolation": { + "title": "Frame Interpolation", + "hint": "Experimental neural frame interpolation (Framegen WebGPU runtime) that synthesizes extra frames between decoded stream frames for smoother motion. Adds display latency and GPU load. Web client mode only; requires WebGPU + shader-f16.", + "nativeUnavailable": "Frame interpolation does not apply while native streaming renders the video outside the app window.", + "factor": "Factor", + "quality": "Model quality", + "qualityHint": "Internal resolution of interpolated frames. Lower uses less GPU; 480p is a good starting point.", + "weightsNotice": "Runtime code is MIT. Model weights are non-commercial research/personal use only (Framegen weight license).", + "reset": "Reset to defaults" + }, "codecDiagnostics": { "advanced": "Advanced - Codec Diagnostics", "title": "Codec Diagnostics", diff --git a/opennow-stable/.gitignore b/opennow-stable/.gitignore index 5152f10cf..613e8ab47 100644 --- a/opennow-stable/.gitignore +++ b/opennow-stable/.gitignore @@ -3,3 +3,6 @@ dist dist-electron dist-release *.log + +# Framegen weights copied at postinstall (non-commercial; see third_party/framegen) +src/renderer/public/framegen-weights/ diff --git a/opennow-stable/bun.lock b/opennow-stable/bun.lock index cbbebe890..3da85f754 100644 --- a/opennow-stable/bun.lock +++ b/opennow-stable/bun.lock @@ -9,6 +9,7 @@ "@react-three/fiber": "^9.6.1", "discord-rpc": "^4.0.1", "electron-updater": "^6.8.9", + "framegen": "^1.4.0", "lucide-react": "^1.25.0", "motion": "^12.42.2", "posthog-js": "^1.406.2", @@ -28,6 +29,7 @@ "@types/three": "^0.185.1", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^5.2.0", + "@webgpu/types": "^0.1.71", "cross-env": "^10.1.0", "electron": "^43.1.1", "electron-builder": "^26.15.3", @@ -547,6 +549,8 @@ "@vitejs/plugin-react": ["@vitejs/plugin-react@5.2.0", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw=="], + "@webgpu/types": ["@webgpu/types@0.1.71", "", {}, "sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A=="], + "@xmldom/xmldom": ["@xmldom/xmldom@0.8.13", "", {}, "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw=="], "abbrev": ["abbrev@4.0.0", "", {}, "sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA=="], @@ -877,6 +881,8 @@ "form-data": ["form-data@4.0.6", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.4", "mime-types": "^2.1.35" } }, "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ=="], + "framegen": ["framegen@1.4.0", "", {}, "sha512-NqK6nckQMqW6l29mKlNzVlbwH89YLPi7wqiPbrmLSm3p9LMqWEnURTjubNQ8k8F4ogz2OWpygT5MSTmxtvojQQ=="], + "framer-motion": ["framer-motion@12.42.2", "", { "dependencies": { "motion-dom": "^12.42.2", "motion-utils": "^12.39.0", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-5XY9luDiu0oHfHBjpDthFMh0ES+122w6p/papSJBweMkO8Sn+PW2QaEgRblQBpWFnuvZS5qvarpt/hO2pjGmnw=="], "fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="], diff --git a/opennow-stable/electron.vite.config.ts b/opennow-stable/electron.vite.config.ts index 9c8f13f70..199103768 100644 --- a/opennow-stable/electron.vite.config.ts +++ b/opennow-stable/electron.vite.config.ts @@ -53,5 +53,8 @@ export default defineConfig({ "@shared": resolve("src/shared"), }, }, + optimizeDeps: { + include: ["framegen"], + }, }, }); diff --git a/opennow-stable/package-lock.json b/opennow-stable/package-lock.json index 0c33fbd79..a95cfcfee 100644 --- a/opennow-stable/package-lock.json +++ b/opennow-stable/package-lock.json @@ -13,6 +13,7 @@ "@react-three/fiber": "^9.6.1", "discord-rpc": "^4.0.1", "electron-updater": "^6.8.9", + "framegen": "^1.4.0", "lucide-react": "^1.25.0", "motion": "^12.42.2", "posthog-js": "^1.406.2", @@ -32,6 +33,7 @@ "@types/three": "^0.185.1", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^5.2.0", + "@webgpu/types": "^0.1.71", "cross-env": "^10.1.0", "electron": "^43.1.1", "electron-builder": "^26.15.3", @@ -3713,6 +3715,13 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/@webgpu/types": { + "version": "0.1.71", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.71.tgz", + "integrity": "sha512-mMy8/ODcKhab808co15eW+yN+HgXoQxRQHTiBV9Mrvl1r0ufnid7YOcI+gi4eUWSWl9ezD6TW2KXccrL8HCh2A==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/@xmldom/xmldom": { "version": "0.8.13", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", @@ -5941,6 +5950,15 @@ "node": ">= 6" } }, + "node_modules/framegen": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/framegen/-/framegen-1.4.0.tgz", + "integrity": "sha512-NqK6nckQMqW6l29mKlNzVlbwH89YLPi7wqiPbrmLSm3p9LMqWEnURTjubNQ8k8F4ogz2OWpygT5MSTmxtvojQQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/framer-motion": { "version": "12.42.2", "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.42.2.tgz", diff --git a/opennow-stable/package.json b/opennow-stable/package.json index 4efa71df8..2a5d67ffe 100644 --- a/opennow-stable/package.json +++ b/opennow-stable/package.json @@ -15,7 +15,7 @@ "type": "module", "main": "dist-electron/main/index.js", "scripts": { - "postinstall": "node scripts/ensure-electron-installed.mjs", + "postinstall": "node scripts/ensure-electron-installed.mjs && node scripts/copy-framegen-weights.mjs", "electron:install": "node scripts/ensure-electron-installed.mjs", "dev": "electron-vite dev", "build": "electron-vite build", @@ -34,6 +34,7 @@ "@react-three/fiber": "^9.6.1", "discord-rpc": "^4.0.1", "electron-updater": "^6.8.9", + "framegen": "^1.4.0", "lucide-react": "^1.25.0", "motion": "^12.42.2", "posthog-js": "^1.406.2", @@ -53,6 +54,7 @@ "@types/three": "^0.185.1", "@types/ws": "^8.18.1", "@vitejs/plugin-react": "^5.2.0", + "@webgpu/types": "^0.1.71", "cross-env": "^10.1.0", "electron": "^43.1.1", "electron-builder": "^26.15.3", diff --git a/opennow-stable/scripts/copy-framegen-weights.mjs b/opennow-stable/scripts/copy-framegen-weights.mjs new file mode 100644 index 000000000..2e749444a --- /dev/null +++ b/opennow-stable/scripts/copy-framegen-weights.mjs @@ -0,0 +1,41 @@ +/** + * Copy Framegen v7-small weights into the renderer public folder so the + * Electron app can load them offline from the app origin. + * + * Weight license is non-commercial (see third_party/framegen/WEIGHTS_LICENSE.md). + */ +import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const src = join(root, "node_modules", "framegen", "weights"); +const dest = join(root, "src", "renderer", "public", "framegen-weights"); +const noticeDir = join(root, "third_party", "framegen"); + +if (!existsSync(src)) { + console.warn(`[copy-framegen-weights] skip: ${src} not found (install framegen first)`); + process.exit(0); +} + +mkdirSync(dest, { recursive: true }); +cpSync(src, dest, { recursive: true }); + +mkdirSync(noticeDir, { recursive: true }); +writeFileSync( + join(noticeDir, "WEIGHTS_LICENSE.md"), + `# Framegen model weights + +The files under \`src/renderer/public/framegen-weights/\` are the Framegen +v7-small neural frame-interpolation weights, redistributed from the +[\`framegen\`](https://www.npmjs.com/package/framegen) npm package. + +**Non-commercial research and personal use only.** See the upstream notice: +https://github.com/MONZikWasTaken/Framegen/blob/main/WEIGHTS_LICENSE.md + +Framegen **runtime code** (JavaScript/WGSL) is MIT-licensed separately. +`, + "utf8", +); + +console.log(`[copy-framegen-weights] copied weights to ${dest}`); diff --git a/opennow-stable/src/main/settings.ts b/opennow-stable/src/main/settings.ts index 6279c0239..859bc6620 100644 --- a/opennow-stable/src/main/settings.ts +++ b/opennow-stable/src/main/settings.ts @@ -17,6 +17,7 @@ import { normalizeStreamPreferences, normalizeTransportModeForPlatform, normalizeVideoShaderSettings, + normalizeFrameInterpolationSettings, normalizeUpdateChannel, } from "@shared/gfn"; @@ -294,6 +295,12 @@ export class SettingsManager { migrated = true; } + const frameInterpolation = normalizeFrameInterpolationSettings(settings.frameInterpolation); + if (JSON.stringify(settings.frameInterpolation) !== JSON.stringify(frameInterpolation)) { + settings.frameInterpolation = frameInterpolation; + migrated = true; + } + const consentBefore = settings.errorReportingConsent; settings.errorReportingConsent = normalizeErrorReportingConsent(settings.errorReportingConsent); if (settings.errorReportingConsent !== consentBefore) { diff --git a/opennow-stable/src/renderer/public/.gitkeep b/opennow-stable/src/renderer/public/.gitkeep new file mode 100644 index 000000000..e02abfc9b --- /dev/null +++ b/opennow-stable/src/renderer/public/.gitkeep @@ -0,0 +1 @@ + diff --git a/opennow-stable/src/renderer/src/App.tsx b/opennow-stable/src/renderer/src/App.tsx index 43637b3e7..54b750db2 100644 --- a/opennow-stable/src/renderer/src/App.tsx +++ b/opennow-stable/src/renderer/src/App.tsx @@ -2754,6 +2754,7 @@ export function App(): JSX.Element { allowEscapeToExitFullscreen={settings.allowEscapeToExitFullscreen} videoShader={settings.videoShader} onVideoShaderChange={handleVideoShaderChange} + frameInterpolation={settings.frameInterpolation} /> )} diff --git a/opennow-stable/src/renderer/src/components/StreamView.tsx b/opennow-stable/src/renderer/src/components/StreamView.tsx index 8d7261e7f..e01f585c6 100644 --- a/opennow-stable/src/renderer/src/components/StreamView.tsx +++ b/opennow-stable/src/renderer/src/components/StreamView.tsx @@ -9,8 +9,14 @@ import type { StreamDiagnosticsStore } from "../utils/streamDiagnosticsStore"; import { useStreamDiagnosticsSelector } from "../utils/streamDiagnosticsStore"; import { getStoreDisplayName, getStoreIconComponent } from "./GameCard"; import { SessionElapsedIndicator } from "./ElapsedSessionIndicators"; -import type { MicrophoneMode, SubscriptionInfo, VideoShaderSettings } from "@shared/gfn"; +import type { + FrameInterpolationSettings, + MicrophoneMode, + SubscriptionInfo, + VideoShaderSettings, +} from "@shared/gfn"; import { VideoShaderPipeline } from "../platforms/gfn/videoShaderPipeline"; +import { FrameInterpolationPipeline } from "../platforms/gfn/frameInterpolationPipeline"; import { formatShortcutForDisplay } from "../shortcuts"; import { useScreenshotGallery } from "../hooks/useScreenshotGallery"; import { useStreamMenuNavigation } from "../hooks/useStreamMenuNavigation"; @@ -101,6 +107,7 @@ interface StreamViewProps { allowEscapeToExitFullscreen?: boolean; videoShader: VideoShaderSettings; onVideoShaderChange: (value: VideoShaderSettings) => void; + frameInterpolation: FrameInterpolationSettings; } export function StreamView({ @@ -156,6 +163,7 @@ export function StreamView({ className, videoShader, onVideoShaderChange, + frameInterpolation, }: StreamViewProps): JSX.Element { const [showHints, setShowHints] = useState(true); const [showSessionClock, setShowSessionClock] = useState(false); @@ -170,6 +178,7 @@ export function StreamView({ const localVideoRef = useRef(null); const localAudioRef = useRef(null); const shaderPipelineRef = useRef(null); + const frameInterpolationPipelineRef = useRef(null); const streamHasVideo = useStreamDiagnosticsSelector( diagnosticsStore, (stats) => hasVisibleStreamVideo(stats), @@ -390,9 +399,26 @@ export function StreamView({ } }, [videoShader, gstreamerEnabled, nativeRendererActive]); + // Experimental Framegen WebGPU frame interpolation (web client path only). + useEffect(() => { + const video = localVideoRef.current; + if (!video) return; + const effective = gstreamerEnabled || nativeRendererActive + ? { ...frameInterpolation, enabled: false } + : frameInterpolation; + if (!frameInterpolationPipelineRef.current) { + if (!effective.enabled) return; + frameInterpolationPipelineRef.current = new FrameInterpolationPipeline(video, effective); + } else { + frameInterpolationPipelineRef.current.updateSettings(effective); + } + }, [frameInterpolation, gstreamerEnabled, nativeRendererActive]); + useEffect(() => () => { shaderPipelineRef.current?.dispose(); shaderPipelineRef.current = null; + frameInterpolationPipelineRef.current?.dispose(); + frameInterpolationPipelineRef.current = null; }, []); const setVideoRef = useCallback((element: HTMLVideoElement | null) => { diff --git a/opennow-stable/src/renderer/src/components/settings/stream/FrameInterpolationControls.tsx b/opennow-stable/src/renderer/src/components/settings/stream/FrameInterpolationControls.tsx new file mode 100644 index 000000000..17bc8d1a3 --- /dev/null +++ b/opennow-stable/src/renderer/src/components/settings/stream/FrameInterpolationControls.tsx @@ -0,0 +1,142 @@ +import { type JSX } from "react"; +import type { + FrameInterpolationFactor, + FrameInterpolationQuality, + Settings, +} from "@shared/gfn"; +import { DEFAULT_FRAME_INTERPOLATION_SETTINGS } from "@shared/gfn"; +import { useTranslation } from "../../../i18n"; +import type { SettingsChangeHandler } from "./streamSettingsTypes"; + +const FACTOR_OPTIONS: FrameInterpolationFactor[] = [2, 3, 4]; +const QUALITY_OPTIONS: FrameInterpolationQuality[] = [360, 480, 720]; + +interface FrameInterpolationControlsProps { + settings: Settings; + handleChange: SettingsChangeHandler; +} + +export function FrameInterpolationControls({ + settings, + handleChange, +}: FrameInterpolationControlsProps): JSX.Element { + const { t } = useTranslation(); + const fi = settings.frameInterpolation; + const nativeBlocked = settings.streamClientMode === "native"; + + return ( +
+
+ + +
+ + {nativeBlocked + ? t("settings.frameInterpolation.nativeUnavailable") + : t("settings.frameInterpolation.hint")} + + {fi.enabled && !nativeBlocked && ( + <> +
+
+ + {fi.factor}× +
+
+ {FACTOR_OPTIONS.map((factor) => ( + + ))} +
+
+
+
+ + {fi.quality}p +
+
+ {QUALITY_OPTIONS.map((quality) => ( + + ))} +
+ + {t("settings.frameInterpolation.qualityHint")} + +
+

+ {t("settings.frameInterpolation.weightsNotice")} +

+
+ +
+ + )} +
+ ); +} diff --git a/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx b/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx index 046667359..5879d76db 100644 --- a/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx +++ b/opennow-stable/src/renderer/src/components/settings/stream/StreamVideoSection.tsx @@ -7,6 +7,7 @@ import type { SettingsChangeHandler } from "./streamSettingsTypes"; import { StreamQualityControls } from "./StreamQualityControls"; import { SessionProxySettings } from "./SessionProxySettings"; import { VideoShaderControls } from "./VideoShaderControls"; +import { FrameInterpolationControls } from "./FrameInterpolationControls"; import { StreamSessionOptions } from "./StreamSessionOptions"; interface StreamVideoSectionProps { @@ -65,6 +66,10 @@ export function StreamVideoSection({ handleChange={handleChange} handlePreview={handlePreview} /> + ); diff --git a/opennow-stable/src/renderer/src/platforms/gfn/frameInterpolationPipeline.ts b/opennow-stable/src/renderer/src/platforms/gfn/frameInterpolationPipeline.ts new file mode 100644 index 000000000..22576edc8 --- /dev/null +++ b/opennow-stable/src/renderer/src/platforms/gfn/frameInterpolationPipeline.ts @@ -0,0 +1,740 @@ +import type { FrameInterpolationSettings } from "@shared/gfn"; +import { frameInterpolationIsActive } from "@shared/gfn"; +import { createRT, type RT } from "framegen"; + +/** + * WebGPU neural frame-interpolation overlay for the embedded WebRTC stream. + * + * Uses the Framegen WGSL runtime (`framegen` npm, MIT code) to synthesize + * intermediate frames between consecutive decoded